def test___init__properties(self):
for lst in [self.p_eps, self.c_eps]:
for tep in lst:
try:
ep = Gw2Endpoint(tep, 'en', [])
except Exception as e:
self.fail(tep.__name__ + str(e))
self.assertEquals(ep.table_name, tep.__name__)
self.assertEquals(ep.rights,
tep.__table_args__['info']['rights'])
parents = list(set([x.__table__.info['parent'] for x in self.c_eps]))
for tep in self.p_eps:
try:
ep = Gw2Endpoint(tep, 'en', self.c_eps)
if tep.__name__ in parents:
self.assertGreater(len(ep._children), 0, ep.table_name)
else:
self.assertEquals(len(ep._children), 0, ep.table_name)
except Exception as e:
self.fail(tep.__name__ + str(e))
def test__make_args(self):
for lst in [self.p_eps, self.c_eps]:
for tep in lst:
ep = Gw2Endpoint(tep, 'en', [])
if (ep._type & EPType.auth) == 0:
# can't test auths endpoint without access token...
self.assertGreater(len(ep._make_args()), 0, ep.table_name)
elif len(self.test_key) > 0:
self.assertGreater(len(ep._make_args(self.test_key)), 0,
ep.table_name)
else:
# without access token, return empty list
self.assertEquals(len(ep._make_args()), 0, ep.table_name)
def test__size(self):
for lst in [self.p_eps, self.c_eps]:
for tep in lst:
ep = Gw2Endpoint(tep, 'en', [])
if (ep._type & EPType.auth) == 0:
# can't test auths endpoint without access token...
self.assertGreater(ep._size(), -1, ep.table_name)
elif len(self.test_key) > 0:
self.assertGreater(
ep._size({'access_token': self.test_key}), -1,
ep.table_name)
else:
# without access token, return 0 if single else -1
self.assertEquals(
ep._size(), 0 if
(ep._type & EPType.single) != 0 else -1, ep.table_name)
def test_upgrade(self):
for tep in self.p_eps:
ep = Gw2Endpoint(tep, 'en', self.c_eps)
if (ep._type & EPType.auth) != 0:
if len(self.test_key) > 0:
ep.set_params(self.test_key)
if (ep._type & EPType.auth) != 0 and len(self.test_key) == 0:
ep.set_params()
datas = ep.upgrade()
self.assertIsNotNone(datas, ep.table_name)
self.assertEquals(len(datas), 0)
continue
ep.set_params()
datas = ep.upgrade()
self.assertIsNotNone(datas, ep.table_name)
self.assertGreater(len(datas), 0, ep.table_name)
def test_set_params(self):
if len(self.test_key) == 0:
return
eps = list()
for lst in [self.p_eps, self.c_eps]:
for tep in lst:
ep = Gw2Endpoint(tep, 'en', [])
ep.set_params(self.test_key)
ep.set_params()
eps.append(ep)
for ep in eps:
if (ep._type & EPType.auth) != 0 and (ep._type
& EPType.param) == 0:
_len = 2
elif (ep._type & EPType.std) != 0:
_len = 0
else:
_len = 1
self.assertEquals(len(ep._pqueue), _len, ep.table_name)
def test__read(self):
for tep in self.p_eps:
ep = Gw2Endpoint(tep, 'en', self.c_eps)
if (ep._type & EPType.auth) == 0:
# can't test auths endpoint without access token...
args = ep._make_args()
elif len(self.test_key) > 0:
args = ep._make_args(self.test_key)
else:
ep._pqueue.appendleft((None, None, None))
datas = ep._read()
self.assertIsNone(datas, ep.table_name)
continue
ep._pqueue.appendleft((args[0], None, None))
datas = ep._read()
self.assertIsNotNone(datas, ep.table_name)
self.assertGreater(len(datas), 0, ep.table_name)
# check children if any - args made automatically
for ch in ep._children:
datas = ch._read()
self.assertIsNotNone(datas, ch.table_name)
self.assertGreater(len(datas), 0, ch.table_name)
def test__next_pkid(self):
ep = Gw2Endpoint(Gw2Currency, 'en', [])
for i in range(1, 11):
self.assertEquals(ep._next_pkid, i)
def test_on_error(self):
ep = Gw2Endpoint(self.p_eps[0], 'en', [])
ep.on_error()
self.assertIsNone(ep._read(), ep.table_name)
self.assertIsNone(ep._build(), ep.table_name)
self.assertIsNone(ep.upgrade(), ep.table_name)
def _fill_datas(self, lang, params):
"""Download, map and store all declared endpoints datas
:param lang: the language to use as url argument
:param params: current parameters stored in db, as dictionnary (k=name, v=value)
:return: True on success, False on error
"""
# TODO: start img dl
keys = [v for k, v in params.items() if k.startswith('KEY_')]
self.running_status(DbUpgradeStatus.downloading,
len(Gw2Db.__endpoints__))
import time
st = time.time()
chs = [
x for x in Gw2Db.__endpoints__
if (x.__table__.info['ep_type'] & EPType.child) != 0
]
eps = [
Gw2Endpoint(x, lang, chs) for x in Gw2Db.__endpoints__
if (x.__table__.info['ep_type'] & EPType.child) == 0
]
with ThreadPoolExecutor(max_workers=len(eps)) as mapper:
ths = {}
for ep in eps:
ths[mapper.submit(ep.upgrade)] = ep
self.endpoint_status(EndpointUpgradeStatus.downloading,
ep.table_name)
for key in keys:
ep.set_params(key=key)
ep.set_params()
def _on_error(_cls_):
self.endpoint_status(EndpointUpgradeStatus.error, _cls_)
for _ep_ in eps:
_ep_.on_error()
return False
ok = True
for future in as_completed(ths):
if future.exception() is not None:
traceback.print_exc()
ok = _on_error(ths[future].table_name)
continue
if not ok:
continue
ep = ths[future]
datas = future.result()
if datas is not None:
self.endpoint_status(EndpointUpgradeStatus.commiting,
ep.table_name)
try:
for k, v in datas.items():
self._session.bulk_insert_mappings(
k, v, return_defaults=True)
self._session.commit()
except SQLAlchemyError:
traceback.print_exc()
ok = _on_error(ep.table_name)
continue
self.endpoint_status(EndpointUpgradeStatus.success,
ep.table_name)
else:
ok = _on_error(ths[future].table_name)
print('all dl: ', (time.time() - st))
# TODO: end img dl
if not ok:
self.running_status(DbUpgradeStatus.error, -1)
return False
return True