async def async_download(self, id_, *args,
attempts=1,
nores_attempts=1,
timeout=None,
save=True,
**kwgs):
curr_attempts, curr_nores_attempts = 0, 0
rit = get_relax_iterator()
errors = []
for i, relax in enumerate(rit):
try:
result = await self.async_fetch(id_, timeout=timeout)
except exc.NoResultError as e:
errors.append(e)
curr_nores_attempts += 1
if (nores_attempts <= 0
or curr_nores_attempts < nores_attempts):
fs = ('{:0>8}: download failed: no result; '
'relax: {} seconds')
self.logger.warning(fs.format(id_, relax))
await asyncio.sleep(relax, loop=self.loop)
continue
else:
raise e
except socket.error as e:
errors.append(e)
curr_attempts += 1
if attempts <= 0 or curr_attempts < attempts:
fs = '{:0>8}: download failed: {}; relax: {} seconds'
self.logger.warning(fs.format(id_, e, relax))
await asyncio.sleep(relax, loop=self.loop)
continue
else:
raise e
else:
fs = '{:0>8}: downloaded'
self.logger.info(fs.format(id_))
break
if save:
f = functools.partial(self._save, id_, result, *args,
**kwgs)
await self.loop.run_in_executor(None, f)
return result, tuple(errors)
def start(self):
with self:
conn = self.conn
idgen = self.get_idgen()
rit = get_relax_iterator()
while True:
try:
id_ = next(idgen)
except StopIteration:
break
except KeyboardInterrupt:
fs = '{:0>8}: keyboard interrupted: STOPPED'
self.logger.info(fs.format(id_))
break
except urllib.error.URLError as err:
relax = next(rit)
fs = 'id refresh failed: {}; relax: {} seconds'
self.logger.warning(fs.format(err, relax))
time.sleep(relax)
gen = fumbbl_trans.xml_matches.UpdateGenerator(
prev_id, relax=self.relax)
idgen = iter(gen)
else:
try:
hs = fumbbl_trans.html_match.Session.of_match(id_,
netloc=':'.join((self.hhost, str(self.hport))))
htext = hs.get()
xs = fumbbl_trans.xml_matches.Session.of_match(id_,
netloc=':'.join((self.hhost, str(self.hport))))
xtext = xs.get()
except KeyboardInterrupt:
fs = '{:0>8}: keyboard interrupted: STOPPED'
self.logger.info(fs.format(id_))
break
except fumbbl_trans.exc.NoResultError:
if self.nores_stop:
fs = '{:0>8}: no result: STOPPED'
self.logger.error(fs.format(id_))
break
else:
fs = '{:0>8}: no result'
self.logger.warning(fs.format(id_))
else:
d = {}
fplusdb_main.transimp.html_match.import_(htext, d)
fplusdb_main.transimp.xml_matches.import_(xtext, d)
with conn:
with conn.cursor() as cur:
for tablename in MAIN.tables.values():
if tablename not in d:
continue
for r in d[tablename].values():
funcs = sql_commands.serts[tablename]
func = funcs[self.upsert]
s = func(cur, data=r)
try:
cur.execute(s)
except psycopg2.IntegrityError as e:
errarg = e.args[0]
m = common_regex.key_exists.search(errarg)
if m:
fs = '({})->({}): already exists'
errmsg =fs.format(m.group('constraint'),
m.group('values'))
self.logger.error(errmsg)
raise e
else:
raise e
# TODO: would be nice to know whether insert
# or update happened
# print(cur.statusmessage)
funcs = sql_commands.notifs[MAIN.MATCH]
func = funcs[self.upsert]
s = func(cur, payload=id_)
cur.execute(s)
fs = '{:0>8}: inserted'
self.logger.info(fs.format(id_))
prev_id = id_
rit = get_relax_iterator()
def _main_download(args, collection_cls):
collection = collection_cls(args.path)
main_setup_logger(collection, args)
if args.overwrite:
method = getattr(collection, 'download')
else:
method = getattr(collection, 'ensure')
for i, id_ in enumerate(args.ID):
if id_ == 'L':
args.ID[i] = collection.get_lowest() - 1
elif id_ == 'H':
args.ID[i] = collection.get_highest() + 1
if len(args.ID) == 2 and args.ID[1] == 'INF':
start = args.ID[0] - 1
if not args.ncli:
try:
gen = xml_matches.UpdateGenerator(start,
relax=args.relax)
except urllib.error.URLError as err:
fs = 'initial connection failed: STOPPED'
collection.logger.error(fs)
return
else:
try:
gen = fumnotifwscli.UpdateGenerator(start,
host=args.nhost, port=args.nport)
except socket.error:
fs = 'initial connection failed: STOPPED'
collection.logger.error(fs)
return
ids = iter(gen)
next(ids) # skip first ID since we have it already
elif len(args.ID) == 2:
if args.ID[0] <= args.ID[1]:
ids = iter(range(args.ID[0], args.ID[1] + 1))
else:
ids_range = range(args.ID[1], args.ID[0] + 1)
ids = reversed(ids_range)
else:
ids = iter(range(args.ID[0], args.ID[0] + 1))
rit = get_relax_iterator()
while True:
try:
id_ = next(ids)
except StopIteration:
break
except KeyboardInterrupt:
fs = '{:0>8}: keyboard interrupted: STOPPED'
collection.logger.info(fs.format(id_))
break
except urllib.error.URLError as err:
relax = next(rit)
fs = 'id refresh failed: {}; relax: {} seconds'
collection.logger.warning(fs.format(err, relax))
time.sleep(relax)
gen = xml_matches.UpdateGenerator(prev_id,
relax=args.relax)
ids = iter(gen)
else:
try:
result, errors = method(id_,
attempts=args.attempts,
nores_attempts=args.nores_attempts,
timeout=args.timeout)
except KeyboardInterrupt:
fs = '{:0>8}: keyboard interrupted: STOPPED'
collection.logger.info(fs.format(id_))
break
except exc.NoResultError:
if args.nores_stop:
fs = '{:0>8}: no result: STOPPED'
collection.logger.error(fs.format(id_))
break
else:
fs = '{:0>8}: no result'
collection.logger.warning(fs.format(id_))
except Exception as err:
relax = next(rit)
fs = '{:0>8}: exception: {}; relax: {} seconds'
collection.logger.error(fs.format(id_, err, relax))
time.sleep(relax)
ids = itertools.chain((id_,), ids)
continue
else:
if result is True:
fs = '{:0>8}: already exists'
collection.logger.warning(fs.format(id_))
prev_id = id_
rit = get_relax_iterator()
