async def resolve_deviant(deviant):
try:
deviant, _group = manager.get_dagr().resolve_deviant(deviant)
return deviant
except DagrException:
logger.warning(f"Unable to resolve deviant {deviant}")
raise web.HTTPBadRequest(reason='not ok: unable to resolve deviant')
async def process_item(item):
try:
item.process()
http_errors = manager.get_dagr().report_http_errors()
if http_errors.get(400, 0) > 1:
raise Exception('Detected 400 error(s)')
except:
logger.exception('Error while processing item')
try:
manager.get_cache().update('error_items', item.params)
except:
logger.exception('Error while saving error item')
try:
flush_errors_to_queue()
except:
pass
async def __main__():
manager.set_stop_check(check_stop_file)
with manager.get_dagr() as dagr:
logger.info('Flushing previous errors')
flush_errors_to_queue()
logger.info("Worker ready")
while not check_stop_file('STOP_WORKER'):
logger.info("Fetching work item")
item = await fetch_item()
if not item is None:
logger.info(f"Got work item {item.params}")
await process_item(item)
dagr.print_errors()
dagr.print_dl_total()
dagr.reset_stats()
else:
logger.warning('Unable to fetch workitem')
sleep(30)
deviant = input('Enter username: '******'Enter mode: ')
from functions import rip_queue, manager
manager.set_mode('rip_queue')
manager.init_logging()
with manager.get_dagr():
manager.get_browser().do_login()
rip_queue(mode, deviant)