def main_worker(file):
dbc = Db()
file_check = dbc.file_check(file)
if file_check is None:
xml_filename = download_file(file['url'])
if xml_filename is not None:
inserted_id = dbc.file_insert(file, os.path.basename(xml_filename))
try:
parse_file(xml_filename, inserted_id)
except Exception:
logger.exception('message')
raise
elif file_check['status'] in ['new', 'reparsing'] or args.force:
logger.warning('File %s exists into database. Going to process again',
file_check['filename'])
if not os.path.isfile(os.path.join(WORK_DIR, file_check['filename'])):
xml_filename = download_file(file['url'])
else:
xml_filename = file_check['filename']
try:
parse_file(xml_filename, file_check['id'])
except Exception:
logger.exception('message')
raise
else:
logger.info('File %s is already inserted into database. Skiping it',
file_check['filename'])
if args.parse:
logger.info('Nothing to work. Exiting.')
exit()
def sub_main(args):
files_tuple = get_urls(MAIN_URL)
dbc = Db()
for file in files_tuple:
file_check = dbc.file_check(file)
if file_check is None:
xml_filename = download_file(file['url'])
if xml_filename is not None:
inserted_id = dbc.file_insert(file, xml_filename)
parse_file(xml_filename, inserted_id)
elif file_check['status'] == 'new':
logger.warning('File %s exists into database. Going to process again', file_check['filename'])
if not os.path.isfile(os.path.join(WORK_DIR, file_check['filename'])):
xml_filename = download_file(file['url'])
else:
xml_filename = file_check['filename']
parse_file(xml_filename, file_check['id'])
else:
logger.info('File %s is already inserted into database.', file_check['filename'])
if args.parse:
logger.info('Nothing to work. Exiting.')
sys.exit()