def inject_from_master(records_json, database_connection, check_only, no_check, sequencer_only, _eval, abort_on_error):
"""
This is the main function that creates datastructures from files, controls main classes instances and saves products
record_json is a file containing a list of records.
database_connection is a file containing connection parameters.
"""
check_prefixes(products['prefixes'].values())
# set logging
log = products.get('log', None)
logging = logger.getLogger(filename=log, prefix=products['prefixes'].get('log', None), screen=logger.WARNING,
color=True)
with logging as log:
log.info(
"Launched %s records_json=%s, database_connection=%s. Options: check_only=%s, no_check=%s, sequencer_only=%s" %\
(sys.argv[0], records_json, database_connection, check_only, no_check, sequencer_only))
if sequencer_only:
no_check = True
if no_check and check_only:
raise RuntimeError("Incompatible options <check_only> and <no_check>")
# check files access
if not os.access(records_json, os.R_OK):
raise RuntimeError('JSON records file <%s> not found' % records_json)
if not os.access(database_connection, os.R_OK):
raise RuntimeError('Connection file <%s> not found' % database_connection)
flat_seq = check_file_access(products, 'flat_seq', ext=time.strftime('_%Y-%m-%d_%H-%M-%S.json'), required=False,
create=True)
if not sequencer_only:
flat_inj = check_file_access(products, 'flat_inj', ext=time.strftime('_%Y-%m-%d_%H-%M-%S.json'),
required=False, create=True)
connection = json_load(database_connection)
# load records, mapper and sequencer and launch injection
if _eval:
records = json_load(records_json)
else:
records = json_load(records_json, 'json')
if not (isinstance(records, list) and isinstance(records[0], dict)):
raise RuntimeError('JSON records file <%s> should be a list of dictionaries' % records_json)
if sequencer_only:
injector = None
no_check = True
else:
dsn = connection['dsn'] % connection['connection_context']
kwargs = dict(records_json=records_json, connection=connection)
injector = Injector(dsn, kwargs, log)
mapseq = myMapperSequencer(log=log, imports=(('datetime', ('datetime',)),), aoe=abort_on_error)
if no_check or injector.check_mapping(mapseq.mapper):
if not check_only:
if injector:
injector.prepare_session(myMapperSequencer.mapper).prepare_injection()
Shell(mapseq, injector)
result = mapseq.multi_process_records(records)
json_dump(mapseq.flat, flat_seq)
if not sequencer_only:
json_dump(injector.flat, flat_inj)
