def inject_from_master(records_json, database_connection, check_only, no_check, sequencer_only, _eval, abort_on_error): """ This is the main function that creates datastructures from files, controls main classes instances and saves products record_json is a file containing a list of records. database_connection is a file containing connection parameters. """ check_prefixes(products['prefixes'].values()) # set logging log = products.get('log', None) logging = logger.getLogger(filename=log, prefix=products['prefixes'].get('log', None), screen=logger.WARNING, color=True) with logging as log: log.info( "Launched %s records_json=%s, database_connection=%s. Options: check_only=%s, no_check=%s, sequencer_only=%s" %\ (sys.argv[0], records_json, database_connection, check_only, no_check, sequencer_only)) if sequencer_only: no_check = True if no_check and check_only: raise RuntimeError("Incompatible options <check_only> and <no_check>") # check files access if not os.access(records_json, os.R_OK): raise RuntimeError('JSON records file <%s> not found' % records_json) if not os.access(database_connection, os.R_OK): raise RuntimeError('Connection file <%s> not found' % database_connection) flat_seq = check_file_access(products, 'flat_seq', ext=time.strftime('_%Y-%m-%d_%H-%M-%S.json'), required=False, create=True) if not sequencer_only: flat_inj = check_file_access(products, 'flat_inj', ext=time.strftime('_%Y-%m-%d_%H-%M-%S.json'), required=False, create=True) connection = json_load(database_connection) # load records, mapper and sequencer and launch injection if _eval: records = json_load(records_json) else: records = json_load(records_json, 'json') if not (isinstance(records, list) and isinstance(records[0], dict)): raise RuntimeError('JSON records file <%s> should be a list of dictionaries' % records_json) if sequencer_only: injector = None no_check = True else: dsn = connection['dsn'] % connection['connection_context'] kwargs = dict(records_json=records_json, connection=connection) injector = Injector(dsn, kwargs, log) mapseq = myMapperSequencer(log=log, imports=(('datetime', ('datetime',)),), aoe=abort_on_error) if no_check or injector.check_mapping(mapseq.mapper): if not check_only: if injector: injector.prepare_session(myMapperSequencer.mapper).prepare_injection() Shell(mapseq, injector) result = mapseq.multi_process_records(records) json_dump(mapseq.flat, flat_seq) if not sequencer_only: json_dump(injector.flat, flat_inj)