def main(**kwargs): """ Command-line arguments are created and parsed, fed to functions. Parameters ---------- session : SQLAlchemy.orm.Session instance tables : list Canonical names for desired database tables. prefix : str Target core tables using 'account.prefix' naming. absolute_paths : bool bbfk : bool Pull in deast & dnorth info from user7 & user8 header fields file_list : str Name of a text file containing full SAC file names. files : list List of SAC file names. """ print("sac2db: {}".format(kwargs)) session = kwargs['session'] if kwargs.get('file_list'): files = get_files(kwargs['file_list'], file_check=_is_sac) else: files = kwargs['files'] tables = get_or_create_tables(session, create=True) lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid'] last = get_lastids(session, tables['lastid'], lastids, create=True) # for item in iterable: for sacfile in files: print(sacfile) # row_dicts = get_row_dicts(item) tr = read(sacfile, format='SAC', debug_headers=True)[0] # sachdr2tables produces table dictionaries # rows needs to be a dict of lists, for make_atomic dicts = sac.sachdr2tables(tr.stats.sac, tables=tables.keys()) rows = dicts2rows(dicts, tables) # manage dir, dfile, datatype # XXX: hack. replace with an updated obspy.io.sac later. bo = tr.data.dtype.byteorder if bo == '<': datatype = 'f4' elif bo == '>': datatype = 't4' elif bo == '=': if sys.byteorder == 'little': datatype = 'f4' else: datatype = 't4' for wf in rows['wfdisc']: wf.datatype = datatype wf.dfile = os.path.basename(sacfile) if kwargs['absolute_paths']: idir = os.path.dirname(os.path.realpath(sacfile)) else: idir = os.path.dirname(sacfile) # make sure relative paths are non-empty if idir == '': idir = '.' wf.dir = idir # manage the ids make_atomic(last, **rows) # plugins = get_plugins(options) # rows = apply_plugins(plugins, **rows) # add rows to the database # XXX: not done very elegantly. some problem rows are simply skipped. for table, instances in rows.items(): if instances: # could be empty [] try: session.add_all(instances) session.commit() except exc.IntegrityError as e: # duplicate or nonexistant primary keys session.rollback() print("rollback {}".format(table)) except exc.OperationalError as e: # no such table, or database is locked session.rollback() print("rollback {}".format(table))
def main(argv=None): """ Command-line arguments are created and parsed, fed to functions. """ parser = get_parser() options = parser.parse_args(argv) # options_dict = vars(options) session = get_session(options) files = get_files(options) tables = get_or_create_tables(options, session, create=True) lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid'] last = get_lastids(session, tables['lastid'], lastids, create=True) # for item in iterable: last_time = time.time() countI=0 staTOT=[] for msfile in files: #print 'last:', last_time-time.time() countI=countI+1 start_time = time.time() print(msfile) # row_dicts = get_row_dicts(item) tr = read(msfile, format='MSEED', headonly=True)[0] #print time.time()-start_time # rows needs to be a dict of lists, for make_atomic # row_dicts = get_row_dicts(tr.stats.sac) # put in the whole trace, to determine byte order? # sac dicts = ms.mshdr2tables(tr.stats, tables=list(tables.keys())) #print time.time()-start_time # row_instances = dicts_to_instances(row_dicts, tables) rows = dicts2rows(dicts, tables) #print time.time()-start_time # manage dir, dfile, datatype # XXX: hack. replace with an updated obspy.io.sac later. bo = tr.data.dtype.byteorder if bo == '<': datatype = 'f4' elif bo == '>': datatype = 't4' elif bo == '=': if sys.byteorder == 'little': datatype = 'f4' else: datatype = 't4' for wf in rows['wfdisc']: wf.datatype = 'ms' wf.dfile = os.path.basename(msfile) if options.absolute_paths: idir = os.path.dirname(os.path.realpath(msfile)) else: idir = os.path.dirname(msfile) wf.dir = idir # manage the ids #print time.time()-start_time make_atomic(last, **rows) #plugins = get_plugins(options) #rows = apply_plugins(plugins, **rows) # add rows to the database # XXX: not done very elegantly. some problem rows are simply skipped. #countI=0 #staTOT=[] for table, instances in list(rows.items()): #print table if not table=='site': continue if instances[0].sta in staTOT: continue if instances: # could be empty [] try: session.add_all(instances) session.commit() staTOT.append(instances[0].sta) except exc.IntegrityError as e: # duplicate or nonexistant primary keys session.rollback() print("rollback {}".format(table)) except exc.OperationalError as e: # no such table, or database is locked session.rollback() print("rollback {}".format(table)) #countI=0 for table, instances in list(rows.items()): #print table if not table=='wfdisc': continue if instances: try: session.add_all(instances) #countI=countI+1 if countI>100: print('commiting') session.commit() countI=0 except exc.IntegrityError as e: # duplicate or nonexistant primary keys session.rollback() print("rollback {}".format(table)) except exc.OperationalError as e: # no such table, or database is locked session.rollback() print("rollback {}".format(table)) #print time.time()-start_time ''' if (time.time()-start_time)> 0.5: print 'slow' embed() exit() ''' last_time = time.time()
def main(argv=None): """ Command-line arguments are created and parsed, fed to functions. """ parser = get_parser() options = parser.parse_args(argv) # options_dict = vars(options) session = get_session(options) files = get_files(options) tables = get_or_create_tables(options, session, create=True) lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid'] last = get_lastids(session, tables['lastid'], lastids, create=True) # for item in iterable: for sacfile in files: print sacfile # row_dicts = get_row_dicts(item) tr = read(sacfile, format='SAC', debug_headers=True)[0] # rows needs to be a dict of lists, for make_atomic # row_dicts = get_row_dicts(tr.stats.sac) # put in the whole trace, to determine byte order? dicts = sac.sachdr2tables(tr.stats.sac, tables=tables.keys()) # row_instances = dicts_to_instances(row_dicts, tables) rows = dicts2rows(dicts, tables) # manage dir, dfile, datatype # XXX: hack. replace with an updated obspy.io.sac later. bo = tr.data.dtype.byteorder if bo == '<': datatype = 'f4' elif bo == '>': datatype = 't4' elif bo == '=': if sys.byteorder == 'little': datatype = 'f4' else: datatype = 't4' for wf in rows['wfdisc']: wf.datatype = datatype wf.dfile = os.path.basename(sacfile) if options.absolute_paths: idir = os.path.dirname(os.path.realpath(sacfile)) else: idir = os.path.dirname(sacfile) wf.dir = idir # manage the ids make_atomic(last, **rows) plugins = get_plugins(options) rows = apply_plugins(plugins, **rows) # add rows to the database # XXX: not done very elegantly. some problem rows are simply skipped. for table, instances in rows.items(): if instances: # could be empty [] try: session.add_all(instances) session.commit() except exc.IntegrityError as e: # duplicate or nonexistant primary keys session.rollback() print("rollback {}".format(table)) except exc.OperationalError as e: # no such table, or database is locked session.rollback() print("rollback {}".format(table))
def main(**kwargs): """ Command-line arguments are created and parsed, fed to functions. Parameters ---------- session : SQLAlchemy.orm.Session instance tables : list Canonical names for desired database tables. prefix : str Target core tables using 'account.prefix' naming. absolute_paths : bool file_list : str Name of a text file containing full SAC file names. files : list List of SAC file names. """ print("sac2db: {}".format(kwargs)) session = kwargs['session'] if kwargs.get('file_list'): files = get_files(kwargs['file_list']) else: files = kwargs['files'] tables = get_or_create_tables(session, create=True) lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid'] last = get_lastids(session, tables['lastid'], lastids, create=True) # for item in iterable: for sacfile in files: print(sacfile) # row_dicts = get_row_dicts(item) tr = read(sacfile, format='SAC', debug_headers=True)[0] # rows needs to be a dict of lists, for make_atomic # row_dicts = get_row_dicts(tr.stats.sac) # put in the whole trace, to determine byte order? dicts = sac.sachdr2tables(tr.stats.sac, tables=tables.keys()) # row_instances = dicts_to_instances(row_dicts, tables) rows = dicts2rows(dicts, tables) # manage dir, dfile, datatype # XXX: hack. replace with an updated obspy.io.sac later. bo = tr.data.dtype.byteorder if bo == '<': datatype = 'f4' elif bo == '>': datatype = 't4' elif bo == '=': if sys.byteorder == 'little': datatype = 'f4' else: datatype = 't4' for wf in rows['wfdisc']: wf.datatype = datatype wf.dfile = os.path.basename(sacfile) if kwargs['absolute_paths']: idir = os.path.dirname(os.path.realpath(sacfile)) else: idir = os.path.dirname(sacfile) wf.dir = idir # manage the ids make_atomic(last, **rows) # plugins = get_plugins(options) # rows = apply_plugins(plugins, **rows) # add rows to the database # XXX: not done very elegantly. some problem rows are simply skipped. for table, instances in rows.items(): if instances: # could be empty [] try: session.add_all(instances) session.commit() except exc.IntegrityError as e: # duplicate or nonexistant primary keys session.rollback() print("rollback {}".format(table)) except exc.OperationalError as e: # no such table, or database is locked session.rollback() print("rollback {}".format(table))
def main(session, files=None, file_list=None, prefix=None, absolute_paths=False): """ session : SQLAlchemy.orm.Session instance tables : list Canonical names for desired database tables. prefix : str Target core tables using 'account.prefix' naming. absolute_paths : bool """ if file_list: files = get_files(file_list) else: files = files tables = get_or_create_tables(session, prefix=prefix, create=True) lastids = ['chanid', 'wfid'] last = get_lastids(session, tables['lastid'], lastids, create=True) for msfile in files: print(msfile) # row_dicts = get_row_dicts(item) st = read(msfile, format='MSEED', details=True, headonly=True) # a single miniSEED file may be multiple traces. we need to identify the # file offsets (bytes) to each trace. Traces are added to the Stream in # order, with enough header information to calculate that. foff = 0 for tr in st: rows = mseed.mseedhdr2tables(tr.stats, wfdisc=tables['wfdisc'], site=tables['site'], sitechan=tables['sitechan'], affiliation=tables['affiliation']) foff += tr.stats.mseed.number_of_records * tr.stats.mseed.record_length rows['wfdisc'][0].foff = foff rows['wfdisc'][0].dfile = os.path.basename(msfile) if absolute_paths: idir = os.path.dirname(os.path.realpath(msfile)) else: idir = os.path.dirname(msfile) # make sure relative paths are non-empty if idir == '': idir = '.' rows['wfdisc'][0].dir = idir # manage the ids make_atomic(last, **rows) # add rows to the database # XXX: not done very elegantly. some problem rows are simply skipped. for table, instances in rows.items(): if instances: # could be empty [] try: session.add_all(instances) session.commit() except exc.IntegrityError as e: # duplicate or nonexistant primary keys session.rollback() print("rollback {}".format(table)) except exc.OperationalError as e: # no such table, or database is locked session.rollback() print("rollback {}".format(table))