def main(**kwargs):
"""
Command-line arguments are created and parsed, fed to functions.
Parameters
----------
session : SQLAlchemy.orm.Session instance
tables : list
Canonical names for desired database tables.
prefix : str
Target core tables using 'account.prefix' naming.
absolute_paths : bool
bbfk : bool
Pull in deast & dnorth info from user7 & user8 header fields
file_list : str
Name of a text file containing full SAC file names.
files : list
List of SAC file names.
"""
print("sac2db: {}".format(kwargs))
session = kwargs['session']
if kwargs.get('file_list'):
files = get_files(kwargs['file_list'], file_check=_is_sac)
else:
files = kwargs['files']
tables = get_or_create_tables(session, create=True)
lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid']
last = get_lastids(session, tables['lastid'], lastids, create=True)
# for item in iterable:
for sacfile in files:
print(sacfile)
# row_dicts = get_row_dicts(item)
tr = read(sacfile, format='SAC', debug_headers=True)[0]
# sachdr2tables produces table dictionaries
# rows needs to be a dict of lists, for make_atomic
dicts = sac.sachdr2tables(tr.stats.sac, tables=tables.keys())
rows = dicts2rows(dicts, tables)
# manage dir, dfile, datatype
# XXX: hack. replace with an updated obspy.io.sac later.
bo = tr.data.dtype.byteorder
if bo == '<':
datatype = 'f4'
elif bo == '>':
datatype = 't4'
elif bo == '=':
if sys.byteorder == 'little':
datatype = 'f4'
else:
datatype = 't4'
for wf in rows['wfdisc']:
wf.datatype = datatype
wf.dfile = os.path.basename(sacfile)
if kwargs['absolute_paths']:
idir = os.path.dirname(os.path.realpath(sacfile))
else:
idir = os.path.dirname(sacfile)
# make sure relative paths are non-empty
if idir == '':
idir = '.'
wf.dir = idir
# manage the ids
make_atomic(last, **rows)
# plugins = get_plugins(options)
# rows = apply_plugins(plugins, **rows)
# add rows to the database
# XXX: not done very elegantly. some problem rows are simply skipped.
for table, instances in rows.items():
if instances:
# could be empty []
try:
session.add_all(instances)
session.commit()
except exc.IntegrityError as e:
# duplicate or nonexistant primary keys
session.rollback()
print("rollback {}".format(table))
except exc.OperationalError as e:
# no such table, or database is locked
session.rollback()
print("rollback {}".format(table))
def main(argv=None):
"""
Command-line arguments are created and parsed, fed to functions.
"""
parser = get_parser()
options = parser.parse_args(argv)
# options_dict = vars(options)
session = get_session(options)
files = get_files(options)
tables = get_or_create_tables(options, session, create=True)
lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid']
last = get_lastids(session, tables['lastid'], lastids, create=True)
# for item in iterable:
last_time = time.time()
countI=0
staTOT=[]
for msfile in files:
#print 'last:', last_time-time.time()
countI=countI+1
start_time = time.time()
print(msfile)
# row_dicts = get_row_dicts(item)
tr = read(msfile, format='MSEED', headonly=True)[0]
#print time.time()-start_time
# rows needs to be a dict of lists, for make_atomic
# row_dicts = get_row_dicts(tr.stats.sac) # put in the whole trace, to determine byte order?
# sac
dicts = ms.mshdr2tables(tr.stats, tables=list(tables.keys()))
#print time.time()-start_time
# row_instances = dicts_to_instances(row_dicts, tables)
rows = dicts2rows(dicts, tables)
#print time.time()-start_time
# manage dir, dfile, datatype
# XXX: hack. replace with an updated obspy.io.sac later.
bo = tr.data.dtype.byteorder
if bo == '<':
datatype = 'f4'
elif bo == '>':
datatype = 't4'
elif bo == '=':
if sys.byteorder == 'little':
datatype = 'f4'
else:
datatype = 't4'
for wf in rows['wfdisc']:
wf.datatype = 'ms'
wf.dfile = os.path.basename(msfile)
if options.absolute_paths:
idir = os.path.dirname(os.path.realpath(msfile))
else:
idir = os.path.dirname(msfile)
wf.dir = idir
# manage the ids
#print time.time()-start_time
make_atomic(last, **rows)
#plugins = get_plugins(options)
#rows = apply_plugins(plugins, **rows)
# add rows to the database
# XXX: not done very elegantly. some problem rows are simply skipped.
#countI=0
#staTOT=[]
for table, instances in list(rows.items()):
#print table
if not table=='site':
continue
if instances[0].sta in staTOT:
continue
if instances:
# could be empty []
try:
session.add_all(instances)
session.commit()
staTOT.append(instances[0].sta)
except exc.IntegrityError as e:
# duplicate or nonexistant primary keys
session.rollback()
print("rollback {}".format(table))
except exc.OperationalError as e:
# no such table, or database is locked
session.rollback()
print("rollback {}".format(table))
#countI=0
for table, instances in list(rows.items()):
#print table
if not table=='wfdisc':
continue
if instances:
try:
session.add_all(instances)
#countI=countI+1
if countI>100:
print('commiting')
session.commit()
countI=0
except exc.IntegrityError as e:
# duplicate or nonexistant primary keys
session.rollback()
print("rollback {}".format(table))
except exc.OperationalError as e:
# no such table, or database is locked
session.rollback()
print("rollback {}".format(table))
#print time.time()-start_time
'''
if (time.time()-start_time)> 0.5:
print 'slow'
embed()
exit()
'''
last_time = time.time()
def main(argv=None):
"""
Command-line arguments are created and parsed, fed to functions.
"""
parser = get_parser()
options = parser.parse_args(argv)
# options_dict = vars(options)
session = get_session(options)
files = get_files(options)
tables = get_or_create_tables(options, session, create=True)
lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid']
last = get_lastids(session, tables['lastid'], lastids, create=True)
# for item in iterable:
for sacfile in files:
print sacfile
# row_dicts = get_row_dicts(item)
tr = read(sacfile, format='SAC', debug_headers=True)[0]
# rows needs to be a dict of lists, for make_atomic
# row_dicts = get_row_dicts(tr.stats.sac) # put in the whole trace, to determine byte order?
dicts = sac.sachdr2tables(tr.stats.sac, tables=tables.keys())
# row_instances = dicts_to_instances(row_dicts, tables)
rows = dicts2rows(dicts, tables)
# manage dir, dfile, datatype
# XXX: hack. replace with an updated obspy.io.sac later.
bo = tr.data.dtype.byteorder
if bo == '<':
datatype = 'f4'
elif bo == '>':
datatype = 't4'
elif bo == '=':
if sys.byteorder == 'little':
datatype = 'f4'
else:
datatype = 't4'
for wf in rows['wfdisc']:
wf.datatype = datatype
wf.dfile = os.path.basename(sacfile)
if options.absolute_paths:
idir = os.path.dirname(os.path.realpath(sacfile))
else:
idir = os.path.dirname(sacfile)
wf.dir = idir
# manage the ids
make_atomic(last, **rows)
plugins = get_plugins(options)
rows = apply_plugins(plugins, **rows)
# add rows to the database
# XXX: not done very elegantly. some problem rows are simply skipped.
for table, instances in rows.items():
if instances:
# could be empty []
try:
session.add_all(instances)
session.commit()
except exc.IntegrityError as e:
# duplicate or nonexistant primary keys
session.rollback()
print("rollback {}".format(table))
except exc.OperationalError as e:
# no such table, or database is locked
session.rollback()
print("rollback {}".format(table))
def main(**kwargs):
"""
Command-line arguments are created and parsed, fed to functions.
Parameters
----------
session : SQLAlchemy.orm.Session instance
tables : list
Canonical names for desired database tables.
prefix : str
Target core tables using 'account.prefix' naming.
absolute_paths : bool
file_list : str
Name of a text file containing full SAC file names.
files : list
List of SAC file names.
"""
print("sac2db: {}".format(kwargs))
session = kwargs['session']
if kwargs.get('file_list'):
files = get_files(kwargs['file_list'])
else:
files = kwargs['files']
tables = get_or_create_tables(session, create=True)
lastids = ['arid', 'chanid', 'evid', 'orid', 'wfid']
last = get_lastids(session, tables['lastid'], lastids, create=True)
# for item in iterable:
for sacfile in files:
print(sacfile)
# row_dicts = get_row_dicts(item)
tr = read(sacfile, format='SAC', debug_headers=True)[0]
# rows needs to be a dict of lists, for make_atomic
# row_dicts = get_row_dicts(tr.stats.sac) # put in the whole trace, to determine byte order?
dicts = sac.sachdr2tables(tr.stats.sac, tables=tables.keys())
# row_instances = dicts_to_instances(row_dicts, tables)
rows = dicts2rows(dicts, tables)
# manage dir, dfile, datatype
# XXX: hack. replace with an updated obspy.io.sac later.
bo = tr.data.dtype.byteorder
if bo == '<':
datatype = 'f4'
elif bo == '>':
datatype = 't4'
elif bo == '=':
if sys.byteorder == 'little':
datatype = 'f4'
else:
datatype = 't4'
for wf in rows['wfdisc']:
wf.datatype = datatype
wf.dfile = os.path.basename(sacfile)
if kwargs['absolute_paths']:
idir = os.path.dirname(os.path.realpath(sacfile))
else:
idir = os.path.dirname(sacfile)
wf.dir = idir
# manage the ids
make_atomic(last, **rows)
# plugins = get_plugins(options)
# rows = apply_plugins(plugins, **rows)
# add rows to the database
# XXX: not done very elegantly. some problem rows are simply skipped.
for table, instances in rows.items():
if instances:
# could be empty []
try:
session.add_all(instances)
session.commit()
except exc.IntegrityError as e:
# duplicate or nonexistant primary keys
session.rollback()
print("rollback {}".format(table))
except exc.OperationalError as e:
# no such table, or database is locked
session.rollback()
print("rollback {}".format(table))
def main(session, files=None, file_list=None, prefix=None, absolute_paths=False):
"""
session : SQLAlchemy.orm.Session instance
tables : list
Canonical names for desired database tables.
prefix : str
Target core tables using 'account.prefix' naming.
absolute_paths : bool
"""
if file_list:
files = get_files(file_list)
else:
files = files
tables = get_or_create_tables(session, prefix=prefix, create=True)
lastids = ['chanid', 'wfid']
last = get_lastids(session, tables['lastid'], lastids, create=True)
for msfile in files:
print(msfile)
# row_dicts = get_row_dicts(item)
st = read(msfile, format='MSEED', details=True, headonly=True)
# a single miniSEED file may be multiple traces. we need to identify the
# file offsets (bytes) to each trace. Traces are added to the Stream in
# order, with enough header information to calculate that.
foff = 0
for tr in st:
rows = mseed.mseedhdr2tables(tr.stats, wfdisc=tables['wfdisc'],
site=tables['site'], sitechan=tables['sitechan'],
affiliation=tables['affiliation'])
foff += tr.stats.mseed.number_of_records * tr.stats.mseed.record_length
rows['wfdisc'][0].foff = foff
rows['wfdisc'][0].dfile = os.path.basename(msfile)
if absolute_paths:
idir = os.path.dirname(os.path.realpath(msfile))
else:
idir = os.path.dirname(msfile)
# make sure relative paths are non-empty
if idir == '':
idir = '.'
rows['wfdisc'][0].dir = idir
# manage the ids
make_atomic(last, **rows)
# add rows to the database
# XXX: not done very elegantly. some problem rows are simply skipped.
for table, instances in rows.items():
if instances:
# could be empty []
try:
session.add_all(instances)
session.commit()
except exc.IntegrityError as e:
# duplicate or nonexistant primary keys
session.rollback()
print("rollback {}".format(table))
except exc.OperationalError as e:
# no such table, or database is locked
session.rollback()
print("rollback {}".format(table))
def main(session,
files=None,
file_list=None,
prefix=None,
absolute_paths=False):
"""
session : SQLAlchemy.orm.Session instance
tables : list
Canonical names for desired database tables.
prefix : str
Target core tables using 'account.prefix' naming.
absolute_paths : bool
"""
if file_list:
files = get_files(file_list)
else:
files = files
tables = get_or_create_tables(session, prefix=prefix, create=True)
lastids = ['chanid', 'wfid']
last = get_lastids(session, tables['lastid'], lastids, create=True)
for msfile in files:
print(msfile)
# row_dicts = get_row_dicts(item)
st = read(msfile, format='MSEED', details=True, headonly=True)
# a single miniSEED file may be multiple traces. we need to identify the
# file offsets (bytes) to each trace. Traces are added to the Stream in
# order, with enough header information to calculate that.
foff = 0
for tr in st:
rows = mseed.mseedhdr2tables(tr.stats,
wfdisc=tables['wfdisc'],
site=tables['site'],
sitechan=tables['sitechan'],
affiliation=tables['affiliation'])
foff += tr.stats.mseed.number_of_records * tr.stats.mseed.record_length
rows['wfdisc'][0].foff = foff
rows['wfdisc'][0].dfile = os.path.basename(msfile)
if absolute_paths:
idir = os.path.dirname(os.path.realpath(msfile))
else:
idir = os.path.dirname(msfile)
# make sure relative paths are non-empty
if idir == '':
idir = '.'
rows['wfdisc'][0].dir = idir
# manage the ids
make_atomic(last, **rows)
# add rows to the database
# XXX: not done very elegantly. some problem rows are simply skipped.
for table, instances in rows.items():
if instances:
# could be empty []
try:
session.add_all(instances)
session.commit()
except exc.IntegrityError as e:
# duplicate or nonexistant primary keys
session.rollback()
print("rollback {}".format(table))
except exc.OperationalError as e:
# no such table, or database is locked
session.rollback()
print("rollback {}".format(table))
