def make_msid_file(colname, content, content_def):
ft['content'] = content
ft['msid'] = colname
filename = msid_files['data'].abs
if os.path.exists(filename):
return
logger.info('Making MSID data file %s', filename)
if colname == 'TIME':
dp_vals, indexes = derived.times_indexes(opt.start, opt.stop,
content_def['time_step'])
else:
dp = content_def['classes'][colname]()
dataset = dp.fetch(opt.start, opt.stop)
dp_vals = np.asarray(dp.calc(dataset), dtype=dp.dtype)
# Finally make the actual MSID data file
filters = tables.Filters(complevel=5, complib='zlib')
h5 = tables.openFile(filename, mode='w', filters=filters)
n_rows = int(20 * 3e7 / content_def['time_step'])
h5shape = (0,)
h5type = tables.Atom.from_dtype(dp_vals.dtype)
h5.createEArray(h5.root, 'data', h5type, h5shape, title=colname,
expectedrows=n_rows)
h5.createEArray(h5.root, 'quality', tables.BoolAtom(), (0,), title='Quality',
expectedrows=n_rows)
logger.info('Made {} shape={} with n_rows(1e6)={}'.format(colname, h5shape, n_rows / 1.0e6))
h5.close()
def make_archfiles_db(filename, content_def):
# Do nothing if it is already there
if os.path.exists(filename):
return
datestart = DateTime(DateTime(opt.start).secs - 60)
tstart = datestart.secs
tstop = tstart
year, doy = datestart.date.split(':')[:2]
times, indexes = derived.times_indexes(tstart, tstop,
content_def['time_step'])
logger.info('Creating db {}'.format(filename))
archfiles_def = open(Path(__file__).parent / 'archfiles_def.sql').read()
db = Ska.DBI.DBI(dbi='sqlite', server=filename)
db.execute(archfiles_def)
archfiles_row = dict(
filename='{}:0:1'.format(content_def['content']),
filetime=0,
year=year,
doy=doy,
tstart=tstart,
tstop=tstop,
rowstart=0,
rowstop=0,
startmjf=indexes[0], # really index0
stopmjf=indexes[-1], # really index1
date=datestart.date)
db.insert(archfiles_row, 'archfiles')
def make_archfiles_db(filename, content_def):
# Do nothing if it is already there
if os.path.exists(filename):
return
datestart = DateTime(DateTime(opt.start).secs - 60)
tstart = datestart.secs
tstop = tstart
year, doy = datestart.date.split(':')[:2]
times, indexes = derived.times_indexes(tstart, tstop, content_def['time_step'])
logger.info('Creating db {}'.format(filename))
archfiles_def = open('archfiles_def.sql').read()
db = Ska.DBI.DBI(dbi='sqlite', server=filename)
db.execute(archfiles_def)
archfiles_row = dict(filename='{}:0:1'.format(content_def['content']),
filetime=0,
year=year,
doy=doy,
tstart=tstart,
tstop=tstop,
rowstart=0,
rowstop=0,
startmjf=indexes[0], # really index0
stopmjf=indexes[-1], # really index1
date=datestart.date)
db.insert(archfiles_row, 'archfiles')
def make_msid_file(colname, content, content_def):
ft['content'] = content
ft['msid'] = colname
filename = msid_files['data'].abs
if os.path.exists(filename):
return
logger.info('Making MSID data file %s', filename)
if colname == 'TIME':
dp_vals, indexes = derived.times_indexes(opt.start, opt.stop,
content_def['time_step'])
else:
dp = content_def['classes'][colname]()
dataset = dp.fetch(opt.start, opt.stop)
dp_vals = np.asarray(dp.calc(dataset), dtype=dp.dtype)
# Finally make the actual MSID data file
filters = tables.Filters(complevel=5, complib='zlib')
h5 = tables.open_file(filename, mode='w', filters=filters)
n_rows = int(20 * 3e7 / content_def['time_step'])
h5shape = (0, )
h5type = tables.Atom.from_dtype(dp_vals.dtype)
h5.create_earray(h5.root,
'data',
h5type,
h5shape,
title=colname,
expectedrows=n_rows)
h5.create_earray(h5.root,
'quality',
tables.BoolAtom(), (0, ),
title='Quality',
expectedrows=n_rows)
logger.info('Made {} shape={} with n_rows(1e6)={}'.format(
colname, h5shape, n_rows / 1.0e6))
h5.close()
