def insert_data( known_columns ):
sensor_already_reported = dict()
for instrument in data:
if instrument == get_header_key():
continue
for timestamp in data[instrument]:
values = dict()
for field in data[instrument][timestamp]:
if field in known_columns:
value = data[instrument][timestamp][field]
if is_number( value ):
values[field] = value
#print( 'debug: found', field, value )
else:
if field not in sensor_already_reported:
print( 'WARNING: data contains field with no database match:', field )
sensor_already_reported[field] = True
for field in values:
print( 'Inserting', site_id, timestamp, field, values[field] )
stmt = 'update data set ' + field + '=%s'
stmt += ' where data_time=%s and site_id=%s'
cur.execute( stmt, (values[field], timestamp, site_id) )
def get_unique_timestamps():
times = dict()
for instrument in data:
if instrument == get_header_key():
continue
for timestamp in data[instrument]:
times[timestamp] = True
return times
if error:
print(error, file=sys.stderr)
sys.exit(1)
results = dict()
results['total'] = dict()
results['total']['records'] = 0
results['total']['min'] = ''
results['total']['max'] = ''
total_first_time = True
total_sortable_min = ''
total_sortable_max = ''
for instrument in sorted(data.keys()):
if instrument == get_header_key():
pass
else:
n_records = 0
min_time = ''
max_time = ''
first_time = True
sortable_min = ''
sortable_max = ''
for time in data[instrument]:
n_records += 1
error += date_is_ok('Minimum', min_date)
error += date_is_ok('Maximum', max_date)
if error:
print(error, file=sys.stderr)
sys.exit(1)
else:
if min_date > max_date:
print('Min date greater than max date', file=sys.stderr)
sys.exit(1)
error, data = read_framer_results()
if error:
print(error, file=sys.stderr)
sys.exit(1)
header_key = get_header_key()
# make a new file
results = dict()
for instrument in sorted(data.keys()):
if instrument == header_key:
results[instrument] = copy.deepcopy(data[instrument])
results[instrument]['date select'] = True
results[instrument]['min date'] = min_date
results[instrument]['max date'] = max_date
if 'operations' in results[instrument]:
op = results[instrument]['operations']
results[instrument]['operations'] = op + '/date-select'
else:
# the output will be tab separated
# get all the timestamps to be sorted that way
# all the field names
# all the instruments
field_names = dict()
times = dict()
instruments = dict()
# if there is an error while handling the data, just let it go
# because this data format should be well defined
# and not going to strip the strings either
for instrument in data:
# skip this metadata item
if instrument == get_header_key(): continue
instruments[instrument] = True
for time in data[instrument]:
times[time] = True
for field in data[instrument][time]:
field_names[field] = True
# make sorted arrays out of those unique keys
sorted_field_names = []
sorted_times = []
sorted_instruments = []
for item in sorted(field_names.keys()):