def insert_data( known_columns ): sensor_already_reported = dict() for instrument in data: if instrument == get_header_key(): continue for timestamp in data[instrument]: values = dict() for field in data[instrument][timestamp]: if field in known_columns: value = data[instrument][timestamp][field] if is_number( value ): values[field] = value #print( 'debug: found', field, value ) else: if field not in sensor_already_reported: print( 'WARNING: data contains field with no database match:', field ) sensor_already_reported[field] = True for field in values: print( 'Inserting', site_id, timestamp, field, values[field] ) stmt = 'update data set ' + field + '=%s' stmt += ' where data_time=%s and site_id=%s' cur.execute( stmt, (values[field], timestamp, site_id) )
def get_unique_timestamps(): times = dict() for instrument in data: if instrument == get_header_key(): continue for timestamp in data[instrument]: times[timestamp] = True return times
if error: print(error, file=sys.stderr) sys.exit(1) results = dict() results['total'] = dict() results['total']['records'] = 0 results['total']['min'] = '' results['total']['max'] = '' total_first_time = True total_sortable_min = '' total_sortable_max = '' for instrument in sorted(data.keys()): if instrument == get_header_key(): pass else: n_records = 0 min_time = '' max_time = '' first_time = True sortable_min = '' sortable_max = '' for time in data[instrument]: n_records += 1
error += date_is_ok('Minimum', min_date) error += date_is_ok('Maximum', max_date) if error: print(error, file=sys.stderr) sys.exit(1) else: if min_date > max_date: print('Min date greater than max date', file=sys.stderr) sys.exit(1) error, data = read_framer_results() if error: print(error, file=sys.stderr) sys.exit(1) header_key = get_header_key() # make a new file results = dict() for instrument in sorted(data.keys()): if instrument == header_key: results[instrument] = copy.deepcopy(data[instrument]) results[instrument]['date select'] = True results[instrument]['min date'] = min_date results[instrument]['max date'] = max_date if 'operations' in results[instrument]: op = results[instrument]['operations'] results[instrument]['operations'] = op + '/date-select' else:
# the output will be tab separated # get all the timestamps to be sorted that way # all the field names # all the instruments field_names = dict() times = dict() instruments = dict() # if there is an error while handling the data, just let it go # because this data format should be well defined # and not going to strip the strings either for instrument in data: # skip this metadata item if instrument == get_header_key(): continue instruments[instrument] = True for time in data[instrument]: times[time] = True for field in data[instrument][time]: field_names[field] = True # make sorted arrays out of those unique keys sorted_field_names = [] sorted_times = [] sorted_instruments = [] for item in sorted(field_names.keys()):