def create_json(root): """ Create the json file containing reading data. Params: root string """ data_dir = defaults.downloads(root) output_dir = defaults.json_dir(root) archive = defaults.data_archive(root) catalog = [] data = [] json_file = {} data_files = utils.get_files_in_dir(data_dir) if not data_files: utils.warn('No csv files to process. Terminating') exit() utils.print_time('PROCESSOR START') print('Begin JSON file generation') for data_file in data_files: with open(data_file, 'rb') as f: reader = csv.reader(f) meterId, meterName = reader.next() print('Processing meterId %s ...' % (meterId)), info = {'meterId': meterId, 'meterName': meterName} catalog.append(info) for row in reader: ts = row[0] val = float(row[1]) reading = {'timestamp': ts, 'value': val, 'meterId': meterId} data.append(reading) print('done') utils.move(data_file, archive) json_file['datasource'] = defaults.URI json_file['meterCatalog'] = catalog json_file['readings'] = data print('End JSON file generation') curr_dt = datetime.now() json_fname = 'dump_%s.json' % (utils.format_dt(curr_dt)) save_path = os.path.join(output_dir, json_fname) print('Writing JSON to file %s ...' % (save_path)), with open(save_path, 'wb') as out: json.dump(json_file, out) print('done') utils.print_time('PROCESSOR END')
def run_batch(root, start, end, idx=None): """ Run this script in batch mode. Download reading data whose timestamps lie within start and end dates. The date must follow the following format (Note the T between date and time): YYYY-MM-DDTHH:MM:SS where 24 hour time is used. If idx is a non-negative integer, instead download the meter at that index. idx is zero-indexed. If idx is greater than the number of meters, nothing happens; no files are downloaded. Default behavior is to download data for all meters. Params: root string start string end string idx integer """ s_date = get_date(start) e_date = get_date(end) if not s_date or not e_date: raise ValueError('Invalid/missing dates') elif start > end: raise ValueError('Start date must come before end date') elif not utils.exists_dir(root): raise ValueError('Root directory not found') elif idx is not None and not is_valid_index(idx): raise ValueError('Index must be non-negative integer') creds_file = defaults.creds(root) cnxn_str = utils.get_cnxn_str(creds_file) output_dir = defaults.downloads(root) meter_file = defaults.meter_file(root) utils.print_time('GETTER START') with Cursor.Cursor(cnxn_str) as cursor: dq = get_reading_from_name_query_str() meters = utils.read_meter_file(meter_file) for i, m in enumerate(meters): if idx is not None and idx != i: continue ion_name = utils.get_ion_name(m) qid = utils.get_ion_qid(m) try: cursor.execute(dq, ion_name, qid, str(s_date), str(e_date)) except pyodbc.Error: utils.error( 'Problem with query to get data for meter %s qid %d' % (ion_name, qid)) continue if cursor.rowcount == 0: utils.warn('No data found for meter %s qid %d' % (ion_name, qid)) continue meterId, meterName = utils.get_lucid_id_and_name(m) s_date_str = utils.make_lucid_ts(str(s_date)) e_date_str = utils.make_lucid_ts(str(e_date)) dl_fname = "%sT%sT%s.csv" % (meterId, s_date_str, e_date_str) path = os.path.join(output_dir, dl_fname) print('Writing data for meter %s qid %d to file: %s ...' % (ion_name, qid, path)), with open(path, 'wb') as data_file: writer = csv.writer(data_file) writer.writerow([meterId, meterName]) for row in cursor: ts = row.TimestampUTC val = row.Value data_row = [utils.make_lucid_ts(ts), val] writer.writerow(data_row) print('done') utils.print_time('GETTER END')
def run_batch(root, start, end, idx=None): """ Run this script in batch mode. Download reading data whose timestamps lie within start and end dates. The date must follow the following format (Note the T between date and time): YYYY-MM-DDTHH:MM:SS where 24 hour time is used. If idx is a non-negative integer, instead download the meter at that index. idx is zero-indexed. If idx is greater than the number of meters, nothing happens; no files are downloaded. Default behavior is to download data for all meters. Params: root string start string end string idx integer """ s_date = get_date(start) e_date = get_date(end) if not s_date or not e_date: raise ValueError('Invalid/missing dates') elif start > end: raise ValueError('Start date must come before end date') elif not utils.exists_dir(root): raise ValueError('Root directory not found') elif idx is not None and not is_valid_index(idx): raise ValueError('Index must be non-negative integer') creds_file = defaults.creds(root) cnxn_str = utils.get_cnxn_str(creds_file) output_dir = defaults.downloads(root) meter_file = defaults.meter_file(root) utils.print_time('GETTER START') with Cursor.Cursor(cnxn_str) as cursor: dq = get_reading_from_name_query_str() meters = utils.read_meter_file(meter_file) for i, m in enumerate(meters): if idx is not None and idx != i: continue ion_name = utils.get_ion_name(m) qid = utils.get_ion_qid(m) try: cursor.execute(dq, ion_name, qid, str(s_date), str(e_date)) except pyodbc.Error: utils.error('Problem with query to get data for meter %s qid %d' % (ion_name, qid)) continue if cursor.rowcount == 0: utils.warn('No data found for meter %s qid %d' % (ion_name, qid)) continue meterId, meterName = utils.get_lucid_id_and_name(m) s_date_str = utils.make_lucid_ts(str(s_date)) e_date_str = utils.make_lucid_ts(str(e_date)) dl_fname = "%sT%sT%s.csv" % (meterId, s_date_str, e_date_str) path = os.path.join(output_dir, dl_fname) print('Writing data for meter %s qid %d to file: %s ...' % (ion_name, qid, path)), with open(path, 'wb') as data_file: writer = csv.writer(data_file) writer.writerow([meterId, meterName]) for row in cursor: ts = row.TimestampUTC val = row.Value data_row = [utils.make_lucid_ts(ts), val] writer.writerow(data_row) print('done') utils.print_time('GETTER END')