def get_data_grid_rows(self): utils.debug("get data rows from data grid.") trs = self.slb.get_elements(datagrid.table_trs) if not trs: utils.warn("Not find any data gird data rows!") utils.debug("got total [{}] row data records".format(len(trs))) return trs
def request(req, session, payload=None): try: if 'GET' in req: return session.get( TARGET, verify=False, allow_redirects=False) elif 'POST' in req: return session.post( TARGET, data=payload, verify=False, allow_redirects=False) except exceptions.HTTPError as eh: print(u.error(), 'HTTPError:', eh) quit() except exceptions.ConnectionError as ec: print(u.error(), 'ConnectionError:', ec) quit() except exceptions.Timeout as et: print(u.warn(), 'Timeout:', et) except exceptions.RequestException as er: print(u.warn(), 'RequestException:', er)
def get_target_row(self, row_unique_key_name, row_unique_key): """ get target row in data grid matched unique key value(eg: type, ruleId) """ target_row, target_row_id, action_column_index = [], None, -1 trs = self.get_data_grid_rows() headers = self.get_data_grid_headers() for index, header in enumerate(headers): if header == constants.ActionsColumnName or header == constants.ActionColumnName: action_column_index = index # get the [Action] column index break for tr in trs: tds_in_tr = self.slb.get_elements_via_element(tr, datagrid.td) target_row_id = self.slb.get_element_text( tds_in_tr[0]) # first column value, help find row on ui for td in tds_in_tr: if self.slb.get_element_text(td) == row_unique_key: utils.log("find the data record with {}={}".format( row_unique_key_name, row_unique_key)) target_row.append(tr) break if target_row: break if not target_row: utils.warn("Not find any data record with {}={} !".format( row_unique_key_name, row_unique_key)) return return target_row, target_row_id, action_column_index
def post_json_files(root): """ Post json objects in a designated directory to BuildingOS. Params: root string """ json_dir = defaults.json_dir(root) archive = defaults.json_archive(root) post_url = defaults.BOS_URL json_files = utils.get_files_in_dir(json_dir) if not json_files: utils.warn('No JSON files to process. Terminating') exit() utils.print_time('LOADER START') for json_file in json_files: print('Posting file: %s ...' % (json_file)), with open(json_file, 'rb') as jf: payload = {'data': jf} response = requests.post(post_url, files=payload) print('done') print('Server response: %s' % (response.text)) utils.move(json_file, archive) utils.print_time('LOADER END')
def create_json(root): """ Create the json file containing reading data. Params: root string """ data_dir = defaults.downloads(root) output_dir = defaults.json_dir(root) archive = defaults.data_archive(root) catalog = [] data = [] json_file = {} data_files = utils.get_files_in_dir(data_dir) if not data_files: utils.warn('No csv files to process. Terminating') exit() utils.print_time('PROCESSOR START') print('Begin JSON file generation') for data_file in data_files: with open(data_file, 'rb') as f: reader = csv.reader(f) meterId, meterName = reader.next() print('Processing meterId %s ...' % (meterId)), info = {'meterId': meterId, 'meterName': meterName} catalog.append(info) for row in reader: ts = row[0] val = float(row[1]) reading = {'timestamp': ts, 'value': val, 'meterId': meterId} data.append(reading) print('done') utils.move(data_file, archive) json_file['datasource'] = defaults.URI json_file['meterCatalog'] = catalog json_file['readings'] = data print('End JSON file generation') curr_dt = datetime.now() json_fname = 'dump_%s.json' % (utils.format_dt(curr_dt)) save_path = os.path.join(output_dir, json_fname) print('Writing JSON to file %s ...' % (save_path)), with open(save_path, 'wb') as out: json.dump(json_file, out) print('done') utils.print_time('PROCESSOR END')
def view_add_multiple_mail_recipient(self, mail_recip_list): utils.log("Add multiple mail recipient: {}".format(mail_recip_list)) if not mail_recip_list: utils.warn( "There is No item in EmailRecipients, please check your configuration!!!" ) return for index, item in enumerate(mail_recip_list): self.view_add_mail_recipient(item, index)
def get_data_grid_headers(self): utils.debug("get headers from data grid.") headers = [] headers_elements = self.slb.get_elements(datagrid.headers) for header in headers_elements: headers.append(self.slb.get_element_text(header)) if not headers: utils.warn("not find any headers for this data grid!") utils.debug("got [{}] data gird headers.".format(len(headers))) return headers
def click_action_in_row(self, action_name, row_unique_key_name, row_unique_key): """ Click Action [View], [Edit] etc. in the specific data row. :param action_name: which action button need to click :param row_unique_key_name: the unique key name for current row. eg:[Type], [Email Title], [Rule Id] :param row_unique_key: the unique key value. """ action_btn_list = self.get_action_btn_list(row_unique_key_name, row_unique_key) for action_btn in action_btn_list: if self.slb.get_element_text(action_btn) == action_name: utils.log("click [{}] on row with [{}]={}".format( action_name, row_unique_key_name, row_unique_key)) action_btn.click() return utils.warn( "Not find record in data grid with [{}]={}, please check!".format( row_unique_key_name, row_unique_key)) self.slb.sleep(2)
def get_select_people_complex_payload(crm_complex_data, grp1_label_list, grp2_label_list): """ generate two group search people complex json. :param crm_complex_data: :param grp1_label_list: group 1 labels list, the groupId for labels in list should be same in yml. :param grp2_label_list: group 2 labels list, eg: ['latestRiskScore', 'installDate'] :return: payload json """ if not isinstance(crm_complex_data, dict) or len(crm_complex_data) <= 0: utils.warn('please pass a valid dict parameter') return None group1_label_data_list, group2_label_data_list = [], [] for label in grp1_label_list: group1_label_data_list.append(crm_complex_data[label]) for label in grp2_label_list: group2_label_data_list.append(crm_complex_data[label]) search_complex_payload = SearchLabelGroupReq([ get_logical_condition(group1_label_data_list), get_logical_condition(group2_label_data_list) ]) json_str = utils.dump_obj(search_complex_payload) payload_json = utils.load_json(json_str) return payload_json
def verify_es_details_with_src(self, es_check_list, src_check_list, es_data_list, src_query, partition_0=None): if len(es_check_list) != len(src_check_list): return utils.warn("please make sure the src and es compare fields number equal!") es_data = es_data_list if len(es_data_list) <= 3 else es_data_list[0:3] for num, es in enumerate(es_data, start=1): es_actual, src_expect = [], [] utils.log("check record #{}".format(num)) if partition_0: src = self.reds.query_results(src_query.format(es['accountid'], partition_0))[0] else: src = self.reds.query_results(src_query.format(es['accountid']))[0] for es_col, src_col in zip(es_check_list, src_check_list): es_actual.append(str(es[es_col]) if es[es_col] is not None else es[es_col]) src_expect.append(str(src[src_col]) if src[src_col] is not None else src[src_col]) utils.log("verify es and src [{}] data matched for below fields.\n{}".format(src_query.split(' ')[3], es_check_list)) self.assertListEqual(src_expect, es_actual, 'Not Match between es and src data!')
def get_session_tokens(session, do_print=False): success = True response = request("GET", session) try: soup = BeautifulSoup(response.text, 'html.parser') csrf_token = soup('input', {'name': CSRF_TOKEN})[0]['value'] session_cookie = re \ .search(SESSION_COOKIE + '=(.*?);', response.headers['set-cookie']) \ .group(1) except Exception as e: print(u.warn(), 'NO TOKENS(S) FOUND') return '', '' if success and do_print: print_tokens(csrf_token, session_cookie) return session_cookie, csrf_token
def run_batch(root, start, end, idx=None): """ Run this script in batch mode. Download reading data whose timestamps lie within start and end dates. The date must follow the following format (Note the T between date and time): YYYY-MM-DDTHH:MM:SS where 24 hour time is used. If idx is a non-negative integer, instead download the meter at that index. idx is zero-indexed. If idx is greater than the number of meters, nothing happens; no files are downloaded. Default behavior is to download data for all meters. Params: root string start string end string idx integer """ s_date = get_date(start) e_date = get_date(end) if not s_date or not e_date: raise ValueError('Invalid/missing dates') elif start > end: raise ValueError('Start date must come before end date') elif not utils.exists_dir(root): raise ValueError('Root directory not found') elif idx is not None and not is_valid_index(idx): raise ValueError('Index must be non-negative integer') creds_file = defaults.creds(root) cnxn_str = utils.get_cnxn_str(creds_file) output_dir = defaults.downloads(root) meter_file = defaults.meter_file(root) utils.print_time('GETTER START') with Cursor.Cursor(cnxn_str) as cursor: dq = get_reading_from_name_query_str() meters = utils.read_meter_file(meter_file) for i, m in enumerate(meters): if idx is not None and idx != i: continue ion_name = utils.get_ion_name(m) qid = utils.get_ion_qid(m) try: cursor.execute(dq, ion_name, qid, str(s_date), str(e_date)) except pyodbc.Error: utils.error( 'Problem with query to get data for meter %s qid %d' % (ion_name, qid)) continue if cursor.rowcount == 0: utils.warn('No data found for meter %s qid %d' % (ion_name, qid)) continue meterId, meterName = utils.get_lucid_id_and_name(m) s_date_str = utils.make_lucid_ts(str(s_date)) e_date_str = utils.make_lucid_ts(str(e_date)) dl_fname = "%sT%sT%s.csv" % (meterId, s_date_str, e_date_str) path = os.path.join(output_dir, dl_fname) print('Writing data for meter %s qid %d to file: %s ...' % (ion_name, qid, path)), with open(path, 'wb') as data_file: writer = csv.writer(data_file) writer.writerow([meterId, meterName]) for row in cursor: ts = row.TimestampUTC val = row.Value data_row = [utils.make_lucid_ts(ts), val] writer.writerow(data_row) print('done') utils.print_time('GETTER END')
def run_batch(root, start, end, idx=None): """ Run this script in batch mode. Download reading data whose timestamps lie within start and end dates. The date must follow the following format (Note the T between date and time): YYYY-MM-DDTHH:MM:SS where 24 hour time is used. If idx is a non-negative integer, instead download the meter at that index. idx is zero-indexed. If idx is greater than the number of meters, nothing happens; no files are downloaded. Default behavior is to download data for all meters. Params: root string start string end string idx integer """ s_date = get_date(start) e_date = get_date(end) if not s_date or not e_date: raise ValueError('Invalid/missing dates') elif start > end: raise ValueError('Start date must come before end date') elif not utils.exists_dir(root): raise ValueError('Root directory not found') elif idx is not None and not is_valid_index(idx): raise ValueError('Index must be non-negative integer') creds_file = defaults.creds(root) cnxn_str = utils.get_cnxn_str(creds_file) output_dir = defaults.downloads(root) meter_file = defaults.meter_file(root) utils.print_time('GETTER START') with Cursor.Cursor(cnxn_str) as cursor: dq = get_reading_from_name_query_str() meters = utils.read_meter_file(meter_file) for i, m in enumerate(meters): if idx is not None and idx != i: continue ion_name = utils.get_ion_name(m) qid = utils.get_ion_qid(m) try: cursor.execute(dq, ion_name, qid, str(s_date), str(e_date)) except pyodbc.Error: utils.error('Problem with query to get data for meter %s qid %d' % (ion_name, qid)) continue if cursor.rowcount == 0: utils.warn('No data found for meter %s qid %d' % (ion_name, qid)) continue meterId, meterName = utils.get_lucid_id_and_name(m) s_date_str = utils.make_lucid_ts(str(s_date)) e_date_str = utils.make_lucid_ts(str(e_date)) dl_fname = "%sT%sT%s.csv" % (meterId, s_date_str, e_date_str) path = os.path.join(output_dir, dl_fname) print('Writing data for meter %s qid %d to file: %s ...' % (ion_name, qid, path)), with open(path, 'wb') as data_file: writer = csv.writer(data_file) writer.writerow([meterId, meterName]) for row in cursor: ts = row.TimestampUTC val = row.Value data_row = [utils.make_lucid_ts(ts), val] writer.writerow(data_row) print('done') utils.print_time('GETTER END')