def main(): input_df = pd.read_csv(input_csv) logger.info("All input entries: %s" %len(input_df)) if os.path.exists(output_csv): existing_df = pd.read_csv(output_csv) scanned_fnames = list(set(existing_df['filename'])) else: utility.create_csv(output_csv, config.extract_Columns) scanned_fnames = [] unscanned_entires_count = len(input_df)-len(scanned_fnames) logger.info("Scanned entries Count: %s"% len(scanned_fnames)) logger.info("Unscanned entries Count: %s" %unscanned_entires_count) time.sleep(5) unscanned_df = input_df[~input_df.fname.isin(scanned_fnames)] unscanned_df = unscanned_df.reset_index() for index, row in unscanned_df.iterrows(): fname = row.fname existing_values = [row.cik, row.coname, row.form, row.fdate, row.fname] logger.info("Processing: %s/%s - %s" %(index+1, unscanned_entires_count, fname)) if os.path.exists(os.path.join(input_directory, fname)): last_3_columns = list(split(input_directory, fname, output_directory)) new_rows = [existing_values + ['success'] + item for item in last_3_columns] else: new_rows = [existing_values + ["No source file"] + ['']* 3] utility.add_rows(output_csv, new_rows)
def continous_measure_PM(NRP2=NRP2, power_meter_misure_number=power_meter_misure_number, power_meter_misure_delay=power_meter_misure_delay, # milliseconds result_file_name=result_file_name, createprogressdialog=None ): dialog = createprogressdialog values = [] # variable for results count = 0 continue_progress = (True, True) NRP2.write("SYSTem:SPEed FAST") for n in range(0, power_meter_misure_number): # number of reading data_now = str(datetime.datetime.now()) values.append(readNRP2(None, NRP2, 1, power_meter_misure_delay, 0, unit.Hz, SAB_switch_delay=0, make_zero=False) + [data_now]) count += 1 if not createprogressdialog is None: import wx #wx.MicroSleep(1) message = "Read {count}".format(count=count) newvalue = int(float(count) / power_meter_misure_number * 100) if newvalue >= 100: createprogressdialog = False # dialog.Update(newvalue, message) # wx.MicroSleep(500) dialog.Close() else: continue_progress = dialog.Update(newvalue, message) if not continue_progress[0]: dialog.Destroy() break print("Misure completed\n") # build header for data table on file header = ["value(dB)", "Time"] values_headers = [] # save data on file savefile_path = result_file_name + "_" + return_now_postfix() + ".csv" try: f_csv = open(savefile_path, "wb") create_csv(f_csv, header, values_headers, values) f_csv.close() except: # on error print data on standard output print(values) return savefile_path
def main(): print(input_csv) input_df = pd.read_csv(input_csv) logger.info("All input entries: %s" % len(input_df)) if os.path.exists(output_csv): existing_df = pd.read_csv(output_csv) scanned_fnames = list(set(existing_df['filename'])) else: utility.create_csv(output_csv, config.outputCsv_columns) scanned_fnames = [] unscanned_entires_count = len(input_df) - len(scanned_fnames) logger.info("Scanned entries Count: %s" % len(scanned_fnames)) logger.info("Unscanned entries Count: %s" % unscanned_entires_count) time.sleep(5) unscanned_df = input_df[~input_df.filename.isin(scanned_fnames)] unscanned_df = unscanned_df.reset_index() unscanned_df.fillna('', inplace=True) for index, row in unscanned_df.iterrows(): location = row.location logger.info("Processing: %s/%s - %s" % (index + 1, unscanned_entires_count, location)) existing_value = list(row)[1:] if (location): # if os.path.exists(location): with open(location) as f: raw_text = f.read() no_html = utility.remove_htmlTags(raw_text) readability = utility.get_readability(no_html) new_row = existing_value + readability # else: # new_row = existing_value + ['']* 10 else: new_row = existing_value + [''] * 10 utility.add_row(output_csv, new_row)
def save_on_csv(self, filepath): adev = self.get_adev() fcounter = self.get_fcounter() spectrum = self.get_spectrum() now_postfix = datetime.datetime.now().strftime("%Y%m%d%H%M%S") rows = [] for a in adev: header_01 = ["NEQBW " + str(a['NEQBW']), " TAU " + str(a['TAU0'])] header_02 = ["tau", "adev", "err"] rows.append(header_01) rows.append(header_02) for i in range(0, len(a["adev"]["tau"]) - 1): rows.append([ str(a["adev"]["tau"][i]), str(a["adev"]["adev"][i]), str(a["adev"]["err"][i]) ]) adev_file = open( os.path.join(filepath, "adev_" + now_postfix + ".csv"), "wb") create_csv(adev_file, ["ADEV"], [], rows) rows = [] rows.append(["Frequency", "Density"]) rows.append([ spectrum["Frequency"]["unit_str"], spectrum["Density"]["unit_str"] ]) for i in range(0, len(spectrum["Frequency"]["value"]) - 1): rows.append([ spectrum["Frequency"]["value"][i], spectrum["Density"]["value"][i] ]) spectrum_file = open( os.path.join(filepath, "spectrum_" + now_postfix + ".csv"), "wb") create_csv(spectrum_file, [], [], rows)
def splitSpuriusCfiletablevalueDict(data_file_name, graph_type, table_value, sort_data, group_level_01, SD_LO_Frequency, SD_LO_Level, SD_RF_Level, SD_IF_Min_Level, savefile=True): """ build a list of list of data sorted by sort_data index list and grouped by group_level_01 index list the restult is [[LO_Freq, LO_Freq_unit, LO_Level, ..., n_LO, m_RF, -(RF_Level-IF_level)], [LO_Freq, LO_Freq_unit, LO_Level, ..., n_LO, m_RF, -(RF_Level-IF_level)], ..., [LO_Freq, LO_Freq_unit, LO_Level, ..., n_LO, m_RF, -(RF_Level-IF_level)]] """ # the table format is: [ table_value = sorted(table_value, key=operator.itemgetter(*sort_data)) # add columnd -(power_RF - power_IF) table_result = [] if graph_type in graph_types.values() + ["IP1"]: for row in table_value: table_result.append(row + [-(row[power_RF_index] - row[power_IF_index])]) elif graph_type in generic_graph_types.values(): table_result = table_value if savefile: filename = data_file_name.split( ".")[0] + "_" + graph_type + "_" + return_now_postfix() filepointer = open(filename + ".csv", "wb") create_csv(filepointer, csv_file_header, [], table_result) filepointer.close() create_xlsx(filename + ".xlsx", csv_file_header, [], table_result) group_result = [[table_result[0][:]]] last_tupla = tuple([table_result[0][index] for index in group_level_01]) # last_tupla = (table_result[0][n_LO_index], table_result[0][m_RF_index], table_result[0][frequency_LO_index], table_result[0][power_LO_index]) for row in table_result[1:]: # first row in group_result yet graph_type = str(graph_type) if graph_type == "SD" and row[power_LO_index] == SD_LO_Level and row[ power_RF_index] == SD_RF_Level and row[ power_IF_index] > SD_IF_Min_Level: if row[frequency_LO_index] == SD_LO_Frequency: current_tupla = tuple([row[index] for index in group_level_01]) if last_tupla == current_tupla: group_result[-1].append(row[:]) else: last_tupla = current_tupla group_result.append([row[:]]) elif graph_type == "GG": current_tupla = tuple([row[index] for index in group_level_01]) if last_tupla == current_tupla: group_result[-1].append(row[:]) else: last_tupla = current_tupla group_result.append([row[:]]) elif graph_type == "NN": current_tupla = tuple([row[index] for index in group_level_01]) if last_tupla == current_tupla: group_result[-1].append(row[:]) else: last_tupla = current_tupla group_result.append([row[:]]) # group_result[-1].append(row[:]) elif graph_type in ["LO", "RF", "SP", "IP1"]: current_tupla = tuple([row[index] for index in group_level_01]) if last_tupla == current_tupla: group_result[-1].append(row[:]) else: last_tupla = current_tupla group_result.append([row[:]]) return group_result
for i in range(1, 5): if verbose == 1: print("\n\t\t\t\033[33m", house_rev[i].upper(), "\033[0m\n") for f1 in range(1, len(df.columns) - 1): for f2 in range(f1 + 1, len(df.columns) - 1): x, y = filter_data(df, house_rev[i], f1, f2) if verbose == 1: print(df.columns[f1], " vs ", df.columns[f2]) if visu != 0 and i == visu and f1 == f1v and f2 == f2v: display_data(x, y, house_rev[i], df, f1, f2) x, mean, std = standardize(x) col, row = x.shape[0], x.shape[1] x = np.insert(x, 0, 1, axis=1) y = y.reshape(col, 1) theta = np.zeros((row + 1, 1)) theta, error_history = train_theta(x, y, theta, 1, 400) if visu != 0 and i == visu and f1 == f1v and f2 == f2v: display_standardize(x, y, house_rev[i], df, f1, f2, theta) display_cost(error_history) ac = get_accuracy(x, y, theta) if verbose == 1: print( "\t\t\t\t\t\t\t\033[36mAccuracy: {}\033[0m".format(ac)) if ac >= 97: row_list.append([ house_rev[i], df.columns[f1], df.columns[f2], theta[0][0], theta[1][0], theta[2][0], mean[0], mean[1], std[0], std[1], ac ]) create_csv(row_list, "weights.csv") print("Weights saved in './weights.csv'")
def measure_calibration_cable( SMB=SMB_RF, NRP2=NRP2, SAB=SAB, synthetizer_frequency=synthetizer_frequency, synthetizer_fix_power=synthetizer_fix_power, # dBm power_meter_make_zero=power_meter_make_zero, power_meter_make_zero_delay=power_meter_make_zero_delay, power_meter_misure_number=power_meter_misure_number, power_meter_misure_delay=power_meter_misure_delay, # milliseconds SAB_switch_01_delay=SAB_switch_01_delay, SAB_switch_02_delay=SAB_switch_02_delay, result_file_name=result_file_name, dummy_cable_power_level=0, create_dummy_cable=False, createprogressdialog=None): dialog = createprogressdialog values = [] # variable for results # reset the synthetizer SMB100A SMB.write("*rst") # imposta la modalit� di funzionamento del SMB100A SMB.write("FREQ:MODE FIX") SMB.write("POW:MODE FIX") command = "POW " + str(synthetizer_fix_power) SMB.write(command) frequency_range = synthetizer_frequency.return_range() maxcount = len(frequency_range) count = 0 continue_progress = (True, True) for f in frequency_range: # frequency loop # set SMB100A frequency f_value = str(f) current_frequency = f_value + unit.return_unit_str(unit.Hz) current_frequency_human_readable = unit.return_human_readable_str(f) command = "FREQ " + current_frequency # Ex. FREQ 500kHz SMB.write(command) SMB.write("OUTP ON") time.sleep(2) data_now = str(datetime.datetime.now()) if create_dummy_cable: values.append([f_value, unit.return_unit_str(unit.Hz)] + [ str(dummy_cable_power_level) for x in range(power_meter_misure_number) ] + [data_now]) else: values.append([f_value, unit.return_unit_str(unit.Hz)] + readNRP2(SAB, NRP2, power_meter_misure_number, power_meter_misure_delay, f, unit.Hz, SAB_switch_01_delay, make_zero=True) + [data_now]) # turn off RF SMB.write("OUTP OFF") count += 1 if not createprogressdialog is None: import wx wx.MicroSleep(500) message = "{lo_freq}".format( lo_freq=current_frequency_human_readable) newvalue = int(float(count) / maxcount * 100) if newvalue >= 100: createprogressdialog = False # dialog.Update(newvalue, message) # wx.MicroSleep(500) dialog.Close() else: continue_progress = dialog.Update(newvalue, message) if not continue_progress[0]: dialog.Destroy() break if f == frequency_range[-1]: # safety turn Off # dialog.Update(100, "Measure completed) dialog.Destroy() # turn off RF SMB.write("OUTP OFF") # Output [Frequenza, output power meter(Loss), time] print("Misure completed\n") # build header for data table on file header = ["Frequency", "Unit"] values_headers = [] for v in range(0, power_meter_misure_number): values_headers.append("Loss value(dB)") values_headers.append("Time") # save data on file savefile_path = result_file_name + "_" + return_now_postfix() + ".csv" try: f_csv = open(savefile_path, "wb") create_csv(f_csv, header, values_headers, values) f_csv.close() except: # on error print data on standard output print(values) return savefile_path
def create_calibration_cable( synthetizer_frequency=synthetizer_frequency, fixed_output_level=fixed_output_level, # dBm frequency_output_unit=frequency_output_unit, result_file_name=result_file_name, createprogressdialog=None): dialog = createprogressdialog # if createprogressdialog: # import wx # # app = wx.App() # dialog = wx.ProgressDialog("Progress", "Time remaining", maximum = 100, # style=wx.PD_CAN_ABORT | wx.PD_ELAPSED_TIME | wx.PD_REMAINING_TIME ) values = [] # variable for results # synthetizer_frequency_min = unit.unit_conversion(synthetizer_frequency_min, synthetizer_frequency_min_unit, frequency_output_unit) # synthetizer_frequency_max = unit.unit_conversion(synthetizer_frequency_max, synthetizer_frequency_max_unit, frequency_output_unit) # synthetizer_frequency_step = unit.unit_conversion(synthetizer_frequency_step, synthetizer_frequency_step_unit, frequency_output_unit) frequency_range = synthetizer_frequency.return_range() maxcount = len(frequency_range) count = 0 for f in frequency_range: # frequency loop current_frequency_human_readable = unit.return_human_readable_str(f) data_now = str(datetime.datetime.now()) values.append([ str(unit.convertion_from_base(f, frequency_output_unit)), unit.return_unit_str(frequency_output_unit), str(fixed_output_level), data_now ]) count += 1 if not createprogressdialog is None: import wx wx.MicroSleep(500) message = "{lo_freq} {lo_pow}dB".format( lo_freq=current_frequency_human_readable, lo_pow=str(fixed_output_level)) newvalue = min([int(count / maxcount * 100), 100]) dialog.Update(newvalue, message) if newvalue == 100: createprogressdialog = False # dialog.Update(newvalue, message) # wx.MicroSleep(500) dialog.Close() else: dialog.Update(newvalue, message) if f == frequency_range[-1]: # safety turn Off # dialog.Update(100, "Measure completed) dialog.Destroy() # Output [Frequenza, unit, output power meter(Loss), time] # if createprogressdialog: # wx.MicroSleep(500) # wx.CallAfter(dialog.Destroy()) print("Misure completed\n") # build header for data table on file header = ["Frequency", "Unit", "Loss value(dB)", "Time"] values_headers = [] # save data on file try: f_csv = open(result_file_name + "_" + return_now_postfix() + ".csv", "wb") create_csv(f_csv, header, values_headers, values) f_csv.close() return 1 except: # on error print data on standard output print(values) return 0
def measure_100GHz_cal(SMB=SMB_RF, PM5=PM5, synthetizer_frequency=synthetizer_frequency, synthetizer_level=synthetizer_level, calibration_cable_file_name=calibration_cable_file_name, pm5_misure_number=pm5_misure_number, pm5_misure_delay=pm5_misure_delay, result_file_name=result_file_name, createprogressdialog=None): dialog = createprogressdialog calibration_LO = readcalibrationfile(calibration_cable_file_name) calibration_function_LO, calibration_function_LO_unit = calibrationfilefunction( calibration_LO) values = [] # variable for results # reset the synthetizer SMB100A SMB.write("*rst") # synthetizer_frequency_min = unit.convertion_to_base(synthetizer_frequency_min, synthetizer_frequency_unit) # synthetizer_frequency_max = unit.convertion_to_base(synthetizer_frequency_max, synthetizer_frequency_unit) # synthetizer_frequency_step = unit.convertion_to_base(synthetizer_frequency_step, synthetizer_frequency_unit) frequency_range = synthetizer_frequency.return_range() level_range = synthetizer_level.return_arange() maxcount = len(frequency_range) * len(level_range) count = 0 SMB.write("OUTP ON") for f in frequency_range: # frequency loop # set SMB100A frequency f_value = str(f) current_frequency = f_value + unit.return_unit_str(unit.Hz) current_frequency_human_readable = str( unit.convertion_from_base(f_value, human_readable_frequency_unit) ) + unit.return_unit_str(human_readable_frequency_unit) command = "FREQ " + current_frequency # Ex. FREQ 500kHz SMB.write(command) time.sleep(1) for l in level_range: current_LO_level, calibration_LO_result = calibrate( l, f, unit.Hz, calibration_LO, calibration_function=calibration_function_LO, calibration_function_unit=calibration_function_LO_unit) current_level = str(current_LO_level) command = "POW " + current_level SMB.write(command) data_now = str(datetime.datetime.now()) values.append([ f_value, unit.return_unit_str(unit.Hz), str(l), current_level, calibration_LO_result ] + readPM5(PM5, pm5_misure_number, pm5_misure_delay) + [data_now]) # turn off RF count += 1 if not createprogressdialog is None: import wx wx.MicroSleep(500) message = "{lo_freq} {lo_level}dB".format( lo_freq=current_frequency_human_readable, lo_level=current_level) newvalue = int(float(count) / maxcount * 100) if newvalue >= 100: createprogressdialog = False # dialog.Update(newvalue, message) # wx.MicroSleep(500) dialog.Close() else: dialog.Update(newvalue, message) if f == frequency_range[-1] and l == level_range[-1]: dialog.Destroy() # turn off RF SMB.write("OUTP OFF") PM5.closeport() # Output [Frequenza, output power meter(Loss), time] print("Misure completed\n") # build header for data table on file header = [ "Frequency", "Unit", "Input Level", "Calibrated Level", "Calibration" ] values_headers = [] for v in range(0, pm5_misure_number): values_headers.append("Power (mW)") values_headers.append("Time") # save data on file try: f_csv = open(result_file_name + "_" + return_now_postfix() + ".csv", "wb") create_csv(f_csv, header, values_headers, values) f_csv.close() except: # on error print data on standard output print(values)
def measure_calibration_cable( SMB=SMB_RF, FSV=FSV, synthetizer_frequency=synthetizer_frequency, synthetizer_fix_power=synthetizer_fix_power, # dBm spectrum_analyzer_state=spectrum_analyzer_state, spectrum_analyzer_sweep_points=spectrum_analyzer_sweep_points, spectrum_analyzer_resolution_bandwidth=spectrum_analyzer_resolution_bandwidth, spectrum_analyzer_resolution_bandwidth_unit=spectrum_analyzer_resolution_bandwidth_unit, spectrum_analyzer_video_bandwidth=spectrum_analyzer_video_bandwidth, spectrum_analyzer_video_bandwidth_unit=spectrum_analyzer_video_bandwidth_unit, spectrum_analyzer_frequency_span=spectrum_analyzer_frequency_span, spectrum_analyzer_frequency_span_unit=spectrum_analyzer_frequency_span_unit, # spectrum_analyzer_attenuation = spectrum_analyzer_attenuation, gainAmplifier=gainAmplifier, # dB spectrum_analyzer_IF_atten_enable=spectrum_analyzer_IF_atten_enable, spectrum_analyzer_IF_atten=spectrum_analyzer_IF_atten, spectrum_analyzer_IF_relative_level=spectrum_analyzer_IF_relative_level_enable, spectrum_analyzer_IF_relative_level_enable=spectrum_analyzer_IF_relative_level_enable, threshold_power=threshold_power, # dB spectrum_analyzer_frequency_marker_unit=spectrum_analyzer_frequency_marker_unit, FSV_delay=FSV_delay, result_file_name=result_file_name, dummy_cable_power_level=0, create_dummy_cable=False, createprogressdialog=None): dialog = createprogressdialog values = [] # variable for results # reset the synthetizer SMB100A SMB.write("*rst") # imposta la modalit� di funzionamento del SMB100A SMB.write("FREQ:MODE FIX") SMB.write("POW:MODE FIX") command = "POW " + str(synthetizer_fix_power) SMB.write(command) FSV_reset_setup(FSV, spectrum_analyzer_sweep_points, spectrum_analyzer_resolution_bandwidth, spectrum_analyzer_resolution_bandwidth_unit, spectrum_analyzer_video_bandwidth, spectrum_analyzer_video_bandwidth_unit, spectrum_analyzer_frequency_span, spectrum_analyzer_frequency_span_unit, spectrum_analyzer_IF_atten_enable, spectrum_analyzer_IF_atten, spectrum_analyzer_IF_relative_level_enable, spectrum_analyzer_IF_relative_level) FSV.write("CALC:MARKER ON") # enable marker mode calibration_IF = [] calibration_function_IF = None calibration_function_IF_unit = unit.Hz frequency_range = synthetizer_frequency.return_range() maxcount = len(frequency_range) count = 0 continue_progress = (True, True) for f in frequency_range: # frequency loop # set SMB100A frequency f_value = str(f) current_frequency = f_value + unit.return_unit_str(unit.Hz) current_frequency_human_readable = unit.return_human_readable_str(f) command = "FREQ " + current_frequency # Ex. FREQ 500kHz SMB.write(command) SMB.write("OUTP ON") time.sleep(2) data_now = str(datetime.datetime.now()) if create_dummy_cable: values.append([f_value, unit.return_unit_str(unit.Hz)] + [str(dummy_cable_power_level)] + [data_now]) else: values.append( readFSV_marker(FSV, FSV_delay, f, unit.Hz, unit.Hz, calibration_IF=[], calibration_IF_function=None, calibration_IF_function_unit=None) + [data_now]) # turn off RF SMB.write("OUTP OFF") count += 1 if not createprogressdialog is None: import wx wx.MicroSleep(500) message = "{lo_freq}".format( lo_freq=current_frequency_human_readable) newvalue = int(float(count) / maxcount * 100) if newvalue >= 100: createprogressdialog = False # dialog.Update(newvalue, message) # wx.MicroSleep(500) dialog.Close() else: continue_progress = dialog.Update(newvalue, message) if not continue_progress[0]: dialog.Destroy() break if f == frequency_range[-1]: # safety turn Off # dialog.Update(100, "Measure completed) dialog.Destroy() # turn off RF SMB.write("OUTP OFF") # Output [Frequenza, output power meter(Loss), time] print("Misure completed\n") # build header for data table on file header = ["Frequency", "Unit"] values_headers = [] values_headers.append("Loss value(dB)") values_headers.append("Time") # save data on file savefile_path = result_file_name + "_" + return_now_postfix() + ".csv" try: f_csv = open(savefile_path, "wb") create_csv(f_csv, header, values_headers, values) f_csv.close() except: # on error print data on standard output print(values) return savefile_path