def main():
input_df = pd.read_csv(input_csv)
logger.info("All input entries: %s" %len(input_df))
if os.path.exists(output_csv):
existing_df = pd.read_csv(output_csv)
scanned_fnames = list(set(existing_df['filename']))
else:
utility.create_csv(output_csv, config.extract_Columns)
scanned_fnames = []
unscanned_entires_count = len(input_df)-len(scanned_fnames)
logger.info("Scanned entries Count: %s"% len(scanned_fnames))
logger.info("Unscanned entries Count: %s" %unscanned_entires_count)
time.sleep(5)
unscanned_df = input_df[~input_df.fname.isin(scanned_fnames)]
unscanned_df = unscanned_df.reset_index()
for index, row in unscanned_df.iterrows():
fname = row.fname
existing_values = [row.cik, row.coname, row.form, row.fdate, row.fname]
logger.info("Processing: %s/%s - %s" %(index+1, unscanned_entires_count, fname))
if os.path.exists(os.path.join(input_directory, fname)):
last_3_columns = list(split(input_directory, fname, output_directory))
new_rows = [existing_values + ['success'] + item for item in last_3_columns]
else:
new_rows = [existing_values + ["No source file"] + ['']* 3]
utility.add_rows(output_csv, new_rows)
def continous_measure_PM(NRP2=NRP2,
power_meter_misure_number=power_meter_misure_number,
power_meter_misure_delay=power_meter_misure_delay, # milliseconds
result_file_name=result_file_name,
createprogressdialog=None
):
dialog = createprogressdialog
values = [] # variable for results
count = 0
continue_progress = (True, True)
NRP2.write("SYSTem:SPEed FAST")
for n in range(0, power_meter_misure_number): # number of reading
data_now = str(datetime.datetime.now())
values.append(readNRP2(None, NRP2, 1, power_meter_misure_delay, 0, unit.Hz, SAB_switch_delay=0, make_zero=False) + [data_now])
count += 1
if not createprogressdialog is None:
import wx
#wx.MicroSleep(1)
message = "Read {count}".format(count=count)
newvalue = int(float(count) / power_meter_misure_number * 100)
if newvalue >= 100:
createprogressdialog = False
# dialog.Update(newvalue, message)
# wx.MicroSleep(500)
dialog.Close()
else:
continue_progress = dialog.Update(newvalue, message)
if not continue_progress[0]:
dialog.Destroy()
break
print("Misure completed\n")
# build header for data table on file
header = ["value(dB)", "Time"]
values_headers = []
# save data on file
savefile_path = result_file_name + "_" + return_now_postfix() + ".csv"
try:
f_csv = open(savefile_path, "wb")
create_csv(f_csv, header, values_headers, values)
f_csv.close()
except:
# on error print data on standard output
print(values)
return savefile_path
def main():
print(input_csv)
input_df = pd.read_csv(input_csv)
logger.info("All input entries: %s" % len(input_df))
if os.path.exists(output_csv):
existing_df = pd.read_csv(output_csv)
scanned_fnames = list(set(existing_df['filename']))
else:
utility.create_csv(output_csv, config.outputCsv_columns)
scanned_fnames = []
unscanned_entires_count = len(input_df) - len(scanned_fnames)
logger.info("Scanned entries Count: %s" % len(scanned_fnames))
logger.info("Unscanned entries Count: %s" % unscanned_entires_count)
time.sleep(5)
unscanned_df = input_df[~input_df.filename.isin(scanned_fnames)]
unscanned_df = unscanned_df.reset_index()
unscanned_df.fillna('', inplace=True)
for index, row in unscanned_df.iterrows():
location = row.location
logger.info("Processing: %s/%s - %s" %
(index + 1, unscanned_entires_count, location))
existing_value = list(row)[1:]
if (location):
# if os.path.exists(location):
with open(location) as f:
raw_text = f.read()
no_html = utility.remove_htmlTags(raw_text)
readability = utility.get_readability(no_html)
new_row = existing_value + readability
# else:
# new_row = existing_value + ['']* 10
else:
new_row = existing_value + [''] * 10
utility.add_row(output_csv, new_row)
def save_on_csv(self, filepath):
adev = self.get_adev()
fcounter = self.get_fcounter()
spectrum = self.get_spectrum()
now_postfix = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
rows = []
for a in adev:
header_01 = ["NEQBW " + str(a['NEQBW']), " TAU " + str(a['TAU0'])]
header_02 = ["tau", "adev", "err"]
rows.append(header_01)
rows.append(header_02)
for i in range(0, len(a["adev"]["tau"]) - 1):
rows.append([
str(a["adev"]["tau"][i]),
str(a["adev"]["adev"][i]),
str(a["adev"]["err"][i])
])
adev_file = open(
os.path.join(filepath, "adev_" + now_postfix + ".csv"), "wb")
create_csv(adev_file, ["ADEV"], [], rows)
rows = []
rows.append(["Frequency", "Density"])
rows.append([
spectrum["Frequency"]["unit_str"], spectrum["Density"]["unit_str"]
])
for i in range(0, len(spectrum["Frequency"]["value"]) - 1):
rows.append([
spectrum["Frequency"]["value"][i],
spectrum["Density"]["value"][i]
])
spectrum_file = open(
os.path.join(filepath, "spectrum_" + now_postfix + ".csv"), "wb")
create_csv(spectrum_file, [], [], rows)
def splitSpuriusCfiletablevalueDict(data_file_name,
graph_type,
table_value,
sort_data,
group_level_01,
SD_LO_Frequency,
SD_LO_Level,
SD_RF_Level,
SD_IF_Min_Level,
savefile=True):
"""
build a list of list of data sorted by sort_data index list and grouped by group_level_01 index list
the restult is [[LO_Freq, LO_Freq_unit, LO_Level, ..., n_LO, m_RF, -(RF_Level-IF_level)],
[LO_Freq, LO_Freq_unit, LO_Level, ..., n_LO, m_RF, -(RF_Level-IF_level)],
...,
[LO_Freq, LO_Freq_unit, LO_Level, ..., n_LO, m_RF, -(RF_Level-IF_level)]]
"""
# the table format is: [
table_value = sorted(table_value, key=operator.itemgetter(*sort_data))
# add columnd -(power_RF - power_IF)
table_result = []
if graph_type in graph_types.values() + ["IP1"]:
for row in table_value:
table_result.append(row +
[-(row[power_RF_index] - row[power_IF_index])])
elif graph_type in generic_graph_types.values():
table_result = table_value
if savefile:
filename = data_file_name.split(
".")[0] + "_" + graph_type + "_" + return_now_postfix()
filepointer = open(filename + ".csv", "wb")
create_csv(filepointer, csv_file_header, [], table_result)
filepointer.close()
create_xlsx(filename + ".xlsx", csv_file_header, [], table_result)
group_result = [[table_result[0][:]]]
last_tupla = tuple([table_result[0][index] for index in group_level_01])
# last_tupla = (table_result[0][n_LO_index], table_result[0][m_RF_index], table_result[0][frequency_LO_index], table_result[0][power_LO_index])
for row in table_result[1:]: # first row in group_result yet
graph_type = str(graph_type)
if graph_type == "SD" and row[power_LO_index] == SD_LO_Level and row[
power_RF_index] == SD_RF_Level and row[
power_IF_index] > SD_IF_Min_Level:
if row[frequency_LO_index] == SD_LO_Frequency:
current_tupla = tuple([row[index] for index in group_level_01])
if last_tupla == current_tupla:
group_result[-1].append(row[:])
else:
last_tupla = current_tupla
group_result.append([row[:]])
elif graph_type == "GG":
current_tupla = tuple([row[index] for index in group_level_01])
if last_tupla == current_tupla:
group_result[-1].append(row[:])
else:
last_tupla = current_tupla
group_result.append([row[:]])
elif graph_type == "NN":
current_tupla = tuple([row[index] for index in group_level_01])
if last_tupla == current_tupla:
group_result[-1].append(row[:])
else:
last_tupla = current_tupla
group_result.append([row[:]])
# group_result[-1].append(row[:])
elif graph_type in ["LO", "RF", "SP", "IP1"]:
current_tupla = tuple([row[index] for index in group_level_01])
if last_tupla == current_tupla:
group_result[-1].append(row[:])
else:
last_tupla = current_tupla
group_result.append([row[:]])
return group_result
for i in range(1, 5):
if verbose == 1:
print("\n\t\t\t\033[33m", house_rev[i].upper(), "\033[0m\n")
for f1 in range(1, len(df.columns) - 1):
for f2 in range(f1 + 1, len(df.columns) - 1):
x, y = filter_data(df, house_rev[i], f1, f2)
if verbose == 1:
print(df.columns[f1], " vs ", df.columns[f2])
if visu != 0 and i == visu and f1 == f1v and f2 == f2v:
display_data(x, y, house_rev[i], df, f1, f2)
x, mean, std = standardize(x)
col, row = x.shape[0], x.shape[1]
x = np.insert(x, 0, 1, axis=1)
y = y.reshape(col, 1)
theta = np.zeros((row + 1, 1))
theta, error_history = train_theta(x, y, theta, 1, 400)
if visu != 0 and i == visu and f1 == f1v and f2 == f2v:
display_standardize(x, y, house_rev[i], df, f1, f2, theta)
display_cost(error_history)
ac = get_accuracy(x, y, theta)
if verbose == 1:
print(
"\t\t\t\t\t\t\t\033[36mAccuracy: {}\033[0m".format(ac))
if ac >= 97:
row_list.append([
house_rev[i], df.columns[f1], df.columns[f2],
theta[0][0], theta[1][0], theta[2][0], mean[0],
mean[1], std[0], std[1], ac
])
create_csv(row_list, "weights.csv")
print("Weights saved in './weights.csv'")
def measure_calibration_cable(
SMB=SMB_RF,
NRP2=NRP2,
SAB=SAB,
synthetizer_frequency=synthetizer_frequency,
synthetizer_fix_power=synthetizer_fix_power, # dBm
power_meter_make_zero=power_meter_make_zero,
power_meter_make_zero_delay=power_meter_make_zero_delay,
power_meter_misure_number=power_meter_misure_number,
power_meter_misure_delay=power_meter_misure_delay, # milliseconds
SAB_switch_01_delay=SAB_switch_01_delay,
SAB_switch_02_delay=SAB_switch_02_delay,
result_file_name=result_file_name,
dummy_cable_power_level=0,
create_dummy_cable=False,
createprogressdialog=None):
dialog = createprogressdialog
values = [] # variable for results
# reset the synthetizer SMB100A
SMB.write("*rst")
# imposta la modalit� di funzionamento del SMB100A
SMB.write("FREQ:MODE FIX")
SMB.write("POW:MODE FIX")
command = "POW " + str(synthetizer_fix_power)
SMB.write(command)
frequency_range = synthetizer_frequency.return_range()
maxcount = len(frequency_range)
count = 0
continue_progress = (True, True)
for f in frequency_range: # frequency loop
# set SMB100A frequency
f_value = str(f)
current_frequency = f_value + unit.return_unit_str(unit.Hz)
current_frequency_human_readable = unit.return_human_readable_str(f)
command = "FREQ " + current_frequency # Ex. FREQ 500kHz
SMB.write(command)
SMB.write("OUTP ON")
time.sleep(2)
data_now = str(datetime.datetime.now())
if create_dummy_cable:
values.append([f_value, unit.return_unit_str(unit.Hz)] + [
str(dummy_cable_power_level)
for x in range(power_meter_misure_number)
] + [data_now])
else:
values.append([f_value, unit.return_unit_str(unit.Hz)] +
readNRP2(SAB,
NRP2,
power_meter_misure_number,
power_meter_misure_delay,
f,
unit.Hz,
SAB_switch_01_delay,
make_zero=True) + [data_now])
# turn off RF
SMB.write("OUTP OFF")
count += 1
if not createprogressdialog is None:
import wx
wx.MicroSleep(500)
message = "{lo_freq}".format(
lo_freq=current_frequency_human_readable)
newvalue = int(float(count) / maxcount * 100)
if newvalue >= 100:
createprogressdialog = False
# dialog.Update(newvalue, message)
# wx.MicroSleep(500)
dialog.Close()
else:
continue_progress = dialog.Update(newvalue, message)
if not continue_progress[0]:
dialog.Destroy()
break
if f == frequency_range[-1]:
# safety turn Off
# dialog.Update(100, "Measure completed)
dialog.Destroy()
# turn off RF
SMB.write("OUTP OFF")
# Output [Frequenza, output power meter(Loss), time]
print("Misure completed\n")
# build header for data table on file
header = ["Frequency", "Unit"]
values_headers = []
for v in range(0, power_meter_misure_number):
values_headers.append("Loss value(dB)")
values_headers.append("Time")
# save data on file
savefile_path = result_file_name + "_" + return_now_postfix() + ".csv"
try:
f_csv = open(savefile_path, "wb")
create_csv(f_csv, header, values_headers, values)
f_csv.close()
except:
# on error print data on standard output
print(values)
return savefile_path
def create_calibration_cable(
synthetizer_frequency=synthetizer_frequency,
fixed_output_level=fixed_output_level, # dBm
frequency_output_unit=frequency_output_unit,
result_file_name=result_file_name,
createprogressdialog=None):
dialog = createprogressdialog
# if createprogressdialog:
# import wx
#
# app = wx.App()
# dialog = wx.ProgressDialog("Progress", "Time remaining", maximum = 100,
# style=wx.PD_CAN_ABORT | wx.PD_ELAPSED_TIME | wx.PD_REMAINING_TIME )
values = [] # variable for results
# synthetizer_frequency_min = unit.unit_conversion(synthetizer_frequency_min, synthetizer_frequency_min_unit, frequency_output_unit)
# synthetizer_frequency_max = unit.unit_conversion(synthetizer_frequency_max, synthetizer_frequency_max_unit, frequency_output_unit)
# synthetizer_frequency_step = unit.unit_conversion(synthetizer_frequency_step, synthetizer_frequency_step_unit, frequency_output_unit)
frequency_range = synthetizer_frequency.return_range()
maxcount = len(frequency_range)
count = 0
for f in frequency_range: # frequency loop
current_frequency_human_readable = unit.return_human_readable_str(f)
data_now = str(datetime.datetime.now())
values.append([
str(unit.convertion_from_base(f, frequency_output_unit)),
unit.return_unit_str(frequency_output_unit),
str(fixed_output_level), data_now
])
count += 1
if not createprogressdialog is None:
import wx
wx.MicroSleep(500)
message = "{lo_freq} {lo_pow}dB".format(
lo_freq=current_frequency_human_readable,
lo_pow=str(fixed_output_level))
newvalue = min([int(count / maxcount * 100), 100])
dialog.Update(newvalue, message)
if newvalue == 100:
createprogressdialog = False
# dialog.Update(newvalue, message)
# wx.MicroSleep(500)
dialog.Close()
else:
dialog.Update(newvalue, message)
if f == frequency_range[-1]:
# safety turn Off
# dialog.Update(100, "Measure completed)
dialog.Destroy()
# Output [Frequenza, unit, output power meter(Loss), time]
# if createprogressdialog:
# wx.MicroSleep(500)
# wx.CallAfter(dialog.Destroy())
print("Misure completed\n")
# build header for data table on file
header = ["Frequency", "Unit", "Loss value(dB)", "Time"]
values_headers = []
# save data on file
try:
f_csv = open(result_file_name + "_" + return_now_postfix() + ".csv",
"wb")
create_csv(f_csv, header, values_headers, values)
f_csv.close()
return 1
except:
# on error print data on standard output
print(values)
return 0
def measure_100GHz_cal(SMB=SMB_RF,
PM5=PM5,
synthetizer_frequency=synthetizer_frequency,
synthetizer_level=synthetizer_level,
calibration_cable_file_name=calibration_cable_file_name,
pm5_misure_number=pm5_misure_number,
pm5_misure_delay=pm5_misure_delay,
result_file_name=result_file_name,
createprogressdialog=None):
dialog = createprogressdialog
calibration_LO = readcalibrationfile(calibration_cable_file_name)
calibration_function_LO, calibration_function_LO_unit = calibrationfilefunction(
calibration_LO)
values = [] # variable for results
# reset the synthetizer SMB100A
SMB.write("*rst")
# synthetizer_frequency_min = unit.convertion_to_base(synthetizer_frequency_min, synthetizer_frequency_unit)
# synthetizer_frequency_max = unit.convertion_to_base(synthetizer_frequency_max, synthetizer_frequency_unit)
# synthetizer_frequency_step = unit.convertion_to_base(synthetizer_frequency_step, synthetizer_frequency_unit)
frequency_range = synthetizer_frequency.return_range()
level_range = synthetizer_level.return_arange()
maxcount = len(frequency_range) * len(level_range)
count = 0
SMB.write("OUTP ON")
for f in frequency_range: # frequency loop
# set SMB100A frequency
f_value = str(f)
current_frequency = f_value + unit.return_unit_str(unit.Hz)
current_frequency_human_readable = str(
unit.convertion_from_base(f_value, human_readable_frequency_unit)
) + unit.return_unit_str(human_readable_frequency_unit)
command = "FREQ " + current_frequency # Ex. FREQ 500kHz
SMB.write(command)
time.sleep(1)
for l in level_range:
current_LO_level, calibration_LO_result = calibrate(
l,
f,
unit.Hz,
calibration_LO,
calibration_function=calibration_function_LO,
calibration_function_unit=calibration_function_LO_unit)
current_level = str(current_LO_level)
command = "POW " + current_level
SMB.write(command)
data_now = str(datetime.datetime.now())
values.append([
f_value,
unit.return_unit_str(unit.Hz),
str(l), current_level, calibration_LO_result
] + readPM5(PM5, pm5_misure_number, pm5_misure_delay) + [data_now])
# turn off RF
count += 1
if not createprogressdialog is None:
import wx
wx.MicroSleep(500)
message = "{lo_freq} {lo_level}dB".format(
lo_freq=current_frequency_human_readable,
lo_level=current_level)
newvalue = int(float(count) / maxcount * 100)
if newvalue >= 100:
createprogressdialog = False
# dialog.Update(newvalue, message)
# wx.MicroSleep(500)
dialog.Close()
else:
dialog.Update(newvalue, message)
if f == frequency_range[-1] and l == level_range[-1]:
dialog.Destroy()
# turn off RF
SMB.write("OUTP OFF")
PM5.closeport()
# Output [Frequenza, output power meter(Loss), time]
print("Misure completed\n")
# build header for data table on file
header = [
"Frequency", "Unit", "Input Level", "Calibrated Level", "Calibration"
]
values_headers = []
for v in range(0, pm5_misure_number):
values_headers.append("Power (mW)")
values_headers.append("Time")
# save data on file
try:
f_csv = open(result_file_name + "_" + return_now_postfix() + ".csv",
"wb")
create_csv(f_csv, header, values_headers, values)
f_csv.close()
except:
# on error print data on standard output
print(values)
def measure_calibration_cable(
SMB=SMB_RF,
FSV=FSV,
synthetizer_frequency=synthetizer_frequency,
synthetizer_fix_power=synthetizer_fix_power, # dBm
spectrum_analyzer_state=spectrum_analyzer_state,
spectrum_analyzer_sweep_points=spectrum_analyzer_sweep_points,
spectrum_analyzer_resolution_bandwidth=spectrum_analyzer_resolution_bandwidth,
spectrum_analyzer_resolution_bandwidth_unit=spectrum_analyzer_resolution_bandwidth_unit,
spectrum_analyzer_video_bandwidth=spectrum_analyzer_video_bandwidth,
spectrum_analyzer_video_bandwidth_unit=spectrum_analyzer_video_bandwidth_unit,
spectrum_analyzer_frequency_span=spectrum_analyzer_frequency_span,
spectrum_analyzer_frequency_span_unit=spectrum_analyzer_frequency_span_unit,
# spectrum_analyzer_attenuation = spectrum_analyzer_attenuation,
gainAmplifier=gainAmplifier, # dB
spectrum_analyzer_IF_atten_enable=spectrum_analyzer_IF_atten_enable,
spectrum_analyzer_IF_atten=spectrum_analyzer_IF_atten,
spectrum_analyzer_IF_relative_level=spectrum_analyzer_IF_relative_level_enable,
spectrum_analyzer_IF_relative_level_enable=spectrum_analyzer_IF_relative_level_enable,
threshold_power=threshold_power, # dB
spectrum_analyzer_frequency_marker_unit=spectrum_analyzer_frequency_marker_unit,
FSV_delay=FSV_delay,
result_file_name=result_file_name,
dummy_cable_power_level=0,
create_dummy_cable=False,
createprogressdialog=None):
dialog = createprogressdialog
values = [] # variable for results
# reset the synthetizer SMB100A
SMB.write("*rst")
# imposta la modalit� di funzionamento del SMB100A
SMB.write("FREQ:MODE FIX")
SMB.write("POW:MODE FIX")
command = "POW " + str(synthetizer_fix_power)
SMB.write(command)
FSV_reset_setup(FSV, spectrum_analyzer_sweep_points,
spectrum_analyzer_resolution_bandwidth,
spectrum_analyzer_resolution_bandwidth_unit,
spectrum_analyzer_video_bandwidth,
spectrum_analyzer_video_bandwidth_unit,
spectrum_analyzer_frequency_span,
spectrum_analyzer_frequency_span_unit,
spectrum_analyzer_IF_atten_enable,
spectrum_analyzer_IF_atten,
spectrum_analyzer_IF_relative_level_enable,
spectrum_analyzer_IF_relative_level)
FSV.write("CALC:MARKER ON") # enable marker mode
calibration_IF = []
calibration_function_IF = None
calibration_function_IF_unit = unit.Hz
frequency_range = synthetizer_frequency.return_range()
maxcount = len(frequency_range)
count = 0
continue_progress = (True, True)
for f in frequency_range: # frequency loop
# set SMB100A frequency
f_value = str(f)
current_frequency = f_value + unit.return_unit_str(unit.Hz)
current_frequency_human_readable = unit.return_human_readable_str(f)
command = "FREQ " + current_frequency # Ex. FREQ 500kHz
SMB.write(command)
SMB.write("OUTP ON")
time.sleep(2)
data_now = str(datetime.datetime.now())
if create_dummy_cable:
values.append([f_value, unit.return_unit_str(unit.Hz)] +
[str(dummy_cable_power_level)] + [data_now])
else:
values.append(
readFSV_marker(FSV,
FSV_delay,
f,
unit.Hz,
unit.Hz,
calibration_IF=[],
calibration_IF_function=None,
calibration_IF_function_unit=None) + [data_now])
# turn off RF
SMB.write("OUTP OFF")
count += 1
if not createprogressdialog is None:
import wx
wx.MicroSleep(500)
message = "{lo_freq}".format(
lo_freq=current_frequency_human_readable)
newvalue = int(float(count) / maxcount * 100)
if newvalue >= 100:
createprogressdialog = False
# dialog.Update(newvalue, message)
# wx.MicroSleep(500)
dialog.Close()
else:
continue_progress = dialog.Update(newvalue, message)
if not continue_progress[0]:
dialog.Destroy()
break
if f == frequency_range[-1]:
# safety turn Off
# dialog.Update(100, "Measure completed)
dialog.Destroy()
# turn off RF
SMB.write("OUTP OFF")
# Output [Frequenza, output power meter(Loss), time]
print("Misure completed\n")
# build header for data table on file
header = ["Frequency", "Unit"]
values_headers = []
values_headers.append("Loss value(dB)")
values_headers.append("Time")
# save data on file
savefile_path = result_file_name + "_" + return_now_postfix() + ".csv"
try:
f_csv = open(savefile_path, "wb")
create_csv(f_csv, header, values_headers, values)
f_csv.close()
except:
# on error print data on standard output
print(values)
return savefile_path