def main(name,iterations,eva,count,processCt): data = getData(name) intervals = [] intervals.append(1) for x in range(1,count+1): intervals.append(x*(eva/count)) evaluations = eva travWorkers = mp.Pool(processes=processCt) genPmxJobs = [travWorkers.apply_async(genetic, args=(data,evaluations,intervals,False)) for x in range(iterations)] genHalfJobs = [travWorkers.apply_async(genetic, args=(data,evaluations,intervals,True)) for x in range(iterations)] stochJobs = [travWorkers.apply_async(stochClimb, args=(data,evaluations,intervals)) for x in range(iterations)] randJobs= [travWorkers.apply_async(randSearch, args=(data,evaluations,intervals)) for x in range(iterations)] # Close the pool and wait until all processes are finished travWorkers.close() travWorkers.join() # Retrieve process outputs genPmx = [t.get() for t in genPmxJobs] genHalf = [t.get() for t in genHalfJobs] stoch = [t.get() for t in stochJobs] rand = [t.get() for t in randJobs] # Display graph of simulation results graph(genPmx,genHalf,stoch,rand,intervals)
def readData(input_file):
result = {}
data = tools.getData(input_file)
for i, row in enumerate(data):
# skip empty rows
if row[0]:
# skip first row of labels
if i > 0:
cable_number = int(row[0])
separation = int(row[1])
channel = row[2]
values = []
# skip rows that are not channel data
if channel in channels:
for j in range(3, len(row)):
values.append(int(row[j]))
dataset_1 = values[:3]
dataset_2 = [values[3], values[1], values[4]]
#print(row)
#print(values)
#print(dataset_1)
#print(dataset_2)
if cable_number not in result:
result[cable_number] = {}
if separation not in result[cable_number]:
result[cable_number][separation] = {}
if channel not in result[cable_number][separation]:
result[cable_number][separation][channel] = {}
result[cable_number][separation][channel]["dataset_1"] = dataset_1
result[cable_number][separation][channel]["dataset_2"] = dataset_2
return result
def getData():
df = pandas.read_csv(consts.filename)
# print(df.head())
X, Y = tools.getData(df)
theta = numpy.zeros(X.shape[1] + 1)
return X, Y, theta
def run(input_files, output_file):
# define decimal precision for values saved in table
precision = 3
# maps to define columns that contain data from raw data csv files
# row and column indices start from 0 for data matrix
row_map = {"label1": 0, "label2": 0, "value": 1}
column_map = {"height": 13, "jitter": 43, "width": 53}
unit_map = {"height": 10**3, "jitter": 10**12, "width": 10**12}
output_column_titles = [
"Index", "TAP0", "Height (mV)", "Jitter (ps)", "Width (ps)"
]
# write to output file
with open(output_file, 'w', newline='') as output_csv:
output_writer = csv.writer(output_csv)
output_writer.writerow(output_column_titles)
index = 1
for tap in settings:
f = input_files[tap]
print("{0}: {1}, {2}".format(index, tap, f))
data = tools.getData(f)
# get values, convert to floats, and convert to standard units
height = float(data[row_map["value"]][
column_map["height"]]) * unit_map["height"]
jitter = float(data[row_map["value"]][
column_map["jitter"]]) * unit_map["jitter"]
width = float(data[row_map["value"]][
column_map["width"]]) * unit_map["width"]
# columns for output table: index, TAP0, height, jitter, width
output_row = [
index, tap,
round(height, precision),
round(jitter, precision),
round(width, precision)
]
output_writer.writerow(output_row)
index += 1
def plotData(input_file, output_file, plot_dir, title, column_indices, xlim,
ylim, drawMean):
verbose = False
tools.makeDir(plot_dir)
data = tools.getData(input_file)
output_png = "{0}/{1}.png".format(plot_dir, output_file)
output_pdf = "{0}/{1}.pdf".format(plot_dir, output_file)
# get default colors
prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color']
# get data from column
column_index_1 = column_indices[0]
column_index_2 = column_indices[1]
data_label = ""
x_vals = []
y1_vals = []
y2_vals = []
for i, row in enumerate(data):
# skip first row
if i < 1:
continue
# second row has labels
elif i == 1:
data_label = row[column_index_1]
# third row is the beginning of data values
else:
# WARNING: make sure to convert strings to floats!
x = float(i - 1)
y1 = float(row[column_index_1])
y2 = float(row[column_index_2])
x_vals.append(x)
y1_vals.append(y1)
y2_vals.append(y2)
if verbose:
print("{0}: {1}, {2}".format(i, row[column_index_1],
row[column_index_2]))
# plot
fig, ax = plt.subplots(figsize=(6, 6))
x_array = np.array(x_vals)
y1_array = np.array(y1_vals)
y2_array = np.array(y2_vals)
y1_mean = np.mean(y1_array)
y2_mean = np.mean(y2_array)
y1_std = np.std(y1_array)
y2_std = np.std(y2_array)
# extend x to plot mean and std dev
x_extended = np.insert(x_array, 0, float(x_array[0] - 1))
x_extended = np.append(x_extended, float(x_array[-1] + 1))
if verbose:
print("x = {0}".format(x_array))
print("x_extended = {0}".format(x_extended))
print("y1 = {0}".format(y1_array))
print("y2 = {0}".format(y2_array))
print("y1_mean = {0}".format(y1_mean))
print("y2_mean = {0}".format(y2_mean))
print("y1_std = {0}".format(y1_std))
print("y2_std = {0}".format(y2_std))
ax.set_xlim(xlim)
ax.set_ylim(ylim)
ax.set_title(title, fontsize=16)
ax.set_xlabel("Channel", fontsize=12)
ax.set_ylabel(data_label, fontsize=12)
p1 = plt.scatter(x_array,
y1_array,
color=colors[0],
label="before lashing")
p2 = plt.scatter(x_array, y2_array, color=colors[1], label="after lashing")
objects = [p1, p2]
if drawMean:
p3 = plt.axline((x_extended[0], y1_mean), (x_extended[-1], y1_mean),
color=colors[0],
linestyle="--",
label="mean")
p4 = plt.axline((x_extended[0], y2_mean), (x_extended[-1], y2_mean),
color=colors[1],
linestyle="--",
label="mean")
p5 = plt.fill_between(x_extended,
y1_mean - y1_std,
y1_mean + y1_std,
color=colors[0],
alpha=0.2,
label="std dev")
p6 = plt.fill_between(x_extended,
y2_mean - y2_std,
y2_mean + y2_std,
color=colors[1],
alpha=0.2,
label="std dev")
objects = [p1, p2, p3, p4, p5, p6]
# specify order for legend
labels = [o.get_label() for o in objects]
plt.legend(objects, labels, loc='upper right', prop={'size': 12})
plt.savefig(output_png)
plt.savefig(output_pdf)
def plotData(input_file_1, input_file_2, label_1, label_2, output_file,
plot_dir, title, column_index, xlim, ylim, drawMean):
verbose = False
tools.makeDir(plot_dir)
data_1 = tools.getData(input_file_1)
data_2 = tools.getData(input_file_2)
output_png = "{0}/{1}.png".format(plot_dir, output_file)
output_pdf = "{0}/{1}.pdf".format(plot_dir, output_file)
# get default colors
prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color']
# get data from column
data_label = ""
x_vals = []
y1_vals = []
y2_vals = []
# get x, y1, and data label
for i, row in enumerate(data_1):
# first row has labels
if i == 0:
data_label = row[column_index]
# second row is the beginning of data values
else:
# WARNING: make sure to convert strings to floats!
x = float(row[0])
y1 = float(row[column_index])
x_vals.append(x)
y1_vals.append(y1)
if verbose:
print("{0}: {1}".format(i, row[column_index]))
# get y2
for i, row in enumerate(data_2):
# second row is the beginning of data values
if i > 0:
# WARNING: make sure to convert strings to floats!
y2 = float(row[column_index])
y2_vals.append(y2)
if verbose:
print("{0}: {1}".format(i, row[column_index]))
# plot
fig, ax = plt.subplots(figsize=(6, 6))
x_array = np.array(x_vals)
y1_array = np.array(y1_vals)
y2_array = np.array(y2_vals)
y1_mean = np.mean(y1_array)
y2_mean = np.mean(y2_array)
y1_std = np.std(y1_array)
y2_std = np.std(y2_array)
# take ratio of means
r21 = y2_mean / y1_mean
# get error of ratio (assuming independent variables without correlation)
# q = x / y
# getMultiplicationError(q, x, dx, y, dy)
r21_err = tools.getMultiplicationError(r21, y2_mean, y2_std, y1_mean,
y1_std)
# extend x to plot mean and std dev
x_extended = np.insert(x_array, 0, float(x_array[0] - 1))
x_extended = np.append(x_extended, float(x_array[-1] + 1))
if verbose:
print("x = {0}".format(x_array))
print("x_extended = {0}".format(x_extended))
print("y1 = {0}".format(y1_array))
print("y2 = {0}".format(y2_array))
print("y1_mean = {0}".format(y1_mean))
print("y2_mean = {0}".format(y2_mean))
print("y1_std = {0}".format(y1_std))
print("y2_std = {0}".format(y2_std))
print("r21 = {0}".format(r21))
print("r21_err = {0}".format(r21_err))
ax.set_xlim(xlim)
ax.set_ylim(ylim)
ax.set_title(title, fontsize=16)
ax.set_xlabel("Channel", fontsize=12)
ax.set_ylabel(data_label, fontsize=12)
p1 = plt.scatter(x_array, y1_array, color=colors[0], label=label_1)
p2 = plt.scatter(x_array, y2_array, color=colors[1], label=label_2)
objects = [p1, p2]
if drawMean:
p3 = plt.axline((x_extended[0], y1_mean), (x_extended[-1], y1_mean),
color=colors[0],
linestyle="--",
label="mean")
p4 = plt.axline((x_extended[0], y2_mean), (x_extended[-1], y2_mean),
color=colors[1],
linestyle="--",
label="mean")
p5 = plt.fill_between(x_extended,
y1_mean - y1_std,
y1_mean + y1_std,
color=colors[0],
alpha=0.2,
label="std dev")
p6 = plt.fill_between(x_extended,
y2_mean - y2_std,
y2_mean + y2_std,
color=colors[1],
alpha=0.2,
label="std dev")
objects = [p1, p2, p3, p4, p5, p6]
# text
text_x = xlim[0] + 0.1 * (xlim[1] - xlim[0])
text_y1 = ylim[0] + 0.9 * (ylim[1] - ylim[0])
text_y2 = ylim[0] + 0.8 * (ylim[1] - ylim[0])
text_y3 = ylim[0] + 0.7 * (ylim[1] - ylim[0])
equation1 = r"$\mu_1 = {0:.2f} \pm {1:.2f}$".format(y1_mean, y1_std)
equation2 = r"$\mu_2 = {0:.2f} \pm {1:.2f}$".format(y2_mean, y2_std)
# use double curly brackets {{21}} so that this is not used as an index by format
equation3 = r"$r_{{21}} = {0:.2f} \pm {1:.2f}$".format(r21, r21_err)
ax.text(text_x, text_y1, equation1, fontsize=15)
ax.text(text_x, text_y2, equation2, fontsize=15)
ax.text(text_x, text_y3, equation3, fontsize=15)
# specify order for legend
labels = [o.get_label() for o in objects]
plt.legend(objects, labels, loc='upper right', prop={'size': 12})
plt.savefig(output_png)
plt.savefig(output_pdf)
remove_zero = True
for filename in os.listdir("../tables"):
if filename.endswith(".csv"):
print(filename)
cable_num = -1
matches = p.findall(filename)
if len(matches) == 1:
cable_num = int(matches[0])
cable_nums.append(cable_num)
else:
print("Error: Unique Number not found")
print(matches)
#TO DO: FIX CABLE HEIGHTS (they are currently overwritten for multiple measurements)
cable_data[filename] = getData("../tables/" + filename)
cable_heights[cable_num] = getHeights(cable_data[filename])
print(cable_heights[cable_num])
means.append(np.mean(cable_heights[cable_num]))
print(np.std(cable_heights[cable_num]))
print(cable_nums)
print(np.mean(means))
#print(cable_heights[])
#print(np.mean(cable_heights))
#print(np.std(cable_heights))
cableInfo = getData("../HarnessSerialNumber.csv")
cableMap = {}
def getArea(input_file):
data = tools.getData(input_file)
area = int(data[6][1])
return area
def run(input_files, output_file, modules, channels, requireModule):
verbose = False
# define decimal precision for values saved in table
precision = 3
# maps to define columns that contain data from raw data csv files
# row and column indices start from 0 for data matrix
# raw data
#row_map = {"label1" : 32, "label2" : 33, "value" : 34}
#column_map = {"height" : 13, "jitter" : 43, "width" : 53}
# clean data
row_map = {"label1": 0, "label2": 0, "value": 1}
column_map = {"height": 13, "jitter": 43, "width": 53}
unit_map = {"height": 10**3, "jitter": 10**12, "width": 10**12}
output_column_titles = [
"Index", "Module", "Channel", "Height (mV)", "Jitter (ps)",
"Width (ps)"
]
# write to output file
with open(output_file, 'w', newline='') as output_csv:
output_writer = csv.writer(output_csv)
output_writer.writerow(output_column_titles)
index = 1
for module in modules:
for channel in channels:
f = getFile(input_files, module, channel, requireModule)
if not f:
print(
"ERROR: Unique file not found for module {0} channel {1}."
.format(module, channel))
return
data = tools.getData(f)
if verbose:
print(f)
for key in column_map:
# remove space from strings
# convert values to floats
label1 = data[row_map["label1"]][
column_map[key]].strip()
label2 = data[row_map["label2"]][
column_map[key]].strip()
value = float(data[row_map["value"]][column_map[key]])
print(" - {0}: {1} = {2:.2E}".format(
label1, label2, value))
# get values, convert to floats, and convert to standard units
height = float(data[row_map["value"]][
column_map["height"]]) * unit_map["height"]
jitter = float(data[row_map["value"]][
column_map["jitter"]]) * unit_map["jitter"]
width = float(data[row_map["value"]][
column_map["width"]]) * unit_map["width"]
# columns for output table: index, module, channel, height, jitter, width
output_row = [
index, module, channel,
round(height, precision),
round(jitter, precision),
round(width, precision)
]
output_writer.writerow(output_row)
index += 1
searches = ["75889", "75890", "75891", "75892", "75893", "75894", "75895", "75896", "75897", "75898", "75899", "75900", "75901", "75902", "75903", "75904"]
response = requests.get(
"https://mfwkweb-api.clarovideo.net/services/content/list?device_id=web&device_category=web&device_model=web&device_type=web&device_so=Chrome&format=json&device_manufacturer=generic&authpn=webclient&authpt=tfg1h3j4k6fd7&api_version=v5.92®ion=argentina&HKS=f9p95s8eo8mjmt4oe80fc7u9r4&quantity=1000&from=0&level_id=GPS&order_way=ASC&order_id=100&filter_id=39263")
values = response.json()
tools.getNumId(array, values)
for b in searches:
response = requests.get(
"https://mfwkweb-api.clarovideo.net/services/content/list?device_id=web&device_category=web&device_model=web&device_type=web&device_so=Chrome&format=json&device_manufacturer=generic&authpn=webclient&authpt=tfg1h3j4k6fd7&api_version=v5.92®ion=argentina&HKS=f9p95s8eo8mjmt4oe80fc7u9r4&order_id=100&order_way=ASC&level_id=GPS&from=0&quantity=1000&node_id="+b)
values = response.json()
tools.getNumId(array, values)
array.sort()
for e in array:
arrayAux.append(tools.getData(e))
with open('claro.json', 'w') as outfile:
for e in arrayAux:
json.dump(e, outfile)
outfile.write("\n")
orig_urls = [ ]
for x in range(1, 31):
print("# Generating Urls... (%d/30)" % x)
orig_urls.append('http://list.youku.com/category/show/c_97_s_1_d_2_p_'+ str(x) +'.html')
codes = [ ]
for x, url in enumerate(orig_urls):
print("# Fetching urls from pages... (%d/30)" % (x + 1))
fetched_urls = tools.fetchUrls(url)
print("## Total fetched: %d" % len(fetched_urls))
code = tools.parseUrls(fetched_urls)
print("## Invalid code parsed: %d" % len(code))
list(map(lambda x: codes.append(x), code))
codes = list(set(codes))
total = len(codes)
print("# Total invalied code: %d" % total)
datas = [ ]
for x, code in enumerate(codes):
print("# Fetching data for %s (%d/%d)" % (code, x + 1, total))
datas.append(tools.getData(code))
print("# Fetching finished. Saving to file: data.json")
f = open('data.json', 'w')
f.write(json.dumps(datas))
f.close()
def plotData(input_file, output_file, plot_dir, title, x_column_index,
y_column_index, ylim, drawMean, drawFit):
verbose = False
tools.makeDir(plot_dir)
data = tools.getData(input_file)
output_png = "{0}/{1}.png".format(plot_dir, output_file)
output_pdf = "{0}/{1}.pdf".format(plot_dir, output_file)
# get default colors
prop_cycle = plt.rcParams['axes.prop_cycle']
colors = prop_cycle.by_key()['color']
# get data from column
x_label = ""
y_label = ""
x_vals = []
y_vals = []
# loop over data
for i, row in enumerate(data):
# first row has labels
if i == 0:
x_label = row[x_column_index]
y_label = row[y_column_index]
# second row is the beginning of data values
else:
# WARNING: make sure to convert strings to floats!
x = float(row[x_column_index])
y = float(row[y_column_index])
x_vals.append(x)
y_vals.append(y)
if verbose:
print("{0}: {1}".format(i, row[y_column_index]))
# plot
fig, ax = plt.subplots(figsize=(6, 6))
x_array = np.array(x_vals)
y_array = np.array(y_vals)
y_mean = np.mean(y_array)
y_std = np.std(y_array)
# extend x to define xlim and plot mean and std dev
step = x_array[1] - x_array[0]
x_extended = np.insert(x_array, 0, float(x_array[0] - step))
x_extended = np.append(x_extended, float(x_array[-1] + step))
xlim = [x_extended[0], x_extended[-1]]
if verbose:
print("x = {0}".format(x_array))
print("x_extended = {0}".format(x_extended))
print("xlim = {0}".format(xlim))
print("y = {0}".format(y_array))
print("y_mean = {0}".format(y_mean))
print("y_std = {0}".format(y_std))
# format plot
ax.set_xlim(xlim)
ax.set_ylim(ylim)
ax.set_title(title, fontsize=16)
ax.set_xlabel(x_label, fontsize=12)
ax.set_ylabel(y_label, fontsize=12)
p1 = plt.scatter(x_array, y_array, color=colors[0], label="data")
objects = [p1]
# draw mean and std dev on plot
if drawMean:
p2 = plt.axline((x_extended[0], y_mean), (x_extended[-1], y_mean),
color=colors[0],
linestyle="--",
label="mean")
p3 = plt.fill_between(x_extended,
y_mean - y_std,
y_mean + y_std,
color=colors[0],
alpha=0.2,
label="std dev")
objects = [p1, p2, p3]
# text
text_x = xlim[0] + 0.1 * (xlim[1] - xlim[0])
text_y = ylim[0] + 0.9 * (ylim[1] - ylim[0])
equation = r"$\mu = {0:.2f} \pm {1:.2f}$".format(y_mean, y_std)
ax.text(text_x, text_y, equation, fontsize=15)
# draw fit on plot
if drawFit:
model = np.polyfit(x_array, y_array, 1)
predict = np.poly1d(model)
#print(predict)
p4 = plt.plot(x_array,
predict(x_array),
label='fit',
color='red',
linestyle='dashed')
#objects = [p1, p4]
# text
text_x = xlim[0] + 0.1 * (xlim[1] - xlim[0])
text_y = ylim[0] + 0.9 * (ylim[1] - ylim[0])
f_x = "{0}".format(predict)
# remove whitespace and newlines
f_x = f_x.strip()
equation = r"y = {0}".format(f_x)
ax.text(text_x, text_y, equation, fontsize=15)
# specify order for legend
labels = [o.get_label() for o in objects]
plt.legend(objects, labels, loc='upper right', prop={'size': 12})
plt.savefig(output_png)
plt.savefig(output_pdf)