def write_master_list(parent_dir, version):
"""Writes a master data lis in the parent directory for batch mode.
Args:
parent_dir (string): path of the parent directory
version (string): the version of the software (for naming output file)
"""
dir_list_path = parent_dir + '/dir_list.dat'
subdir_list = [_[0] for _ in uf.read_file(dir_list_path)][1:]
master_data = []
for i in range(len(subdir_list)):
data_dir = parent_dir + '/' + subdir_list[i]
data = uf.read_file(data_dir + '/ls_data/ls_data.dat')
if i == 0:
for line in data:
master_data.append(line)
else:
for line in data[1:]:
master_data.append(line)
# print master_data
uf.save_data_array(master_data,
parent_dir + '/master_list_v%s.dat' % version)
def __run_test(self, filename, output_directory):
"""
Handles running of data through algorithm not live, parses the file and puts data line by line
through the algorithm, algorithm will react to it as if the data is being collected real time. Used in testing
mode as can be run at home to see how algorithm reacts.
Args:
filename: name of recorded data to run through, defaults to latest file
output_directory: relative path to output_directory folder
Returns:
None, but stores the logs of the test in a file in output_directory
Example:
> self.__run_test('data_to_load', '../Output_data')
"""
# Read old data
print(
'\n\033[1mUsing test mode, will apply algorithm to data from file {}\033[0m\n'
.format(filename))
data = read_file(output_directory + filename)
self.all_data = self.initialize_all_data()
# some functions depend on sampling period, therefore extract correct
# period and place into algorithm data so that it can be passed through
average_cycle_time = numpy.mean(numpy.diff(data['time']))
for algorithm in self.order:
algorithm['period'] = average_cycle_time
switch = 'switch'
for i in xrange(len(data)):
algo = self.algo_name
# Make current row out of real values from data minus the position and algorithm
# as those are the things we are running testing to watch
row_no_pos_algo = list(data[i])[:-2]
current_values = convert_list_dict(row_no_pos_algo +
[algo, self.position])
try:
switch = self.__run_algorithm(switch, current_values)
except AlgorithmFinished:
print '\n\033[1mAlgorithm finished, stopping now\033[0m\n'
break
# Data loaded in will have ' Org' file so remove that and replace with ' Tst'
store(filename[:-4] + ' Tst', self.all_data)
def load_ls(ls_path, px_size=1.):
"""Loads a linescan file
Args:
ls_path (str): path of the average linescan file to be loaded
px_size (float): pixel size in microns
Returns:
x (numpy array): the positions (in microns)
i (numpy array): the intensities
"""
ls_data = uf.read_file(ls_path)
x = np.array([float(_[0]) for _ in ls_data]) * px_size
i = np.array([float(_[1]) for _ in ls_data])
return x, i
def __run_test(self, filename, output_directory):
"""
Function to run the interface with a fake robot and fake encoders
Args:
filename: The name of the file containing prvious data to run the interface off
output_directory: name of the Directory containg the data file
"""
# Read old data
print(
'\n\033[1mUsing test mode, will apply algorithm to data from file {}\033[0m\n'
.format(filename))
data = read_file(output_directory + filename)
self.all_data = self.initialize_all_data()
# some functions depend on sampling period, therefore extract correct
# period and place into algorithm data so that it can be passed through
average_cycle_time = numpy.mean(numpy.diff(data['time']))
for algorithm in self.order:
algorithm['period'] = average_cycle_time
switch = 'switch'
for i in xrange(len(data)):
algo = self.algo_name
# Make current row out of real values from data minus the position and algorithm
# as those are the things we are running testing to watch
row_no_pos_algo = list(data[i])[:-2]
current_values = convert_list_dict(row_no_pos_algo +
[algo, self.position])
try:
switch = self.__run_algorithm(switch, current_values)
except AlgorithmFinished:
print '\n\033[1mAlgorithm finished, stopping now\033[0m\n'
break
# Data loaded in will have ' Org' file so remove that and replace with ' Tst'
store(filename[:-4] + ' Tst.txt', self.all_data)
from scipy.signal import find_peaks
import numpy as np
from sys import path
path.insert(0, '..')
from graph_functions import *
from utility_functions import read_file
fig, ax = plt.subplots(1)
ax = format_graph(ax)
files = ['Quarter Period', 'Angular Velocity', 'Max Angle']
files = ['Quarter Period']
for filename in files:
data = read_file('../Output_data/' + filename)
be = data['be']
t = data['time']
algorithm = data['algo']
change_indexes = shade_background_based_on_algorithm(t,
algorithm,
plot=False)
t = t[change_indexes[0]:]
be = be[change_indexes[0]:]
t = t[be > 0]
be = be[be > 0]
max_indexes = find_peaks(be)[0]
be = be[max_indexes]
fig, ax = plt.subplots(1, 1, figsize=(13, 8))
ax = format_graph(ax)
information = OrderedDict({})
for offset in sorted(files_offsets.keys()):
print 'Offset: {:.2f}'.format(offset)
information[offset] = {
'gradients': [],
'periods': [],
}
for i, filename in enumerate(files_offsets[offset]):
if filename != '':
data = read_file(output_data_directory + filename)
time = data['time']
be = data['be']
algorithm = data['algo']
for pos in range(2):
# find indexes where algorithm changes
algorithm_change_indexes = np.append(
np.where(algorithm[:-1] != algorithm[1:])[0],
np.array(len(algorithm) - 1))
# only take data after has switched to quarter period
time = time[algorithm_change_indexes[1]:]
be = be[algorithm_change_indexes[1]:]
window = 21
def plot_maxima_curve(filename, plot=True):
angles = read_file(filename)
time = angles['time']
be = angles['be']
algo = angles['algo']
algo_change_indexes = shade_background_based_on_algorithm(time,
algo,
plot=False)
# only take values from when algorithm is switched to increasing
time = time[algo_change_indexes[1]:]
be = be[algo_change_indexes[1]:]
# there is a weird spike that goes really high, this filters that out
time = time[be < 30]
be = be[be < 30]
# again something else to filter
time = time[be > 1]
be = be[be > 1]
# calculate maximas after smoothing results together
window_size = 5
avg_be = np.array(moving_average(be, window_size=window_size))
angle_max_index = (np.diff(np.sign(np.diff(avg_be))) <
0).nonzero()[0] + 1 + (window_size - 1) / 2
# take times and values at maximas
time = time[angle_max_index]
be = be[angle_max_index]
# Some specific filtering for the rotational datafile
if filename == '../Output_data/Rotational No Masses 400secs':
filtered_time, filtered_be = [], []
for time_, be_ in zip(time, be):
# print time_, be_
if time_ <= 165 or time_ >= 185:
filtered_time.append(time_)
filtered_be.append(be_)
elif 165 < time_ < 185:
if be_ > 8.4:
filtered_time.append(time_)
filtered_be.append(be_)
else:
print "Doesn't fit category", time_, be_
time, be = filtered_time, filtered_be
# Remove as many weird spikes as possible
filtered_time, filtered_be = [], []
for time_, be_ in zip(time, be):
if time_ <= 260 or 400 <= time_ <= 600 or time_ >= 700:
filtered_time.append(time_)
filtered_be.append(be_)
elif 260 < time_ < 400:
if be_ > 6.8:
filtered_time.append(time_)
filtered_be.append(be_)
elif 600 < time_ < 700:
if be_ > 17.5:
filtered_time.append(time_)
filtered_be.append(be_)
else:
print "Doesn't fit category", time_, be_
# convert to numpy arrays get some weird error without
time, be = np.array(filtered_time), np.array(filtered_be)
# centre them so they both start the plot at zero
print 'Time offset for file {}: {}s'.format(filename, time[0])
time -= time[0]
label = get_name(each_file)[:-1]
if label == 'Rotational No Masses 400secs':
label = 'Quarter Period'
if plot:
# plot against each other
plt.plot(time, be, label=label)
return time, be
import matplotlib.pyplot as plt
import numpy as np
from graph_functions import *
from sys import path
path.insert(0, '..')
from utility_functions import read_file
output_data = '../Output_data/'
filenames = ['Box Masses', 'Box No Masses']
for filename in filenames:
data = read_file(output_data + filename)
t = data['time']
be = data['be']
plt.plot(t, be, label=filename)
plt.legend(loc='best')
plt.show()
files_to_compare = [latest_filename]
fits = ['linear']
fit_dict = {
'linear': linear_fit
}
# setup figure
fig, ax = plt.subplots(
1, 1, figsize=(
13, 8))
ax = format_graph(ax)
for i, _file in enumerate(files_to_compare):
# collect the data
angles = read_file(output_data_directory + _file)
# Extract data
t = angles['time']
be = angles['be']
algorithm = angles['algo']
# find indexes where the algorithm changes
algo_change_indexes = algorithm_change_indexes(algorithm)
# [0] is first algorithm (start), [1] is second (increasing)
increase_index = algo_change_indexes[1]
# only take time from switch point, change time relative to this point
t = t[increase_index:] - t[increase_index]
# only take encoder values from this point
be = be[increase_index:]
from sys import path
path.append("..")
import numpy as np
from scipy.signal import find_peaks
from scipy.optimize import curve_fit
from matplotlib import pyplot as plt
from utility_functions import read_file
from graph_functions import format_graph
#parametric = read_file("../Output_data/Parametric No Masses 800secs")['be']
#rotational = read_file("../Output_data/Rotational No Masses 400secs")['be']
parametric = read_file("../Output_data/26-03-2019 18:17:02 Para 600 secs Org")
rotational = read_file("../Output_data/26-03-2019 18:30:45 Rot 600 secs Org")
para_dt = np.mean(np.diff(parametric['time']))
rot_dt = np.mean(np.diff(rotational['time']))
index_para = find_peaks(parametric['be'])[0]
index_rot = find_peaks(rotational['be'])[0]
peaks_para = abs(parametric['be'][index_para])
peaks_rot = abs(rotational['be'][index_rot])
times_para = parametric['time'][index_para]
times_rot = abs(rotational['time'][index_rot])
##### George's magical filter.
be = peaks_para
t = times_para
if location[-i] == "/":
break
result = location[-i] + result
return result
files_to_compare = get_files(output_data_directory)
fig, ax = plt.subplots(1, 1, figsize=(8, 6))
ax = format_graph(ax)
ax.set_facecolor('#eeeeee')
for each_file in files_to_compare:
angles = read_file(each_file)
#angles = convert_read_numpy(angles) # Not sure if this should be here, it seems it was removed since my last pull.
time = angles['time']
be = angles['be']
angle_max_index = (np.diff(np.sign(np.diff(be))) < 0).nonzero()[0] + 1
be = be[time > 12.06]
time = time[time > 12.06]
print each_file
if each_file == '../Output_data/Decrease Parametric':
label = 'Parametric Decrease'
be -= 1.2
elif each_file == '../Output_data/decreasingdamping':
label = 'Natural Damping'
# true_max = time[angle_max_index][5]
def main():
"""__main__ function"""
version = '5'
#set up root for asking questions
# root = Tk() #moved this up to the imports
root.withdraw()
#chooses analysis mode
mode = askinteger(
title="Analysis Mode Selection",
prompt=
"Please enter:\n1 for pairwise analysis or\n2 for batch analysis",
minvalue=1,
maxvalue=2)
if mode == 1:
ch1_path = askopenfilename(
title='Select an average linescan file for channel 1',
filetypes=[("dat", "*.dat")],
initialdir='.',
initialfile="")
ch2_path = askopenfilename(
title='Select an average linescan file for channel 2',
filetypes=[("dat", "*.dat")],
initialdir='/'.join(ch1_path.split('/')[:-1]),
initialfile=ch1_path.split('/')[-1])
px_size = askfloat(title='Pixel Size',
prompt='Please enter your pixel size')
ch_actin = askinteger(title='Actin Channel',
prompt='Please enter the actin channel',
minvalue=1,
maxvalue=2)
sigma_actin = askfloat(
title='Actin Sigma',
prompt=
'Please enter the sigma value\nfor the PSF for the actin channel\n(in microns)'
)
analyze_ls_pair(ch1_path, ch2_path, px_size, ch_actin, sigma_actin,
version)
if mode == 2:
parent_dir = askdirectory(
title=
'Select the parent directory (be sure it contains dir_list.dat!)',
initialdir=os.path.split(os.path.realpath(__file__))[0])
# parent_dir = './test_data'
dir_list = uf.get_dict_list(uf.read_file(parent_dir + '/dir_list.dat'))
for line in dir_list:
sub_dir = line['sub_dir']
px_size = float(line['px_size'])
category = line['category']
ch_actin = int(line['ch_actin'])
sigma_actin = float(line['sigma_actin'])
data_dir = parent_dir + '/' + sub_dir
print(data_dir)
analyze_dir(data_dir, px_size, category, ch_actin, sigma_actin,
version)
write_master_list(parent_dir, version)
clear_file(FLAGGED_SNR_FILE)
clear_file(GOOD_FIT_FILE)
clear_file(ORIGINAL_FILE)
clear_file(UNFLAGGED_FILE)
append_row_to_csv(FLAGGED_ABSORPTION_FILE, fields)
append_row_to_csv(FLAGGED_BAD_FIT_FILE, fields)
append_row_to_csv(FLAGGED_SNR_FILE, fields_snr)
append_row_to_csv(GOOD_FIT_FILE, fields)
append_row_to_csv(ORIGINAL_FILE, field)
append_row_to_csv(UNFLAGGED_FILE, fields)
clear_file(LOG_FILE)
clear_file(LOG_NO_LOW_SNR_FILE)
redshift_value_list, snr_value_list, spectra_list = read_file(CONFIG_FILE)
indices, spectra_indices, processed_spectra_file_names, powerlaw_final_b_values, powerlaw_final_c_values = [], [], [], [], []
flagged_indices, flagged_spectra_indices, flagged_spectra_file_names = [], [], []
flagged_snr_indices, flagged_snr_spectra_indices, flagged_snr_spectra_file_names, flagged_snr_in_ehvo_values = [], [], [], []
for spectra_index in range(STARTS_FROM, ENDS_AT + 1):
z = round(redshift_value_list[spectra_index - 1], 5)
snr = round(snr_value_list[spectra_index - 1], 5)
current_spectrum_file_name = spectra_list[spectra_index - 1]
print(str(spectra_index) + ": " + current_spectrum_file_name)
print_to_file(
str(spectra_index) + ": " + current_spectrum_file_name, LOG_FILE)
current_spectra_data = np.loadtxt(SPEC_DIREC + current_spectrum_file_name)
# infoDR16_all = sys.argv[1] if len(sys.argv) > 1 else "DR16_sorted_norm.csv"
# infoDR16_snr = sys.argv[1] if len(sys.argv) > 1 else os.getcwd() + "/OUTPUT_FILES/NORMALIZATION/parent_sample.csv" ### ALL CASES WITH SNR > 10
infoDR16_parent = sys.argv[1] if len(
sys.argv) > 1 else "DR16_parent_sample.csv"
#infoDR16_BAL = sys.argv[1] if len(sys.argv) > 1 else "DR16_BAL.csv"
#infoDR16_EHVO = sys.argv[1] if len(sys.argv) > 1 else "DR16_EHVO.csv"
#spectra_list, zem, calc_snr_list = read_list_spectra(infoDR16_snr, ["NORM SPECTRA FILE NAME", "REDSHIFT", "CALCULATED SNR"])
# zem_orig, snr, spectra_name = read_file(infoDR16)
zem = np.zeros((specnum1) - 1) ## Redshift
zem_EHVO = [] ## EHVO Redshift
zem_BAL = [] ## BAL Redshift
# zem, snr, spectra_list = read_file(infoDR16_snr)
# zem_all, snr_all, spectra_list_all = read_file(infoDR16_all)
zem, snr, spectra_list = read_file(infoDR16_parent)
#zem_BAL, snr_BAL, spectra_list_BAL = read_file(infoDR16_BAL)
#zem_EHVO, snr_EHVO, spectra_list_EHVO = read_file(infoDR16_EHVO)
## Read EHVO
# for m in range(0,specnum3):
# ee=infoEHVO[m]
# columns3=ee.split()
# QUASARNAME_EHVO.append(str(columns3[0]))
# PLATE_inEHVO.append(int(columns3[1]))
# MJD_inEHVO.append(int(columns3[2]))
# FIBERID_inEHVO.append(int(columns3[3]))
# vmax_inEHVO.append(int(columns3[4]))
# vmin_inEHVO.append(int(columns3[5]))
# Linear.
return m * n + c
def fit_parametric(n, m, c, q, k):
# Linear then exponential.
# y(n<q) = mn+c
# y(n>=q) = (mq+c) e^k(n-q)
lin = lambda n: m * n + c
exp = lambda n: (m * q + c) * np.exp(k * (n - q))
ans = np.piecewise(n, [n < q, n >= q], [lin, exp])
print ans
return float(ans)
para_angles = read_file(files_to_compare[1])['be']
rot_angles = read_file(files_to_compare[0])['be']
def plot_maxima_curve(filename, plot=True, fit_func=None, p0=[]):
angles = read_file(filename)
time = angles['time']
be = angles['be']
algo = angles['algo']
algo_change_indexes = shade_background_based_on_algorithm(time,
algo,
plot=False)
# only take values from when algorithm is switched to increasing
time = time[algo_change_indexes[1]:]
