def main():
options = parse_arguments()
yes_should_continue = query_yes_no(
"All the images in %s will be modified.\n"
"No backup will be created, ARE YOU SURE?" % options.input_path,
default="no")
if not yes_should_continue:
print("End of game. No file has been edited.")
return
filenames = [
os.path.join(options.input_path, x)
for x in os.listdir(options.input_path)
]
use_multithreads = True
#use_multithreads = False
if use_multithreads:
# multithreaded processing of the files
num_processes = cpu_count() + 1
pool = Pool(processes=num_processes)
pool.map(fix_image_borders, filenames)
else:
for filename in filenames:
fix_image_borders(filename)
print("\nEnd of game. Have a nice day")
return
def main():
options = parse_arguments()
yes_should_continue = query_yes_no(
"All the images in %s will be modified.\n"
"No backup will be created, ARE YOU SURE?" % options.input_path,
default="no")
if not yes_should_continue:
print("End of game. No file has been edited.")
return
filenames = [os.path.join(options.input_path, x)
for x in os.listdir(options.input_path)]
use_multithreads = True
#use_multithreads = False
if use_multithreads:
# multithreaded processing of the files
num_processes = cpu_count() + 1
pool = Pool(processes=num_processes)
pool.map(fix_image_borders, filenames)
else:
for filename in filenames:
fix_image_borders(filename)
print("\nEnd of game. Have a nice day")
return
def equalize_colors(input_path, method_name, on_condor=False):
if not on_condor:
yes_should_continue = query_yes_no(
"All the images in %s will be modified.\n"
"No backup will be created, ARE YOU SURE?" % input_path,
default="no")
if not yes_should_continue:
print("End of game. No file has been edited.")
return
equalize_image = methods[method_name]
filenames = os.listdir(input_path)
progress_bar = None
if not on_condor:
progress_bar = progressbar.ProgressBarWithMessage(len(filenames),
" Processing images")
files_counter = 0
for filename in filenames:
print (filename)
file_path = os.path.join(input_path, filename)
try:
image = Image.open(file_path)
except:
print("Failed to open image %s, skipping." % file_path)
continue
new_image = to_pil(equalize_image(from_pil(image)))
new_image.save(file_path)
files_counter += 1
if progress_bar:
progress_bar.update(files_counter)
else:
# on condor
f = open(file_path + ".done", "w")
f.writelines("test\n")
f.close()
# end of "for each image in the folder"
if progress_bar:
progress_bar.finish()
print("%i images inside %s have been equalized" % (files_counter,
input_path))
return
def equalize_colors(input_path, method_name, on_condor=False):
if not on_condor:
yes_should_continue = query_yes_no(
"All the images in %s will be modified.\n"
"No backup will be created, ARE YOU SURE?" % input_path,
default="no")
if not yes_should_continue:
print("End of game. No file has been edited.")
return
equalize_image = methods[method_name]
filenames = os.listdir(input_path)
progress_bar = None
if not on_condor:
progress_bar = progressbar.ProgressBarWithMessage(
len(filenames), " Processing images")
files_counter = 0
for filename in filenames:
print(filename)
file_path = os.path.join(input_path, filename)
try:
image = Image.open(file_path)
except:
print("Failed to open image %s, skipping." % file_path)
continue
new_image = to_pil(equalize_image(from_pil(image)))
new_image.save(file_path)
files_counter += 1
if progress_bar:
progress_bar.update(files_counter)
else:
# on condor
f = open(file_path + ".done", "w")
f.writelines("test\n")
f.close()
# end of "for each image in the folder"
if progress_bar:
progress_bar.finish()
print("%i images inside %s have been equalized" %
(files_counter, input_path))
return
def ask_for_urls():
repeat = True
urls = list()
while repeat is True:
urls.append(
input('Enter a Digikey part url: ')) # TODO: make a function that is more robust then a simple input
answer = query_yes_no('Do you want to enter another url? [yes/no]')
if answer is True:
continue
else:
repeat = False
return urls
def check_hive_database(database):
' returns valid database or raises error if not valid '
host, port = get_hive_host_port()
databases = get_hive_databases(host, port)
names = databases.database_name.values
# TODO: should this be case-insensitive?
if database in names:
return database
close_matches = difflib.get_close_matches(database, names)
if len(close_matches) > 0:
message = 'Did you mean {}?'.format(close_matches[0])
response = query_yes_no(message)
if response:
return close_matches[0]
raise ValueError('Invalid database name {}'.format(database))
def check_presto_catalog(catalog):
' returns valid catalog or raises error if not valid '
host, port = get_presto_host_port()
catalogs = get_presto_catalogs(host, port)
names = catalogs.Catalog.values
# TODO: should this be case-insensitive?
if catalog in names:
return catalog
close_matches = difflib.get_close_matches(catalog, names)
if len(close_matches) > 0:
message = 'Did you mean {}?'.format(close_matches[0])
response = query_yes_no(message)
if response:
return close_matches[0]
raise ValueError('Invalid catalog name {}'.format(catalog))
def reset_to_bootloader(args) -> int:
dbg_print("======== RESET TO BOOTLOADER ========")
s = serial.Serial(port=args.port,
baudrate=args.baud,
bytesize=8,
parity='N',
stopbits=1,
timeout=1,
xonxoff=0,
rtscts=0)
rl = SerialHelper.SerialHelper(s, 3.)
rl.clear()
if args.half_duplex:
BootloaderInitSeq = bootloader.get_init_seq('GHST', args.type)
dbg_print(" * Using half duplex (GHST)")
else:
BootloaderInitSeq = bootloader.get_init_seq('CRSF', args.type)
dbg_print(" * Using full duplex (CRSF)")
#this is the training sequ for the ROM bootloader, we send it here so it doesn't auto-neg to the wrong baudrate by the BootloaderInitSeq that we send to reset ELRS
rl.write(b'\x07\x07\x12\x20' + 32 * b'\x55')
time.sleep(0.2)
rl.write(BootloaderInitSeq)
s.flush()
rx_target = rl.read_line().strip()
flash_target = re.sub("_VIA_.*", "", args.target.upper())
ignore_incorrect_target = args.action == "uploadforce"
if rx_target == "":
dbg_print("Cannot detect RX target, blindly flashing!")
elif ignore_incorrect_target:
dbg_print(f"Force flashing {flash_target}, detected {rx_target}")
elif rx_target != flash_target:
if query_yes_no(
"\n\n\nWrong target selected! your RX is '%s', trying to flash '%s', continue? Y/N\n"
% (rx_target, flash_target)):
dbg_print("Ok, flashing anyway!")
else:
dbg_print(
"Wrong target selected your RX is '%s', trying to flash '%s'" %
(rx_target, flash_target))
return ElrsUploadResult.ErrorMismatch
elif flash_target != "":
dbg_print("Verified RX target '%s'" % (flash_target))
time.sleep(.5)
s.close()
return ElrsUploadResult.Success
def validate_reported_rx_type(rx_target, target):
from query_yes_no import query_yes_no
if rx_target:
print_log(" Receiver reported target: '%s'" % rx_target)
if rx_target.endswith("_RX"):
# remove suffix
rx_target = rx_target[:-3].upper()
if rx_target == "":
print_warning(" Cannot detect RX target, blindly flashing!")
elif target == "":
print_warning(" Unable to verify without target, blindly flashing!")
elif rx_target != target:
message = " Wrong target selected! your RX is '%s', trying to flash '%s'" % (
rx_target, target)
if query_yes_no(f"\n\n\n{message}, continue? Y/N\n"):
print_log(" Ok, flashing anyway!")
else:
raise WrongTargetSelected(message)
def search_operational_stations():
"""Find active HiSPARC shower and weather stations"""
if os.path.isfile('active_station_IDs.txt'):
read_list_from_file()
else:
print ''
print 'You do not have a list with operational stations in your current working directory.'
update = query_yes_no('Do you want to update/create the list with operational stations?')
print ''
if update == True:
active_shower, active_weather = get_active_stations()
with open('active_station_IDs.txt', 'w') as file:
file.write('active_shower = [')
file.write(', '.join(str(id) for id in active_shower))
file.write(']\nactive_weather = [')
file.write(', '.join(str(id) for id in active_weather))
file.write(']')
read_list_from_file()
def ask_for_excel_file():
yes_no = query_yes_no('Do you want to continue on an existing file? [yes/no]')
if yes_no is True:
excel_file_name = input('Enter the files name:')
path = os.getcwd()
excel_files_names = [f for f in os.listdir(path) if f.endswith('.xlsx')]
if excel_file_name in excel_files_names:
try:
excel_file = pd.read_excel(r'' + path + '/' + excel_file_name, sheet_name=None)
if excel_file is not None:
try:
parts_dict = reformat_excel_file(excel_file)
return parts_dict
except Exception as e:
raise ('Problem occurred while reformatting your xlsx file. \n'
'make sure the file was initially generated by this program.')
except Exception as e:
raise ('Error while reading your excel file' + e)
else:
os.error('No files of named' + excel_file_name + ' on that path' + path)
else:
excel_file = None
return excel_file
def reset_to_bootloader(args):
dbg_print("======== RESET TO BOOTLOADER ========")
s = serial.Serial(port=args.port,
baudrate=args.baud,
bytesize=8,
parity='N',
stopbits=1,
timeout=1,
xonxoff=0,
rtscts=0)
rl = SerialHelper.SerialHelper(s, 3.)
rl.clear()
if args.half_duplex:
BootloaderInitSeq = bootloader.get_init_seq('GHST', args.type)
dbg_print(" * Using half duplex (GHST)")
else:
BootloaderInitSeq = bootloader.get_init_seq('CRSF', args.type)
dbg_print(" * Using full duplex (CRSF)")
rl.write(BootloaderInitSeq)
s.flush()
rx_target = rl.read_line().strip()
flash_target = re.sub("_VIA_.*", "", args.target.upper())
if rx_target == "":
dbg_print("Cannot detect RX target, blindly flashing!")
elif rx_target != flash_target:
if query_yes_no(
"\n\n\nWrong target selected! your RX is '%s', trying to flash '%s', continue? Y/N\n"
% (rx_target, flash_target)):
dbg_print("Ok, flashing anyway!")
else:
raise WrongTargetSelected(
"Wrong target selected your RX is '%s', trying to flash '%s'" %
(rx_target, flash_target))
elif flash_target != "":
dbg_print("Verified RX target '%s'" % (flash_target))
time.sleep(.5)
s.close()
def find_MPV_pulseheights(pulseheights_list, plot_variable, time, number_of_plates):
print ''
show_plot = query_yes_no('Do you want to see a plot of every individual fit for every plate?')
MPV_list = []
timing = []
for p in range(len(pulseheights_list)):
#print 'Time interval %d of %d.' % (pulseheights_list.index(p) + 1, len(pulseheights_list))
MPV_MIPs = []
for i in range(number_of_plates):
print 'plate ', i + 1
p_plate_total = pulseheights_list[p][:, i] # select for this time interval the data of plate i
# this histogram is for finding pulseheight value corresponding to the MPV bin (the highest bin besides the first photon bin)
hist = plt.hist(p_plate_total, 150) # make a histogram from the pulseheights of plate i
binsize = hist[0] # array with the number of pulseheights in each bin
value = hist[1] # array with the pulseheight value corresponding to the left side of each bin.
del hist
plt.clf()
# sort the bins on the number of pulseheights in each bin
binsize_sort = sorted(binsize)
# Here we select the the 'photon bin': the bin with the most pulseheights
photon_bin = 0
for g in range(len(binsize)):
if binsize[g] == binsize_sort[-1]:
photon_bin = g
break
# Here the minimum after the photon bin is selected
skip = False
for h in range(len(binsize)):
try:
if h > photon_bin and binsize[h + 1] > binsize[h]: # the bin that is more to the right than the photon bin and if the histogram is rising again.
lowest_bin = h
break # if the lowest bin is found stop searching
except IndexError:
skip = True
"""
plt.hist(p_plate_total,150)
plt.show()
print 'h = ',h
print 'len(binsize) = ', len(binsize)
print 'photon_bin = ', photon_bin
print 'p = ',p
print 'time[p] = ', time[p]
"""
if skip == False:
# throw away that part of the histogram left from the lowest bin
binsize = binsize[lowest_bin:] #array with number of pulseheights in each bin from the lowest bin to the right of the histogram
value = value[lowest_bin:] #array with the pulseheight value corresponding to the left side of each bin. from the lowest bin to the right of the histogram
binsize_sort = sorted(binsize) # sort this selection of the bins on the number of pulseheights in each bin
MPV_bin = binsize_sort[-1] # the largest bin of this selection is the best guess for the MPV bin.
# calculate pulseheight value of the centre of the MPV-bin.
for j in range(len(binsize)):
if binsize[j] == MPV_bin:
MPV_value = (value[j] - value[j - 1]) / 2 + value[j]
interval = [100, 80, 60, 40, 30] # we fit the data in the range (MPV_value - interval[i], MPV_value + interval[i])
fit_values_list = []
success = False
# fit for every interval
for k in interval:
# select the fit data that is within the interval
ph_red = [p for p in p_plate_total if (MPV_value - k < p < MPV_value + k)]
plo = plt.hist(ph_red, 40)
del ph_red
values = plo[0]
#print values
bins = plo[1]
del plo
bin_centers = (bins[:-1] + bins[1:]) / 2
# bins = [0,500,1000,1500] i.e. the x-value for the LEFT side of each bin
# bins[:-1] = [0 ,500 ,1000]
# bins[1:] = [500,1000,1500]
# Therefore (bins[:-1] + bins[1:])/2 = [(0+500)/2 , (500+1000)/2, (1000+1500)/2]
# = [ 250 , 750 , 1250 ] i.e. the x-value for the CENTER side of each bin
del bins
#print bin_centers
plt.clf()
try:
popt, pcov = curve_fit(func, bin_centers, values, [1, MPV_value, 1])
del pcov
a = popt[0]
b = popt[1]
c = popt[2]
deviation = abs(MPV_value - b)
#print 'THE VALUES deviation,a,b,c,k: ', deviation,a,b,c,k
if a < 0 and deviation < 60:
success = True
#print success
fit_values_list.append([deviation, a, b, c, k])
except RuntimeError:
print 'Error'
if success:
fit_values_list = sorted(fit_values_list)
MPV_MIPs.append(fit_values_list[0][2])
"""
print ''
afw = "Deviation = %g" % fit_values_list[0][0]
print afw
ap = "a = %g" % fit_values_list[0][1]
print ap
bp = "b = %g" % fit_values_list[0][2]
print bp
cp = "c = %g" % fit_values_list[0][3]
print cp
kp = "k = %g" % fit_values_list[0][4]
print kp
"""
if show_plot:
hist = plt.hist(p_plate_total, 150)
plt.yscale('log')
plt.axvline(fit_values_list[0][2])
print 'Close plot to continue...'
print ''
plt.show()
print 'MPV value = %.2f %s ' % (fit_values_list[0][2], units[plot_variable[0][0]])
print ''
success = False
else:
MPV_MIPs.append(MPV_value)
print ''
print 'WARNING: Bad fit'
print ''
print ''
print 'MPV value = %.2f %s ' % (MPV_value, units[plot_variable[0][0]])
print ''
if show_plot:
hist = plt.hist(p_plate_total, 150)
plt.yscale('log')
plt.axvline(MPV_value)
print ''
print 'Close plot to continue...'
plt.show()
elif skip == True:
MPV_MIPs.append(None)
print '------------------------------------------------------------'
MPV_list.append(MPV_MIPs)
timing.append(time[p])
return MPV_list, number_of_plates, timing
def get_corresponding_values_MPV(plot_variable2, times):
data_sorted = []
var_list_without_bad_data2 = []
for i in range(len(plot_variable2)):
# open datafile
with tables.openFile(plot_variable2[i][1], 'r') as data:
# get variable values from data file
var_string = "data.root.s%s.%s.col('%s')" % (plot_variable2[i][2], plot_variable2[i][3], plot_variable2[i][0])
var = eval(var_string)
# get timestamp values corresponding to the variable values from datafile
ts_string = "data.root.s%s.%s.col('timestamp')" % (plot_variable2[i][2], plot_variable2[i][3])
ts = eval(ts_string)
data_sorted.extend(sorted(zip(ts, var))) # one list with timestamps and variable values
if plot_variable2[0][0] in low_limit:
bad_data2 = []
for t2, v2 in data_sorted:
if v2 >= low_limit[plot_variable2[0][0]] and v2 <= high_limit[plot_variable2[0][0]]:
var_list_without_bad_data2.append((t2, v2))
else:
bad_data2.append((t2, v2))
if bad_data2:
print 'Removed %d rows of bad %s data.' % (len(data_sorted) - len(var_list_without_bad_data2), plot_variable2[0][0])
print_bad_data2 = query_yes_no('Do you want to print the BAD data?')
if print_bad_data2:
print bad_data2
if len(times) <= 1: # a correlation analysis for one datapoint is pointless
print 'not enough data'
else: # the plot module gives the center of every time interval. From this the begin- and end-timestamp for every time-interval is calculated
# times[0] = 1000 s, times[1] = 2000 s, times[2] = 3000 s.
begin_end_timestamp_list = [times[0] - ((times[1] - times[0]) / 2) + i * (times[1] - times[0]) for i in range(len(times) + 1)]
# = 1000 - (2000 - 1000)/2 + 0 * (2000 - 1000) = 1000 - 500 + 0 = 500 s
# = 1000 - 500 + 1 * 1000 = 1000 - 500 + 1000 = 1500 s
# = 1000 - 500 + 2 * 1000 = 1000 - 500 + 2000 = 2500 s
# for every timeinterval the variable values are brought together
data_list_to_calculate_mean = []
for i in range(len(times)):
mean_data_list_int = []
for ts, var in var_list_without_bad_data2:
if ts > begin_end_timestamp_list[i] and ts < begin_end_timestamp_list[i + 1]:
mean_data_list_int.append([ts, var])
data_list_to_calculate_mean.append(array(mean_data_list_int))
# for every time interval the mean of the variable is calculated
mean_variable_list = []
for j in data_list_to_calculate_mean:
if len(j.shape) == 2:
if len(j[0]):
var_list = j[:, 1]
mean = sum(var_list) / len(var_list)
mean_variable_list.append(mean)
else:
print 'No data for this time interval: list is empty.'
elif len(j.shape) == 1:
print 'No data for this time interval: list is empty.'
print 'weird'
mean_variable_list = array(mean_variable_list)
return mean_variable_list
def uart_upload(port, filename, baudrate, ghst=False, ignore_incorrect_target=False, key=None, target="") -> int:
SCRIPT_DEBUG = False
half_duplex = False
dbg_print("=================== FIRMWARE UPLOAD ===================\n")
dbg_print(" Bin file '%s'\n" % filename)
dbg_print(" Port %s @ %s\n" % (port, baudrate))
logging.basicConfig(level=logging.ERROR)
if ghst:
BootloaderInitSeq1 = bootloader.get_init_seq('GHST', key)
half_duplex = True
dbg_print(" Using GHST (half duplex)!\n")
else:
BootloaderInitSeq1 = bootloader.get_init_seq('CRSF', key)
dbg_print(" Using CRSF (full duplex)!\n")
BootloaderInitSeq2 = bytes([0x62,0x62,0x62,0x62,0x62,0x62])
if not os.path.exists(filename):
msg = "[FAILED] file '%s' does not exist\n" % filename
dbg_print(msg)
return ElrsUploadResult.ErrorGeneral
s = serial.Serial(port=port, baudrate=baudrate,
bytesize=8, parity='N', stopbits=1,
timeout=5, inter_byte_timeout=None, xonxoff=0, rtscts=0)
rl = SerialHelper.SerialHelper(s, 2., ["CCC"], half_duplex)
# Check if bootloader *and* passthrough is already active
gotBootloader = 'CCC' in rl.read_line()
if not gotBootloader:
s.close()
# Init Betaflight passthrough
try:
BFinitPassthrough.bf_passthrough_init(port, baudrate, half_duplex)
except BFinitPassthrough.PassthroughEnabled as info:
dbg_print(" Warning: '%s'\n" % info)
except BFinitPassthrough.PassthroughFailed as failed:
raise
# Prepare to upload
s = serial.Serial(port=port, baudrate=baudrate,
bytesize=8, parity='N', stopbits=1,
timeout=1, xonxoff=0, rtscts=0)
rl.set_serial(s)
rl.clear()
# Check again if we're in the bootloader now that passthrough is setup;
# Note: This is for button-method flashing
gotBootloader = 'CCC' in rl.read_line()
# Init bootloader
if not gotBootloader:
# legacy bootloader requires a 500ms delay
delay_seq2 = .5
rl.set_timeout(2.)
rl.set_delimiters(["\n", "CCC"])
currAttempt = 0
dbg_print("\nAttempting to reboot into bootloader...\n")
while gotBootloader == False:
currAttempt += 1
if 10 < currAttempt:
msg = "[FAILED] to get to BL in reasonable time\n"
dbg_print(msg)
return ElrsUploadResult.ErrorGeneral
if 5 < currAttempt:
# Enable debug logs after 5 retries
SCRIPT_DEBUG = True
dbg_print("[%1u] retry...\n" % currAttempt)
# clear RX buffer before continuing
rl.clear()
# request reboot
rl.write(BootloaderInitSeq1)
start = time.time()
while ((time.time() - start) < 2.):
line = rl.read_line().strip()
if not line:
# timeout
continue
if SCRIPT_DEBUG and line:
dbg_print(" **DBG : '%s'\n" % line)
if "BL_TYPE" in line:
bl_type = line[8:].strip()
dbg_print(" Bootloader type found : '%s'\n" % bl_type)
# Newer bootloaders do not require delay but keep
# a 100ms just in case
delay_seq2 = .1
continue
versionMatch = re.search('=== (v.*) ===', line, re.IGNORECASE)
if versionMatch:
bl_version = versionMatch.group(1)
dbg_print(" Bootloader version found : '%s'\n" % bl_version)
elif "hold down button" in line:
time.sleep(delay_seq2)
rl.write(BootloaderInitSeq2)
gotBootloader = True
break
elif "CCC" in line:
# Button method has been used so we're not catching the header;
# let's jump right to the sanity check if we're getting enough C's
gotBootloader = True
break
elif "_RX_" in line:
flash_target = re.sub("_VIA_.*", "", target.upper())
if line != flash_target and not ignore_incorrect_target:
if query_yes_no("\n\n\nWrong target selected! your RX is '%s', trying to flash '%s', continue? Y/N\n" % (line, flash_target)):
ignore_incorrect_target = True
continue
else:
dbg_print("Wrong target selected your RX is '%s', trying to flash '%s'" % (line, flash_target))
return ElrsUploadResult.ErrorMismatch
elif flash_target != "":
dbg_print("Verified RX target '%s'" % flash_target)
dbg_print(" Got into bootloader after: %u attempts\n" % currAttempt)
# sanity check! Make sure the bootloader is started
dbg_print("Wait sync...")
rl.set_delimiters(["CCC"])
if "CCC" not in rl.read_line(15.):
msg = "[FAILED] Unable to communicate with bootloader...\n"
dbg_print(msg)
return ElrsUploadResult.ErrorGeneral
dbg_print(" sync OK\n")
else:
dbg_print("\nWe were already in bootloader\n")
else:
dbg_print("\nWe were already in bootloader\n")
# change timeout to 5sec
s.timeout = 5.
s.write_timeout = .3
# open binary
stream = open(filename, 'rb')
filesize = os.stat(filename).st_size
filechunks = filesize / 128
dbg_print("\nuploading %d bytes...\n" % filesize)
def StatusCallback(total_packets, success_count, error_count):
#sys.stdout.flush()
if (total_packets % 10 == 0):
dbg = str(round((total_packets / filechunks) * 100)) + "%"
if (error_count > 0):
dbg += ", err: " + str(error_count)
dbg_print(dbg + "\n")
def getc(size, timeout=3):
return s.read(size) or None
def putc(data, timeout=3):
cnt = s.write(data)
if half_duplex:
s.flush()
# Clean RX buffer in case of half duplex
# All written data is read into RX buffer
s.read(cnt)
return cnt
s.reset_input_buffer()
modem = XMODEM(getc, putc, mode='xmodem')
#modem.log.setLevel(logging.DEBUG)
status = modem.send(stream, retry=10, callback=StatusCallback)
s.close()
stream.close()
if (status):
dbg_print("Success!!!!\n\n")
return ElrsUploadResult.Success
dbg_print("[FAILED] Upload failed!\n\n")
return ElrsUploadResult.ErrorGeneral
####Tomas Dardet 2014####
# local_textfile_wrangler requires query_yes_no in the same directory.
# You must modify the path for where you wish to save your file
import glob
import os
import sys
import query_yes_no
import tempfile
exit = query_yes_no.query_yes_no('''This code will first find all the *.txt files in the directory you
provide. It will then write all of the information of those text files into one file. Do you want to proceed?''')
if not exit:
print("Goodbye")
sys.exit()
directory = input("Yay for continuing! Now enter your desired directory's path: ")
os.chdir(directory)
fileNames = []
for file in glob.glob("*.txt"):
fileNames.append(file)
fileNames.sort()
outName = input('What shall we name your file? Please write it now and press Enter when you are done: ')
ext = '.txt' # Change this if you want a different sort of file copied.
with open('C:/Users/TomasD/Documents/Documents/Python Scripts/outfiles/' + outName + ext,'w') as outfile:
for fname in fileNames:
with open(fname) as infile:
def least_squares_fit(filename, variable1, variable2):
with tables.openFile(filename, 'r') as data:
# fetch values variable 1 and 2
variable_1 = data.root.correlation.table.col('variable1')
variable_2 = data.root.correlation.table.col('variable2')
y_axis = query_yes_no("Do you want to plot %s on the y-axis?" % variable1[0][0])
if len(variable_1.shape) != 1:
print 'There are %d plates with an individual %s value.' % (variable_1.shape[1], variable1[0][0])
plate_number1 = int(question.digit_plate("Enter the plate number that you want to you use in your correlation analysis ( e.g. '1' ): ", variable_1.shape[1]))
variable_1 = variable_1[:, plate_number1 - 1]
if len(variable_2.shape) != 1:
print 'There are %d plates with an individual %s value.' % (variable_2.shape[1], variable2[0][0])
plate_number2 = int(question.digit_plate("Enter the plate number that you want to you use in your correlation analysis ( e.g. '1' ): ", variable_2.shape[1]))
variable_2 = variable_2[:, plate_number2 - 1]
if y_axis == True:
y = variable_1 # e.g. 'event_rates'
x = variable_2 # e.g. 'barometric pressure'
x, y = lose_nans(x, y)
elif y_axis == False:
x = variable_1 # e.g. 'event_rates'
y = variable_2 # e.g. 'barometric pressure'
else:
print 'weird'
del variable_1, variable_2
# Apply a linear least square fit:
# a line, ``y = mx + c``, through the data-points:
# We can rewrite the line equation as ``y = Ap``, where ``A = [[x 1]]``
# and ``p = [[m], [c]]``. Now use `lstsq` to solve for `p`:
A = np.vstack([x, np.ones(len(x))]).T
a, b = np.linalg.lstsq(A, y)[0]
del A
if y_axis == True:
print ''
print "The equation for the linear fit line is: ( y = a * x + b ) y = " + str(a) + " * x + " + str(b)
print ''
print "or '" + variable1[0][0] + "' = " + str(a) + " * '" + variable2[0][0] + "' + " + str(b)
elif y_axis == False:
print ''
print "The equation for the linear fit line is: ( y = a * x + b ) y = " + str(a) + " * x + " + str(b)
print ''
print "or '" + variable2[0][0] + "' = " + str(a) + " * '" + variable1[0][0] + "' + " + str(b)
# Calculate sample pearson correlation coefficient
cor_coef = np.corrcoef([x, y])[0, 1]
absolute_cor_coef = abs(cor_coef)
print ''
pearson_text = "The Pearson correlation coefficient between '%s' and '%s' is: %s" % (variable1[0][0], variable2[0][0], str(cor_coef))
print pearson_text
print ''
if absolute_cor_coef < 0.1:
correlation = 'NO'
elif 0.1 <= absolute_cor_coef <= 0.3:
correlation = 'a SMALL'
elif 0.3 <= absolute_cor_coef <= 0.5:
correlation = 'a MEDIUM'
elif 0.5 <= absolute_cor_coef <= 1:
correlation = 'a STRONG'
else:
correlation = ''
if cor_coef >= 0.1:
pos_neg = ' POSITIVE'
elif cor_coef <= -0.1:
pos_neg = ' NEGATIVE'
else:
pos_neg = ''
conclusion = "For this sample you have found %s%s correlation between '%s' and '%s'." % (correlation, pos_neg, variable1[0][0], variable2[0][0])
print conclusion
"""
# calculate chi squared
list_exp = array([a*i + b for i in x])
begin3 = datetime.now()
chi2, p = chisquare(y,list_exp)
end3 = datetime.now()
print end3 - begin3
combo = zip(y,list_exp)
begin = datetime.now()
ch2 = 0
for i in combo:
ch2 = ch2 + (i[0]-i[1]-0.5)**2/i[1]
print 'chi squared is ', ch2
end = datetime.now()
print end - begin
print ''
print 'chi squared:', chi2
print 'associated p-value: ', p
print ''
degrees_of_freedom = (len(x) - 1)
print 'chi squared divided by the number of measurements: ', chi2/degrees_of_freedom
chi2_prob = chisqprob(chi2,degrees_of_freedom) # probability value associated with the provided chi-square value and degrees of freedom
print 'probability value associated with the provided chi-square value and degrees of freedom:', chi2_prob
"""
# Plot the data along with the fitted line:
if(len(x) > 500000):
x, y = downsample(x, y)
plt.plot(x, y, 'o', label='Original data', markersize=1)
plt.plot(x, a * x + b, 'r', label='Fitted line')
if y_axis == True:
plt.ylabel(variable1[0][0] + ' (' + units[variable1[0][0]] + ')')
plt.xlabel(variable2[0][0] + ' (' + units[variable2[0][0]] + ')')
elif y_axis == False:
plt.ylabel(variable2[0][0] + ' (' + units[variable2[0][0]] + ')')
plt.xlabel(variable1[0][0] + ' (' + units[variable1[0][0]] + ')')
tit = "Fit line: ( y = ax + b ) y = " + str(a) + " * x + " + str(b)
plt.legend()
plt.title(tit)
start_date_interval, stop_date_interval = get_date_interval_from_file_names(variable1, variable2)
inter_filename = filename.replace('.h5', '')
fname = inter_filename + ' ' + start_date_interval + '_' + stop_date_interval
plt.savefig(fname + ".png")
plt.show()
fit_info = open(fname + '.txt', 'w')
fit_info.write(tit)
fit_info.write("%s\n" % (''))
fit_info.write(str(pearson_text))
fit_info.write("%s\n" % (''))
fit_info.write(str(conclusion))
fit_info.close
"""
# calculate mean y value
mean_y = sum(y) / len(y)
print 'mean_y = ', mean_y
relative_deviation_from_mean_y_list = []
#relative deviation of the cosmic ray intensity (deltaI/I) from the mean intensity.
for i in range(len(y)):
deviation_of_mean_y = y[i] - mean_y
relative_deviation_from_mean_y = deviation_of_mean_y/mean_y
relative_deviation_from_mean_y_list.append(relative_deviation_from_mean_y)
plt.plot(x,relative_deviation_from_mean_y_list,'o',markersize=1)
plt.ylabel('deltaMPV_p/<MPV_p>')
plt.xlabel('Outside temperature (degrees Celsius)')
tit = "Correlation between the Relative deviation of the MPV of the pulseheight (3h intervals) from the mean MPV value with the outside temperature."
plt.title(tit)
fname = 'Correlation between relative deviation of the MPV of the pulseheights (3h intervals) from the mean MPV value with T_out'
plt.savefig(fname +".png")
plt.show()
"""
"""
def main():
filterstring = raw_input('Filter for this file type (please type the file '
'extension, e.g. ".mov" or leave empty to work on all '
'files): ')
hashing = query_yes_no(
"Do you want to use the much more time intensive, but accurate checksum "
"approach? (WARNING: Not feasible for anything beyond a couple of GB!!)",
default="no")
inputleft = raw_input("Enter your first path (the one copied from): ")
if not os.path.isdir(inputleft):
return "Error, not a valid path"
countleft = 0
pathleft = []
nameleft = []
inputright = raw_input("Enter your second path (the one copied to): ")
if not os.path.isdir(inputright):
return "Error, not a valid path"
countright = 0
pathright = []
nameright = []
if not hashing:
print "\n\nPlease be patient. This might take a while. (About 1.5 minutes for " \
"1TB of data over ethernet.)"
else:
print "\n\nPlease be patient. This might take a while. (About 1.5 days (!!) for \
1TB of data over ethernet.)"
print "\nCaching first path's items…"
for root, dirs, files in os.walk(inputleft):
for name in files:
if filterstring in name:
nameleft.append(name)
pathleft.append(os.path.join(root, name))
countleft += 1
print "Caching second path's items…\n"
for root, dirs, files in os.walk(inputright):
for name in files:
if filterstring in name:
nameright.append(name)
pathright.append(os.path.join(root, name))
countright += 1
leftset = set(nameleft)
matches = leftset.intersection(nameright)
copyagain = None
copyrecord = ""
exists = 0
mismatch = 0
i = 1
for match in matches:
leftindex = nameleft.index(match)
rightindex = nameright.index(match)
if hashing:
print "Comparing file " + str(i) + " of " + str(len(matches))
i += 1
sizeleft = hash_for_file(pathleft[leftindex])
sizeright = hash_for_file(pathright[rightindex])
else:
sizeleft = os.stat(pathleft[leftindex]).st_size
sizeright = os.stat(pathright[rightindex]).st_size
if (sizeleft != sizeright):
mismatch = 1
if (copyagain == None):
copyagain = query_yes_no(
"\n\nThere have been mismatches in size. Do you want to mark the "
"specified files for later processing?")
if (copyagain == True):
copyrecord = inputright + "/copyagain.py"
if exists:
file = open(copyrecord, 'a')
file.write("copyFile(\"" + pathleft[leftindex] + "\", "
"\"" + pathright[rightindex] + "\")\n")
file.close()
else:
file = open(copyrecord, 'w')
file.write(
"#!/usr/bin/env python\n# encoding: utf-8\nfrom copyFile "
"import copyFile\n\ncopyFile(\"" +
pathleft[leftindex] + "\", \"" + pathright[rightindex] + "\")\n")
file.close()
exists = 1
print "\n\n" + match + " has a size mismatch\n"
if exists and os.path.exists(copyrecord):
os.chmod(copyrecord, 0755)
print "\n\nA file with all data to be copied has been written to your " \
"destination directory.\n\n" + copyrecord + "\n\nRefine it as you see " \
"fit or simply run it to get a new copy of the mismatched files.\n"
if not mismatch:
print "There has been no size mismatch!"
rate = 0
# Index of the contracted cell
no = '?'
nbrsnap = 200
Tsnap = round(T * 1000 / nbrsnap)
# Name of the INP file (output of GMSH) that we use to create the Voronoi net
mesh_basename = "Circle_triangle"
# Folder name to save the plots
folder_basename = "6 pull left and right K=10 Kafter10 rate = " + str(
rate) + " area threshold = " + str(
areathreshold) + "A2 = A0 k20 relax lo " + str(T) + "s " + str(
dt) + "dt gamma" + str(gamma) + " cell n" + str(no)
if os.path.exists(folder_basename) == True:
A = query_yes_no(
"The folder already exists. Do you want to overwrite it ?")
if A == 1:
shutil.rmtree(folder_basename)
os.mkdir(folder_basename)
else:
sys.exit("Program stopped")
else:
os.mkdir(folder_basename)
# Creation of CSV file for easy use of results with Excel
f_coords = open(folder_basename + "/coordsexcel.csv", "w")
f_forces = open(folder_basename + "/forcesexcel.csv", "w")
f_areas = open(folder_basename + "/areaexcel.csv", "w")
w_coords = csv.writer(f_coords, delimiter=";")
w_forces = csv.writer(f_forces, delimiter=";")
w_areas = csv.writer(f_areas, delimiter=";")
# Get rid of about: links
if re.search('http://about:', sanitized_url):
print "Skipping {}".format(sanitized_url)
continue
print u"Attempting to open {}".format(sanitized_url)
error_occurred = False
try:
req = urllib2.Request(sanitized_url, headers=hdr)
resp = urllib2.urlopen(req, timeout=30)
except urllib2.HTTPError as e:
print 'HTTP Error requesting: {}'.format(e)
error_occurred = True
except urllib2.URLError as e:
print 'URL Error requesting: {}'.format(e)
error_occurred = True
except socket.error as e:
print 'Socket Error requesting: {}'.format(e)
error_occurred = True
responsecode = resp.getcode()
if responsecode != 200:
print 'Unexpected HTTP response: {}'.format(responsecode)
error_occurred = True
# Logic to deal with broken links
if error_occurred:
shoulddelete = query_yes_no("Delete?", default="no")
if shoulddelete:
# Tell Diigo to delete the link
result = api.bookmark_delete(url=bookmark.url)
print result['message']
offset += 100
def find_MPV_integrals(pulseintegral_list, plot_variable):
print ''
show_plot = query_yes_no('Do you want to see a plot of every individual fit for every plate?')
MPV_list = []
for pi in pulseintegral_list:
print 'Time interval %d of %d.' % (pulseintegral_list.index(pi) + 1, len(pulseintegral_list))
print ''
MPV_MIPs = []
number_of_plates = get_number_of_plates(pi[0])
for i in range(number_of_plates):
print ''
print 'plate %d ' % i + 1
p_plate_total = array(pi[:, i])
hist = plt.hist(p_plate_total, 750)
binsize = hist[0]
value = hist[1]
del hist
plt.clf()
binsize_sort = sorted(binsize)
photon_bin = 0
for g in range(len(binsize)):
if binsize[g] == binsize_sort[-1]:
photon_bin = g
break
for h in range(len(binsize)):
if h > photon_bin and binsize[h + 1] > binsize[h]:
lowest_bin = h
lowest_p = value[h]
break
binsize = binsize[lowest_bin:]
value = value[lowest_bin:]
binsize_sort = sorted(binsize)
MPV_bin = binsize_sort[-1]
for j in range(len(binsize)):
if binsize[j] == MPV_bin:
MPV_value = (value[j] - value[j - 1]) / 2 + value[j]
#print 'MPV = ', MPV_value
extra = [1500, 1250, 1000, 750, 500]
list = []
success = False
for k in extra:
pi_red = [pi for pi in p_plate_total if (MPV_value - k < pi < MPV_value + k)]
plo = plt.hist(pi_red, 500)
del pi_red
values = plo[0]
#print values
bins = plo[1]
del plo
bin_centers = (bins[:-1] + bins[1:]) / 2
del bins
#print bin_centers
plt.clf()
try:
popt, pcov = curve_fit(func, bin_centers, values)
del pcov
a = popt[0]
b = popt[1]
c = popt[2]
deviation = abs(MPV_value - b)
#print 'THE VALUES deviation,a,b,c,k: ', deviation,a,b,c,k
if a < 0 and deviation < 500:
success = True
#print success
list.append([deviation, a, b, c, k])
except RuntimeError:
print 'Error'
if success:
list = sorted(list)
MPV_MIPs.append(list[0][2])
print ''
afw = "Deviation = %g" % list[0][0]
print afw
ap = "a = %g" % list[0][1]
print ap
bp = "b = %g" % list[0][2]
print bp
cp = "c = %g" % list[0][3]
print cp
kp = "k = %g" % list[0][4]
print kp
print ''
if show_plot:
hist = plt.hist(p_plate_total, 750)
plt.yscale('log')
plt.axvline(list[0][2])
print 'Close plot to continue...'
print ''
plt.show()
print 'MPV value = %.2f %s' % (list[0][2], units[plot_variable[0][0]])
print ''
else:
MPV_MIPs.append(MPV_value)
print ''
print 'WARNING: Bad fit'
print ''
print ''
print 'MPV value = %.2f %s' % (MPV_value, units[plot_variable[0][0]])
print ''
if show_plot:
hist = plt.hist(p_plate_total, 750)
plt.yscale('log')
plt.axvline(MPV_value)
print ''
print 'Close plot to continue...'
plt.show()
#print MPV_MIPs
MPV_list.append(MPV_MIPs)
return MPV_list, number_of_plates
prompt = {{ program_name }}
serverurl=unix:///tmp/{{ program_name }}-supervisor.sock
{% for pid in range(num_proc) %}
[program:{{ program_name }}-{{ pid }}]
command={{ command }}
stderr_logfile = /var/log/{{ program_name }}/stderr.log
stdout_logfile = /var/log/{{ program_name }}/stdout.log
{% endfor %}
""")
return Template(cleandoc(template))
if __name__ == '__main__':
inet_http_server = query_yes_no(
"Would you like an inet_http_server section?")
kwargs = {}
if inet_http_server:
kwargs["inet_port"] = raw_input(
"Enter desired inet_http_server port: ")
kwargs["inet_username"] = raw_input(
"Enter desired inet_http_server username: "******"inet_password"] = raw_input(
"Enter desired inet_http_server password: "******"program_name"] = raw_input("Enter name of program: ")
kwargs["command"] = raw_input("Enter command to run to start program: ")
kwargs["num_proc"] = int(
raw_input("Enter number of instances to run of program: "))
print("")
print(get_template(inet_http_server).render(**kwargs))
serverurl=unix:///tmp/{{ program_name }}-supervisor.sock
{% for pid in range(num_proc) %}
[program:{{ program_name }}-{{ pid }}]
command={{ command }}
stderr_logfile = /var/log/{{ program_name }}/stderr.log
stdout_logfile = /var/log/{{ program_name }}/stdout.log
{% endfor %}
""")
return Template(cleandoc(template))
if __name__ == '__main__':
inet_http_server = query_yes_no(
"Would you like an inet_http_server section?"
)
kwargs = {}
if inet_http_server:
kwargs["inet_port"] = raw_input(
"Enter desired inet_http_server port: "
)
kwargs["inet_username"] = raw_input(
"Enter desired inet_http_server username: "******"inet_password"] = raw_input(
"Enter desired inet_http_server password: "******"program_name"] = raw_input("Enter name of program: ")
kwargs["command"] = raw_input("Enter command to run to start program: ")
kwargs["num_proc"] = int(raw_input(
print 'HH HH iii SSSS PPPPPPP A RRRRRRR CCCCCC
print 'HH HH iii SSSSSSSS PPPPPPPPP AAA RRRRRRRRRR CCC CCC
print 'HH HH SSS SSS PPP PPP AAAAA RRR RRR CCC CCC
print 'HH HH iii SSSSS PPP PPP AAA AAA RRR RRR CCC
print 'HHHHHHHHH iii SSSSSS PPPPPPPPP AAA AAA RRRRRRRR CCC
print 'HH HH iii SSS PPP AAAAAAAAAAA RRR RRR CCC CCC
print 'HH HH iii SSS SSS PPP AAA AAA RRR RRR CCC CCC
print 'HH HH iii SSSSSSSS PPP AAA AAA RRR RRR CCCCCC
print ''
print ''
print 'Welcome to HiSPARC download and correlation software!'
print ''
print 'With this program you can download HiSPARC data, plot data and search for a correlation between HiSPARC shower and/or weather variables.'
print ''
download_question = query_yes_no('Do you want to download DATA? (if not, then you must already have a datafile on your pc)')
stations = []
plot_variable1 = []
plot_variable2 = []
if download_question == True:
show_operational_stations = query_yes_no('Do you want to see a list with operational HiSPARC and WEATHER stations?')
if show_operational_stations == True:
search_operational_stations()
user_hisparc_station_id_1 = question.digit('Enter the HiSPARC STATION ID from which you want to download data ( e.g. 501 ): ')
stations.append(user_hisparc_station_id_1)
user_start_date_data_interval = question.digit_and_date('Enter START date data interval ( e.g. 2011,7,21 ) : ')
user_stop_date_data_interval = question.digit_and_date('Enter STOP date data interval ( e.g. 2011,7,22 ) : ')
list_file_names_station_1 = down_data_in_parts(user_hisparc_station_id_1, user_start_date_data_interval, user_stop_date_data_interval)
def main():
filterstring = raw_input(
'Filter for this file type (please type the file '
'extension, e.g. ".mov" or leave empty to work on all '
'files): ')
hashing = query_yes_no(
"Do you want to use the much more time intensive, but accurate checksum "
"approach? (WARNING: Not feasible for anything beyond a couple of GB!!)",
default="no")
inputleft = raw_input("Enter your first path (the one copied from): ")
if not os.path.isdir(inputleft):
return "Error, not a valid path"
countleft = 0
pathleft = []
nameleft = []
inputright = raw_input("Enter your second path (the one copied to): ")
if not os.path.isdir(inputright):
return "Error, not a valid path"
countright = 0
pathright = []
nameright = []
if not hashing:
print "\n\nPlease be patient. This might take a while. (About 1.5 minutes for " \
"1TB of data over ethernet.)"
else:
print "\n\nPlease be patient. This might take a while. (About 1.5 days (!!) for \
1TB of data over ethernet.)"
print "\nCaching first path's items…"
for root, dirs, files in os.walk(inputleft):
for name in files:
if filterstring in name:
nameleft.append(name)
pathleft.append(os.path.join(root, name))
countleft += 1
print "Caching second path's items…\n"
for root, dirs, files in os.walk(inputright):
for name in files:
if filterstring in name:
nameright.append(name)
pathright.append(os.path.join(root, name))
countright += 1
leftset = set(nameleft)
matches = leftset.intersection(nameright)
copyagain = None
copyrecord = ""
exists = 0
mismatch = 0
i = 1
for match in matches:
leftindex = nameleft.index(match)
rightindex = nameright.index(match)
if hashing:
print "Comparing file " + str(i) + " of " + str(len(matches))
i += 1
sizeleft = hash_for_file(pathleft[leftindex])
sizeright = hash_for_file(pathright[rightindex])
else:
sizeleft = os.stat(pathleft[leftindex]).st_size
sizeright = os.stat(pathright[rightindex]).st_size
if (sizeleft != sizeright):
mismatch = 1
if (copyagain == None):
copyagain = query_yes_no(
"\n\nThere have been mismatches in size. Do you want to mark the "
"specified files for later processing?")
if (copyagain == True):
copyrecord = inputright + "/copyagain.py"
if exists:
file = open(copyrecord, 'a')
file.write("copyFile(\"" + pathleft[leftindex] + "\", "
"\"" + pathright[rightindex] + "\")\n")
file.close()
else:
file = open(copyrecord, 'w')
file.write(
"#!/usr/bin/env python\n# encoding: utf-8\nfrom copyFile "
"import copyFile\n\ncopyFile(\"" +
pathleft[leftindex] + "\", \"" +
pathright[rightindex] + "\")\n")
file.close()
exists = 1
print "\n\n" + match + " has a size mismatch\n"
if exists and os.path.exists(copyrecord):
os.chmod(copyrecord, 0755)
print "\n\nA file with all data to be copied has been written to your " \
"destination directory.\n\n" + copyrecord + "\n\nRefine it as you see " \
"fit or simply run it to get a new copy of the mismatched files.\n"
if not mismatch:
print "There has been no size mismatch!"
cf = config.ConfigFile()
baseline_files_path = (cf.configfile[cf.computername]['baseline_files_path'])
if sys.argv[1] == "create_hpk_dic_file":
file_name = "hpk.dic"
elif sys.argv[1] == "create_baseline_aff_file":
file_name = "baseline.aff"
else:
file_name = "oops.oops"
baseline_file_path = baseline_files_path + file_name
if os.path.isfile(baseline_file_path):
# file exists, check to see if it should be overridden
file_exists = True
overwrite_file = query_yes_no.query_yes_no("OVERWRITE the existing file?")
else:
file_exists = False
def create_hpk_dic_file():
if file_exists and not overwrite_file:
print("Existing file left as is")
return None
if file_exists and overwrite_file:
# delete existing file
os.remove(baseline_file_path)
# write the file
cf = config.ConfigFile()
baseline_files_path = (cf.configfile[cf.computername]['baseline_files_path'])
if sys.argv[1] == "create_hpk_dic_file":
file_name = "hpk.dic"
elif sys.argv[1] == "create_baseline_aff_file":
file_name = "baseline.aff"
else:
file_name = "oops.oops"
baseline_file_path = baseline_files_path + file_name
if os.path.isfile(baseline_file_path):
# file exists, check to see if it should be overridden
file_exists = True
overwrite_file = query_yes_no.query_yes_no("OVERWRITE the existing file?")
else:
file_exists = False
def create_hpk_dic_file():
if file_exists and not overwrite_file:
print("Existing file left as is")
return None
if file_exists and overwrite_file:
# delete existing file
os.remove(baseline_file_path)
# write the file
import json_processor_hunspell as jph
def plot_data(plot_variable):
MPV = False
# These variables are stored per plate, so we have 2 or 4 values
# for one event instead of just 1.
if plot_variable[0][0] in ('pulseheights', 'integrals',
'baseline', 'std_dev', "n_peaks"):
if plot_variable[0][0] in ('pulseheights', 'integrals'):
MPV = query_yes_no('Do you want to PLOT the MPV (Most probable value) of the %s?' % plot_variable[0][0])
if MPV == True:
print ''
interval = question.digit("Select the time interval (in seconds) over which the MPV values must be calculated ( e.g. for a day enter '86400' ): ")
seconds = int(interval)
variable_parts, time, number_of_plates = split_data_file_in_parts(plot_variable, seconds)
#e.g. variable_parts = [[p1, p2...pn], [p1, p2...pn], ....]
# time [t1, t2...tn] times are timestamps in the middle of every time interval
if plot_variable[0][0] == 'pulseheights':
MPV_list, number_of_plates, times = find_MPV_pulseheights(variable_parts, plot_variable, time, number_of_plates)
#times = array(time) necessary?
times_dates = [datetime.fromtimestamp(x) for x in times]
plot_variable1 = [('pulseheights', plot_variable[0][1], plot_variable[0][2], 'events')]
plot_variable2 = [('time', plot_variable[0][1], plot_variable[0][2], 'events')]
values1 = MPV_list
values2 = time
filename = create_correlation_table(plot_variable1, plot_variable2, values1, values2, seconds)
elif plot_variable[0][0] == 'integrals':
MPV_list, number_of_plates = find_MPV_integrals(variable_parts, plot_variable)
values = array(MPV_list)
for i in range(number_of_plates):
y = values[:, i]
plt.plot(times_dates, y)
plt.xlabel('time')
plt.ylabel('%s (%s)' % (plot_variable[0][0], units[plot_variable[0][0]]))
# create filename for correlation table from data filenames
intermediate1 = plot_variable[0][1].replace('data_s%s_' % str(plot_variable[0][2]), '')
intermediate2 = intermediate1.partition('_')
start_date = intermediate2[0]
intermediate1b = plot_variable[-1][1].replace('data_s%s_' % str(plot_variable[-1][2]), '')
intermediate2b = intermediate1b.partition('_')
intermediate3b = intermediate2b[2][1:]
end_date = intermediate3b.replace('.h5', '')
fname = ('MPV_%s_s%s_%s-%s_timeinterval_%d_seconds.png' %
(plot_variable[0][0], plot_variable[0][2], start_date, end_date, seconds))
plt.savefig(fname)
plt.show()
returntype = 'MPV'
else:
time_list = []
plate1 = []
plate2 = []
plate3 = []
plate4 = []
number_of_plates = 0
for i in range(len(plot_variable)):
with tables.openFile(plot_variable[i][1], 'r') as data:
tree_time = "data.root.s%s.%s.col('timestamp')" % (plot_variable[i][2], plot_variable[i][3])
time = eval(tree_time)
tree_variable = "data.root.s%s.%s.col('%s')" % (plot_variable[i][2], plot_variable[i][3], plot_variable[i][0])
variable = eval(tree_variable)
time_list.extend(time)
number_of_plates = get_number_of_plates(variable[0])
plate1.extend(list(variable[:, 0]))
plate2.extend(list(variable[:, 1]))
if number_of_plates == 4:
plate3.extend(list(variable[:, 2]))
plate4.extend(list(variable[:, 3]))
dat_sorted1 = array(sorted(zip(time_list, plate1)))
dat_sorted2 = array(sorted(zip(time_list, plate2)))
if number_of_plates == 4:
dat_sorted3 = array(sorted(zip(time_list, plate3)))
dat_sorted4 = array(sorted(zip(time_list, plate4)))
print ''
print 'Your file(s) contain(s) data from ' + str(datetime.fromtimestamp(dat_sorted1[0][0])) + ' until ' + str(datetime.fromtimestamp(dat_sorted1[-1][0]))
print ''
whole = query_yes_no('Do you want to PLOT this whole time interval')
if whole == True:
times = dat_sorted1[:, 0]
x = [datetime.fromtimestamp(i) for i in times]
y = dat_sorted1[:, 1]
plt.plot(x, y)
y = dat_sorted2[:, 1]
plt.plot(x, y)
if number_of_plates == 4:
y = dat_sorted3[:, 1]
plt.plot(x, y)
y = dat_sorted4[:, 1]
plt.plot(x, y)
values = array(zip(dat_sorted1[:, 1], dat_sorted2[:, 1],
dat_sorted3[:, 1], dat_sorted4[:, 1]))
else:
values = array(zip(dat_sorted1[:, 1], dat_sorted2[:, 1]))
plt.ylabel('%s (%s)' % (plot_variable[0][0], units[plot_variable[0][0]]))
plt.grid(True)
plt.show()
returntype = 'whole'
elif whole == False:
x_lim_low = 2
x_lim_up = 2
while True:
start = datetime.fromtimestamp(dat_sorted1[0][0])
print ''
print 'Start time: ' + str(start)
print 'Seconds in interval: ' + str(dat_sorted1[-1][0] - dat_sorted1[0][0])
print ''
print 'First you are going to enter the LOWER time limit.'
while True:
x_lim_low = question.digit('Enter the number of seconds after the start time shown above ( e.g. input "3600" means x_begin = timestamp + 3600 s ): ')
x_lim_low = int(x_lim_low)
if dat_sorted1[0][0] + x_lim_low <= dat_sorted1[-1][0]:
break
else:
print "Oops! Your lower time limit lies beyond your data set. Try again."
print ''
print 'Now you are going to enter the UPPER time limit.'
while True:
x_lim_up = question.digit('Enter the number of seconds after the start time shown above ( e.g. input "86400" means x_end = timestamp + 86400 s ): ')
x_lim_up = int(x_lim_up)
if x_lim_up > x_lim_low and dat_sorted1[0][0] + x_lim_up <= dat_sorted1[-1][0]:
break
elif x_lim_up <= x_lim_low:
print "Oops! The upper time limit less than or equal to the lower time limit. Try again."
elif dat_sorted1[0][0] + x_lim_up > dat_sorted1[-1][0]:
print "Oops! Your upper time limit lies beyond your data set. Try again."
plot_list1 = array([[t, v] for t, v in dat_sorted1 if dat_sorted1[0][0] + x_lim_low < t < dat_sorted1[0][0] + x_lim_up])
plot_list2 = array([[t, v] for t, v in dat_sorted2 if dat_sorted2[0][0] + x_lim_low < t < dat_sorted2[0][0] + x_lim_up])
if number_of_plates == 4:
plot_list3 = array([[t, v] for t, v in dat_sorted3 if dat_sorted3[0][0] + x_lim_low < t < dat_sorted3[0][0] + x_lim_up])
plot_list4 = array([[t, v] for t, v in dat_sorted4 if dat_sorted4[0][0] + x_lim_low < t < dat_sorted4[0][0] + x_lim_up])
values = array(zip(plot_list1[:, 1], plot_list2[:, 1],
plot_list3[:, 1], plot_list4[:, 1]))
else:
values = array(zip(plot_list1[:, 1], plot_list2[:, 1]))
times = plot_list1[:, 0]
x = [datetime.fromtimestamp(i) for i in times]
plt.plot(x, plot_list1[:, 1])
plt.plot(x, plot_list2[:, 1])
if number_of_plates == 4:
plt.plot(x, plot_list3[:, 1])
plt.plot(x, plot_list4[:, 1])
plt.ylabel('%s (%s)' % (plot_variable[0][0], units[plot_variable[0][0]]))
returntype = 'part'
plt.show()
again = query_yes_no('Do you want to plot again with different values for the UPPER and LOWER time?')
if again != True:
break
else:
variable_list = []
time_list = []
for i in range(len(plot_variable)):
with tables.openFile(plot_variable[i][1], 'r') as data:
tree_time = "data.root.s%s.%s.col('timestamp')" % (plot_variable[i][2], plot_variable[i][3])
time = eval(tree_time)
tree_variable = "data.root.s%s.%s.col('%s')" % (plot_variable[i][2], plot_variable[i][3], plot_variable[i][0])
variable = eval(tree_variable)
time_list.extend(time)
variable_list.extend(variable)
dat_sorted = array(sorted(zip(time_list, variable_list)))
print ''
print 'Your file(s) contain(s) data from %s until %s' % (str(datetime.fromtimestamp(dat_sorted[0][0])), str(datetime.fromtimestamp(dat_sorted[-1][0])))
print ''
whole = query_yes_no('Do you want to PLOT this whole time interval?')
if whole == True:
times = dat_sorted[:, 0]
x = [datetime.fromtimestamp(i) for i in times]
dat_sorted = array(dat_sorted)
values = dat_sorted[:, 1]
plt.plot(x, values)
plt.ylabel('%s (%s)' % (plot_variable[0][0], units[plot_variable[0][0]]))
plt.grid(True)
plt.show()
returntype = 'whole'
elif whole == False:
x_lim_low = 2
x_lim_up = 2
plot_list = []
while True:
start = datetime.fromtimestamp(dat_sorted[0][0])
print ''
print 'Start time = ' + str(start)
print 'Seconds in interval: ' + str(dat_sorted[-1][0] - dat_sorted[0][0])
print ''
print 'First you are going to enter the LOWER time limit.'
while True:
x_lim_low = question.digit('Enter the number of seconds after the start time shown above ( e.g. input "3600" means x_begin = timestamp + 3600 s ): ')
x_lim_low = int(x_lim_low)
if dat_sorted[0][0] + x_lim_low <= dat_sorted[-1][0]:
break
else:
print "Oops! Your lower time limit lies beyond your data set. Try again."
print ''
print 'Now you are going to enter the UPPER time limit.'
while True:
x_lim_up = question.digit('Enter the number of seconds after the start time shown above ( e.g. input "86400" means x_end = timestamp + 86400 s ): ')
x_lim_up = int(x_lim_up)
if x_lim_up > x_lim_low and dat_sorted[0][0] + x_lim_up <= dat_sorted[-1][0]:
break
elif x_lim_up <= x_lim_low:
print "Oops! The upper time limit less than or equal to the lower time limit. Try again."
elif dat_sorted[0][0] + x_lim_up > dat_sorted[-1][0]:
print "Oops! Your upper time limit lies beyond your data set. Try again."
plot_list = []
# e.g. dat_sorted = (timestamp, variable)
for t, v in dat_sorted:
if t > dat_sorted[0][0] + x_lim_low and t < dat_sorted[0][0] + x_lim_up:
plot_list.append([t, v])
plot_list = array(plot_list)
times = plot_list[:, 0]
values = plot_list[:, 1]
x = [datetime.fromtimestamp(i) for i in times]
plt.plot(x, values)
plt.ylabel('%s (%s)' % (plot_variable[0][0], units[plot_variable[0][0]]))
plt.grid(True)
plt.show()
returntype = 'part'
again = query_yes_no('Do you want to plot again with a different time limits?')
if again != True:
break
return values, times, returntype
# Get rid of about: links
if re.search('http://about:', sanitized_url):
print "Skipping {}".format(sanitized_url)
continue
print u"Attempting to open {}".format(sanitized_url)
error_occurred = False
try:
req = urllib2.Request(sanitized_url,headers=hdr)
resp = urllib2.urlopen(req, timeout=30)
except urllib2.HTTPError as e:
print 'HTTP Error requesting: {}'.format(e)
error_occurred = True
except urllib2.URLError as e:
print 'URL Error requesting: {}'.format(e)
error_occurred = True
except socket.error as e:
print 'Socket Error requesting: {}'.format(e)
error_occurred = True
responsecode = resp.getcode()
if responsecode != 200:
print 'Unexpected HTTP response: {}'.format(responsecode)
error_occurred = True
# Logic to deal with broken links
if error_occurred:
shoulddelete = query_yes_no("Delete?", default="no")
if shoulddelete:
# Tell Diigo to delete the link
result = api.bookmark_delete(url=bookmark.url)
print result['message']
offset += 100
