def takedataall(observer): a = 0 while True: if a < 10 : # create a check so we know the file is there and has the right name mce_file_name = "/data/cryo/current_data/temp.00%i" %(a) mce_file = Path("/data/cryo/current_data/temp.00%i" %(a+1)) #wait to read new file until old file is complete if mce_file.exists(): a = a + 1 f = mce_data.SmallMCEFile(mce_file_name) readdata(f, mce_file_name) y = readgraphall(y, f, mce_file_name) else: continue if a >= 10 and a < 100 : mce_file_name = "/data/cryo/current_data/temp.0%i"%(a) mce_file = Path("/data/cryo/current_data/temp.0%i"%(a+1)) if mce_file.exists(): a = a + 1 f = mce_data.SmallMCEFile(mce_file_name) readdata(f, mce_file_name) y = readgraphall(y, f, mce_file_name) else: continue if a >= 100 : mce_file_name = "/data/cryo/current_data/temp.%i"%(a) mce_file = Path("/data/cryo/current_data/temp.%i"%(a+1)) if mce_file.exists(): a = a + 1 f = mce_data.SmallMCEFile(mce_file_name) readdata(f, mce_file_name) y = readgraphall(y, f, mce_file_name) else: continue
def readdata(self, files1, files2, mce_file1, mce_file2): ''' This only works under the assumption that both mces are spitting out the same number of files at any given time''' # for i in range(1): f1 = mce_data.SmallMCEFile(files1) self.h1 = f1.Read(row_col=True, unfilter='DC').data f2 = mce_data.SmallMCEFile(files2) self.h2 = f2.Read(row_col=True, unfilter='DC').data # -------CHECK FOR FRAME SIZE CHANGE-------------------------------- # if frame size is wrong, just append zeros instead of partial array to prevent netcdf error # also gives a frame size error flag if self.p == 0: self.h1_shape = self.h1.shape self.h2_shape = self.h2.shape ut.flags.append(0) print(ut.flags) else: if (self.h1.shape != self.h1_shape) and (self.h2.shape != self.h2_shape): print( colored('WARNING! Both MCE Frame Size Has Changed', 'red')) ut.flags[3] = 11 self.h1 = np.zeros( (self.h1_shape[0], self.h1_shape[1], self.h1_shape[2])) self.h2 = np.zeros( (self.h2_shape[0], self.h2_shape[1], self.h2_shape[2])) elif self.h1.shape != self.h1_shape: print(colored('WARNING! MCE0 Frame Size Has Changed', 'red')) ut.flags[3] = 11 self.h1 = np.zeros( (self.h1_shape[0], self.h1_shape[1], self.h1_shape[2])) elif self.h2.shape != self.h2_shape: print(colored('WARNING! MCE1 Frame Size Has Changed', 'red')) ut.flags[3] = 11 self.h2 = np.zeros( (self.h2_shape[0], self.h2_shape[1], self.h2_shape[2])) else: ut.flags[3] = 0 print(ut.flags) self.head1 = self.read_header(f1) self.head2 = self.read_header(f2) self.rc = 'a' self.append_data() self.p += 1 # print(colored("MCE Append",'red')) return
def netcdfdata(n_files): tempfiledir = ('./netcdffiles') if os.path.exists(tempfiledir): print('Hello!') else: print('Making NETCDF File Directory') netcdf_dir = ['mkdir netcdffiles'] subprocess.Popen(netcdf_dir, shell=True).wait() #print('HELLO!') #n_files = len(os.listdir("/data/cryo/current_data")) a = 0 st.a = a mce = 1 n = 0 while True: mce_file = os.path.exists( "/data/cryo/current_data/temp.%0.3i" % (a + 1)) #wait to read new file until old file is complete if mce_file: print(a) mce_file_name = "/data/cryo/current_data/temp.%0.3i" % (a) a = a + 1 st.a = a f = mce_data.SmallMCEFile(mce_file_name) header = read_header(f) mce, n = readdata(f, mce_file_name, mce, header, n, a) mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i" % (a + 1)) else: pass
def readdata(): n = 0 f = mce_data.SmallMCEFile( '/data/TIME_Software/cryo/20171009/data_test4_2.858Arms_3hz') h = f.Read(row_col=True, unfilter='DC').data d = [[[] for i in range(32)] for j in range(31)] stream_ids = tls.get_credentials_file()['stream_ids'] stream_id = stream_ids[0] heatmap = go.Heatmap(stream=dict(token=stream_id)) plot_data = go.Data([heatmap]) py.plot(plot_data) s = py.Stream(stream_id) s.open() time.sleep(5) while n < 5000: t = time.time() for i in range(32): for j in range(31): d[j][i] = (np.std(h[j, i, n:n + 1000])) n = n + 1000 print(d[27][4]) z = ([[d[23][0], d[23][1], d[23][2], d[23][3], d[23][4]], [d[24][0], d[24][1], d[24][2], d[24][3], d[24][4]], [d[25][0], d[25][1], d[25][2], d[25][3], d[25][4]], [d[26][0], d[26][1], d[26][2], d[26][3], d[26][4]], [d[27][0], d[27][1], d[27][2], d[27][3], d[27][4]]]) s.write(dict(z=z, type='heatmap')) print("time elapsed for loop: %s" % (time.time() - t)) t = 0 time.sleep(2) s.close()
def takedata(a, ch, n_files, frameperfile, mce, row): a -= 1 st.a = a y = [] allgraphdata = [] while True: mce_file_name = "/home/pilot1/ssh_stuff/mce1/temp.%0.3i" % (a) mce_file = os.path.exists( "/home/pilot1/ssh_stuff/mce1/temp.%0.3i" % (a + 1)) #wait to read new file until old file is complete if mce_file: #print(len(os.listdir("/home/pilot1/ssh_stuff/mce1")) - 2 - n_files) while mce_file: mce_file_name = "/home/pilot1/ssh_stuff/mce1/temp.%0.3i" % (a) a = a + 1 st.a = a f = mce_data.SmallMCEFile(mce_file_name) #header = read_header(f) header = 22 z, mce = readdata(f, mce_file_name, frameperfile, mce, header) graphdata = readgraph(y, f, mce_file_name, a, ch, row) allgraphdata.append(graphdata) mce_file = os.path.exists( "/home/pilot1/ssh_stuff/mce1/temp.%0.3i" % (a + 1)) break else: pass return z, allgraphdata, mce
def __init__(self, datafile, minlen_nbranch=50, min_slope=1e2, max_slope=1e4, darksquid_std_thres=300, verbose=True): mce = mce_data.SmallMCEFile(datafile) self.data = mce.Read(row_col=True).data #iterate through columns and flip as appropriate for col in range(0, self.data.shape[1]): self.data[:, col] = config.fb_normalize[col] * self.data[:, col] self.bias_data = read_ascii(datafile + '.bias', comment_chars=['<', '#'])[0] self.bias_step = np.abs(self.bias_data[0] - self.bias_data[1]) self.nrows = self.data.shape[0] self.ncols = self.data.shape[1] #initialize self.iv_statuses to nrows*ncols list of None self.iv_statuses = [] for i in range(self.nrows): col_vals = [] for j in range(self.ncols): col_vals.append(None) self.iv_statuses.append(col_vals) self.minlen_nbranch = minlen_nbranch self.min_slope = min_slope self.max_slope = max_slope self.darksquid_std_thres = darksquid_std_thres self.verbose = verbose
def netcdfdata(): print('HELLO!') #n_files = len(os.listdir("/data/cryo/current_data")) a = 0 mce = 1 n = 0 n_files = 8 filestarttime = 0 while True: #mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a) #mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i" %(a+1)) #wait to read new file until old file is complete #if mce_file: #print('NETCDF IS WORKING') #print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files) subprocess.call([ 'ssh -T [email protected] python /home/pilot2/ssh_stuff/mce1_sftp.py %s %s' % (a, n_files) ], shell=True) mce_file = os.path.exists("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1)) print("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1)) if mce_file: print(len(os.listdir("mce1")) - 2) for i in range(len(os.listdir("mce1")) - 2): print('netcdf: %s' % (a)) mce_file_name = "/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % ( a) a = a + 1 f = mce_data.SmallMCEFile(mce_file_name) header = read_header(f) mce, n, filestarttime = readdata(f, mce_file_name, mce, header, n, a, filestarttime) mce_file = os.path.exists( "/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1))
def netcdfdata(n_files): print('HELLO!') #n_files = len(os.listdir("/data/cryo/current_data")) a = 0 st.a = a mce = 1 n = 0 while True: mce_file_name = "/data/cryo/current_data/temp.%0.3i" % (a) mce_file = os.path.exists( "/data/cryo/current_data/temp.%0.3i" % (a + 1)) #wait to read new file until old file is complete if mce_file: #print('NETCDF IS WORKING') #print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files) for i in range( len(os.listdir("/data/cryo/current_data")) - 2 - int(n_files)): mce_file_name = "/data/cryo/current_data/temp.%0.3i" % (a) a = a + 1 st.a = a f = mce_data.SmallMCEFile(mce_file_name) header = read_header(f) mce, n = readdata(f, mce_file_name, mce, header, n, a) else: pass
def gui_data(): dir1 = '/Users/vlb9398/Desktop/test_mce_files/' file1 = (dir1 + 'test_data.%0.3i' % (a)) f1 = mce_data.SmallMCEFile(file1) h1 = file1.Read(row_col=True, unfilter='DC').data print(colored('File Read: %s' % (file1.replace(dir1, '')), 'yellow')) ''' add stuff in for storing to netcdf ''' return h1
def __init__(self, mce=None, head_binary=None): mce_data.SmallMCEFile(self, runfile=False, basic_info=False) self.mce = mce self.runfile = 'fake' self.filename = None self.header = None self.n_ro = 1 if head_binary != None: self._ReadHeader(head_binary=self.head_binary)
def netcdfdata(rc, ch, row): #print('Read files has started') sys.stdout.flush() a = 0 mce = 0 n = 0 filestarttime = 0 dir1 = '/home/pilot1/Desktop/time-data/mce1/' #dir2 = '/home/pilot1/Desktop/time-data/mce2/' while True: mce_file1 = os.path.exists(dir1 + 'temp.%0.3i' % (a + 1)) #mce_file2 = os.path.exists(dir2 + 'temp.%0.3i' %(a+1)) #if (mce_file1 and mce_file2): if mce_file1: files1 = [ dir1 + x for x in os.listdir(dir1) if (x.startswith("temp") and not x.endswith('.run')) ] #print(colored('First if statement passed')) sys.stdout.flush() #files2 = [dir2 + x for x in os.listdir(dir2) if (x.startswith("temp") and not x.endswith('.run'))] # if (len(files1) and len(files2)) != 0: if len(files1) != 0: mce_file1 = min(files1, key=os.path.getctime) #print(colored('Second if statement passed')) sys.stdout.flush() #mce_file2 = min(files2, key = os.path.getctime) f1 = mce_data.SmallMCEFile(mce_file1) #f2 = mce_data.SmallMCEFile(mce_file2) head1 = read_header(f1) #head2 = read_header(f2) #++++++++++++++++++++++++++++++++ TELESCOPE DATA +++++++++++++++++++++++++++++++++++++++++++++++++++ # pa,slew_flag,alt,az,ra,dec = np.loadtxt('tempfiles/tempteledata.txt',delimiter = ',',unpack=True) # t = open('tempfiles/tempteledata.txt','w') # t.close() # tel_size = len(pa) # tt = np.column_stack((pa,slew_flag,alt,az,ra,dec)) #print(tt.shape) #print(tt) #+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ mce, n, filestarttime, d1, graphdata1 = readdata( f1, mce_file1, mce, head1, n, a, filestarttime, rc, ch, row) #mce, n, filestarttime, tel_size, tt = readdata(f, mce_file, mce, header, n, a, filestarttime, rc, tel_size, tt) #mce,n,filestarttime,d1,d2,graphdata1,graphdata2 = readdata(f1,f2,mce_file1,mce_file2,mce,head1,head2,n,a,filestarttime,rc,ch,row) # print(colored('File Read: %s , %s' %(mce_file1.replace(dir,''),mce_file2.replace(dir,''))),'yellow') print( colored('File Read: %s' % (mce_file1.replace(dir1, '')), 'yellow')) a = a + 1 sys.stdout.flush() break else: pass return d1, graphdata1, mce
def takedata(observer): y = [] while True: if st.a < 10: # create a check so we know the file is there and has the right name mce_file_name = "/data/cryo/current_data/temp.00%i" % (st.a) mce_file = Path( "/data/cryo/current_data/temp.00%i" % (st.a + 1)) #wait to read new file until old file is complete if mce_file.exists(): st.a = st.a + 1 f = mce_data.SmallMCEFile(mce_file_name) read_header(f) readdata(f, mce_file_name) readgraph(y, f, mce_file_name, st.a) else: continue if st.a >= 10 and st.a < 100: mce_file_name = "/data/cryo/current_data/temp.0%i" % (st.a) mce_file = Path("/data/cryo/current_data/temp.0%i" % (st.a + 1)) if mce_file.exists(): st.a = st.a + 1 f = mce_data.SmallMCEFile(mce_file_name) read_header(f) readdata(f, mce_file_name) readgraph(y, f, mce_file_name, st.a) else: continue if st.a >= 100: mce_file_name = "/data/cryo/current_data/temp.%i" % (st.a) mce_file = Path("/data/cryo/current_data/temp.%i" % (st.a + 1)) if mce_file.exists(): st.a = st.a + 1 f = mce_data.SmallMCEFile(mce_file_name) read_header(f) readdata(f, mce_file_name) readgraph(y, f, mce_file_name, st.a) else: continue time.sleep(1.0)
def netcdfdata(): print('HELLO!') #n_files = len(os.listdir("/data/cryo/current_data")) a = 0 mcea = 0 mce = 1 n = 0 n_files = 8 filestarttime = 0 mcea = subprocess.Popen([ 'ssh -T [email protected] python /home/pilot2/ssh_stuff/mce1_sftp.py %s %s' % (a, n_files) ], shell=True) while True: #mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a) #mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i" %(a+1)) #wait to read new file until old file is complete #if mce_file: #print('NETCDF IS WORKING') #print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files) mce_file = os.path.exists("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1)) #print("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" %(a+1)) if mce_file: #print(len(os.listdir("mce1")) - 2) for i in range(len(os.listdir("mce1")) - 3): print('netcdf: %s' % (a)) mce_file_name = "/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % ( a) #print(mce_file_name) a = a + 1 f = mce_data.SmallMCEFile(mce_file_name) header = read_header(f) mce, n, filestarttime = readdata(f, mce_file_name, mce, header, n, a, filestarttime) mce_file = os.path.exists( "/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1)) #else: # pass if os.path.exists('tempfiles/stop.txt'): subprocess.call([ "ssh -T [email protected] /usr/mce/bin/mce_cmd '-x stop rcs ret_dat'" ], shell=True) mcea.terminate() subprocess.call([ 'ssh -T [email protected] "rm /data/cryo/current_data/temp.*"' ], shell=True) subprocess.call(['rm mce1/temp.*'], shell=True) sys.exit()
def get_mce_data(datafile, **args): '''Gets mce data from a mce data file using mce_data.SmallMCEFile.Read @return: the mce data @rtype: numpy array, 3 dimensional @param datafile: the file to be passed into mce_data.SmallMCEFile @type datafile: str @param args: arguments to be passed into mce_data.SmallMCEFile.Read ''' if os.path.isdir(datafile): data = mce_dirfile.getdata(datafile) data = fix_dirfile_signed_field(data) elif os.path.isfile(datafile): with open(datafile) as f: mce = mce_data.SmallMCEFile(datafile) data = mce.Read(**args).data return data
def takedataall(a, ch, rc, n_files, frameperfile, mce, row): a -= 1 st.a = a y = [] allgraphdata = [] while True: mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a) mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i"%(a+1)) if mce_file: print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files) for i in range(len(os.listdir("/data/cryo/current_data")) - 2 - n_files): mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a) a = a + 1 st.a = a f = mce_data.SmallMCEFile(mce_file_name) header = read_header(f) z, mce = readdataall(f, mce_file_name, frameperfile, mce, header) graphdata = readgraphall(y, f, mce_file_name, a, ch, rc, row) allgraphdata.append(graphdata) break else: pass return z, allgraphdata, mce
print(i) run = [ "mce_run temp %s %s && cat /data/cryo/current_data/temp >> /data/cryo/current_data/%s" % (numframe, rc, filename) ] b = subprocess.Popen(run, shell=True) b.communicate() if i == 0: copy2('/data/cryo/current_data/temp.run', '/data/cryo/current_data/%s' % (filename) + '.run') open("/data/cryo/current_data/temp", "w").close() else: open("/data/cryo/current_data/temp", "w").close() open("/data/cryo/current_data/temp.run", "w").close() i = i + 1 # create mce data object f = mce_data.SmallMCEFile('/data/cryo/current_data/%s' % (filename)) d = f.Read(row_col=True, unfilter='DC').data x = datetime.datetime.now().strftime('%m-%d-%Y %H:%M:%S.%f') y = (d[1, 1]) s.write(dict(x=x, y=y)) time.sleep(1) #plots one point every second except KeyboardInterrupt: s.close() print("Data acquisition stopped after %s runs" % (i)) exit()
def takedataall(a, ch, rc, n_files, frameperfile, mce): #print('Hello!') #st.init() #nc.new_file(st.n) a -= 1 st.a = a y = [] allgraphdata = [] while True: if a < 10: # create a check so we know the file is there and has the right name mce_file_name = "/data/cryo/current_data/temp.00%i" % (a) if a == 9: mce_file = os.path.exists("/data/cryo/current_data/temp.0%i" % (a + 1)) else: mce_file = os.path.exists( "/data/cryo/current_data/temp.00%i" % (a + 1)) #wait to read new file until old file is complete if mce_file: print(len(os.listdir('/data/cryo/current_data')) - 2 - n_files) for i in range( len(os.listdir('/data/cryo/current_data')) - 2 - n_files): if a < 10: mce_file_name = "/data/cryo/current_data/temp.00%i" % ( a) else: mce_file_name = "/data/cryo/current_data/temp.0%i" % ( a) #print(datafile) a = a + 1 st.a = a f = mce_data.SmallMCEFile(mce_file_name) read_header(f) z, mce = readdataall(f, mce_file_name, frameperfile, mce) graphdata = readgraphall(y, f, mce_file_name, a, ch, rc) allgraphdata.append(graphdata) break else: #continue pass elif a >= 10 and a < 100: mce_file_name = "/data/cryo/current_data/temp.0%i" % (a) if a == 99: mce_file = os.path.exists("/data/cryo/current_data/temp.%i" % (a + 1)) else: mce_file = os.path.exists("/data/cryo/current_data/temp.0%i" % (a + 1)) if mce_file: print(len(os.listdir('/data/cryo/current_data')) - 2 - n_files) for i in range( len(os.listdir('/data/cryo/current_data')) - 2 - n_files): if a < 100: mce_file_name = "/data/cryo/current_data/temp.0%i" % ( a) else: mce_file_name = "/data/cryo/current_data/temp.%i" % (a) #print(datafile) a = a + 1 st.a = a f = mce_data.SmallMCEFile(mce_file_name) read_header(f) z, mce = readdataall(f, mce_file_name, frameperfile, mce) graphdata = readgraphall(y, f, mce_file_name, a, ch, rc) allgraphdata.append(graphdata) break else: #continue pass elif a >= 100: mce_file_name = "/data/cryo/current_data/temp.%i" % (a) mce_file = os.path.exists("/data/cryo/current_data/temp.%i" % (a + 1)) if mce_file: print(len(os.listdir('/data/cryo/current_data')) - 2 - n_files) for i in range( len(os.listdir('/data/cryo/current_data')) - 2 - n_files): mce_file_name = "/data/cryo/current_data/temp.%i" % (a) #print(datafile) a = a + 1 st.a = a f = mce_data.SmallMCEFile(mce_file_name) read_header(f) z, mce = readdataall(f, mce_file_name, frameperfile, mce) graphdata = readgraphall(y, f, mce_file_name, a, ch, rc) allgraphdata.append(graphdata) break else: #continue pass #nc.new_file(st.n, frameperfile) return z, allgraphdata, mce
df = demod_filter(25, 4) col_best = 2 row_best = 14 num_points = 131072 nrows = 22 ncols = 4 c0 = [0, 1, 10, 11, 12, 15, 19, 21] c1 = [1, 2, 5, 6, 7, 14, 15, 17, 18] c2 = [7, 8, 10, 11, 12, 14] c3 = [1, 2, 3, 7, 8, 9, 10, 11, 12, 13] row_arrays = [c0, c1, c2, c3] if 1 == 0: data = mce_data.SmallMCEFile( '/data/cryo/20151204/det_on_2000_triangle_scan_call_0.5mms_25hz0.0') ctime_init = data.header['runfile_id'] demod_all = np.array([]) for i in np.arange(273): int_str = str(i / 1000.) f = '/data/cryo/20151204/det_on_2000_triangle_scan_call_0.5mms_25hz' + int_str print f data = mce_data.SmallMCEFile(f) ctime_det = data.header['runfile_id'] data = fh.get_mce_data(f, row_col=True) ctime_diff = ctime_det - ctime_init tes = data[row_best, col_best, :] - np.mean(data[row_best, col_best, :]) d_filter, d_fft = df.demod(tes, samp_rate) demod_all = np.concatenate(
def read_header(f): for key, value in f.header.items(): if key == '_rc_present': for i in range(len(value)): if value[i] == True: value[i] = "1" elif value[i] == False: value[i] = "0" else: print("I don't know what I am...") value = ''.join(map(str, value)) value = str(value) st.keys.append(key) st.values.append(value) # keys,values = zip(*f.header.items()) st.keys = np.asarray(st.keys, dtype=object) st.values = np.asarray(st.values, dtype=object) st.head = np.array((st.keys, st.values)).T #------------------------------------------------------------------------------- if __name__ == "__main__": st.init() mce_file_name = path f = mce_data.SmallMCEFile(mce_file_name) h = f.Read(row_col=True, unfilter='DC').data read_header(f) st.h_size = h.shape[2] nct.new_file(st.n) readdata(f, mce_file_name, h)