def takedataall(observer):
a = 0
while True:
if a < 10 : # create a check so we know the file is there and has the right name
mce_file_name = "/data/cryo/current_data/temp.00%i" %(a)
mce_file = Path("/data/cryo/current_data/temp.00%i" %(a+1)) #wait to read new file until old file is complete
if mce_file.exists():
a = a + 1
f = mce_data.SmallMCEFile(mce_file_name)
readdata(f, mce_file_name)
y = readgraphall(y, f, mce_file_name)
else:
continue
if a >= 10 and a < 100 :
mce_file_name = "/data/cryo/current_data/temp.0%i"%(a)
mce_file = Path("/data/cryo/current_data/temp.0%i"%(a+1))
if mce_file.exists():
a = a + 1
f = mce_data.SmallMCEFile(mce_file_name)
readdata(f, mce_file_name)
y = readgraphall(y, f, mce_file_name)
else:
continue
if a >= 100 :
mce_file_name = "/data/cryo/current_data/temp.%i"%(a)
mce_file = Path("/data/cryo/current_data/temp.%i"%(a+1))
if mce_file.exists():
a = a + 1
f = mce_data.SmallMCEFile(mce_file_name)
readdata(f, mce_file_name)
y = readgraphall(y, f, mce_file_name)
else:
continue
def readdata(self, files1, files2, mce_file1, mce_file2):
''' This only works under the assumption that both mces are
spitting out the same number of files at any given time'''
# for i in range(1):
f1 = mce_data.SmallMCEFile(files1)
self.h1 = f1.Read(row_col=True, unfilter='DC').data
f2 = mce_data.SmallMCEFile(files2)
self.h2 = f2.Read(row_col=True, unfilter='DC').data
# -------CHECK FOR FRAME SIZE CHANGE--------------------------------
# if frame size is wrong, just append zeros instead of partial array to prevent netcdf error
# also gives a frame size error flag
if self.p == 0:
self.h1_shape = self.h1.shape
self.h2_shape = self.h2.shape
ut.flags.append(0)
print(ut.flags)
else:
if (self.h1.shape != self.h1_shape) and (self.h2.shape !=
self.h2_shape):
print(
colored('WARNING! Both MCE Frame Size Has Changed', 'red'))
ut.flags[3] = 11
self.h1 = np.zeros(
(self.h1_shape[0], self.h1_shape[1], self.h1_shape[2]))
self.h2 = np.zeros(
(self.h2_shape[0], self.h2_shape[1], self.h2_shape[2]))
elif self.h1.shape != self.h1_shape:
print(colored('WARNING! MCE0 Frame Size Has Changed', 'red'))
ut.flags[3] = 11
self.h1 = np.zeros(
(self.h1_shape[0], self.h1_shape[1], self.h1_shape[2]))
elif self.h2.shape != self.h2_shape:
print(colored('WARNING! MCE1 Frame Size Has Changed', 'red'))
ut.flags[3] = 11
self.h2 = np.zeros(
(self.h2_shape[0], self.h2_shape[1], self.h2_shape[2]))
else:
ut.flags[3] = 0
print(ut.flags)
self.head1 = self.read_header(f1)
self.head2 = self.read_header(f2)
self.rc = 'a'
self.append_data()
self.p += 1
# print(colored("MCE Append",'red'))
return
def netcdfdata(n_files):
tempfiledir = ('./netcdffiles')
if os.path.exists(tempfiledir):
print('Hello!')
else:
print('Making NETCDF File Directory')
netcdf_dir = ['mkdir netcdffiles']
subprocess.Popen(netcdf_dir, shell=True).wait()
#print('HELLO!')
#n_files = len(os.listdir("/data/cryo/current_data"))
a = 0
st.a = a
mce = 1
n = 0
while True:
mce_file = os.path.exists(
"/data/cryo/current_data/temp.%0.3i" %
(a + 1)) #wait to read new file until old file is complete
if mce_file:
print(a)
mce_file_name = "/data/cryo/current_data/temp.%0.3i" % (a)
a = a + 1
st.a = a
f = mce_data.SmallMCEFile(mce_file_name)
header = read_header(f)
mce, n = readdata(f, mce_file_name, mce, header, n, a)
mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i" %
(a + 1))
else:
pass
def readdata():
n = 0
f = mce_data.SmallMCEFile(
'/data/TIME_Software/cryo/20171009/data_test4_2.858Arms_3hz')
h = f.Read(row_col=True, unfilter='DC').data
d = [[[] for i in range(32)] for j in range(31)]
stream_ids = tls.get_credentials_file()['stream_ids']
stream_id = stream_ids[0]
heatmap = go.Heatmap(stream=dict(token=stream_id))
plot_data = go.Data([heatmap])
py.plot(plot_data)
s = py.Stream(stream_id)
s.open()
time.sleep(5)
while n < 5000:
t = time.time()
for i in range(32):
for j in range(31):
d[j][i] = (np.std(h[j, i, n:n + 1000]))
n = n + 1000
print(d[27][4])
z = ([[d[23][0], d[23][1], d[23][2], d[23][3], d[23][4]],
[d[24][0], d[24][1], d[24][2], d[24][3], d[24][4]],
[d[25][0], d[25][1], d[25][2], d[25][3], d[25][4]],
[d[26][0], d[26][1], d[26][2], d[26][3], d[26][4]],
[d[27][0], d[27][1], d[27][2], d[27][3], d[27][4]]])
s.write(dict(z=z, type='heatmap'))
print("time elapsed for loop: %s" % (time.time() - t))
t = 0
time.sleep(2)
s.close()
def takedata(a, ch, n_files, frameperfile, mce, row):
a -= 1
st.a = a
y = []
allgraphdata = []
while True:
mce_file_name = "/home/pilot1/ssh_stuff/mce1/temp.%0.3i" % (a)
mce_file = os.path.exists(
"/home/pilot1/ssh_stuff/mce1/temp.%0.3i" %
(a + 1)) #wait to read new file until old file is complete
if mce_file:
#print(len(os.listdir("/home/pilot1/ssh_stuff/mce1")) - 2 - n_files)
while mce_file:
mce_file_name = "/home/pilot1/ssh_stuff/mce1/temp.%0.3i" % (a)
a = a + 1
st.a = a
f = mce_data.SmallMCEFile(mce_file_name)
#header = read_header(f)
header = 22
z, mce = readdata(f, mce_file_name, frameperfile, mce, header)
graphdata = readgraph(y, f, mce_file_name, a, ch, row)
allgraphdata.append(graphdata)
mce_file = os.path.exists(
"/home/pilot1/ssh_stuff/mce1/temp.%0.3i" % (a + 1))
break
else:
pass
return z, allgraphdata, mce
def __init__(self,
datafile,
minlen_nbranch=50,
min_slope=1e2,
max_slope=1e4,
darksquid_std_thres=300,
verbose=True):
mce = mce_data.SmallMCEFile(datafile)
self.data = mce.Read(row_col=True).data
#iterate through columns and flip as appropriate
for col in range(0, self.data.shape[1]):
self.data[:, col] = config.fb_normalize[col] * self.data[:, col]
self.bias_data = read_ascii(datafile + '.bias',
comment_chars=['<', '#'])[0]
self.bias_step = np.abs(self.bias_data[0] - self.bias_data[1])
self.nrows = self.data.shape[0]
self.ncols = self.data.shape[1]
#initialize self.iv_statuses to nrows*ncols list of None
self.iv_statuses = []
for i in range(self.nrows):
col_vals = []
for j in range(self.ncols):
col_vals.append(None)
self.iv_statuses.append(col_vals)
self.minlen_nbranch = minlen_nbranch
self.min_slope = min_slope
self.max_slope = max_slope
self.darksquid_std_thres = darksquid_std_thres
self.verbose = verbose
def netcdfdata():
print('HELLO!')
#n_files = len(os.listdir("/data/cryo/current_data"))
a = 0
mce = 1
n = 0
n_files = 8
filestarttime = 0
while True:
#mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a)
#mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i" %(a+1)) #wait to read new file until old file is complete
#if mce_file:
#print('NETCDF IS WORKING')
#print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files)
subprocess.call([
'ssh -T [email protected] python /home/pilot2/ssh_stuff/mce1_sftp.py %s %s'
% (a, n_files)
],
shell=True)
mce_file = os.path.exists("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" %
(a + 1))
print("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1))
if mce_file:
print(len(os.listdir("mce1")) - 2)
for i in range(len(os.listdir("mce1")) - 2):
print('netcdf: %s' % (a))
mce_file_name = "/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (
a)
a = a + 1
f = mce_data.SmallMCEFile(mce_file_name)
header = read_header(f)
mce, n, filestarttime = readdata(f, mce_file_name, mce, header,
n, a, filestarttime)
mce_file = os.path.exists(
"/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1))
def netcdfdata(n_files):
print('HELLO!')
#n_files = len(os.listdir("/data/cryo/current_data"))
a = 0
st.a = a
mce = 1
n = 0
while True:
mce_file_name = "/data/cryo/current_data/temp.%0.3i" % (a)
mce_file = os.path.exists(
"/data/cryo/current_data/temp.%0.3i" %
(a + 1)) #wait to read new file until old file is complete
if mce_file:
#print('NETCDF IS WORKING')
#print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files)
for i in range(
len(os.listdir("/data/cryo/current_data")) - 2 -
int(n_files)):
mce_file_name = "/data/cryo/current_data/temp.%0.3i" % (a)
a = a + 1
st.a = a
f = mce_data.SmallMCEFile(mce_file_name)
header = read_header(f)
mce, n = readdata(f, mce_file_name, mce, header, n, a)
else:
pass
def gui_data():
dir1 = '/Users/vlb9398/Desktop/test_mce_files/'
file1 = (dir1 + 'test_data.%0.3i' % (a))
f1 = mce_data.SmallMCEFile(file1)
h1 = file1.Read(row_col=True, unfilter='DC').data
print(colored('File Read: %s' % (file1.replace(dir1, '')), 'yellow'))
''' add stuff in for storing to netcdf '''
return h1
def __init__(self, mce=None, head_binary=None):
mce_data.SmallMCEFile(self, runfile=False, basic_info=False)
self.mce = mce
self.runfile = 'fake'
self.filename = None
self.header = None
self.n_ro = 1
if head_binary != None:
self._ReadHeader(head_binary=self.head_binary)
def netcdfdata(rc, ch, row):
#print('Read files has started')
sys.stdout.flush()
a = 0
mce = 0
n = 0
filestarttime = 0
dir1 = '/home/pilot1/Desktop/time-data/mce1/'
#dir2 = '/home/pilot1/Desktop/time-data/mce2/'
while True:
mce_file1 = os.path.exists(dir1 + 'temp.%0.3i' % (a + 1))
#mce_file2 = os.path.exists(dir2 + 'temp.%0.3i' %(a+1))
#if (mce_file1 and mce_file2):
if mce_file1:
files1 = [
dir1 + x for x in os.listdir(dir1)
if (x.startswith("temp") and not x.endswith('.run'))
]
#print(colored('First if statement passed'))
sys.stdout.flush()
#files2 = [dir2 + x for x in os.listdir(dir2) if (x.startswith("temp") and not x.endswith('.run'))]
# if (len(files1) and len(files2)) != 0:
if len(files1) != 0:
mce_file1 = min(files1, key=os.path.getctime)
#print(colored('Second if statement passed'))
sys.stdout.flush()
#mce_file2 = min(files2, key = os.path.getctime)
f1 = mce_data.SmallMCEFile(mce_file1)
#f2 = mce_data.SmallMCEFile(mce_file2)
head1 = read_header(f1)
#head2 = read_header(f2)
#++++++++++++++++++++++++++++++++ TELESCOPE DATA +++++++++++++++++++++++++++++++++++++++++++++++++++
# pa,slew_flag,alt,az,ra,dec = np.loadtxt('tempfiles/tempteledata.txt',delimiter = ',',unpack=True)
# t = open('tempfiles/tempteledata.txt','w')
# t.close()
# tel_size = len(pa)
# tt = np.column_stack((pa,slew_flag,alt,az,ra,dec))
#print(tt.shape)
#print(tt)
#+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
mce, n, filestarttime, d1, graphdata1 = readdata(
f1, mce_file1, mce, head1, n, a, filestarttime, rc, ch,
row)
#mce, n, filestarttime, tel_size, tt = readdata(f, mce_file, mce, header, n, a, filestarttime, rc, tel_size, tt)
#mce,n,filestarttime,d1,d2,graphdata1,graphdata2 = readdata(f1,f2,mce_file1,mce_file2,mce,head1,head2,n,a,filestarttime,rc,ch,row)
# print(colored('File Read: %s , %s' %(mce_file1.replace(dir,''),mce_file2.replace(dir,''))),'yellow')
print(
colored('File Read: %s' % (mce_file1.replace(dir1, '')),
'yellow'))
a = a + 1
sys.stdout.flush()
break
else:
pass
return d1, graphdata1, mce
def takedata(observer):
y = []
while True:
if st.a < 10: # create a check so we know the file is there and has the right name
mce_file_name = "/data/cryo/current_data/temp.00%i" % (st.a)
mce_file = Path(
"/data/cryo/current_data/temp.00%i" %
(st.a + 1)) #wait to read new file until old file is complete
if mce_file.exists():
st.a = st.a + 1
f = mce_data.SmallMCEFile(mce_file_name)
read_header(f)
readdata(f, mce_file_name)
readgraph(y, f, mce_file_name, st.a)
else:
continue
if st.a >= 10 and st.a < 100:
mce_file_name = "/data/cryo/current_data/temp.0%i" % (st.a)
mce_file = Path("/data/cryo/current_data/temp.0%i" % (st.a + 1))
if mce_file.exists():
st.a = st.a + 1
f = mce_data.SmallMCEFile(mce_file_name)
read_header(f)
readdata(f, mce_file_name)
readgraph(y, f, mce_file_name, st.a)
else:
continue
if st.a >= 100:
mce_file_name = "/data/cryo/current_data/temp.%i" % (st.a)
mce_file = Path("/data/cryo/current_data/temp.%i" % (st.a + 1))
if mce_file.exists():
st.a = st.a + 1
f = mce_data.SmallMCEFile(mce_file_name)
read_header(f)
readdata(f, mce_file_name)
readgraph(y, f, mce_file_name, st.a)
else:
continue
time.sleep(1.0)
def netcdfdata():
print('HELLO!')
#n_files = len(os.listdir("/data/cryo/current_data"))
a = 0
mcea = 0
mce = 1
n = 0
n_files = 8
filestarttime = 0
mcea = subprocess.Popen([
'ssh -T [email protected] python /home/pilot2/ssh_stuff/mce1_sftp.py %s %s'
% (a, n_files)
],
shell=True)
while True:
#mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a)
#mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i" %(a+1)) #wait to read new file until old file is complete
#if mce_file:
#print('NETCDF IS WORKING')
#print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files)
mce_file = os.path.exists("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" %
(a + 1))
#print("/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" %(a+1))
if mce_file:
#print(len(os.listdir("mce1")) - 2)
for i in range(len(os.listdir("mce1")) - 3):
print('netcdf: %s' % (a))
mce_file_name = "/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (
a)
#print(mce_file_name)
a = a + 1
f = mce_data.SmallMCEFile(mce_file_name)
header = read_header(f)
mce, n, filestarttime = readdata(f, mce_file_name, mce, header,
n, a, filestarttime)
mce_file = os.path.exists(
"/home/pilot1/netcdf_stuff/mce1/temp.%0.3i" % (a + 1))
#else:
# pass
if os.path.exists('tempfiles/stop.txt'):
subprocess.call([
"ssh -T [email protected] /usr/mce/bin/mce_cmd '-x stop rcs ret_dat'"
],
shell=True)
mcea.terminate()
subprocess.call([
'ssh -T [email protected] "rm /data/cryo/current_data/temp.*"'
],
shell=True)
subprocess.call(['rm mce1/temp.*'], shell=True)
sys.exit()
def get_mce_data(datafile, **args):
'''Gets mce data from a mce data file using mce_data.SmallMCEFile.Read
@return: the mce data
@rtype: numpy array, 3 dimensional
@param datafile: the file to be passed into mce_data.SmallMCEFile
@type datafile: str
@param args: arguments to be passed into mce_data.SmallMCEFile.Read '''
if os.path.isdir(datafile):
data = mce_dirfile.getdata(datafile)
data = fix_dirfile_signed_field(data)
elif os.path.isfile(datafile):
with open(datafile) as f:
mce = mce_data.SmallMCEFile(datafile)
data = mce.Read(**args).data
return data
def takedataall(a, ch, rc, n_files, frameperfile, mce, row):
a -= 1
st.a = a
y = []
allgraphdata = []
while True:
mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a)
mce_file = os.path.exists("/data/cryo/current_data/temp.%0.3i"%(a+1))
if mce_file:
print(len(os.listdir("/data/cryo/current_data")) - 2 - n_files)
for i in range(len(os.listdir("/data/cryo/current_data")) - 2 - n_files):
mce_file_name = "/data/cryo/current_data/temp.%0.3i" %(a)
a = a + 1
st.a = a
f = mce_data.SmallMCEFile(mce_file_name)
header = read_header(f)
z, mce = readdataall(f, mce_file_name, frameperfile, mce, header)
graphdata = readgraphall(y, f, mce_file_name, a, ch, rc, row)
allgraphdata.append(graphdata)
break
else:
pass
return z, allgraphdata, mce
print(i)
run = [
"mce_run temp %s %s && cat /data/cryo/current_data/temp >> /data/cryo/current_data/%s"
% (numframe, rc, filename)
]
b = subprocess.Popen(run, shell=True)
b.communicate()
if i == 0:
copy2('/data/cryo/current_data/temp.run',
'/data/cryo/current_data/%s' % (filename) + '.run')
open("/data/cryo/current_data/temp", "w").close()
else:
open("/data/cryo/current_data/temp", "w").close()
open("/data/cryo/current_data/temp.run", "w").close()
i = i + 1
# create mce data object
f = mce_data.SmallMCEFile('/data/cryo/current_data/%s' % (filename))
d = f.Read(row_col=True, unfilter='DC').data
x = datetime.datetime.now().strftime('%m-%d-%Y %H:%M:%S.%f')
y = (d[1, 1])
s.write(dict(x=x, y=y))
time.sleep(1) #plots one point every second
except KeyboardInterrupt:
s.close()
print("Data acquisition stopped after %s runs" % (i))
exit()
def takedataall(a, ch, rc, n_files, frameperfile, mce):
#print('Hello!')
#st.init()
#nc.new_file(st.n)
a -= 1
st.a = a
y = []
allgraphdata = []
while True:
if a < 10: # create a check so we know the file is there and has the right name
mce_file_name = "/data/cryo/current_data/temp.00%i" % (a)
if a == 9:
mce_file = os.path.exists("/data/cryo/current_data/temp.0%i" %
(a + 1))
else:
mce_file = os.path.exists(
"/data/cryo/current_data/temp.00%i" %
(a + 1)) #wait to read new file until old file is complete
if mce_file:
print(len(os.listdir('/data/cryo/current_data')) - 2 - n_files)
for i in range(
len(os.listdir('/data/cryo/current_data')) - 2 -
n_files):
if a < 10:
mce_file_name = "/data/cryo/current_data/temp.00%i" % (
a)
else:
mce_file_name = "/data/cryo/current_data/temp.0%i" % (
a)
#print(datafile)
a = a + 1
st.a = a
f = mce_data.SmallMCEFile(mce_file_name)
read_header(f)
z, mce = readdataall(f, mce_file_name, frameperfile, mce)
graphdata = readgraphall(y, f, mce_file_name, a, ch, rc)
allgraphdata.append(graphdata)
break
else:
#continue
pass
elif a >= 10 and a < 100:
mce_file_name = "/data/cryo/current_data/temp.0%i" % (a)
if a == 99:
mce_file = os.path.exists("/data/cryo/current_data/temp.%i" %
(a + 1))
else:
mce_file = os.path.exists("/data/cryo/current_data/temp.0%i" %
(a + 1))
if mce_file:
print(len(os.listdir('/data/cryo/current_data')) - 2 - n_files)
for i in range(
len(os.listdir('/data/cryo/current_data')) - 2 -
n_files):
if a < 100:
mce_file_name = "/data/cryo/current_data/temp.0%i" % (
a)
else:
mce_file_name = "/data/cryo/current_data/temp.%i" % (a)
#print(datafile)
a = a + 1
st.a = a
f = mce_data.SmallMCEFile(mce_file_name)
read_header(f)
z, mce = readdataall(f, mce_file_name, frameperfile, mce)
graphdata = readgraphall(y, f, mce_file_name, a, ch, rc)
allgraphdata.append(graphdata)
break
else:
#continue
pass
elif a >= 100:
mce_file_name = "/data/cryo/current_data/temp.%i" % (a)
mce_file = os.path.exists("/data/cryo/current_data/temp.%i" %
(a + 1))
if mce_file:
print(len(os.listdir('/data/cryo/current_data')) - 2 - n_files)
for i in range(
len(os.listdir('/data/cryo/current_data')) - 2 -
n_files):
mce_file_name = "/data/cryo/current_data/temp.%i" % (a)
#print(datafile)
a = a + 1
st.a = a
f = mce_data.SmallMCEFile(mce_file_name)
read_header(f)
z, mce = readdataall(f, mce_file_name, frameperfile, mce)
graphdata = readgraphall(y, f, mce_file_name, a, ch, rc)
allgraphdata.append(graphdata)
break
else:
#continue
pass
#nc.new_file(st.n, frameperfile)
return z, allgraphdata, mce
df = demod_filter(25, 4)
col_best = 2
row_best = 14
num_points = 131072
nrows = 22
ncols = 4
c0 = [0, 1, 10, 11, 12, 15, 19, 21]
c1 = [1, 2, 5, 6, 7, 14, 15, 17, 18]
c2 = [7, 8, 10, 11, 12, 14]
c3 = [1, 2, 3, 7, 8, 9, 10, 11, 12, 13]
row_arrays = [c0, c1, c2, c3]
if 1 == 0:
data = mce_data.SmallMCEFile(
'/data/cryo/20151204/det_on_2000_triangle_scan_call_0.5mms_25hz0.0')
ctime_init = data.header['runfile_id']
demod_all = np.array([])
for i in np.arange(273):
int_str = str(i / 1000.)
f = '/data/cryo/20151204/det_on_2000_triangle_scan_call_0.5mms_25hz' + int_str
print f
data = mce_data.SmallMCEFile(f)
ctime_det = data.header['runfile_id']
data = fh.get_mce_data(f, row_col=True)
ctime_diff = ctime_det - ctime_init
tes = data[row_best, col_best, :] - np.mean(data[row_best,
col_best, :])
d_filter, d_fft = df.demod(tes, samp_rate)
demod_all = np.concatenate(
def read_header(f):
for key, value in f.header.items():
if key == '_rc_present':
for i in range(len(value)):
if value[i] == True:
value[i] = "1"
elif value[i] == False:
value[i] = "0"
else:
print("I don't know what I am...")
value = ''.join(map(str, value))
value = str(value)
st.keys.append(key)
st.values.append(value)
# keys,values = zip(*f.header.items())
st.keys = np.asarray(st.keys, dtype=object)
st.values = np.asarray(st.values, dtype=object)
st.head = np.array((st.keys, st.values)).T
#-------------------------------------------------------------------------------
if __name__ == "__main__":
st.init()
mce_file_name = path
f = mce_data.SmallMCEFile(mce_file_name)
h = f.Read(row_col=True, unfilter='DC').data
read_header(f)
st.h_size = h.shape[2]
nct.new_file(st.n)
readdata(f, mce_file_name, h)
