filepath='/air/incoming/sylvania/2015/'
#filepath='/Users/jthom/Documents/data/sylvania/2012/'
# remove existing biomet files from
#dellist = glob(filepath + '*biomet.txt')
#for fn in dellist:
#os.remove(fn)
filesin=glob(filepath + '/Sylvania_met_data*.dat')
timelist=[]
datalist=[]
for fn in filesin:
datefilenameIn=datetime.strptime(fn,filepath + 'Sylvania_met_data_%Y_%m_%d_%H%M.dat')
print fn
data=toa5head(fn)
timelist=[]
datalist=[]
for i in range(len(data)):
timelist.append(data[i][0])
datalist.append(data[i][1])
# convert the long wave data to include the sensor body temperature adjustment
for i in range(len(data)):
datalist[i]['IR01Dn_Avg'] = datalist[i]['IR01Dn_Avg'] + 5.67e-8 * (datalist[i]['TC_Avg'] + 273.15) ** 4
datalist[i]['IR01Up_Avg'] = datalist[i]['IR01Up_Avg'] + 5.67e-8 * (datalist[i]['TC_Avg'] + 273.15) ** 4
if datefilenameIn < date1:
keystoprint=keystoprint1
elif datefilenameIn >= date2:
files.extend(glob(yesterdaydir + '*/Profiler_' + ft + '*.dat'))
files.extend(glob(todaydir + '*/Profiler_' + ft + '.dat'))
# metdata table
#metdatafiles.extend(glob(yesterdaydir + '*/Profiler_metdata*.dat'))
#metdatafiles.extend(glob(todaydir + '*/Profiler_metdata*.dat'))
# oldsensors table
#oldsensorsfiles.extend(glob(yesterdaydir + '*/Profiler_oldsensors*.dat'))
#oldsensorsfiles.extend(glob(todaydir + '*/Profiler_oldsensors*.dat'))
# profileDiag table
#profileDiagfiles.extend(glob(yesterdaydir + '*/Profiler_profileDiag*.dat'))
#profileDiagfiles.extend(glob(todaydir + '*/Profiler_profileDiag*.dat'))
# loop through the files and read in the data
for filein in fastfiles:
#print filein
filedata = toa5head(filein)
data.extend(filedata)
# get the keys for the data stored in the dictionaries
datakeys = data[0][1].keys()
# make a list of list to put the data in before tranferring it to a numpy array
datalist = [[] for i in range(len(datakeys))]
# make a list of all of the datetime objects
for i in range(len(data)):
datatimelist.append(data[i][0])
# put the data from each dictionary into a list of lists
for j in datakeys:
for i in range(len(data)):
datalist[datakeys.index(j)].append(data[i][1].get(j))
# put the data into a numpy array
dataarray = np.array(datalist, dtype='d')
headtitles = timetitle + ',' + ','.join(titlelist) + '\n'
headunits = timeunits + ',' + ','.join(unitslist) + '\n'
#filepath='/Users/jthom/Documents/data/LostCreek/'
filepath = '/air/incoming/LostCreek/'
# find dates to process
#dates=datetime.now() - timedelta(days=1)
# actually, I need to process todays date. UTC has already flipped to the next day.
dates = datetime.now()
datestr = dates.strftime('%Y%m%d')
filesin = glob(filepath + datestr + '/*metvalues*')
timelist = []
datalist = []
for fn in filesin:
data = toa5head(fn)
for i in range(len(data)):
timelist.append(data[i][0])
datalist.append(data[i][1])
timelistindx = sorted(range(len(timelist)), key=lambda k: timelist[k])
datafrmt = []
printstr = []
for i in timelistindx:
for j in keystoprint:
datafrmt.append('%.2f' % datalist[i][j])
# timstr.append(timelist[i].strftime('%Y-%m-%d %H%M'))
printstr.append(timelist[i].strftime('%Y-%m-%d %H%M') + ',' +
','.join(datafrmt) + '\n')
datafrmt = []
#create directories to look in and make a list of files to read
yesterdaydir = datadir + yesterday.strftime('%Y_%m/%d/*/')
todaydir = datadir + currenttime.strftime('%Y_%m/%d/*/')
fastfiles.extend(glob(yesterdaydir + logger + '_fast_*.dat'))
fastfiles.extend(glob(todaydir + logger + '_fast_*.dat'))
slowfiles.extend(glob(yesterdaydir + logger + '_slow*.dat'))
slowfiles.extend(glob(todaydir + logger + '_slow*.dat'))
# initialize the lists for storing the data
fast = []
slow = []
fasttimelist = []
slowtimelist = []
# loop through the files and read in the data
for filein in fastfiles:
filedata = toa5head(filein)
fast.extend(filedata)
for filein in slowfiles:
filedata = toa5head(filein)
slow.extend(filedata)
# get the keys for the data stored in the dictionaries
fastkeys = fast[0][1].keys()
slowkeys = slow[0][1].keys()
# make a list of list to put the data in before tranferring it to a numpy array
fastlist = [[] for i in range(len(fastkeys))]
slowlist = [[] for i in range(len(slowkeys))]
# make a list of all of the datetime objects
for i in range(len(fast)):
fasttimelist.append(fast[i][0])
for i in range(len(slow)):
slowtimelist.append(slow[i][0])
files.extend(glob(yesterdaydir + '/*/Profiler_' + ft + '*.dat'))
files.extend(glob(todaydir + '/*/Profiler_' + ft + '*.dat'))
# metdata table
#metdatafiles.extend(glob(yesterdaydir + '*/Profiler_metdata*.dat'))
#metdatafiles.extend(glob(todaydir + '*/Profiler_metdata*.dat'))
# oldsensors table
#oldsensorsfiles.extend(glob(yesterdaydir + '*/Profiler_oldsensors*.dat'))
#oldsensorsfiles.extend(glob(todaydir + '*/Profiler_oldsensors*.dat'))
# profileDiag table
#profileDiagfiles.extend(glob(yesterdaydir + '*/Profiler_profileDiag*.dat'))
#profileDiagfiles.extend(glob(todaydir + '*/Profiler_profileDiag*.dat'))
# loop through the files and read in the data
for filein in files:
#print filein
datain = toa5head(filein)
data.extend(datain)
# get the keys for the data stored in the dictionaries
datakeys = data[0][1].keys()
# make a list of list to put the data in before tranferring it to a numpy array
datalist = [[] for i in range(len(datakeys))]
# make a list of all of the datetime objects
for i in range(len(data)):
datatimelist.append(data[i][0])
# put the data from each dictionary into a list of lists
for j in datakeys:
for i in range(len(data)):
datalist[datakeys.index(j)].append(data[i][1].get(j))
# put the data into a numpy array
dataarray = np.array(datalist, dtype='d')
data = []
datatimelist = []
#create directories to look in and make a list of files to read
yesterdaydir = datadir + yesterday.strftime('%Y%m%d')
todaydir = datadir + currenttime.strftime('%Y%m%d')
# create list of files that will be read
#licor table
files.extend(glob(yesterdaydir + '/*/Profiler_' + ft + '*.dat'))
files.extend(glob(todaydir + '/*/Profiler_' + ft + '*.dat'))
# loop through the files and read in the data
for filein in files:
#print filein
datain = toa5head(filein)
data.extend(datain)
# get the keys for the data stored in the dictionaries
datakeys = data[0][1].keys()
# make a list of list to put the data in before tranferring it to a numpy array
datalist = [[] for i in range(len(datakeys))]
IDlist=[]
# make a list of all of the datetime objects
for i in range(len(data)):
datatimelist.append(data[i][0])
# put the data from each dictionary into a list of lists
for j in datakeys:
for i in range(len(data)):
datalist[datakeys.index(j)].append(data[i][1].get(j))
for i in range(len(data)):