# author Sarah Heim
# date create: May 2015
#
# Description: Read all log files and create NetCDFs
# Input: location of log files and location of where to put NCs(both in sass.py)
# Output: netCDFs
#
# Plans:
#import pandas
import time, os
import sass
start = time.time()
#Loop through ALL log files and put in NC files
mnArr = os.listdir(sass.logsdir)
mnArr.sort()
for mn in mnArr:
mnpath = os.path.join(sass.logsdir, mn)
if os.path.isdir(mnpath):
# print "folder:", mn
startfld = time.time()
#Loop through all log files
filesArr = os.listdir(mnpath)
filesArr.sort()
for fn in filesArr:
filename = os.path.join(mnpath, fn)
sass.log2nc(filename)
print time.time() - startfld, mn
print "DONE", time.time()-start, time.strftime(sass.dateformat, time.gmtime())
sig.units = "kg/m^3"
dV = ncfile.createVariable("diagnosticVoltage", "f4", ("time"), **kwargs)
dV.long_name = "diagnostic voltage" # NO standard name???
dV.units = "V"
cDr = ncfile.createVariable("currentDraw", "f4", ("time"), **kwargs)
cDr.long_name = "current draw" # NO standard name???
cDr.units = "mA"
ncfile.Metadata_Conventions = "Unidata Dataset Discovery v1.0"
ncfile.Conventions = "CF-1.6"
ncfile.summary = "Automated shore station with a suite of sensors that are attached to piers"
return ncfile
# mnpath = os.path.join(sass.logsdir, mn)
filename = os.path.join(sass.logsdir, "2015-07", "data-20150701.dat")
# print sass.ncpath
sass.ncpath = ncpath
print sass.ncpath
print filename
sass.log2nc(filename)
print "Made init file", time.time() - start
# Append a few files
sass.log2nc(os.path.join(sass.logsdir, "2015-07", "data-20150702.dat"))
sass.log2nc(os.path.join(sass.logsdir, "2015-07", "data-20150703.dat"))
sass.log2nc(os.path.join(sass.logsdir, "2015-07", "data-20150704.dat"))
print "DONE! Appended a few log files", time.time() - start
def appendToNCs(logfile):
#Make log, for testing!!!!!!!
# logfile = '/data/InSitu/SASS/code/log2nc/appendLog.txt'
looplimit = 30
log = open(logfile, 'a+')
# log.write('STARTING Script: '+time.strftime(sass.dateformat, time.gmtime()))
start = time.time()
LRstr = sass.readLastRecorded()
if (LRstr):
LRdt = dt(LRstr)
# print "Last Recorded date:", LRdt.timetuple()[0:3]
# log.write("\tLast Recorded date: " + str(LRdt.timetuple()[0:3])+"\n")
# print "Today's server date:", time.gmtime()[0:3]
# log.write("\tCurrent server date: " + str(time.gmtime())+"\n")
if LRdt.timetuple()[0:3] < time.gmtime()[0:3]:
# print "initial file:", time.time()-start, datefilename(LRdt)
# log.write('\nDIF day, catching up: ')
# log.write("\n\tinitial file: " + str(time.time()-start)+" "+ datefilename(LRdt))
sass.log2nc(datefilename(LRdt))
loopCount = 0
while LRdt.timetuple()[0:3] < time.gmtime()[0:3]:
nextDayFile = datefilename(LRdt + datetime.timedelta(days=1)) #increment day
# print "looping:", time.time()-start, nextDayFile
# log.write("\n\tlooping: " + str(time.time()-start) +" "+ nextDayFile)
sass.log2nc(nextDayFile)
LRdt = dt(sass.readLastRecorded())
#New Day/Today may not exist just yet!!!
# exit or stuck in loop until file is created
if nextDayFile == datefilename(datetime.datetime.utcnow()):
# log.write('\n\tReached today, STOPPING')
return time.time()-start
loopCount += 1
if loopCount > looplimit:
log.write('\n\tReached loop limit, STOPPING: '+str(datetime.datetime.utcnow()))
return time.time()-start
else:
# Same Day
# if file exists, read today's Last Line Server datetime (LLSdt)
if os.path.isfile(datefilename(LRdt)):
#USE Pandas. Last line could be a bad/ignored line
# for line in open(datefilename(LRdt), 'r'):
# last=line
# LLSstr = last.split(',')[0]
pre = sass.readLastRecorded()
# print "Last Rec'd datetime:", pre
# log.write("\tLast Rec'd datetime: " + pre+"\n")
sass.log2nc(datefilename(LRdt))
post = sass.readLastRecorded()
# print "post datetime:", post
# if (pre==post):
# log.write("\tNO UPDATE")
# else:
# log.write("\tNew: " + post)
# if LLSdt != LRdt:
# print "add new data from TODAY"
# sass.log2nc(datefilename(LRdt))
# else:
# print "nc files are up to date"
else:
# print "today's file doesn't exist yet", datefilename(LRdt)
log.write("\ntoday's file doesn't exist yet: " + datefilename(LRdt)+"\n")
else:
log.write("\nERROR: Latest Recorded file does NOT exist! \n")
#print "DONE APPENDING", time.time()-start
#log.write("\tDONE! " + str(time.time()-start)+"\n")
return time.time()-start
sig.units = 'kg/m^3'
dV = ncfile.createVariable('diagnosticVoltage', 'f4', ('time'), **kwargs)
dV.long_name = 'diagnostic voltage' #NO standard name???
dV.units = 'V'
cDr = ncfile.createVariable('currentDraw', 'f4', ('time'), **kwargs)
cDr.long_name = 'current draw' #NO standard name???
cDr.units = 'mA'
ncfile.Metadata_Conventions = 'Unidata Dataset Discovery v1.0'
ncfile.Conventions = 'CF-1.6'
ncfile.summary = 'Automated shore station with a suite of sensors that are attached to piers'
return ncfile
#mnpath = os.path.join(sass.logsdir, mn)
filename = os.path.join(sass.logsdir, '2015-07', 'data-20150701.dat')
#print sass.ncpath
sass.ncpath = ncpath
print sass.ncpath
print filename
sass.log2nc(filename)
print "Made init file", time.time() - start
#Append a few files
sass.log2nc(os.path.join(sass.logsdir, '2015-07', 'data-20150702.dat'))
sass.log2nc(os.path.join(sass.logsdir, '2015-07', 'data-20150703.dat'))
sass.log2nc(os.path.join(sass.logsdir, '2015-07', 'data-20150704.dat'))
print "DONE! Appended a few log files", time.time() - start
def appendToNCs(logfile):
#Make log, for testing!!!!!!!
# logfile = '/data/InSitu/SASS/code/log2nc/appendLog.txt'
looplimit = 30
log = open(logfile, 'a+')
# log.write('STARTING Script: '+time.strftime(sass.dateformat, time.gmtime()))
start = time.time()
LRstr = sass.readLastRecorded()
if (LRstr):
LRdt = dt(LRstr)
# print "Last Recorded date:", LRdt.timetuple()[0:3]
# log.write("\tLast Recorded date: " + str(LRdt.timetuple()[0:3])+"\n")
# print "Today's server date:", time.gmtime()[0:3]
# log.write("\tCurrent server date: " + str(time.gmtime())+"\n")
if LRdt.timetuple()[0:3] < time.gmtime()[0:3]:
# print "initial file:", time.time()-start, datefilename(LRdt)
# log.write('\nDIF day, catching up: ')
# log.write("\n\tinitial file: " + str(time.time()-start)+" "+ datefilename(LRdt))
sass.log2nc(datefilename(LRdt))
loopCount = 0
while LRdt.timetuple()[0:3] < time.gmtime()[0:3]:
nextDayFile = datefilename(
LRdt + datetime.timedelta(days=1)) #increment day
# print "looping:", time.time()-start, nextDayFile
# log.write("\n\tlooping: " + str(time.time()-start) +" "+ nextDayFile)
sass.log2nc(nextDayFile)
LRdt = dt(sass.readLastRecorded())
#New Day/Today may not exist just yet!!!
# exit or stuck in loop until file is created
if nextDayFile == datefilename(datetime.datetime.utcnow()):
# log.write('\n\tReached today, STOPPING')
return time.time() - start
loopCount += 1
if loopCount > looplimit:
log.write('\n\tReached loop limit, STOPPING: ' +
str(datetime.datetime.utcnow()))
return time.time() - start
else:
# Same Day
# if file exists, read today's Last Line Server datetime (LLSdt)
if os.path.isfile(datefilename(LRdt)):
#USE Pandas. Last line could be a bad/ignored line
# for line in open(datefilename(LRdt), 'r'):
# last=line
# LLSstr = last.split(',')[0]
pre = sass.readLastRecorded()
# print "Last Rec'd datetime:", pre
# log.write("\tLast Rec'd datetime: " + pre+"\n")
sass.log2nc(datefilename(LRdt))
post = sass.readLastRecorded()
# print "post datetime:", post
# if (pre==post):
# log.write("\tNO UPDATE")
# else:
# log.write("\tNew: " + post)
# if LLSdt != LRdt:
# print "add new data from TODAY"
# sass.log2nc(datefilename(LRdt))
# else:
# print "nc files are up to date"
else:
# print "today's file doesn't exist yet", datefilename(LRdt)
log.write("\ntoday's file doesn't exist yet: " +
datefilename(LRdt) + "\n")
else:
log.write("\nERROR: Latest Recorded file does NOT exist! \n")
#print "DONE APPENDING", time.time()-start
#log.write("\tDONE! " + str(time.time()-start)+"\n")
return time.time() - start
# date create: May 2015
#
# Description: Read all log files and create NetCDFs
# Input: location of log files and location of where to put NCs(both in sass.py)
# Output: netCDFs
#
# Plans:
#import pandas
import time, os
import sass
start = time.time()
#Loop through ALL log files and put in NC files
mnArr = os.listdir(sass.logsdir)
mnArr.sort()
for mn in mnArr:
mnpath = os.path.join(sass.logsdir, mn)
if os.path.isdir(mnpath):
# print "folder:", mn
startfld = time.time()
#Loop through all log files
filesArr = os.listdir(mnpath)
filesArr.sort()
for fn in filesArr:
filename = os.path.join(mnpath, fn)
sass.log2nc(filename)
print time.time() - startfld, mn
print "DONE", time.time() - start, time.strftime(sass.dateformat,
time.gmtime())