def getListOfNewSambaFiles(topDir, lastSambaFileDict):
sambafiles = list()
metafiles = list()
sambaLetter = ""
if os.path.isdir(topDir) == False:
print topDir, "is not a directory"
return sambafiles
try:
for dirpath, dirs, files in os.walk(topDir):
for aFileName in files:
parentdir = os.path.basename(dirpath)
sambaLetter = parentdir[4]
# check for data file
if (
sut.isvalidsambadatafilename(aFileName)
and sut.isvalidsambarunname(parentdir)
and isFileOldEnough(os.path.join(dirpath, aFileName))
):
add = False
if lastSambaFileDict["samba"].has_key(sambaLetter):
if aFileName > lastSambaFileDict["samba"][sambaLetter]["lastfile"]:
add = True
else:
add = True
if add:
_appendToList(sambafiles, os.path.join(dirpath, aFileName), "samba data file found:")
# check for meta data file
if (
sut.isvalidsambametadatafilename(aFileName)
and sut.isvalidsambarunname(parentdir)
and isFileOldEnough(os.path.join(dirpath, aFileName))
):
add = False
if lastSambaFileDict["samba"].has_key(sambaLetter):
if aFileName > lastSambaFileDict["samba"][sambaLetter]["last_%s" % aFileName.split("_")[1]]:
add = True
else:
add = True
if add:
_appendToList(metafiles, os.path.join(dirpath, aFileName), "samba meta data file found:")
except Exception as e:
print aFileName, parentdir, sambaLetter, len(sambafiles), len(metafiles)
raise e
return sambafiles, metafiles
def setLastSambaDataFile(self, aSambaDataFileName):
if sut.isvalidsambadatafilename(aSambaDataFileName) is False:
raise TypeError('%s : Invalid Samba Data File Name' % aSambaDataFileName)
trackerDoc = self.getTrackerDoc()
sambakey = os.path.basename(aSambaDataFileName)[4]
trackerDoc['samba'][sambakey]['lastfile'] = os.path.basename(aSambaDataFileName)
self._save_doc(trackerDoc)
def uploadFile(filename, uri, dbname, overWrite = False):
if sut.isvalidsambadatafilename(os.path.basename(filename)) is False:
return False
theServer = Server(uri)
db = theServer.get_or_create_db(dbname)
#print db.info()
#read the run header
sambafile = open(filename)
global _localRunDict
_localRunDict = {}
runheader = readrunheader(sambafile)
_localRunDict['author'] = 'Samba'
_localRunDict['content'] = 'Samba DAQ document for a particular run. Use this database entry to track the progress of the processing of this data'
_localRunDict['type'] = 'daqdocument'
_localRunDict['date_uploaded'] = time.time()
runname = os.path.basename(sambafile.name)
runsplit = runname.split('_')
_localRunDict['file'] = os.path.realpath(filename)
_localRunDict['file_lastmodified'] = os.path.getctime(filename)
_localRunDict['run_name'] = formatvalue(runsplit[0])
_localRunDict['file_number'] = int(runsplit[1])
sambafile.close() #close and then reopen the file, just to make it easy to get to the start
# of the run.
#read the samba file header
sambafile = open(filename)
sambaheader = readsambafileheader(sambafile)
#now add the key/values to the _localRunDict document
for k, v in sambaheader.items():
if _localRunDict.has_key(k) is False:
_localRunDict[k] = v
#now, loop through and read the bolometer header files
boloArray = list()
lastline = ''
while True:
boloheader = readboloheader(sambafile)
if isinstance(boloheader,dict):
boloArray.append(boloheader)
elif isinstance(boloheader, str):
lastline = copy.copy(boloheader)
print 'Not a Dictionary. We are done reading the bolometer headers.'
break
else:
print 'Not a Dictionary. We are done reading the bolometer headers.'
break
_localRunDict['Detecteurs'] = boloArray
#now read through the channel configuration values
# the while loop above quits when the the readboloheader doesn't return
# a dictionary. Instead, it returns the next line in the header, which
# should be the start of the channel configurations. (assuming that Samba
# doesn't change its format
#
channelArray = list()
voiepart = lastline[lastline.find('* Voie'):]
channelName = voiepart[voiepart.find('"'):].strip('":\n')
while True:
chanheaderoutput = readchannelheader(sambafile, channelName)
channelheader = chanheaderoutput[0]
voiepart = chanheaderoutput[1][chanheaderoutput[1].find('* Voie'):]
channelName = voiepart[voiepart.find('"'):].strip('":\n')
if isinstance(channelheader,dict):
channelArray.append(channelheader)
else:
print 'Read Channel Header didn\'t return a dictionary.'
if chanheaderoutput[2] == False:
print 'Channel Header output False - Done reading Channel Headers.' # okay, this tells us that we're done
break
_localRunDict['Voies'] = channelArray
_localRunDict['status'] = 'closed'
_localRunDict['hostipaddress'] = socket.gethostbyname( socket.gethostname() )
_localRunDict['hostname'] = socket.gethostname()
#_localRunDict['size_in_bytes'] = os.path.getsize(_localRunDict['file'])
#don't allow this script to rewrite a doc to the database!
#if you want to do that, then delete the doc you want to recreate
#or use the overWrite option
doc_exist_status = db.doc_exist(_localRunDict['_id'])
if doc_exist_status and overWrite:
print 'doc exists on database, but overWrite was true. so i overwrite it'
_localRunDict['_rev'] = db.get_rev(_localRunDict['_id'])
elif doc_exist_status and not overWrite:
print 'doc exists on database! exiting without uploading to database'
sambafile.close()
return False
res = db.save_doc(_localRunDict)
sambafile.close()
return res['ok']