def getListOfNewSambaFiles(topDir, lastSambaFileDict): sambafiles = list() metafiles = list() sambaLetter = "" if os.path.isdir(topDir) == False: print topDir, "is not a directory" return sambafiles try: for dirpath, dirs, files in os.walk(topDir): for aFileName in files: parentdir = os.path.basename(dirpath) sambaLetter = parentdir[4] # check for data file if ( sut.isvalidsambadatafilename(aFileName) and sut.isvalidsambarunname(parentdir) and isFileOldEnough(os.path.join(dirpath, aFileName)) ): add = False if lastSambaFileDict["samba"].has_key(sambaLetter): if aFileName > lastSambaFileDict["samba"][sambaLetter]["lastfile"]: add = True else: add = True if add: _appendToList(sambafiles, os.path.join(dirpath, aFileName), "samba data file found:") # check for meta data file if ( sut.isvalidsambametadatafilename(aFileName) and sut.isvalidsambarunname(parentdir) and isFileOldEnough(os.path.join(dirpath, aFileName)) ): add = False if lastSambaFileDict["samba"].has_key(sambaLetter): if aFileName > lastSambaFileDict["samba"][sambaLetter]["last_%s" % aFileName.split("_")[1]]: add = True else: add = True if add: _appendToList(metafiles, os.path.join(dirpath, aFileName), "samba meta data file found:") except Exception as e: print aFileName, parentdir, sambaLetter, len(sambafiles), len(metafiles) raise e return sambafiles, metafiles
def setLastSambaDataFile(self, aSambaDataFileName): if sut.isvalidsambadatafilename(aSambaDataFileName) is False: raise TypeError('%s : Invalid Samba Data File Name' % aSambaDataFileName) trackerDoc = self.getTrackerDoc() sambakey = os.path.basename(aSambaDataFileName)[4] trackerDoc['samba'][sambakey]['lastfile'] = os.path.basename(aSambaDataFileName) self._save_doc(trackerDoc)
def uploadFile(filename, uri, dbname, overWrite = False): if sut.isvalidsambadatafilename(os.path.basename(filename)) is False: return False theServer = Server(uri) db = theServer.get_or_create_db(dbname) #print db.info() #read the run header sambafile = open(filename) global _localRunDict _localRunDict = {} runheader = readrunheader(sambafile) _localRunDict['author'] = 'Samba' _localRunDict['content'] = 'Samba DAQ document for a particular run. Use this database entry to track the progress of the processing of this data' _localRunDict['type'] = 'daqdocument' _localRunDict['date_uploaded'] = time.time() runname = os.path.basename(sambafile.name) runsplit = runname.split('_') _localRunDict['file'] = os.path.realpath(filename) _localRunDict['file_lastmodified'] = os.path.getctime(filename) _localRunDict['run_name'] = formatvalue(runsplit[0]) _localRunDict['file_number'] = int(runsplit[1]) sambafile.close() #close and then reopen the file, just to make it easy to get to the start # of the run. #read the samba file header sambafile = open(filename) sambaheader = readsambafileheader(sambafile) #now add the key/values to the _localRunDict document for k, v in sambaheader.items(): if _localRunDict.has_key(k) is False: _localRunDict[k] = v #now, loop through and read the bolometer header files boloArray = list() lastline = '' while True: boloheader = readboloheader(sambafile) if isinstance(boloheader,dict): boloArray.append(boloheader) elif isinstance(boloheader, str): lastline = copy.copy(boloheader) print 'Not a Dictionary. We are done reading the bolometer headers.' break else: print 'Not a Dictionary. We are done reading the bolometer headers.' break _localRunDict['Detecteurs'] = boloArray #now read through the channel configuration values # the while loop above quits when the the readboloheader doesn't return # a dictionary. Instead, it returns the next line in the header, which # should be the start of the channel configurations. (assuming that Samba # doesn't change its format # channelArray = list() voiepart = lastline[lastline.find('* Voie'):] channelName = voiepart[voiepart.find('"'):].strip('":\n') while True: chanheaderoutput = readchannelheader(sambafile, channelName) channelheader = chanheaderoutput[0] voiepart = chanheaderoutput[1][chanheaderoutput[1].find('* Voie'):] channelName = voiepart[voiepart.find('"'):].strip('":\n') if isinstance(channelheader,dict): channelArray.append(channelheader) else: print 'Read Channel Header didn\'t return a dictionary.' if chanheaderoutput[2] == False: print 'Channel Header output False - Done reading Channel Headers.' # okay, this tells us that we're done break _localRunDict['Voies'] = channelArray _localRunDict['status'] = 'closed' _localRunDict['hostipaddress'] = socket.gethostbyname( socket.gethostname() ) _localRunDict['hostname'] = socket.gethostname() #_localRunDict['size_in_bytes'] = os.path.getsize(_localRunDict['file']) #don't allow this script to rewrite a doc to the database! #if you want to do that, then delete the doc you want to recreate #or use the overWrite option doc_exist_status = db.doc_exist(_localRunDict['_id']) if doc_exist_status and overWrite: print 'doc exists on database, but overWrite was true. so i overwrite it' _localRunDict['_rev'] = db.get_rev(_localRunDict['_id']) elif doc_exist_status and not overWrite: print 'doc exists on database! exiting without uploading to database' sambafile.close() return False res = db.save_doc(_localRunDict) sambafile.close() return res['ok']