def createUnitAndJobChain(path, config, terminate=False): path = unicodeToStr(path) if os.path.isdir(path): path = path + "/" logger.debug('Creating unit and job chain for %s with %s', path, config) unit = None if os.path.isdir(path): if config[3] == "SIP": UUID = findOrCreateSipInDB(path) unit = unitSIP(path, UUID) elif config[3] == "DIP": UUID = findOrCreateSipInDB(path, unit_type='DIP') unit = unitDIP(path, UUID) elif config[3] == "Transfer": unit = unitTransfer(path) elif os.path.isfile(path): if config[3] == "Transfer": unit = unitTransfer(path) else: return UUID = uuid.uuid4() unit = unitFile(path, UUID) else: return jobChain(unit, config[1]) if terminate: exit(0)
def reloadFileList(self): """Match files to their UUID's via their location and the File table's currentLocation""" self.fileList = {} # currentPath must be a string to return all filenames as bytestrings, # and to safely concatenate with other bytestrings currentPath = os.path.join(self.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), "").encode('utf-8') try: for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace(currentPath, self.pathString, 1) for file_ in files: if self.pathString != directory: filePath = os.path.join(directory, file_) else: filePath = directory + file_ self.fileList[filePath] = unitFile(filePath, owningUnit=self) if self.unitType == "Transfer": files = File.objects.filter(transfer_id=self.UUID) else: files = File.objects.filter(sip_id=self.UUID) for f in files: if f.currentlocation in self.fileList: self.fileList[f.currentlocation].UUID = f.uuid self.fileList[f.currentlocation].fileGrpUse = f.filegrpuse else: LOGGER.warning('%s %s has file (%s) %s in the database, but file does not exist in the file system', self.unitType, self.UUID, f.uuid, f.currentlocation) except Exception: LOGGER.exception('Error reloading file list for %s', currentPath) exit(1)
def reloadFileList(self): self.fileList = {} #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]]) currentPath = self.currentPath.replace("%sharedPath%", \ archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/" for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace(currentPath, "%SIPDirectory%", 1) for file in files: filePath = os.path.join(directory, file) #print filePath self.fileList[filePath] = unitFile(filePath) sql = """SELECT fileUUID, currentLocation FROM Files WHERE sipUUID = '""" + self.UUID + "'" #AND Files.removedTime = 0; TODO c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: #print row UUID = row[0] currentPath = row[1] if currentPath in self.fileList: self.fileList[currentPath].UUID = UUID else: print "todo: find deleted files/exclude" print row[99] #fail row = c.fetchone() self.fileList[filePath].UUID = UUID sqlLock.release()
def reloadFileList(self): self.fileList = {} #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]]) currentPath = self.currentPath.replace("%sharedPath%", \ archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/" for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace( currentPath, "%SIPDirectory%", 1) for file in files: if directory != "%SIPDirectory%": filePath = os.path.join(directory, file) else: filePath = directory + file #print filePath self.fileList[filePath] = unitFile(filePath, owningUnit=self) sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND sipUUID = '""" + self.UUID + "'" c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: #print row UUID = row[0] currentPath = row[1] fileGrpUse = row[2] if currentPath in self.fileList: self.fileList[currentPath].UUID = UUID self.fileList[currentPath].fileGrpUse = fileGrpUse else: print >>sys.stderr, self.fileList eventDetail = "SIP {" + self.UUID + "} has file {" + UUID + "}\"" + currentPath + "\" in the database, but file doesn't exist in the file system." print >>sys.stderr, "!!!", eventDetail, "!!!" row = c.fetchone() sqlLock.release()
def createUnitAndJobChain(path, config, terminate=False): path = unicodeToStr(path) if os.path.isdir(path): path = path + "/" print "createUnitAndJobChain", path, config unit = None if os.path.isdir(path): if config[3] == "SIP": UUID = findOrCreateSipInDB(path) unit = unitSIP(path, UUID) elif config[3] == "DIP": UUID = findOrCreateSipInDB(path) unit = unitDIP(path, UUID) elif config[3] == "Transfer": #UUID = findOrCreateSipInDB(path) unit = unitTransfer(path) elif os.path.isfile(path): if config[3] == "Transfer": unit = unitTransfer(path) else: return UUID = uuid.uuid4() unit = unitFile(path, UUID) else: return jobChain(unit, config[1]) if terminate: exit(0)
def reloadFileList(self): self.fileList = {} #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]]) currentPath = self.currentPath.replace("%sharedPath%", \ archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/" for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace( currentPath, "%SIPDirectory%", 1) for file in files: filePath = os.path.join(directory, file) #print filePath self.fileList[filePath] = unitFile(filePath) sql = """SELECT fileUUID, currentLocation FROM Files WHERE sipUUID = '""" + self.UUID + "'" #AND Files.removedTime = 0; TODO c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: #print row UUID = row[0] currentPath = row[1] if currentPath in self.fileList: self.fileList[currentPath].UUID = UUID else: print "todo: find deleted files/exclude" print row[99]#fail row = c.fetchone() self.fileList[filePath].UUID = UUID sqlLock.release()
def reloadFileList(self): self.fileList = {} #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]]) currentPath = self.currentPath.replace("%sharedPath%", \ archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/" for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace(currentPath, "%SIPDirectory%", 1) for file in files: if directory != "%SIPDirectory%": filePath = os.path.join(directory, file) else: filePath = directory + file #print filePath self.fileList[filePath] = unitFile(filePath, owningUnit=self) sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND sipUUID = '""" + self.UUID + "'" c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: #print row UUID = row[0] currentPath = row[1] fileGrpUse = row[2] if currentPath in self.fileList: self.fileList[currentPath].UUID = UUID self.fileList[currentPath].fileGrpUse = fileGrpUse else: print >> sys.stderr, self.fileList eventDetail = "SIP {" + self.UUID + "} has file {" + UUID + "}\"" + currentPath + "\" in the database, but file doesn't exist in the file system." print >> sys.stderr, "!!!", eventDetail, "!!!" row = c.fetchone() sqlLock.release()
def reloadFileList(self): """Match files to their UUID's via their location and the File table's currentLocation""" self.fileList = {} #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]]) currentPath = self.currentPath.replace("%sharedPath%", \ archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/" #print "currentPath: ", currentPath, type(currentPath) try: #print currentPath, type(currentPath) for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace(currentPath, "%transferDirectory%", 1) for file in files: if "%transferDirectory%" != directory: filePath = os.path.join(directory, file) else: filePath = directory + file self.fileList[filePath] = unitFile(filePath, owningUnit=self) sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND transferUUID = '""" + self.UUID + "'" #print sql c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() #print self.fileList while row != None: #print row UUID = row[0] currentPath = row[1].encode("utf-8") fileGrpUse = row[2] #print currentPath in self.fileList, row if currentPath in self.fileList: self.fileList[currentPath].UUID = UUID self.fileList[currentPath].fileGrpUse = fileGrpUse else: print >> sys.stderr, "!!!", "Transfer {" + self.UUID + "} has file {" + UUID + "}\"", currentPath, "\" in the database, but file doesn't exist in the file system.", "!!!" row = c.fetchone() sqlLock.release() except Exception as inst: traceback.print_exc(file=sys.stdout) print type(inst) print inst.args exit(1)
def reloadFileList(self): """Match files to their UUID's via their location and the File table's currentLocation""" self.fileList = {} #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]]) currentPath = self.currentPath.replace("%sharedPath%", \ archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/" #print "currentPath: ", currentPath, type(currentPath) try: #print currentPath, type(currentPath) for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace( currentPath, "%transferDirectory%", 1) for file in files: if "%transferDirectory%" != directory: filePath = os.path.join(directory, file) else: filePath = directory + file self.fileList[filePath] = unitFile(filePath, owningUnit=self) sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND transferUUID = '""" + self.UUID + "'" #print sql c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() #print self.fileList while row != None: #print row UUID = row[0] currentPath = row[1].encode("utf-8") fileGrpUse = row[2] #print currentPath in self.fileList, row if currentPath in self.fileList: self.fileList[currentPath].UUID = UUID self.fileList[currentPath].fileGrpUse = fileGrpUse else: print >>sys.stderr, "!!!", "Transfer {" + self.UUID + "} has file {" + UUID + "}\"", currentPath, "\" in the database, but file doesn't exist in the file system.", "!!!" row = c.fetchone() sqlLock.release() except Exception as inst: traceback.print_exc(file=sys.stdout) print type(inst) print inst.args exit(1)
def reloadFileList(self): """Match files to their UUID's via their location and the File table's currentLocation""" self.fileList = {} # currentPath must be a string to return all filenames as bytestrings, # and to safely concatenate with other bytestrings currentPath = os.path.join( self.currentPath.replace("%sharedPath%", django_settings.SHARED_DIRECTORY, 1), "", ).encode("utf-8") try: for directory, subDirectories, files in os.walk(currentPath): directory = directory.replace(currentPath, self.pathString, 1) for file_ in files: if self.pathString != directory: filePath = os.path.join(directory, file_) else: filePath = directory + file_ self.fileList[filePath] = unitFile(filePath, owningUnit=self) if self.unitType == "Transfer": files = File.objects.filter(transfer_id=self.UUID) else: files = File.objects.filter(sip_id=self.UUID) for f in files: currentlocation = archivematicaFunctions.unicodeToStr( f.currentlocation) if currentlocation in self.fileList: self.fileList[currentlocation].UUID = f.uuid self.fileList[currentlocation].fileGrpUse = f.filegrpuse else: LOGGER.warning( "%s %s has file (%s) %s in the database, but file does not exist in the file system", self.unitType, self.UUID, f.uuid, f.currentlocation, ) except Exception: LOGGER.exception("Error reloading file list for %s", currentPath) exit(1)