def createUnitAndJobChain(path, config, terminate=False):
path = unicodeToStr(path)
if os.path.isdir(path):
path = path + "/"
logger.debug('Creating unit and job chain for %s with %s', path, config)
unit = None
if os.path.isdir(path):
if config[3] == "SIP":
UUID = findOrCreateSipInDB(path)
unit = unitSIP(path, UUID)
elif config[3] == "DIP":
UUID = findOrCreateSipInDB(path, unit_type='DIP')
unit = unitDIP(path, UUID)
elif config[3] == "Transfer":
unit = unitTransfer(path)
elif os.path.isfile(path):
if config[3] == "Transfer":
unit = unitTransfer(path)
else:
return
UUID = uuid.uuid4()
unit = unitFile(path, UUID)
else:
return
jobChain(unit, config[1])
if terminate:
exit(0)
def reloadFileList(self):
"""Match files to their UUID's via their location and the File table's currentLocation"""
self.fileList = {}
# currentPath must be a string to return all filenames as bytestrings,
# and to safely concatenate with other bytestrings
currentPath = os.path.join(self.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), "").encode('utf-8')
try:
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace(currentPath, self.pathString, 1)
for file_ in files:
if self.pathString != directory:
filePath = os.path.join(directory, file_)
else:
filePath = directory + file_
self.fileList[filePath] = unitFile(filePath, owningUnit=self)
if self.unitType == "Transfer":
files = File.objects.filter(transfer_id=self.UUID)
else:
files = File.objects.filter(sip_id=self.UUID)
for f in files:
if f.currentlocation in self.fileList:
self.fileList[f.currentlocation].UUID = f.uuid
self.fileList[f.currentlocation].fileGrpUse = f.filegrpuse
else:
LOGGER.warning('%s %s has file (%s) %s in the database, but file does not exist in the file system',
self.unitType, self.UUID, f.uuid, f.currentlocation)
except Exception:
LOGGER.exception('Error reloading file list for %s', currentPath)
exit(1)
def reloadFileList(self):
self.fileList = {}
#os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
currentPath = self.currentPath.replace("%sharedPath%", \
archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace(currentPath, "%SIPDirectory%", 1)
for file in files:
filePath = os.path.join(directory, file)
#print filePath
self.fileList[filePath] = unitFile(filePath)
sql = """SELECT fileUUID, currentLocation FROM Files WHERE sipUUID = '""" + self.UUID + "'" #AND Files.removedTime = 0; TODO
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
#print row
UUID = row[0]
currentPath = row[1]
if currentPath in self.fileList:
self.fileList[currentPath].UUID = UUID
else:
print "todo: find deleted files/exclude"
print row[99] #fail
row = c.fetchone()
self.fileList[filePath].UUID = UUID
sqlLock.release()
def reloadFileList(self):
self.fileList = {}
#os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
currentPath = self.currentPath.replace("%sharedPath%", \
archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace( currentPath, "%SIPDirectory%", 1)
for file in files:
if directory != "%SIPDirectory%":
filePath = os.path.join(directory, file)
else:
filePath = directory + file
#print filePath
self.fileList[filePath] = unitFile(filePath, owningUnit=self)
sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND sipUUID = '""" + self.UUID + "'"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
#print row
UUID = row[0]
currentPath = row[1]
fileGrpUse = row[2]
if currentPath in self.fileList:
self.fileList[currentPath].UUID = UUID
self.fileList[currentPath].fileGrpUse = fileGrpUse
else:
print >>sys.stderr, self.fileList
eventDetail = "SIP {" + self.UUID + "} has file {" + UUID + "}\"" + currentPath + "\" in the database, but file doesn't exist in the file system."
print >>sys.stderr, "!!!", eventDetail, "!!!"
row = c.fetchone()
sqlLock.release()
def createUnitAndJobChain(path, config, terminate=False):
path = unicodeToStr(path)
if os.path.isdir(path):
path = path + "/"
print "createUnitAndJobChain", path, config
unit = None
if os.path.isdir(path):
if config[3] == "SIP":
UUID = findOrCreateSipInDB(path)
unit = unitSIP(path, UUID)
elif config[3] == "DIP":
UUID = findOrCreateSipInDB(path)
unit = unitDIP(path, UUID)
elif config[3] == "Transfer":
#UUID = findOrCreateSipInDB(path)
unit = unitTransfer(path)
elif os.path.isfile(path):
if config[3] == "Transfer":
unit = unitTransfer(path)
else:
return
UUID = uuid.uuid4()
unit = unitFile(path, UUID)
else:
return
jobChain(unit, config[1])
if terminate:
exit(0)
def createUnitAndJobChain(path, config, terminate=False):
path = unicodeToStr(path)
if os.path.isdir(path):
path = path + "/"
print "createUnitAndJobChain", path, config
unit = None
if os.path.isdir(path):
if config[3] == "SIP":
UUID = findOrCreateSipInDB(path)
unit = unitSIP(path, UUID)
elif config[3] == "DIP":
UUID = findOrCreateSipInDB(path)
unit = unitDIP(path, UUID)
elif config[3] == "Transfer":
#UUID = findOrCreateSipInDB(path)
unit = unitTransfer(path)
elif os.path.isfile(path):
if config[3] == "Transfer":
unit = unitTransfer(path)
else:
return
UUID = uuid.uuid4()
unit = unitFile(path, UUID)
else:
return
jobChain(unit, config[1])
if terminate:
exit(0)
def reloadFileList(self):
self.fileList = {}
#os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
currentPath = self.currentPath.replace("%sharedPath%", \
archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace( currentPath, "%SIPDirectory%", 1)
for file in files:
filePath = os.path.join(directory, file)
#print filePath
self.fileList[filePath] = unitFile(filePath)
sql = """SELECT fileUUID, currentLocation FROM Files WHERE sipUUID = '""" + self.UUID + "'" #AND Files.removedTime = 0; TODO
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
#print row
UUID = row[0]
currentPath = row[1]
if currentPath in self.fileList:
self.fileList[currentPath].UUID = UUID
else:
print "todo: find deleted files/exclude"
print row[99]#fail
row = c.fetchone()
self.fileList[filePath].UUID = UUID
sqlLock.release()
def reloadFileList(self):
self.fileList = {}
#os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
currentPath = self.currentPath.replace("%sharedPath%", \
archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace(currentPath, "%SIPDirectory%", 1)
for file in files:
if directory != "%SIPDirectory%":
filePath = os.path.join(directory, file)
else:
filePath = directory + file
#print filePath
self.fileList[filePath] = unitFile(filePath, owningUnit=self)
sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND sipUUID = '""" + self.UUID + "'"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
#print row
UUID = row[0]
currentPath = row[1]
fileGrpUse = row[2]
if currentPath in self.fileList:
self.fileList[currentPath].UUID = UUID
self.fileList[currentPath].fileGrpUse = fileGrpUse
else:
print >> sys.stderr, self.fileList
eventDetail = "SIP {" + self.UUID + "} has file {" + UUID + "}\"" + currentPath + "\" in the database, but file doesn't exist in the file system."
print >> sys.stderr, "!!!", eventDetail, "!!!"
row = c.fetchone()
sqlLock.release()
def reloadFileList(self):
"""Match files to their UUID's via their location and the File table's currentLocation"""
self.fileList = {}
#os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
currentPath = self.currentPath.replace("%sharedPath%", \
archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
#print "currentPath: ", currentPath, type(currentPath)
try:
#print currentPath, type(currentPath)
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace(currentPath,
"%transferDirectory%", 1)
for file in files:
if "%transferDirectory%" != directory:
filePath = os.path.join(directory, file)
else:
filePath = directory + file
self.fileList[filePath] = unitFile(filePath,
owningUnit=self)
sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND transferUUID = '""" + self.UUID + "'"
#print sql
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
#print self.fileList
while row != None:
#print row
UUID = row[0]
currentPath = row[1].encode("utf-8")
fileGrpUse = row[2]
#print currentPath in self.fileList, row
if currentPath in self.fileList:
self.fileList[currentPath].UUID = UUID
self.fileList[currentPath].fileGrpUse = fileGrpUse
else:
print >> sys.stderr, "!!!", "Transfer {" + self.UUID + "} has file {" + UUID + "}\"", currentPath, "\" in the database, but file doesn't exist in the file system.", "!!!"
row = c.fetchone()
sqlLock.release()
except Exception as inst:
traceback.print_exc(file=sys.stdout)
print type(inst)
print inst.args
exit(1)
def reloadFileList(self):
"""Match files to their UUID's via their location and the File table's currentLocation"""
self.fileList = {}
#os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
currentPath = self.currentPath.replace("%sharedPath%", \
archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
#print "currentPath: ", currentPath, type(currentPath)
try:
#print currentPath, type(currentPath)
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace( currentPath, "%transferDirectory%", 1)
for file in files:
if "%transferDirectory%" != directory:
filePath = os.path.join(directory, file)
else:
filePath = directory + file
self.fileList[filePath] = unitFile(filePath, owningUnit=self)
sql = """SELECT fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND transferUUID = '""" + self.UUID + "'"
#print sql
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
#print self.fileList
while row != None:
#print row
UUID = row[0]
currentPath = row[1].encode("utf-8")
fileGrpUse = row[2]
#print currentPath in self.fileList, row
if currentPath in self.fileList:
self.fileList[currentPath].UUID = UUID
self.fileList[currentPath].fileGrpUse = fileGrpUse
else:
print >>sys.stderr, "!!!", "Transfer {" + self.UUID + "} has file {" + UUID + "}\"", currentPath, "\" in the database, but file doesn't exist in the file system.", "!!!"
row = c.fetchone()
sqlLock.release()
except Exception as inst:
traceback.print_exc(file=sys.stdout)
print type(inst)
print inst.args
exit(1)
def reloadFileList(self):
"""Match files to their UUID's via their location and the File table's currentLocation"""
self.fileList = {}
# currentPath must be a string to return all filenames as bytestrings,
# and to safely concatenate with other bytestrings
currentPath = os.path.join(
self.currentPath.replace("%sharedPath%",
django_settings.SHARED_DIRECTORY, 1),
"",
).encode("utf-8")
try:
for directory, subDirectories, files in os.walk(currentPath):
directory = directory.replace(currentPath, self.pathString, 1)
for file_ in files:
if self.pathString != directory:
filePath = os.path.join(directory, file_)
else:
filePath = directory + file_
self.fileList[filePath] = unitFile(filePath,
owningUnit=self)
if self.unitType == "Transfer":
files = File.objects.filter(transfer_id=self.UUID)
else:
files = File.objects.filter(sip_id=self.UUID)
for f in files:
currentlocation = archivematicaFunctions.unicodeToStr(
f.currentlocation)
if currentlocation in self.fileList:
self.fileList[currentlocation].UUID = f.uuid
self.fileList[currentlocation].fileGrpUse = f.filegrpuse
else:
LOGGER.warning(
"%s %s has file (%s) %s in the database, but file does not exist in the file system",
self.unitType,
self.UUID,
f.uuid,
f.currentlocation,
)
except Exception:
LOGGER.exception("Error reloading file list for %s", currentPath)
exit(1)
