def __init__(self, jobChainLink, pk, unit):
global outputLock
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
opts = {"inputFile":"%relativeLocation%", "fileUUID":"%fileUUID%", 'commandClassifications':'%commandClassifications%', "taskUUID":"%taskUUID%", "objectsDirectory":"%SIPObjectsDirectory%", "logsDirectory":"%SIPLogsDirectory%", "sipUUID":"%SIPUUID%", "sipPath":"%SIPDirectory%", "fileGrpUse":"%fileGrpUse%", "normalizeFileGrpUse":"%normalizeFileGrpUse%", "excludeDirectory":"%excludeDirectory%", "standardErrorFile":"%standardErrorFile%", "standardOutputFile":"%standardOutputFile%"}
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
for optsKey, optsValue in opts.iteritems():
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
opts[optsKey] = self.jobChainLink.passVar.replace(opts[optsKey])[0]
commandReplacementDic = unit.getReplacementDic()
for key, value in commandReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
for key, value in SIPReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
commandReplacementDic = unit.getReplacementDic()
sql = """SELECT CommandRelationships.pk FROM CommandRelationships JOIN Commands ON CommandRelationships.command = Commands.pk WHERE CommandRelationships.pk = '%s';""" % (pk.__str__())
rows = databaseInterface.queryAllSQL(sql)
taskCount = 0
tasksList = []
if rows:
self.tasksLock.acquire()
for row in rows:
UUID = uuid.uuid4().__str__()
opts["taskUUID"] = UUID
opts["CommandRelationship"] = pk.__str__()
execute = "transcoder_cr%s" % (pk)
deUnicode(execute)
arguments = row.__str__()
standardOutputFile = opts["standardOutputFile"]
standardErrorFile = opts["standardErrorFile"]
self.standardOutputFile = standardOutputFile
self.standardErrorFile = standardErrorFile
self.execute = execute
self.arguments = arguments
task = taskStandard(self, execute, opts, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
taskCount += 1
tasksList.append(task)
self.tasksLock.release()
for task in tasksList:
task.performTask()
else:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def reload(self):
sql = """SELECT * FROM SIPs WHERE sipUUID = '""" + self.UUID + "'"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
print row
#self.UUID = row[0]
self.createdTime = deUnicode(row[1])
self.currentPath = deUnicode(row[2])
row = c.fetchone()
sqlLock.release()
def reload(self):
sql = """SELECT * FROM SIPs WHERE sipUUID = '""" + self.UUID + "'"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
print row
#self.UUID = row[0]
self.createdTime = deUnicode(row[1])
self.currentPath = deUnicode(row[2])
row = c.fetchone()
sqlLock.release()
def reload(self):
sql = """SELECT transferUUID, currentLocation FROM Transfers WHERE transferUUID = '""" + self.UUID + "'"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
self.UUID = deUnicode(row[0])
#self.createdTime = row[1]
self.currentPath = deUnicode(row[1])
row = c.fetchone()
sqlLock.release()
return
def reload(self):
sql = """SELECT transferUUID, currentLocation FROM Transfers WHERE transferUUID = '""" + self.UUID + "'"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
self.UUID = deUnicode(row[0])
#self.createdTime = row[1]
self.currentPath = deUnicode(row[1])
row = c.fetchone()
sqlLock.release()
return
def __init__(self, jobChainLink, pk, unit):
self.tasks = []
self.pk = pk
self.jobChainLink = jobChainLink
sql = """SELECT * FROM StandardTasksConfigs where pk = """ + pk.__str__()
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
print row
#pk = row[0]
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
self.requiresOutputLock = deUnicode(row[4])
standardOutputFile = deUnicode(row[5])
standardErrorFile = deUnicode(row[6])
execute = deUnicode(row[7])
self.execute = execute
arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
#if reloadFileList:
# unit.reloadFileList()
# "%taskUUID%": task.UUID.__str__(), \
if filterSubDir:
directory = os.path.join(unit.currentPath, filterSubDir)
else:
directory = unit.currentPath
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
commandReplacementDic = unit.getReplacementDic(directory)
#for each key replace all instances of the key in the command string
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=UUID)
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
t = threading.Thread(target=self.task.performTask)
t.daemon = True
t.start()
def call(jobs):
with transaction.atomic():
for job in jobs:
with job.JobContext():
objectsDirectory = job.args[1]
transferName = job.args[2]
transferUUID = job.args[3]
processingDirectory = job.args[4]
autoProcessSIPDirectory = job.args[5]
sharedPath = job.argv[6]
transfer_objects_directory = '%transferDirectory%objects'
for container in os.listdir(objectsDirectory):
sipUUID = uuid.uuid4().__str__()
containerPath = os.path.join(objectsDirectory, container)
if not os.path.isdir(containerPath):
job.pyprint("file (not container) found: ", container, file=sys.stderr)
continue
sipName = "%s-%s" % (transferName, container)
tmpSIPDir = os.path.join(processingDirectory, sipName) + "/"
destSIPDir = os.path.join(autoProcessSIPDirectory, sipName) + "/"
archivematicaFunctions.create_structured_directory(tmpSIPDir, manual_normalization=True)
databaseFunctions.createSIP(destSIPDir.replace(sharedPath, '%sharedPath%'), sipUUID, printfn=job.pyprint)
# move the objects to the SIPDir
for item in os.listdir(containerPath):
shutil.move(os.path.join(containerPath, item), os.path.join(tmpSIPDir, "objects", item))
# get the database list of files in the objects directory
# for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP'
directory = os.path.join(transfer_objects_directory, container)
files = File.objects.filter(removedtime__isnull=True,
currentlocation__startswith=directory,
transfer_id=transferUUID)
for f in files:
currentPath = databaseFunctions.deUnicode(f.currentlocation).replace(directory, transfer_objects_directory)
currentSIPFilePath = currentPath.replace("%transferDirectory%", tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
f.currentlocation = currentPath.replace("%transferDirectory%", "%SIPDirectory%")
f.sip_id = sipUUID
f.save()
else:
job.pyprint("file not found: ", currentSIPFilePath, file=sys.stderr)
# moveSIPTo autoProcessSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
commandReplacementDic = fileUnit.getReplacementDic()
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
for key in SIPReplacementDic.iterkeys():
value = SIPReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
t = threading.Thread(target=task.performTask)
t.daemon = True
while(archivematicaMCP.limitTaskThreads <= threading.activeCount()):
#print "Waiting for active threads", threading.activeCount()
self.tasksLock.release()
time.sleep(archivematicaMCP.limitTaskThreadsSleep)
self.tasksLock.acquire()
print "Active threads:", threading.activeCount()
t.start()
self.clearToNextLink = True
self.tasksLock.release()
if self.tasks == {} :
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
SIPUUID = unit.owningUnit.UUID
sql = """SELECT variableValue FROM UnitVariables WHERE unitType = 'SIP' AND variable = 'normalizationFileIdentificationToolIdentifierTypes' AND unitUUID = '%s';""" % (SIPUUID)
rows = databaseInterface.queryAllSQL(sql)
if len(rows):
fileIdentificationRestriction = rows[0][0]
else:
fileIdentificationRestriction = None
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
print "skipping file", file, filterSubDir, " : \t Doesn't start with: ", unit.pathString + filterSubDir
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
fileUUID = unit.UUID
ComandClassification = self.execute
#passVar=self.jobChainLink.passVar
toPassVar = eval(arguments)
toPassVar.update({"%standardErrorFile%":standardErrorFile, "%standardOutputFile%":standardOutputFile, '%commandClassifications%':ComandClassification})
#print "debug", toPassVar, toPassVar['%normalizeFileGrpUse%'], unit.fileGrpUse
passVar=replacementDic(toPassVar)
if toPassVar['%normalizeFileGrpUse%'] != unit.fileGrpUse or self.alreadyNormalizedManually(unit, ComandClassification):
#print "debug: ", unit.currentPath, unit.fileGrpUse
self.jobChainLink.linkProcessingComplete(self.exitCode, passVar=self.jobChainLink.passVar)
else:
taskType = databaseInterface.queryAllSQL("SELECT pk FROM TaskTypes WHERE description = '%s';" % ("Transcoder task type"))[0][0]
#find out if fileIDType superseded
sql = """SELECT MicroServiceChainLinks.pk, CommandRelationships.pk, CommandRelationships.command FROM FilesIdentifiedIDs JOIN FileIDs ON FilesIdentifiedIDs.fileID = FileIDs.pk JOIN FileIDTypes ON FileIDs.fileIDType = FileIDTypes.pk JOIN CommandRelationships ON FilesIdentifiedIDs.fileID = CommandRelationships.fileID JOIN CommandClassifications ON CommandClassifications.pk = CommandRelationships.commandClassification JOIN TasksConfigs ON TasksConfigs.taskTypePKReference = CommandRelationships.pk JOIN MicroServiceChainLinks ON MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE TasksConfigs.taskType = '%s' AND FilesIdentifiedIDs.fileUUID = '%s' AND CommandClassifications.classification = '%s' AND (%s) AND CommandRelationships.enabled = TRUE AND CommandClassifications.enabled = TRUE AND FileIDTypes.enabled = TRUE GROUP BY MicroServiceChainLinks.pk;""" % (taskType, fileUUID, ComandClassification, "FileIDTypes.pk = 'ceff7e8c-3e78-4d51-9654-eb72efd28777'")
rows = databaseInterface.queryAllSQL(sql)
#-not superseded check
if not len(rows):
if fileIdentificationRestriction:
sql = """SELECT MicroServiceChainLinks.pk, CommandRelationships.pk, CommandRelationships.command FROM FilesIdentifiedIDs JOIN FileIDs ON FilesIdentifiedIDs.fileID = FileIDs.pk JOIN FileIDTypes ON FileIDs.fileIDType = FileIDTypes.pk JOIN CommandRelationships ON FilesIdentifiedIDs.fileID = CommandRelationships.fileID JOIN CommandClassifications ON CommandClassifications.pk = CommandRelationships.commandClassification JOIN TasksConfigs ON TasksConfigs.taskTypePKReference = CommandRelationships.pk JOIN MicroServiceChainLinks ON MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE TasksConfigs.taskType = '%s' AND FilesIdentifiedIDs.fileUUID = '%s' AND CommandClassifications.classification = '%s' AND (%s) AND CommandRelationships.enabled = TRUE AND CommandClassifications.enabled = TRUE AND FileIDTypes.enabled = TRUE GROUP BY MicroServiceChainLinks.pk;""" % (taskType, fileUUID, ComandClassification, fileIdentificationRestriction)
else:
sql = """SELECT MicroServiceChainLinks.pk, CommandRelationships.pk, CommandRelationships.command FROM FilesIdentifiedIDs JOIN CommandRelationships ON FilesIdentifiedIDs.fileID = CommandRelationships.fileID JOIN CommandClassifications ON CommandClassifications.pk = CommandRelationships.commandClassification JOIN TasksConfigs ON TasksConfigs.taskTypePKReference = CommandRelationships.pk JOIN MicroServiceChainLinks ON MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE TasksConfigs.taskType = '%s' AND FilesIdentifiedIDs.fileUUID = '%s' AND CommandClassifications.classification = '%s' AND CommandRelationships.enabled = TRUE AND CommandClassifications.enabled = TRUE GROUP BY MicroServiceChainLinks.pk;""" % (taskType, fileUUID, ComandClassification)
rows = databaseInterface.queryAllSQL(sql)
commandsRun={}
#if the rule is defined
if rows and len(rows):
for row in rows:
microServiceChainLink, commandRelationship, command = row
if command in commandsRun:
link = commandsRun[command]
sql = """SELECT exitCode FROM Tasks JOIN Jobs ON Jobs.jobUUID = Tasks.jobUUID WHERE Tasks.jobUUID IN (SELECT jobUUID FROM Jobs WHERE subJobOf = '%s') AND Jobs.MicroServiceChainLinksPK = '%s';""" % (self.jobChainLink.UUID, link)
rows = databaseInterface.queryAllSQL(sql)
if len(rows) != 1:
print sys.stderr, "Bad query:", sql
for row in rows:
ret = row[0]
sql = "UPDATE CommandRelationships SET countAttempts=countAttempts+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
if ret:
column = "countNotOK"
else:
column = "countOK"
sql = "UPDATE CommandRelationships SET " + column + "=" + column + "+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
else:
commandsRun[command] = microServiceChainLink
jobChainLink.jobChain.nextChainLink(row[0], passVar=passVar, incrementLinkSplit=True, subJobOf=self.jobChainLink.UUID)
#no rule defined, use default
else:
sql = """SELECT MicroserviceChainLink, CommandRelationships.pk, CommandRelationships.command FROM DefaultCommandsForClassifications JOIN MicroServiceChainLinks ON MicroServiceChainLinks.pk = DefaultCommandsForClassifications.MicroserviceChainLink JOIN TasksConfigs ON TasksConfigs.pk = MicroServiceChainLinks.currentTask JOIN CommandRelationships ON CommandRelationships.pk = TasksConfigs.taskTypePKReference JOIN CommandClassifications ON CommandClassifications.pk = DefaultCommandsForClassifications.forClassification WHERE TasksConfigs.taskType = '5e70152a-9c5b-4c17-b823-c9298c546eeb' AND CommandClassifications.classification = '%s' AND DefaultCommandsForClassifications.enabled = TRUE;""" % (ComandClassification)
rows = databaseInterface.queryAllSQL(sql)
for row in rows:
microServiceChainLink, commandRelationship, command = row
if command in commandsRun:
link = commandsRun[command]
sql = """SELECT exitCode FROM Tasks JOIN Jobs ON Jobs.jobUUID = Tasks.jobUUID WHERE Tasks.jobUUID IN (SELECT jobUUID FROM Jobs WHERE subJobOf = '%s') AND Jobs.MicroServiceChainLinksPK = '%s';""" % (self.jobChainLink.UUID, link)
rows = databaseInterface.queryAllSQL(sql)
if len(rows) != 1:
print sys.stderr, "Bad query:", sql
for row in rows:
ret = row[0]
sql = "UPDATE CommandRelationships SET countAttempts=countAttempts+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
if ret:
column = "countNotOK"
else:
column = "countOK"
sql = "UPDATE CommandRelationships SET " + column + "=" + column + "+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
else:
jobChainLink.jobChain.nextChainLink(microServiceChainLink, passVar=passVar, incrementLinkSplit=True, subJobOf=self.jobChainLink.UUID)
commandsRun[command] = microServiceChainLink
self.jobChainLink.linkProcessingComplete(self.exitCode, passVar=self.jobChainLink.passVar)
def process_transfer(request, transfer_uuid):
response = {}
if request.user.id:
# get transfer info
transfer = models.Transfer.objects.get(uuid=transfer_uuid)
transfer_path = transfer.currentlocation.replace(
'%sharedPath%',
helpers.get_server_config_value('sharedDirectory')
)
import MySQLdb
import databaseInterface
import databaseFunctions
import shutil
from archivematicaCreateStructuredDirectory import createStructuredDirectory
from archivematicaCreateStructuredDirectory import createManualNormalizedDirectoriesList
createStructuredDirectory(transfer_path, createManualNormalizedDirectories=False)
processingDirectory = helpers.get_server_config_value('processingDirectory')
transfer_directory_name = os.path.basename(transfer_path[:-1])
transfer_name = transfer_directory_name[:-37]
sharedPath = helpers.get_server_config_value('sharedDirectory')
tmpSIPDir = os.path.join(processingDirectory, transfer_name) + "/"
#processSIPDirectory = os.path.join(sharedPath, 'watchedDirectories/system/autoProcessSIP') + '/'
processSIPDirectory = os.path.join(sharedPath, 'watchedDirectories/SIPCreation/SIPsUnderConstruction') + '/'
#destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/"
#destSIPDir = os.path.join(processSIPDirectory, transfer_name + '-' + ) + "/"
createStructuredDirectory(tmpSIPDir, createManualNormalizedDirectories=False)
objectsDirectory = os.path.join(transfer_path, 'objects') + '/'
"""
#create row in SIPs table if one doesn't already exist
lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%')
#lookup_path = '%sharedPath%watchedDirectories/workFlowDecisions/createDip/' + transfer_name + '/'
sql = " " "SELECT sipUUID FROM SIPs WHERE currentPath = '" " " + MySQLdb.escape_string(lookup_path) + "';"
rows = databaseInterface.queryAllSQL(sql)
if len(rows) > 0:
row = rows[0]
sipUUID = row[0]
else:
sipUUID = uuid.uuid4().__str__()
databaseFunctions.createSIP(lookup_path, sipUUID)
"""
sipUUID = uuid.uuid4().__str__()
destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/"
lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%')
databaseFunctions.createSIP(lookup_path, sipUUID)
#move the objects to the SIPDir
for item in os.listdir(objectsDirectory):
shutil.move(os.path.join(objectsDirectory, item), os.path.join(tmpSIPDir, "objects", item))
#get the database list of files in the objects directory
#for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP'
sql = """SELECT fileUUID, currentLocation FROM Files WHERE removedTime = 0 AND currentLocation LIKE '\%transferDirectory\%objects%' AND transferUUID = '""" + transfer_uuid + "'"
for row in databaseInterface.queryAllSQL(sql):
fileUUID = row[0]
currentPath = databaseFunctions.deUnicode(row[1])
currentSIPFilePath = currentPath.replace("%transferDirectory%", tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
sql = """UPDATE Files SET currentLocation='%s', sipUUID='%s' WHERE fileUUID='%s'""" % (MySQLdb.escape_string(currentPath.replace("%transferDirectory%", "%SIPDirectory%")), sipUUID, fileUUID)
databaseInterface.runSQL(sql)
else:
print >>sys.stderr, "file not found: ", currentSIPFilePath
#copy processingMCP.xml file
src = os.path.join(os.path.dirname(objectsDirectory[:-1]), "processingMCP.xml")
dst = os.path.join(tmpSIPDir, "processingMCP.xml")
shutil.copy(src, dst)
#moveSIPTo processSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
elasticSearchFunctions.connect_and_change_transfer_file_status(transfer_uuid, '')
response['message'] = 'SIP ' + sipUUID + ' created.'
else:
response['error'] = True
response['message'] = 'Must be logged in.'
return HttpResponse(
simplejson.JSONEncoder(encoding='utf-8').encode(response),
mimetype='application/json'
)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (
pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(
execute, arguments, standardOutputFile,
standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
execute, arguments, standardOutputFile,
standardErrorFile)
commandReplacementDic = fileUnit.getReplacementDic()
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
for key in SIPReplacementDic.iterkeys():
value = SIPReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
task = taskStandard(self,
execute,
arguments,
standardOutputFile,
standardErrorFile,
outputLock=outputLock,
UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic,
UUID, arguments)
t = threading.Thread(target=task.performTask)
t.daemon = True
while (archivematicaMCP.limitTaskThreads <=
threading.activeCount()):
#print "Waiting for active threads", threading.activeCount()
self.tasksLock.release()
time.sleep(archivematicaMCP.limitTaskThreadsSleep)
self.tasksLock.acquire()
print "Active threads:", threading.activeCount()
t.start()
self.clearToNextLink = True
self.tasksLock.release()
if self.tasks == {}:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT filterSubDir, execute FROM TasksConfigsStartLinkForEachFile where pk = '%s'""" % (
pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = "" #deUnicode(row[1])
filterFileStart = "" #deUnicode(row[2])
filterSubDir = deUnicode(row[0])
requiresOutputLock = "" #row[4]
self.standardOutputFile = "" #deUnicode(row[5])
self.standardErrorFile = "" #deUnicode(row[6])
self.execute = deUnicode(row[1])
self.arguments = "" #deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
print "skipping file", file, filterSubDir, " : \t Doesn't start with: ", unit.pathString + filterSubDir
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
execute, arguments, standardOutputFile,
standardErrorFile)
commandReplacementDic = fileUnit.getReplacementDic()
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
for key in SIPReplacementDic.iterkeys():
value = SIPReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
self.tasks[UUID] = None
## passVar = [{preservationJobUUID, accessJobUUID, thumbnailsJobUUID}] #an idea not in use
t = threading.Thread(target=jobChain.jobChain,
args=(
fileUnit,
execute,
self.taskCompletedCallBackFunction,
),
kwargs={
"passVar": self.jobChainLink.passVar,
"UUID": UUID,
"subJobOf":
self.jobChainLink.UUID.__str__()
})
t.daemon = True
while (archivematicaMCP.limitTaskThreads / 2 <=
threading.activeCount()):
#print "Waiting for active threads", threading.activeCount()
self.tasksLock.release()
time.sleep(archivematicaMCP.limitTaskThreadsSleep)
self.tasksLock.acquire()
print "Active threads:", threading.activeCount()
t.start()
self.clearToNextLink = True
self.tasksLock.release()
if self.tasks == {}:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = """ + pk.__str__()
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
self.tasksLock.acquire()
print "Debug - ", unit.fileList.items()
for file, fileUnit in unit.fileList.items():
# print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
# print "file", file, type(file)
# print unit.pathString, type(unit.pathString)
# filterSubDir = filterSubDir.encode('utf-8')
# print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
print "skipping file", file, filterSubDir
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
execute, arguments, standardOutputFile, standardErrorFile
)
commandReplacementDic = fileUnit.getReplacementDic()
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace('"', ('\\"'))
# print "key", type(key), key
# print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
# key = key.encode("utf-8")
# value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
for key in SIPReplacementDic.iterkeys():
value = SIPReplacementDic[key].replace('"', ('\\"'))
# print "key", type(key), key
# print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
# key = key.encode("utf-8")
# value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
self.tasks[UUID] = None
## passVar = [{preservationJobUUID, accessJobUUID, thumbnailsJobUUID}] #an idea not in use
t = threading.Thread(
target=jobChain.jobChain,
args=(fileUnit, execute, self.taskCompletedCallBackFunction),
kwargs={
"passVar": self.jobChainLink.passVar,
"UUID": UUID,
"subJobOf": self.jobChainLink.UUID.__str__(),
},
)
t.daemon = True
while archivematicaMCP.limitTaskThreads / 2 <= threading.activeCount():
# print "Waiting for active threads", threading.activeCount()
self.tasksLock.release()
time.sleep(archivematicaMCP.limitTaskThreadsSleep)
self.tasksLock.acquire()
print "Active threads:", threading.activeCount()
t.start()
self.clearToNextLink = True
self.tasksLock.release()
if self.tasks == {}:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def process_transfer(request, transfer_uuid):
response = {}
if request.user.id:
# get transfer info
transfer = models.Transfer.objects.get(uuid=transfer_uuid)
transfer_path = transfer.currentlocation.replace(
'%sharedPath%', helpers.get_server_config_value('sharedDirectory'))
import MySQLdb
import databaseInterface
import databaseFunctions
import shutil
from archivematicaCreateStructuredDirectory import createStructuredDirectory
from archivematicaCreateStructuredDirectory import createManualNormalizedDirectoriesList
createStructuredDirectory(transfer_path,
createManualNormalizedDirectories=False)
processingDirectory = helpers.get_server_config_value(
'processingDirectory')
transfer_directory_name = os.path.basename(transfer_path[:-1])
transfer_name = transfer_directory_name[:-37]
sharedPath = helpers.get_server_config_value('sharedDirectory')
tmpSIPDir = os.path.join(processingDirectory, transfer_name) + "/"
#processSIPDirectory = os.path.join(sharedPath, 'watchedDirectories/system/autoProcessSIP') + '/'
processSIPDirectory = os.path.join(
sharedPath,
'watchedDirectories/SIPCreation/SIPsUnderConstruction') + '/'
#destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/"
#destSIPDir = os.path.join(processSIPDirectory, transfer_name + '-' + ) + "/"
createStructuredDirectory(tmpSIPDir,
createManualNormalizedDirectories=False)
objectsDirectory = os.path.join(transfer_path, 'objects') + '/'
"""
#create row in SIPs table if one doesn't already exist
lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%')
#lookup_path = '%sharedPath%watchedDirectories/workFlowDecisions/createDip/' + transfer_name + '/'
sql = " " "SELECT sipUUID FROM SIPs WHERE currentPath = '" " " + MySQLdb.escape_string(lookup_path) + "';"
rows = databaseInterface.queryAllSQL(sql)
if len(rows) > 0:
row = rows[0]
sipUUID = row[0]
else:
sipUUID = uuid.uuid4().__str__()
databaseFunctions.createSIP(lookup_path, sipUUID)
"""
sipUUID = uuid.uuid4().__str__()
destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/"
lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%')
databaseFunctions.createSIP(lookup_path, sipUUID)
#move the objects to the SIPDir
for item in os.listdir(objectsDirectory):
shutil.move(os.path.join(objectsDirectory, item),
os.path.join(tmpSIPDir, "objects", item))
#get the database list of files in the objects directory
#for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP'
sql = """SELECT fileUUID, currentLocation FROM Files WHERE removedTime = 0 AND currentLocation LIKE '\%transferDirectory\%objects%' AND transferUUID = '""" + transfer_uuid + "'"
for row in databaseInterface.queryAllSQL(sql):
fileUUID = row[0]
currentPath = databaseFunctions.deUnicode(row[1])
currentSIPFilePath = currentPath.replace("%transferDirectory%",
tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
sql = """UPDATE Files SET currentLocation='%s', sipUUID='%s' WHERE fileUUID='%s'""" % (
MySQLdb.escape_string(
currentPath.replace(
"%transferDirectory%",
"%SIPDirectory%")), sipUUID, fileUUID)
databaseInterface.runSQL(sql)
else:
print >> sys.stderr, "file not found: ", currentSIPFilePath
#copy processingMCP.xml file
src = os.path.join(os.path.dirname(objectsDirectory[:-1]),
"processingMCP.xml")
dst = os.path.join(tmpSIPDir, "processingMCP.xml")
shutil.copy(src, dst)
#moveSIPTo processSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
elasticSearchFunctions.connect_and_change_transfer_file_status(
transfer_uuid, '')
response['message'] = 'SIP ' + sipUUID + ' created.'
else:
response['error'] = True
response['message'] = 'Must be logged in.'
return HttpResponse(
simplejson.JSONEncoder(encoding='utf-8').encode(response),
mimetype='application/json')
def call(jobs):
with transaction.atomic():
for job in jobs:
with job.JobContext():
objectsDirectory = job.args[1]
transferName = job.args[2]
transferUUID = job.args[3]
processingDirectory = job.args[4]
autoProcessSIPDirectory = job.args[5]
sharedPath = job.args[6]
sipName = transferName
tmpSIPDir = os.path.join(processingDirectory, sipName) + "/"
destSIPDir = os.path.join(autoProcessSIPDirectory,
sipName) + "/"
archivematicaFunctions.create_structured_directory(
tmpSIPDir, manual_normalization=False)
# If transfer is a reingested AIP, then pass that info to the SIP
sip_type = 'SIP'
sip_uuid = None
transfer = Transfer.objects.get(uuid=transferUUID)
if transfer.type == 'Archivematica AIP':
sip_type = 'AIP-REIN'
# Use reingested AIP's UUID as the SIP UUID
# Get AIP UUID from reingest METS name
job.pyprint(
'path', os.path.join(objectsDirectory, '..',
'metadata'), 'listdir',
os.listdir(
os.path.join(objectsDirectory, '..', 'metadata')))
for item in os.listdir(
os.path.join(objectsDirectory, '..', 'metadata')):
if item.startswith('METS'):
sip_uuid = item.replace('METS.',
'').replace('.xml', '')
job.pyprint('sip_uuid', sip_uuid)
job.pyprint('sip_type', sip_type)
# Find out if any ``Directory`` models were created for the source
# ``Transfer``. If so, this fact gets recorded in the new ``SIP`` model.
dir_mdls = Directory.objects.filter(
transfer_id=transferUUID,
currentlocation__startswith='%transferDirectory%objects')
diruuids = len(dir_mdls) > 0
# Create row in SIPs table if one doesn't already exist
lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%')
try:
sip = SIP.objects.get(currentpath=lookup_path).uuid
if diruuids:
sip.diruuids = True
sip.save()
except SIP.DoesNotExist:
sip_uuid = databaseFunctions.createSIP(lookup_path,
UUID=sip_uuid,
sip_type=sip_type,
diruuids=diruuids,
printfn=job.pyprint)
sip = SIP.objects.get(uuid=sip_uuid)
# Move the objects to the SIPDir
for item in os.listdir(objectsDirectory):
src_path = os.path.join(objectsDirectory, item)
dst_path = os.path.join(tmpSIPDir, "objects", item)
# If dst_path already exists and is a directory, shutil.move
# will move src_path into it rather than overwriting it;
# to avoid incorrectly-nested paths, move src_path's contents
# into it instead.
if os.path.exists(dst_path) and os.path.isdir(src_path):
for subitem in os.listdir(src_path):
shutil.move(os.path.join(src_path, subitem),
dst_path)
else:
shutil.move(src_path, dst_path)
# Get the ``Directory`` models representing the subdirectories in the
# objects/ directory. For each subdirectory, confirm it's in the SIP
# objects/ directory, and update the current location and owning SIP.
for dir_mdl in dir_mdls:
currentPath = databaseFunctions.deUnicode(
dir_mdl.currentlocation)
currentSIPDirPath = currentPath.replace(
"%transferDirectory%", tmpSIPDir)
if os.path.isdir(currentSIPDirPath):
dir_mdl.currentlocation = currentPath.replace(
"%transferDirectory%", "%SIPDirectory%")
dir_mdl.sip = sip
dir_mdl.save()
else:
job.pyprint("directory not found: ",
currentSIPDirPath,
file=sys.stderr)
# Get the database list of files in the objects directory.
# For each file, confirm it's in the SIP objects directory, and update the
# current location/ owning SIP'
files = File.objects.filter(
transfer_id=transferUUID,
currentlocation__startswith='%transferDirectory%objects',
removedtime__isnull=True)
for f in files:
currentPath = databaseFunctions.deUnicode(
f.currentlocation)
currentSIPFilePath = currentPath.replace(
"%transferDirectory%", tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
f.currentlocation = currentPath.replace(
"%transferDirectory%", "%SIPDirectory%")
f.sip = sip
f.save()
else:
job.pyprint("file not found: ",
currentSIPFilePath,
file=sys.stderr)
archivematicaFunctions.create_directories(
archivematicaFunctions.MANUAL_NORMALIZATION_DIRECTORIES,
basepath=tmpSIPDir)
# Copy the JSON metadata file, if present; this contains a
# serialized copy of DC metadata entered in the dashboard UI
# during the transfer.
src = os.path.normpath(
os.path.join(objectsDirectory, "..", "metadata",
"dc.json"))
dst = os.path.join(tmpSIPDir, "metadata", "dc.json")
if os.path.exists(src):
shutil.copy(src, dst)
# Copy processingMCP.xml file
src = os.path.join(os.path.dirname(objectsDirectory[:-1]),
"processingMCP.xml")
dst = os.path.join(tmpSIPDir, "processingMCP.xml")
shutil.copy(src, dst)
# moveSIPTo autoProcessSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = """ + pk.__str__()
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
print "skipping file", file, filterSubDir
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
fileUUID = unit.UUID
ComandClassification = self.execute
#passVar=self.jobChainLink.passVar
toPassVar = eval(arguments)
toPassVar.update({"%standardErrorFile%":standardErrorFile, "%standardOutputFile%":standardOutputFile, '%commandClassifications%':ComandClassification})
print "debug", toPassVar
passVar=replacementDic(toPassVar)
sql = """SELECT MicroServiceChainLinks.pk FROM FilesIdentifiedIDs JOIN CommandRelationships ON FilesIdentifiedIDs.fileID = CommandRelationships.fileID JOIN CommandClassifications ON CommandClassifications.pk = CommandRelationships.commandClassification JOIN TasksConfigs ON TasksConfigs.taskTypePKReference = CommandRelationships.pk JOIN MicroServiceChainLinks ON MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE TasksConfigs.taskType = 8 AND FilesIdentifiedIDs.fileUUID = '%s' AND CommandClassifications.classification = '%s';""" % (fileUUID, ComandClassification)
rows = databaseInterface.queryAllSQL(sql)
if rows and len(rows):
print "DEBUGGING 6772: ", fileUUID, ComandClassification, rows
for row in rows:
jobChainLink.jobChain.nextChainLink(row[0], passVar=passVar, incrementLinkSplit=True, subJobOf=self.jobChainLink.UUID)
else:
sql = """SELECT MicroserviceChainLink FROM DefaultCommandsForClassifications JOIN CommandClassifications ON CommandClassifications.pk = DefaultCommandsForClassifications.forClassification WHERE CommandClassifications.classification = '%s'""" % (ComandClassification)
rows = databaseInterface.queryAllSQL(sql)
print "DEBUGGING2 6772: ", fileUUID, ComandClassification, rows
for row in rows:
jobChainLink.jobChain.nextChainLink(row[0], passVar=passVar, incrementLinkSplit=True, subJobOf=self.jobChainLink.UUID)
self.jobChainLink.linkProcessingComplete(self.exitCode, passVar=self.jobChainLink.passVar)
if not os.path.isdir(containerPath):
print >>sys.stderr, "file (not container) found: ", container
continue
sipName = "%s-%s" % (transferName, container)
tmpSIPDir = os.path.join(processingDirectory, sipName) + "/"
destSIPDir = os.path.join(autoProcessSIPDirectory, sipName) + "/"
createStructuredDirectory(tmpSIPDir, createManualNormalizedDirectories=True)
databaseFunctions.createSIP(destSIPDir.replace(sharedPath, '%sharedPath%'), sipUUID)
#move the objects to the SIPDir
for item in os.listdir(containerPath):
shutil.move(os.path.join(containerPath, item), os.path.join(tmpSIPDir, "objects", item))
#get the database list of files in the objects directory
#for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP'
sql = """SELECT fileUUID, currentLocation FROM Files WHERE removedTime = 0 AND currentLocation LIKE '\%transferDirectory\%objects/""" + container + """/%' AND transferUUID = '""" + transferUUID + "'"
for row in databaseInterface.queryAllSQL(sql):
fileUUID = row[0]
currentPath = databaseFunctions.deUnicode(row[1]).replace('%transferDirectory%objects/' + container, '%transferDirectory%objects')
currentSIPFilePath = currentPath.replace("%transferDirectory%", tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
sql = """UPDATE Files SET currentLocation='%s', sipUUID='%s' WHERE fileUUID='%s'""" % (MySQLdb.escape_string(currentPath.replace("%transferDirectory%", "%SIPDirectory%")), sipUUID, fileUUID)
databaseInterface.runSQL(sql)
else:
print >>sys.stderr, "file not found: ", currentSIPFilePath
#moveSIPTo autoProcessSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
sharedPath = sys.argv[6]
sipName = transferName
sipUUID = uuid.uuid4().__str__()
tmpSIPDir = os.path.join(processingDirectory, sipName) + "/"
destSIPDir = os.path.join(autoProcessSIPDirectory, sipName) + "/"
createStructuredDirectory(tmpSIPDir)
databaseFunctions.createSIP(destSIPDir.replace(sharedPath, '%sharedPath%'), sipUUID)
#move the objects to the SIPDir
for item in os.listdir(objectsDirectory):
shutil.move(os.path.join(objectsDirectory, item), os.path.join(tmpSIPDir, "objects", item))
#get the database list of files in the objects directory
#for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP'
sql = """SELECT fileUUID, currentLocation FROM Files WHERE removedTime = 0 AND currentLocation LIKE '\%transferDirectory\%objects%' AND transferUUID = '""" + transferUUID + "'"
for row in databaseInterface.queryAllSQL(sql):
fileUUID = row[0]
currentPath = databaseFunctions.deUnicode(row[1])
currentSIPFilePath = currentPath.replace("%transferDirectory%", tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
sql = """UPDATE Files SET currentLocation='%s', sipUUID='%s' WHERE fileUUID='%s'""" % (MySQLdb.escape_string(currentPath.replace("%transferDirectory%", "%SIPDirectory%")), sipUUID, fileUUID)
databaseInterface.runSQL(sql)
else:
print >>sys.stderr, "file not found: ", currentSIPFilePath
#moveSIPTo autoProcessSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
def call(jobs):
with transaction.atomic():
for job in jobs:
with job.JobContext():
objectsDirectory = job.args[1]
transferName = job.args[2]
transferUUID = job.args[3]
processingDirectory = job.args[4]
autoProcessSIPDirectory = job.args[5]
sharedPath = job.args[6]
sipName = transferName
tmpSIPDir = os.path.join(processingDirectory, sipName) + "/"
destSIPDir = os.path.join(autoProcessSIPDirectory,
sipName) + "/"
archivematicaFunctions.create_structured_directory(
tmpSIPDir, manual_normalization=False)
# If transfer is a reingested AIP, then pass that info to the SIP
sip_type = "SIP"
sip_uuid = None
transfer = Transfer.objects.get(uuid=transferUUID)
if transfer.type == "Archivematica AIP":
sip_type = "AIP-REIN"
# Use reingested AIP's UUID as the SIP UUID
# Get AIP UUID from reingest METS name
job.pyprint(
"path",
os.path.join(objectsDirectory, "..", "metadata"),
"listdir",
os.listdir(
os.path.join(objectsDirectory, "..", "metadata")),
)
for item in os.listdir(
os.path.join(objectsDirectory, "..", "metadata")):
if item.startswith("METS"):
sip_uuid = item.replace("METS.",
"").replace(".xml", "")
job.pyprint("sip_uuid", sip_uuid)
job.pyprint("sip_type", sip_type)
# Find out if any ``Directory`` models were created for the source
# ``Transfer``. If so, this fact gets recorded in the new ``SIP`` model.
dir_mdls = Directory.objects.filter(
transfer_id=transferUUID,
currentlocation__startswith="%transferDirectory%objects",
)
diruuids = len(dir_mdls) > 0
# Create row in SIPs table if one doesn't already exist
lookup_path = destSIPDir.replace(sharedPath, "%sharedPath%")
try:
sip = SIP.objects.get(currentpath=lookup_path)
if diruuids:
sip.diruuids = True
sip.save()
except SIP.DoesNotExist:
sip_uuid = databaseFunctions.createSIP(
lookup_path,
UUID=sip_uuid,
sip_type=sip_type,
diruuids=diruuids,
printfn=job.pyprint,
)
sip = SIP.objects.get(uuid=sip_uuid)
# Set activeAgent using the value in Transfer. This ensures
# that events generated in Ingest can fall to this value in
# scenarios where the processing config does not require user
# interfactions, e.g. in the "automated" processing config.
try:
unit_variable = UnitVariable.objects.get(
unittype="Transfer",
unituuid=transferUUID,
variable="activeAgent",
)
except UnitVariable.DoesNotExist:
unit_variable = None
if unit_variable:
try:
agent = Agent.objects.get(
id=unit_variable.variablevalue)
except Agent.DoesNotExist:
pass
else:
sip.update_active_agent(agent.userprofile.user_id)
# Move the objects to the SIPDir
for item in os.listdir(objectsDirectory):
src_path = os.path.join(objectsDirectory, item)
dst_path = os.path.join(tmpSIPDir, "objects", item)
# If dst_path already exists and is a directory, shutil.move
# will move src_path into it rather than overwriting it;
# to avoid incorrectly-nested paths, move src_path's contents
# into it instead.
if os.path.exists(dst_path) and os.path.isdir(src_path):
for subitem in os.listdir(src_path):
shutil.move(os.path.join(src_path, subitem),
dst_path)
else:
shutil.move(src_path, dst_path)
# Get the ``Directory`` models representing the subdirectories in the
# objects/ directory. For each subdirectory, confirm it's in the SIP
# objects/ directory, and update the current location and owning SIP.
for dir_mdl in dir_mdls:
currentPath = databaseFunctions.deUnicode(
dir_mdl.currentlocation)
currentSIPDirPath = currentPath.replace(
"%transferDirectory%", tmpSIPDir)
if os.path.isdir(currentSIPDirPath):
dir_mdl.currentlocation = currentPath.replace(
"%transferDirectory%", "%SIPDirectory%")
dir_mdl.sip = sip
dir_mdl.save()
else:
job.pyprint("directory not found: ",
currentSIPDirPath,
file=sys.stderr)
# Get the database list of files in the objects directory.
# For each file, confirm it's in the SIP objects directory, and update the
# current location/ owning SIP'
files = File.objects.filter(
transfer_id=transferUUID,
currentlocation__startswith="%transferDirectory%objects",
removedtime__isnull=True,
)
for f in files:
currentPath = databaseFunctions.deUnicode(
f.currentlocation)
currentSIPFilePath = currentPath.replace(
"%transferDirectory%", tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
f.currentlocation = currentPath.replace(
"%transferDirectory%", "%SIPDirectory%")
f.sip = sip
f.save()
else:
job.pyprint("file not found: ",
currentSIPFilePath,
file=sys.stderr)
archivematicaFunctions.create_directories(
archivematicaFunctions.MANUAL_NORMALIZATION_DIRECTORIES,
basepath=tmpSIPDir,
)
# Copy the JSON metadata file, if present; this contains a
# serialized copy of DC metadata entered in the dashboard UI
# during the transfer.
src = os.path.normpath(
os.path.join(objectsDirectory, "..", "metadata",
"dc.json"))
dst = os.path.join(tmpSIPDir, "metadata", "dc.json")
if os.path.exists(src):
shutil.copy(src, dst)
# Copy processingMCP.xml file
src = os.path.join(os.path.dirname(objectsDirectory[:-1]),
"processingMCP.xml")
dst = os.path.join(tmpSIPDir, "processingMCP.xml")
shutil.copy(src, dst)
# moveSIPTo autoProcessSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
dst_path = os.path.join(tmpSIPDir, "objects", item)
# If dst_path already exists and is a directory, shutil.move
# will move src_path into it rather than overwriting it;
# to avoid incorrectly-nested paths, move src_path's contents
# into it instead.
if os.path.exists(dst_path) and os.path.isdir(src_path):
for subitem in os.listdir(src_path):
shutil.move(os.path.join(src_path, subitem), dst_path)
else:
shutil.move(src_path, dst_path)
# Get the ``Directory`` models representing the subdirectories in the
# objects/ directory. For each subdirectory, confirm it's in the SIP
# objects/ directory, and update the current location and owning SIP.
for dir_mdl in dir_mdls:
currentPath = databaseFunctions.deUnicode(dir_mdl.currentlocation)
currentSIPDirPath = currentPath.replace("%transferDirectory%",
tmpSIPDir)
if os.path.isdir(currentSIPDirPath):
dir_mdl.currentlocation = currentPath.replace(
"%transferDirectory%", "%SIPDirectory%")
dir_mdl.sip = sip
dir_mdl.save()
else:
print("directory not found: ", currentSIPDirPath, file=sys.stderr)
# Get the database list of files in the objects directory.
# For each file, confirm it's in the SIP objects directory, and update the
# current location/ owning SIP'
files = File.objects.filter(
transfer_id=transferUUID,
row = rows[0]
sipUUID = row[0]
else:
sipUUID = uuid.uuid4().__str__()
databaseFunctions.createSIP(lookup_path, sipUUID)
#move the objects to the SIPDir
for item in os.listdir(objectsDirectory):
shutil.move(os.path.join(objectsDirectory, item), os.path.join(tmpSIPDir, "objects", item))
#get the database list of files in the objects directory
#for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP'
sql = """SELECT fileUUID, currentLocation FROM Files WHERE removedTime = 0 AND currentLocation LIKE '\%transferDirectory\%objects%' AND transferUUID = '""" + transferUUID + "'"
for row in databaseInterface.queryAllSQL(sql):
fileUUID = row[0]
currentPath = databaseFunctions.deUnicode(row[1])
currentSIPFilePath = currentPath.replace("%transferDirectory%", tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
sql = """UPDATE Files SET currentLocation='%s', sipUUID='%s' WHERE fileUUID='%s'""" % (MySQLdb.escape_string(currentPath.replace("%transferDirectory%", "%SIPDirectory%")), sipUUID, fileUUID)
databaseInterface.runSQL(sql)
else:
print >>sys.stderr, "file not found: ", currentSIPFilePath
for directory in createManualNormalizedDirectoriesList:
path = os.path.join(tmpSIPDir, directory)
if not os.path.isdir(path):
os.makedirs(path)
#copy processingMCP.xml file
src = os.path.join(os.path.dirname(objectsDirectory[:-1]), "processingMCP.xml")
dst = os.path.join(tmpSIPDir, "processingMCP.xml")
databaseFunctions.createSIP(
destSIPDir.replace(sharedPath, '%sharedPath%'), sipUUID)
# move the objects to the SIPDir
for item in os.listdir(containerPath):
shutil.move(os.path.join(containerPath, item),
os.path.join(tmpSIPDir, "objects", item))
# get the database list of files in the objects directory
# for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP'
directory = os.path.join(transfer_objects_directory, container)
files = File.objects.filter(removedtime__isnull=True,
currentlocation__startswith=directory,
transfer_id=transferUUID)
for f in files:
currentPath = databaseFunctions.deUnicode(
f.currentlocation).replace(directory,
transfer_objects_directory)
currentSIPFilePath = currentPath.replace("%transferDirectory%",
tmpSIPDir)
if os.path.isfile(currentSIPFilePath):
f.currentlocation = currentPath.replace(
"%transferDirectory%", "%SIPDirectory%")
f.sip_id = sipUUID
f.save()
else:
print >> sys.stderr, "file not found: ", currentSIPFilePath
# moveSIPTo autoProcessSIPDirectory
shutil.move(tmpSIPDir, destSIPDir)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (
pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
SIPUUID = unit.owningUnit.UUID
sql = """SELECT variableValue FROM UnitVariables WHERE unitType = 'SIP' AND variable = 'normalizationFileIdentificationToolIdentifierTypes' AND unitUUID = '%s';""" % (
SIPUUID)
rows = databaseInterface.queryAllSQL(sql)
if len(rows):
fileIdentificationRestriction = rows[0][0]
else:
fileIdentificationRestriction = None
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
print "skipping file", file, filterSubDir, " : \t Doesn't start with: ", unit.pathString + filterSubDir
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
execute, arguments, standardOutputFile,
standardErrorFile)
fileUUID = unit.UUID
ComandClassification = self.execute
#passVar=self.jobChainLink.passVar
toPassVar = eval(arguments)
toPassVar.update({
"%standardErrorFile%": standardErrorFile,
"%standardOutputFile%": standardOutputFile,
'%commandClassifications%': ComandClassification
})
#print "debug", toPassVar, toPassVar['%normalizeFileGrpUse%'], unit.fileGrpUse
passVar = replacementDic(toPassVar)
if toPassVar[
'%normalizeFileGrpUse%'] != unit.fileGrpUse or self.alreadyNormalizedManually(
unit, ComandClassification):
#print "debug: ", unit.currentPath, unit.fileGrpUse
self.jobChainLink.linkProcessingComplete(
self.exitCode, passVar=self.jobChainLink.passVar)
else:
taskType = databaseInterface.queryAllSQL(
"SELECT pk FROM TaskTypes WHERE description = '%s';" %
("Transcoder task type"))[0][0]
if fileIdentificationRestriction:
sql = """SELECT MicroServiceChainLinks.pk, CommandRelationships.pk, CommandRelationships.command FROM FilesIdentifiedIDs JOIN FileIDs ON FilesIdentifiedIDs.fileID = FileIDs.pk JOIN FileIDTypes ON FileIDs.fileIDType = FileIDTypes.pk JOIN CommandRelationships ON FilesIdentifiedIDs.fileID = CommandRelationships.fileID JOIN CommandClassifications ON CommandClassifications.pk = CommandRelationships.commandClassification JOIN TasksConfigs ON TasksConfigs.taskTypePKReference = CommandRelationships.pk JOIN MicroServiceChainLinks ON MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE TasksConfigs.taskType = '%s' AND FilesIdentifiedIDs.fileUUID = '%s' AND CommandClassifications.classification = '%s' AND (%s) AND CommandRelationships.enabled = TRUE AND CommandClassifications.enabled = TRUE AND FileIDTypes.enabled = TRUE GROUP BY MicroServiceChainLinks.pk;""" % (
taskType, fileUUID, ComandClassification,
fileIdentificationRestriction)
else:
sql = """SELECT MicroServiceChainLinks.pk, CommandRelationships.pk, CommandRelationships.command FROM FilesIdentifiedIDs JOIN CommandRelationships ON FilesIdentifiedIDs.fileID = CommandRelationships.fileID JOIN CommandClassifications ON CommandClassifications.pk = CommandRelationships.commandClassification JOIN TasksConfigs ON TasksConfigs.taskTypePKReference = CommandRelationships.pk JOIN MicroServiceChainLinks ON MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE TasksConfigs.taskType = '%s' AND FilesIdentifiedIDs.fileUUID = '%s' AND CommandClassifications.classification = '%s' AND CommandRelationships.enabled = TRUE AND CommandClassifications.enabled = TRUE GROUP BY MicroServiceChainLinks.pk;""" % (
taskType, fileUUID, ComandClassification)
rows = databaseInterface.queryAllSQL(sql)
commandsRun = {}
if rows and len(rows):
for row in rows:
microServiceChainLink, commandRelationship, command = row
if command in commandsRun:
link = commandsRun[command]
sql = """SELECT exitCode FROM Tasks JOIN Jobs ON Jobs.jobUUID = Tasks.jobUUID WHERE Tasks.jobUUID IN (SELECT jobUUID FROM Jobs WHERE subJobOf = '%s') AND Jobs.MicroServiceChainLinksPK = '%s';""" % (
self.jobChainLink.UUID, link)
rows = databaseInterface.queryAllSQL(sql)
if len(rows) != 1:
print sys.stderr, "Bad query:", sql
for row in rows:
ret = row[0]
sql = "UPDATE CommandRelationships SET countAttempts=countAttempts+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
if ret:
column = "countNotOK"
else:
column = "countOK"
sql = "UPDATE CommandRelationships SET " + column + "=" + column + "+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
else:
commandsRun[command] = microServiceChainLink
jobChainLink.jobChain.nextChainLink(
row[0],
passVar=passVar,
incrementLinkSplit=True,
subJobOf=self.jobChainLink.UUID)
else:
sql = """SELECT MicroserviceChainLink, CommandRelationships.pk, CommandRelationships.command FROM DefaultCommandsForClassifications JOIN MicroServiceChainLinks ON MicroServiceChainLinks.pk = DefaultCommandsForClassifications.MicroserviceChainLink JOIN TasksConfigs ON TasksConfigs.pk = MicroServiceChainLinks.currentTask JOIN CommandRelationships ON CommandRelationships.pk = TasksConfigs.taskTypePKReference JOIN CommandClassifications ON CommandClassifications.pk = DefaultCommandsForClassifications.forClassification WHERE TasksConfigs.taskType = '5e70152a-9c5b-4c17-b823-c9298c546eeb' AND CommandClassifications.classification = '%s' AND DefaultCommandsForClassifications.enabled = TRUE;""" % (
ComandClassification)
rows = databaseInterface.queryAllSQL(sql)
for row in rows:
microServiceChainLink, commandRelationship, command = row
if command in commandsRun:
link = commandsRun[command]
sql = """SELECT exitCode FROM Tasks JOIN Jobs ON Jobs.jobUUID = Tasks.jobUUID WHERE Tasks.jobUUID IN (SELECT jobUUID FROM Jobs WHERE subJobOf = '%s') AND Jobs.MicroServiceChainLinksPK = '%s';""" % (
self.jobChainLink.UUID, link)
rows = databaseInterface.queryAllSQL(sql)
if len(rows) != 1:
print sys.stderr, "Bad query:", sql
for row in rows:
ret = row[0]
sql = "UPDATE CommandRelationships SET countAttempts=countAttempts+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
if ret:
column = "countNotOK"
else:
column = "countOK"
sql = "UPDATE CommandRelationships SET " + column + "=" + column + "+1 WHERE pk='" + commandRelationship + "';"
databaseInterface.runSQL(sql)
else:
jobChainLink.jobChain.nextChainLink(
microServiceChainLink,
passVar=passVar,
incrementLinkSplit=True,
subJobOf=self.jobChainLink.UUID)
commandsRun[command] = microServiceChainLink
self.jobChainLink.linkProcessingComplete(
self.exitCode, passVar=self.jobChainLink.passVar)
def __init__(self, jobChainLink, pk, unit):
global outputLock
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
opts = {
"inputFile": "%relativeLocation%",
"fileUUID": "%fileUUID%",
'commandClassifications': '%commandClassifications%',
"taskUUID": "%taskUUID%",
"objectsDirectory": "%SIPObjectsDirectory%",
"logsDirectory": "%SIPLogsDirectory%",
"sipUUID": "%SIPUUID%",
"sipPath": "%SIPDirectory%",
"fileGrpUse": "%fileGrpUse%",
"normalizeFileGrpUse": "%normalizeFileGrpUse%",
"excludeDirectory": "%excludeDirectory%",
"standardErrorFile": "%standardErrorFile%",
"standardOutputFile": "%standardOutputFile%"
}
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
for optsKey, optsValue in opts.iteritems():
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
opts[optsKey] = self.jobChainLink.passVar.replace(
opts[optsKey])[0]
commandReplacementDic = unit.getReplacementDic()
for key, value in commandReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
for key, value in SIPReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
commandReplacementDic = unit.getReplacementDic()
sql = """SELECT CommandRelationships.pk FROM CommandRelationships JOIN Commands ON CommandRelationships.command = Commands.pk WHERE CommandRelationships.pk = '%s';""" % (
pk.__str__())
rows = databaseInterface.queryAllSQL(sql)
taskCount = 0
tasksList = []
if rows:
self.tasksLock.acquire()
for row in rows:
UUID = uuid.uuid4().__str__()
opts["taskUUID"] = UUID
opts["CommandRelationship"] = pk.__str__()
execute = "transcoder_cr%s" % (pk)
deUnicode(execute)
arguments = row.__str__()
standardOutputFile = opts["standardOutputFile"]
standardErrorFile = opts["standardErrorFile"]
self.standardOutputFile = standardOutputFile
self.standardErrorFile = standardErrorFile
self.execute = execute
self.arguments = arguments
task = taskStandard(self,
execute,
opts,
standardOutputFile,
standardErrorFile,
outputLock=outputLock,
UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self,
commandReplacementDic,
UUID, arguments)
taskCount += 1
tasksList.append(task)
self.tasksLock.release()
for task in tasksList:
task.performTask()
else:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
self.tasks = []
self.pk = pk
self.jobChainLink = jobChainLink
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (
pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
if row == None:
print >> sys.stderr, "\nfind me\n"
traceback.print_exc(file=sys.stderr)
return None
while row != None:
print row
#pk = row[0]
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
self.requiresOutputLock = deUnicode(row[4])
standardOutputFile = deUnicode(row[5])
standardErrorFile = deUnicode(row[6])
execute = deUnicode(row[7])
self.execute = execute
arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
#if reloadFileList:
# unit.reloadFileList()
# "%taskUUID%": task.UUID.__str__(), \
if filterSubDir:
directory = os.path.join(unit.currentPath, filterSubDir)
else:
directory = unit.currentPath
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(
execute, arguments, standardOutputFile,
standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
execute, arguments, standardOutputFile, standardErrorFile)
commandReplacementDic = unit.getReplacementDic(directory)
#for each key replace all instances of the key in the command string
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
self.task = taskStandard(self,
execute,
arguments,
standardOutputFile,
standardErrorFile,
UUID=UUID)
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID,
arguments)
t = threading.Thread(target=self.task.performTask)
t.daemon = True
t.start()