def __init__(self, jobChainLink, pk, unit):
self.tasks = []
self.pk = pk
self.jobChainLink = jobChainLink
sql = """SELECT * FROM StandardTasksConfigs where pk = """ + pk.__str__()
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
print row
#pk = row[0]
filterFileEnd = row[1]
filterFileStart = row[2]
filterSubDir = row[3]
self.requiresOutputLock = row[4]
standardOutputFile = row[5]
standardErrorFile = row[6]
execute = row[7]
self.execute = execute
arguments = row[8]
row = c.fetchone()
sqlLock.release()
#if reloadFileList:
# unit.reloadFileList()
# "%taskUUID%": task.UUID.__str__(), \
if filterSubDir:
directory = os.path.join(unit.currentPath, filterSubDir)
else:
directory = unit.currentPath
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
commandReplacementDic = unit.getReplacementDic(directory)
#for each key replace all instances of the key in the command string
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=UUID)
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
t = threading.Thread(target=self.task.performTask)
t.daemon = True
t.start()
def __init__(self, jobChainLink, pk, unit):
self.tasks = []
self.pk = pk
self.jobChainLink = jobChainLink
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
print row
#pk = row[0]
filterFileEnd = row[1]
filterFileStart = row[2]
filterSubDir = row[3]
self.requiresOutputLock = row[4]
standardOutputFile = row[5]
standardErrorFile = row[6]
execute = row[7]
self.execute = execute
arguments = row[8]
row = c.fetchone()
sqlLock.release()
#if reloadFileList:
# unit.reloadFileList()
# "%taskUUID%": task.UUID.__str__(), \
if filterSubDir:
directory = os.path.join(unit.currentPath, filterSubDir)
else:
directory = unit.currentPath
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
commandReplacementDic = unit.getReplacementDic(directory)
#for each key replace all instances of the key in the command string
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=UUID)
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
t = threading.Thread(target=self.task.performTask)
t.daemon = True
t.start()
def __init__(self, jobChainLink, pk, unit):
global outputLock
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
opts = {"inputFile":"%relativeLocation%", "fileUUID":"%fileUUID%", 'commandClassifications':'%commandClassifications%', "taskUUID":"%taskUUID%", "objectsDirectory":"%SIPObjectsDirectory%", "logsDirectory":"%SIPLogsDirectory%", "sipUUID":"%SIPUUID%", "sipPath":"%SIPDirectory%", "fileGrpUse":"%fileGrpUse%", "normalizeFileGrpUse":"%normalizeFileGrpUse%", "excludeDirectory":"%excludeDirectory%", "standardErrorFile":"%standardErrorFile%", "standardOutputFile":"%standardOutputFile%"}
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
for optsKey, optsValue in opts.iteritems():
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
opts[optsKey] = self.jobChainLink.passVar.replace(opts[optsKey])[0]
commandReplacementDic = unit.getReplacementDic()
for key, value in commandReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
for key, value in SIPReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
commandReplacementDic = unit.getReplacementDic()
sql = """SELECT CommandRelationships.pk FROM CommandRelationships JOIN Commands ON CommandRelationships.command = Commands.pk WHERE CommandRelationships.pk = '%s';""" % (pk.__str__())
rows = databaseInterface.queryAllSQL(sql)
taskCount = 0
tasksList = []
if rows:
self.tasksLock.acquire()
for row in rows:
UUID = uuid.uuid4().__str__()
opts["taskUUID"] = UUID
opts["CommandRelationship"] = pk.__str__()
execute = "transcoder_cr%s" % (pk)
deUnicode(execute)
arguments = row.__str__()
standardOutputFile = opts["standardOutputFile"]
standardErrorFile = opts["standardErrorFile"]
self.standardOutputFile = standardOutputFile
self.standardErrorFile = standardErrorFile
self.execute = execute
self.arguments = arguments
task = taskStandard(self, execute, opts, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
taskCount += 1
tasksList.append(task)
self.tasksLock.release()
for task in tasksList:
task.performTask()
else:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
super(linkTaskManagerDirectories,
self).__init__(jobChainLink, pk, unit)
self.tasks = []
stc = StandardTaskConfig.objects.get(id=str(pk))
filterFileEnd = stc.filter_file_end
filterFileStart = stc.filter_file_start
filterSubDir = stc.filter_subdir
self.requiresOutputLock = stc.requires_output_lock
standardOutputFile = stc.stdout_file
standardErrorFile = stc.stderr_file
execute = stc.execute
self.execute = execute
arguments = stc.arguments
if filterSubDir:
directory = os.path.join(unit.currentPath, filterSubDir)
else:
directory = unit.currentPath
# Apply passvar replacement values
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, ReplacementDict):
arguments, standardOutputFile, standardErrorFile = passVar.replace(
arguments, standardOutputFile, standardErrorFile)
elif isinstance(self.jobChainLink.passVar, ReplacementDict):
arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
arguments, standardOutputFile, standardErrorFile)
# Apply unit (SIP/Transfer) replacement values
commandReplacementDic = unit.getReplacementDic(directory)
# Escape all values for shell
for key, value in commandReplacementDic.items():
commandReplacementDic[
key] = archivematicaFunctions.escapeForCommand(value)
arguments, standardOutputFile, standardErrorFile = commandReplacementDic.replace(
arguments, standardOutputFile, standardErrorFile)
self.task = taskStandard(self,
execute,
arguments,
standardOutputFile,
standardErrorFile,
UUID=self.UUID)
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic,
self.UUID, arguments)
t = threading.Thread(target=self.task.performTask)
t.daemon = True
t.start()
def __init__(self, jobChainLink, pk, unit):
global outputLock
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
opts = {
"inputFile": "%relativeLocation%",
"fileUUID": "%fileUUID%",
'commandClassifications': '%commandClassifications%',
"taskUUID": "%taskUUID%",
"objectsDirectory": "%SIPObjectsDirectory%",
"logsDirectory": "%SIPLogsDirectory%",
"sipUUID": "%SIPUUID%",
"sipPath": "%SIPDirectory%",
"fileGrpUse": "%fileGrpUse%",
"normalizeFileGrpUse": "%normalizeFileGrpUse%",
"excludeDirectory": "%excludeDirectory%",
"standardErrorFile": "%standardErrorFile%",
"standardOutputFile": "%standardOutputFile%"
}
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
for optsKey, optsValue in opts.iteritems():
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, replacementDic):
opts[optsKey] = self.jobChainLink.passVar.replace(
opts[optsKey])[0]
commandReplacementDic = unit.getReplacementDic()
for key, value in commandReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
for key, value in SIPReplacementDic.iteritems():
opts[optsKey] = opts[optsKey].replace(key, value)
commandReplacementDic = unit.getReplacementDic()
sql = """SELECT CommandRelationships.pk FROM CommandRelationships JOIN Commands ON CommandRelationships.command = Commands.pk WHERE CommandRelationships.pk = '%s';""" % (
pk.__str__())
rows = databaseInterface.queryAllSQL(sql)
taskCount = 0
tasksList = []
if rows:
self.tasksLock.acquire()
for row in rows:
UUID = uuid.uuid4().__str__()
opts["taskUUID"] = UUID
opts["CommandRelationship"] = pk.__str__()
execute = "transcoder_cr%s" % (pk)
deUnicode(execute)
arguments = row.__str__()
standardOutputFile = opts["standardOutputFile"]
standardErrorFile = opts["standardErrorFile"]
self.standardOutputFile = standardOutputFile
self.standardErrorFile = standardErrorFile
self.execute = execute
self.arguments = arguments
task = taskStandard(self,
execute,
opts,
standardOutputFile,
standardErrorFile,
outputLock=outputLock,
UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self,
commandReplacementDic,
UUID, arguments)
taskCount += 1
tasksList.append(task)
self.tasksLock.release()
for task in tasksList:
task.performTask()
else:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
super(linkTaskManagerFiles, self).__init__(jobChainLink, pk, unit)
self.tasks = {}
self.tasksLock = threading.Lock()
self.exitCode = 0
self.clearToNextLink = False
stc = StandardTaskConfig.objects.get(id=str(pk))
# These three may be concatenated/compared with other strings,
# so they need to be bytestrings here
filterFileEnd = str(stc.filter_file_end) if stc.filter_file_end else ''
filterFileStart = str(
stc.filter_file_start) if stc.filter_file_start else ''
filterSubDir = str(stc.filter_subdir) if stc.filter_subdir else ''
self.standardOutputFile = stc.stdout_file
self.standardErrorFile = stc.stderr_file
self.execute = stc.execute
self.arguments = stc.arguments
if stc.requires_output_lock:
outputLock = threading.Lock()
else:
outputLock = None
# Check if filterSubDir has been overridden for this Transfer/SIP
try:
var = UnitVariable.objects.get(unittype=self.unit.unitType,
unituuid=self.unit.UUID,
variable=self.execute)
except (UnitVariable.DoesNotExist,
UnitVariable.MultipleObjectsReturned):
var = None
if var:
try:
variableValue = ast.literal_eval(var.variablevalue)
except SyntaxError:
# SyntaxError = contents of variableValue weren't the expected dict
pass
else:
filterSubDir = variableValue['filterSubDir']
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
# Escape all values for shell
for key, value in SIPReplacementDic.items():
SIPReplacementDic[key] = archivematicaFunctions.escapeForCommand(
value)
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
if not file.startswith(unit.pathString + filterSubDir):
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
# Apply passvar replacement values
if self.jobChainLink.passVar is not None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, ReplacementDict):
arguments, standardOutputFile, standardErrorFile = passVar.replace(
arguments, standardOutputFile,
standardErrorFile)
elif isinstance(self.jobChainLink.passVar, ReplacementDict):
arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
arguments, standardOutputFile, standardErrorFile)
# Apply file replacement values
commandReplacementDic = fileUnit.getReplacementDic()
for key, value in commandReplacementDic.items():
# Escape values for shell
commandReplacementDic[
key] = archivematicaFunctions.escapeForCommand(value)
arguments, standardOutputFile, standardErrorFile = commandReplacementDic.replace(
arguments, standardOutputFile, standardErrorFile)
# Apply unit (SIP/Transfer) replacement values
arguments, standardOutputFile, standardErrorFile = SIPReplacementDic.replace(
arguments, standardOutputFile, standardErrorFile)
UUID = str(uuid.uuid4())
task = taskStandard(self,
execute,
arguments,
standardOutputFile,
standardErrorFile,
outputLock=outputLock,
UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic,
UUID, arguments)
t = threading.Thread(target=task.performTask)
t.daemon = True
while (archivematicaMCP.limitTaskThreads <=
threading.activeCount()):
self.tasksLock.release()
time.sleep(archivematicaMCP.limitTaskThreadsSleep)
self.tasksLock.acquire()
t.start()
self.clearToNextLink = True
self.tasksLock.release()
if self.tasks == {}:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
commandReplacementDic = fileUnit.getReplacementDic()
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
for key in SIPReplacementDic.iterkeys():
value = SIPReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
t = threading.Thread(target=task.performTask)
t.daemon = True
while(archivematicaMCP.limitTaskThreads <= threading.activeCount()):
#print "Waiting for active threads", threading.activeCount()
self.tasksLock.release()
time.sleep(archivematicaMCP.limitTaskThreadsSleep)
self.tasksLock.acquire()
print "Active threads:", threading.activeCount()
t.start()
self.clearToNextLink = True
self.tasksLock.release()
if self.tasks == {} :
self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit):
self.tasks = {}
self.tasksLock = threading.Lock()
self.pk = pk
self.jobChainLink = jobChainLink
self.exitCode = 0
self.clearToNextLink = False
sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (
pk.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filterFileEnd = deUnicode(row[1])
filterFileStart = deUnicode(row[2])
filterSubDir = deUnicode(row[3])
requiresOutputLock = row[4]
self.standardOutputFile = deUnicode(row[5])
self.standardErrorFile = deUnicode(row[6])
self.execute = deUnicode(row[7])
self.arguments = deUnicode(row[8])
row = c.fetchone()
sqlLock.release()
if requiresOutputLock:
outputLock = threading.Lock()
else:
outputLock = None
SIPReplacementDic = unit.getReplacementDic(unit.currentPath)
self.tasksLock.acquire()
for file, fileUnit in unit.fileList.items():
#print "file:", file, fileUnit
if filterFileEnd:
if not file.endswith(filterFileEnd):
continue
if filterFileStart:
if not os.path.basename(file).startswith(filterFileStart):
continue
if filterSubDir:
#print "file", file, type(file)
#print unit.pathString, type(unit.pathString)
#filterSubDir = filterSubDir.encode('utf-8')
#print filterSubDir, type(filterSubDir)
if not file.startswith(unit.pathString + filterSubDir):
continue
standardOutputFile = self.standardOutputFile
standardErrorFile = self.standardErrorFile
execute = self.execute
arguments = self.arguments
if self.jobChainLink.passVar != None:
if isinstance(self.jobChainLink.passVar, list):
for passVar in self.jobChainLink.passVar:
if isinstance(passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(
execute, arguments, standardOutputFile,
standardErrorFile)
elif isinstance(self.jobChainLink.passVar, replacementDic):
execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(
execute, arguments, standardOutputFile,
standardErrorFile)
commandReplacementDic = fileUnit.getReplacementDic()
for key in commandReplacementDic.iterkeys():
value = commandReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
for key in SIPReplacementDic.iterkeys():
value = SIPReplacementDic[key].replace("\"", ("\\\""))
#print "key", type(key), key
#print "value", type(value), value
if isinstance(value, unicode):
value = value.encode("utf-8")
#key = key.encode("utf-8")
#value = value.encode("utf-8")
if execute:
execute = execute.replace(key, value)
if arguments:
arguments = arguments.replace(key, value)
if standardOutputFile:
standardOutputFile = standardOutputFile.replace(key, value)
if standardErrorFile:
standardErrorFile = standardErrorFile.replace(key, value)
UUID = uuid.uuid4().__str__()
task = taskStandard(self,
execute,
arguments,
standardOutputFile,
standardErrorFile,
outputLock=outputLock,
UUID=UUID)
self.tasks[UUID] = task
databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic,
UUID, arguments)
t = threading.Thread(target=task.performTask)
t.daemon = True
while (archivematicaMCP.limitTaskThreads <=
threading.activeCount()):
#print "Waiting for active threads", threading.activeCount()
self.tasksLock.release()
time.sleep(archivematicaMCP.limitTaskThreadsSleep)
self.tasksLock.acquire()
print "Active threads:", threading.activeCount()
t.start()
self.clearToNextLink = True
self.tasksLock.release()
if self.tasks == {}:
self.jobChainLink.linkProcessingComplete(self.exitCode)
def insertTasks():
with transaction.atomic():
for task in self.groupTasks:
databaseFunctions.logTaskCreatedSQL(
self.linkTaskManager, task.commandReplacementDic,
task.UUID, task.arguments)