def __init__(self, jobChainLink, pk, unit): self.tasks = [] self.pk = pk self.jobChainLink = jobChainLink sql = """SELECT * FROM StandardTasksConfigs where pk = """ + pk.__str__() c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: print row #pk = row[0] filterFileEnd = row[1] filterFileStart = row[2] filterSubDir = row[3] self.requiresOutputLock = row[4] standardOutputFile = row[5] standardErrorFile = row[6] execute = row[7] self.execute = execute arguments = row[8] row = c.fetchone() sqlLock.release() #if reloadFileList: # unit.reloadFileList() # "%taskUUID%": task.UUID.__str__(), \ if filterSubDir: directory = os.path.join(unit.currentPath, filterSubDir) else: directory = unit.currentPath if self.jobChainLink.passVar != None: if isinstance(self.jobChainLink.passVar, list): for passVar in self.jobChainLink.passVar: if isinstance(passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile) elif isinstance(self.jobChainLink.passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile) commandReplacementDic = unit.getReplacementDic(directory) #for each key replace all instances of the key in the command string for key in commandReplacementDic.iterkeys(): value = commandReplacementDic[key].replace("\"", ("\\\"")) if execute: execute = execute.replace(key, value) if arguments: arguments = arguments.replace(key, value) if standardOutputFile: standardOutputFile = standardOutputFile.replace(key, value) if standardErrorFile: standardErrorFile = standardErrorFile.replace(key, value) UUID = uuid.uuid4().__str__() self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=UUID) databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments) t = threading.Thread(target=self.task.performTask) t.daemon = True t.start()
def __init__(self, jobChainLink, pk, unit): self.tasks = [] self.pk = pk self.jobChainLink = jobChainLink sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (pk.__str__()) c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: print row #pk = row[0] filterFileEnd = row[1] filterFileStart = row[2] filterSubDir = row[3] self.requiresOutputLock = row[4] standardOutputFile = row[5] standardErrorFile = row[6] execute = row[7] self.execute = execute arguments = row[8] row = c.fetchone() sqlLock.release() #if reloadFileList: # unit.reloadFileList() # "%taskUUID%": task.UUID.__str__(), \ if filterSubDir: directory = os.path.join(unit.currentPath, filterSubDir) else: directory = unit.currentPath if self.jobChainLink.passVar != None: if isinstance(self.jobChainLink.passVar, list): for passVar in self.jobChainLink.passVar: if isinstance(passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile) elif isinstance(self.jobChainLink.passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile) commandReplacementDic = unit.getReplacementDic(directory) #for each key replace all instances of the key in the command string for key in commandReplacementDic.iterkeys(): value = commandReplacementDic[key].replace("\"", ("\\\"")) if execute: execute = execute.replace(key, value) if arguments: arguments = arguments.replace(key, value) if standardOutputFile: standardOutputFile = standardOutputFile.replace(key, value) if standardErrorFile: standardErrorFile = standardErrorFile.replace(key, value) UUID = uuid.uuid4().__str__() self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=UUID) databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments) t = threading.Thread(target=self.task.performTask) t.daemon = True t.start()
def __init__(self, jobChainLink, pk, unit): global outputLock self.tasks = {} self.tasksLock = threading.Lock() self.pk = pk self.jobChainLink = jobChainLink self.exitCode = 0 self.clearToNextLink = False opts = {"inputFile":"%relativeLocation%", "fileUUID":"%fileUUID%", 'commandClassifications':'%commandClassifications%', "taskUUID":"%taskUUID%", "objectsDirectory":"%SIPObjectsDirectory%", "logsDirectory":"%SIPLogsDirectory%", "sipUUID":"%SIPUUID%", "sipPath":"%SIPDirectory%", "fileGrpUse":"%fileGrpUse%", "normalizeFileGrpUse":"%normalizeFileGrpUse%", "excludeDirectory":"%excludeDirectory%", "standardErrorFile":"%standardErrorFile%", "standardOutputFile":"%standardOutputFile%"} SIPReplacementDic = unit.getReplacementDic(unit.currentPath) for optsKey, optsValue in opts.iteritems(): if self.jobChainLink.passVar != None: if isinstance(self.jobChainLink.passVar, replacementDic): opts[optsKey] = self.jobChainLink.passVar.replace(opts[optsKey])[0] commandReplacementDic = unit.getReplacementDic() for key, value in commandReplacementDic.iteritems(): opts[optsKey] = opts[optsKey].replace(key, value) for key, value in SIPReplacementDic.iteritems(): opts[optsKey] = opts[optsKey].replace(key, value) commandReplacementDic = unit.getReplacementDic() sql = """SELECT CommandRelationships.pk FROM CommandRelationships JOIN Commands ON CommandRelationships.command = Commands.pk WHERE CommandRelationships.pk = '%s';""" % (pk.__str__()) rows = databaseInterface.queryAllSQL(sql) taskCount = 0 tasksList = [] if rows: self.tasksLock.acquire() for row in rows: UUID = uuid.uuid4().__str__() opts["taskUUID"] = UUID opts["CommandRelationship"] = pk.__str__() execute = "transcoder_cr%s" % (pk) deUnicode(execute) arguments = row.__str__() standardOutputFile = opts["standardOutputFile"] standardErrorFile = opts["standardErrorFile"] self.standardOutputFile = standardOutputFile self.standardErrorFile = standardErrorFile self.execute = execute self.arguments = arguments task = taskStandard(self, execute, opts, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID) self.tasks[UUID] = task databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments) taskCount += 1 tasksList.append(task) self.tasksLock.release() for task in tasksList: task.performTask() else: self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit): super(linkTaskManagerDirectories, self).__init__(jobChainLink, pk, unit) self.tasks = [] stc = StandardTaskConfig.objects.get(id=str(pk)) filterFileEnd = stc.filter_file_end filterFileStart = stc.filter_file_start filterSubDir = stc.filter_subdir self.requiresOutputLock = stc.requires_output_lock standardOutputFile = stc.stdout_file standardErrorFile = stc.stderr_file execute = stc.execute self.execute = execute arguments = stc.arguments if filterSubDir: directory = os.path.join(unit.currentPath, filterSubDir) else: directory = unit.currentPath # Apply passvar replacement values if self.jobChainLink.passVar != None: if isinstance(self.jobChainLink.passVar, list): for passVar in self.jobChainLink.passVar: if isinstance(passVar, ReplacementDict): arguments, standardOutputFile, standardErrorFile = passVar.replace( arguments, standardOutputFile, standardErrorFile) elif isinstance(self.jobChainLink.passVar, ReplacementDict): arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace( arguments, standardOutputFile, standardErrorFile) # Apply unit (SIP/Transfer) replacement values commandReplacementDic = unit.getReplacementDic(directory) # Escape all values for shell for key, value in commandReplacementDic.items(): commandReplacementDic[ key] = archivematicaFunctions.escapeForCommand(value) arguments, standardOutputFile, standardErrorFile = commandReplacementDic.replace( arguments, standardOutputFile, standardErrorFile) self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=self.UUID) databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, self.UUID, arguments) t = threading.Thread(target=self.task.performTask) t.daemon = True t.start()
def __init__(self, jobChainLink, pk, unit): global outputLock self.tasks = {} self.tasksLock = threading.Lock() self.pk = pk self.jobChainLink = jobChainLink self.exitCode = 0 self.clearToNextLink = False opts = { "inputFile": "%relativeLocation%", "fileUUID": "%fileUUID%", 'commandClassifications': '%commandClassifications%', "taskUUID": "%taskUUID%", "objectsDirectory": "%SIPObjectsDirectory%", "logsDirectory": "%SIPLogsDirectory%", "sipUUID": "%SIPUUID%", "sipPath": "%SIPDirectory%", "fileGrpUse": "%fileGrpUse%", "normalizeFileGrpUse": "%normalizeFileGrpUse%", "excludeDirectory": "%excludeDirectory%", "standardErrorFile": "%standardErrorFile%", "standardOutputFile": "%standardOutputFile%" } SIPReplacementDic = unit.getReplacementDic(unit.currentPath) for optsKey, optsValue in opts.iteritems(): if self.jobChainLink.passVar != None: if isinstance(self.jobChainLink.passVar, replacementDic): opts[optsKey] = self.jobChainLink.passVar.replace( opts[optsKey])[0] commandReplacementDic = unit.getReplacementDic() for key, value in commandReplacementDic.iteritems(): opts[optsKey] = opts[optsKey].replace(key, value) for key, value in SIPReplacementDic.iteritems(): opts[optsKey] = opts[optsKey].replace(key, value) commandReplacementDic = unit.getReplacementDic() sql = """SELECT CommandRelationships.pk FROM CommandRelationships JOIN Commands ON CommandRelationships.command = Commands.pk WHERE CommandRelationships.pk = '%s';""" % ( pk.__str__()) rows = databaseInterface.queryAllSQL(sql) taskCount = 0 tasksList = [] if rows: self.tasksLock.acquire() for row in rows: UUID = uuid.uuid4().__str__() opts["taskUUID"] = UUID opts["CommandRelationship"] = pk.__str__() execute = "transcoder_cr%s" % (pk) deUnicode(execute) arguments = row.__str__() standardOutputFile = opts["standardOutputFile"] standardErrorFile = opts["standardErrorFile"] self.standardOutputFile = standardOutputFile self.standardErrorFile = standardErrorFile self.execute = execute self.arguments = arguments task = taskStandard(self, execute, opts, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID) self.tasks[UUID] = task databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments) taskCount += 1 tasksList.append(task) self.tasksLock.release() for task in tasksList: task.performTask() else: self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit): super(linkTaskManagerFiles, self).__init__(jobChainLink, pk, unit) self.tasks = {} self.tasksLock = threading.Lock() self.exitCode = 0 self.clearToNextLink = False stc = StandardTaskConfig.objects.get(id=str(pk)) # These three may be concatenated/compared with other strings, # so they need to be bytestrings here filterFileEnd = str(stc.filter_file_end) if stc.filter_file_end else '' filterFileStart = str( stc.filter_file_start) if stc.filter_file_start else '' filterSubDir = str(stc.filter_subdir) if stc.filter_subdir else '' self.standardOutputFile = stc.stdout_file self.standardErrorFile = stc.stderr_file self.execute = stc.execute self.arguments = stc.arguments if stc.requires_output_lock: outputLock = threading.Lock() else: outputLock = None # Check if filterSubDir has been overridden for this Transfer/SIP try: var = UnitVariable.objects.get(unittype=self.unit.unitType, unituuid=self.unit.UUID, variable=self.execute) except (UnitVariable.DoesNotExist, UnitVariable.MultipleObjectsReturned): var = None if var: try: variableValue = ast.literal_eval(var.variablevalue) except SyntaxError: # SyntaxError = contents of variableValue weren't the expected dict pass else: filterSubDir = variableValue['filterSubDir'] SIPReplacementDic = unit.getReplacementDic(unit.currentPath) # Escape all values for shell for key, value in SIPReplacementDic.items(): SIPReplacementDic[key] = archivematicaFunctions.escapeForCommand( value) self.tasksLock.acquire() for file, fileUnit in unit.fileList.items(): if filterFileEnd: if not file.endswith(filterFileEnd): continue if filterFileStart: if not os.path.basename(file).startswith(filterFileStart): continue if filterSubDir: if not file.startswith(unit.pathString + filterSubDir): continue standardOutputFile = self.standardOutputFile standardErrorFile = self.standardErrorFile execute = self.execute arguments = self.arguments # Apply passvar replacement values if self.jobChainLink.passVar is not None: if isinstance(self.jobChainLink.passVar, list): for passVar in self.jobChainLink.passVar: if isinstance(passVar, ReplacementDict): arguments, standardOutputFile, standardErrorFile = passVar.replace( arguments, standardOutputFile, standardErrorFile) elif isinstance(self.jobChainLink.passVar, ReplacementDict): arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace( arguments, standardOutputFile, standardErrorFile) # Apply file replacement values commandReplacementDic = fileUnit.getReplacementDic() for key, value in commandReplacementDic.items(): # Escape values for shell commandReplacementDic[ key] = archivematicaFunctions.escapeForCommand(value) arguments, standardOutputFile, standardErrorFile = commandReplacementDic.replace( arguments, standardOutputFile, standardErrorFile) # Apply unit (SIP/Transfer) replacement values arguments, standardOutputFile, standardErrorFile = SIPReplacementDic.replace( arguments, standardOutputFile, standardErrorFile) UUID = str(uuid.uuid4()) task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID) self.tasks[UUID] = task databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments) t = threading.Thread(target=task.performTask) t.daemon = True while (archivematicaMCP.limitTaskThreads <= threading.activeCount()): self.tasksLock.release() time.sleep(archivematicaMCP.limitTaskThreadsSleep) self.tasksLock.acquire() t.start() self.clearToNextLink = True self.tasksLock.release() if self.tasks == {}: self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit): self.tasks = {} self.tasksLock = threading.Lock() self.pk = pk self.jobChainLink = jobChainLink self.exitCode = 0 self.clearToNextLink = False sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (pk.__str__()) c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: filterFileEnd = deUnicode(row[1]) filterFileStart = deUnicode(row[2]) filterSubDir = deUnicode(row[3]) requiresOutputLock = row[4] self.standardOutputFile = deUnicode(row[5]) self.standardErrorFile = deUnicode(row[6]) self.execute = deUnicode(row[7]) self.arguments = deUnicode(row[8]) row = c.fetchone() sqlLock.release() if requiresOutputLock: outputLock = threading.Lock() else: outputLock = None SIPReplacementDic = unit.getReplacementDic(unit.currentPath) self.tasksLock.acquire() for file, fileUnit in unit.fileList.items(): #print "file:", file, fileUnit if filterFileEnd: if not file.endswith(filterFileEnd): continue if filterFileStart: if not os.path.basename(file).startswith(filterFileStart): continue if filterSubDir: #print "file", file, type(file) #print unit.pathString, type(unit.pathString) #filterSubDir = filterSubDir.encode('utf-8') #print filterSubDir, type(filterSubDir) if not file.startswith(unit.pathString + filterSubDir): continue standardOutputFile = self.standardOutputFile standardErrorFile = self.standardErrorFile execute = self.execute arguments = self.arguments if self.jobChainLink.passVar != None: if isinstance(self.jobChainLink.passVar, list): for passVar in self.jobChainLink.passVar: if isinstance(passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile) elif isinstance(self.jobChainLink.passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile) commandReplacementDic = fileUnit.getReplacementDic() for key in commandReplacementDic.iterkeys(): value = commandReplacementDic[key].replace("\"", ("\\\"")) #print "key", type(key), key #print "value", type(value), value if isinstance(value, unicode): value = value.encode("utf-8") #key = key.encode("utf-8") #value = value.encode("utf-8") if execute: execute = execute.replace(key, value) if arguments: arguments = arguments.replace(key, value) if standardOutputFile: standardOutputFile = standardOutputFile.replace(key, value) if standardErrorFile: standardErrorFile = standardErrorFile.replace(key, value) for key in SIPReplacementDic.iterkeys(): value = SIPReplacementDic[key].replace("\"", ("\\\"")) #print "key", type(key), key #print "value", type(value), value if isinstance(value, unicode): value = value.encode("utf-8") #key = key.encode("utf-8") #value = value.encode("utf-8") if execute: execute = execute.replace(key, value) if arguments: arguments = arguments.replace(key, value) if standardOutputFile: standardOutputFile = standardOutputFile.replace(key, value) if standardErrorFile: standardErrorFile = standardErrorFile.replace(key, value) UUID = uuid.uuid4().__str__() task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID) self.tasks[UUID] = task databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments) t = threading.Thread(target=task.performTask) t.daemon = True while(archivematicaMCP.limitTaskThreads <= threading.activeCount()): #print "Waiting for active threads", threading.activeCount() self.tasksLock.release() time.sleep(archivematicaMCP.limitTaskThreadsSleep) self.tasksLock.acquire() print "Active threads:", threading.activeCount() t.start() self.clearToNextLink = True self.tasksLock.release() if self.tasks == {} : self.jobChainLink.linkProcessingComplete(self.exitCode)
def __init__(self, jobChainLink, pk, unit): self.tasks = {} self.tasksLock = threading.Lock() self.pk = pk self.jobChainLink = jobChainLink self.exitCode = 0 self.clearToNextLink = False sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % ( pk.__str__()) c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: filterFileEnd = deUnicode(row[1]) filterFileStart = deUnicode(row[2]) filterSubDir = deUnicode(row[3]) requiresOutputLock = row[4] self.standardOutputFile = deUnicode(row[5]) self.standardErrorFile = deUnicode(row[6]) self.execute = deUnicode(row[7]) self.arguments = deUnicode(row[8]) row = c.fetchone() sqlLock.release() if requiresOutputLock: outputLock = threading.Lock() else: outputLock = None SIPReplacementDic = unit.getReplacementDic(unit.currentPath) self.tasksLock.acquire() for file, fileUnit in unit.fileList.items(): #print "file:", file, fileUnit if filterFileEnd: if not file.endswith(filterFileEnd): continue if filterFileStart: if not os.path.basename(file).startswith(filterFileStart): continue if filterSubDir: #print "file", file, type(file) #print unit.pathString, type(unit.pathString) #filterSubDir = filterSubDir.encode('utf-8') #print filterSubDir, type(filterSubDir) if not file.startswith(unit.pathString + filterSubDir): continue standardOutputFile = self.standardOutputFile standardErrorFile = self.standardErrorFile execute = self.execute arguments = self.arguments if self.jobChainLink.passVar != None: if isinstance(self.jobChainLink.passVar, list): for passVar in self.jobChainLink.passVar: if isinstance(passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = passVar.replace( execute, arguments, standardOutputFile, standardErrorFile) elif isinstance(self.jobChainLink.passVar, replacementDic): execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace( execute, arguments, standardOutputFile, standardErrorFile) commandReplacementDic = fileUnit.getReplacementDic() for key in commandReplacementDic.iterkeys(): value = commandReplacementDic[key].replace("\"", ("\\\"")) #print "key", type(key), key #print "value", type(value), value if isinstance(value, unicode): value = value.encode("utf-8") #key = key.encode("utf-8") #value = value.encode("utf-8") if execute: execute = execute.replace(key, value) if arguments: arguments = arguments.replace(key, value) if standardOutputFile: standardOutputFile = standardOutputFile.replace(key, value) if standardErrorFile: standardErrorFile = standardErrorFile.replace(key, value) for key in SIPReplacementDic.iterkeys(): value = SIPReplacementDic[key].replace("\"", ("\\\"")) #print "key", type(key), key #print "value", type(value), value if isinstance(value, unicode): value = value.encode("utf-8") #key = key.encode("utf-8") #value = value.encode("utf-8") if execute: execute = execute.replace(key, value) if arguments: arguments = arguments.replace(key, value) if standardOutputFile: standardOutputFile = standardOutputFile.replace(key, value) if standardErrorFile: standardErrorFile = standardErrorFile.replace(key, value) UUID = uuid.uuid4().__str__() task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, outputLock=outputLock, UUID=UUID) self.tasks[UUID] = task databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments) t = threading.Thread(target=task.performTask) t.daemon = True while (archivematicaMCP.limitTaskThreads <= threading.activeCount()): #print "Waiting for active threads", threading.activeCount() self.tasksLock.release() time.sleep(archivematicaMCP.limitTaskThreadsSleep) self.tasksLock.acquire() print "Active threads:", threading.activeCount() t.start() self.clearToNextLink = True self.tasksLock.release() if self.tasks == {}: self.jobChainLink.linkProcessingComplete(self.exitCode)
def insertTasks(): with transaction.atomic(): for task in self.groupTasks: databaseFunctions.logTaskCreatedSQL( self.linkTaskManager, task.commandReplacementDic, task.UUID, task.arguments)