def removeFile(filePath, utcDate=databaseInterface.getUTCDate()): global separator print "removing: ", filePath filesWithMatchingPath = [] sqlLoggingLock.acquire() #Find the file pk/UUID c = MCPloggingSQL.database.cursor() sql = """SELECT fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '""" + MySQLdb.escape_string( filePath) + """';""" c.execute(sql) row = c.fetchone() while row != None: filesWithMatchingPath.append(row[0]) row = c.fetchone() sqlLoggingLock.release() #Update the database for file in filesWithMatchingPath: eventIdentifierUUID = uuid.uuid4().__str__() eventType = "file removed" eventDateTime = utcDate eventDetail = "" eventOutcomeDetailNote = "removed from: " + filePath databaseInterface.insertIntoEvents(fileUUID=fileUUID, \ eventIdentifierUUID=eventIdentifierUUID, \ eventType=eventType, \ eventDateTime=eventDateTime, \ eventDetail=eventDetail, \ eventOutcome=eventOutcome, \ eventOutcomeDetailNote=eventOutcomeDetailNote) removeFileByFileUUID(fileUUID, utcDate)
def insertIntoEvents( fileUUID="", eventIdentifierUUID="", eventType="", eventDateTime=databaseInterface.getUTCDate(), eventDetail="", eventOutcome="", eventOutcomeDetailNote="", ): if eventIdentifierUUID == "": eventIdentifierUUID = uuid.uuid4().__str__() databaseInterface.runSQL( """INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote) VALUES ( '""" + escapeForDB(fileUUID) + databaseInterface.separator + escapeForDB(eventIdentifierUUID) + databaseInterface.separator + escapeForDB(eventType) + databaseInterface.separator + escapeForDB(eventDateTime) + databaseInterface.separator + escapeForDB(eventDetail) + databaseInterface.separator + escapeForDB(eventOutcome) + databaseInterface.separator + escapeForDB(eventOutcomeDetailNote) + "' )" )
def xmlCreateFileAssociationBetween(originalFileFullPath, outputFromNormalizationFileFullPath, SIPFullPath, sipUUID, eventDetailText, eventOutcomeDetailNote, outputFileUUID=""): #assign file UUID date = databaseInterface.getUTCDate() if outputFileUUID == "": outputFileUUID = uuid.uuid4().__str__() originalFilePathRelativeToSIP = originalFileFullPath.replace(SIPFullPath,"%SIPDirectory%", 1) sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(originalFilePathRelativeToSIP) + "' AND Files.sipUUID = '" + sipUUID + "';" print sql rows = databaseInterface.queryAllSQL(sql) print rows fileUUID = rows[0][0] filePathRelativeToSIP = outputFromNormalizationFileFullPath.replace(SIPFullPath,"%SIPDirectory%", 1) addFileToSIP(filePathRelativeToSIP, outputFileUUID, sipUUID, uuid.uuid4().__str__(), date, sourceType="creation", use="preservation") updateSizeAndChecksum(outputFileUUID, outputFromNormalizationFileFullPath, date, uuid.uuid4().__str__()) taskUUID = uuid.uuid4().__str__() insertIntoEvents(fileUUID=fileUUID, \ eventIdentifierUUID=taskUUID, \ eventType="normalization", \ eventDateTime=date, \ eventDetail=eventDetailText, \ eventOutcome="", \ eventOutcomeDetailNote=eventOutcomeDetailNote) insertIntoDerivations(sourceFileUUID=fileUUID, derivedFileUUID=outputFileUUID, relatedEventUUID=taskUUID)
def fileWasRemoved( fileUUID, utcDate=databaseInterface.getUTCDate(), eventDetail="", eventOutcomeDetailNote="", eventOutcome="" ): eventIdentifierUUID = uuid.uuid4().__str__() eventType = "file removed" eventDateTime = utcDate insertIntoEvents( fileUUID=fileUUID, eventIdentifierUUID=eventIdentifierUUID, eventType=eventType, eventDateTime=eventDateTime, eventDetail=eventDetail, eventOutcome=eventOutcome, eventOutcomeDetailNote=eventOutcomeDetailNote, ) databaseInterface.runSQL( "UPDATE Files " + "SET removedTime='" + utcDate + "', currentLocation=NULL " + "WHERE fileUUID='" + fileUUID + "'" )
def logTaskCreatedSQL(taskManager, commandReplacementDic, taskUUID, arguments): taskUUID = taskUUID jobUUID = taskManager.jobChainLink.UUID fileUUID = "" if "%fileUUID%" in commandReplacementDic: fileUUID = commandReplacementDic["%fileUUID%"] taskexec = taskManager.execute fileName = os.path.basename(os.path.abspath(commandReplacementDic["%relativeLocation%"])) databaseInterface.runSQL( """INSERT INTO Tasks (taskUUID, jobUUID, fileUUID, fileName, exec, arguments, createdTime) VALUES ( '""" + taskUUID + databaseInterface.separator + jobUUID + databaseInterface.separator + escapeForDB(fileUUID) + databaseInterface.separator + escapeForDB(fileName) + databaseInterface.separator + escapeForDB(taskexec) + databaseInterface.separator + escapeForDB(arguments) + databaseInterface.separator + databaseInterface.getUTCDate() + "' )" )
def insertIntoFiles(fileUUID, filePath, enteredSystem=databaseInterface.getUTCDate(), transferUUID="", sipUUID="", use="original"): if transferUUID != "" and sipUUID == "": databaseInterface.runSQL("""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, transferUUID) VALUES ( '""" + fileUUID + databaseInterface.separator \ + escapeForDB(filePath) + databaseInterface.separator \ + escapeForDB(filePath) + databaseInterface.separator \ + enteredSystem + databaseInterface.separator \ + use + databaseInterface.separator \ + transferUUID + "' )" ) elif transferUUID == "" and sipUUID != "": databaseInterface.runSQL("""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, sipUUID) VALUES ( '""" + fileUUID + databaseInterface.separator \ + escapeForDB(filePath) + databaseInterface.separator \ + escapeForDB(filePath) + databaseInterface.separator \ + enteredSystem + databaseInterface.separator \ + use + databaseInterface.separator \ + sipUUID + "' )" ) else: print >> sys.stderr, "not supported yet - both SIP and transfer UUID's defined (or neither defined)" print >> sys.stderr, "SIP UUID:", sipUUID print >> sys.stderr, "transferUUID:", transferUUID raise Exception( "not supported yet - both SIP and transfer UUID's defined (or neither defined)", sipUUID + "-" + transferUUID)
def process_IN_MOVED_FROM(self, event): global movedFromCount #print event #print "Transfer IN_MOVED_FROM" #Wait for a moved to, and if one doesn't occur, consider it moved outside of the system. movedFromPath = os.path.join(event.path, event.name).replace(\ self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), \ "%transferDirectory%", 1) filesMoved = [] sql = """SELECT fileUUID, currentLocation FROM Files WHERE transferUUID = '""" + self.unit.UUID + "' AND removedTime = 0 AND currentLocation LIKE '" + MySQLdb.escape_string(movedFromPath).replace("%", "\%") + "%';" c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() while row != None: #print row filesMoved.append(row) row = c.fetchone() sqlLock.release() movedFromLock.acquire() utcDate = databaseInterface.getUTCDate() timer = threading.Timer(archivematicaMCP.config.getint('MCPServer', "delayTimer"), timerExpired, args=[event, utcDate], kwargs={}) movedFrom[event.cookie] = (movedFromPath, filesMoved, timer) movedFromCount.value = movedFromCount.value + 1 movedFromLock.release() #create timer to check if it's claimed by a move to timer.start()
def removeFile(filePath, utcDate = databaseInterface.getUTCDate()): global separator print "removing: ", filePath filesWithMatchingPath = [] sqlLoggingLock.acquire() #Find the file pk/UUID c=MCPloggingSQL.database.cursor() sql = """SELECT fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '""" + MySQLdb.escape_string(filePath) + """';""" c.execute(sql) row = c.fetchone() while row != None: filesWithMatchingPath.append(row[0]) row = c.fetchone() sqlLoggingLock.release() #Update the database for file in filesWithMatchingPath: eventIdentifierUUID = uuid.uuid4().__str__() eventType = "file removed" eventDateTime = utcDate eventDetail = "" eventOutcomeDetailNote = "removed from: " + filePath databaseInterface.insertIntoEvents(fileUUID=fileUUID, \ eventIdentifierUUID=eventIdentifierUUID, \ eventType=eventType, \ eventDateTime=eventDateTime, \ eventDetail=eventDetail, \ eventOutcome=eventOutcome, \ eventOutcomeDetailNote=eventOutcomeDetailNote) removeFileByFileUUID(fileUUID, utcDate)
def executeCommandReleationship(gearman_worker, gearman_job): """some text""" try: execute = gearman_job.task print "executing:", execute, "{", gearman_job.unique, "}" data = cPickle.loads(gearman_job.data) utcDate = databaseInterface.getUTCDate() opts = data["arguments"]#.encode("utf-8") #if isinstance(arguments, unicode): # arguments = arguments.encode("utf-8") #if isinstance(arguments, str): # arguments = unicode(arguments) sInput = "" clientID = gearman_worker.worker_client_id opts["date"] = utcDate opts["accessDirectory"] = os.path.join(opts['sipPath'], "DIP/objects") + "/" opts["thumbnailDirectory"] = os.path.join(opts['sipPath'], "thumbnails") + "/" print opts for key, value in archivematicaClient.replacementDic.iteritems(): for key2 in opts: opts[key2] = opts[key2].replace(key, value) replacementDic = getReplacementDic(opts) #if True: opts["prependStdOut"] = """Operating on file: {%s}%s \r\nUsing %s command classifications""" % (opts["fileUUID"], replacementDic["%fileName%"], opts["commandClassification"]) opts["prependStdError"] = "\r\nSTDError:" # print clientID, execute, data archivematicaClient.logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate) cl = transcoder.CommandLinker(opts["CommandRelationship"], replacementDic, opts, onceNormalized) cl.execute() co = cl.commandObject exitCode = co.exitCode stdOut = "%s \r\n%s" % (opts["prependStdOut"], co.stdOut) if not co.stdError or co.stdError.isspace(): stdError = "" else: stdError = "%s \r\n%s" % (opts["prependStdError"], co.stdError) #TODO add date to ops #Replace replacement strings #archivematicaClient.printOutputLock.acquire() #print >>sys.stderr, "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>" #archivematicaClient.printOutputLock.release() #exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False) return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError}) #catch OS errors except OSError, ose: archivematicaClient.printOutputLock.acquire() traceback.print_exc(file=sys.stdout) print >>sys.stderr, "Execution failed:", ose archivematicaClient.printOutputLock.release() output = ["Config Error!", ose.__str__() ] exitCode = 1 return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
def debugMonitor(): global countOfCreateUnitAndJobChainThreaded while True: dblockstatus = "SQL Lock: Locked" if databaseInterface.sqlLock.acquire(False): databaseInterface.sqlLock.release() dblockstatus = "SQL Lock: Unlocked" print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate(), "\tThreadCount: ", threading.activeCount(), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>" time.sleep(60)
def debugMonitor(): """Periodically prints out status of MCP, including whether the database lock is locked, thread count, etc.""" global countOfCreateUnitAndJobChainThreaded while True: dblockstatus = "SQL Lock: Locked" if databaseInterface.sqlLock.acquire(False): databaseInterface.sqlLock.release() dblockstatus = "SQL Lock: Unlocked" print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate(), "\tThreadCount: ", threading.activeCount(), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>" time.sleep(3600)
def logJobStepCompletedSQL(job): databaseInterface.runSQL( """INSERT INTO jobStepCompleted (jobUUID, step, completedTime) VALUES ( '""" + job.UUID.__str__() + databaseInterface.separator + job.step + databaseInterface.separator + databaseInterface.getUTCDate() + "' )" )
def logTaskCompletedSQL(task): print "Logging task output to db", task.UUID taskUUID = task.UUID.__str__() exitCode = task.results["exitCode"].__str__() stdOut = task.results["stdOut"] stdError = task.results["stdError"] databaseInterface.runSQL("UPDATE Tasks " + \ "SET endTime='" + databaseInterface.getUTCDate() +"', exitCode='" + exitCode + "', " + \ "stdOut='" + escapeForDB(stdOut) + "', stdError='" + escapeForDB(stdError) + "' " "WHERE taskUUID='" + taskUUID + "'" )
def executeCommand(gearman_worker, gearman_job): try: execute = gearman_job.task print "executing:", execute, "{", gearman_job.unique, "}" data = cPickle.loads(gearman_job.data) utcDate = databaseInterface.getUTCDate() arguments = data["arguments"]#.encode("utf-8") if isinstance(arguments, unicode): arguments = arguments.encode("utf-8") #if isinstance(arguments, str): # arguments = unicode(arguments) sInput = "" clientID = gearman_worker.worker_client_id #if True: # print clientID, execute, data logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate) if execute not in supportedModules: output = ["Error!", "Error! - Tried to run and unsupported command." ] exitCode = -1 return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]}) command = supportedModules[execute] replacementDic["%date%"] = utcDate replacementDic["%jobCreatedDate%"] = data["createdDate"] #Replace replacement strings for key in replacementDic.iterkeys(): command = command.replace ( key, replacementDic[key] ) arguments = arguments.replace ( key, replacementDic[key] ) key = "%taskUUID%" value = gearman_job.unique.__str__() arguments = arguments.replace(key, value) #execute command command += " " + arguments printOutputLock.acquire() print >>sys.stderr, "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>" printOutputLock.release() exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False) return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError}) #catch OS errors except OSError, ose: traceback.print_exc(file=sys.stdout) printOutputLock.acquire() print >>sys.stderr, "Execution failed:", ose printOutputLock.release() output = ["Config Error!", ose.__str__() ] exitCode = 1 return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
def debugMonitor(): global countOfCreateUnitAndJobChainThreaded while True: dblockstatus = "SQL Lock: Locked" if databaseInterface.sqlLock.acquire(False): databaseInterface.sqlLock.release() dblockstatus = "SQL Lock: Unlocked" print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate( ), "\tThreadCount: ", threading.activeCount( ), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>" time.sleep(60)
def debugMonitor(): """Periodically prints out status of MCP, including whether the database lock is locked, thread count, etc.""" global countOfCreateUnitAndJobChainThreaded while True: dblockstatus = "SQL Lock: Locked" if databaseInterface.sqlLock.acquire(False): databaseInterface.sqlLock.release() dblockstatus = "SQL Lock: Unlocked" print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate( ), "\tThreadCount: ", threading.activeCount( ), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>" time.sleep(3600)
def xmlCreateFileAssociationBetween(originalFileFullPath, outputFromNormalizationFileFullPath, SIPFullPath, sipUUID, eventDetailText, eventOutcomeDetailNote, outputFileUUID=""): #assign file UUID date = databaseInterface.getUTCDate() if outputFileUUID == "": outputFileUUID = uuid.uuid4().__str__() originalFilePathRelativeToSIP = originalFileFullPath.replace( SIPFullPath, "%SIPDirectory%", 1) sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string( originalFilePathRelativeToSIP ) + "' AND Files.sipUUID = '" + sipUUID + "';" print sql rows = databaseInterface.queryAllSQL(sql) print rows fileUUID = rows[0][0] filePathRelativeToSIP = outputFromNormalizationFileFullPath.replace( SIPFullPath, "%SIPDirectory%", 1) addFileToSIP(filePathRelativeToSIP, outputFileUUID, sipUUID, uuid.uuid4().__str__(), date, sourceType="creation", use="preservation") updateSizeAndChecksum(outputFileUUID, outputFromNormalizationFileFullPath, date, uuid.uuid4().__str__()) taskUUID = uuid.uuid4().__str__() insertIntoEvents(fileUUID=fileUUID, \ eventIdentifierUUID=taskUUID, \ eventType="normalization", \ eventDateTime=date, \ eventDetail=eventDetailText, \ eventOutcome="", \ eventOutcomeDetailNote=eventOutcomeDetailNote) insertIntoDerivations(sourceFileUUID=fileUUID, derivedFileUUID=outputFileUUID, relatedEventUUID=taskUUID)
def insertIntoFiles( fileUUID, filePath, enteredSystem=databaseInterface.getUTCDate(), transferUUID="", sipUUID="", use="original" ): if transferUUID != "" and sipUUID == "": databaseInterface.runSQL( """INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, transferUUID) VALUES ( '""" + fileUUID + databaseInterface.separator + escapeForDB(filePath) + databaseInterface.separator + escapeForDB(filePath) + databaseInterface.separator + enteredSystem + databaseInterface.separator + use + databaseInterface.separator + transferUUID + "' )" ) elif transferUUID == "" and sipUUID != "": databaseInterface.runSQL( """INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, sipUUID) VALUES ( '""" + fileUUID + databaseInterface.separator + escapeForDB(filePath) + databaseInterface.separator + escapeForDB(filePath) + databaseInterface.separator + enteredSystem + databaseInterface.separator + use + databaseInterface.separator + sipUUID + "' )" ) else: print >> sys.stderr, "not supported yet - both SIP and transfer UUID's defined (or neither defined)" print >> sys.stderr, "SIP UUID:", sipUUID print >> sys.stderr, "transferUUID:", transferUUID raise Exception( "not supported yet - both SIP and transfer UUID's defined (or neither defined)", sipUUID + "-" + transferUUID, )
def logTaskCreatedSQL(taskManager, commandReplacementDic, taskUUID, arguments): taskUUID = taskUUID jobUUID = taskManager.jobChainLink.UUID fileUUID = "" if "%fileUUID%" in commandReplacementDic: fileUUID = commandReplacementDic["%fileUUID%"] taskexec = taskManager.execute fileName = os.path.basename( os.path.abspath(commandReplacementDic["%relativeLocation%"])) databaseInterface.runSQL("""INSERT INTO Tasks (taskUUID, jobUUID, fileUUID, fileName, exec, arguments, createdTime) VALUES ( '""" + taskUUID + databaseInterface.separator \ + jobUUID + databaseInterface.separator \ + escapeForDB(fileUUID) + databaseInterface.separator \ + escapeForDB(fileName) + databaseInterface.separator \ + escapeForDB(taskexec) + databaseInterface.separator \ + escapeForDB(arguments) + databaseInterface.separator \ + databaseInterface.getUTCDate() + "' )" )
def __init__(self, jobChain, jobChainLinkPK, unit, passVar=None, subJobOf=""): if jobChainLinkPK == None: return None self.UUID = uuid.uuid4().__str__() self.jobChain = jobChain self.pk = jobChainLinkPK self.unit = unit self.passVar = passVar self.createdDate = databaseInterface.getUTCDate() self.subJobOf = subJobOf sql = """SELECT MicroServiceChainLinks.currentTask, MicroServiceChainLinks.defaultNextChainLink, TasksConfigs.taskType, TasksConfigs.taskTypePKReference, TasksConfigs.description, MicroServiceChainLinks.reloadFileList, Sounds.fileLocation, MicroServiceChainLinks.defaultExitMessage, MicroServiceChainLinks.microserviceGroup FROM MicroServiceChainLinks LEFT OUTER JOIN Sounds ON MicroServiceChainLinks.defaultPlaySound = Sounds.pk JOIN TasksConfigs on MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE MicroServiceChainLinks.pk = '%s'""" % ( jobChainLinkPK.__str__()) c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() if row == None: sqlLock.release() return None while row != None: print row self.currentTask = row[0] self.defaultNextChainLink = row[1] taskType = row[2] taskTypePKReference = row[3] self.description = row[4] self.reloadFileList = row[5] self.defaultSoundFile = row[6] self.defaultExitMessage = row[7] self.microserviceGroup = row[8] row = c.fetchone() sqlLock.release() print "<<<<<<<<< ", self.description, " >>>>>>>>>" self.unit.reload() logJobCreatedSQL(self) if self.createTasks(taskType, taskTypePKReference) == None: self.getNextChainLinkPK(None)
def insertIntoEvents(fileUUID="", eventIdentifierUUID="", eventType="", eventDateTime=databaseInterface.getUTCDate(), eventDetail="", eventOutcome="", eventOutcomeDetailNote=""): agent = getAgentForFileUUID(fileUUID) if not agent: agent = 'NULL' if eventIdentifierUUID == "": eventIdentifierUUID = uuid.uuid4().__str__() databaseInterface.runSQL("""INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote, linkingAgentIdentifier) VALUES ( '""" + escapeForDB(fileUUID) + databaseInterface.separator \ + escapeForDB(eventIdentifierUUID) + databaseInterface.separator \ + escapeForDB(eventType) + databaseInterface.separator \ + escapeForDB(eventDateTime) + databaseInterface.separator \ + escapeForDB(eventDetail) + databaseInterface.separator \ + escapeForDB(eventOutcome) + databaseInterface.separator \ + escapeForDB(eventOutcomeDetailNote) + "', " + escapeForDB(agent) + " )" )
def fileWasRemoved(fileUUID, utcDate=databaseInterface.getUTCDate(), eventDetail="", eventOutcomeDetailNote="", eventOutcome=""): eventIdentifierUUID = uuid.uuid4().__str__() eventType = "file removed" eventDateTime = utcDate insertIntoEvents(fileUUID=fileUUID, \ eventIdentifierUUID=eventIdentifierUUID, \ eventType=eventType, \ eventDateTime=eventDateTime, \ eventDetail=eventDetail, \ eventOutcome=eventOutcome, \ eventOutcomeDetailNote=eventOutcomeDetailNote) databaseInterface.runSQL("UPDATE Files " + \ "SET removedTime='" + utcDate + "', currentLocation=NULL " + \ "WHERE fileUUID='" + fileUUID + "'" )
def checksumFile(filePath, fileUUID): global transferDirectory truePath = filePath.replace("transfer/", transferDirectory, 1) checksum = sha_for_file(truePath) utcDate = databaseInterface.getUTCDate() #Create Event eventIdentifierUUID = uuid.uuid4().__str__() eventType = "message digest calculation" eventDateTime = utcDate eventDetail = 'program="python"; module="hashlib.sha256()" ; file="/usr/lib/python2.6/hashlib.pyc"' eventOutcome = "" eventOutcomeDetailNote = checksum.__str__() databaseInterface.insertIntoEvents(fileUUID=fileUUID, \ eventIdentifierUUID=eventIdentifierUUID, \ eventType=eventType, \ eventDateTime=eventDateTime, \ eventDetail=eventDetail, \ eventOutcome=eventOutcome, \ eventOutcomeDetailNote=eventOutcomeDetailNote)
def __init__(self, jobChain, jobChainLinkPK, unit, passVar=None, subJobOf=""): if jobChainLinkPK == None: return None self.UUID = uuid.uuid4().__str__() self.jobChain = jobChain self.pk = jobChainLinkPK self.unit = unit self.passVar=passVar self.createdDate = databaseInterface.getUTCDate() self.subJobOf = subJobOf sql = """SELECT MicroServiceChainLinks.currentTask, MicroServiceChainLinks.defaultNextChainLink, TasksConfigs.taskType, TasksConfigs.taskTypePKReference, TasksConfigs.description, MicroServiceChainLinks.reloadFileList, Sounds.fileLocation, MicroServiceChainLinks.defaultExitMessage, MicroServiceChainLinks.microserviceGroup FROM MicroServiceChainLinks LEFT OUTER JOIN Sounds ON MicroServiceChainLinks.defaultPlaySound = Sounds.pk JOIN TasksConfigs on MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE MicroServiceChainLinks.pk = '%s'""" % (jobChainLinkPK.__str__()) c, sqlLock = databaseInterface.querySQL(sql) row = c.fetchone() if row == None: sqlLock.release() return None while row != None: print row self.currentTask = row[0] self.defaultNextChainLink = row[1] taskType = row[2] taskTypePKReference = row[3] self.description = row[4] self.reloadFileList = row[5] self.defaultSoundFile = row[6] self.defaultExitMessage = row[7] self.microserviceGroup = row[8] row = c.fetchone() sqlLock.release() print "<<<<<<<<< ", self.description, " >>>>>>>>>" self.unit.reload() logJobCreatedSQL(self) if self.createTasks(taskType, taskTypePKReference) == None: self.getNextChainLinkPK(None)
def removeFileByFileUUID(fileUUID, utcDate = databaseInterface.getUTCDate()): databaseInterface.runSQL("UPDATE Files " + \ "SET removedTime='" + utcDate + "', currentLocation=NULL " + \ "WHERE fileUUID='" + fileUUID + "'" )
def logJobStepCompletedSQL(job): databaseInterface.runSQL( """INSERT INTO jobStepCompleted (jobUUID, step, completedTime) VALUES ( '""" + job.UUID.__str__() + databaseInterface.separator + job.step + databaseInterface.separator + databaseInterface.getUTCDate() + "' )")
def executeCommand(gearman_worker, gearman_job): try: execute = gearman_job.task print "executing:", execute, "{", gearman_job.unique, "}" data = cPickle.loads(gearman_job.data) utcDate = databaseInterface.getUTCDate() arguments = data["arguments"]#.encode("utf-8") if isinstance(arguments, unicode): arguments = arguments.encode("utf-8") #if isinstance(arguments, str): # arguments = unicode(arguments) sInput = "" clientID = gearman_worker.worker_client_id sql = """SELECT Tasks.taskUUID FROM Tasks WHERE taskUUID='%s' AND startTime != 0;""" % (gearman_job.unique.__str__()) rows = databaseInterface.queryAllSQL(sql) if len(rows): exitCode = -1 stdOut = "" stdError = """Detected this task has already started! Unable to determine if it completed successfully.""" return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError}) logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate) if execute not in supportedModules: output = ["Error!", "Error! - Tried to run and unsupported command." ] exitCode = -1 return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]}) command = supportedModules[execute] replacementDic["%date%"] = utcDate replacementDic["%jobCreatedDate%"] = data["createdDate"] #Replace replacement strings for key in replacementDic.iterkeys(): command = command.replace ( key, replacementDic[key] ) arguments = arguments.replace ( key, replacementDic[key] ) key = "%taskUUID%" value = gearman_job.unique.__str__() arguments = arguments.replace(key, value) #execute command command += " " + arguments printOutputLock.acquire() print "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>" printOutputLock.release() exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False) return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError}) #catch OS errors except OSError, ose: traceback.print_exc(file=sys.stdout) printOutputLock.acquire() print >>sys.stderr, "Execution failed:", ose printOutputLock.release() output = ["Archivematica Client Error!", ose.__str__() ] exitCode = 1 return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
createFileSec(os.path.join(baseDirectoryPath, "objects"), structMapDiv) fileSec = etree.Element("fileSec") for group in globalFileGrpsUses: #globalFileGrps.itervalues(): grp = globalFileGrps[group] if len(grp) > 0: fileSec.append(grp) rootNSMap = {None: metsNS} rootNSMap.update(NSMAP) root = etree.Element( "mets", \ nsmap = rootNSMap, \ attrib = { "{" + xsiNS + "}schemaLocation" : "http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/version18/mets.xsd" } ) etree.SubElement(root, "metsHdr").set( "CREATEDATE", databaseInterface.getUTCDate().split(".")[0]) dc = createDublincoreDMDSecFromDBData(SIPMetadataAppliesToType, fileGroupIdentifier) if dc != None: (dmdSec, ID) = dc structMapDiv.set("DMDID", ID) root.append(dmdSec) for dmdSec in dmdSecs: root.append(dmdSec) for amdSec in amdSecs: root.append(amdSec) root.append(fileSec)
def executeCommand(gearman_worker, gearman_job): try: execute = gearman_job.task print "executing:", execute, "{", gearman_job.unique, "}" data = cPickle.loads(gearman_job.data) utcDate = databaseInterface.getUTCDate() arguments = data["arguments"] #.encode("utf-8") if isinstance(arguments, unicode): arguments = arguments.encode("utf-8") #if isinstance(arguments, str): # arguments = unicode(arguments) sInput = "" clientID = gearman_worker.worker_client_id sql = """SELECT Tasks.taskUUID FROM Tasks WHERE taskUUID='%s' AND startTime != 0;""" % ( gearman_job.unique.__str__()) rows = databaseInterface.queryAllSQL(sql) if len(rows): exitCode = -1 stdOut = "" stdError = """Detected this task has already started! Unable to determine if it completed successfully.""" return cPickle.dumps({ "exitCode": exitCode, "stdOut": stdOut, "stdError": stdError }) logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate) if execute not in supportedModules: output = [ "Error!", "Error! - Tried to run and unsupported command." ] exitCode = -1 return cPickle.dumps({ "exitCode": exitCode, "stdOut": output[0], "stdError": output[1] }) command = supportedModules[execute] replacementDic["%date%"] = utcDate replacementDic["%jobCreatedDate%"] = data["createdDate"] #Replace replacement strings for key in replacementDic.iterkeys(): command = command.replace(key, replacementDic[key]) arguments = arguments.replace(key, replacementDic[key]) key = "%taskUUID%" value = gearman_job.unique.__str__() arguments = arguments.replace(key, value) #execute command command += " " + arguments printOutputLock.acquire() print "<processingCommand>{" + gearman_job.unique + "}" + command.__str__( ) + "</processingCommand>" printOutputLock.release() exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False) return cPickle.dumps({ "exitCode": exitCode, "stdOut": stdOut, "stdError": stdError }) #catch OS errors except OSError, ose: traceback.print_exc(file=sys.stdout) printOutputLock.acquire() print >> sys.stderr, "Execution failed:", ose printOutputLock.release() output = ["Archivematica Client Error!", ose.__str__()] exitCode = 1 return cPickle.dumps({ "exitCode": exitCode, "stdOut": output[0], "stdError": output[1] })
def executeCommandReleationship(gearman_worker, gearman_job): """some text""" try: execute = gearman_job.task print "executing:", execute, "{", gearman_job.unique, "}" data = cPickle.loads(gearman_job.data) utcDate = databaseInterface.getUTCDate() opts = data["arguments"] #.encode("utf-8") #if isinstance(arguments, unicode): # arguments = arguments.encode("utf-8") #if isinstance(arguments, str): # arguments = unicode(arguments) sInput = "" clientID = gearman_worker.worker_client_id opts["date"] = utcDate opts["accessDirectory"] = os.path.join(opts['sipPath'], "DIP/objects") + "/" opts["thumbnailDirectory"] = os.path.join(opts['sipPath'], "thumbnails") + "/" print opts for key, value in archivematicaClient.replacementDic.iteritems(): for key2 in opts: opts[key2] = opts[key2].replace(key, value) replacementDic = getReplacementDic(opts) opts[ "prependStdOut"] = """Operating on file: {%s}%s \r\nUsing %s command classifications""" % ( opts["fileUUID"], replacementDic["%fileName%"], opts["commandClassification"]) opts["prependStdError"] = "\r\nSTDError:" # print clientID, execute, data archivematicaClient.logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate) cl = transcoder.CommandLinker(opts["CommandRelationship"], replacementDic, opts, onceNormalized) cl.execute() co = cl.commandObject exitCode = co.exitCode stdOut = "%s \r\n%s" % (opts["prependStdOut"], co.stdOut) if not co.stdError or co.stdError.isspace(): stdError = "" else: stdError = "%s \r\n%s" % (opts["prependStdError"], co.stdError) #TODO add date to ops #Replace replacement strings #archivematicaClient.printOutputLock.acquire() #print >>sys.stderr, "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>" #archivematicaClient.printOutputLock.release() #exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False) return cPickle.dumps({ "exitCode": exitCode, "stdOut": stdOut, "stdError": stdError }) #catch OS errors except OSError, ose: archivematicaClient.printOutputLock.acquire() traceback.print_exc(file=sys.stdout) print >> sys.stderr, "Execution failed:", ose archivematicaClient.printOutputLock.release() output = ["Config Error!", ose.__str__()] exitCode = 1 return cPickle.dumps({ "exitCode": exitCode, "stdOut": output[0], "stdError": output[1] })
structMapDiv = newChild(structMapDiv, "div", sets=[("TYPE","Directory"), ("LABEL","objects") ]) createFileSec(os.path.join(baseDirectoryPath, "objects"), structMapDiv) fileSec = etree.Element( "fileSec") for group in globalFileGrpsUses: #globalFileGrps.itervalues(): grp = globalFileGrps[group] if len(grp) > 0: fileSec.append(grp) rootNSMap = {None: metsNS} rootNSMap.update(NSMAP) root = etree.Element( "mets", \ nsmap = rootNSMap, \ attrib = { "{" + xsiNS + "}schemaLocation" : "http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/version18/mets.xsd" } ) etree.SubElement(root,"metsHdr").set("CREATEDATE", databaseInterface.getUTCDate().split(".")[0]) dc = createDublincoreDMDSecFromDBData(SIPMetadataAppliesToType, fileGroupIdentifier) if dc != None: (dmdSec, ID) = dc structMapDiv.set("DMDID", ID) root.append(dmdSec) for dmdSec in dmdSecs: root.append(dmdSec) for amdSec in amdSecs: root.append(amdSec)
def removeFileByFileUUID(fileUUID, utcDate=databaseInterface.getUTCDate()): databaseInterface.runSQL("UPDATE Files " + \ "SET removedTime='" + utcDate + "', currentLocation=NULL " + \ "WHERE fileUUID='" + fileUUID + "'" )