def removeFile(filePath, utcDate=databaseInterface.getUTCDate()):
global separator
print "removing: ", filePath
filesWithMatchingPath = []
sqlLoggingLock.acquire()
#Find the file pk/UUID
c = MCPloggingSQL.database.cursor()
sql = """SELECT fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '""" + MySQLdb.escape_string(
filePath) + """';"""
c.execute(sql)
row = c.fetchone()
while row != None:
filesWithMatchingPath.append(row[0])
row = c.fetchone()
sqlLoggingLock.release()
#Update the database
for file in filesWithMatchingPath:
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "file removed"
eventDateTime = utcDate
eventDetail = ""
eventOutcomeDetailNote = "removed from: " + filePath
databaseInterface.insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=eventIdentifierUUID, \
eventType=eventType, \
eventDateTime=eventDateTime, \
eventDetail=eventDetail, \
eventOutcome=eventOutcome, \
eventOutcomeDetailNote=eventOutcomeDetailNote)
removeFileByFileUUID(fileUUID, utcDate)
def insertIntoEvents(
fileUUID="",
eventIdentifierUUID="",
eventType="",
eventDateTime=databaseInterface.getUTCDate(),
eventDetail="",
eventOutcome="",
eventOutcomeDetailNote="",
):
if eventIdentifierUUID == "":
eventIdentifierUUID = uuid.uuid4().__str__()
databaseInterface.runSQL(
"""INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote)
VALUES ( '"""
+ escapeForDB(fileUUID)
+ databaseInterface.separator
+ escapeForDB(eventIdentifierUUID)
+ databaseInterface.separator
+ escapeForDB(eventType)
+ databaseInterface.separator
+ escapeForDB(eventDateTime)
+ databaseInterface.separator
+ escapeForDB(eventDetail)
+ databaseInterface.separator
+ escapeForDB(eventOutcome)
+ databaseInterface.separator
+ escapeForDB(eventOutcomeDetailNote)
+ "' )"
)
def xmlCreateFileAssociationBetween(originalFileFullPath, outputFromNormalizationFileFullPath, SIPFullPath, sipUUID, eventDetailText, eventOutcomeDetailNote, outputFileUUID=""):
#assign file UUID
date = databaseInterface.getUTCDate()
if outputFileUUID == "":
outputFileUUID = uuid.uuid4().__str__()
originalFilePathRelativeToSIP = originalFileFullPath.replace(SIPFullPath,"%SIPDirectory%", 1)
sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(originalFilePathRelativeToSIP) + "' AND Files.sipUUID = '" + sipUUID + "';"
print sql
rows = databaseInterface.queryAllSQL(sql)
print rows
fileUUID = rows[0][0]
filePathRelativeToSIP = outputFromNormalizationFileFullPath.replace(SIPFullPath,"%SIPDirectory%", 1)
addFileToSIP(filePathRelativeToSIP, outputFileUUID, sipUUID, uuid.uuid4().__str__(), date, sourceType="creation", use="preservation")
updateSizeAndChecksum(outputFileUUID, outputFromNormalizationFileFullPath, date, uuid.uuid4().__str__())
taskUUID = uuid.uuid4().__str__()
insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=taskUUID, \
eventType="normalization", \
eventDateTime=date, \
eventDetail=eventDetailText, \
eventOutcome="", \
eventOutcomeDetailNote=eventOutcomeDetailNote)
insertIntoDerivations(sourceFileUUID=fileUUID, derivedFileUUID=outputFileUUID, relatedEventUUID=taskUUID)
def fileWasRemoved(
fileUUID, utcDate=databaseInterface.getUTCDate(), eventDetail="", eventOutcomeDetailNote="", eventOutcome=""
):
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "file removed"
eventDateTime = utcDate
insertIntoEvents(
fileUUID=fileUUID,
eventIdentifierUUID=eventIdentifierUUID,
eventType=eventType,
eventDateTime=eventDateTime,
eventDetail=eventDetail,
eventOutcome=eventOutcome,
eventOutcomeDetailNote=eventOutcomeDetailNote,
)
databaseInterface.runSQL(
"UPDATE Files "
+ "SET removedTime='"
+ utcDate
+ "', currentLocation=NULL "
+ "WHERE fileUUID='"
+ fileUUID
+ "'"
)
def logTaskCreatedSQL(taskManager, commandReplacementDic, taskUUID, arguments):
taskUUID = taskUUID
jobUUID = taskManager.jobChainLink.UUID
fileUUID = ""
if "%fileUUID%" in commandReplacementDic:
fileUUID = commandReplacementDic["%fileUUID%"]
taskexec = taskManager.execute
fileName = os.path.basename(os.path.abspath(commandReplacementDic["%relativeLocation%"]))
databaseInterface.runSQL(
"""INSERT INTO Tasks (taskUUID, jobUUID, fileUUID, fileName, exec, arguments, createdTime)
VALUES ( '"""
+ taskUUID
+ databaseInterface.separator
+ jobUUID
+ databaseInterface.separator
+ escapeForDB(fileUUID)
+ databaseInterface.separator
+ escapeForDB(fileName)
+ databaseInterface.separator
+ escapeForDB(taskexec)
+ databaseInterface.separator
+ escapeForDB(arguments)
+ databaseInterface.separator
+ databaseInterface.getUTCDate()
+ "' )"
)
def insertIntoFiles(fileUUID,
filePath,
enteredSystem=databaseInterface.getUTCDate(),
transferUUID="",
sipUUID="",
use="original"):
if transferUUID != "" and sipUUID == "":
databaseInterface.runSQL("""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, transferUUID)
VALUES ( '""" + fileUUID + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ enteredSystem + databaseInterface.separator \
+ use + databaseInterface.separator \
+ transferUUID + "' )" )
elif transferUUID == "" and sipUUID != "":
databaseInterface.runSQL("""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, sipUUID)
VALUES ( '""" + fileUUID + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ enteredSystem + databaseInterface.separator \
+ use + databaseInterface.separator \
+ sipUUID + "' )" )
else:
print >> sys.stderr, "not supported yet - both SIP and transfer UUID's defined (or neither defined)"
print >> sys.stderr, "SIP UUID:", sipUUID
print >> sys.stderr, "transferUUID:", transferUUID
raise Exception(
"not supported yet - both SIP and transfer UUID's defined (or neither defined)",
sipUUID + "-" + transferUUID)
def process_IN_MOVED_FROM(self, event):
global movedFromCount
#print event
#print "Transfer IN_MOVED_FROM"
#Wait for a moved to, and if one doesn't occur, consider it moved outside of the system.
movedFromPath = os.path.join(event.path, event.name).replace(\
self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), \
"%transferDirectory%", 1)
filesMoved = []
sql = """SELECT fileUUID, currentLocation FROM Files WHERE transferUUID = '""" + self.unit.UUID + "' AND removedTime = 0 AND currentLocation LIKE '" + MySQLdb.escape_string(movedFromPath).replace("%", "\%") + "%';"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
#print row
filesMoved.append(row)
row = c.fetchone()
sqlLock.release()
movedFromLock.acquire()
utcDate = databaseInterface.getUTCDate()
timer = threading.Timer(archivematicaMCP.config.getint('MCPServer', "delayTimer"), timerExpired, args=[event, utcDate], kwargs={})
movedFrom[event.cookie] = (movedFromPath, filesMoved, timer)
movedFromCount.value = movedFromCount.value + 1
movedFromLock.release()
#create timer to check if it's claimed by a move to
timer.start()
def removeFile(filePath, utcDate = databaseInterface.getUTCDate()):
global separator
print "removing: ", filePath
filesWithMatchingPath = []
sqlLoggingLock.acquire()
#Find the file pk/UUID
c=MCPloggingSQL.database.cursor()
sql = """SELECT fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '""" + MySQLdb.escape_string(filePath) + """';"""
c.execute(sql)
row = c.fetchone()
while row != None:
filesWithMatchingPath.append(row[0])
row = c.fetchone()
sqlLoggingLock.release()
#Update the database
for file in filesWithMatchingPath:
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "file removed"
eventDateTime = utcDate
eventDetail = ""
eventOutcomeDetailNote = "removed from: " + filePath
databaseInterface.insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=eventIdentifierUUID, \
eventType=eventType, \
eventDateTime=eventDateTime, \
eventDetail=eventDetail, \
eventOutcome=eventOutcome, \
eventOutcomeDetailNote=eventOutcomeDetailNote)
removeFileByFileUUID(fileUUID, utcDate)
def executeCommandReleationship(gearman_worker, gearman_job):
"""some text"""
try:
execute = gearman_job.task
print "executing:", execute, "{", gearman_job.unique, "}"
data = cPickle.loads(gearman_job.data)
utcDate = databaseInterface.getUTCDate()
opts = data["arguments"]#.encode("utf-8")
#if isinstance(arguments, unicode):
# arguments = arguments.encode("utf-8")
#if isinstance(arguments, str):
# arguments = unicode(arguments)
sInput = ""
clientID = gearman_worker.worker_client_id
opts["date"] = utcDate
opts["accessDirectory"] = os.path.join(opts['sipPath'], "DIP/objects") + "/"
opts["thumbnailDirectory"] = os.path.join(opts['sipPath'], "thumbnails") + "/"
print opts
for key, value in archivematicaClient.replacementDic.iteritems():
for key2 in opts:
opts[key2] = opts[key2].replace(key, value)
replacementDic = getReplacementDic(opts)
#if True:
opts["prependStdOut"] = """Operating on file: {%s}%s \r\nUsing %s command classifications""" % (opts["fileUUID"], replacementDic["%fileName%"], opts["commandClassification"])
opts["prependStdError"] = "\r\nSTDError:"
# print clientID, execute, data
archivematicaClient.logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate)
cl = transcoder.CommandLinker(opts["CommandRelationship"], replacementDic, opts, onceNormalized)
cl.execute()
co = cl.commandObject
exitCode = co.exitCode
stdOut = "%s \r\n%s" % (opts["prependStdOut"], co.stdOut)
if not co.stdError or co.stdError.isspace():
stdError = ""
else:
stdError = "%s \r\n%s" % (opts["prependStdError"], co.stdError)
#TODO add date to ops
#Replace replacement strings
#archivematicaClient.printOutputLock.acquire()
#print >>sys.stderr, "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>"
#archivematicaClient.printOutputLock.release()
#exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False)
return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError})
#catch OS errors
except OSError, ose:
archivematicaClient.printOutputLock.acquire()
traceback.print_exc(file=sys.stdout)
print >>sys.stderr, "Execution failed:", ose
archivematicaClient.printOutputLock.release()
output = ["Config Error!", ose.__str__() ]
exitCode = 1
return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
def debugMonitor():
global countOfCreateUnitAndJobChainThreaded
while True:
dblockstatus = "SQL Lock: Locked"
if databaseInterface.sqlLock.acquire(False):
databaseInterface.sqlLock.release()
dblockstatus = "SQL Lock: Unlocked"
print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate(), "\tThreadCount: ", threading.activeCount(), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>"
time.sleep(60)
def debugMonitor():
"""Periodically prints out status of MCP, including whether the database lock is locked, thread count, etc."""
global countOfCreateUnitAndJobChainThreaded
while True:
dblockstatus = "SQL Lock: Locked"
if databaseInterface.sqlLock.acquire(False):
databaseInterface.sqlLock.release()
dblockstatus = "SQL Lock: Unlocked"
print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate(), "\tThreadCount: ", threading.activeCount(), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>"
time.sleep(3600)
def logJobStepCompletedSQL(job):
databaseInterface.runSQL(
"""INSERT INTO jobStepCompleted (jobUUID, step, completedTime)
VALUES ( '"""
+ job.UUID.__str__()
+ databaseInterface.separator
+ job.step
+ databaseInterface.separator
+ databaseInterface.getUTCDate()
+ "' )"
)
def logTaskCompletedSQL(task):
print "Logging task output to db", task.UUID
taskUUID = task.UUID.__str__()
exitCode = task.results["exitCode"].__str__()
stdOut = task.results["stdOut"]
stdError = task.results["stdError"]
databaseInterface.runSQL("UPDATE Tasks " + \
"SET endTime='" + databaseInterface.getUTCDate() +"', exitCode='" + exitCode + "', " + \
"stdOut='" + escapeForDB(stdOut) + "', stdError='" + escapeForDB(stdError) + "' "
"WHERE taskUUID='" + taskUUID + "'" )
def executeCommand(gearman_worker, gearman_job):
try:
execute = gearman_job.task
print "executing:", execute, "{", gearman_job.unique, "}"
data = cPickle.loads(gearman_job.data)
utcDate = databaseInterface.getUTCDate()
arguments = data["arguments"]#.encode("utf-8")
if isinstance(arguments, unicode):
arguments = arguments.encode("utf-8")
#if isinstance(arguments, str):
# arguments = unicode(arguments)
sInput = ""
clientID = gearman_worker.worker_client_id
#if True:
# print clientID, execute, data
logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate)
if execute not in supportedModules:
output = ["Error!", "Error! - Tried to run and unsupported command." ]
exitCode = -1
return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
command = supportedModules[execute]
replacementDic["%date%"] = utcDate
replacementDic["%jobCreatedDate%"] = data["createdDate"]
#Replace replacement strings
for key in replacementDic.iterkeys():
command = command.replace ( key, replacementDic[key] )
arguments = arguments.replace ( key, replacementDic[key] )
key = "%taskUUID%"
value = gearman_job.unique.__str__()
arguments = arguments.replace(key, value)
#execute command
command += " " + arguments
printOutputLock.acquire()
print >>sys.stderr, "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>"
printOutputLock.release()
exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False)
return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError})
#catch OS errors
except OSError, ose:
traceback.print_exc(file=sys.stdout)
printOutputLock.acquire()
print >>sys.stderr, "Execution failed:", ose
printOutputLock.release()
output = ["Config Error!", ose.__str__() ]
exitCode = 1
return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
def debugMonitor():
global countOfCreateUnitAndJobChainThreaded
while True:
dblockstatus = "SQL Lock: Locked"
if databaseInterface.sqlLock.acquire(False):
databaseInterface.sqlLock.release()
dblockstatus = "SQL Lock: Unlocked"
print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate(
), "\tThreadCount: ", threading.activeCount(
), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>"
time.sleep(60)
def logTaskCompletedSQL(task):
print "Logging task output to db", task.UUID
taskUUID = task.UUID.__str__()
exitCode = task.results["exitCode"].__str__()
stdOut = task.results["stdOut"]
stdError = task.results["stdError"]
databaseInterface.runSQL("UPDATE Tasks " + \
"SET endTime='" + databaseInterface.getUTCDate() +"', exitCode='" + exitCode + "', " + \
"stdOut='" + escapeForDB(stdOut) + "', stdError='" + escapeForDB(stdError) + "' "
"WHERE taskUUID='" + taskUUID + "'" )
def debugMonitor():
"""Periodically prints out status of MCP, including whether the database lock is locked, thread count, etc."""
global countOfCreateUnitAndJobChainThreaded
while True:
dblockstatus = "SQL Lock: Locked"
if databaseInterface.sqlLock.acquire(False):
databaseInterface.sqlLock.release()
dblockstatus = "SQL Lock: Unlocked"
print "<DEBUG type=\"archivematicaMCP\">", "\tDate Time: ", databaseInterface.getUTCDate(
), "\tThreadCount: ", threading.activeCount(
), "\tcountOfCreateUnitAndJobChainThreaded", countOfCreateUnitAndJobChainThreaded, dblockstatus, "</DEBUG>"
time.sleep(3600)
def xmlCreateFileAssociationBetween(originalFileFullPath,
outputFromNormalizationFileFullPath,
SIPFullPath,
sipUUID,
eventDetailText,
eventOutcomeDetailNote,
outputFileUUID=""):
#assign file UUID
date = databaseInterface.getUTCDate()
if outputFileUUID == "":
outputFileUUID = uuid.uuid4().__str__()
originalFilePathRelativeToSIP = originalFileFullPath.replace(
SIPFullPath, "%SIPDirectory%", 1)
sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(
originalFilePathRelativeToSIP
) + "' AND Files.sipUUID = '" + sipUUID + "';"
print sql
rows = databaseInterface.queryAllSQL(sql)
print rows
fileUUID = rows[0][0]
filePathRelativeToSIP = outputFromNormalizationFileFullPath.replace(
SIPFullPath, "%SIPDirectory%", 1)
addFileToSIP(filePathRelativeToSIP,
outputFileUUID,
sipUUID,
uuid.uuid4().__str__(),
date,
sourceType="creation",
use="preservation")
updateSizeAndChecksum(outputFileUUID, outputFromNormalizationFileFullPath,
date,
uuid.uuid4().__str__())
taskUUID = uuid.uuid4().__str__()
insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=taskUUID, \
eventType="normalization", \
eventDateTime=date, \
eventDetail=eventDetailText, \
eventOutcome="", \
eventOutcomeDetailNote=eventOutcomeDetailNote)
insertIntoDerivations(sourceFileUUID=fileUUID,
derivedFileUUID=outputFileUUID,
relatedEventUUID=taskUUID)
def insertIntoFiles(
fileUUID, filePath, enteredSystem=databaseInterface.getUTCDate(), transferUUID="", sipUUID="", use="original"
):
if transferUUID != "" and sipUUID == "":
databaseInterface.runSQL(
"""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, transferUUID)
VALUES ( '"""
+ fileUUID
+ databaseInterface.separator
+ escapeForDB(filePath)
+ databaseInterface.separator
+ escapeForDB(filePath)
+ databaseInterface.separator
+ enteredSystem
+ databaseInterface.separator
+ use
+ databaseInterface.separator
+ transferUUID
+ "' )"
)
elif transferUUID == "" and sipUUID != "":
databaseInterface.runSQL(
"""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, sipUUID)
VALUES ( '"""
+ fileUUID
+ databaseInterface.separator
+ escapeForDB(filePath)
+ databaseInterface.separator
+ escapeForDB(filePath)
+ databaseInterface.separator
+ enteredSystem
+ databaseInterface.separator
+ use
+ databaseInterface.separator
+ sipUUID
+ "' )"
)
else:
print >> sys.stderr, "not supported yet - both SIP and transfer UUID's defined (or neither defined)"
print >> sys.stderr, "SIP UUID:", sipUUID
print >> sys.stderr, "transferUUID:", transferUUID
raise Exception(
"not supported yet - both SIP and transfer UUID's defined (or neither defined)",
sipUUID + "-" + transferUUID,
)
def logTaskCreatedSQL(taskManager, commandReplacementDic, taskUUID, arguments):
taskUUID = taskUUID
jobUUID = taskManager.jobChainLink.UUID
fileUUID = ""
if "%fileUUID%" in commandReplacementDic:
fileUUID = commandReplacementDic["%fileUUID%"]
taskexec = taskManager.execute
fileName = os.path.basename(
os.path.abspath(commandReplacementDic["%relativeLocation%"]))
databaseInterface.runSQL("""INSERT INTO Tasks (taskUUID, jobUUID, fileUUID, fileName, exec, arguments, createdTime)
VALUES ( '""" + taskUUID + databaseInterface.separator \
+ jobUUID + databaseInterface.separator \
+ escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(fileName) + databaseInterface.separator \
+ escapeForDB(taskexec) + databaseInterface.separator \
+ escapeForDB(arguments) + databaseInterface.separator \
+ databaseInterface.getUTCDate() + "' )" )
def __init__(self,
jobChain,
jobChainLinkPK,
unit,
passVar=None,
subJobOf=""):
if jobChainLinkPK == None:
return None
self.UUID = uuid.uuid4().__str__()
self.jobChain = jobChain
self.pk = jobChainLinkPK
self.unit = unit
self.passVar = passVar
self.createdDate = databaseInterface.getUTCDate()
self.subJobOf = subJobOf
sql = """SELECT MicroServiceChainLinks.currentTask, MicroServiceChainLinks.defaultNextChainLink, TasksConfigs.taskType, TasksConfigs.taskTypePKReference, TasksConfigs.description, MicroServiceChainLinks.reloadFileList, Sounds.fileLocation, MicroServiceChainLinks.defaultExitMessage, MicroServiceChainLinks.microserviceGroup FROM MicroServiceChainLinks LEFT OUTER JOIN Sounds ON MicroServiceChainLinks.defaultPlaySound = Sounds.pk JOIN TasksConfigs on MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE MicroServiceChainLinks.pk = '%s'""" % (
jobChainLinkPK.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
if row == None:
sqlLock.release()
return None
while row != None:
print row
self.currentTask = row[0]
self.defaultNextChainLink = row[1]
taskType = row[2]
taskTypePKReference = row[3]
self.description = row[4]
self.reloadFileList = row[5]
self.defaultSoundFile = row[6]
self.defaultExitMessage = row[7]
self.microserviceGroup = row[8]
row = c.fetchone()
sqlLock.release()
print "<<<<<<<<< ", self.description, " >>>>>>>>>"
self.unit.reload()
logJobCreatedSQL(self)
if self.createTasks(taskType, taskTypePKReference) == None:
self.getNextChainLinkPK(None)
def insertIntoEvents(fileUUID="",
eventIdentifierUUID="",
eventType="",
eventDateTime=databaseInterface.getUTCDate(),
eventDetail="",
eventOutcome="",
eventOutcomeDetailNote=""):
agent = getAgentForFileUUID(fileUUID)
if not agent:
agent = 'NULL'
if eventIdentifierUUID == "":
eventIdentifierUUID = uuid.uuid4().__str__()
databaseInterface.runSQL("""INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote, linkingAgentIdentifier)
VALUES ( '""" + escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(eventIdentifierUUID) + databaseInterface.separator \
+ escapeForDB(eventType) + databaseInterface.separator \
+ escapeForDB(eventDateTime) + databaseInterface.separator \
+ escapeForDB(eventDetail) + databaseInterface.separator \
+ escapeForDB(eventOutcome) + databaseInterface.separator \
+ escapeForDB(eventOutcomeDetailNote) + "', " + escapeForDB(agent) + " )" )
def fileWasRemoved(fileUUID,
utcDate=databaseInterface.getUTCDate(),
eventDetail="",
eventOutcomeDetailNote="",
eventOutcome=""):
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "file removed"
eventDateTime = utcDate
insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=eventIdentifierUUID, \
eventType=eventType, \
eventDateTime=eventDateTime, \
eventDetail=eventDetail, \
eventOutcome=eventOutcome, \
eventOutcomeDetailNote=eventOutcomeDetailNote)
databaseInterface.runSQL("UPDATE Files " + \
"SET removedTime='" + utcDate + "', currentLocation=NULL " + \
"WHERE fileUUID='" + fileUUID + "'" )
def checksumFile(filePath, fileUUID):
global transferDirectory
truePath = filePath.replace("transfer/", transferDirectory, 1)
checksum = sha_for_file(truePath)
utcDate = databaseInterface.getUTCDate()
#Create Event
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "message digest calculation"
eventDateTime = utcDate
eventDetail = 'program="python"; module="hashlib.sha256()" ; file="/usr/lib/python2.6/hashlib.pyc"'
eventOutcome = ""
eventOutcomeDetailNote = checksum.__str__()
databaseInterface.insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=eventIdentifierUUID, \
eventType=eventType, \
eventDateTime=eventDateTime, \
eventDetail=eventDetail, \
eventOutcome=eventOutcome, \
eventOutcomeDetailNote=eventOutcomeDetailNote)
def checksumFile(filePath, fileUUID):
global transferDirectory
truePath = filePath.replace("transfer/", transferDirectory, 1)
checksum = sha_for_file(truePath)
utcDate = databaseInterface.getUTCDate()
#Create Event
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "message digest calculation"
eventDateTime = utcDate
eventDetail = 'program="python"; module="hashlib.sha256()" ; file="/usr/lib/python2.6/hashlib.pyc"'
eventOutcome = ""
eventOutcomeDetailNote = checksum.__str__()
databaseInterface.insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=eventIdentifierUUID, \
eventType=eventType, \
eventDateTime=eventDateTime, \
eventDetail=eventDetail, \
eventOutcome=eventOutcome, \
eventOutcomeDetailNote=eventOutcomeDetailNote)
def __init__(self, jobChain, jobChainLinkPK, unit, passVar=None, subJobOf=""):
if jobChainLinkPK == None:
return None
self.UUID = uuid.uuid4().__str__()
self.jobChain = jobChain
self.pk = jobChainLinkPK
self.unit = unit
self.passVar=passVar
self.createdDate = databaseInterface.getUTCDate()
self.subJobOf = subJobOf
sql = """SELECT MicroServiceChainLinks.currentTask, MicroServiceChainLinks.defaultNextChainLink, TasksConfigs.taskType, TasksConfigs.taskTypePKReference, TasksConfigs.description, MicroServiceChainLinks.reloadFileList, Sounds.fileLocation, MicroServiceChainLinks.defaultExitMessage, MicroServiceChainLinks.microserviceGroup FROM MicroServiceChainLinks LEFT OUTER JOIN Sounds ON MicroServiceChainLinks.defaultPlaySound = Sounds.pk JOIN TasksConfigs on MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE MicroServiceChainLinks.pk = '%s'""" % (jobChainLinkPK.__str__())
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
if row == None:
sqlLock.release()
return None
while row != None:
print row
self.currentTask = row[0]
self.defaultNextChainLink = row[1]
taskType = row[2]
taskTypePKReference = row[3]
self.description = row[4]
self.reloadFileList = row[5]
self.defaultSoundFile = row[6]
self.defaultExitMessage = row[7]
self.microserviceGroup = row[8]
row = c.fetchone()
sqlLock.release()
print "<<<<<<<<< ", self.description, " >>>>>>>>>"
self.unit.reload()
logJobCreatedSQL(self)
if self.createTasks(taskType, taskTypePKReference) == None:
self.getNextChainLinkPK(None)
def insertIntoEvents(fileUUID="", eventIdentifierUUID="", eventType="", eventDateTime=databaseInterface.getUTCDate(), eventDetail="", eventOutcome="", eventOutcomeDetailNote=""):
agent = getAgentForFileUUID(fileUUID)
if not agent:
agent = 'NULL'
if eventIdentifierUUID == "":
eventIdentifierUUID = uuid.uuid4().__str__()
databaseInterface.runSQL("""INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote, linkingAgentIdentifier)
VALUES ( '""" + escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(eventIdentifierUUID) + databaseInterface.separator \
+ escapeForDB(eventType) + databaseInterface.separator \
+ escapeForDB(eventDateTime) + databaseInterface.separator \
+ escapeForDB(eventDetail) + databaseInterface.separator \
+ escapeForDB(eventOutcome) + databaseInterface.separator \
+ escapeForDB(eventOutcomeDetailNote) + "', " + escapeForDB(agent) + " )" )
def removeFileByFileUUID(fileUUID, utcDate = databaseInterface.getUTCDate()):
databaseInterface.runSQL("UPDATE Files " + \
"SET removedTime='" + utcDate + "', currentLocation=NULL " + \
"WHERE fileUUID='" + fileUUID + "'" )
def logJobStepCompletedSQL(job):
databaseInterface.runSQL(
"""INSERT INTO jobStepCompleted (jobUUID, step, completedTime)
VALUES ( '""" + job.UUID.__str__() + databaseInterface.separator +
job.step + databaseInterface.separator +
databaseInterface.getUTCDate() + "' )")
def executeCommand(gearman_worker, gearman_job):
try:
execute = gearman_job.task
print "executing:", execute, "{", gearman_job.unique, "}"
data = cPickle.loads(gearman_job.data)
utcDate = databaseInterface.getUTCDate()
arguments = data["arguments"]#.encode("utf-8")
if isinstance(arguments, unicode):
arguments = arguments.encode("utf-8")
#if isinstance(arguments, str):
# arguments = unicode(arguments)
sInput = ""
clientID = gearman_worker.worker_client_id
sql = """SELECT Tasks.taskUUID FROM Tasks WHERE taskUUID='%s' AND startTime != 0;""" % (gearman_job.unique.__str__())
rows = databaseInterface.queryAllSQL(sql)
if len(rows):
exitCode = -1
stdOut = ""
stdError = """Detected this task has already started!
Unable to determine if it completed successfully."""
return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError})
logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate)
if execute not in supportedModules:
output = ["Error!", "Error! - Tried to run and unsupported command." ]
exitCode = -1
return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
command = supportedModules[execute]
replacementDic["%date%"] = utcDate
replacementDic["%jobCreatedDate%"] = data["createdDate"]
#Replace replacement strings
for key in replacementDic.iterkeys():
command = command.replace ( key, replacementDic[key] )
arguments = arguments.replace ( key, replacementDic[key] )
key = "%taskUUID%"
value = gearman_job.unique.__str__()
arguments = arguments.replace(key, value)
#execute command
command += " " + arguments
printOutputLock.acquire()
print "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>"
printOutputLock.release()
exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False)
return cPickle.dumps({"exitCode" : exitCode, "stdOut": stdOut, "stdError": stdError})
#catch OS errors
except OSError, ose:
traceback.print_exc(file=sys.stdout)
printOutputLock.acquire()
print >>sys.stderr, "Execution failed:", ose
printOutputLock.release()
output = ["Archivematica Client Error!", ose.__str__() ]
exitCode = 1
return cPickle.dumps({"exitCode" : exitCode, "stdOut": output[0], "stdError": output[1]})
createFileSec(os.path.join(baseDirectoryPath, "objects"), structMapDiv)
fileSec = etree.Element("fileSec")
for group in globalFileGrpsUses: #globalFileGrps.itervalues():
grp = globalFileGrps[group]
if len(grp) > 0:
fileSec.append(grp)
rootNSMap = {None: metsNS}
rootNSMap.update(NSMAP)
root = etree.Element( "mets", \
nsmap = rootNSMap, \
attrib = { "{" + xsiNS + "}schemaLocation" : "http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/version18/mets.xsd" } )
etree.SubElement(root, "metsHdr").set(
"CREATEDATE",
databaseInterface.getUTCDate().split(".")[0])
dc = createDublincoreDMDSecFromDBData(SIPMetadataAppliesToType,
fileGroupIdentifier)
if dc != None:
(dmdSec, ID) = dc
structMapDiv.set("DMDID", ID)
root.append(dmdSec)
for dmdSec in dmdSecs:
root.append(dmdSec)
for amdSec in amdSecs:
root.append(amdSec)
root.append(fileSec)
def executeCommand(gearman_worker, gearman_job):
try:
execute = gearman_job.task
print "executing:", execute, "{", gearman_job.unique, "}"
data = cPickle.loads(gearman_job.data)
utcDate = databaseInterface.getUTCDate()
arguments = data["arguments"] #.encode("utf-8")
if isinstance(arguments, unicode):
arguments = arguments.encode("utf-8")
#if isinstance(arguments, str):
# arguments = unicode(arguments)
sInput = ""
clientID = gearman_worker.worker_client_id
sql = """SELECT Tasks.taskUUID FROM Tasks WHERE taskUUID='%s' AND startTime != 0;""" % (
gearman_job.unique.__str__())
rows = databaseInterface.queryAllSQL(sql)
if len(rows):
exitCode = -1
stdOut = ""
stdError = """Detected this task has already started!
Unable to determine if it completed successfully."""
return cPickle.dumps({
"exitCode": exitCode,
"stdOut": stdOut,
"stdError": stdError
})
logTaskAssignedSQL(gearman_job.unique.__str__(), clientID, utcDate)
if execute not in supportedModules:
output = [
"Error!", "Error! - Tried to run and unsupported command."
]
exitCode = -1
return cPickle.dumps({
"exitCode": exitCode,
"stdOut": output[0],
"stdError": output[1]
})
command = supportedModules[execute]
replacementDic["%date%"] = utcDate
replacementDic["%jobCreatedDate%"] = data["createdDate"]
#Replace replacement strings
for key in replacementDic.iterkeys():
command = command.replace(key, replacementDic[key])
arguments = arguments.replace(key, replacementDic[key])
key = "%taskUUID%"
value = gearman_job.unique.__str__()
arguments = arguments.replace(key, value)
#execute command
command += " " + arguments
printOutputLock.acquire()
print "<processingCommand>{" + gearman_job.unique + "}" + command.__str__(
) + "</processingCommand>"
printOutputLock.release()
exitCode, stdOut, stdError = executeOrRun("command",
command,
sInput,
printing=False)
return cPickle.dumps({
"exitCode": exitCode,
"stdOut": stdOut,
"stdError": stdError
})
#catch OS errors
except OSError, ose:
traceback.print_exc(file=sys.stdout)
printOutputLock.acquire()
print >> sys.stderr, "Execution failed:", ose
printOutputLock.release()
output = ["Archivematica Client Error!", ose.__str__()]
exitCode = 1
return cPickle.dumps({
"exitCode": exitCode,
"stdOut": output[0],
"stdError": output[1]
})
def executeCommandReleationship(gearman_worker, gearman_job):
"""some text"""
try:
execute = gearman_job.task
print "executing:", execute, "{", gearman_job.unique, "}"
data = cPickle.loads(gearman_job.data)
utcDate = databaseInterface.getUTCDate()
opts = data["arguments"] #.encode("utf-8")
#if isinstance(arguments, unicode):
# arguments = arguments.encode("utf-8")
#if isinstance(arguments, str):
# arguments = unicode(arguments)
sInput = ""
clientID = gearman_worker.worker_client_id
opts["date"] = utcDate
opts["accessDirectory"] = os.path.join(opts['sipPath'],
"DIP/objects") + "/"
opts["thumbnailDirectory"] = os.path.join(opts['sipPath'],
"thumbnails") + "/"
print opts
for key, value in archivematicaClient.replacementDic.iteritems():
for key2 in opts:
opts[key2] = opts[key2].replace(key, value)
replacementDic = getReplacementDic(opts)
opts[
"prependStdOut"] = """Operating on file: {%s}%s \r\nUsing %s command classifications""" % (
opts["fileUUID"], replacementDic["%fileName%"],
opts["commandClassification"])
opts["prependStdError"] = "\r\nSTDError:"
# print clientID, execute, data
archivematicaClient.logTaskAssignedSQL(gearman_job.unique.__str__(),
clientID, utcDate)
cl = transcoder.CommandLinker(opts["CommandRelationship"],
replacementDic, opts, onceNormalized)
cl.execute()
co = cl.commandObject
exitCode = co.exitCode
stdOut = "%s \r\n%s" % (opts["prependStdOut"], co.stdOut)
if not co.stdError or co.stdError.isspace():
stdError = ""
else:
stdError = "%s \r\n%s" % (opts["prependStdError"], co.stdError)
#TODO add date to ops
#Replace replacement strings
#archivematicaClient.printOutputLock.acquire()
#print >>sys.stderr, "<processingCommand>{" + gearman_job.unique + "}" + command.__str__() + "</processingCommand>"
#archivematicaClient.printOutputLock.release()
#exitCode, stdOut, stdError = executeOrRun("command", command, sInput, printing=False)
return cPickle.dumps({
"exitCode": exitCode,
"stdOut": stdOut,
"stdError": stdError
})
#catch OS errors
except OSError, ose:
archivematicaClient.printOutputLock.acquire()
traceback.print_exc(file=sys.stdout)
print >> sys.stderr, "Execution failed:", ose
archivematicaClient.printOutputLock.release()
output = ["Config Error!", ose.__str__()]
exitCode = 1
return cPickle.dumps({
"exitCode": exitCode,
"stdOut": output[0],
"stdError": output[1]
})
structMapDiv = newChild(structMapDiv, "div", sets=[("TYPE","Directory"), ("LABEL","objects") ])
createFileSec(os.path.join(baseDirectoryPath, "objects"), structMapDiv)
fileSec = etree.Element( "fileSec")
for group in globalFileGrpsUses: #globalFileGrps.itervalues():
grp = globalFileGrps[group]
if len(grp) > 0:
fileSec.append(grp)
rootNSMap = {None: metsNS}
rootNSMap.update(NSMAP)
root = etree.Element( "mets", \
nsmap = rootNSMap, \
attrib = { "{" + xsiNS + "}schemaLocation" : "http://www.loc.gov/METS/ http://www.loc.gov/standards/mets/version18/mets.xsd" } )
etree.SubElement(root,"metsHdr").set("CREATEDATE", databaseInterface.getUTCDate().split(".")[0])
dc = createDublincoreDMDSecFromDBData(SIPMetadataAppliesToType, fileGroupIdentifier)
if dc != None:
(dmdSec, ID) = dc
structMapDiv.set("DMDID", ID)
root.append(dmdSec)
for dmdSec in dmdSecs:
root.append(dmdSec)
for amdSec in amdSecs:
root.append(amdSec)
def removeFileByFileUUID(fileUUID, utcDate=databaseInterface.getUTCDate()):
databaseInterface.runSQL("UPDATE Files " + \
"SET removedTime='" + utcDate + "', currentLocation=NULL " + \
"WHERE fileUUID='" + fileUUID + "'" )
