def logTaskCreatedSQL(taskManager, commandReplacementDic, taskUUID, arguments):
taskUUID = taskUUID
jobUUID = taskManager.jobChainLink.UUID
fileUUID = ""
if "%fileUUID%" in commandReplacementDic:
fileUUID = commandReplacementDic["%fileUUID%"]
taskexec = taskManager.execute
fileName = os.path.basename(os.path.abspath(commandReplacementDic["%relativeLocation%"]))
databaseInterface.runSQL(
"""INSERT INTO Tasks (taskUUID, jobUUID, fileUUID, fileName, exec, arguments, createdTime)
VALUES ( '"""
+ taskUUID
+ databaseInterface.separator
+ jobUUID
+ databaseInterface.separator
+ escapeForDB(fileUUID)
+ databaseInterface.separator
+ escapeForDB(fileName)
+ databaseInterface.separator
+ escapeForDB(taskexec)
+ databaseInterface.separator
+ escapeForDB(arguments)
+ databaseInterface.separator
+ databaseInterface.getUTCDate()
+ "' )"
)
def create(self, table, entry):
sets = []
for key, value in entry.iteritems():
if key == "resource_uri":
continue
if key == "uuid":
key = "pk"
if key == "lastmodified" and value > self.maxLastUpdate:
self.maxLastUpdate = value
#print type(value)
if value == None:
sets.append("%s=NULL" % (key))
elif isinstance(value, int):
sets.append("%s=%s" % (key, value))
elif isinstance(value, unicode):
sets.append(
"%s='%s'" %
(key, databaseInterface.MySQLdb.escape_string(value)))
elif isinstance(value, str):
sets.append(
"%s='%s'" %
(key, databaseInterface.MySQLdb.escape_string(value)))
sets = ", ".join(sets)
sql = """INSERT INTO %s SET %s;""" % (table, sets)
#print sql
databaseInterface.runSQL(sql)
def addKeyFileToNormalizeMaildirOffOf(relativePathToRepresent,
mirrorDir,
transferPath,
transferUUID,
date,
eventDetail="",
fileUUID=uuid.uuid4().__str__()):
basename = os.path.basename(mirrorDir)
dirname = os.path.dirname(mirrorDir)
outFile = os.path.join(dirname, basename + ".archivematicaMaildir")
content = """#This file is used in the archivematica system to represent a maildir dirctory, for normalization and permission purposes.
[archivematicaMaildir]
path = %s
""" % (relativePathToRepresent)
f = open(outFile, 'w')
f.write(content)
f.close()
addFile(outFile,
transferPath,
transferUUID,
date,
eventDetail=eventDetail,
fileUUID=fileUUID)
sql = """INSERT INTO FilesIdentifiedIDs (fileUUID, fileID) VALUES ('%s', (SELECT pk FROM FileIDs WHERE enabled = TRUE AND description = 'A .archivematicaMaildir file')); """ % (
fileUUID)
databaseInterface.runSQL(sql)
return
def updateFileLocation(src, dst, eventType, eventDateTime, eventDetail, eventIdentifierUUID = uuid.uuid4().__str__(), fileUUID="None", sipUUID = None, transferUUID=None, eventOutcomeDetailNote = ""):
"""If the file uuid is not provided, will use the sip uuid and old path to find the file uuid"""
src = unicodeToStr(src)
dst = unicodeToStr(dst)
fileUUID = unicodeToStr(fileUUID)
if not fileUUID or fileUUID == "None":
sql = "Need to define transferUUID or sipUUID"
if sipUUID:
sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(src) + "' AND Files.sipUUID = '" + sipUUID + "';"
elif transferUUID:
sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(src) + "' AND Files.transferUUID = '" + transferUUID + "';"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
fileUUID = unicodeToStr(row[0])
row = c.fetchone()
sqlLock.release()
if eventOutcomeDetailNote == "":
eventOutcomeDetailNote = "Original name=\"%s\"; cleaned up name=\"%s\"" %(src, dst)
#eventOutcomeDetailNote = eventOutcomeDetailNote.decode('utf-8')
#CREATE THE EVENT
if not fileUUID:
print >>sys.stderr, "Unable to find file uuid for: ", src, " -> ", dst
exit(6)
insertIntoEvents(fileUUID=fileUUID, eventIdentifierUUID=eventIdentifierUUID, eventType=eventType, eventDateTime=eventDateTime, eventDetail=eventDetail, eventOutcome="", eventOutcomeDetailNote=eventOutcomeDetailNote)
#UPDATE THE CURRENT FILE PATH
sql = """UPDATE Files SET currentLocation='%s' WHERE fileUUID='%s';""" % (MySQLdb.escape_string(dst), fileUUID)
databaseInterface.runSQL(sql)
def verifyMetsFileSecChecksums(metsFile, date, taskUUID, transferDirectory, transferUUID, relativeDirectory="./"):
print metsFile
DspaceLicenses = "metadata/submissionDocumentation/DspaceLicenses"
try:
path = os.path.join(transferDirectory, DspaceLicenses)
if not os.path.isdir(path):
os.mkdir(path)
except:
print "error creating DspaceLicenses directory."
exitCode = 0
tree = etree.parse(metsFile)
root = tree.getroot()
for item in root.findall("{http://www.loc.gov/METS/}fileSec/{http://www.loc.gov/METS/}fileGrp"):
#print etree.tostring(item)
#print item
USE = item.get("USE")
if USE == "LICENSE":
for item2 in item:
if item2.tag == "{http://www.loc.gov/METS/}file":
for item3 in item2:
if item3.tag == "{http://www.loc.gov/METS/}FLocat":
fileLocation = item3.get("{http://www.w3.org/1999/xlink}href")
fileFullPath = os.path.join(relativeDirectory, fileLocation)
#dest = os.path.join(transferDirectory, DspaceLicenses, os.path.basename(fileLocation))
#renameAsSudo(fileFullPath, dest)
dbLocation = fileFullPath.replace(transferDirectory, "%transferDirectory%")
sql = """UPDATE Files SET fileGrpUse = 'license' WHERE currentLocation = '%s' AND transferUUID = '%s';""" % (MySQLdb.escape_string(dbLocation), transferUUID)
databaseInterface.runSQL(sql)
#dst = dest.replace(transferDirectory, "%transferDirectory%")
#eventDetail = ""
#eventOutcomeDetailNote = "moved from=\"" + src + "\"; moved to=\"" + dst + "\""
#updateFileLocation(src, dst, "movement", date, eventDetail, transferUUID=transferUUID, eventOutcomeDetailNote = eventOutcomeDetailNote)
return exitCode
def threaded_process_IN_MOVED_TO(self, event):
time.sleep(archivematicaMCP.config.getint('MCPServer', "waitToActOnMoves"))
#print event
#print "SIP IN_MOVED_TO"
movedFromLock.acquire()
if event.cookie not in movedFrom:
#if there isn't one - error
#print event.cookie, movedFrom
#print >>sys.stderr, "#error. No adding files to a sip in this manner."
movedFromLock.release()
return
#remove it from the list of unfound moves
movedFromPath, filesMoved, timer = movedFrom.pop(event.cookie)
movedFromCount.value = movedFromCount.value - 1
movedFromLock.release()
movedToPath = os.path.join(event.path, event.name).replace(\
self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), \
"%SIPDirectory%", 1)
for fileUUID, oldLocation in filesMoved:
newFilePath = oldLocation.replace(movedFromPath, movedToPath, 1)
#print "Moved: ", oldLocation, "-> (" + self.unit.UUID + ")" + newFilePath
databaseInterface.runSQL("UPDATE Files " + \
"SET currentLocation='" + newFilePath + "', " + \
"Files.sipUUID = '" + self.unit.UUID + "' " \
"WHERE fileUUID='" + fileUUID + "'" )
def something(SIPDirectory, serviceDirectory, objectsDirectory, SIPUUID, date):
#exitCode = 435
exitCode = 0
print SIPDirectory
#For every file, & directory Try to find the matching file & directory in the objects directory
for (path, dirs, files) in os.walk(serviceDirectory):
for file in files:
servicePreExtension = "_me"
originalPreExtension = "_m"
file1Full = os.path.join(path,
file).replace(SIPDirectory,
"%SIPDirectory%",
1) #service
a = file.rfind(servicePreExtension + ".")
file2 = ""
if a != -1:
file2Full = os.path.join(
path, file[:a] + originalPreExtension + ".").replace(
SIPDirectory + "objects/service/",
"%SIPDirectory%objects/", 1) #service
else:
a = file.rfind(".")
if a != -1: #if a period is found
a += 1 #include the period
file2Full = os.path.join(path, file[:a]).replace(
SIPDirectory + "objects/service/",
"%SIPDirectory%objects/", 1) #service
accessPath = os.path.join(path, file)
sql = "UPDATE Files SET fileGrpUse='service' WHERE currentLocation = '" + file1Full + "' AND removedTime = 0 AND SIPUUID = '" + SIPUUID + "'"
rows = databaseInterface.runSQL(sql)
sql = "UPDATE Files SET fileGrpUUID= (SELECT fileUUID FROM (SELECT * FROM Files WHERE removedTime = 0 AND SIPUUID = '" + SIPUUID + "') AS F WHERE currentLocation LIKE '" + file2Full + "%') WHERE currentLocation = '" + file1Full + "' AND removedTime = 0 AND SIPUUID = '" + SIPUUID + "'"
print sql
rows = databaseInterface.runSQL(sql)
return exitCode
def fileWasRemoved(
fileUUID, utcDate=databaseInterface.getUTCDate(), eventDetail="", eventOutcomeDetailNote="", eventOutcome=""
):
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "file removed"
eventDateTime = utcDate
insertIntoEvents(
fileUUID=fileUUID,
eventIdentifierUUID=eventIdentifierUUID,
eventType=eventType,
eventDateTime=eventDateTime,
eventDetail=eventDetail,
eventOutcome=eventOutcome,
eventOutcomeDetailNote=eventOutcomeDetailNote,
)
databaseInterface.runSQL(
"UPDATE Files "
+ "SET removedTime='"
+ utcDate
+ "', currentLocation=NULL "
+ "WHERE fileUUID='"
+ fileUUID
+ "'"
)
def something(SIPDirectory, serviceDirectory, objectsDirectory, SIPUUID, date):
#exitCode = 435
exitCode = 0
print SIPDirectory
#For every file, & directory Try to find the matching file & directory in the objects directory
for (path, dirs, files) in os.walk(serviceDirectory):
for file in files:
servicePreExtension = "_me"
originalPreExtension = "_m"
file1Full = os.path.join(path, file).replace(SIPDirectory, "%SIPDirectory%", 1) #service
a = file.rfind(servicePreExtension + ".")
file2 = ""
if a != -1:
file2Full = os.path.join(path, file[:a] + originalPreExtension + ".").replace(SIPDirectory + "objects/service/", "%SIPDirectory%objects/", 1) #service
else:
a = file.rfind(".")
if a != -1: #if a period is found
a += 1 #include the period
file2Full = os.path.join(path, file[:a]).replace(SIPDirectory + "objects/service/", "%SIPDirectory%objects/", 1) #service
accessPath = os.path.join(path, file)
sql = "UPDATE Files SET fileGrpUse='service' WHERE currentLocation = '" + file1Full + "' AND removedTime = 0 AND SIPUUID = '"+ SIPUUID + "'"
rows = databaseInterface.runSQL(sql)
sql = "UPDATE Files SET fileGrpUUID= (SELECT fileUUID FROM (SELECT * FROM Files WHERE removedTime = 0 AND SIPUUID = '"+ SIPUUID + "') AS F WHERE currentLocation LIKE '" + file2Full + "%') WHERE currentLocation = '" + file1Full + "' AND removedTime = 0 AND SIPUUID = '"+ SIPUUID + "'"
print sql
rows = databaseInterface.runSQL(sql)
return exitCode
def updateDirectoryLocation(src, dst, unitPath, unitIdentifier,
unitIdentifierType, unitPathReplaceWith):
srcDB = src.replace(unitPath, unitPathReplaceWith)
if not srcDB.endswith("/") and srcDB != unitPathReplaceWith:
srcDB += "/"
dstDB = dst.replace(unitPath, unitPathReplaceWith)
if not dstDB.endswith("/") and dstDB != unitPathReplaceWith:
dstDB += "/"
sql = "SELECT Files.fileUUID, Files.currentLocation FROM Files WHERE removedTime = 0 AND Files.currentLocation LIKE '" + MySQLdb.escape_string(
srcDB
) + "%' AND " + unitIdentifierType + " = '" + unitIdentifier + "';"
rows = databaseInterface.queryAllSQL(sql)
for row in rows:
fileUUID = row[0]
location = row[1]
destDB = location.replace(srcDB, dstDB)
sql = """UPDATE Files SET currentLocation='%s' WHERE fileUUID='%s';""" % (
MySQLdb.escape_string(destDB), fileUUID)
databaseInterface.runSQL(sql)
if os.path.isdir(dst):
if dst.endswith("/"):
dst += "."
else:
dst += "/."
print "moving: ", src, dst
shutil.move(src, dst)
def __init__(self, currentPath, UUID=""):
self.owningUnit = None
self.unitType = "Transfer"
#Just Use the end of the directory name
self.pathString = "%transferDirectory%"
currentPath2 = currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), \
"%sharedPath%", 1)
if UUID == "":
sql = """SELECT transferUUID FROM Transfers WHERE currentLocation = '""" + MySQLdb.escape_string(
currentPath2) + "'"
time.sleep(.5)
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
UUID = row[0]
print "Opening existing Transfer:", UUID, "-", currentPath2
row = c.fetchone()
sqlLock.release()
if UUID == "":
uuidLen = -36
if archivematicaMCP.isUUID(currentPath[uuidLen - 1:-1]):
UUID = currentPath[uuidLen - 1:-1]
else:
UUID = uuid.uuid4().__str__()
self.UUID = UUID
sql = """INSERT INTO Transfers (transferUUID, currentLocation)
VALUES ('""" + UUID + databaseInterface.separator + MySQLdb.escape_string(
currentPath2) + "');"
databaseInterface.runSQL(sql)
self.currentPath = currentPath2
self.UUID = UUID
self.fileList = {}
def regular(SIPDirectory, objectsDirectory, SIPUUID, date):
searchForRegularExpressions = True
if not searchForRegularExpressions:
return
original = ""
service = ""
for (path, dirs, files) in os.walk(objectsDirectory):
for file in files:
m = re.search("_me\.[a-zA-Z0-9]{2,4}$", file)
if m != None:
file1Full = os.path.join(path, file).replace(SIPDirectory, "%SIPDirectory%", 1) # service
file2 = file.replace(m.group(0), m.group(0).replace("_me", "_m", 1))
file2Full = os.path.join(path, file2).replace(SIPDirectory, "%SIPDirectory%", 1) # original
accessPath = os.path.join(path, file)
sql = (
"UPDATE Files SET fileGrpUse='service' WHERE currentLocation = '"
+ file1Full
+ "' AND removedTime = 0 AND SIPUUID = '"
+ SIPUUID
+ "'"
)
rows = databaseInterface.runSQL(sql)
sql = (
"UPDATE Files SET fileGrpUUID= (SELECT fileUUID FROM (SELECT * FROM Files WHERE removedTime = 0 AND SIPUUID = '"
+ SIPUUID
+ "') AS F WHERE currentLocation = '"
+ file2Full
+ "') WHERE currentLocation = '"
+ file1Full
+ "' AND removedTime = 0 AND SIPUUID = '"
+ SIPUUID
+ "'"
)
rows = databaseInterface.runSQL(sql)
def verifyMetsFileSecChecksums(metsFile,
date,
taskUUID,
transferDirectory,
transferUUID,
relativeDirectory="./"):
print metsFile
exitCode = 0
tree = etree.parse(metsFile)
root = tree.getroot()
for item in root.findall(
"{http://www.loc.gov/METS/}fileSec/{http://www.loc.gov/METS/}fileGrp"
):
#print etree.tostring(item)
#print item
USE = item.get("USE")
if USE == "LICENSE":
for item2 in item:
if item2.tag == "{http://www.loc.gov/METS/}file":
for item3 in item2:
if item3.tag == "{http://www.loc.gov/METS/}FLocat":
fileLocation = item3.get(
"{http://www.w3.org/1999/xlink}href")
fileFullPath = os.path.join(
relativeDirectory, fileLocation)
dbLocation = fileFullPath.replace(
transferDirectory, "%transferDirectory%")
sql = """UPDATE Files SET fileGrpUse = 'license' WHERE currentLocation = '%s' AND transferUUID = '%s';""" % (
MySQLdb.escape_string(dbLocation),
transferUUID)
databaseInterface.runSQL(sql)
return exitCode
def logJobCreatedSQL(job):
separator = databaseInterface.getSeparator()
unitUUID = job.unit.UUID
if job.unit.owningUnit != None:
unitUUID = job.unit.owningUnit.UUID
databaseInterface.runSQL(
"""INSERT INTO Jobs (jobUUID, jobType, directory, SIPUUID, currentStep, unitType, microserviceGroup, createdTime, createdTimeDec, MicroServiceChainLinksPK, subJobOf)
VALUES ( '"""
+ job.UUID.__str__()
+ separator
+ escapeForDB(job.description)
+ separator
+ escapeForDB(job.unit.currentPath)
+ separator
+ escapeForDB(unitUUID)
+ separator
+ "Executing command(s)"
+ separator
+ job.unit.__class__.__name__
+ separator
+ job.microserviceGroup.__str__()
+ separator
+ job.createdDate
+ separator
+ databaseInterface.getDeciDate("." + job.createdDate.split(".")[-1])
+ "', "
+ job.pk.__str__()
+ ", '"
+ job.subJobOf.__str__()
+ "' )"
)
def verifyMetsFileSecChecksums(metsFile, date, taskUUID, transferDirectory, transferUUID, relativeDirectory="./"):
print metsFile
DspaceLicenses = "metadata/submissionDocumentation/DspaceLicenses"
try:
path = os.path.join(transferDirectory, DspaceLicenses)
if not os.path.isdir(path):
os.mkdir(path)
except:
print "error creating DspaceLicenses directory."
exitCode = 0
tree = etree.parse(metsFile)
root = tree.getroot()
for item in root.findall("{http://www.loc.gov/METS/}fileSec/{http://www.loc.gov/METS/}fileGrp"):
#print etree.tostring(item)
#print item
USE = item.get("USE")
if USE == "TEXT":
for item2 in item:
if item2.tag == "{http://www.loc.gov/METS/}file":
for item3 in item2:
if item3.tag == "{http://www.loc.gov/METS/}FLocat":
fileLocation = item3.get("{http://www.w3.org/1999/xlink}href")
fileFullPath = os.path.join(relativeDirectory, fileLocation)
dbLocation = fileFullPath.replace(transferDirectory, "%transferDirectory%")
sql = """UPDATE Files SET fileGrpUse = 'text/ocr' WHERE currentLocation = '%s' AND transferUUID = '%s';""" % (MySQLdb.escape_string(dbLocation), transferUUID)
databaseInterface.runSQL(sql)
return exitCode
def __init__(self, currentPath, UUID=""):
self.owningUnit = None
self.unitType = "Transfer"
#Just Use the end of the directory name
self.pathString = "%transferDirectory%"
currentPath2 = currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), \
"%sharedPath%", 1)
if UUID == "":
sql = """SELECT transferUUID FROM Transfers WHERE currentLocation = '""" + MySQLdb.escape_string(currentPath2) + "'"
time.sleep(.5)
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
UUID = row[0]
print "Opening existing Transfer:", UUID, "-", currentPath2
row = c.fetchone()
sqlLock.release()
if UUID == "":
uuidLen = -36
if archivematicaMCP.isUUID(currentPath[uuidLen-1:-1]):
UUID = currentPath[uuidLen-1:-1]
else:
UUID = uuid.uuid4().__str__()
self.UUID = UUID
sql = """INSERT INTO Transfers (transferUUID, currentLocation)
VALUES ('""" + UUID + databaseInterface.separator + MySQLdb.escape_string(currentPath2) + "');"
databaseInterface.runSQL(sql)
self.currentPath = currentPath2
self.UUID = UUID
self.fileList = {}
def insertIntoFiles(fileUUID,
filePath,
enteredSystem=databaseInterface.getUTCDate(),
transferUUID="",
sipUUID="",
use="original"):
if transferUUID != "" and sipUUID == "":
databaseInterface.runSQL("""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, transferUUID)
VALUES ( '""" + fileUUID + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ enteredSystem + databaseInterface.separator \
+ use + databaseInterface.separator \
+ transferUUID + "' )" )
elif transferUUID == "" and sipUUID != "":
databaseInterface.runSQL("""INSERT INTO Files (fileUUID, originalLocation, currentLocation, enteredSystem, fileGrpUse, sipUUID)
VALUES ( '""" + fileUUID + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ escapeForDB(filePath) + databaseInterface.separator \
+ enteredSystem + databaseInterface.separator \
+ use + databaseInterface.separator \
+ sipUUID + "' )" )
else:
print >> sys.stderr, "not supported yet - both SIP and transfer UUID's defined (or neither defined)"
print >> sys.stderr, "SIP UUID:", sipUUID
print >> sys.stderr, "transferUUID:", transferUUID
raise Exception(
"not supported yet - both SIP and transfer UUID's defined (or neither defined)",
sipUUID + "-" + transferUUID)
def insertIntoEvents(
fileUUID="",
eventIdentifierUUID="",
eventType="",
eventDateTime=databaseInterface.getUTCDate(),
eventDetail="",
eventOutcome="",
eventOutcomeDetailNote="",
):
if eventIdentifierUUID == "":
eventIdentifierUUID = uuid.uuid4().__str__()
databaseInterface.runSQL(
"""INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote)
VALUES ( '"""
+ escapeForDB(fileUUID)
+ databaseInterface.separator
+ escapeForDB(eventIdentifierUUID)
+ databaseInterface.separator
+ escapeForDB(eventType)
+ databaseInterface.separator
+ escapeForDB(eventDateTime)
+ databaseInterface.separator
+ escapeForDB(eventDetail)
+ databaseInterface.separator
+ escapeForDB(eventOutcome)
+ databaseInterface.separator
+ escapeForDB(eventOutcomeDetailNote)
+ "' )"
)
def insertIntoDerivations(sourceFileUUID="", derivedFileUUID="", relatedEventUUID=""):
databaseInterface.runSQL("""INSERT INTO Derivations
(sourceFileUUID, derivedFileUUID, relatedEventUUID)
VALUES ( '""" \
+ sourceFileUUID + databaseInterface.separator \
+ derivedFileUUID + databaseInterface.separator \
+ relatedEventUUID + "');")
def setMagicLink(self, link, exitStatus=""):
"""Assign a link to the unit to process when loaded.
Deprecated! Replaced with Set/Load Unit Variable"""
if exitStatus != "":
sql = """UPDATE Transfers SET magicLink='""" + link.__str__() + """', magicLinkExitMessage='""" + exitStatus + """' WHERE transferUUID='""" + self.UUID.__str__() + """';"""
else:
sql = """UPDATE Transfers SET magicLink='""" + link.__str__() + """' WHERE transferUUID='""" + self.UUID.__str__() + """';"""
databaseInterface.runSQL(sql)
def run(target, fileUUID):
basename = os.path.basename(target)
extensionIndex = basename.rfind(".")
if extensionIndex != -1:
extension = basename[extensionIndex+1:]
print "extension:", extension
sql = """INSERT INTO FilesIdentifiedIDs (fileUUID, fileID) VALUES ('%s', (SELECT FileIDs FROM FileIDsByExtension WHERE Extension = '%s'))""" % (escapeForDB(fileUUID), escapeForDB(extension.lower()))
databaseInterface.runSQL(sql)
def setMagicLink(self, link, exitStatus=""):
"""Assign a link to the unit to process when loaded.
Deprecated! Replaced with Set/Load Unit Variable"""
if exitStatus != "":
sql = """UPDATE SIPs SET magicLink='""" + link + """', magicLinkExitMessage='""" + exitStatus + """' WHERE sipUUID='""" + self.UUID + """';"""
else:
sql = """UPDATE SIPs SET magicLink='""" + link + """' WHERE sipUUID='""" + self.UUID + """';"""
databaseInterface.runSQL(sql)
def backup_indexed_document(result, indexData, index, type):
sql = "INSERT INTO ElasticsearchIndexBackup (docId, data, indexName, typeName) VALUES ('%s', '%s', '%s', '%s')"
sql = sql % (MySQLdb.escape_string(
result['_id']), unicode(base64.encodestring(cPickle.dumps(indexData))),
MySQLdb.escape_string(index), MySQLdb.escape_string(type))
databaseInterface.runSQL(sql)
def createSIP(path, UUID=None):
if UUID == None:
UUID = uuid.uuid4().__str__()
print "Creating SIP:", UUID, "-", path
sql = """INSERT INTO SIPs (sipUUID, currentPath)
VALUES ('""" + UUID + databaseInterface.separator + escapeForDB(path) + "');"
databaseInterface.runSQL(sql)
return UUID
def insertIntoDerivations(sourceFileUUID="",
derivedFileUUID="",
relatedEventUUID=""):
databaseInterface.runSQL("""INSERT INTO Derivations
(sourceFileUUID, derivedFileUUID, relatedEventUUID)
VALUES ( '""" \
+ sourceFileUUID + databaseInterface.separator \
+ derivedFileUUID + databaseInterface.separator \
+ relatedEventUUID + "');")
def insertIntoFilesIDs(fileUUID="", formatName="", formatVersion="", formatRegistryName="", formatRegistryKey=""):
databaseInterface.runSQL("""INSERT INTO FilesIDs
(fileUUID, formatName, formatVersion, formatRegistryName, formatRegistryKey)
VALUES ( '""" \
+ escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(formatName) + databaseInterface.separator \
+ escapeForDB(formatVersion) + databaseInterface.separator \
+ escapeForDB(formatRegistryName) + databaseInterface.separator \
+ escapeForDB(formatRegistryKey) + "');")
def createSIP(path, UUID=None):
if UUID == None:
UUID = uuid.uuid4().__str__()
print "Creating SIP:", UUID, "-", path
sql = """INSERT INTO SIPs (sipUUID, currentPath)
VALUES ('""" + UUID + databaseInterface.separator + escapeForDB(
path) + "');"
databaseInterface.runSQL(sql)
return UUID
def setMagicLink(self, link, exitStatus=""):
if exitStatus != "":
sql = """UPDATE Transfers SET magicLink='""" + link.__str__(
) + """', magicLinkExitMessage='""" + exitStatus + """' WHERE transferUUID='""" + self.UUID.__str__(
) + """';"""
else:
sql = """UPDATE Transfers SET magicLink='""" + link.__str__(
) + """' WHERE transferUUID='""" + self.UUID.__str__() + """';"""
databaseInterface.runSQL(sql)
def insertIntoFilesFits(fileUUID="", fitsXMLString=""):
databaseInterface.runSQL(
"""INSERT INTO FilesFits
(fileUUID, FITSxml)
VALUES ( '"""
+ escapeForDB(fileUUID)
+ databaseInterface.separator
+ escapeForDB(fitsXMLString)
+ "');"
)
def setMaildirFiles(sipUUID, sipPath):
for root, dirs, files in os.walk(os.path.join(sipPath, "objects", "Maildir")):
for file in files:
fileRelativePath = os.path.join(root, file).replace(sipPath, "%SIPDirectory%", 1)
sql = """SELECT fileUUID FROM Files WHERE removedTime = 0 AND sipUUID = '%s' AND currentLocation = '%s';""" % (sipUUID, fileRelativePath)
rows = databaseInterface.queryAllSQL(sql)
if len(rows):
fileUUID = rows[0][0]
sql = """INSERT INTO FilesIdentifiedIDs (fileUUID, fileID) VALUES ('%s', (SELECT pk FROM FileIDs WHERE enabled = TRUE AND description = 'A maildir email file')); """ % (fileUUID)
databaseInterface.runSQL(sql)
def printFidoInsert(itemdirectoryPath):
global runSQLInserts
ext = findExtension(itemdirectoryPath).lower()
if not ext:
return
fileID = findExistingFileID(ext)
if not fileID:
return
fileID, validPreservationFormat, validAccessFormat = fileID
FidoFileID = getFidoID(itemdirectoryPath).strip()
if not FidoFileID:
return
# check for existing rule
sql = (
"""SELECT pk FROM FileIDs WHERE fileIDType = 'afdbee13-eec5-4182-8c6c-f5638ee290f3' AND description = '%s';"""
% FidoFileID
)
if databaseInterface.queryAllSQL(sql):
a = "skip"
# return
if FidoFileID in idsDone:
return
fileIDUUID = uuid.uuid4().__str__()
sql = """INSERT INTO FileIDs (pk, description, validPreservationFormat, validAccessFormat, fileIDType)
VALUES ('%s', '%s', %s, %s, 'afdbee13-eec5-4182-8c6c-f5638ee290f3');""" % (
fileIDUUID,
FidoFileID,
validPreservationFormat,
validAccessFormat,
)
idsDone.append(FidoFileID)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
FileIDsBySingleIDUUID = uuid.uuid4().__str__()
sql = """INSERT INTO FileIDsBySingleID (pk, fileID, id, tool, toolVersion)
VALUES ('%s', '%s', '%s', 'Fido', '1.1.2');""" % (
FileIDsBySingleIDUUID,
fileIDUUID,
FidoFileID,
)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
printNewCommandRelationships(fileID, fileIDUUID)
print
def logJobCreatedSQL(job):
separator = databaseInterface.getSeparator()
unitUUID = job.unit.UUID
decDate = databaseInterface.getDeciDate("." +
job.createdDate.split(".")[-1])
if job.unit.owningUnit != None:
unitUUID = job.unit.owningUnit.UUID
databaseInterface.runSQL("""INSERT INTO Jobs (jobUUID, jobType, directory, SIPUUID, currentStep, unitType, microserviceGroup, createdTime, createdTimeDec, MicroServiceChainLinksPK, subJobOf)
VALUES ( '""" + job.UUID.__str__() + separator + escapeForDB(job.description) + separator \
+ escapeForDB(job.unit.currentPath) + separator + escapeForDB(unitUUID) + \
separator + "Executing command(s)" + separator + job.unit.__class__.__name__ + separator + job.microserviceGroup.__str__() + separator + job.createdDate + separator + decDate + separator + job.pk.__str__() + separator + job.subJobOf.__str__() + "' )" )
def setMaxLastUpdate(maxLastUpdate):
sql = """SELECT pk FROM UnitVariables WHERE unitType = 'FPR' AND unitUUID = 'Client' AND variable = 'maxLastUpdate'; """
rows = databaseInterface.queryAllSQL(sql)
if rows:
sql = """UPDATE UnitVariables SET variableValue='%s' WHERE unitType = 'FPR' AND unitUUID = 'Client' AND variable = 'maxLastUpdate';""" % (maxLastUpdate)
databaseInterface.runSQL(sql)
else:
pk = uuid.uuid4().__str__()
sql = """INSERT INTO UnitVariables SET pk='%s', variableValue='%s', unitType='FPR', unitUUID = 'Client', variable = 'maxLastUpdate';""" % (pk, maxLastUpdate)
databaseInterface.runSQL(sql)
return maxLastUpdate
def execute(self):
sql = "UPDATE CommandRelationships SET countAttempts=countAttempts+1 WHERE pk=" + self.pk.__str__() + ";"
databaseInterface.runSQL(sql)
ret = self.commandObject.execute()
if ret:
column = "countNotOK"
else:
column = "countOK"
sql = "UPDATE CommandRelationships SET " + column + "=" + column + "+1 WHERE pk=" + self.pk.__str__() + ";"
databaseInterface.runSQL(sql)
return ret
def logTaskCompletedSQL(task):
print "Logging task output to db", task.UUID
taskUUID = task.UUID.__str__()
exitCode = task.results["exitCode"].__str__()
stdOut = task.results["stdOut"]
stdError = task.results["stdError"]
databaseInterface.runSQL("UPDATE Tasks " + \
"SET endTime='" + databaseInterface.getUTCDate() +"', exitCode='" + exitCode + "', " + \
"stdOut='" + escapeForDB(stdOut) + "', stdError='" + escapeForDB(stdError) + "' "
"WHERE taskUUID='" + taskUUID + "'" )
def logTaskCompletedSQL(task):
print "Logging task output to db", task.UUID
taskUUID = task.UUID.__str__()
exitCode = task.results["exitCode"].__str__()
stdOut = task.results["stdOut"]
stdError = task.results["stdError"]
databaseInterface.runSQL("UPDATE Tasks " + \
"SET endTime='" + databaseInterface.getUTCDate() +"', exitCode='" + exitCode + "', " + \
"stdOut='" + escapeForDB(stdOut) + "', stdError='" + escapeForDB(stdError) + "' "
"WHERE taskUUID='" + taskUUID + "'" )
def backup_indexed_document(result, indexData, index, type):
sql = "INSERT INTO ElasticsearchIndexBackup (docId, data, indexName, typeName) VALUES ('%s', '%s', '%s', '%s')"
sql = sql % (
MySQLdb.escape_string(result["_id"]),
unicode(base64.encodestring(cPickle.dumps(indexData))),
MySQLdb.escape_string(index),
MySQLdb.escape_string(type),
)
databaseInterface.runSQL(sql)
def run(target, fileUUID):
basename = os.path.basename(target)
extensionIndex = basename.rfind(".")
if extensionIndex != -1:
extension = basename[extensionIndex+1:]
print "extension:", extension
sql1= """SELECT FileIDsBySingleID.fileID FROM FileIDsBySingleID JOIN FileIDs ON FileIDsBySingleID.fileID = FileIDs.pk WHERE FileIDs.fileIDType = '16ae42ff-1018-4815-aac8-cceacd8d88a8' AND FileIDsBySingleID.id = '%s' AND FileIDs.enabled = TRUE AND FileIDsBySingleID.enabled = TRUE;""" % (escapeForDB(extension.lower()))
rows = databaseInterface.queryAllSQL(sql1)
for row in rows:
sql2 = """INSERT INTO FilesIdentifiedIDs (fileUUID, fileID) VALUES ('%s', '%s');""" % (escapeForDB(fileUUID), row[0])
databaseInterface.runSQL(sql2)
def setMaxLastUpdate(self, maxLastUpdate):
sql = """SELECT pk FROM UnitVariables WHERE unitType = 'FPR' AND unitUUID = 'Client' AND variable = 'maxLastUpdate'; """
rows = databaseInterface.queryAllSQL(sql)
if rows:
sql = """UPDATE UnitVariables SET variableValue='%s' WHERE unitType = 'FPR' AND unitUUID = 'Client' AND variable = 'maxLastUpdate';""" % (self.maxLastUpdate)
databaseInterface.runSQL(sql)
else:
pk = uuid.uuid4().__str__()
sql = """INSERT INTO UnitVariables SET pk='%s', variableValue='%s', unitType='FPR', unitUUID = 'Client', variable = 'maxLastUpdate';""" % (pk, self.maxLastUpdate)
databaseInterface.runSQL(sql)
return maxLastUpdate
def logJobStepCompletedSQL(job):
databaseInterface.runSQL(
"""INSERT INTO jobStepCompleted (jobUUID, step, completedTime)
VALUES ( '"""
+ job.UUID.__str__()
+ databaseInterface.separator
+ job.step
+ databaseInterface.separator
+ databaseInterface.getUTCDate()
+ "' )"
)
def logTaskAssignedSQL(taskUUID, client, date):
databaseInterface.runSQL(
"UPDATE Tasks "
+ "SET startTime='"
+ date
+ "', client='"
+ client
+ "' "
+ "WHERE taskUUID='"
+ taskUUID
+ "';"
)
def printNewCommandRelationships(fileID, fileIDUUID):
global runSQLInserts
sql = """SELECT commandClassification, command FROM CommandRelationships WHERE fileID = '%s';""" % (fileID)
rows = databaseInterface.queryAllSQL(sql)
for row in rows:
commandClassification, command = row
CommandRelationshipUUID = uuid.uuid4().__str__()
sql = """INSERT INTO CommandRelationships (pk, fileID, commandClassification, command)
VALUES ('%s', '%s', '%s', '%s');""" % (CommandRelationshipUUID, fileIDUUID, commandClassification, command)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
def insertIntoFilesIDs(fileUUID="",
formatName="",
formatVersion="",
formatRegistryName="",
formatRegistryKey=""):
databaseInterface.runSQL("""INSERT INTO FilesIDs
(fileUUID, formatName, formatVersion, formatRegistryName, formatRegistryKey)
VALUES ( '""" \
+ escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(formatName) + databaseInterface.separator \
+ escapeForDB(formatVersion) + databaseInterface.separator \
+ escapeForDB(formatRegistryName) + databaseInterface.separator \
+ escapeForDB(formatRegistryKey) + "');")
def execute(self):
sql = "UPDATE CommandRelationships SET countAttempts=countAttempts+1 WHERE pk='" + self.pk.__str__(
) + "';"
databaseInterface.runSQL(sql)
ret = self.commandObject.execute()
if ret:
column = "countNotOK"
else:
column = "countOK"
sql = "UPDATE CommandRelationships SET " + column + "=" + column + "+1 WHERE pk='" + self.pk.__str__(
) + "';"
databaseInterface.runSQL(sql)
return ret
def run(target, fileUUID):
basename = os.path.basename(target)
extensionIndex = basename.rfind(".")
if extensionIndex != -1:
extension = basename[extensionIndex + 1:]
print "extension:", extension
sql1 = """SELECT FileIDsBySingleID.fileID FROM FileIDsBySingleID JOIN FileIDs ON FileIDsBySingleID.fileID = FileIDs.pk WHERE FileIDs.fileIDType = '16ae42ff-1018-4815-aac8-cceacd8d88a8' AND FileIDsBySingleID.id = '%s' AND FileIDs.enabled = TRUE AND FileIDsBySingleID.enabled = TRUE;""" % (
escapeForDB(extension.lower()))
rows = databaseInterface.queryAllSQL(sql1)
for row in rows:
sql2 = """INSERT INTO FilesIdentifiedIDs (fileUUID, fileID) VALUES ('%s', '%s');""" % (
escapeForDB(fileUUID), row[0])
databaseInterface.runSQL(sql2)
def setMaildirFiles(sipUUID, sipPath):
for root, dirs, files in os.walk(
os.path.join(sipPath, "objects", "Maildir")):
for file in files:
fileRelativePath = os.path.join(root, file).replace(
sipPath, "%SIPDirectory%", 1)
sql = """SELECT fileUUID FROM Files WHERE removedTime = 0 AND sipUUID = %s AND currentLocation = %s;"""
rows = databaseInterface.queryAllSQL(sql,
(sipUUID, fileRelativePath))
if len(rows):
fileUUID = rows[0][0]
sql = """INSERT INTO FilesIdentifiedIDs (fileUUID, fileID) VALUES (%s, (SELECT pk FROM FileIDs WHERE enabled = TRUE AND description = 'A maildir email file')); """
databaseInterface.runSQL(sql, (fileUUID, ))
def insertIntoEvents(fileUUID="", eventIdentifierUUID="", eventType="", eventDateTime=databaseInterface.getUTCDate(), eventDetail="", eventOutcome="", eventOutcomeDetailNote=""):
agent = getAgentForFileUUID(fileUUID)
if not agent:
agent = 'NULL'
if eventIdentifierUUID == "":
eventIdentifierUUID = uuid.uuid4().__str__()
databaseInterface.runSQL("""INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote, linkingAgentIdentifier)
VALUES ( '""" + escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(eventIdentifierUUID) + databaseInterface.separator \
+ escapeForDB(eventType) + databaseInterface.separator \
+ escapeForDB(eventDateTime) + databaseInterface.separator \
+ escapeForDB(eventDetail) + databaseInterface.separator \
+ escapeForDB(eventOutcome) + databaseInterface.separator \
+ escapeForDB(eventOutcomeDetailNote) + "', " + escapeForDB(agent) + " )" )
def onceNormalized(command, opts, replacementDic):
transcodedFiles = []
if not command.outputLocation:
command.outputLocation = ""
if os.path.isfile(command.outputLocation):
transcodedFiles.append(command.outputLocation)
elif os.path.isdir(command.outputLocation):
for w in os.walk(command.outputLocation):
path, directories, files = w
for p in files:
p = os.path.join(path, p)
if os.path.isfile(p):
transcodedFiles.append(p)
elif command.outputLocation:
print >>sys.stderr, command
print >>sys.stderr, "Error - output file does not exist [" + command.outputLocation + "]"
command.exitCode = -2
derivationEventUUID = uuid.uuid4().__str__()
eventDetail = "ArchivematicaFPRCommandID=\"%s\"" % (command.pk)
if command.eventDetailCommand != None:
eventDetail = '%s; %s' % (eventDetail, command.eventDetailCommand.stdOut)
for ef in transcodedFiles:
if opts["commandClassifications"] == "preservation":
# TODO Add manual normalization for files of same name mapping
#Add the new file to the sip
filePathRelativeToSIP = ef.replace(opts["sipPath"], "%SIPDirectory%", 1)
# addFileToSIP(filePathRelativeToSIP, fileUUID, sipUUID, taskUUID, date, sourceType="ingestion"):
addFileToSIP(filePathRelativeToSIP, replacementDic["%outputFileUUID%"], opts["sipUUID"], uuid.uuid4().__str__(), opts["date"], sourceType="creation", use="preservation")
#Calculate new file checksum
#Add event information to current file
insertIntoEvents(fileUUID=opts["fileUUID"], \
eventIdentifierUUID=derivationEventUUID, \
eventType="normalization", \
eventDateTime=opts["date"], \
eventDetail=eventDetail, \
eventOutcome="", \
eventOutcomeDetailNote=filePathRelativeToSIP)
updateSizeAndChecksum(replacementDic["%outputFileUUID%"], ef, opts["date"], uuid.uuid4().__str__())
#Add linking information between files
insertIntoDerivations(sourceFileUUID=opts["fileUUID"], derivedFileUUID=replacementDic["%outputFileUUID%"], relatedEventUUID=derivationEventUUID)
sql = "INSERT INTO FilesIDs (fileUUID, formatName, formatVersion, formatRegistryName, formatRegistryKey) VALUES ('%s', '%s', NULL, NULL, NULL);" % (replacementDic["%outputFileUUID%"], command.outputFormat)
databaseInterface.runSQL(sql)
replacementDic["%outputFileUUID%"] = uuid.uuid4().__str__()
replacementDic["%postfix%"] = "-" + replacementDic["%outputFileUUID%"]
def printNewCommandRelationships(fileID, fileIDUUID):
global runSQLInserts
sql = """SELECT commandClassification, command FROM CommandRelationships WHERE fileID = '%s';""" % (
fileID)
rows = databaseInterface.queryAllSQL(sql)
for row in rows:
commandClassification, command = row
CommandRelationshipUUID = uuid.uuid4().__str__()
sql = """INSERT INTO CommandRelationships (pk, fileID, commandClassification, command)
VALUES ('%s', '%s', '%s', '%s');""" % (
CommandRelationshipUUID, fileIDUUID, commandClassification,
command)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
def setArchivematicaMaildirFiles(sipUUID, sipPath):
for root, dirs, files in os.walk(
os.path.join(sipPath, "objects", "attachments")):
for file in files:
if file.endswith('.archivematicaMaildir'):
fileRelativePath = os.path.join(root, file).replace(
sipPath, "%SIPDirectory%", 1)
sql = """SELECT fileUUID FROM Files WHERE removedTime = 0 AND sipUUID = '%s' AND currentLocation = '%s';""" % (
sipUUID, fileRelativePath)
rows = databaseInterface.queryAllSQL(sql)
if len(rows):
fileUUID = rows[0][0]
sql = """INSERT INTO FilesIdentifiedIDs (fileUUID, fileID) VALUES ('%s', (SELECT pk FROM FileIDs WHERE enabled = TRUE AND description = 'A .archivematicaMaildir file')); """ % (
fileUUID)
databaseInterface.runSQL(sql)
def updateSizeAndChecksum(fileUUID, filePath, date, eventIdentifierUUID):
fileSize = os.path.getsize(filePath).__str__()
checksum = sha_for_file(filePath).__str__()
sql = "UPDATE Files " + \
"SET fileSize='" + fileSize +"', checksum='" + checksum + "' " + \
"WHERE fileUUID='" + fileUUID + "'"
databaseInterface.runSQL(sql)
insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=eventIdentifierUUID, \
eventType="message digest calculation", \
eventDateTime=date, \
eventDetail="program=\"python\"; module=\"hashlib.sha256()\"", \
eventOutcomeDetailNote=checksum)
def updateFileLocation2(src, dst, unitPath, unitIdentifier, unitIdentifierType, unitPathReplaceWith):
"""Dest needs to be the actual full destination path with filename."""
srcDB = src.replace(unitPath, unitPathReplaceWith)
dstDB = dst.replace(unitPath, unitPathReplaceWith)
sql = "SELECT Files.fileUUID, Files.currentLocation FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(srcDB) + "' AND " + unitIdentifierType + " = '" + unitIdentifier + "';"
rows = databaseInterface.queryAllSQL(sql)
if len(rows) != 1:
print sys.stderr, len(rows), "rows", sql, rows
exit(4)
for row in rows:
fileUUID = row[0]
location = row[1]
sql = """UPDATE Files SET currentLocation='%s' WHERE fileUUID='%s';""" % (MySQLdb.escape_string(dstDB), fileUUID)
databaseInterface.runSQL(sql)
print "moving: ", src, dst
shutil.move(src, dst)
def printFidoInsert(itemdirectoryPath):
global runSQLInserts
ext = findExtension(itemdirectoryPath).lower()
if not ext:
return
fileID = findExistingFileID(ext)
if not fileID:
return
fileID, validPreservationFormat, validAccessFormat = fileID
FidoFileID = getFidoID(itemdirectoryPath).strip()
if not FidoFileID:
return
#check for existing rule
sql = """SELECT pk FROM FileIDs WHERE fileIDType = 'afdbee13-eec5-4182-8c6c-f5638ee290f3' AND description = '%s';""" % FidoFileID
if databaseInterface.queryAllSQL(sql):
a = "skip"
#return
if FidoFileID in idsDone:
return
fileIDUUID = uuid.uuid4().__str__()
sql = """INSERT INTO FileIDs (pk, description, validPreservationFormat, validAccessFormat, fileIDType)
VALUES ('%s', '%s', %s, %s, 'afdbee13-eec5-4182-8c6c-f5638ee290f3');""" % (
fileIDUUID, FidoFileID, validPreservationFormat, validAccessFormat)
idsDone.append(FidoFileID)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
FileIDsBySingleIDUUID = uuid.uuid4().__str__()
sql = """INSERT INTO FileIDsBySingleID (pk, fileID, id, tool, toolVersion)
VALUES ('%s', '%s', '%s', 'Fido', '1.1.2');""" % (
FileIDsBySingleIDUUID, fileIDUUID, FidoFileID)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
printNewCommandRelationships(fileID, fileIDUUID)
print
def printTikaInsert(itemdirectoryPath):
global runSQLInserts
ext = findExtension(itemdirectoryPath).lower()
if not ext:
return
fileID = findExistingFileID(ext)
if not fileID:
return
fileID, validPreservationFormat, validAccessFormat = fileID
tikaFileID = getTikaID(itemdirectoryPath).strip()
if not tikaFileID:
return
#check for existing rule
sql = """SELECT pk FROM FileIDs WHERE fileIDType = '1d8f3bb3-da8a-4ef6-bac7-b65942df83fc' AND description = '%s';""" % tikaFileID
if databaseInterface.queryAllSQL(sql):
a = "skip"
#return
if tikaFileID in idsDone:
return
fileIDUUID = uuid.uuid4().__str__()
sql = """INSERT INTO FileIDs (pk, description, validPreservationFormat, validAccessFormat, fileIDType)
VALUES ('%s', '%s', %s, %s, '1d8f3bb3-da8a-4ef6-bac7-b65942df83fc');""" % (
fileIDUUID, tikaFileID, validPreservationFormat, validAccessFormat)
idsDone.append(tikaFileID)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
FileIDsBySingleIDUUID = uuid.uuid4().__str__()
sql = """INSERT INTO FileIDsBySingleID (pk, fileID, id, tool, toolVersion)
VALUES ('%s', '%s', '%s', 'Tika', '1.3');""" % (FileIDsBySingleIDUUID,
fileIDUUID, tikaFileID)
print sql
if runSQLInserts:
databaseInterface.runSQL(sql)
printNewCommandRelationships(fileID, fileIDUUID)
print
def logTaskCreatedSQL(taskManager, commandReplacementDic, taskUUID, arguments):
taskUUID = taskUUID
jobUUID = taskManager.jobChainLink.UUID
fileUUID = ""
if "%fileUUID%" in commandReplacementDic:
fileUUID = commandReplacementDic["%fileUUID%"]
taskexec = taskManager.execute
fileName = os.path.basename(
os.path.abspath(commandReplacementDic["%relativeLocation%"]))
databaseInterface.runSQL("""INSERT INTO Tasks (taskUUID, jobUUID, fileUUID, fileName, exec, arguments, createdTime)
VALUES ( '""" + taskUUID + databaseInterface.separator \
+ jobUUID + databaseInterface.separator \
+ escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(fileName) + databaseInterface.separator \
+ escapeForDB(taskexec) + databaseInterface.separator \
+ escapeForDB(arguments) + databaseInterface.separator \
+ databaseInterface.getUTCDate() + "' )" )
def fileWasRemoved(fileUUID,
utcDate=databaseInterface.getUTCDate(),
eventDetail="",
eventOutcomeDetailNote="",
eventOutcome=""):
eventIdentifierUUID = uuid.uuid4().__str__()
eventType = "file removed"
eventDateTime = utcDate
insertIntoEvents(fileUUID=fileUUID, \
eventIdentifierUUID=eventIdentifierUUID, \
eventType=eventType, \
eventDateTime=eventDateTime, \
eventDetail=eventDetail, \
eventOutcome=eventOutcome, \
eventOutcomeDetailNote=eventOutcomeDetailNote)
databaseInterface.runSQL("UPDATE Files " + \
"SET removedTime='" + utcDate + "', currentLocation=NULL " + \
"WHERE fileUUID='" + fileUUID + "'" )
def insertIntoEvents(fileUUID="",
eventIdentifierUUID="",
eventType="",
eventDateTime=databaseInterface.getUTCDate(),
eventDetail="",
eventOutcome="",
eventOutcomeDetailNote=""):
agent = getAgentForFileUUID(fileUUID)
if not agent:
agent = 'NULL'
if eventIdentifierUUID == "":
eventIdentifierUUID = uuid.uuid4().__str__()
databaseInterface.runSQL("""INSERT INTO Events (fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote, linkingAgentIdentifier)
VALUES ( '""" + escapeForDB(fileUUID) + databaseInterface.separator \
+ escapeForDB(eventIdentifierUUID) + databaseInterface.separator \
+ escapeForDB(eventType) + databaseInterface.separator \
+ escapeForDB(eventDateTime) + databaseInterface.separator \
+ escapeForDB(eventDetail) + databaseInterface.separator \
+ escapeForDB(eventOutcome) + databaseInterface.separator \
+ escapeForDB(eventOutcomeDetailNote) + "', " + escapeForDB(agent) + " )" )
def setVariable(self, variable, variableValue, microServiceChainLink):
if not variableValue:
variableValue = ""
if not microServiceChainLink:
microServiceChainLink = "NULL"
else:
microServiceChainLink = "'%s'" % (microServiceChainLink)
variableValue = databaseInterface.MySQLdb.escape_string(variableValue)
sql = """SELECT pk FROM UnitVariables WHERE unitType = '%s' AND unitUUID = '%s' AND variable = '%s';""" % (
self.unitType, self.UUID, variable)
rows = databaseInterface.queryAllSQL(sql)
if rows:
for row in rows:
sql = """UPDATE UnitVariables SET variable='%s', variableValue='%s', microServiceChainLink=%s WHERE pk = '%s'; """ % (
variable, variableValue, microServiceChainLink, row[0])
databaseInterface.runSQL(sql)
else:
sql = """INSERT INTO UnitVariables (pk, unitType, unitUUID, variable, variableValue, microserviceChainLink) VALUES ('%s', '%s', '%s', '%s', '%s', %s);""" % (
uuid.uuid4().__str__(), self.unitType, self.UUID, variable,
variableValue, microServiceChainLink)
databaseInterface.runSQL(sql)
