Example #1
0
def createDigiprovMDAgents():
    ret = []
    global globalDigiprovMDCounter
    #AGENTS
    sql = """SELECT agentIdentifierType, agentIdentifierValue, agentName, agentType FROM Agents;"""
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        globalDigiprovMDCounter += 1
        digiprovMD = etree.Element("digiprovMD")
        digiprovMD.set("ID", "digiprovMD_" + globalDigiprovMDCounter.__str__())
        ret.append(digiprovMD)  #newChild(amdSec, "digiprovMD")
        mdWrap = newChild(digiprovMD, "mdWrap")
        mdWrap.set("MDTYPE", "PREMIS:AGENT")
        xmlData = newChild(mdWrap, "xmlData")
        #agents = etree.SubElement(xmlData, "agents")
        xmlData.append(createAgent(row[0], row[1], row[2], row[3]))
        row = c.fetchone()
    sqlLock.release()

    sql = """SELECT auth_user.id, auth_user.username, auth_user.first_name, auth_user.last_name FROM SIPs JOIN Files ON SIPs.sipUUID = Files.sipUUID JOIN Events ON Files.fileUUID = Events.fileUUID JOIN auth_user ON Events.linkingAgentIdentifier = auth_user.id WHERE SIPs.sipUUID = '%s' GROUP BY auth_user.id;""" % (
        fileGroupIdentifier)
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        globalDigiprovMDCounter += 1
        digiprovMD = etree.Element("digiprovMD")
        digiprovMD.set("ID", "digiprovMD_" + globalDigiprovMDCounter.__str__())
        ret.append(digiprovMD)  #newChild(amdSec, "digiprovMD")
        mdWrap = newChild(digiprovMD, "mdWrap")
        mdWrap.set("MDTYPE", "PREMIS:AGENT")
        xmlData = newChild(mdWrap, "xmlData")
        #agents = etree.SubElement(xmlData, "agents")

        id, username, first_name, last_name = row
        id = id.__str__()
        if not username:
            username = ""
        if not first_name:
            first_name = ""
        if not last_name:
            last_name = ""
        agentIdentifierType = "Archivematica user pk"
        agentIdentifierValue = id
        agentName = 'username="******", first_name="%s", last_name="%s"' % (
            username, first_name, last_name)
        agentType = "Archivematica user"
        xmlData.append(
            createAgent(agentIdentifierType, agentIdentifierValue, agentName,
                        agentType))
        row = c.fetchone()
    sqlLock.release()

    return ret
def createDigiprovMDAgents():
    ret = []
    global globalDigiprovMDCounter
    #AGENTS
    sql = """SELECT agentIdentifierType, agentIdentifierValue, agentName, agentType FROM Agents;"""
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        globalDigiprovMDCounter += 1
        digiprovMD = etree.Element("digiprovMD")
        digiprovMD.set("ID", "digiprovMD_"+ globalDigiprovMDCounter.__str__())
        ret.append(digiprovMD) #newChild(amdSec, "digiprovMD")
        mdWrap = newChild(digiprovMD,"mdWrap")
        mdWrap.set("MDTYPE", "PREMIS:AGENT")
        xmlData = newChild(mdWrap,"xmlData")
        #agents = etree.SubElement(xmlData, "agents")
        xmlData.append(createAgent(row[0], row[1], row[2], row[3]))
        row = c.fetchone()
    sqlLock.release()
    
    sql = """SELECT auth_user.id, auth_user.username, auth_user.first_name, auth_user.last_name FROM SIPs JOIN Files ON SIPs.sipUUID = Files.sipUUID JOIN Events ON Files.fileUUID = Events.fileUUID JOIN auth_user ON Events.linkingAgentIdentifier = auth_user.id WHERE SIPs.sipUUID = '%s' GROUP BY auth_user.id;""" % (fileGroupIdentifier)
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        globalDigiprovMDCounter += 1
        digiprovMD = etree.Element("digiprovMD")
        digiprovMD.set("ID", "digiprovMD_"+ globalDigiprovMDCounter.__str__())
        ret.append(digiprovMD) #newChild(amdSec, "digiprovMD")
        mdWrap = newChild(digiprovMD,"mdWrap")
        mdWrap.set("MDTYPE", "PREMIS:AGENT")
        xmlData = newChild(mdWrap,"xmlData")
        #agents = etree.SubElement(xmlData, "agents")
        
        id, username, first_name, last_name = row
        id = id.__str__()
        if not username:
            username = ""
        if not first_name:
            first_name = ""
        if not last_name:
            last_name = ""
        agentIdentifierType = "Archivematica user pk"
        agentIdentifierValue = id
        agentName = 'username="******", first_name="%s", last_name="%s"' % (username, first_name, last_name)       
        agentType = "Archivematica user"
        xmlData.append(createAgent(agentIdentifierType, agentIdentifierValue, agentName, agentType))
        row = c.fetchone()
    sqlLock.release()
    
    return ret
Example #3
0
    def __init__(self, currentPath, UUID=""):
        self.owningUnit = None
        self.unitType = "Transfer"
        #Just Use the end of the directory name
        self.pathString = "%transferDirectory%"
        currentPath2 = currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), \
                       "%sharedPath%", 1)

        if UUID == "":
            sql = """SELECT transferUUID FROM Transfers WHERE currentLocation = '""" + MySQLdb.escape_string(
                currentPath2) + "'"
            time.sleep(.5)
            c, sqlLock = databaseInterface.querySQL(sql)
            row = c.fetchone()
            while row != None:
                UUID = row[0]
                print "Opening existing Transfer:", UUID, "-", currentPath2
                row = c.fetchone()
            sqlLock.release()

        if UUID == "":
            uuidLen = -36
            if archivematicaMCP.isUUID(currentPath[uuidLen - 1:-1]):
                UUID = currentPath[uuidLen - 1:-1]
            else:
                UUID = uuid.uuid4().__str__()
                self.UUID = UUID
                sql = """INSERT INTO Transfers (transferUUID, currentLocation)
                VALUES ('""" + UUID + databaseInterface.separator + MySQLdb.escape_string(
                    currentPath2) + "');"
                databaseInterface.runSQL(sql)

        self.currentPath = currentPath2
        self.UUID = UUID
        self.fileList = {}
    def __init__(self, jobChainLink, pk, unit):
        """Used to get a selection, from a list of chains, to process"""
        self.choices = []
        self.pk = pk
        self.jobChainLink = jobChainLink
        self.UUID = uuid.uuid4().__str__()
        self.unit = unit
        self.delayTimerLock = threading.Lock()
        self.delayTimer = None
        sql = """SELECT chainAvailable, description FROM MicroServiceChainChoice JOIN MicroServiceChains on chainAvailable = MicroServiceChains.pk WHERE choiceAvailableAtLink = '%s' ORDER BY MicroServiceChainChoice.pk;""" % (jobChainLink.pk.__str__())
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            print row
            chainAvailable = row[0]
            description = row[1]
            self.choices.append((chainAvailable, description,))
            row = c.fetchone()
        sqlLock.release()

        preConfiguredChain = self.checkForPreconfiguredXML()
        if preConfiguredChain != None:
            time.sleep(archivematicaMCP.config.getint('MCPServer', "waitOnAutoApprove"))
            self.jobChainLink.setExitMessage("Completed successfully")
            jobChain.jobChain(self.unit, preConfiguredChain)

        else:
            choicesAvailableForUnitsLock.acquire()
            if self.delayTimer == None:
                self.jobChainLink.setExitMessage('Awaiting decision')
            choicesAvailableForUnits[self.jobChainLink.UUID] = self
            choicesAvailableForUnitsLock.release()
Example #5
0
def getDublinCore(type_, id):
    sql = """SELECT     title, creator, subject, description, publisher, contributor, date, type, format, identifier, source, relation, language, coverage, rights
    FROM Dublincore WHERE metadataAppliesToType = '%s' AND metadataAppliesToidentifier = '%s';""" % \
    (type_.__str__(), id.__str__())

    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    if row == None:
        sqlLock.release()
        return None
    ret = etree.Element("dublincore", nsmap={None: dctermsNS})
    ret.set(
        xsiBNS + "schemaLocation", dctermsNS +
        " http://dublincore.org/schemas/xmls/qdc/2008/02/11/dcterms.xsd")
    dctermsElements = ["isPartOf"]
    while row != None:
        key = [
            "title", "creator", "subject", "description", "publisher",
            "contributor", "date", "type", "format", "identifier", "source",
            "relation", "language", "coverage", "rights"
        ]
        #title, creator, subject, description, publisher, contributor, date, type, format, identifier, source, relation, language, coverage, rights = row
        #key.index("title") == title
        i = 0
        for term in key:
            if row[i] != None:
                txt = row[i]
            else:
                txt = ""
            newChild(ret, term, text=txt)
            i += 1

        row = c.fetchone()
    sqlLock.release()
    return ret
Example #6
0
def findOrCreateSipInDB(path, waitSleep=dbWaitSleep):
    UUID = ""
    path = path.replace(config.get('MCPServer', "sharedDirectory"),
                        "%sharedPath%", 1)

    #find UUID on end of SIP path
    uuidLen = -36
    if isUUID(path[uuidLen - 1:-1]):
        UUID = path[uuidLen - 1:-1]

    if UUID == "":
        #Find it in the database
        databaseInterface.printSQL = True
        sql = """SELECT sipUUID FROM SIPs WHERE currentPath = '""" + MySQLdb.escape_string(
            path) + "';"
        #if waitSleep != 0:
        #time.sleep(waitSleep) #let db be updated by the microservice that moved it.
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        if not row:
            print "Not opening existing SIP:", UUID, "-", path
        while row != None:
            UUID = row[0]
            print "Opening existing SIP:", UUID, "-", path
            row = c.fetchone()
        sqlLock.release()

    #Create it
    if UUID == "":
        UUID = databaseFunctions.createSIP(path)
        print "DEBUG creating sip", path, UUID
    return UUID
Example #7
0
    def reloadFileList(self):
        self.fileList = {}
        #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
        currentPath = self.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
        for directory, subDirectories, files in os.walk(currentPath):
            directory = directory.replace( currentPath, "%SIPDirectory%", 1)
            for file in files:
                filePath = os.path.join(directory, file)
                #print filePath
                self.fileList[filePath] = unitFile(filePath)

        sql = """SELECT  fileUUID, currentLocation FROM Files WHERE sipUUID =  '""" + self.UUID + "'" #AND Files.removedTime = 0; TODO
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            #print row
            UUID = row[0]
            currentPath = row[1]
            if currentPath in self.fileList:
                self.fileList[currentPath].UUID = UUID
            else:
                print "todo: find deleted files/exclude"
                print row[99]#fail
            row = c.fetchone()
            self.fileList[filePath].UUID = UUID
        sqlLock.release()
Example #8
0
    def __init__(self, currentPath, UUID=""):
        self.owningUnit = None
        self.unitType = "Transfer"
        #Just Use the end of the directory name
        self.pathString = "%transferDirectory%"
        currentPath2 = currentPath.replace(archivematicaMCP.config.get('MCPServer', "sharedDirectory"), \
                       "%sharedPath%", 1)

        if UUID == "":
            sql = """SELECT transferUUID FROM Transfers WHERE currentLocation = '""" + MySQLdb.escape_string(currentPath2) + "'"
            time.sleep(.5)
            c, sqlLock = databaseInterface.querySQL(sql)
            row = c.fetchone()
            while row != None:
                UUID = row[0]
                print "Opening existing Transfer:", UUID, "-", currentPath2
                row = c.fetchone()
            sqlLock.release()

        if UUID == "":
            uuidLen = -36
            if  archivematicaMCP.isUUID(currentPath[uuidLen-1:-1]):
                UUID = currentPath[uuidLen-1:-1]
            else:
                UUID = uuid.uuid4().__str__()
                self.UUID = UUID
                sql = """INSERT INTO Transfers (transferUUID, currentLocation)
                VALUES ('""" + UUID + databaseInterface.separator + MySQLdb.escape_string(currentPath2) + "');"
                databaseInterface.runSQL(sql)

        self.currentPath = currentPath2
        self.UUID = UUID
        self.fileList = {}
    def __init__(self, commandID):
        self.pk = commandID
        self.stdOut = ""
        self.stdErr = ""
        self.exitCode=None
        self.failedCount=0
        sql = """SELECT CT.type, C.verificationCommand, C.eventDetailCommand, C.command, C.outputLocation, C.description
        FROM Commands AS C
        JOIN CommandTypes AS CT ON C.commandType = CT.pk
        WHERE C.pk = """ + commandID.__str__() + """
        ;"""
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            rowSTR = []
            for colIndex in range(len(row)):
                rowSTR.append(toStrFromUnicode(row[colIndex])) 
            self.type, \
            self.verificationCommand, \
            self.eventDetailCommand, \
            self.command, \
            self.outputLocation, \
            self.description = \
            rowSTR
            if isinstance(self.command, unicode):
                self.command = self.command.encode('utf-8')
            row = c.fetchone()
        sqlLock.release()
        if self.verificationCommand:
            self.verificationCommand = Command(self.verificationCommand)
            self.verificationCommand.command = self.verificationCommand.command.replace("%outputLocation%", self.outputLocation)

        if self.eventDetailCommand:
            self.eventDetailCommand = Command(self.eventDetailCommand)
            self.eventDetailCommand.command = self.eventDetailCommand.command.replace("%outputLocation%", self.outputLocation)
Example #10
0
def updateFileLocation(src, dst, eventType, eventDateTime, eventDetail, eventIdentifierUUID = uuid.uuid4().__str__(), fileUUID="None", sipUUID = None, transferUUID=None, eventOutcomeDetailNote = ""):
    """If the file uuid is not provided, will use the sip uuid and old path to find the file uuid"""
    src = unicodeToStr(src)
    dst = unicodeToStr(dst)
    fileUUID = unicodeToStr(fileUUID)
    if not fileUUID or fileUUID == "None":
        sql = "Need to define transferUUID or sipUUID"
        if sipUUID:
            sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(src) + "' AND Files.sipUUID = '" + sipUUID + "';"
        elif transferUUID:
            sql = "SELECT Files.fileUUID FROM Files WHERE removedTime = 0 AND Files.currentLocation = '" + MySQLdb.escape_string(src) + "' AND Files.transferUUID = '" + transferUUID + "';"
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            fileUUID = unicodeToStr(row[0])
            row = c.fetchone()
        sqlLock.release()

    if eventOutcomeDetailNote == "":
        eventOutcomeDetailNote = "Original name=\"%s\"; cleaned up name=\"%s\"" %(src, dst)
        #eventOutcomeDetailNote = eventOutcomeDetailNote.decode('utf-8')
    #CREATE THE EVENT
    if not fileUUID:
        print >>sys.stderr, "Unable to find file uuid for: ", src, " -> ", dst
        exit(6)
    insertIntoEvents(fileUUID=fileUUID, eventIdentifierUUID=eventIdentifierUUID, eventType=eventType, eventDateTime=eventDateTime, eventDetail=eventDetail, eventOutcome="", eventOutcomeDetailNote=eventOutcomeDetailNote)

    #UPDATE THE CURRENT FILE PATH
    sql =  """UPDATE Files SET currentLocation='%s' WHERE fileUUID='%s';""" % (MySQLdb.escape_string(dst), fileUUID)
    databaseInterface.runSQL(sql)
 def __init__(self, unit, chainPK, notifyComplete=None, passVar=None, UUID=None, subJobOf=""):
     print "jobChain",  unit, chainPK
     if chainPK == None:
         return None
     self.unit = unit
     self.pk = chainPK
     self.notifyComplete = notifyComplete
     self.UUID = UUID
     self.linkSplitCount = 1
     self.subJobOf = subJobOf
     sql = """SELECT * FROM MicroServiceChains WHERE pk =  """ + chainPK.__str__()
     print sql
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     if row == None:
         sqlLock.release()
         return None
     while row != None:
         print "jobChain", row
         #self.pk = row[0]
         self.startingChainLink = row[1]
         self.description = row[2]
         row = c.fetchone()
     sqlLock.release()
     self.currentLink = jobChainLink(self, self.startingChainLink, unit, passVar=passVar, subJobOf=subJobOf)
     if self.currentLink == None:
         return None
Example #12
0
    def process_IN_MOVED_FROM(self, event):
        global movedFromCount
        #print event
        #print "Transfer IN_MOVED_FROM"
        #Wait for a moved to, and if one doesn't occur, consider it moved outside of the system.


        movedFromPath = os.path.join(event.path, event.name).replace(\
                             self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), \
                             "%transferDirectory%", 1)
        filesMoved = []
        sql = """SELECT fileUUID, currentLocation FROM Files WHERE transferUUID = '""" + self.unit.UUID + "' AND removedTime = 0 AND currentLocation LIKE '" + MySQLdb.escape_string(movedFromPath).replace("%", "\%") + "%';"
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            #print row
            filesMoved.append(row)
            row = c.fetchone()
        sqlLock.release()

        movedFromLock.acquire()
        utcDate = databaseInterface.getUTCDate()
        timer = threading.Timer(archivematicaMCP.config.getint('MCPServer', "delayTimer"), timerExpired, args=[event, utcDate], kwargs={})
        movedFrom[event.cookie] = (movedFromPath, filesMoved, timer)
        movedFromCount.value = movedFromCount.value + 1
        movedFromLock.release()

        #create timer to check if it's claimed by a move to
        timer.start()
def getDublinCore(type_, id):
    sql = """SELECT     title, creator, subject, description, publisher, contributor, date, type, format, identifier, source, relation, language, coverage, rights
    FROM Dublincore WHERE metadataAppliesToType = '%s' AND metadataAppliesToidentifier = '%s';""" % \
    (type_.__str__(), id.__str__())
    
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    if row == None:
        sqlLock.release()
        return None
    ret = etree.Element( "dublincore", nsmap = {None:dctermsNS} )
    ret.set(xsiBNS+"schemaLocation", dctermsNS + " http://dublincore.org/schemas/xmls/qdc/2008/02/11/dcterms.xsd")
    dctermsElements= ["isPartOf"]
    while row != None:
        key = ["title", "creator", "subject", "description", "publisher", "contributor", "date", "type", "format", "identifier", "source", "relation", "language", "coverage", "rights"]
        #title, creator, subject, description, publisher, contributor, date, type, format, identifier, source, relation, language, coverage, rights = row
        #key.index("title") == title
        i = 0
        for term in key:
            if row[i] != None:
                txt = row[i]
            else:
                txt = ""
            newChild(ret, term, text=txt)
            i+=1

        row = c.fetchone()
    sqlLock.release()
    return ret
Example #14
0
 def __init__(self, unit, chainPK, notifyComplete=None, passVar=None, UUID=None, subJobOf=""):
     """Create an instance of a chain from the MicroServiceChains table"""
     print "jobChain",  unit, chainPK
     if chainPK == None:
         return None
     self.unit = unit
     self.pk = chainPK
     self.notifyComplete = notifyComplete
     self.UUID = UUID
     self.linkSplitCount = 1
     self.subJobOf = subJobOf
     sql = """SELECT * FROM MicroServiceChains WHERE pk =  '%s'""" % (chainPK.__str__())
     print sql
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     if row == None:
         sqlLock.release()
         return None
     while row != None:
         print "jobChain", row
         #self.pk = row[0]
         self.startingChainLink = row[1]
         self.description = row[2]
         row = c.fetchone()
     sqlLock.release()
     self.currentLink = jobChainLink(self, self.startingChainLink, unit, passVar=passVar, subJobOf=subJobOf)
     if self.currentLink == None:
         return None
Example #15
0
    def reloadFileList(self):
        self.fileList = {}
        #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
        currentPath = self.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
        for directory, subDirectories, files in os.walk(currentPath):
            directory = directory.replace(currentPath, "%SIPDirectory%", 1)
            for file in files:
                filePath = os.path.join(directory, file)
                #print filePath
                self.fileList[filePath] = unitFile(filePath)

        sql = """SELECT  fileUUID, currentLocation FROM Files WHERE sipUUID =  '""" + self.UUID + "'"  #AND Files.removedTime = 0; TODO
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            #print row
            UUID = row[0]
            currentPath = row[1]
            if currentPath in self.fileList:
                self.fileList[currentPath].UUID = UUID
            else:
                print "todo: find deleted files/exclude"
                print row[99]  #fail
            row = c.fetchone()
            self.fileList[filePath].UUID = UUID
        sqlLock.release()
Example #16
0
    def reloadFileList(self):
        self.fileList = {}
        #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
        currentPath = self.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
        for directory, subDirectories, files in os.walk(currentPath):
            directory = directory.replace(currentPath, "%SIPDirectory%", 1)
            for file in files:
                if directory != "%SIPDirectory%":
                    filePath = os.path.join(directory, file)
                else:
                    filePath = directory + file
                #print filePath
                self.fileList[filePath] = unitFile(filePath, owningUnit=self)

        sql = """SELECT  fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND sipUUID =  '""" + self.UUID + "'"
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            #print row
            UUID = row[0]
            currentPath = row[1]
            fileGrpUse = row[2]
            if currentPath in self.fileList:
                self.fileList[currentPath].UUID = UUID
                self.fileList[currentPath].fileGrpUse = fileGrpUse
            else:
                print >> sys.stderr, self.fileList
                eventDetail = "SIP {" + self.UUID + "} has file {" + UUID + "}\"" + currentPath + "\" in the database, but file doesn't exist in the file system."
                print >> sys.stderr, "!!!", eventDetail, "!!!"
            row = c.fetchone()
        sqlLock.release()
    def reloadFileList(self):
        self.fileList = {}
        #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
        currentPath = self.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
        for directory, subDirectories, files in os.walk(currentPath):
            directory = directory.replace( currentPath, "%SIPDirectory%", 1)
            for file in files:
                if directory != "%SIPDirectory%":
                    filePath = os.path.join(directory, file)
                else:
                    filePath = directory + file
                #print filePath
                self.fileList[filePath] = unitFile(filePath, owningUnit=self)

        sql = """SELECT  fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND sipUUID =  '""" + self.UUID + "'"
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            #print row
            UUID = row[0]
            currentPath = row[1]
            fileGrpUse = row[2]
            if currentPath in self.fileList:
                self.fileList[currentPath].UUID = UUID
                self.fileList[currentPath].fileGrpUse = fileGrpUse
            else:
                print >>sys.stderr, self.fileList
                eventDetail = "SIP {" + self.UUID + "} has file {" + UUID + "}\"" + currentPath + "\" in the database, but file doesn't exist in the file system."
                print >>sys.stderr, "!!!", eventDetail, "!!!"
            row = c.fetchone()
        sqlLock.release()
Example #18
0
def findOrCreateSipInDB(path, waitSleep=dbWaitSleep):
    UUID = ""
    path = path.replace(config.get('MCPServer', "sharedDirectory"), "%sharedPath%", 1)

    #find UUID on end of SIP path
    uuidLen = -36
    if isUUID(path[uuidLen-1:-1]):
        UUID = path[uuidLen-1:-1]


    if UUID == "":
        #Find it in the database
        databaseInterface.printSQL = True
        sql = """SELECT sipUUID FROM SIPs WHERE currentPath = '""" + MySQLdb.escape_string(path) + "';"
        #if waitSleep != 0:
            #time.sleep(waitSleep) #let db be updated by the microservice that moved it.
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        if not row:
            print "Not opening existing SIP:", UUID, "-", path
        while row != None:
            UUID = row[0]
            print "Opening existing SIP:", UUID, "-", path
            row = c.fetchone()
        sqlLock.release()


    #Create it
    if UUID == "":
        UUID = databaseFunctions.createSIP(path)
        print "DEBUG creating sip", path, UUID
    return UUID
Example #19
0
def watchDirectories():
    """Start watching the watched directories defined in the WatchedDirectories table in the database."""
    rows = []
    sql = """SELECT watchedDirectoryPath, chain, onlyActOnDirectories, description FROM WatchedDirectories LEFT OUTER JOIN WatchedDirectoriesExpectedTypes ON WatchedDirectories.expectedType = WatchedDirectoriesExpectedTypes.pk"""
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        rows.append(row)
        row = c.fetchone()
    sqlLock.release()

    for row in rows:
        directory = row[0].replace("%watchDirectoryPath%", config.get('MCPServer', "watchDirectoryPath"), 1)
        if not os.path.isdir(directory):
            os.makedirs(directory)
        for item in os.listdir(directory):
            if item == ".gitignore":
                continue
            item = item.decode("utf-8")
            path = os.path.join(unicode(directory), item)
            #createUnitAndJobChain(path, row)
            while(limitTaskThreads <= threading.activeCount() + reservedAsTaskProcessingThreads ):
                time.sleep(1)
            createUnitAndJobChainThreaded(path, row, terminate=False)
        actOnFiles=True
        if row[2]: #onlyActOnDirectories
            actOnFiles=False
        watchDirectory.archivematicaWatchDirectory(directory,variablesAdded=row, callBackFunctionAdded=createUnitAndJobChainThreaded, alertOnFiles=actOnFiles, interval=config.getint('MCPServer', "watchDirectoriesPollInterval"))
    def __init__(self, jobChainLink, pk, unit):
        self.tasks = []
        self.pk = pk
        self.jobChainLink = jobChainLink
        sql = """SELECT * FROM StandardTasksConfigs where pk = '%s'""" % (pk.__str__())
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            print row
            #pk = row[0]
            filterFileEnd = row[1]
            filterFileStart = row[2]
            filterSubDir = row[3]
            self.requiresOutputLock = row[4]
            standardOutputFile = row[5]
            standardErrorFile = row[6]
            execute = row[7]
            self.execute = execute
            arguments = row[8]
            row = c.fetchone()
        sqlLock.release()

        #if reloadFileList:
        #    unit.reloadFileList()

        #        "%taskUUID%": task.UUID.__str__(), \

        if filterSubDir:
            directory = os.path.join(unit.currentPath, filterSubDir)
        else:
            directory = unit.currentPath
        
        if self.jobChainLink.passVar != None:
            if isinstance(self.jobChainLink.passVar, list):
                for passVar in self.jobChainLink.passVar:
                    if isinstance(passVar, replacementDic):
                        execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
            elif isinstance(self.jobChainLink.passVar, replacementDic):
                execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
                    
        commandReplacementDic = unit.getReplacementDic(directory)
                #for each key replace all instances of the key in the command string
        for key in commandReplacementDic.iterkeys():
            value = commandReplacementDic[key].replace("\"", ("\\\""))
            if execute:
                execute = execute.replace(key, value)
            if arguments:
                arguments = arguments.replace(key, value)
            if standardOutputFile:
                standardOutputFile = standardOutputFile.replace(key, value)
            if standardErrorFile:
                standardErrorFile = standardErrorFile.replace(key, value)

        UUID = uuid.uuid4().__str__()
        self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=UUID)
        databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
        t = threading.Thread(target=self.task.performTask)
        t.daemon = True
        t.start()
    def checkForPreconfiguredXML(self):
        ret = None
        xmlFilePath = os.path.join( \
                                        self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), \
                                        archivematicaMCP.config.get('MCPServer', "processingXMLFile") \
                                    )
        xmlFilePath = unicodeToStr(xmlFilePath)
        if os.path.isfile(xmlFilePath):
            # For a list of items with pks:
            # SELECT TasksConfigs.description, choiceAvailableAtLink, ' ' AS 'SPACE', MicroServiceChains.description, chainAvailable FROM MicroServiceChainChoice Join MicroServiceChains on MicroServiceChainChoice.chainAvailable = MicroServiceChains.pk Join MicroServiceChainLinks on MicroServiceChainLinks.pk = MicroServiceChainChoice.choiceAvailableAtLink Join TasksConfigs on TasksConfigs.pk = MicroServiceChainLinks.currentTask ORDER BY choiceAvailableAtLink desc;
            try:
                command = "sudo chmod 774 \"" + xmlFilePath + "\""
                if isinstance(command, unicode):
                    command = command.encode("utf-8")
                exitCode, stdOut, stdError = executeOrRun("command", command, "", printing=False)
                tree = etree.parse(xmlFilePath)
                root = tree.getroot()
                for preconfiguredChoice in root.find("preconfiguredChoices"):
                    #if int(preconfiguredChoice.find("appliesTo").text) == self.jobChainLink.pk:
                    if preconfiguredChoice.find("appliesTo").text == self.jobChainLink.description:
                        desiredChoice = preconfiguredChoice.find("goToChain").text
                        sql = """SELECT MicroServiceChains.pk FROM MicroServiceChainChoice Join MicroServiceChains on MicroServiceChainChoice.chainAvailable = MicroServiceChains.pk WHERE MicroServiceChains.description = '%s' AND MicroServiceChainChoice.choiceAvailableAtLink = %s;""" % (desiredChoice, self.jobChainLink.pk.__str__())
                        c, sqlLock = databaseInterface.querySQL(sql)
                        row = c.fetchone()
                        while row != None:
                            ret = row[0]
                            row = c.fetchone()
                        sqlLock.release()
                        try:
                            #<delay unitAtime="yes">30</delay>
                            delayXML = preconfiguredChoice.find("delay")
                            unitAtimeXML = delayXML.get("unitCtime")
                            if unitAtimeXML != None and unitAtimeXML.lower() != "no":
                                delaySeconds=int(delayXML.text)
                                unitTime = os.path.getmtime(self.unit.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1))
                                nowTime=time.time()
                                timeDifference = nowTime - unitTime
                                timeToGo = delaySeconds - timeDifference
                                print "time to go:", timeToGo
                                #print "that will be: ", (nowTime + timeToGo)
                                self.jobChainLink.setExitMessage("Waiting till: " + datetime.datetime.fromtimestamp((nowTime + timeToGo)).ctime())

                                t = threading.Timer(timeToGo, self.proceedWithChoice, args=[ret], kwargs={"delayTimerStart":True})
                                t.daemon = True
                                self.delayTimer = t
                                t.start()
                                return None

                        except Exception as inst:
                            print >>sys.stderr, "Error parsing xml:"
                            print >>sys.stderr, type(inst)
                            print >>sys.stderr, inst.args

            except Exception as inst:
                print >>sys.stderr, "Error parsing xml:"
                print >>sys.stderr, type(inst)
                print >>sys.stderr, inst.args
        return ret
    def __init__(self, jobChainLink, pk, unit):
        self.tasks = []
        self.pk = pk
        self.jobChainLink = jobChainLink
        sql = """SELECT * FROM StandardTasksConfigs where pk = """ + pk.__str__()
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            print row
            #pk = row[0]
            filterFileEnd = row[1]
            filterFileStart = row[2]
            filterSubDir = row[3]
            self.requiresOutputLock = row[4]
            standardOutputFile = row[5]
            standardErrorFile = row[6]
            execute = row[7]
            self.execute = execute
            arguments = row[8]
            row = c.fetchone()
        sqlLock.release()

        #if reloadFileList:
        #    unit.reloadFileList()

        #        "%taskUUID%": task.UUID.__str__(), \

        if filterSubDir:
            directory = os.path.join(unit.currentPath, filterSubDir)
        else:
            directory = unit.currentPath
        
        if self.jobChainLink.passVar != None:
            if isinstance(self.jobChainLink.passVar, list):
                for passVar in self.jobChainLink.passVar:
                    if isinstance(passVar, replacementDic):
                        execute, arguments, standardOutputFile, standardErrorFile = passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
            elif isinstance(self.jobChainLink.passVar, replacementDic):
                execute, arguments, standardOutputFile, standardErrorFile = self.jobChainLink.passVar.replace(execute, arguments, standardOutputFile, standardErrorFile)
                    
        commandReplacementDic = unit.getReplacementDic(directory)
                #for each key replace all instances of the key in the command string
        for key in commandReplacementDic.iterkeys():
            value = commandReplacementDic[key].replace("\"", ("\\\""))
            if execute:
                execute = execute.replace(key, value)
            if arguments:
                arguments = arguments.replace(key, value)
            if standardOutputFile:
                standardOutputFile = standardOutputFile.replace(key, value)
            if standardErrorFile:
                standardErrorFile = standardErrorFile.replace(key, value)

        UUID = uuid.uuid4().__str__()
        self.task = taskStandard(self, execute, arguments, standardOutputFile, standardErrorFile, UUID=UUID)
        databaseFunctions.logTaskCreatedSQL(self, commandReplacementDic, UUID, arguments)
        t = threading.Thread(target=self.task.performTask)
        t.daemon = True
        t.start()
    def checkForPreconfiguredXML(self):
        ret = None
        xmlFilePath = os.path.join( \
                                        self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/", \
                                        archivematicaMCP.config.get('MCPServer', "processingXMLFile") \
                                    )

        if os.path.isfile(xmlFilePath):
            # For a list of items with pks:
            # SELECT TasksConfigs.description, choiceAvailableAtLink, ' ' AS 'SPACE', MicroServiceChains.description, chainAvailable FROM MicroServiceChainChoice Join MicroServiceChains on MicroServiceChainChoice.chainAvailable = MicroServiceChains.pk Join MicroServiceChainLinks on MicroServiceChainLinks.pk = MicroServiceChainChoice.choiceAvailableAtLink Join TasksConfigs on TasksConfigs.pk = MicroServiceChainLinks.currentTask ORDER BY choiceAvailableAtLink desc;
            try:
                tree = etree.parse(xmlFilePath)
                root = tree.getroot()
                for preconfiguredChoice in root.find("preconfiguredChoices"):
                    #if int(preconfiguredChoice.find("appliesTo").text) == self.jobChainLink.pk:
                    if preconfiguredChoice.find("appliesTo").text == self.jobChainLink.description:
                        desiredChoice = preconfiguredChoice.find("goToChain").text
                        sql = """SELECT MicroServiceChoiceReplacementDic.replacementDic FROM MicroServiceChoiceReplacementDic  WHERE MicroServiceChoiceReplacementDic.description = '%s' AND MicroServiceChoiceReplacementDic.choiceAvailableAtLink = '%s';""" % (desiredChoice, self.jobChainLink.pk.__str__())
                        c, sqlLock = databaseInterface.querySQL(sql)
                        row = c.fetchone()
                        while row != None:
                            ret = row[0]
                            row = c.fetchone()
                        sqlLock.release()
                        try:
                            #<delay unitAtime="yes">30</delay>
                            delayXML = preconfiguredChoice.find("delay")
                            unitAtimeXML = delayXML.get("unitCtime")
                            if unitAtimeXML != None and unitAtimeXML.lower() != "no":
                                delaySeconds=int(delayXML.text)
                                unitTime = os.path.getmtime(self.unit.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1))
                                nowTime=time.time()
                                timeDifference = nowTime - unitTime
                                timeToGo = delaySeconds - timeDifference
                                print "time to go:", timeToGo
                                #print "that will be: ", (nowTime + timeToGo)
                                self.jobChainLink.setExitMessage("Waiting till: " + datetime.datetime.fromtimestamp((nowTime + timeToGo)).ctime())
                                rd = replacementDic(eval(ret))
                                if self.jobChainLink.passVar != None:
                                        if isinstance(self.jobChainLink.passVar, replacementDic):
                                            new = {}
                                            new.update(self.jobChainLink.passVar.dic)
                                            new.update(rd.dic)
                                            rd.dic = new
                                t = threading.Timer(timeToGo, self.jobChainLink.linkProcessingComplete, args=[0, rd], kwargs={})
                                t.daemon = True
                                t.start()

                                t2 = threading.Timer(timeToGo, self.jobChainLink.setExitMessage, args=["Completed successfully"], kwargs={})
                                t2.start()
                                return waitingOnTimer

                        except Exception as inst:
                            print >>sys.stderr, "Error parsing xml for pre-configured choice"
            except Exception as inst:
                print >>sys.stderr, "Error parsing xml for pre-configured choice"
        return ret
Example #24
0
 def getNextChainLinkPK(self, exitCode):
     if exitCode != None:
         ret = self.defaultNextChainLink
         sql = "SELECT nextMicroServiceChainLink FROM MicroServiceChainLinksExitCodes WHERE microServiceChainLink = '%s' AND exitCode = %s" % (self.pk.__str__(), exitCode.__str__())
         c, sqlLock = databaseInterface.querySQL(sql)
         row = c.fetchone()
         if row != None:
             ret = row[0]
         sqlLock.release()
         return ret
Example #25
0
 def getSoundFileToPlay(self, exitCode):
     if exitCode != None:
         ret = self.defaultSoundFile
         sql = "SELECT Sounds.fileLocation FROM MicroServiceChainLinksExitCodes LEFT OUTER JOIN Sounds ON MicroServiceChainLinksExitCodes.playSound = Sounds.pk WHERE microServiceChainLink = '%s' AND exitCode = %s" % (self.pk.__str__(), exitCode.__str__())
         c, sqlLock = databaseInterface.querySQL(sql)
         row = c.fetchone()
         if row != None:
             ret = row[0]
         sqlLock.release()
         return ret
def something(SIPDirectory, accessDirectory, objectsDirectory, DIPDirectory, SIPUUID, date, copy=False):
    #exitCode = 435
    exitCode = 179
    print SIPDirectory
    #For every file, & directory Try to find the matching file & directory in the objects directory
    for (path, dirs, files) in os.walk(accessDirectory):
        for file in files:
            accessPath = os.path.join(path, file)
            objectPath = accessPath.replace(accessDirectory, objectsDirectory, 1)
            objectName = os.path.basename(objectPath)
            objectNameExtensionIndex = objectName.rfind(".")

            if objectNameExtensionIndex != -1:
                objectName = objectName[:objectNameExtensionIndex + 1]
                objectNameLike = os.path.join( os.path.dirname(objectPath), objectName).replace(SIPDirectory, "%SIPDirectory%", 1)
                #sql = "SELECT fileUUID, currentLocation FROM Files WHERE currentLocation LIKE  '%s%' AND removedTime = 0 AND SIPUUID = '%s'" % (objectNameLike, SIPUUID)
                #ValueError: unsupported format character ''' (0x27) at index 76
                sql = "SELECT fileUUID, currentLocation FROM Files WHERE currentLocation LIKE  '" + objectNameLike + "%' AND removedTime = 0 AND SIPUUID = '"+ SIPUUID + "'"
                c, sqlLock = databaseInterface.querySQL(sql)
                row = c.fetchone()
                if not row:
                    print >>sys.stderr, "No corresponding object for:", accessPath.replace(SIPDirectory, "%SIPDirectory%", 1)
                    exitCode = 1
                update = []
                while row != None:
                    objectUUID = row[0]
                    objectPath = row[1]
                    objectExtension = objectPath.replace(objectNameLike, "", 1)
                    print objectName[objectNameExtensionIndex + 1:], objectExtension, "\t",
                    if objectExtension.find(".") != -1:
                        print
                        row = c.fetchone()
                        continue
                    print objectName[objectNameExtensionIndex + 1:], objectExtension, "\t",
                    print row  
                    dipPath = os.path.join(DIPDirectory,  "objects", "%s-%s" % (objectUUID, os.path.basename(accessPath)))
                    if copy:
                        print "TODO - copy not supported yet"
                    else:
                        #
                        dest = dipPath
                        renameAsSudo(accessPath, dest)

                        src = accessPath.replace(SIPDirectory, "%SIPDirectory%")
                        dst = dest.replace(SIPDirectory, "%SIPDirectory%")
                        update.append((src, dst))

                        #
                    row = c.fetchone()
                sqlLock.release()
                for src, dst in update:
                    eventDetail = ""
                    eventOutcomeDetailNote = "moved from=\"" + src + "\"; moved to=\"" + dst + "\""
                    updateFileLocation(src, dst, "movement", date, eventDetail, sipUUID=SIPUUID, eventOutcomeDetailNote = eventOutcomeDetailNote)
    return exitCode
 def reload(self):
     sql = """SELECT * FROM SIPs WHERE sipUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         print row
         #self.UUID = row[0]
         self.createdTime = deUnicode(row[1])
         self.currentPath = deUnicode(row[2])
         row = c.fetchone()
     sqlLock.release()
Example #28
0
 def getMagicLink(self):
     ret = None
     sql = """SELECT magicLink, magicLinkExitMessage FROM Transfers WHERE transferUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         print row
         ret = row
         row = c.fetchone()
     sqlLock.release()
     return ret
Example #29
0
 def reload(self):
     sql = """SELECT * FROM SIPs WHERE sipUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         print row
         #self.UUID = row[0]
         self.createdTime = deUnicode(row[1])
         self.currentPath = deUnicode(row[2])
         row = c.fetchone()
     sqlLock.release()
Example #30
0
 def getSoundFileToPlay(self, exitCode):
     if exitCode != None:
         ret = self.defaultSoundFile
         sql = "SELECT Sounds.fileLocation FROM MicroServiceChainLinksExitCodes LEFT OUTER JOIN Sounds ON MicroServiceChainLinksExitCodes.playSound = Sounds.pk WHERE microServiceChainLink = '%s' AND exitCode = %s" % (
             self.pk.__str__(), exitCode.__str__())
         c, sqlLock = databaseInterface.querySQL(sql)
         row = c.fetchone()
         if row != None:
             ret = row[0]
         sqlLock.release()
         return ret
Example #31
0
 def getNextChainLinkPK(self, exitCode):
     if exitCode != None:
         ret = self.defaultNextChainLink
         sql = "SELECT nextMicroServiceChainLink FROM MicroServiceChainLinksExitCodes WHERE microServiceChainLink = '%s' AND exitCode = %s" % (
             self.pk.__str__(), exitCode.__str__())
         c, sqlLock = databaseInterface.querySQL(sql)
         row = c.fetchone()
         if row != None:
             ret = row[0]
         sqlLock.release()
         return ret
Example #32
0
 def reload(self):
     sql = """SELECT transferUUID, currentLocation FROM Transfers WHERE transferUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         self.UUID = deUnicode(row[0])
         #self.createdTime = row[1]
         self.currentPath = deUnicode(row[1])
         row = c.fetchone()
     sqlLock.release()
     return
 def getMagicLink(self):
     ret = None
     sql = """SELECT magicLink, magicLinkExitMessage FROM SIPs WHERE sipUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         print row
         ret = row
         row = c.fetchone()
     sqlLock.release()
     return ret
    def __init__(self, jobChainLink, pk, unit):
        self.choices = []
        self.pk = pk
        self.jobChainLink = jobChainLink
        self.UUID = uuid.uuid4().__str__()
        self.unit = unit
        sql = """SELECT replacementDic, description FROM MicroServiceChoiceReplacementDic WHERE choiceAvailableAtLink = '%s'""" % (
            jobChainLink.pk.__str__())
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        choiceIndex = 0
        while row != None:
            print row
            replacementDic_ = row[0]
            description_ = row[1]
            self.choices.append((choiceIndex, description_, replacementDic_))
            row = c.fetchone()
            choiceIndex += 1
        sqlLock.release()
        #print "choices", self.choices

        preConfiguredChain = self.checkForPreconfiguredXML()
        if preConfiguredChain != None:
            if preConfiguredChain != waitingOnTimer:
                #time.sleep(archivematicaMCP.config.getint('MCPServer', "waitOnAutoApprove"))
                #print "checking for xml file for processing rules. TODO"
                self.jobChainLink.setExitMessage("Completed successfully")
                #jobChain.jobChain(self.unit, preConfiguredChain)
                rd = replacementDic(eval(preConfiguredChain))
                if self.jobChainLink.passVar != None:
                    if isinstance(self.jobChainLink.passVar, list):
                        found = False
                        for passVar in self.jobChainLink.passVar:
                            if isinstance(self.jobChainLink.passVar,
                                          replacementDic):
                                new = {}
                                new.update(self.jobChainLink.passVar.dic)
                                new.update(rd.dic)
                                rd.dic = [new]
                                found = True
                                break
                        if not found:
                            self.jobChainLink.passVar.append(rd)
                            rd = self.jobChainLink.passVar
                else:
                    rd = [rd]
                self.jobChainLink.linkProcessingComplete(0, rd)
            else:
                print "waiting on delay to resume processing on unit:", unit
        else:
            choicesAvailableForUnitsLock.acquire()
            self.jobChainLink.setExitMessage('Awaiting decision')
            choicesAvailableForUnits[self.jobChainLink.UUID] = self
            choicesAvailableForUnitsLock.release()
Example #35
0
 def reload(self):
     sql = """SELECT transferUUID, currentLocation FROM Transfers WHERE transferUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         self.UUID = deUnicode(row[0])
         #self.createdTime = row[1]
         self.currentPath = deUnicode(row[1])
         row = c.fetchone()
     sqlLock.release()
     return
    def __init__(self, jobChainLink, pk, unit):
        self.choices = []
        self.pk = pk
        self.jobChainLink = jobChainLink
        self.UUID = uuid.uuid4().__str__()
        self.unit = unit
        sql = """SELECT replacementDic, description FROM MicroServiceChoiceReplacementDic WHERE choiceAvailableAtLink = """ + jobChainLink.pk.__str__()
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        choiceIndex = 0
        while row != None:
            print row
            replacementDic_ = row[0]
            description_ = row[1]
            self.choices.append((choiceIndex, description_, replacementDic_))
            row = c.fetchone()
            choiceIndex += 1
        sqlLock.release()
        #print "choices", self.choices

        preConfiguredChain = self.checkForPreconfiguredXML()
        if preConfiguredChain != None:
            if preConfiguredChain != waitingOnTimer:
                #time.sleep(archivematicaMCP.config.getint('MCPServer', "waitOnAutoApprove"))
                #print "checking for xml file for processing rules. TODO"
                self.jobChainLink.setExitMessage("Completed successfully")
                #jobChain.jobChain(self.unit, preConfiguredChain)
                rd = replacementDic(eval(preConfiguredChain))
                if self.jobChainLink.passVar != None:
                    if isinstance(self.jobChainLink.passVar, list):
                        found = False
                        for passVar in self.jobChainLink.passVar:
                            if isinstance(self.jobChainLink.passVar, replacementDic):
                                new = {}
                                new.update(self.jobChainLink.passVar.dic)
                                new.update(rd.dic)
                                rd.dic = [new]
                                found = True
                                break
                        if not found:
                            self.jobChainLink.passVar.append(rd)
                            rd = self.jobChainLink.passVar 
                else:
                    rd = [rd]
                self.jobChainLink.linkProcessingComplete(0, rd)
            else:
                print "waiting on delay to resume processing on unit:", unit
        else:
            choicesAvailableForUnitsLock.acquire()
            self.jobChainLink.setExitMessage('Awaiting decision')
            choicesAvailableForUnits[self.jobChainLink.UUID] = self
            choicesAvailableForUnitsLock.release()
Example #37
0
 def getMagicLink(self):
     """Load a link from the unit to process.
     Deprecated! Replaced with Set/Load Unit Variable"""
     ret = None
     sql = """SELECT magicLink, magicLinkExitMessage FROM Transfers WHERE transferUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         print row
         ret = row
         row = c.fetchone()
     sqlLock.release()
     return ret
Example #38
0
 def updateExitMessage(self, exitCode):
     ret = self.defaultExitMessage
     if exitCode != None:
         sql = "SELECT exitMessage FROM MicroServiceChainLinksExitCodes WHERE microServiceChainLink = '%s' AND exitCode = %s" % (self.pk.__str__(), exitCode.__str__())
         c, sqlLock = databaseInterface.querySQL(sql)
         row = c.fetchone()
         if row != None:
             ret = row[0]
         sqlLock.release()
     if ret != None:
         self.setExitMessage(ret)
     else:
         print "No exit message"
Example #39
0
 def getMagicLink(self):
     """Load a link from the unit to process.
     Deprecated! Replaced with Set/Load Unit Variable"""
     ret = None
     sql = """SELECT magicLink, magicLinkExitMessage FROM Transfers WHERE transferUUID =  '""" + self.UUID + "'"
     c, sqlLock = databaseInterface.querySQL(sql)
     row = c.fetchone()
     while row != None:
         print row
         ret = row
         row = c.fetchone()
     sqlLock.release()
     return ret
def main(sipUUID, transfersMetadataDirectory, transfersLogsDirectory, sharedPath=""):
    if not os.path.exists(transfersMetadataDirectory):
        os.makedirs(transfersMetadataDirectory)
    if not os.path.exists(transfersLogsDirectory):
        os.makedirs(transfersLogsDirectory)

    exitCode = 0
    sql = """SELECT Files.transferUUID, Transfers.currentLocation FROM Files
        JOIN Transfers on Transfers.transferUUID = Files.transferUUID
        WHERE sipUUID = '%s'
        GROUP BY Files.transferUUID;""" % (sipUUID)

    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        try:
            transferUUID = row[0]
            transferPath = row[1]
            if sharedPath != "":
                transferPath = transferPath.replace("%sharedPath%", sharedPath, 1)
            transferBasename = os.path.basename(os.path.abspath(transferPath))
            transferMetaDestDir = os.path.join(transfersMetadataDirectory, transferBasename)
            transfersLogsDestDir = os.path.join(transfersLogsDirectory, transferBasename)
            if not os.path.exists(transferMetaDestDir):
                os.makedirs(transferMetaDestDir)
                transferMetadataDirectory = os.path.join(transferPath, "metadata")
                for met in os.listdir(transferMetadataDirectory):
                    if met == "submissionDocumentation":
                        continue
                    item = os.path.join(transferMetadataDirectory, met)
                    if os.path.isdir(item):
                        shutil.copytree(item, os.path.join(transferMetaDestDir, met))
                    else:
                        shutil.copy(item, os.path.join(transferMetaDestDir, met))
                print "copied: ", transferPath + "metadata", " -> ", os.path.join(transferMetaDestDir, "metadata")
            if not os.path.exists(transfersLogsDestDir):
                os.makedirs(transfersLogsDestDir)
                shutil.copytree(transferPath + "logs", os.path.join(transfersLogsDestDir, "logs"))
                print "copied: ", transferPath + "logs", " -> ", os.path.join(transfersLogsDestDir, "logs")
                

        except Exception as inst:
            print >>sys.stderr, type(inst)
            print >>sys.stderr, inst.args
            traceback.print_exc(file=sys.stderr)
            print >>sys.stderr, "Error with transfer: ", row
            exitCode += 1
        row = c.fetchone()

    sqlLock.release()
    exit(exitCode)
def createDigiprovMD(fileUUID):
    ret = []
    #EVENTS

    sql = "SELECT pk, fileUUID, eventIdentifierUUID, eventType, eventDateTime, eventDetail, eventOutcome, eventOutcomeDetailNote, linkingAgentIdentifier FROM Events WHERE fileUUID = '" + fileUUID + "';"
    rows = databaseInterface.queryAllSQL(sql)
    for row in rows:
        digiprovMD = etree.Element("digiprovMD")
        ret.append(digiprovMD) #newChild(amdSec, "digiprovMD")
        #digiprovMD.set("ID", "digiprov-"+ os.path.basename(filename) + "-" + fileUUID)
        global globalDigiprovMDCounter
        globalDigiprovMDCounter += 1
        digiprovMD.set("ID", "digiprovMD_"+ globalDigiprovMDCounter.__str__())

        mdWrap = newChild(digiprovMD,"mdWrap")
        mdWrap.set("MDTYPE", "PREMIS:EVENT")
        xmlData = newChild(mdWrap,"xmlData")
        event = etree.SubElement(xmlData, "event", nsmap={None: premisNS})
        event.set(xsiBNS+"schemaLocation", premisNS + " http://www.loc.gov/standards/premis/v2/premis-v2-2.xsd")
        event.set("version", "2.2")

        eventIdentifier = etree.SubElement(event, "eventIdentifier")
        etree.SubElement(eventIdentifier, "eventIdentifierType").text = "UUID"
        etree.SubElement(eventIdentifier, "eventIdentifierValue").text = row[2]

        etree.SubElement(event, "eventType").text = row[3]
        etree.SubElement(event, "eventDateTime").text = row[4].__str__().replace(" ", "T")
        etree.SubElement(event, "eventDetail").text = escape(row[5])

        eventOutcomeInformation  = etree.SubElement(event, "eventOutcomeInformation")
        etree.SubElement(eventOutcomeInformation, "eventOutcome").text = row[6]
        eventOutcomeDetail = etree.SubElement(eventOutcomeInformation, "eventOutcomeDetail")
        etree.SubElement(eventOutcomeDetail, "eventOutcomeDetailNote").text = escape(row[7])
        
        if row[8]:
            linkingAgentIdentifier = etree.SubElement(event, "linkingAgentIdentifier")
            etree.SubElement(linkingAgentIdentifier, "linkingAgentIdentifierType").text = "Archivematica user pk"
            etree.SubElement(linkingAgentIdentifier, "linkingAgentIdentifierValue").text = row[8].__str__()
        
        #linkingAgentIdentifier
        sql = """SELECT agentIdentifierType, agentIdentifierValue, agentName, agentType FROM Agents;"""
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            linkingAgentIdentifier = etree.SubElement(event, "linkingAgentIdentifier")
            etree.SubElement(linkingAgentIdentifier, "linkingAgentIdentifierType").text = row[0]
            etree.SubElement(linkingAgentIdentifier, "linkingAgentIdentifierValue").text = row[1]
            row = c.fetchone()
        sqlLock.release()
    return ret
Example #42
0
def get_pairs(dip_uuid):
    pairs = dict()
    #connect to archivematica db, make a set of pairs from pairs table
   
    sql = """SELECT fileUUID, resourceId, resourceComponentId from AtkDIPObjectResourcePairing where dipUUID = %s"""
    c, sqlLock = databaseInterface.querySQL(sql, (dip_uuid,))
    dbresult = c.fetchall()
    for item in dbresult:
        ids = dict()
        ids['rid'] = item[1]
        ids['rcid'] = item[2]
        pairs[item[0]] =  ids
    sqlLock.release()
    return pairs
Example #43
0
 def updateExitMessage(self, exitCode):
     ret = self.defaultExitMessage
     if exitCode != None:
         sql = "SELECT exitMessage FROM MicroServiceChainLinksExitCodes WHERE microServiceChainLink = '%s' AND exitCode = %s" % (
             self.pk.__str__(), exitCode.__str__())
         c, sqlLock = databaseInterface.querySQL(sql)
         row = c.fetchone()
         if row != None:
             ret = row[0]
         sqlLock.release()
     if ret != None:
         self.setExitMessage(ret)
     else:
         print "No exit message"
def get_pairs(dip_uuid):
    pairs = dict()
    #connect to archivematica db, make a set of pairs from pairs table
   
    sql = """SELECT fileUUID, resourceId, resourceComponentId from AtkDIPObjectResourcePairing where dipUUID = '{}'""".format(dip_uuid)
    logger.debug(sql)
    c, sqlLock = databaseInterface.querySQL(sql)
    dbresult = c.fetchall()
    for item in dbresult:
        ids = dict()
        ids['rid'] = item[1]
        ids['rcid'] = item[2]
        pairs[item[0]] =  ids
    sqlLock.release()
    return pairs
def cleanWatchedDirectories():
    sql = """SELECT watchedDirectoryPath FROM WatchedDirectories;"""
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        try:
            directory = row[0].replace(
                "%watchDirectoryPath%", "/var/archivematica/sharedDirectory/watchedDirectories/", 1
            )
            removeEverythingInDirectory(directory)
        except Exception as inst:
            print "debug except 2"
            print type(inst)  # the exception instance
            print inst.args  # arguments stored in .args
        row = c.fetchone()
    sqlLock.release()
Example #46
0
def cleanWatchedDirectories():
    sql = """SELECT watchedDirectoryPath FROM WatchedDirectories;"""
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        try:
            directory = row[0].replace(
                "%watchDirectoryPath%",
                "/var/archivematica/sharedDirectory/watchedDirectories/", 1)
            removeEverythingInDirectory(directory)
        except Exception as inst:
            print "debug except 2"
            print type(inst)  # the exception instance
            print inst.args  # arguments stored in .args
        row = c.fetchone()
    sqlLock.release()
Example #47
0
    def process_IN_DELETE(self, event):
        #print event
        #print "Transfer IN_DELETE"
        #Wait for a moved to, and if one doesn't occur, consider it moved outside of the system.

        movedFromPath = os.path.join(event.path, event.name).replace(\
                             self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), \
                             "%transferDirectory%", 1)
        filesMoved = []
        sql = """SELECT fileUUID, currentLocation FROM Files WHERE transferUUID = '""" + self.unit.UUID + "' AND removedTime = 0 AND currentLocation LIKE '" + MySQLdb.escape_string(movedFromPath).replace("%", "\%") + "%';"
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            filesMoved.append(row)
            row = c.fetchone()
        sqlLock.release()
        for fileUUID, currentLocation in filesMoved:
            fileWasRemoved(fileUUID, eventOutcomeDetailNote = "removed from: " + currentLocation)
Example #48
0
    def reloadFileList(self):
        """Match files to their UUID's via their location and the File table's currentLocation"""
        self.fileList = {}
        #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
        currentPath = self.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
        #print "currentPath: ", currentPath, type(currentPath)
        try:
            #print currentPath, type(currentPath)
            for directory, subDirectories, files in os.walk(currentPath):
                directory = directory.replace(currentPath,
                                              "%transferDirectory%", 1)
                for file in files:
                    if "%transferDirectory%" != directory:
                        filePath = os.path.join(directory, file)
                    else:
                        filePath = directory + file
                    self.fileList[filePath] = unitFile(filePath,
                                                       owningUnit=self)

            sql = """SELECT  fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND transferUUID =  '""" + self.UUID + "'"
            #print sql
            c, sqlLock = databaseInterface.querySQL(sql)
            row = c.fetchone()
            #print self.fileList
            while row != None:
                #print row
                UUID = row[0]
                currentPath = row[1].encode("utf-8")
                fileGrpUse = row[2]
                #print currentPath in self.fileList, row
                if currentPath in self.fileList:
                    self.fileList[currentPath].UUID = UUID
                    self.fileList[currentPath].fileGrpUse = fileGrpUse
                else:
                    print >> sys.stderr, "!!!", "Transfer {" + self.UUID + "} has file {" + UUID + "}\"", currentPath, "\" in the database, but file doesn't exist in the file system.", "!!!"
                row = c.fetchone()
            sqlLock.release()

        except Exception as inst:
            traceback.print_exc(file=sys.stdout)
            print type(inst)
            print inst.args
            exit(1)
Example #49
0
    def __init__(self,
                 jobChain,
                 jobChainLinkPK,
                 unit,
                 passVar=None,
                 subJobOf=""):
        if jobChainLinkPK == None:
            return None
        self.UUID = uuid.uuid4().__str__()
        self.jobChain = jobChain
        self.pk = jobChainLinkPK
        self.unit = unit
        self.passVar = passVar
        self.createdDate = databaseInterface.getUTCDate()
        self.subJobOf = subJobOf
        sql = """SELECT MicroServiceChainLinks.currentTask, MicroServiceChainLinks.defaultNextChainLink, TasksConfigs.taskType, TasksConfigs.taskTypePKReference, TasksConfigs.description, MicroServiceChainLinks.reloadFileList, Sounds.fileLocation, MicroServiceChainLinks.defaultExitMessage, MicroServiceChainLinks.microserviceGroup FROM MicroServiceChainLinks LEFT OUTER JOIN Sounds ON MicroServiceChainLinks.defaultPlaySound = Sounds.pk JOIN TasksConfigs on MicroServiceChainLinks.currentTask = TasksConfigs.pk WHERE MicroServiceChainLinks.pk = '%s'""" % (
            jobChainLinkPK.__str__())
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        if row == None:
            sqlLock.release()
            return None
        while row != None:
            print row
            self.currentTask = row[0]
            self.defaultNextChainLink = row[1]
            taskType = row[2]
            taskTypePKReference = row[3]
            self.description = row[4]
            self.reloadFileList = row[5]
            self.defaultSoundFile = row[6]
            self.defaultExitMessage = row[7]
            self.microserviceGroup = row[8]
            row = c.fetchone()
        sqlLock.release()

        print "<<<<<<<<< ", self.description, " >>>>>>>>>"
        self.unit.reload()

        logJobCreatedSQL(self)

        if self.createTasks(taskType, taskTypePKReference) == None:
            self.getNextChainLinkPK(None)
Example #50
0
    def __init__(self, commandID, replacementDic, onSuccess=None, opts=None):
        self.pk = commandID
        self.replacementDic = replacementDic
        self.onSuccess = onSuccess
        self.stdOut = ""
        self.stdErr = ""
        self.exitCode = None
        self.failedCount = 0
        self.opts = opts
        sql = """SELECT CT.type, C.verificationCommand, C.eventDetailCommand, C.command, C.outputLocation, C.description, C.outputFileFormat
        FROM Commands AS C
        JOIN CommandTypes AS CT ON C.commandType = CT.pk
        WHERE C.pk = '""" + commandID.__str__() + """'
        ;"""
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            rowSTR = []
            for colIndex in range(len(row)):
                rowSTR.append(toStrFromUnicode(row[colIndex]))
            self.type, \
            self.verificationCommand, \
            self.eventDetailCommand, \
            self.command, \
            self.outputLocation, \
            self.description, \
            self.outputFormat = \
            rowSTR
            if isinstance(self.command, unicode):
                self.command = self.command.encode('utf-8')
            row = c.fetchone()
        sqlLock.release()
        if self.verificationCommand:
            self.verificationCommand = Command(self.verificationCommand,
                                               replacementDic)
            self.verificationCommand.command = self.verificationCommand.command.replace(
                "%outputLocation%", self.outputLocation)

        if self.eventDetailCommand:
            self.eventDetailCommand = Command(self.eventDetailCommand,
                                              replacementDic)
            self.eventDetailCommand.command = self.eventDetailCommand.command.replace(
                "%outputLocation%", self.outputLocation)
Example #51
0
    def reloadFileList(self):
        """Match files to their UUID's via their location and the File table's currentLocation"""
        self.fileList = {}
        #os.walk(top[, topdown=True[, onerror=None[, followlinks=False]]])
        currentPath = self.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/"
        #print "currentPath: ", currentPath, type(currentPath)
        try:
            #print currentPath, type(currentPath)
            for directory, subDirectories, files in os.walk(currentPath):
                directory = directory.replace( currentPath, "%transferDirectory%", 1) 
                for file in files:
                    if "%transferDirectory%" !=  directory:
                        filePath = os.path.join(directory, file)
                    else:
                        filePath = directory + file
                    self.fileList[filePath] = unitFile(filePath, owningUnit=self)

            sql = """SELECT  fileUUID, currentLocation, fileGrpUse FROM Files WHERE removedTime = 0 AND transferUUID =  '""" + self.UUID + "'"
            #print sql
            c, sqlLock = databaseInterface.querySQL(sql)
            row = c.fetchone()
            #print self.fileList
            while row != None:
                #print row
                UUID = row[0]
                currentPath = row[1].encode("utf-8")
                fileGrpUse = row[2]
                #print currentPath in self.fileList, row
                if currentPath in self.fileList:
                    self.fileList[currentPath].UUID = UUID
                    self.fileList[currentPath].fileGrpUse = fileGrpUse
                else:
                    print >>sys.stderr, "!!!", "Transfer {" + self.UUID + "} has file {" + UUID + "}\"", currentPath, "\" in the database, but file doesn't exist in the file system.", "!!!"
                row = c.fetchone()
            sqlLock.release()

        except Exception as inst:
            traceback.print_exc(file=sys.stdout)
            print  type(inst)
            print  inst.args
            exit(1)
Example #52
0
    def __init__(self, jobChainLink, pk, unit):
        self.pk = pk
        self.jobChainLink = jobChainLink
        self.UUID = uuid.uuid4().__str__()
        self.unit = unit

        ###GET THE MAGIC NUMBER FROM THE TASK stuff
        link = 0
        sql = """SELECT execute FROM TasksConfigsAssignMagicLink where pk = '%s'""" % (
            pk)
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            print row
            link = row[0]
            row = c.fetchone()
        sqlLock.release()

        ###Update the unit
        #set the magic number
        self.unit.setMagicLink(link, exitStatus="")
        self.jobChainLink.linkProcessingComplete(0)
def verifyChecksum(fileUUID, filePath, date, eventIdentifierUUID):
    sql = """SELECT checksum FROM Files WHERE fileUUID = '""" + fileUUID + "'"
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    checksumDB = ""
    while row != None:
        checksumDB = row[0]
        row = c.fetchone()
    sqlLock.release()
    if checksumDB == None or checksumDB == "" or checksumDB == "None":
        print >> sys.stderr, "No checksum found in database for file:", fileUUID, filePath
        exit(1)
    checksumFile = sha_for_file(filePath)

    eventOutcome = ""
    eventOutcomeDetailNote = ""
    exitCode = 0
    if checksumFile != checksumDB:
        eventOutcomeDetailNote = checksumFile.__str__(
        ) + " != " + checksumDB.__str__()
        eventOutcome = "Fail"
        exitCode = 2
        print >> sys.stderr, "Checksums do not match:", fileUUID, filePath
        print >> sys.stderr, eventOutcomeDetailNote
    else:
        eventOutcomeDetailNote = "%s %s" % (checksumFile.__str__(), "verified")
        eventOutcome = "Pass"
        exitCode = 0

    #insertIntoEvents(fileUUID="", eventIdentifierUUID="", eventType="", eventDateTime=databaseInterface.getUTCDate(), eventDetail="", eventOutcome="", eventOutcomeDetailNote="")
    databaseFunctions.insertIntoEvents(fileUUID=fileUUID, \
                 eventIdentifierUUID=eventIdentifierUUID, \
                 eventType="fixity check", \
                 eventDateTime=date, \
                 eventOutcome=eventOutcome, \
                 eventOutcomeDetailNote=eventOutcomeDetailNote, \
                 eventDetail="program=\"python\"; module=\"hashlib.sha256()\"")

    exit(exitCode)
Example #54
0
    def __init__(self, jobChainLink, pk, unit):
        self.pk = pk
        self.jobChainLink = jobChainLink
        self.UUID = uuid.uuid4().__str__()
        self.unit = unit

        sql = """SELECT variable, variableValue, defaultMicroServiceChainLink FROM TasksConfigsUnitVariableLinkPull where pk = '%s'""" % (
            pk)
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            print row
            variable, variableValue, defaultMicroServiceChainLink = row
            row = c.fetchone()
        sqlLock.release()
        link = self.unit.getmicroServiceChainLink(
            variable, variableValue, defaultMicroServiceChainLink)

        ###Update the unit
        if link != None:
            self.jobChainLink.setExitMessage("Completed successfully")
            self.jobChainLink.jobChain.nextChainLink(link)
Example #55
0
    def __init__(self, jobChainLink, pk, unit):
        self.pk = pk
        self.jobChainLink = jobChainLink
        self.UUID = uuid.uuid4().__str__()
        self.unit = unit

        ###GET THE MAGIC NUMBER FROM THE TASK stuff
        link = ""
        sql = """SELECT variable, variableValue, microServiceChainLink FROM TasksConfigsSetUnitVariable where pk = '%s'""" % (
            pk)
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            print row
            variable, variableValue, microServiceChainLink = row
            row = c.fetchone()
        sqlLock.release()

        ###Update the unit
        #set the magic number
        self.unit.setVariable(variable, variableValue, microServiceChainLink)
        self.jobChainLink.linkProcessingComplete(0)
Example #56
0
    def __init__(self, jobChainLink, pk, unit):
        """Used to get a selection, from a list of chains, to process"""
        self.choices = []
        self.pk = pk
        self.jobChainLink = jobChainLink
        self.UUID = uuid.uuid4().__str__()
        self.unit = unit
        self.delayTimerLock = threading.Lock()
        self.delayTimer = None
        sql = """SELECT chainAvailable, description FROM MicroServiceChainChoice JOIN MicroServiceChains on chainAvailable = MicroServiceChains.pk WHERE choiceAvailableAtLink = '%s' ORDER BY MicroServiceChainChoice.pk;""" % (
            jobChainLink.pk.__str__())
        c, sqlLock = databaseInterface.querySQL(sql)
        row = c.fetchone()
        while row != None:
            print row
            chainAvailable = row[0]
            description = row[1]
            self.choices.append((
                chainAvailable,
                description,
            ))
            row = c.fetchone()
        sqlLock.release()

        preConfiguredChain = self.checkForPreconfiguredXML()
        if preConfiguredChain != None:
            time.sleep(
                archivematicaMCP.config.getint('MCPServer',
                                               "waitOnAutoApprove"))
            self.jobChainLink.setExitMessage("Completed successfully")
            jobChain.jobChain(self.unit, preConfiguredChain)

        else:
            choicesAvailableForUnitsLock.acquire()
            if self.delayTimer == None:
                self.jobChainLink.setExitMessage('Awaiting decision')
            choicesAvailableForUnits[self.jobChainLink.UUID] = self
            choicesAvailableForUnitsLock.release()
Example #57
0
def watchDirectories():
    """Start watching the watched directories defined in the WatchedDirectories table in the database."""
    rows = []
    sql = """SELECT watchedDirectoryPath, chain, onlyActOnDirectories, description FROM WatchedDirectories LEFT OUTER JOIN WatchedDirectoriesExpectedTypes ON WatchedDirectories.expectedType = WatchedDirectoriesExpectedTypes.pk"""
    c, sqlLock = databaseInterface.querySQL(sql)
    row = c.fetchone()
    while row != None:
        rows.append(row)
        row = c.fetchone()
    sqlLock.release()

    for row in rows:
        directory = row[0].replace(
            "%watchDirectoryPath%",
            config.get('MCPServer', "watchDirectoryPath"), 1)
        if not os.path.isdir(directory):
            os.makedirs(directory)
        for item in os.listdir(directory):
            if item == ".gitignore":
                continue
            item = item.decode("utf-8")
            path = os.path.join(unicode(directory), item)
            #createUnitAndJobChain(path, row)
            while (limitTaskThreads <=
                   threading.activeCount() + reservedAsTaskProcessingThreads):
                time.sleep(1)
            createUnitAndJobChainThreaded(path, row, terminate=False)
        actOnFiles = True
        if row[2]:  #onlyActOnDirectories
            actOnFiles = False
        watchDirectory.archivematicaWatchDirectory(
            directory,
            variablesAdded=row,
            callBackFunctionAdded=createUnitAndJobChainThreaded,
            alertOnFiles=actOnFiles,
            interval=config.getint('MCPServer',
                                   "watchDirectoriesPollInterval"))
    def checkForPreconfiguredXML(self):
        ret = None
        xmlFilePath = os.path.join( \
                                        self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1) + "/", \
                                        archivematicaMCP.config.get('MCPServer', "processingXMLFile") \
                                    )

        if os.path.isfile(xmlFilePath):
            # For a list of items with pks:
            # SELECT TasksConfigs.description, choiceAvailableAtLink, ' ' AS 'SPACE', MicroServiceChains.description, chainAvailable FROM MicroServiceChainChoice Join MicroServiceChains on MicroServiceChainChoice.chainAvailable = MicroServiceChains.pk Join MicroServiceChainLinks on MicroServiceChainLinks.pk = MicroServiceChainChoice.choiceAvailableAtLink Join TasksConfigs on TasksConfigs.pk = MicroServiceChainLinks.currentTask ORDER BY choiceAvailableAtLink desc;
            try:
                tree = etree.parse(xmlFilePath)
                root = tree.getroot()
                for preconfiguredChoice in root.find("preconfiguredChoices"):
                    #if int(preconfiguredChoice.find("appliesTo").text) == self.jobChainLink.pk:
                    if preconfiguredChoice.find(
                            "appliesTo").text == self.jobChainLink.description:
                        desiredChoice = preconfiguredChoice.find(
                            "goToChain").text
                        sql = """SELECT MicroServiceChoiceReplacementDic.replacementDic FROM MicroServiceChoiceReplacementDic  WHERE MicroServiceChoiceReplacementDic.description = '%s' AND MicroServiceChoiceReplacementDic.choiceAvailableAtLink = '%s';""" % (
                            desiredChoice, self.jobChainLink.pk.__str__())
                        c, sqlLock = databaseInterface.querySQL(sql)
                        row = c.fetchone()
                        while row != None:
                            ret = row[0]
                            row = c.fetchone()
                        sqlLock.release()
                        try:
                            #<delay unitAtime="yes">30</delay>
                            delayXML = preconfiguredChoice.find("delay")
                            unitAtimeXML = delayXML.get("unitCtime")
                            if unitAtimeXML != None and unitAtimeXML.lower(
                            ) != "no":
                                delaySeconds = int(delayXML.text)
                                unitTime = os.path.getmtime(self.unit.currentPath.replace("%sharedPath%", \
                                               archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1))
                                nowTime = time.time()
                                timeDifference = nowTime - unitTime
                                timeToGo = delaySeconds - timeDifference
                                print "time to go:", timeToGo
                                #print "that will be: ", (nowTime + timeToGo)
                                self.jobChainLink.setExitMessage(
                                    "Waiting till: " +
                                    datetime.datetime.fromtimestamp(
                                        (nowTime + timeToGo)).ctime())
                                rd = replacementDic(eval(ret))
                                if self.jobChainLink.passVar != None:
                                    if isinstance(self.jobChainLink.passVar,
                                                  replacementDic):
                                        new = {}
                                        new.update(
                                            self.jobChainLink.passVar.dic)
                                        new.update(rd.dic)
                                        rd.dic = new
                                t = threading.Timer(
                                    timeToGo,
                                    self.jobChainLink.linkProcessingComplete,
                                    args=[0, rd],
                                    kwargs={})
                                t.daemon = True
                                t.start()

                                t2 = threading.Timer(
                                    timeToGo,
                                    self.jobChainLink.setExitMessage,
                                    args=["Completed successfully"],
                                    kwargs={})
                                t2.start()
                                return waitingOnTimer

                        except Exception as inst:
                            print >> sys.stderr, "Error parsing xml:"
                            print >> sys.stderr, type(inst)
                            print >> sys.stderr, inst.args

            except Exception as inst:
                print >> sys.stderr, "Error parsing xml:"
                print >> sys.stderr, type(inst)
                print >> sys.stderr, inst.args
        return ret