def process_transfer(request, transfer_uuid): response = {} if request.user.id: # get transfer info transfer = models.Transfer.objects.get(uuid=transfer_uuid) transfer_path = transfer.currentlocation.replace( '%sharedPath%', helpers.get_server_config_value('sharedDirectory') ) import MySQLdb import databaseInterface import databaseFunctions import shutil from archivematicaCreateStructuredDirectory import createStructuredDirectory from archivematicaCreateStructuredDirectory import createManualNormalizedDirectoriesList createStructuredDirectory(transfer_path, createManualNormalizedDirectories=False) processingDirectory = helpers.get_server_config_value('processingDirectory') transfer_directory_name = os.path.basename(transfer_path[:-1]) transfer_name = transfer_directory_name[:-37] sharedPath = helpers.get_server_config_value('sharedDirectory') tmpSIPDir = os.path.join(processingDirectory, transfer_name) + "/" #processSIPDirectory = os.path.join(sharedPath, 'watchedDirectories/system/autoProcessSIP') + '/' processSIPDirectory = os.path.join(sharedPath, 'watchedDirectories/SIPCreation/SIPsUnderConstruction') + '/' #destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/" #destSIPDir = os.path.join(processSIPDirectory, transfer_name + '-' + ) + "/" createStructuredDirectory(tmpSIPDir, createManualNormalizedDirectories=False) objectsDirectory = os.path.join(transfer_path, 'objects') + '/' """ #create row in SIPs table if one doesn't already exist lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%') #lookup_path = '%sharedPath%watchedDirectories/workFlowDecisions/createDip/' + transfer_name + '/' sql = " " "SELECT sipUUID FROM SIPs WHERE currentPath = '" " " + MySQLdb.escape_string(lookup_path) + "';" rows = databaseInterface.queryAllSQL(sql) if len(rows) > 0: row = rows[0] sipUUID = row[0] else: sipUUID = uuid.uuid4().__str__() databaseFunctions.createSIP(lookup_path, sipUUID) """ sipUUID = uuid.uuid4().__str__() destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/" lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%') databaseFunctions.createSIP(lookup_path, sipUUID) #move the objects to the SIPDir for item in os.listdir(objectsDirectory): shutil.move(os.path.join(objectsDirectory, item), os.path.join(tmpSIPDir, "objects", item)) #get the database list of files in the objects directory #for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP' sql = """SELECT fileUUID, currentLocation FROM Files WHERE removedTime = 0 AND currentLocation LIKE '\%transferDirectory\%objects%' AND transferUUID = '""" + transfer_uuid + "'" for row in databaseInterface.queryAllSQL(sql): fileUUID = row[0] currentPath = databaseFunctions.deUnicode(row[1]) currentSIPFilePath = currentPath.replace("%transferDirectory%", tmpSIPDir) if os.path.isfile(currentSIPFilePath): sql = """UPDATE Files SET currentLocation='%s', sipUUID='%s' WHERE fileUUID='%s'""" % (MySQLdb.escape_string(currentPath.replace("%transferDirectory%", "%SIPDirectory%")), sipUUID, fileUUID) databaseInterface.runSQL(sql) else: print >>sys.stderr, "file not found: ", currentSIPFilePath #copy processingMCP.xml file src = os.path.join(os.path.dirname(objectsDirectory[:-1]), "processingMCP.xml") dst = os.path.join(tmpSIPDir, "processingMCP.xml") shutil.copy(src, dst) #moveSIPTo processSIPDirectory shutil.move(tmpSIPDir, destSIPDir) elasticSearchFunctions.connect_and_change_transfer_file_status(transfer_uuid, '') response['message'] = 'SIP ' + sipUUID + ' created.' else: response['error'] = True response['message'] = 'Must be logged in.' return HttpResponse( simplejson.JSONEncoder(encoding='utf-8').encode(response), mimetype='application/json' )
def process_transfer(request, transfer_uuid): response = {} if request.user.id: # get transfer info transfer = models.Transfer.objects.get(uuid=transfer_uuid) transfer_path = transfer.currentlocation.replace( '%sharedPath%', helpers.get_server_config_value('sharedDirectory')) import MySQLdb import databaseInterface import databaseFunctions import shutil from archivematicaCreateStructuredDirectory import createStructuredDirectory from archivematicaCreateStructuredDirectory import createManualNormalizedDirectoriesList createStructuredDirectory(transfer_path, createManualNormalizedDirectories=False) processingDirectory = helpers.get_server_config_value( 'processingDirectory') transfer_directory_name = os.path.basename(transfer_path[:-1]) transfer_name = transfer_directory_name[:-37] sharedPath = helpers.get_server_config_value('sharedDirectory') tmpSIPDir = os.path.join(processingDirectory, transfer_name) + "/" #processSIPDirectory = os.path.join(sharedPath, 'watchedDirectories/system/autoProcessSIP') + '/' processSIPDirectory = os.path.join( sharedPath, 'watchedDirectories/SIPCreation/SIPsUnderConstruction') + '/' #destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/" #destSIPDir = os.path.join(processSIPDirectory, transfer_name + '-' + ) + "/" createStructuredDirectory(tmpSIPDir, createManualNormalizedDirectories=False) objectsDirectory = os.path.join(transfer_path, 'objects') + '/' """ #create row in SIPs table if one doesn't already exist lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%') #lookup_path = '%sharedPath%watchedDirectories/workFlowDecisions/createDip/' + transfer_name + '/' sql = " " "SELECT sipUUID FROM SIPs WHERE currentPath = '" " " + MySQLdb.escape_string(lookup_path) + "';" rows = databaseInterface.queryAllSQL(sql) if len(rows) > 0: row = rows[0] sipUUID = row[0] else: sipUUID = uuid.uuid4().__str__() databaseFunctions.createSIP(lookup_path, sipUUID) """ sipUUID = uuid.uuid4().__str__() destSIPDir = os.path.join(processSIPDirectory, transfer_name) + "/" lookup_path = destSIPDir.replace(sharedPath, '%sharedPath%') databaseFunctions.createSIP(lookup_path, sipUUID) #move the objects to the SIPDir for item in os.listdir(objectsDirectory): shutil.move(os.path.join(objectsDirectory, item), os.path.join(tmpSIPDir, "objects", item)) #get the database list of files in the objects directory #for each file, confirm it's in the SIP objects directory, and update the current location/ owning SIP' sql = """SELECT fileUUID, currentLocation FROM Files WHERE removedTime = 0 AND currentLocation LIKE '\%transferDirectory\%objects%' AND transferUUID = '""" + transfer_uuid + "'" for row in databaseInterface.queryAllSQL(sql): fileUUID = row[0] currentPath = databaseFunctions.deUnicode(row[1]) currentSIPFilePath = currentPath.replace("%transferDirectory%", tmpSIPDir) if os.path.isfile(currentSIPFilePath): sql = """UPDATE Files SET currentLocation='%s', sipUUID='%s' WHERE fileUUID='%s'""" % ( MySQLdb.escape_string( currentPath.replace( "%transferDirectory%", "%SIPDirectory%")), sipUUID, fileUUID) databaseInterface.runSQL(sql) else: print >> sys.stderr, "file not found: ", currentSIPFilePath #copy processingMCP.xml file src = os.path.join(os.path.dirname(objectsDirectory[:-1]), "processingMCP.xml") dst = os.path.join(tmpSIPDir, "processingMCP.xml") shutil.copy(src, dst) #moveSIPTo processSIPDirectory shutil.move(tmpSIPDir, destSIPDir) elasticSearchFunctions.connect_and_change_transfer_file_status( transfer_uuid, '') response['message'] = 'SIP ' + sipUUID + ' created.' else: response['error'] = True response['message'] = 'Must be logged in.' return HttpResponse( simplejson.JSONEncoder(encoding='utf-8').encode(response), mimetype='application/json')
from fileOperations import renameAsSudo import elasticSearchFunctions def updateDB(dst, transferUUID): sql = """UPDATE Transfers SET currentLocation='""" + MySQLdb.escape_string(dst) + """' WHERE transferUUID='""" + transferUUID + """';""" databaseInterface.runSQL(sql) def moveSIP(src, dst, transferUUID, sharedDirectoryPath): # os.rename(src, dst) if src.endswith("/"): src = src[:-1] dest = dst.replace(sharedDirectoryPath, "%sharedPath%", 1) if dest.endswith("/"): dest = os.path.join(dest, os.path.basename(src)) if dest.endswith("/."): dest = os.path.join(dest[:-1], os.path.basename(src)) updateDB(dest + "/", transferUUID) renameAsSudo(src, dst) if __name__ == '__main__': #"%SIPUUID%" "%SIPName%" "%SIPDirectory%" transferUUID = sys.argv[1] transferName = sys.argv[2] transferDirectory = sys.argv[3] print 'Processing ' + transferUUID + '...' found = elasticSearchFunctions.connect_and_change_transfer_file_status(transferUUID, 'backlog') print 'Updated ' + str(found) + ' transfer file entries.' #moveSIP(src, dst, transferUUID, sharedDirectoryPath)