def remove_file(target_file, file_uuid):
basename = os.path.basename(target_file)
if basename in REMOVEABLE_FILES:
print "Removing {filename} (UUID: {uuid})".format(uuid=file_uuid, filename=basename)
os.remove(target_file)
# Gearman passes parameters as strings, so None (NoneType) becomes
# "None" (string)
if file_uuid and file_uuid != "None":
fileWasRemoved(file_uuid)
def onceExtracted(command):
extractedFiles = []
print "TODO - Metadata regarding removal of extracted archive"
if removeOnceExtracted:
packageFileUUID = sys.argv[6].__str__()
sipDirectory = sys.argv[2].__str__()
os.remove(replacementDic["%inputFile%"])
currentLocation = replacementDic["%inputFile%"].replace(sipDirectory, "%transferDirectory%", 1)
fileWasRemoved(packageFileUUID, eventOutcomeDetailNote = "removed from: " + currentLocation)
print "OUTPUT DIRECTORY: ", replacementDic["%outputDirectory%"]
for w in os.walk(replacementDic["%outputDirectory%"].replace("*", "asterisk*")):
path, directories, files = w
for p in files:
p = os.path.join(path, p)
#print "path: ", p
if os.path.isfile(p):
extractedFiles.append(p)
for ef in extractedFiles:
fileUUID = uuid.uuid4().__str__()
#print "File Extracted:", ef
if True: #Add the file to the SIP
#<arguments>"%relativeLocation%" "%SIPObjectsDirectory%" "%SIPLogsDirectory%" "%date%" "%taskUUID%" "%fileUUID%"</arguments>
sipDirectory = sys.argv[2].__str__()
transferUUID = sys.argv[3].__str__()
date = sys.argv[4].__str__()
taskUUID = sys.argv[5].__str__()
packageFileUUID = sys.argv[6].__str__()
filePathRelativeToSIP = ef.replace(sipDirectory,"%transferDirectory%", 1)
print "File Extracted:: {" + fileUUID + "} ", filePathRelativeToSIP
eventDetail="Unpacked from: {" + packageFileUUID + "}" + filePathRelativeToSIP
addFileToTransfer(filePathRelativeToSIP, fileUUID, transferUUID, taskUUID, date, sourceType="unpacking", eventDetail=eventDetail)
updateSizeAndChecksum(fileUUID, ef, date, uuid.uuid4.__str__())
run = sys.argv[0].__str__() + \
" \"" + transcoder.escapeForCommand(ef) + "\""
if True: #Add the file to the SIP
run = run + " \"" + transcoder.escapeForCommand(sys.argv[2].__str__()) + "\"" + \
" \"" + transcoder.escapeForCommand(sys.argv[3].__str__()) + "\"" + \
" \"" + transcoder.escapeForCommand(sys.argv[4].__str__()) + "\"" + \
" \"" + transcoder.escapeForCommand(sys.argv[5].__str__()) + "\"" + \
" \"" + fileUUID + "\""
exitCode, stdOut, stdError = executeOrRun("command", run)
print stdOut
print >>sys.stderr, stdError
if exitCode != 0 and command.exitCode == 0:
command.exitCode = exitCode
global extractedCount
date = sys.argv[4].__str__().split(".", 1)[0]
extractedCount = extractedCount + 1
replacementDic["%outputDirectory%"] = transcoder.fileFullName + '-' + extractedCount.__str__() + '-' + date
def onceExtracted(command):
extractedFiles = []
print "TODO - Metadata regarding removal of extracted archive"
if removeOnceExtracted:
packageFileUUID = sys.argv[6].__str__()
sipDirectory = sys.argv[2].__str__()
os.remove(replacementDic["%inputFile%"])
currentLocation = replacementDic["%inputFile%"].replace(sipDirectory, "%transferDirectory%", 1)
fileWasRemoved(packageFileUUID, eventOutcomeDetailNote = "removed from: " + currentLocation)
print "OUTPUT DIRECTORY: ", replacementDic["%outputDirectory%"]
for w in os.walk(replacementDic["%outputDirectory%"].replace("*", "asterisk*")):
path, directories, files = w
for p in files:
p = os.path.join(path, p)
#print "path: ", p
if os.path.isfile(p):
extractedFiles.append(p)
for ef in extractedFiles:
fileUUID = uuid.uuid4().__str__()
#print "File Extracted:", ef
if True: #Add the file to the SIP
#<arguments>"%relativeLocation%" "%SIPObjectsDirectory%" "%SIPLogsDirectory%" "%date%" "%taskUUID%" "%fileUUID%"</arguments>
sipDirectory = sys.argv[2].__str__()
transferUUID = sys.argv[3].__str__()
date = sys.argv[4].__str__()
taskUUID = sys.argv[5].__str__()
packageFileUUID = sys.argv[6].__str__()
filePathRelativeToSIP = ef.replace(sipDirectory,"%transferDirectory%", 1)
print "File Extracted:: {" + fileUUID + "} ", filePathRelativeToSIP
eventDetail="Unpacked from: {" + packageFileUUID + "}" + filePathRelativeToSIP
addFileToTransfer(filePathRelativeToSIP, fileUUID, transferUUID, taskUUID, date, sourceType="unpacking", eventDetail=eventDetail)
updateSizeAndChecksum(fileUUID, ef, date, uuid.uuid4.__str__())
run = sys.argv[0].__str__() + \
" \"" + transcoder.escapeForCommand(ef) + "\""
if True: #Add the file to the SIP
run = run + " \"" + transcoder.escapeForCommand(sys.argv[2].__str__()) + "\"" + \
" \"" + transcoder.escapeForCommand(sys.argv[3].__str__()) + "\"" + \
" \"" + transcoder.escapeForCommand(sys.argv[4].__str__()) + "\"" + \
" \"" + transcoder.escapeForCommand(sys.argv[5].__str__()) + "\"" + \
" \"" + fileUUID + "\""
exitCode, stdOut, stdError = executeOrRun("command", run)
print stdOut
print >>sys.stderr, stdError
if exitCode != 0 and command.exitCode == 0:
command.exitCode = exitCode
global extractedCount
date = sys.argv[4].__str__().split(".", 1)[0]
extractedCount = extractedCount + 1
replacementDic["%outputDirectory%"] = transcoder.fileFullName + '-' + extractedCount.__str__() + '-' + date
def timerExpired(event, utcDate):
global movedFromCount
movedFromLock.acquire()
if event.cookie in movedFrom:
#remove it from the list of unfound moves
movedFromPath, filesMoved, timer = movedFrom.pop(event.cookie)
movedFromCount.value = movedFromCount.value - 1
movedFromLock.release()
for fileUUID, oldLocation in filesMoved:
fileWasRemoved(fileUUID, utcDate = utcDate, eventOutcomeDetailNote = "removed from: " + oldLocation)
else:
movedFromLock.release()
def remove_file(target_file, file_uuid):
removableFiles = [
e.strip() for e in mcpclient_settings.REMOVABLE_FILES.split(',')
]
basename = os.path.basename(target_file)
if basename in removableFiles:
print("Removing {filename} (UUID: {uuid})".format(uuid=file_uuid,
filename=basename))
try:
os.remove(target_file)
except OSError:
shutil.rmtree(target_file)
# Gearman passes parameters as strings, so None (NoneType) becomes
# "None" (string)
if file_uuid and file_uuid != "None":
fileWasRemoved(file_uuid)
def process_IN_DELETE(self, event):
#print event
#print "Transfer IN_DELETE"
#Wait for a moved to, and if one doesn't occur, consider it moved outside of the system.
movedFromPath = os.path.join(event.path, event.name).replace(\
self.unit.currentPath.replace("%sharedPath%", archivematicaMCP.config.get('MCPServer', "sharedDirectory"), 1), \
"%transferDirectory%", 1)
filesMoved = []
sql = """SELECT fileUUID, currentLocation FROM Files WHERE transferUUID = '""" + self.unit.UUID + "' AND removedTime = 0 AND currentLocation LIKE '" + MySQLdb.escape_string(movedFromPath).replace("%", "\%") + "%';"
c, sqlLock = databaseInterface.querySQL(sql)
row = c.fetchone()
while row != None:
filesMoved.append(row)
row = c.fetchone()
sqlLock.release()
for fileUUID, currentLocation in filesMoved:
fileWasRemoved(fileUUID, eventOutcomeDetailNote = "removed from: " + currentLocation)
def call(jobs):
with transaction.atomic():
for job in jobs:
with job.JobContext():
SIPDirectory = job.args[1]
manual_normalization_dir = os.path.join(
SIPDirectory, "objects", "manualNormalization")
errorCount = 0
if os.path.isdir(manual_normalization_dir):
# Delete normalization.csv if present
normalization_csv = os.path.join(manual_normalization_dir,
"normalization.csv")
if os.path.isfile(normalization_csv):
os.remove(normalization_csv)
# Need SIP UUID to get file UUID to remove file in DB
sipUUID = SIPDirectory[-37:
-1] # Account for trailing /
f = File.objects.get(
removedtime__isnull=True,
originallocation__endswith="normalization.csv",
sip_id=sipUUID,
)
databaseFunctions.fileWasRemoved(f.uuid)
# Recursively delete empty manual normalization dir
try:
errorCount += recursivelyRemoveEmptyDirectories(
job, manual_normalization_dir)
os.rmdir(manual_normalization_dir)
except OSError as e:
job.pyprint(
"{0} could not be deleted: {1}".format(
manual_normalization_dir, e.args),
file=sys.stderr,
)
errorCount += 1
job.set_status(errorCount)
def remove_file(target_file, file_uuid):
clientConfigFilePath = '/etc/archivematica/MCPClient/clientConfig.conf'
config = ConfigParser.SafeConfigParser()
config.read(clientConfigFilePath)
try:
removableFiles = [
e.strip()
for e in config.get('MCPClient', 'removableFiles').split(',')
]
except ConfigParser.NoOptionError:
removableFiles = ["Thumbs.db", "Icon", u"Icon\u000D", ".DS_Store"]
basename = os.path.basename(target_file)
if basename in removableFiles:
print("Removing {filename} (UUID: {uuid})".format(uuid=file_uuid,
filename=basename))
try:
os.remove(target_file)
except OSError:
shutil.rmtree(target_file)
# Gearman passes parameters as strings, so None (NoneType) becomes
# "None" (string)
if file_uuid and file_uuid != "None":
fileWasRemoved(file_uuid)
def delete_and_record_package_file(job, file_path, file_uuid, current_location):
os.remove(file_path)
job.pyprint("Package removed: " + file_path)
event_detail_note = "removed from: " + current_location
fileWasRemoved(file_uuid, eventDetail=event_detail_note)
import os
sys.path.append("/usr/lib/archivematica/archivematicaCommon")
from databaseFunctions import fileWasRemoved
removeIfFileNameIs = ["Thumbs.db", "Icon", u"Icon\u000D"]
def removableFile(target):
global eventDetailText
basename = os.path.basename(target)
if basename in removeIfFileNameIs:
eventDetailText = basename + " is noted as a removable file."
return True
return False
if __name__ == '__main__':
target = sys.argv[1]
fileUUID = sys.argv[2]
logsDirectory = sys.argv[3]
date = sys.argv[4]
eIDValue = sys.argv[5]
global eventDetailText
eventDetailText = "fileRemoved"
if removableFile(target):
print fileUUID + " -> " + os.path.basename(target)
os.remove(target)
fileWasRemoved(fileUUID)
if os.path.isdir(manual_normalization_dir):
# Delete normalization.csv if present
normalization_csv = os.path.join(manual_normalization_dir,
'normalization.csv')
if os.path.isfile(normalization_csv):
os.remove(normalization_csv)
# Need SIP UUID to get file UUID to remove file in DB
sipUUID = SIPDirectory[-37:-1] # Account for trailing /
sql = """SELECT fileUUID
FROM Files
WHERE removedTime = 0 AND
Files.originalLocation LIKE '%normalization.csv' AND
SIPUUID='{sipUUID}';""".format(sipUUID=sipUUID)
rows = databaseInterface.queryAllSQL(sql)
fileUUID = rows[0][0]
databaseFunctions.fileWasRemoved(fileUUID)
# Recursively delete empty manual normalization dir
try:
errorCount += recursivelyRemoveEmptyDirectories(
manual_normalization_dir)
os.rmdir(manual_normalization_dir)
except OSError as e:
print >> sys.stderr, "{0} could not be deleted: {1}".format(
directory, e.args)
errorCount += 1
exit(errorCount)
error_count += 1
return error_count
if os.path.isdir(manual_normalization_dir):
# Delete normalization.csv if present
normalization_csv = os.path.join(manual_normalization_dir,
'normalization.csv')
if os.path.isfile(normalization_csv):
os.remove(normalization_csv)
# Need SIP UUID to get file UUID to remove file in DB
sipUUID = SIPDirectory[-37:-1] # Account for trailing /
f = File.objects.get(removedtime__isnull=True,
originallocation__endswith='normalization.csv',
sip_id=sipUUID)
databaseFunctions.fileWasRemoved(f.uuid)
# Recursively delete empty manual normalization dir
try:
errorCount += recursivelyRemoveEmptyDirectories(
manual_normalization_dir)
os.rmdir(manual_normalization_dir)
except OSError as e:
print("{0} could not be deleted: {1}".format(manual_normalization_dir,
e.args),
file=sys.stderr)
errorCount += 1
exit(errorCount)
# @subpackage archivematicaClientScript
# @author Joseph Perry <*****@*****.**>
import sys
import os
sys.path.append("/usr/lib/archivematica/archivematicaCommon")
from databaseFunctions import fileWasRemoved
removeIfFileNameIs = ["Thumbs.db", "Icon", u"Icon\u000D"]
def removableFile(target):
global eventDetailText
basename = os.path.basename(target)
if basename in removeIfFileNameIs:
eventDetailText = basename + " is noted as a removable file."
return True
return False
if __name__ == '__main__':
target = sys.argv[1]
fileUUID = sys.argv[2]
logsDirectory = sys.argv[3]
date = sys.argv[4]
eIDValue = sys.argv[5]
global eventDetailText
eventDetailText = "fileRemoved"
if removableFile(target):
print fileUUID + " -> " + os.path.basename(target)
os.remove(target)
fileWasRemoved(fileUUID)
return error_count
if os.path.isdir(manual_normalization_dir):
# Delete normalization.csv if present
normalization_csv = os.path.join(manual_normalization_dir, "normalization.csv")
if os.path.isfile(normalization_csv):
os.remove(normalization_csv)
# Need SIP UUID to get file UUID to remove file in DB
sipUUID = SIPDirectory[-37:-1] # Account for trailing /
sql = """SELECT fileUUID
FROM Files
WHERE removedTime = 0 AND
Files.originalLocation LIKE '%normalization.csv' AND
SIPUUID='{sipUUID}';""".format(
sipUUID=sipUUID
)
rows = databaseInterface.queryAllSQL(sql)
fileUUID = rows[0][0]
databaseFunctions.fileWasRemoved(fileUUID)
# Recursively delete empty manual normalization dir
try:
errorCount += recursivelyRemoveEmptyDirectories(manual_normalization_dir)
os.rmdir(manual_normalization_dir)
except OSError as e:
print >>sys.stderr, "{0} could not be deleted: {1}".format(directory, e.args)
errorCount += 1
exit(errorCount)
