def _create_file(
transfer_id, current_location, relative_transfer_path, backlog, backlog_path, size
):
"""Store the transfer in the backlog location.
Storage Service will copy the contents of the transfers.
"""
try:
new_file = storage_service.create_file(
uuid=transfer_id,
origin_location=current_location["resource_uri"],
origin_path=relative_transfer_path,
current_location=backlog["resource_uri"],
current_path=backlog_path,
package_type="transfer", # TODO use constant from storage service
size=size,
)
except storage_service.Error as err:
raise StorageServiceCreateFileError(err)
if new_file is None:
raise StorageServiceCreateFileError(
"Value returned by Storage Service is unexpected"
)
if new_file.get("status") == "FAIL":
raise StorageServiceCreateFileError(
'Object returned by Storage Service has status "FAIL"'
)
return new_file
def _create_file(uuid, current_location, relative_aip_path,
aip_destination_uri, current_path, package_type, aip_subtype,
size, sip_type, related_package_uuid):
try:
new_file = storage_service.create_file(
uuid=uuid,
origin_location=current_location['resource_uri'],
origin_path=relative_aip_path,
current_location=aip_destination_uri,
current_path=current_path,
package_type=package_type,
aip_subtype=aip_subtype,
size=size,
update='REIN' in sip_type,
related_package_uuid=related_package_uuid,
events=get_events_from_db(uuid),
agents=get_agents_from_db(uuid)
)
except storage_service.Error as err:
raise StorageServiceCreateFileError(err)
if new_file is None:
raise StorageServiceCreateFileError(
'Value returned by Storage Service is unexpected')
if new_file.get('status') == 'FAIL':
raise StorageServiceCreateFileError(
'Object returned by Storage Service has status "FAIL"')
return new_file
def main(transfer_uuid, transfer_path):
current_location = storage_service.get_location(purpose="CP")[0]
backlog = storage_service.get_location(purpose="BL")[0]
# Get size recursively for Transfer
size = 0
for dirpath, _, filenames in os.walk(transfer_path):
for filename in filenames:
file_path = os.path.join(dirpath, filename)
size += os.path.getsize(file_path)
# Make Transfer path relative to Location
shared_path = os.path.join(current_location['path'], '')
relative_transfer_path = transfer_path.replace(shared_path, '')
# TODO this should use the same value as
# dashboard/src/components/filesystem_ajax/views.py DEFAULT_BACKLOG_PATH
transfer_name = os.path.basename(transfer_path.rstrip('/'))
backlog_path = os.path.join('originals', transfer_name)
(new_file, error_msg) = storage_service.create_file(
uuid=transfer_uuid,
origin_location=current_location['resource_uri'],
origin_path=relative_transfer_path,
current_location=backlog['resource_uri'],
current_path=backlog_path,
package_type='transfer', # TODO use constant from storage service
size=size,
)
if new_file is not None and new_file.get('status', '') != "FAIL":
message = "Transfer moved to backlog: {}".format(new_file)
logging.info(message)
print(message)
# TODO update transfer location? Files location?
# Delete transfer from processing space
shutil.rmtree(transfer_path)
return 0
else:
print(
"Moving to backlog failed. See Storage Service logs for more details",
file=sys.stderr)
print(error_msg or "Package status: Failed", file=sys.stderr)
logging.warning(
"Moving to backlog failed: {}. See logs for more details.".format(
error_msg))
return 1
def _create_file(transfer_uuid, current_location, relative_transfer_path,
backlog, backlog_path, size):
new_file = storage_service.create_file(
uuid=transfer_uuid,
origin_location=current_location['resource_uri'],
origin_path=relative_transfer_path,
current_location=backlog['resource_uri'],
current_path=backlog_path,
package_type='transfer', # TODO use constant from storage service
size=size,
)
if new_file is None:
raise StorageServiceCreateFileError(
'Value returned by Storage Service is unexpected')
if new_file.get('status') == 'FAIL':
raise StorageServiceCreateFileError(
'Object returned by Storage Service has status "FAIL"')
return new_file
def store_aip(aip_destination_uri, aip_path, sip_uuid, sip_name, sip_type):
""" Stores an AIP with the storage service.
aip_destination_uri = storage service destination URI, should be of purpose
AIP Store (AS)
aip_path = Full absolute path to the AIP's current location on the local
filesystem
sip_uuid = UUID of the SIP, which will become the UUID of the AIP
sip_name = SIP name. Not used directly, but part of the AIP name
Example inputs:
storeAIP.py
"/api/v1/location/9c2b5bb7-abd6-477b-88e0-57107219dace/"
"/var/archivematica/sharedDirectory/currentlyProcessing/ep6-0737708e-9b99-471a-b331-283e2244164f/ep6-0737708e-9b99-471a-b331-283e2244164f.7z"
"0737708e-9b99-471a-b331-283e2244164f"
"ep6"
"""
# FIXME Assume current Location is the one set up by default until location
# is passed in properly, or use Agent to make sure is correct CP
current_location = storage_service.get_location(purpose="CP")[0]
# If ``aip_path`` does not exist, this may be a DIP that was not uploaded.
# In that case, it will be in the uploadDIP/ directory instead of the
# uploadedDIPs/ directory.
if not os.path.exists(aip_path):
aip_path = get_upload_dip_path(aip_path)
# Make aip_path relative to the Location
shared_path = os.path.join(current_location['path'],
'') # Ensure ends with /
relative_aip_path = aip_path.replace(shared_path, '')
# Get the package type: AIC or AIP
if 'SIP' in sip_type or 'AIP' in sip_type: # Also matches AIP-REIN
package_type = "AIP"
elif 'AIC' in sip_type: # Also matches AIC-REIN
package_type = 'AIC'
elif 'DIP' in sip_type:
package_type = 'DIP'
# Uncompressed directory AIPs must be terminated in a /,
# otherwise the storage service will place the directory
# inside another directory of the same name.
current_path = os.path.basename(aip_path)
if os.path.isdir(aip_path) and not aip_path.endswith('/'):
relative_aip_path = relative_aip_path + '/'
# DIPs cannot share the AIP UUID, as the storage service depends on
# having a unique UUID; assign a new one before uploading.
# TODO allow mapping the AIP UUID to the DIP UUID for retrieval.
related_package_uuid = None
if sip_type == 'DIP':
uuid = str(uuid4())
print('Checking if DIP {} parent AIP has been created...'.format(uuid))
# Set related package UUID, so a relationship to the parent AIP can be
# created if if AIP has been stored. If the AIP hasn't yet been stored
# take note of the DIP's UUID so it the relationship can later be
# created when the AIP is stored.
try:
storage_service.get_file_info(uuid=sip_uuid)[0] # Check existence
related_package_uuid = sip_uuid
print('Parent AIP exists so relationship can be created.')
except IndexError:
UnitVariable.objects.create(unittype='SIP',
unituuid=sip_uuid,
variable='relatedPackage',
variablevalue=uuid)
print(
'Noting DIP UUID {} related to AIP so relationship can be created when AIP is stored.'
.format(uuid))
else:
uuid = sip_uuid
related_package = get_object_or_None(UnitVariable,
unituuid=sip_uuid,
variable='relatedPackage')
related_package_uuid = related_package.variablevalue if related_package is not None else None
# If AIP is a directory, calculate size recursively
if os.path.isdir(aip_path):
size = 0
for dirpath, _, filenames in os.walk(aip_path):
for filename in filenames:
file_path = os.path.join(dirpath, filename)
size += os.path.getsize(file_path)
else:
size = os.path.getsize(aip_path)
# Get the AIP subtype from any DC type attribute supplied by the user for
# the AIP. If found, this will replace 'Archival Information Package' in
# ``<mets:div TYPE='Archival Information Package'>`` in the pointer file.
sip_metadata_uuid = '3e48343d-e2d2-4956-aaa3-b54d26eb9761'
try:
dc = DublinCore.objects.get(metadataappliestotype_id=sip_metadata_uuid,
metadataappliestoidentifier=uuid)
except DublinCore.DoesNotExist:
aip_subtype = 'Archival Information Package'
else:
aip_subtype = dc.type
# Store the AIP
(new_file, error_msg) = storage_service.create_file(
uuid=uuid,
origin_location=current_location['resource_uri'],
origin_path=relative_aip_path,
current_location=aip_destination_uri,
current_path=current_path,
package_type=package_type,
aip_subtype=aip_subtype,
size=size,
update='REIN' in sip_type,
related_package_uuid=related_package_uuid,
events=get_events_from_db(uuid),
agents=get_agents_from_db(uuid))
if new_file is not None and new_file.get('status', '') != "FAIL":
message = "Storage service created {}: {}".format(sip_type, new_file)
LOGGER.info(message)
print(message)
sys.exit(0)
else:
print("{} creation failed. See Storage Service logs for more details".
format(sip_type),
file=sys.stderr)
print(error_msg or "Package status: Failed", file=sys.stderr)
LOGGER.warning(
"{} unabled to be created: {}. See logs for more details.".format(
sip_type, error_msg))
sys.exit(1)
def store_aip(aip_destination_uri, aip_path, sip_uuid, sip_name, sip_type):
""" Stores an AIP with the storage service.
aip_destination_uri = storage service destination URI, should be of purpose
AIP Store (AS)
aip_path = Full absolute path to the AIP's current location on the local
filesystem
sip_uuid = UUID of the SIP, which will become the UUID of the AIP
sip_name = SIP name. Not used directly, but part of the AIP name
Example inputs:
storeAIP.py
"/api/v1/location/9c2b5bb7-abd6-477b-88e0-57107219dace/"
"/var/archivematica/sharedDirectory/currentlyProcessing/ep6-0737708e-9b99-471a-b331-283e2244164f/ep6-0737708e-9b99-471a-b331-283e2244164f.7z"
"0737708e-9b99-471a-b331-283e2244164f"
"ep6"
"""
# FIXME Assume current Location is the one set up by default until location
# is passed in properly, or use Agent to make sure is correct CP
current_location = storage_service.get_location(purpose="CP")[0]
# Make aip_path relative to the Location
shared_path = os.path.join(current_location['path'], '') # Ensure ends with /
relative_aip_path = aip_path.replace(shared_path, '')
# Get the package type: AIC or AIP
if 'SIP' in sip_type or 'AIP' in sip_type: # Also matches AIP-REIN
package_type = "AIP"
elif 'AIC' in sip_type: # Also matches AIC-REIN
package_type = 'AIC'
elif 'DIP' in sip_type:
package_type = 'DIP'
# Uncompressed directory AIPs must be terminated in a /,
# otherwise the storage service will place the directory
# inside another directory of the same name.
current_path = os.path.basename(aip_path)
if os.path.isdir(aip_path) and not aip_path.endswith('/'):
relative_aip_path = relative_aip_path + '/'
# DIPs cannot share the AIP UUID, as the storage service depends on
# having a unique UUID; assign a new one before uploading.
# TODO allow mapping the AIP UUID to the DIP UUID for retrieval.
related_package_uuid = None
if sip_type == 'DIP':
uuid = str(uuid4())
print('Checking if DIP {} parent AIP has been created...'.format(uuid))
# Set related package UUID, so a relationship to the parent AIP can be
# created if if AIP has been stored. If the AIP hasn't yet been stored
# take note of the DIP's UUID so it the relationship can later be
# created when the AIP is stored.
api = storage_service._storage_api()
try:
api.file(sip_uuid).get()
related_package_uuid = sip_uuid
print('Parent AIP exists so relationship can be created.')
except slumber.exceptions.HttpClientError:
UnitVariable.objects.create(unittype='SIP', unituuid=sip_uuid, variable='relatedPackage', variablevalue=uuid)
print('Noting DIP UUID {} related to AIP so relationship can be created when AIP is stored.'.format(uuid))
else:
uuid = sip_uuid
related_package = get_object_or_None(UnitVariable, unituuid=sip_uuid, variable='relatedPackage')
related_package_uuid = related_package.variablevalue if related_package is not None else None
# If AIP is a directory, calculate size recursively
if os.path.isdir(aip_path):
size = 0
for dirpath, _, filenames in os.walk(aip_path):
for filename in filenames:
file_path = os.path.join(dirpath, filename)
size += os.path.getsize(file_path)
else:
size = os.path.getsize(aip_path)
#Store the AIP
(new_file, error_msg) = storage_service.create_file(
uuid=uuid,
origin_location=current_location['resource_uri'],
origin_path=relative_aip_path,
current_location=aip_destination_uri,
current_path=current_path,
package_type=package_type,
size=size,
update='REIN' in sip_type,
related_package_uuid=related_package_uuid,
)
if new_file is not None and new_file.get('status', '') != "FAIL":
message = "Storage service created {}: {}".format(sip_type, new_file)
logging.info(message)
print(message)
sys.exit(0)
else:
print("{} creation failed. See Storage Service logs for more details".format(sip_type), file=sys.stderr)
print(error_msg or "Package status: Failed", file=sys.stderr)
logging.warning("{} unabled to be created: {}. See logs for more details.".format(sip_type, error_msg))
sys.exit(1)
def store_aip(aip_destination_uri, aip_path, sip_uuid, sip_name, sip_type):
""" Stores an AIP with the storage service.
aip_destination_uri = storage service destination URI, should be of purpose
AIP Store (AS)
aip_path = Full absolute path to the AIP's current location on the local
filesystem
sip_uuid = UUID of the SIP, which will become the UUID of the AIP
sip_name = SIP name. Not used directly, but part of the AIP name
Example inputs:
storeAIP.py
"/api/v1/location/9c2b5bb7-abd6-477b-88e0-57107219dace/"
"/var/archivematica/sharedDirectory/currentlyProcessing/ep6-0737708e-9b99-471a-b331-283e2244164f/ep6-0737708e-9b99-471a-b331-283e2244164f.7z"
"0737708e-9b99-471a-b331-283e2244164f"
"ep6"
"""
# FIXME Assume current Location is the one set up by default until location
# is passed in properly, or use Agent to make sure is correct CP
current_location = storage_service.get_location(purpose="CP")[0]
# Make aip_path relative to the Location
shared_path = os.path.join(current_location['path'], '') # Ensure ends with /
relative_aip_path = aip_path.replace(shared_path, '')
# Get the package type: AIC or AIP
if sip_type == "SIP":
package_type = "AIP"
elif sip_type == 'AIC' or sip_type == 'DIP':
package_type = sip_type
# Uncompressed directory AIPs must be terminated in a /,
# otherwise the storage service will place the directory
# inside another directory of the same name.
current_path = os.path.basename(aip_path)
if os.path.isdir(aip_path) and not aip_path.endswith('/'):
relative_aip_path = relative_aip_path + '/'
# DIPs cannot share the AIP UUID, as the storage service depends on
# having a unique UUID; assign a new one before uploading.
# TODO allow mapping the AIP UUID to the DIP UUID for retrieval.
if sip_type == 'DIP':
uuid = str(uuid4())
else:
uuid = sip_uuid
# If AIP is a directory, calculate size recursively
if os.path.isdir(aip_path):
size = 0
for dirpath, _, filenames in os.walk(aip_path):
for filename in filenames:
file_path = os.path.join(dirpath, filename)
size += os.path.getsize(file_path)
else:
size = os.path.getsize(aip_path)
#Store the AIP
(new_file, error_msg) = storage_service.create_file(
uuid=uuid,
origin_location=current_location['resource_uri'],
origin_path=relative_aip_path,
current_location=aip_destination_uri,
current_path=current_path,
package_type=package_type,
size=size
)
if new_file is not None and new_file.get('status', '') != "FAIL":
message = "Storage service created {}: {}".format(sip_type, new_file)
logging.info(message)
print message
sys.exit(0)
else:
print >>sys.stderr, "{} creation failed. See Storage Service logs for more details".format(sip_type)
print >>sys.stderr, error_msg or "Package status: Failed"
logging.warning("{} unabled to be created: {}. See logs for more details.".format(sip_type, error_msg))
sys.exit(1)
