def move_from_storage_service(self,
source_path,
destination_path,
package=None):
""" Moves self.staging_path/src_path to dest_path. """
if os.path.isdir(source_path):
# Both source and destination paths should end with /
destination_path = os.path.join(destination_path, '')
# Swift does not accept folders, so upload each file individually
for path, dirs, files in os.walk(source_path):
for basename in files:
entry = os.path.join(path, basename)
dest = entry.replace(source_path, destination_path, 1)
checksum = utils.generate_checksum(entry)
with open(entry, 'rb') as f:
self.connection.put_object(
self.container,
obj=dest,
contents=f,
etag=checksum.hexdigest(),
content_length=os.path.getsize(entry))
elif os.path.isfile(source_path):
checksum = utils.generate_checksum(source_path)
with open(source_path, 'rb') as f:
self.connection.put_object(
self.container,
obj=destination_path,
contents=f,
etag=checksum.hexdigest(),
content_length=os.path.getsize(source_path),
)
else:
raise StorageException(
_('%(path)s is neither a file nor a directory, may not exist')
% {'path': source_path})
def post_move_from_storage_service(self, staging_path, destination_path,
package):
""" POST to Arkivum with information about the newly stored Package. """
if package is None:
return
relative_path = os.path.relpath(destination_path, self.space.path)
if package.is_compressed: # Single-file package
url = 'https://' + self.host + '/api/2/files/release/' + relative_path
headers = {'Content-Type': 'application/json'}
# Get size, checksum, checksum algorithm (md5sum), compression algorithm
checksum = utils.generate_checksum(staging_path, 'md5')
payload = {
'size': str(os.path.getsize(staging_path)),
'checksum': checksum.hexdigest(),
'checksumAlgorithm': 'md5',
'compressionAlgorithm':
os.path.splitext(package.current_path)[1],
}
payload = json.dumps(payload)
files = None
else: # uncompressed bag
url = 'https://' + self.host + '/api/3/ingest-manifest/release'
headers = None
# FIXME Destination path has to exclude mount path, but what is part of the mounth path? Let's pretend it's the Space path
payload = {'bagitPath': os.path.join('/', relative_path)}
files = {'': ('', '')}
LOGGER.debug('POST URL: %s; Header: %s; Payload: %s; Files: %s', url,
headers, payload, files)
try:
response = requests.post(url,
headers=headers,
data=payload,
files=files,
verify=VERIFY)
except requests.exceptions.ConnectionError:
LOGGER.exception('Error in connection for POST to %s', url)
raise StorageException('Error in connection for POST to %s', url)
LOGGER.debug('Response: %s, Response text: %s', response.status_code,
response.text)
if response.status_code not in (requests.codes.ok,
requests.codes.accepted):
LOGGER.warning('Arkivum responded with %s: %s',
response.status_code, response.text)
raise StorageException('Unable to notify Arkivum of %s', package)
# Response has request ID for polling status
try:
response_json = response.json()
except json.JSONDecodeError:
raise StorageException(
"Could not get request ID from Arkivum's response %s",
response.text)
# Store request ID in misc_attributes
request_id = response_json['id']
package.misc_attributes.update({'arkivum_identifier': request_id})
package.save()
def _download_file(self, remote_path, download_path):
"""
Download the file from download_path in this Space to remote_path.
:param str remote_path: Full path in Swift
:param str download_path: Full path to save the file to
:raises: swiftclient.exceptions.ClientException may be raised and is not caught
"""
# TODO find a way to stream content to dest_path, instead of having to put it in memory
headers, content = self.connection.get_object(self.container,
remote_path)
self.space.create_local_directory(download_path)
with open(download_path, 'wb') as f:
f.write(content)
# Check ETag matches checksum of this file
if 'etag' in headers:
checksum = utils.generate_checksum(download_path)
if checksum.hexdigest() != headers['etag']:
message = _(
'ETag %(remote_path)s for %(etag)s does not match %(checksum)s'
) % {
'remote_path': remote_path,
'etag': headers['etag'],
'checksum': checksum.hexdigest()
}
logging.warning(message)
raise StorageException(message)
def post_move_from_storage_service(self, staging_path, destination_path,
package):
""" POST to Arkivum with information about the newly stored Package. """
if package is None:
return
# Get size, checksum, checksum algorithm (md5sum), compression algorithm
checksum = utils.generate_checksum(staging_path, 'md5')
payload = {
'size': str(os.path.getsize(staging_path)),
'checksum': checksum.hexdigest(),
'checksumAlgorithm': 'md5',
'compressionAlgorithm': os.path.splitext(package.current_path)[1],
}
payload = json.dumps(payload)
# POST to Arkivum host/api/2/files/release/relative_path
relative_path = os.path.relpath(destination_path, self.space.path)
url = 'https://' + self.host + '/api/2/files/release/' + relative_path
LOGGER.info('URL: %s, Payload: %s', url, payload)
try:
response = requests.post(
url,
headers={'Content-Type': 'application/json'},
data=payload,
verify=VERIFY)
except requests.exceptions.ConnectionError:
LOGGER.exception('Error in connection for POST to %s', url)
raise StorageException('Error in connection for POST to %s', url)
LOGGER.info('Response: %s, Response text: %s', response.status_code,
response.text)
if response.status_code not in (requests.codes.ok,
requests.codes.accepted):
LOGGER.warning('Arkivum responded with %s: %s',
response.status_code, response.text)
raise StorageException('Unable to notify Arkivum of %s', package)
# Response has request ID for polling status
try:
response_json = response.json()
except json.JSONDecodeError:
raise StorageException(
"Could not get request ID from Arkivum's response %s",
response.text)
request_id = response_json['id']
# Store request ID in misc_attributes
package.misc_attributes.update({'request_id': request_id})
package.save()
def _fetch_content(deposit_uuid, objects, subdirs=None):
"""
Download a number of files, keeping track of progress and success using a
database record. After downloading, finalize deposit if requested.
If subdirs is provided, the file will be moved into a subdirectory of the
new transfer; otherwise, it will be placed in the transfer's root.
"""
# add download task to keep track of progress
deposit = get_deposit(deposit_uuid)
task = models.PackageDownloadTask(package=deposit)
task.downloads_attempted = len(objects)
task.downloads_completed = 0
task.save()
# Get deposit protocol info
deposit_space = deposit.current_location.space.get_child_space()
fedora_username = getattr(deposit_space, 'fedora_user', None)
fedora_password = getattr(deposit_space, 'fedora_password', None)
# download the files
temp_dir = tempfile.mkdtemp()
completed = 0
for item in objects:
# create download task file record
task_file = models.PackageDownloadTaskFile(task=task)
task_file.save()
try:
filename = item['filename']
task_file.filename = filename
task_file.url = item['url']
task_file.save()
download_resource(url=item['url'],
destination_path=temp_dir,
filename=filename,
username=fedora_username,
password=fedora_password)
temp_filename = os.path.join(temp_dir, filename)
if item['checksum'] is not None and item[
'checksum'] != generate_checksum(temp_filename,
'md5').hexdigest():
os.unlink(temp_filename)
raise Exception("Incorrect checksum")
# Some MODS records have no proper filenames
if filename == 'MODS Record':
filename = item['object_id'].replace(':', '-') + '-MODS.xml'
if subdirs:
base_path = os.path.join(deposit.full_path, *subdirs)
else:
base_path = deposit.full_path
new_path = os.path.join(base_path, filename)
shutil.move(temp_filename, new_path)
# mark download task file record complete or failed
task_file.completed = True
task_file.save()
LOGGER.info('Saved file to ' + new_path)
completed += 1
file_record = models.File(name=item['filename'],
source_id=item['object_id'],
checksum=generate_checksum(
new_path, 'sha512').hexdigest())
file_record.save()
except Exception as e:
LOGGER.error('Package download task encountered an error:' +
str(e))
# an error occurred
task_file.failed = True
task_file.save()
# remove temp dir
shutil.rmtree(temp_dir)
# record the number of successful downloads and completion time
task.downloads_completed = completed
task.download_completion_time = timezone.now()
task.save()
# if the deposit is ready for finalization and this is the last batch
# download to complete, then finalize
ready_for_finalization = deposit.misc_attributes.get(
'ready_for_finalization', False)
if ready_for_finalization and deposit_downloading_status(
deposit) == models.PackageDownloadTask.COMPLETE:
_finalize_if_not_empty(deposit_uuid)
def _create_resource(self, package, output_files):
""" Given a package, create an Atom resource entry to send to LOCKSS.
Parses metadata for the Atom entry from the METS file, uses
LOCKSS-o-matic-specific tags to describe size and checksums.
"""
# Parse METS to get information for atom entry
relative_mets_path = os.path.join(
os.path.splitext(os.path.basename(package.current_path))[0],
"data",
'METS.{}.xml'.format(package.uuid))
(mets_path, temp_dir) = package.extract_file(relative_mets_path)
mets = etree.parse(mets_path)
# Delete temp dir if created
if os.path.exists(temp_dir):
shutil.rmtree(temp_dir)
# Parse out name and description if found
slug = str(package.uuid)
title = os.path.basename(package.current_path)
summary = 'AIP generated by Archivematica with uuid {}'.format(package.uuid)
dublincore = mets.find('mets:dmdSec/mets:mdWrap[@MDTYPE="DC"]/mets:xmlData/dcterms:dublincore', namespaces=utils.NSMAP)
if dublincore is not None:
title = dublincore.findtext('dcterms:title', namespaces=utils.NSMAP, default=title)
slug = dublincore.findtext('dcterms:title', namespaces=utils.NSMAP, default=slug)
summary = dublincore.findtext('dcterms:description', namespaces=utils.NSMAP, default=summary)
# Parse out Agent for author
authors = mets.xpath(".//mets:mdWrap[@MDTYPE='PREMIS:AGENT']//mets:agentType[text()='organization']/ancestor::mets:agent/*/mets:agentIdentifierValue", namespaces=utils.NSMAP)
author = authors[0].text if authors else None
# Create atom entry
entry = sword2.Entry(
title=title,
id='urn:uuid:' + package.uuid,
author={'name': author},
summary=summary)
# Add each chunk to the atom entry
if not self.pointer_root:
self.pointer_root = etree.parse(package.full_pointer_file_path)
entry.register_namespace('lom', utils.NSMAP['lom'])
for index, file_path in enumerate(output_files):
# Get external URL
if len(output_files) == 1:
external_url = self._download_url(package.uuid)
else:
external_url = self._download_url(package.uuid, index + 1)
# Get checksum and size from pointer file (or generate if not found)
file_e = self.pointer_root.find(".//mets:fileGrp[@USE='LOCKSS chunk']/mets:file[@ID='{}']".format(os.path.basename(file_path)), namespaces=utils.NSMAP)
if file_e is not None:
checksum_name = file_e.get('CHECKSUMTYPE')
checksum_value = file_e.get('CHECKSUM')
size = int(file_e.get('SIZE'))
else:
# Not split, generate
try:
checksum = utils.generate_checksum(file_path,
self.checksum_type)
except ValueError: # Invalid checksum type
checksum = utils.generate_checksum(file_path, 'md5')
checksum_name = checksum.name.upper().replace('SHA', 'SHA-')
checksum_value = checksum.hexdigest()
size = os.path.getsize(file_path)
# Convert size to kB
size = str(math.ceil(size / 1000))
# Add new content entry and values
entry.add_field('lom_content', external_url)
content_entry = entry.entry[-1]
content_entry.set('size', size)
content_entry.set('checksumType', checksum_name)
content_entry.set('checksumValue', checksum_value)
LOGGER.debug('LOCKSS atom entry: %s', entry)
return entry, slug
def _split_package(self, package):
"""
Splits the package into chunks of size self.au_size. Returns list of paths to the chunks.
If the package has already been split (and an event is in the pointer
file), returns the list if file paths from the pointer file.
Updates the pointer file with the new LOCKSS chunks, and adds 'division'
event.
"""
# Parse pointer file
if not self.pointer_root:
self.pointer_root = etree.parse(package.full_pointer_file_path)
# Check if file is already split, and if so just return split files
if self.pointer_root.xpath('.//premis:eventType[text()="division"]', namespaces=utils.NSMAP):
chunks = self.pointer_root.findall(".//mets:div[@TYPE='Archival Information Package']/mets:div[@TYPE='LOCKSS chunk']", namespaces=utils.NSMAP)
output_files = [c.find('mets:fptr', namespaces=utils.NSMAP).get('FILEID') for c in chunks]
return output_files
file_path = package.full_path
expected_num_files = math.ceil(os.path.getsize(file_path) / self.au_size)
LOGGER.debug('expected_num_files: %s', expected_num_files)
# No split needed - just return the file path
if expected_num_files <= 1:
LOGGER.debug('Only one file expected, not splitting')
output_files = [file_path]
# No events or structMap changes needed
LOGGER.info('LOCKSS: after splitting: %s', output_files)
return output_files
# Split file
# Strip extension, add .tar-1 ('-1' to make rename script happy)
output_path = os.path.splitext(file_path)[0] + '.tar-1'
command = ['tar', '--create', '--multi-volume',
'--tape-length', str(self.au_size),
'--new-volume-script', 'common/tar_new_volume.sh',
'-f', output_path, file_path]
# TODO reserve space in quota for extra files
LOGGER.info('LOCKSS split command: %s', command)
try:
subprocess.check_call(command)
except Exception:
LOGGER.exception("Split of %s failed with command %s", file_path, command)
raise
output_path = output_path[:-2] # Remove '-1'
dirname, basename = os.path.split(output_path)
output_files = sorted([os.path.join(dirname, entry) for entry in os.listdir(dirname) if entry.startswith(basename)])
# Update pointer file
amdsec = self.pointer_root.find('mets:amdSec', namespaces=utils.NSMAP)
# Add 'division' PREMIS:EVENT
try:
event_detail = subprocess.check_output(['tar', '--version'])
except subprocess.CalledProcessError as e:
event_detail = e.output or _('Error: getting tool info; probably GNU tar')
utils.mets_add_event(
amdsec,
event_type='division',
event_detail=event_detail,
event_outcome_detail_note='{} LOCKSS chunks created'.format(len(output_files)),
)
# Update structMap & fileSec
self.pointer_root.find('mets:structMap', namespaces=utils.NSMAP).set('TYPE', 'logical')
aip_div = self.pointer_root.find("mets:structMap/mets:div[@TYPE='Archival Information Package']", namespaces=utils.NSMAP)
filesec = self.pointer_root.find('mets:fileSec', namespaces=utils.NSMAP)
filegrp = etree.SubElement(filesec, utils.PREFIX_NS['mets'] + 'fileGrp', USE='LOCKSS chunk')
# Move ftpr to Local copy div
local_ftpr = aip_div.find('mets:fptr', namespaces=utils.NSMAP)
if local_ftpr is not None:
div = etree.SubElement(aip_div, utils.PREFIX_NS['mets'] + 'div', TYPE='Local copy')
div.append(local_ftpr) # This moves local_fptr
# Add each split chunk to structMap & fileSec
for idx, out_path in enumerate(output_files):
# Add div to structMap
div = etree.SubElement(aip_div, utils.PREFIX_NS['mets'] + 'div', TYPE='LOCKSS chunk', ORDER=str(idx + 1))
etree.SubElement(div, utils.PREFIX_NS['mets'] + 'fptr', FILEID=os.path.basename(out_path))
# Get checksum and size for fileSec
try:
checksum = utils.generate_checksum(out_path, self.checksum_type)
except ValueError: # Invalid checksum type
checksum = utils.generate_checksum(out_path, 'md5')
checksum_name = checksum.name.upper().replace('SHA', 'SHA-')
size = os.path.getsize(out_path)
# Add file & FLocat to fileSec
file_e = etree.SubElement(filegrp, utils.PREFIX_NS['mets'] + 'file',
ID=os.path.basename(out_path), SIZE=str(size),
CHECKSUM=checksum.hexdigest(), CHECKSUMTYPE=checksum_name)
flocat = etree.SubElement(file_e, utils.PREFIX_NS['mets'] + 'FLocat', OTHERLOCTYPE="SYSTEM", LOCTYPE="OTHER")
flocat.set(utils.NSMAP['xlink'] + 'href', out_path)
# Write out pointer file again
with open(package.full_pointer_file_path, 'w') as f:
f.write(etree.tostring(self.pointer_root, pretty_print=True, xml_declaration=True, encoding='utf-8'))
return output_files
def _upload_file(self, url, upload_file, resume=False):
"""
Upload a file of any size to Duracloud.
If the file is larger that self.CHUNK_SIZE, will chunk it and upload chunks and manifest.
:param url: URL to upload the file to.
:param upload_file: Absolute path to the file to upload.
:returns: None
:raises: StorageException if error storing file
"""
LOGGER.debug('Upload %s to %s', upload_file, url)
filesize = os.path.getsize(upload_file)
if filesize > self.CHUNK_SIZE:
LOGGER.debug('%s size (%s) larger than %s', upload_file, filesize,
self.CHUNK_SIZE)
# Create manifest info for complete file. Eg:
# <header schemaVersion="0.2">
# <sourceContent contentId="chunked/chunked_image.jpg">
# <mimetype>application/octet-stream</mimetype>
# <byteSize>2222135</byteSize>
# <md5>9497f70a1a17943ddfcbed567538900d</md5>
# </sourceContent>
# </header>
relative_path = urllib.unquote(
url.replace(self.duraspace_url, '', 1))
LOGGER.debug('File name: %s', relative_path)
checksum = utils.generate_checksum(upload_file, 'md5')
LOGGER.debug('Checksum for %s: %s', upload_file,
checksum.hexdigest())
root = etree.Element('{duracloud.org}chunksManifest',
nsmap={'dur': 'duracloud.org'})
header = etree.SubElement(root, 'header', schemaVersion="0.2")
content = etree.SubElement(header,
'sourceContent',
contentId=relative_path)
etree.SubElement(content,
'mimetype').text = 'application/octet-stream'
etree.SubElement(content, 'byteSize').text = str(filesize)
etree.SubElement(content, 'md5').text = checksum.hexdigest()
chunks = etree.SubElement(root, 'chunks')
# Split file into chunks
with open(upload_file, 'rb') as f:
# If resume, check if chunks already exists
if resume:
chunklist = set(self._get_files_list(relative_path))
LOGGER.debug('Chunklist %s', chunklist)
file_complete = False
i = 0
while not file_complete:
# Setup chunk info
chunk_suffix = '.dura-chunk-' + str(i).zfill(4)
chunk_path = upload_file + chunk_suffix
LOGGER.debug('Chunk path: %s', chunk_path)
chunk_url = url + chunk_suffix
LOGGER.debug('Chunk URL: %s', chunk_url)
chunkid = relative_path + chunk_suffix
LOGGER.debug('Chunk ID: %s', chunkid)
try:
self._process_chunk(f, chunk_path)
except StopIteration:
file_complete = True
# Make chunk element
# <chunk chunkId="chunked/chunked_image.jpg.dura-chunk-0000" index="0">
# <byteSize>2097152</byteSize>
# <md5>ddbb227beaac5a9dc34eb49608997abf</md5>
# </chunk>
checksum = utils.generate_checksum(chunk_path)
chunk_e = etree.SubElement(chunks,
'chunk',
chunkId=chunkid)
etree.SubElement(chunk_e, 'byteSize').text = str(
os.path.getsize(chunk_path))
etree.SubElement(chunk_e,
'md5').text = checksum.hexdigest()
# Upload chunk
# Check if chunk exists already
if resume and chunkid in chunklist:
LOGGER.info('%s already in Duracloud, skipping upload',
chunk_path)
else:
self._upload_chunk(chunk_url, chunk_path)
# Delete chunk
os.remove(chunk_path)
i += 1
# Write .dura-manifest
manifest_path = upload_file + self.MANIFEST_SUFFIX
manifest_url = url + self.MANIFEST_SUFFIX
with open(manifest_path, 'w') as f:
f.write(
etree.tostring(root,
pretty_print=True,
xml_declaration=True,
encoding='UTF-8'))
# Upload .dura-manifest
self._upload_chunk(manifest_url, manifest_path)
os.remove(manifest_path)
# TODO what if .dura-manifest over chunksize?
else:
# Example URL: https://trial.duracloud.org/durastore/trial261//ts/test.txt
self._upload_chunk(url, upload_file)
def _download_file(self,
url,
download_path,
expected_size=0,
checksum=None):
"""
Helper to download files from DuraCloud.
:param url: URL to fetch the file from.
:param download_path: Absolute path to store the downloaded file at.
:return: True on success, False if file not found
:raises: StorageException if response code not 200 or 404
"""
LOGGER.debug('URL: %s', url)
response = self.session.get(url)
LOGGER.debug('Response: %s', response)
if response.status_code == 404:
# Check if chunked by looking for a .dura-manifest
manifest_url = url + self.MANIFEST_SUFFIX
LOGGER.debug('Manifest URL: %s', manifest_url)
response = self.session.get(manifest_url)
LOGGER.debug('Response: %s', response)
# No manifest - this file does not exist
if not response.ok:
return False
# Get chunks, expected size, checksum
root = etree.fromstring(response.content)
expected_size = int(root.findtext('header/sourceContent/byteSize'))
checksum = root.findtext('header/sourceContent/md5')
chunk_elements = [e for e in root.findall('chunks/chunk')]
# Download each chunk and append to original file
self.space.create_local_directory(download_path)
LOGGER.debug('Writing to %s', download_path)
with open(download_path, 'wb') as output_f:
for e in chunk_elements:
# Parse chunk element
chunk = e.attrib['chunkId']
size = int(e.findtext('byteSize'))
md5 = e.findtext('md5')
# Download
chunk_url = self.duraspace_url + urllib.quote(chunk)
LOGGER.debug('Chunk URL: %s', chunk_url)
chunk_path = chunk_url.replace(url, download_path)
LOGGER.debug('Chunk path: %s', chunk_path)
self._download_file(chunk_url, chunk_path, size, md5)
# Append to output
with open(chunk_path, 'rb') as chunk_f:
shutil.copyfileobj(chunk_f, output_f)
# Delete chunk_path
os.remove(chunk_path)
elif response.status_code != 200:
LOGGER.warning('Response: %s when fetching %s', response, url)
LOGGER.warning('Response text: %s', response.text)
raise StorageException('Unable to fetch %s' % url)
else: # Status code 200 - file exists
self.space.create_local_directory(download_path)
LOGGER.debug('Writing to %s', download_path)
with open(download_path, 'wb') as f:
f.write(response.content)
# Verify file, if size or checksum is known
if expected_size and os.path.getsize(download_path) != expected_size:
raise StorageException(
_('File %(path)s does not match expected size of %(expected_size)s bytes, but was actually %(actual_size)s bytes'
), {
'path': download_path,
'expected_size': expected_size,
'actual_size': os.path.getsize(download_path)
})
calculated_checksum = utils.generate_checksum(download_path, 'md5')
if checksum and checksum != calculated_checksum.hexdigest():
raise StorageException(
'File %s does not match expected checksum of %s, but was actually %s',
download_path, checksum, calculated_checksum.hexdigest())
return True
def _upload_file(self, url, upload_file):
"""
Helper to upload files to Duracloud.
:param url: URL to upload the file to.
:param upload_file: Absolute path to the file to upload.
:returns: None
:raises: StorageException if error storing file
"""
LOGGER.debug('Upload %s to %s', upload_file, url)
filesize = os.path.getsize(upload_file)
if filesize > self.CHUNK_SIZE:
LOGGER.debug('%s size (%s) larger than %s', upload_file, filesize,
self.CHUNK_SIZE)
# Create manifest info for complete file. Eg:
# <header schemaVersion="0.2">
# <sourceContent contentId="chunked/chunked_image.jpg">
# <mimetype>application/octet-stream</mimetype>
# <byteSize>2222135</byteSize>
# <md5>9497f70a1a17943ddfcbed567538900d</md5>
# </sourceContent>
# </header>
relative_path = urllib.unquote(
url.replace(self.duraspace_url, '', 1))
LOGGER.debug('File name: %s', relative_path)
checksum = utils.generate_checksum(upload_file, 'md5')
root = etree.Element('{duracloud.org}chunksManifest',
nsmap={'dur': 'duracloud.org'})
header = etree.SubElement(root, 'header', schemaVersion="0.2")
content = etree.SubElement(header,
'sourceContent',
contentId=relative_path)
etree.SubElement(content,
'mimetype').text = 'application/octet-stream'
etree.SubElement(content, 'byteSize').text = str(filesize)
etree.SubElement(content, 'md5').text = checksum.hexdigest()
chunks = etree.SubElement(root, 'chunks')
# Split file into chunks
with open(upload_file, 'rb') as f:
i = 0
chunk_data = f.read(self.CHUNK_SIZE)
while chunk_data:
# Setup chunk info
chunk_suffix = '.dura-chunk-' + str(i).zfill(4)
chunk_path = upload_file + chunk_suffix
LOGGER.debug('Chunk path: %s', chunk_path)
chunk_url = url + chunk_suffix
LOGGER.debug('Chunk URL: %s', chunk_url)
chunkid = relative_path + chunk_suffix
LOGGER.debug('Chunk ID: %s', chunkid)
# Write chunk
with open(chunk_path, 'wb') as fchunk:
fchunk.write(chunk_data)
# Make chunk element
# <chunk chunkId="chunked/chunked_image.jpg.dura-chunk-0000" index="0">
# <byteSize>2097152</byteSize>
# <md5>ddbb227beaac5a9dc34eb49608997abf</md5>
# </chunk>
checksum = utils.generate_checksum(chunk_path)
chunk_e = etree.SubElement(chunks,
'chunk',
chunkId=chunkid)
etree.SubElement(chunk_e, 'byteSize').text = str(
os.path.getsize(chunk_path))
etree.SubElement(chunk_e,
'md5').text = checksum.hexdigest()
# Upload chunk
self._upload_file(chunk_url, chunk_path)
# Delete chunk
os.remove(chunk_path)
# Read next chunk
chunk_data = f.read(self.CHUNK_SIZE)
i += 1
# Write .dura-manifest
manifest_path = upload_file + self.MANIFEST_SUFFIX
manifest_url = url + self.MANIFEST_SUFFIX
with open(manifest_path, 'w') as f:
f.write(
etree.tostring(root,
pretty_print=True,
xml_declaration=True,
encoding='UTF-8'))
# Upload .dura-manifest
self._upload_file(manifest_url, manifest_path)
os.remove(manifest_path)
# TODO what if .dura-manifest over chunksize?
else:
# Example URL: https://trial.duracloud.org/durastore/trial261//ts/test.txt
LOGGER.debug('PUT URL: %s', url)
with open(upload_file, 'rb') as f:
response = self.session.put(url, data=f)
LOGGER.debug('Response: %s', response)
if response.status_code != 201:
LOGGER.warning('%s: Response: %s', response, response.text)
raise StorageException('Unable to store %s' % upload_file)
