archive.add( ldda.dataset.file_name, path )
else:
archive.add( ldda.dataset.file_name, path, check_file=True )
except IOError:
log.exception( "Unable to write %s to temporary library download archive" % ldda.dataset.file_name )
raise exceptions.InternalServerError( "Unable to create archive for download" )
except ObjectNotFound:
log.exception( "Requested dataset %s does not exist on the host." % ldda.dataset.file_name )
raise exceptions.ObjectNotFound( "Requested dataset not found." )
except Exception, e:
log.exception( "Unable to add %s to temporary library download archive %s" % ( fname, outfname ) )
raise exceptions.InternalServerError( "Unknown error. " + str( e ) )
lname = 'selected_dataset'
fname = lname.replace( ' ', '_' ) + '_files'
if format == 'zip':
archive.close()
trans.response.set_content_type( "application/octet-stream" )
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % ( fname, outext )
archive = util.streamball.ZipBall( tmpf, tmpd )
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
else:
trans.response.set_content_type( "application/x-tar" )
trans.response.headers[ "Content-Disposition" ] = 'attachment; filename="%s.%s"' % ( fname, outext )
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
elif format == 'uncompressed':
if len(library_datasets) != 1:
raise exceptions.RequestParameterInvalidException( "You can download only one uncompressed file at once." )
def download(self, trans, format, **kwd):
"""
Download requested datasets (identified by encoded IDs) in requested format.
* GET /api/libraries/datasets/download/{format}
* POST /api/libraries/datasets/download/{format}
example: ``GET localhost:8080/api/libraries/datasets/download/tbz?ld_ids%255B%255D=a0d84b45643a2678&ld_ids%255B%255D=fe38c84dcd46c828``
.. note:: supported format values are: 'zip', 'tgz', 'tbz', 'uncompressed'
:param format: string representing requested archive format
:type format: string
:param ld_ids[]: an array of encoded dataset ids
:type ld_ids[]: an array
:param folder_ids[]: an array of encoded folder ids
:type folder_ids[]: an array
:returns: either archive with the requested datasets packed inside or a single uncompressed dataset
:rtype: file
:raises: MessageException, ItemDeletionException, ItemAccessibilityException, HTTPBadRequest, OSError, IOError, ObjectNotFound
"""
library_datasets = []
datasets_to_download = kwd.get('ld_ids%5B%5D', None)
if datasets_to_download is None:
datasets_to_download = kwd.get('ld_ids', None)
if datasets_to_download is not None:
datasets_to_download = util.listify(datasets_to_download)
for dataset_id in datasets_to_download:
try:
library_dataset = self.get_library_dataset(trans, id=dataset_id, check_ownership=False, check_accessible=True)
library_datasets.append(library_dataset)
except HTTPBadRequest:
raise exceptions.RequestParameterInvalidException('Bad Request.')
except HTTPInternalServerError:
raise exceptions.InternalServerError('Internal error.')
except Exception as e:
raise exceptions.InternalServerError('Unknown error.' + str(e))
folders_to_download = kwd.get('folder_ids%5B%5D', None)
if folders_to_download is None:
folders_to_download = kwd.get('folder_ids', None)
if folders_to_download is not None:
folders_to_download = util.listify(folders_to_download)
current_user_roles = trans.get_current_user_roles()
def traverse(folder):
admin = trans.user_is_admin()
rval = []
for subfolder in folder.active_folders:
if not admin:
can_access, folder_ids = trans.app.security_agent.check_folder_contents(trans.user, current_user_roles, subfolder)
if (admin or can_access) and not subfolder.deleted:
rval.extend(traverse(subfolder))
for ld in folder.datasets:
if not admin:
can_access = trans.app.security_agent.can_access_dataset(
current_user_roles,
ld.library_dataset_dataset_association.dataset
)
if (admin or can_access) and not ld.deleted:
rval.append(ld)
return rval
for encoded_folder_id in folders_to_download:
folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id)
folder = self.folder_manager.get(trans, folder_id)
library_datasets.extend(traverse(folder))
if not library_datasets:
raise exceptions.RequestParameterMissingException('Request has to contain a list of dataset ids or folder ids to download.')
if format in ['zip', 'tgz', 'tbz']:
# error = False
killme = string.punctuation + string.whitespace
trantab = string.maketrans(killme, '_' * len(killme))
try:
outext = 'zip'
if format == 'zip':
# Can't use mkstemp - the file must not exist first
tmpd = tempfile.mkdtemp()
util.umask_fix_perms(tmpd, trans.app.config.umask, 0777, self.app.config.gid)
tmpf = os.path.join(tmpd, 'library_download.' + format)
if trans.app.config.upstream_gzip:
archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_STORED, True)
else:
archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_DEFLATED, True)
archive.add = lambda x, y: archive.write(x, y.encode('CP437'))
elif format == 'tgz':
if trans.app.config.upstream_gzip:
archive = StreamBall('w|')
outext = 'tar'
else:
archive = StreamBall('w|gz')
outext = 'tgz'
elif format == 'tbz':
archive = StreamBall('w|bz2')
outext = 'tbz2'
except (OSError, zipfile.BadZipfile):
log.exception("Unable to create archive for download")
raise exceptions.InternalServerError("Unable to create archive for download.")
except Exception:
log.exception("Unexpected error in create archive for download")
raise exceptions.InternalServerError("Unable to create archive for download.")
composite_extensions = trans.app.datatypes_registry.get_composite_extensions()
seen = []
for ld in library_datasets:
ldda = ld.library_dataset_dataset_association
ext = ldda.extension
is_composite = ext in composite_extensions
path = ""
parent_folder = ldda.library_dataset.folder
while parent_folder is not None:
# Exclude the now-hidden "root folder"
if parent_folder.parent is None:
path = os.path.join(parent_folder.library_root[0].name, path)
break
path = os.path.join(parent_folder.name, path)
parent_folder = parent_folder.parent
path += ldda.name
while path in seen:
path += '_'
seen.append(path)
zpath = os.path.split(path)[-1] # comes as base_name/fname
outfname, zpathext = os.path.splitext(zpath)
if is_composite:
# need to add all the components from the extra_files_path to the zip
if zpathext == '':
zpath = '%s.html' % zpath # fake the real nature of the html file
try:
if format == 'zip':
archive.add(ldda.dataset.file_name, zpath) # add the primary of a composite set
else:
archive.add(ldda.dataset.file_name, zpath, check_file=True) # add the primary of a composite set
except IOError:
log.exception("Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name)
raise exceptions.InternalServerError("Unable to create archive for download.")
except ObjectNotFound:
log.exception("Requested dataset %s does not exist on the host.", ldda.dataset.file_name)
raise exceptions.ObjectNotFound("Requested dataset not found. ")
except Exception as e:
log.exception("Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name)
raise exceptions.InternalServerError("Unable to add composite parent to temporary library download archive. " + str(e))
flist = glob.glob(os.path.join(ldda.dataset.extra_files_path, '*.*')) # glob returns full paths
for fpath in flist:
efp, fname = os.path.split(fpath)
if fname > '':
fname = fname.translate(trantab)
try:
if format == 'zip':
archive.add(fpath, fname)
else:
archive.add(fpath, fname, check_file=True)
except IOError:
log.exception("Unable to add %s to temporary library download archive %s", fname, outfname)
raise exceptions.InternalServerError("Unable to create archive for download.")
except ObjectNotFound:
log.exception("Requested dataset %s does not exist on the host.", fpath)
raise exceptions.ObjectNotFound("Requested dataset not found.")
except Exception as e:
log.exception("Unable to add %s to temporary library download archive %s" % (fname, outfname))
raise exceptions.InternalServerError("Unable to add dataset to temporary library download archive . " + str(e))
else:
try:
if format == 'zip':
archive.add(ldda.dataset.file_name, path)
else:
archive.add(ldda.dataset.file_name, path, check_file=True)
except IOError:
log.exception("Unable to write %s to temporary library download archive", ldda.dataset.file_name)
raise exceptions.InternalServerError("Unable to create archive for download")
except ObjectNotFound:
log.exception("Requested dataset %s does not exist on the host.", ldda.dataset.file_name)
raise exceptions.ObjectNotFound("Requested dataset not found.")
except Exception as e:
log.exception("Unable to add %s to temporary library download archive %s", fname, outfname)
raise exceptions.InternalServerError("Unknown error. " + str(e))
lname = 'selected_dataset'
fname = lname.replace(' ', '_') + '_files'
if format == 'zip':
archive.close()
trans.response.set_content_type("application/octet-stream")
trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.%s"' % (fname, outext)
archive = util.streamball.ZipBall(tmpf, tmpd)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
else:
trans.response.set_content_type("application/x-tar")
trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.%s"' % (fname, outext)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
elif format == 'uncompressed':
if len(library_datasets) != 1:
raise exceptions.RequestParameterInvalidException("You can download only one uncompressed file at once.")
else:
single_ld = library_datasets[0]
ldda = single_ld.library_dataset_dataset_association
dataset = ldda.dataset
fStat = os.stat(dataset.file_name)
trans.response.set_content_type(ldda.get_mime())
trans.response.headers['Content-Length'] = int(fStat.st_size)
fname = ldda.name
fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in fname)[0:150]
trans.response.headers["Content-Disposition"] = 'attachment; filename="%s"' % fname
try:
return open(dataset.file_name)
except:
raise exceptions.InternalServerError("This dataset contains no content.")
else:
raise exceptions.RequestParameterInvalidException("Wrong format parameter specified")
except ObjectNotFound:
log.exception(
"Requested dataset %s does not exist on the host."
% ldda.dataset.file_name)
raise exceptions.ObjectNotFound(
"Requested dataset not found.")
except Exception, e:
log.exception(
"Unable to add %s to temporary library download archive %s"
% (fname, outfname))
raise exceptions.InternalServerError(
"Unknown error. " + str(e))
lname = 'selected_dataset'
fname = lname.replace(' ', '_') + '_files'
if format == 'zip':
archive.close()
trans.response.set_content_type("application/octet-stream")
trans.response.headers[
"Content-Disposition"] = 'attachment; filename="%s.%s"' % (
fname, outext)
archive = util.streamball.ZipBall(tmpf, tmpd)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
else:
trans.response.set_content_type("application/x-tar")
trans.response.headers[
"Content-Disposition"] = 'attachment; filename="%s.%s"' % (
fname, outext)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
def download(self, trans, format, **kwd):
"""
GET /api/libraries/datasets/download/{format}
POST /api/libraries/datasets/download/{format}
Download requested datasets (identified by encoded IDs) in requested format.
example: ``GET localhost:8080/api/libraries/datasets/download/tbz?ld_ids%255B%255D=a0d84b45643a2678&ld_ids%255B%255D=fe38c84dcd46c828``
.. note:: supported format values are: 'zip', 'tgz', 'tbz', 'uncompressed'
:param format: string representing requested archive format
:type format: string
:param ld_ids[]: an array of encoded dataset ids
:type ld_ids[]: an array
:param folder_ids[]: an array of encoded folder ids
:type folder_ids[]: an array
:returns: either archive with the requested datasets packed inside or a single uncompressed dataset
:rtype: file
:raises: MessageException, ItemDeletionException, ItemAccessibilityException, HTTPBadRequest, OSError, IOError, ObjectNotFound
"""
library_datasets = []
datasets_to_download = kwd.get('ld_ids%5B%5D', None)
if datasets_to_download is None:
datasets_to_download = kwd.get('ld_ids', None)
if datasets_to_download is not None:
datasets_to_download = util.listify(datasets_to_download)
for dataset_id in datasets_to_download:
try:
library_dataset = self.get_library_dataset(
trans,
id=dataset_id,
check_ownership=False,
check_accessible=True)
library_datasets.append(library_dataset)
except HTTPBadRequest:
raise exceptions.RequestParameterInvalidException(
'Bad Request.')
except HTTPInternalServerError:
raise exceptions.InternalServerError('Internal error.')
except Exception as e:
raise exceptions.InternalServerError('Unknown error.' +
util.unicodify(e))
folders_to_download = kwd.get('folder_ids%5B%5D', None)
if folders_to_download is None:
folders_to_download = kwd.get('folder_ids', None)
if folders_to_download is not None:
folders_to_download = util.listify(folders_to_download)
current_user_roles = trans.get_current_user_roles()
def traverse(folder):
admin = trans.user_is_admin
rval = []
for subfolder in folder.active_folders:
if not admin:
can_access, folder_ids = trans.app.security_agent.check_folder_contents(
trans.user, current_user_roles, subfolder)
if (admin or can_access) and not subfolder.deleted:
rval.extend(traverse(subfolder))
for ld in folder.datasets:
if not admin:
can_access = trans.app.security_agent.can_access_dataset(
current_user_roles,
ld.library_dataset_dataset_association.dataset)
if (admin or can_access) and not ld.deleted:
rval.append(ld)
return rval
for encoded_folder_id in folders_to_download:
folder_id = self.folder_manager.cut_and_decode(
trans, encoded_folder_id)
folder = self.folder_manager.get(trans, folder_id)
library_datasets.extend(traverse(folder))
if not library_datasets:
raise exceptions.RequestParameterMissingException(
'Request has to contain a list of dataset ids or folder ids to download.'
)
if format in ['zip', 'tgz', 'tbz']:
# error = False
killme = string.punctuation + string.whitespace
trantab = string.maketrans(killme, '_' * len(killme))
try:
outext = 'zip'
if format == 'zip':
# Can't use mkstemp - the file must not exist first
tmpd = tempfile.mkdtemp()
util.umask_fix_perms(tmpd, trans.app.config.umask, 0o777,
self.app.config.gid)
tmpf = os.path.join(tmpd, 'library_download.' + format)
if trans.app.config.upstream_gzip:
archive = zipfile.ZipFile(tmpf, 'w',
zipfile.ZIP_STORED, True)
else:
archive = zipfile.ZipFile(tmpf, 'w',
zipfile.ZIP_DEFLATED, True)
archive.add = lambda x, y: archive.write(
x, y.encode('CP437'))
elif format == 'tgz':
if trans.app.config.upstream_gzip:
archive = StreamBall('w|')
outext = 'tar'
else:
archive = StreamBall('w|gz')
outext = 'tgz'
elif format == 'tbz':
archive = StreamBall('w|bz2')
outext = 'tbz2'
except (OSError, zipfile.BadZipfile):
log.exception("Unable to create archive for download")
raise exceptions.InternalServerError(
"Unable to create archive for download.")
except Exception:
log.exception(
"Unexpected error in create archive for download")
raise exceptions.InternalServerError(
"Unable to create archive for download.")
composite_extensions = trans.app.datatypes_registry.get_composite_extensions(
)
seen = []
for ld in library_datasets:
ldda = ld.library_dataset_dataset_association
ext = ldda.extension
is_composite = ext in composite_extensions
path = ""
parent_folder = ldda.library_dataset.folder
while parent_folder is not None:
# Exclude the now-hidden "root folder"
if parent_folder.parent is None:
path = os.path.join(parent_folder.library_root[0].name,
path)
break
path = os.path.join(parent_folder.name, path)
parent_folder = parent_folder.parent
path += ldda.name
while path in seen:
path += '_'
path = "{path}.{extension}".format(path=path,
extension=ldda.extension)
seen.append(path)
zpath = os.path.split(path)[-1] # comes as base_name/fname
outfname, zpathext = os.path.splitext(zpath)
if is_composite:
# need to add all the components from the extra_files_path to the zip
if zpathext == '':
zpath = '%s.html' % zpath # fake the real nature of the html file
try:
if format == 'zip':
archive.add(
ldda.dataset.file_name,
zpath) # add the primary of a composite set
else:
archive.add(ldda.dataset.file_name,
zpath,
check_file=True
) # add the primary of a composite set
except IOError:
log.exception(
"Unable to add composite parent %s to temporary library download archive",
ldda.dataset.file_name)
raise exceptions.InternalServerError(
"Unable to create archive for download.")
except ObjectNotFound:
log.exception(
"Requested dataset %s does not exist on the host.",
ldda.dataset.file_name)
raise exceptions.ObjectNotFound(
"Requested dataset not found. ")
except Exception as e:
log.exception(
"Unable to add composite parent %s to temporary library download archive",
ldda.dataset.file_name)
raise exceptions.InternalServerError(
"Unable to add composite parent to temporary library download archive. "
+ util.unicodify(e))
flist = glob.glob(
os.path.join(ldda.dataset.extra_files_path,
'*.*')) # glob returns full paths
for fpath in flist:
efp, fname = os.path.split(fpath)
if fname > '':
fname = fname.translate(trantab)
try:
if format == 'zip':
archive.add(fpath, fname)
else:
archive.add(fpath, fname, check_file=True)
except IOError:
log.exception(
"Unable to add %s to temporary library download archive %s",
fname, outfname)
raise exceptions.InternalServerError(
"Unable to create archive for download.")
except ObjectNotFound:
log.exception(
"Requested dataset %s does not exist on the host.",
fpath)
raise exceptions.ObjectNotFound(
"Requested dataset not found.")
except Exception as e:
log.exception(
"Unable to add %s to temporary library download archive %s",
fname, outfname)
raise exceptions.InternalServerError(
"Unable to add dataset to temporary library download archive . "
+ util.unicodify(e))
else:
try:
if format == 'zip':
archive.add(ldda.dataset.file_name, path)
else:
archive.add(ldda.dataset.file_name,
path,
check_file=True)
except IOError:
log.exception(
"Unable to write %s to temporary library download archive",
ldda.dataset.file_name)
raise exceptions.InternalServerError(
"Unable to create archive for download")
except ObjectNotFound:
log.exception(
"Requested dataset %s does not exist on the host.",
ldda.dataset.file_name)
raise exceptions.ObjectNotFound(
"Requested dataset not found.")
except Exception as e:
log.exception(
"Unable to add %s to temporary library download archive %s",
ldda.dataset.file_name, outfname)
raise exceptions.InternalServerError(
"Unknown error. " + util.unicodify(e))
lname = 'selected_dataset'
fname = lname.replace(' ', '_') + '_files'
if format == 'zip':
archive.close()
trans.response.set_content_type("application/octet-stream")
trans.response.headers[
"Content-Disposition"] = 'attachment; filename="%s.%s"' % (
fname, outext)
archive = util.streamball.ZipBall(tmpf, tmpd)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
else:
trans.response.set_content_type("application/x-tar")
trans.response.headers[
"Content-Disposition"] = 'attachment; filename="%s.%s"' % (
fname, outext)
archive.wsgi_status = trans.response.wsgi_status()
archive.wsgi_headeritems = trans.response.wsgi_headeritems()
return archive.stream
elif format == 'uncompressed':
if len(library_datasets) != 1:
raise exceptions.RequestParameterInvalidException(
"You can download only one uncompressed file at once.")
else:
single_ld = library_datasets[0]
ldda = single_ld.library_dataset_dataset_association
dataset = ldda.dataset
fStat = os.stat(dataset.file_name)
trans.response.set_content_type(ldda.get_mime())
trans.response.headers['Content-Length'] = int(fStat.st_size)
fname = "{path}.{extension}".format(path=ldda.name,
extension=ldda.extension)
fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_'
for c in fname)[0:150]
trans.response.headers[
"Content-Disposition"] = 'attachment; filename="%s"' % fname
try:
return open(dataset.file_name, 'rb')
except Exception:
raise exceptions.InternalServerError(
"This dataset contains no content.")
else:
raise exceptions.RequestParameterInvalidException(
"Wrong format parameter specified")