def download(self, trans, archive_format, **kwd): """ GET /api/libraries/datasets/download/{archive_format} POST /api/libraries/datasets/download/{archive_format} Download requested datasets (identified by encoded IDs) in requested archive_format. example: ``GET localhost:8080/api/libraries/datasets/download/tbz?ld_ids%255B%255D=a0d84b45643a2678&ld_ids%255B%255D=fe38c84dcd46c828`` .. note:: supported archive_format values are: 'zip', 'tgz', 'tbz', 'uncompressed' :param archive_format: string representing requested archive archive_format :type archive_format: string :param ld_ids[]: an array of encoded dataset ids :type ld_ids[]: an array :param folder_ids[]: an array of encoded folder ids :type folder_ids[]: an array :returns: either archive with the requested datasets packed inside or a single uncompressed dataset :rtype: file :raises: MessageException, ItemDeletionException, ItemAccessibilityException, HTTPBadRequest, OSError, IOError, ObjectNotFound """ library_datasets = [] datasets_to_download = kwd.get('ld_ids%5B%5D', None) if datasets_to_download is None: datasets_to_download = kwd.get('ld_ids', None) if datasets_to_download is not None: datasets_to_download = util.listify(datasets_to_download) for dataset_id in datasets_to_download: try: library_dataset = self.get_library_dataset(trans, id=dataset_id, check_ownership=False, check_accessible=True) library_datasets.append(library_dataset) except HTTPBadRequest: raise exceptions.RequestParameterInvalidException('Bad Request.') except HTTPInternalServerError: raise exceptions.InternalServerError('Internal error.') except Exception as e: raise exceptions.InternalServerError(f"Unknown error.{util.unicodify(e)}") folders_to_download = kwd.get('folder_ids%5B%5D', None) if folders_to_download is None: folders_to_download = kwd.get('folder_ids', None) if folders_to_download is not None: folders_to_download = util.listify(folders_to_download) current_user_roles = trans.get_current_user_roles() def traverse(folder): admin = trans.user_is_admin rval = [] for subfolder in folder.active_folders: if not admin: can_access, folder_ids = trans.app.security_agent.check_folder_contents(trans.user, current_user_roles, subfolder) if (admin or can_access) and not subfolder.deleted: rval.extend(traverse(subfolder)) for ld in folder.datasets: if not admin: can_access = trans.app.security_agent.can_access_dataset( current_user_roles, ld.library_dataset_dataset_association.dataset ) if (admin or can_access) and not ld.deleted: rval.append(ld) return rval for encoded_folder_id in folders_to_download: folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id) folder = self.folder_manager.get(trans, folder_id) library_datasets.extend(traverse(folder)) if not library_datasets: raise exceptions.RequestParameterMissingException('Request has to contain a list of dataset ids or folder ids to download.') if archive_format == 'zip': archive = ZipstreamWrapper( archive_name="selected_library_files", upstream_mod_zip=self.app.config.upstream_mod_zip, upstream_gzip=self.app.config.upstream_gzip, ) killme = string.punctuation + string.whitespace trantab = str.maketrans(killme, '_' * len(killme)) seen = [] for ld in library_datasets: ldda = ld.library_dataset_dataset_association is_composite = ldda.datatype.composite_type path = "" parent_folder = ldda.library_dataset.folder while parent_folder is not None: # Exclude the now-hidden "root folder" if parent_folder.parent is None: path = os.path.join(parent_folder.library_root[0].name, path) break path = os.path.join(parent_folder.name, path) parent_folder = parent_folder.parent path += ldda.name while path in seen: path += '_' path = f"{path}.{ldda.extension}" seen.append(path) zpath = os.path.split(path)[-1] # comes as base_name/fname outfname, zpathext = os.path.splitext(zpath) if is_composite: # need to add all the components from the extra_files_path to the zip if zpathext == '': zpath = f'{zpath}.html' # fake the real nature of the html file try: if archive_format == 'zip': archive.write(ldda.dataset.file_name, zpath) # add the primary of a composite set else: archive.write(ldda.dataset.file_name, zpath) # add the primary of a composite set except OSError: log.exception("Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError("Unable to create archive for download.") except ObjectNotFound: log.exception("Requested dataset %s does not exist on the host.", ldda.dataset.file_name) raise exceptions.ObjectNotFound("Requested dataset not found. ") except Exception as e: log.exception("Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError(f"Unable to add composite parent to temporary library download archive. {util.unicodify(e)}") flist = glob.glob(os.path.join(ldda.dataset.extra_files_path, '*.*')) # glob returns full paths for fpath in flist: efp, fname = os.path.split(fpath) if fname > '': fname = fname.translate(trantab) try: archive.write(fpath, fname) except OSError: log.exception("Unable to add %s to temporary library download archive %s", fname, outfname) raise exceptions.InternalServerError("Unable to create archive for download.") except ObjectNotFound: log.exception("Requested dataset %s does not exist on the host.", fpath) raise exceptions.ObjectNotFound("Requested dataset not found.") except Exception as e: log.exception("Unable to add %s to temporary library download archive %s", fname, outfname) raise exceptions.InternalServerError(f"Unable to add dataset to temporary library download archive . {util.unicodify(e)}") else: try: archive.write(ldda.dataset.file_name, path) except OSError: log.exception("Unable to write %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError("Unable to create archive for download") except ObjectNotFound: log.exception("Requested dataset %s does not exist on the host.", ldda.dataset.file_name) raise exceptions.ObjectNotFound("Requested dataset not found.") except Exception as e: log.exception("Unable to add %s to temporary library download archive %s", ldda.dataset.file_name, outfname) raise exceptions.InternalServerError(f"Unknown error. {util.unicodify(e)}") trans.response.headers.update(archive.get_headers()) return archive.response() elif archive_format == 'uncompressed': if len(library_datasets) != 1: raise exceptions.RequestParameterInvalidException("You can download only one uncompressed file at once.") else: single_ld = library_datasets[0] ldda = single_ld.library_dataset_dataset_association dataset = ldda.dataset fStat = os.stat(dataset.file_name) trans.response.set_content_type(ldda.get_mime()) trans.response.headers['Content-Length'] = str(fStat.st_size) fname = f"{ldda.name}.{ldda.extension}" fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in fname)[0:150] trans.response.headers["Content-Disposition"] = f'attachment; filename="{fname}"' try: return open(dataset.file_name, 'rb') except Exception: raise exceptions.InternalServerError("This dataset contains no content.") else: raise exceptions.RequestParameterInvalidException("Wrong archive_format parameter specified")
def download(self, trans, format, **kwd): """ Download requested datasets (identified by encoded IDs) in requested format. * GET /api/libraries/datasets/download/{format} * POST /api/libraries/datasets/download/{format} example: ``GET localhost:8080/api/libraries/datasets/download/tbz?ld_ids%255B%255D=a0d84b45643a2678&ld_ids%255B%255D=fe38c84dcd46c828`` .. note:: supported format values are: 'zip', 'tgz', 'tbz', 'uncompressed' :param format: string representing requested archive format :type format: string :param ld_ids[]: an array of encoded dataset ids :type ld_ids[]: an array :param folder_ids[]: an array of encoded folder ids :type folder_ids[]: an array :returns: either archive with the requested datasets packed inside or a single uncompressed dataset :rtype: file :raises: MessageException, ItemDeletionException, ItemAccessibilityException, HTTPBadRequest, OSError, IOError, ObjectNotFound """ library_datasets = [] datasets_to_download = kwd.get('ld_ids%5B%5D', None) if datasets_to_download is None: datasets_to_download = kwd.get('ld_ids', None) if datasets_to_download is not None: datasets_to_download = util.listify(datasets_to_download) for dataset_id in datasets_to_download: try: library_dataset = self.get_library_dataset(trans, id=dataset_id, check_ownership=False, check_accessible=True) library_datasets.append(library_dataset) except HTTPBadRequest: raise exceptions.RequestParameterInvalidException('Bad Request.') except HTTPInternalServerError: raise exceptions.InternalServerError('Internal error.') except Exception as e: raise exceptions.InternalServerError('Unknown error.' + str(e)) folders_to_download = kwd.get('folder_ids%5B%5D', None) if folders_to_download is None: folders_to_download = kwd.get('folder_ids', None) if folders_to_download is not None: folders_to_download = util.listify(folders_to_download) current_user_roles = trans.get_current_user_roles() def traverse(folder): admin = trans.user_is_admin() rval = [] for subfolder in folder.active_folders: if not admin: can_access, folder_ids = trans.app.security_agent.check_folder_contents(trans.user, current_user_roles, subfolder) if (admin or can_access) and not subfolder.deleted: rval.extend(traverse(subfolder)) for ld in folder.datasets: if not admin: can_access = trans.app.security_agent.can_access_dataset( current_user_roles, ld.library_dataset_dataset_association.dataset ) if (admin or can_access) and not ld.deleted: rval.append(ld) return rval for encoded_folder_id in folders_to_download: folder_id = self.folder_manager.cut_and_decode(trans, encoded_folder_id) folder = self.folder_manager.get(trans, folder_id) library_datasets.extend(traverse(folder)) if not library_datasets: raise exceptions.RequestParameterMissingException('Request has to contain a list of dataset ids or folder ids to download.') if format in ['zip', 'tgz', 'tbz']: # error = False killme = string.punctuation + string.whitespace trantab = string.maketrans(killme, '_' * len(killme)) try: outext = 'zip' if format == 'zip': # Can't use mkstemp - the file must not exist first tmpd = tempfile.mkdtemp() util.umask_fix_perms(tmpd, trans.app.config.umask, 0777, self.app.config.gid) tmpf = os.path.join(tmpd, 'library_download.' + format) if trans.app.config.upstream_gzip: archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_STORED, True) else: archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_DEFLATED, True) archive.add = lambda x, y: archive.write(x, y.encode('CP437')) elif format == 'tgz': if trans.app.config.upstream_gzip: archive = StreamBall('w|') outext = 'tar' else: archive = StreamBall('w|gz') outext = 'tgz' elif format == 'tbz': archive = StreamBall('w|bz2') outext = 'tbz2' except (OSError, zipfile.BadZipfile): log.exception("Unable to create archive for download") raise exceptions.InternalServerError("Unable to create archive for download.") except Exception: log.exception("Unexpected error in create archive for download") raise exceptions.InternalServerError("Unable to create archive for download.") composite_extensions = trans.app.datatypes_registry.get_composite_extensions() seen = [] for ld in library_datasets: ldda = ld.library_dataset_dataset_association ext = ldda.extension is_composite = ext in composite_extensions path = "" parent_folder = ldda.library_dataset.folder while parent_folder is not None: # Exclude the now-hidden "root folder" if parent_folder.parent is None: path = os.path.join(parent_folder.library_root[0].name, path) break path = os.path.join(parent_folder.name, path) parent_folder = parent_folder.parent path += ldda.name while path in seen: path += '_' seen.append(path) zpath = os.path.split(path)[-1] # comes as base_name/fname outfname, zpathext = os.path.splitext(zpath) if is_composite: # need to add all the components from the extra_files_path to the zip if zpathext == '': zpath = '%s.html' % zpath # fake the real nature of the html file try: if format == 'zip': archive.add(ldda.dataset.file_name, zpath) # add the primary of a composite set else: archive.add(ldda.dataset.file_name, zpath, check_file=True) # add the primary of a composite set except IOError: log.exception("Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError("Unable to create archive for download.") except ObjectNotFound: log.exception("Requested dataset %s does not exist on the host.", ldda.dataset.file_name) raise exceptions.ObjectNotFound("Requested dataset not found. ") except Exception as e: log.exception("Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError("Unable to add composite parent to temporary library download archive. " + str(e)) flist = glob.glob(os.path.join(ldda.dataset.extra_files_path, '*.*')) # glob returns full paths for fpath in flist: efp, fname = os.path.split(fpath) if fname > '': fname = fname.translate(trantab) try: if format == 'zip': archive.add(fpath, fname) else: archive.add(fpath, fname, check_file=True) except IOError: log.exception("Unable to add %s to temporary library download archive %s", fname, outfname) raise exceptions.InternalServerError("Unable to create archive for download.") except ObjectNotFound: log.exception("Requested dataset %s does not exist on the host.", fpath) raise exceptions.ObjectNotFound("Requested dataset not found.") except Exception as e: log.exception("Unable to add %s to temporary library download archive %s" % (fname, outfname)) raise exceptions.InternalServerError("Unable to add dataset to temporary library download archive . " + str(e)) else: try: if format == 'zip': archive.add(ldda.dataset.file_name, path) else: archive.add(ldda.dataset.file_name, path, check_file=True) except IOError: log.exception("Unable to write %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError("Unable to create archive for download") except ObjectNotFound: log.exception("Requested dataset %s does not exist on the host.", ldda.dataset.file_name) raise exceptions.ObjectNotFound("Requested dataset not found.") except Exception as e: log.exception("Unable to add %s to temporary library download archive %s", fname, outfname) raise exceptions.InternalServerError("Unknown error. " + str(e)) lname = 'selected_dataset' fname = lname.replace(' ', '_') + '_files' if format == 'zip': archive.close() trans.response.set_content_type("application/octet-stream") trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.%s"' % (fname, outext) archive = util.streamball.ZipBall(tmpf, tmpd) archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream else: trans.response.set_content_type("application/x-tar") trans.response.headers["Content-Disposition"] = 'attachment; filename="%s.%s"' % (fname, outext) archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream elif format == 'uncompressed': if len(library_datasets) != 1: raise exceptions.RequestParameterInvalidException("You can download only one uncompressed file at once.") else: single_ld = library_datasets[0] ldda = single_ld.library_dataset_dataset_association dataset = ldda.dataset fStat = os.stat(dataset.file_name) trans.response.set_content_type(ldda.get_mime()) trans.response.headers['Content-Length'] = int(fStat.st_size) fname = ldda.name fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in fname)[0:150] trans.response.headers["Content-Disposition"] = 'attachment; filename="%s"' % fname try: return open(dataset.file_name) except: raise exceptions.InternalServerError("This dataset contains no content.") else: raise exceptions.RequestParameterInvalidException("Wrong format parameter specified")
def download(self, trans, format, **kwd): """ GET /api/libraries/datasets/download/{format} POST /api/libraries/datasets/download/{format} Download requested datasets (identified by encoded IDs) in requested format. example: ``GET localhost:8080/api/libraries/datasets/download/tbz?ld_ids%255B%255D=a0d84b45643a2678&ld_ids%255B%255D=fe38c84dcd46c828`` .. note:: supported format values are: 'zip', 'tgz', 'tbz', 'uncompressed' :param format: string representing requested archive format :type format: string :param ld_ids[]: an array of encoded dataset ids :type ld_ids[]: an array :param folder_ids[]: an array of encoded folder ids :type folder_ids[]: an array :returns: either archive with the requested datasets packed inside or a single uncompressed dataset :rtype: file :raises: MessageException, ItemDeletionException, ItemAccessibilityException, HTTPBadRequest, OSError, IOError, ObjectNotFound """ library_datasets = [] datasets_to_download = kwd.get('ld_ids%5B%5D', None) if datasets_to_download is None: datasets_to_download = kwd.get('ld_ids', None) if datasets_to_download is not None: datasets_to_download = util.listify(datasets_to_download) for dataset_id in datasets_to_download: try: library_dataset = self.get_library_dataset( trans, id=dataset_id, check_ownership=False, check_accessible=True) library_datasets.append(library_dataset) except HTTPBadRequest: raise exceptions.RequestParameterInvalidException( 'Bad Request.') except HTTPInternalServerError: raise exceptions.InternalServerError('Internal error.') except Exception as e: raise exceptions.InternalServerError('Unknown error.' + util.unicodify(e)) folders_to_download = kwd.get('folder_ids%5B%5D', None) if folders_to_download is None: folders_to_download = kwd.get('folder_ids', None) if folders_to_download is not None: folders_to_download = util.listify(folders_to_download) current_user_roles = trans.get_current_user_roles() def traverse(folder): admin = trans.user_is_admin rval = [] for subfolder in folder.active_folders: if not admin: can_access, folder_ids = trans.app.security_agent.check_folder_contents( trans.user, current_user_roles, subfolder) if (admin or can_access) and not subfolder.deleted: rval.extend(traverse(subfolder)) for ld in folder.datasets: if not admin: can_access = trans.app.security_agent.can_access_dataset( current_user_roles, ld.library_dataset_dataset_association.dataset) if (admin or can_access) and not ld.deleted: rval.append(ld) return rval for encoded_folder_id in folders_to_download: folder_id = self.folder_manager.cut_and_decode( trans, encoded_folder_id) folder = self.folder_manager.get(trans, folder_id) library_datasets.extend(traverse(folder)) if not library_datasets: raise exceptions.RequestParameterMissingException( 'Request has to contain a list of dataset ids or folder ids to download.' ) if format in ['zip', 'tgz', 'tbz']: # error = False killme = string.punctuation + string.whitespace trantab = string.maketrans(killme, '_' * len(killme)) try: outext = 'zip' if format == 'zip': # Can't use mkstemp - the file must not exist first tmpd = tempfile.mkdtemp() util.umask_fix_perms(tmpd, trans.app.config.umask, 0o777, self.app.config.gid) tmpf = os.path.join(tmpd, 'library_download.' + format) if trans.app.config.upstream_gzip: archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_STORED, True) else: archive = zipfile.ZipFile(tmpf, 'w', zipfile.ZIP_DEFLATED, True) archive.add = lambda x, y: archive.write( x, y.encode('CP437')) elif format == 'tgz': if trans.app.config.upstream_gzip: archive = StreamBall('w|') outext = 'tar' else: archive = StreamBall('w|gz') outext = 'tgz' elif format == 'tbz': archive = StreamBall('w|bz2') outext = 'tbz2' except (OSError, zipfile.BadZipfile): log.exception("Unable to create archive for download") raise exceptions.InternalServerError( "Unable to create archive for download.") except Exception: log.exception( "Unexpected error in create archive for download") raise exceptions.InternalServerError( "Unable to create archive for download.") composite_extensions = trans.app.datatypes_registry.get_composite_extensions( ) seen = [] for ld in library_datasets: ldda = ld.library_dataset_dataset_association ext = ldda.extension is_composite = ext in composite_extensions path = "" parent_folder = ldda.library_dataset.folder while parent_folder is not None: # Exclude the now-hidden "root folder" if parent_folder.parent is None: path = os.path.join(parent_folder.library_root[0].name, path) break path = os.path.join(parent_folder.name, path) parent_folder = parent_folder.parent path += ldda.name while path in seen: path += '_' path = "{path}.{extension}".format(path=path, extension=ldda.extension) seen.append(path) zpath = os.path.split(path)[-1] # comes as base_name/fname outfname, zpathext = os.path.splitext(zpath) if is_composite: # need to add all the components from the extra_files_path to the zip if zpathext == '': zpath = '%s.html' % zpath # fake the real nature of the html file try: if format == 'zip': archive.add( ldda.dataset.file_name, zpath) # add the primary of a composite set else: archive.add(ldda.dataset.file_name, zpath, check_file=True ) # add the primary of a composite set except IOError: log.exception( "Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError( "Unable to create archive for download.") except ObjectNotFound: log.exception( "Requested dataset %s does not exist on the host.", ldda.dataset.file_name) raise exceptions.ObjectNotFound( "Requested dataset not found. ") except Exception as e: log.exception( "Unable to add composite parent %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError( "Unable to add composite parent to temporary library download archive. " + util.unicodify(e)) flist = glob.glob( os.path.join(ldda.dataset.extra_files_path, '*.*')) # glob returns full paths for fpath in flist: efp, fname = os.path.split(fpath) if fname > '': fname = fname.translate(trantab) try: if format == 'zip': archive.add(fpath, fname) else: archive.add(fpath, fname, check_file=True) except IOError: log.exception( "Unable to add %s to temporary library download archive %s", fname, outfname) raise exceptions.InternalServerError( "Unable to create archive for download.") except ObjectNotFound: log.exception( "Requested dataset %s does not exist on the host.", fpath) raise exceptions.ObjectNotFound( "Requested dataset not found.") except Exception as e: log.exception( "Unable to add %s to temporary library download archive %s", fname, outfname) raise exceptions.InternalServerError( "Unable to add dataset to temporary library download archive . " + util.unicodify(e)) else: try: if format == 'zip': archive.add(ldda.dataset.file_name, path) else: archive.add(ldda.dataset.file_name, path, check_file=True) except IOError: log.exception( "Unable to write %s to temporary library download archive", ldda.dataset.file_name) raise exceptions.InternalServerError( "Unable to create archive for download") except ObjectNotFound: log.exception( "Requested dataset %s does not exist on the host.", ldda.dataset.file_name) raise exceptions.ObjectNotFound( "Requested dataset not found.") except Exception as e: log.exception( "Unable to add %s to temporary library download archive %s", ldda.dataset.file_name, outfname) raise exceptions.InternalServerError( "Unknown error. " + util.unicodify(e)) lname = 'selected_dataset' fname = lname.replace(' ', '_') + '_files' if format == 'zip': archive.close() trans.response.set_content_type("application/octet-stream") trans.response.headers[ "Content-Disposition"] = 'attachment; filename="%s.%s"' % ( fname, outext) archive = util.streamball.ZipBall(tmpf, tmpd) archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream else: trans.response.set_content_type("application/x-tar") trans.response.headers[ "Content-Disposition"] = 'attachment; filename="%s.%s"' % ( fname, outext) archive.wsgi_status = trans.response.wsgi_status() archive.wsgi_headeritems = trans.response.wsgi_headeritems() return archive.stream elif format == 'uncompressed': if len(library_datasets) != 1: raise exceptions.RequestParameterInvalidException( "You can download only one uncompressed file at once.") else: single_ld = library_datasets[0] ldda = single_ld.library_dataset_dataset_association dataset = ldda.dataset fStat = os.stat(dataset.file_name) trans.response.set_content_type(ldda.get_mime()) trans.response.headers['Content-Length'] = int(fStat.st_size) fname = "{path}.{extension}".format(path=ldda.name, extension=ldda.extension) fname = ''.join(c in util.FILENAME_VALID_CHARS and c or '_' for c in fname)[0:150] trans.response.headers[ "Content-Disposition"] = 'attachment; filename="%s"' % fname try: return open(dataset.file_name, 'rb') except Exception: raise exceptions.InternalServerError( "This dataset contains no content.") else: raise exceptions.RequestParameterInvalidException( "Wrong format parameter specified")