Beispiel #1
0
        def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
            """
            Upload |source_file| (a stream) to |worksheet_uuid|.
            """
            # Construct info for creating the bundle.
            bundle_subclass = get_bundle_subclass(
                bundle_type)  # program or data
            metadata = metadata_util.fill_missing_metadata(
                bundle_subclass, {},
                initial_metadata={
                    'name': source_file.name,
                    'description': 'Upload ' + source_file.name
                })
            info = {'bundle_type': bundle_type, 'metadata': metadata}

            # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
            remote_file_uuid = self.client.open_temp_file()
            dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
            file_util.copy(source_file,
                           dest,
                           autoflush=False,
                           print_status='Uploading %s' %
                           info['metadata']['name'])
            dest.close()

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.upload_bundle_zip(
                remote_file_uuid, info, worksheet_uuid, False, True)
            return new_bundle_uuid
    def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks, exclude_patterns, add_to_worksheet):
        # URLs can be directly passed to the local client.
        if path and not isinstance(path, list) and path_util.path_is_url(path):
            return self.upload_bundle_url(path, info, worksheet_uuid, follow_symlinks, exclude_patterns)

        # First, zip path up (temporary local zip file).
        if path:
            name = info['metadata']['name']
            zip_path = zip_util.zip(path, follow_symlinks=follow_symlinks, exclude_patterns=exclude_patterns, file_name=name)
            # Copy it up to the server (temporary remote zip file)
            with open(zip_path, 'rb') as source:
                remote_file_uuid = self.open_temp_file()
                dest = RPCFileHandle(remote_file_uuid, self.proxy)
                # FileServer does not expose an API for forcibly flushing writes, so
                # we rely on closing the file to flush it.
                file_util.copy(source, dest, autoflush=False, print_status='Uploading %s%s to %s' % (zip_path, ' ('+info['uuid']+')' if 'uuid' in info else '', self.address))
                dest.close()
        else:
            remote_file_uuid = None
            zip_path = None

        # Finally, install the zip file (this will be in charge of deleting that zip file).
        result = self.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, follow_symlinks, add_to_worksheet)

        if zip_path:
            path_util.remove(zip_path)  # Remove local zip
        return result
    def upload_bundle(self, sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet):
        """
        See local_bundle_client.py for documentation on the usage.
        Strategy:
        1) We copy the |sources| to a temporary directory on the server
          (streaming either a tar or tar.gz depending on whether compression is
          needed).
        2) We politely ask the server to finish_upload_bundle (performs a
          LocalBundleClient.upload_bundle from the temporary directory).
        """
        # URLs can be directly passed to the local client.
        if all(path_util.path_is_url(source) for source in sources):
            return self.upload_bundle_url(sources, follow_symlinks, exclude_patterns, git, unpack, remove_sources, info, worksheet_uuid, add_to_worksheet)

        # 1) Copy sources up to the server (temporary remote zip file)
        remote_file_uuids = []
        for source in sources:
            remote_file_uuid = self.open_temp_file(zip_util.add_packed_suffix(os.path.basename(source)))
            remote_file_uuids.append(remote_file_uuid)
            dest_handle = RPCFileHandle(remote_file_uuid, self.proxy)
            if zip_util.path_is_archive(source):
                source_handle = open(source)
            else:
                source_handle = zip_util.open_packed_path(source, follow_symlinks, exclude_patterns)
                unpack = True  # We packed it, so we have to unpack it
            status = 'Uploading %s%s to %s' % (source, ' ('+info['uuid']+')' if 'uuid' in info else '', self.address)
            # FileServer does not expose an API for forcibly flushing writes, so
            # we rely on closing the file to flush it.
            file_util.copy(source_handle, dest_handle, autoflush=False, print_status=status)
            dest_handle.close()

        # 2) Install upload (this call will be in charge of deleting the temporary file).
        result = self.finish_upload_bundle(remote_file_uuids, unpack, info, worksheet_uuid, add_to_worksheet)

        return result
Beispiel #4
0
    def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
        """
        Upload |source_file| (a stream) to |worksheet_uuid|.
        """
        # Construct info for creating the bundle.
        bundle_subclass = get_bundle_subclass(bundle_type) # program or data
        metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
        info = {'bundle_type': bundle_type, 'metadata': metadata}

        # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
        remote_file_uuid = self.client.open_temp_file(metadata['name'])
        dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
        file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % metadata['name'])
        dest.close()

        pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
        if not pack and zip_util.path_is_archive(metadata['name']):
            metadata['name'] = zip_util.strip_archive_ext(metadata['name'])

        # Then tell the client that the uploaded file handle is there.
        new_bundle_uuid = self.client.finish_upload_bundle(
            [remote_file_uuid],
            not pack,  # unpack
            info,
            worksheet_uuid,
            True)  # add_to_worksheet
        return new_bundle_uuid
    def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks):
        # URLs can be directly passed to the local client.
        if path and not isinstance(path, list) and path_util.path_is_url(path):
            return self.upload_bundle_url(path, info, worksheet_uuid, follow_symlinks)

        # First, zip path up (temporary local zip file).
        if path:
            zip_path, sub_path = zip_util.zip(path, follow_symlinks=follow_symlinks)
            # Copy it up to the server (temporary remote zip file)
            with open(zip_path, 'rb') as source:
                remote_file_uuid = self.open_temp_file()
                dest = RPCFileHandle(remote_file_uuid, self.proxy)
                # FileServer does not expose an API for forcibly flushing writes, so
                # we rely on closing the file to flush it.
                file_util.copy(source, dest, autoflush=False, print_status=True)
                dest.close()
        else:
            remote_file_uuid = None
            zip_path = None

        # Finally, install the zip file (this will be in charge of deleting that zip file).
        result = self.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, follow_symlinks)

        if zip_path:
            path_util.remove(zip_path)  # Remove local zip
        return result
    def copy_bundle(self, source_bundle_uuid, info, dest_client,
                    dest_worksheet_uuid, add_to_worksheet):
        '''
        A streamlined combination of download_target and upload_bundle.
        Copy from self to dest_client.
        '''
        # Open source
        source_file_uuid, name = self.open_target_zip((source_bundle_uuid, ''),
                                                      False)
        source = RPCFileHandle(source_file_uuid, self.proxy)
        # Open target
        dest_file_uuid = dest_client.open_temp_file()
        dest = RPCFileHandle(dest_file_uuid, dest_client.proxy)

        # Copy contents over
        file_util.copy(source,
                       dest,
                       autoflush=False,
                       print_status='Copying %s from %s to %s' %
                       (source_bundle_uuid, self.address, dest_client.address))
        dest.close()
        # Finally, install the zip file (this will be in charge of deleting that zip file).
        result = dest_client.upload_bundle_zip(dest_file_uuid, info,
                                               dest_worksheet_uuid, False,
                                               add_to_worksheet)

        self.finalize_file(source_file_uuid, True)  # Delete remote zip file
        return result
Beispiel #7
0
        def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
            '''
            Upload |source_file| (a stream) to |worksheet_uuid|.
            '''
            # Construct info for creating the bundle.
            bundle_subclass = get_bundle_subclass(bundle_type) # program or data
            metadata = metadata_util.fill_missing_metadata(bundle_subclass, {}, initial_metadata={'name': source_file.name, 'description': 'Upload ' + source_file.name})
            info = {'bundle_type': bundle_type, 'metadata': metadata}

            # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
            remote_file_uuid = self.client.open_temp_file()
            dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
            file_util.copy(source_file, dest, autoflush=False, print_status='Uploading %s' % info['metadata']['name'])
            dest.close()

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid, False, True)
            return new_bundle_uuid
    def copy_bundle(self, source_bundle_uuid, info, dest_client, dest_worksheet_uuid, add_to_worksheet):
        '''
        A streamlined combination of download_target and upload_bundle.
        Copy from self to dest_client.
        '''
        # Open source
        source_file_uuid, name = self.open_target_zip((source_bundle_uuid, ''), False)
        source = RPCFileHandle(source_file_uuid, self.proxy)
        # Open target
        dest_file_uuid = dest_client.open_temp_file()
        dest = RPCFileHandle(dest_file_uuid, dest_client.proxy)

        # Copy contents over
        file_util.copy(source, dest, autoflush=False, print_status='Copying %s from %s to %s' % (source_bundle_uuid, self.address, dest_client.address))
        dest.close()
        # Finally, install the zip file (this will be in charge of deleting that zip file).
        result = dest_client.upload_bundle_zip(dest_file_uuid, info, dest_worksheet_uuid, False, add_to_worksheet)

        self.finalize_file(source_file_uuid, True)  # Delete remote zip file
        return result
    def upload_bundle(self, path, info, worksheet_uuid, follow_symlinks,
                      exclude_patterns, add_to_worksheet):
        # URLs can be directly passed to the local client.
        if path and not isinstance(path, list) and path_util.path_is_url(path):
            return self.upload_bundle_url(path, info, worksheet_uuid,
                                          follow_symlinks, exclude_patterns)

        # First, zip path up (temporary local zip file).
        if path:
            name = info['metadata']['name']
            zip_path = zip_util.zip(path,
                                    follow_symlinks=follow_symlinks,
                                    exclude_patterns=exclude_patterns,
                                    file_name=name)
            # Copy it up to the server (temporary remote zip file)
            with open(zip_path, 'rb') as source:
                remote_file_uuid = self.open_temp_file()
                dest = RPCFileHandle(remote_file_uuid, self.proxy)
                # FileServer does not expose an API for forcibly flushing writes, so
                # we rely on closing the file to flush it.
                file_util.copy(source,
                               dest,
                               autoflush=False,
                               print_status='Uploading %s%s to %s' %
                               (zip_path, ' (' + info['uuid'] +
                                ')' if 'uuid' in info else '', self.address))
                dest.close()
        else:
            remote_file_uuid = None
            zip_path = None

        # Finally, install the zip file (this will be in charge of deleting that zip file).
        result = self.upload_bundle_zip(remote_file_uuid, info, worksheet_uuid,
                                        follow_symlinks, add_to_worksheet)

        if zip_path:
            path_util.remove(zip_path)  # Remove local zip
        return result
Beispiel #10
0
        def upload_bundle(self, source_file, bundle_type, worksheet_uuid):
            """
            Upload |source_file| (a stream) to |worksheet_uuid|.
            """
            # Construct info for creating the bundle.
            bundle_subclass = get_bundle_subclass(
                bundle_type)  # program or data
            metadata = metadata_util.fill_missing_metadata(
                bundle_subclass, {},
                initial_metadata={
                    'name': source_file.name,
                    'description': 'Upload ' + source_file.name
                })
            info = {'bundle_type': bundle_type, 'metadata': metadata}

            # Upload it by creating a file handle and copying source_file to it (see RemoteBundleClient.upload_bundle in the CLI).
            remote_file_uuid = self.client.open_temp_file(metadata['name'])
            dest = RPCFileHandle(remote_file_uuid, self.client.proxy)
            file_util.copy(source_file,
                           dest,
                           autoflush=False,
                           print_status='Uploading %s' % metadata['name'])
            dest.close()

            pack = False  # For now, always unpack (note: do this after set remote_file_uuid, which needs the extension)
            if not pack and zip_util.path_is_archive(metadata['name']):
                metadata['name'] = zip_util.strip_archive_ext(metadata['name'])

            # Then tell the client that the uploaded file handle is there.
            new_bundle_uuid = self.client.finish_upload_bundle(
                [remote_file_uuid],
                not pack,  # unpack
                info,
                worksheet_uuid,
                True)  # add_to_worksheet
            return new_bundle_uuid