def _upload_application_zip(self, app: Entity, path: str): _logger.debug('Unzipping file %s', path) tmp_dir = tempfile.mkdtemp() try: FileHelper.unzip(path, tmp_dir) self._upload_application_directory(app, tmp_dir) finally: shutil.rmtree(tmp_dir)
def _load_all_resources(path): application_items = {} for directory, file_names in FileHelper.walk(path): for file_name in file_names: file_location = os.path.join(path, directory, file_name) application_items[os.path.join(directory, file_name)] = dict( sha1=FileHelper.sha1(file_location), size=FileHelper.size(file_location), mode=FileHelper.mode(file_location)) return application_items
def _load_all_resources(top_directory: str) -> dict: application_items = {} cf_ignore = CfIgnore(top_directory) for directory, file_names in FileHelper.walk(top_directory): for file_name in file_names: relative_file_location = os.path.join(directory, file_name) if not cf_ignore.is_entry_ignored(relative_file_location): absolute_file_location = os.path.join( top_directory, relative_file_location) application_items[relative_file_location] = dict( sha1=FileHelper.sha1(absolute_file_location), size=FileHelper.size(absolute_file_location), mode=FileHelper.mode(absolute_file_location)) return application_items
def _upload_application_directory(self, app: Entity, application_path: str): _logger.debug("Uploading application from directory %s", application_path) _, temp_file = tempfile.mkstemp() try: resource_descriptions_by_path = PushOperation._load_all_resources( application_path) def generate_key(item: dict): return "%s-%d" % (item["sha1"], item["size"]) already_uploaded_entries = [ generate_key(item) for item in self.client.v2.resources.match([ dict(sha1=item["sha1"], size=item["size"]) for item in resource_descriptions_by_path.values() ]) ] _logger.debug("Already uploaded %d / %d items", len(already_uploaded_entries), len(resource_descriptions_by_path)) FileHelper.zip( temp_file, application_path, lambda item: item in resource_descriptions_by_path and generate_key(resource_descriptions_by_path[item] ) not in already_uploaded_entries, ) _logger.debug("Diff zip file built: %s", temp_file) resources = [ dict( fn=resource_path, sha1=resource_description["sha1"], size=resource_description["size"], mode=resource_description["mode"], ) for resource_path, resource_description in resource_descriptions_by_path.items() if generate_key( resource_description) in already_uploaded_entries ] _logger.debug("Uploading bits of application") job = self.client.v2.apps.upload(app["metadata"]["guid"], resources, temp_file, True) self._poll_job(job) finally: _logger.debug("Skipping remove of zip file")
def _upload_application_directory(self, app, path): _logger.debug('Uploading application from directory %s', path) _, temp_file = tempfile.mkstemp() try: resource_descriptions_by_path = PushOperation._load_all_resources( path) def generate_key(item): return '%s-%d' % (item["sha1"], item["size"]) already_uploaded_entries = [ generate_key(item) for item in self.client.v2.resources.match([ dict(sha1=item["sha1"], size=item["size"]) for item in resource_descriptions_by_path.values() ]) ] _logger.debug('Already uploaded %d / %d items', len(already_uploaded_entries), len(resource_descriptions_by_path)) FileHelper.zip( temp_file, path, lambda item: generate_key(resource_descriptions_by_path[item] ) not in already_uploaded_entries) _logger.debug('Diff zip file built: %s', temp_file) resources = [ dict(fn=resource_path, sha1=resource_description["sha1"], size=resource_description["size"], mode=resource_description["mode"]) for resource_path, resource_description in resource_descriptions_by_path.items() if generate_key( resource_description) in already_uploaded_entries ] _logger.debug('Uploading bits of application') job = self.client.v2.apps.upload(app['metadata']['guid'], resources, temp_file, True) self._poll_job(job) finally: _logger.debug('Skipping remove of zip file')
def unzip(self): FileHelper.unzip(os.path.join(self.input_dirpath, 'myzip.zip'), self.output_dirpath)