def _upload_application_zip(self, app: Entity, path: str):
_logger.debug('Unzipping file %s', path)
tmp_dir = tempfile.mkdtemp()
try:
FileHelper.unzip(path, tmp_dir)
self._upload_application_directory(app, tmp_dir)
finally:
shutil.rmtree(tmp_dir)
def _load_all_resources(path):
application_items = {}
for directory, file_names in FileHelper.walk(path):
for file_name in file_names:
file_location = os.path.join(path, directory, file_name)
application_items[os.path.join(directory, file_name)] = dict(
sha1=FileHelper.sha1(file_location),
size=FileHelper.size(file_location),
mode=FileHelper.mode(file_location))
return application_items
def _load_all_resources(top_directory: str) -> dict:
application_items = {}
cf_ignore = CfIgnore(top_directory)
for directory, file_names in FileHelper.walk(top_directory):
for file_name in file_names:
relative_file_location = os.path.join(directory, file_name)
if not cf_ignore.is_entry_ignored(relative_file_location):
absolute_file_location = os.path.join(
top_directory, relative_file_location)
application_items[relative_file_location] = dict(
sha1=FileHelper.sha1(absolute_file_location),
size=FileHelper.size(absolute_file_location),
mode=FileHelper.mode(absolute_file_location))
return application_items
def _upload_application_directory(self, app: Entity,
application_path: str):
_logger.debug("Uploading application from directory %s",
application_path)
_, temp_file = tempfile.mkstemp()
try:
resource_descriptions_by_path = PushOperation._load_all_resources(
application_path)
def generate_key(item: dict):
return "%s-%d" % (item["sha1"], item["size"])
already_uploaded_entries = [
generate_key(item) for item in self.client.v2.resources.match([
dict(sha1=item["sha1"], size=item["size"])
for item in resource_descriptions_by_path.values()
])
]
_logger.debug("Already uploaded %d / %d items",
len(already_uploaded_entries),
len(resource_descriptions_by_path))
FileHelper.zip(
temp_file,
application_path,
lambda item: item in resource_descriptions_by_path and
generate_key(resource_descriptions_by_path[item]
) not in already_uploaded_entries,
)
_logger.debug("Diff zip file built: %s", temp_file)
resources = [
dict(
fn=resource_path,
sha1=resource_description["sha1"],
size=resource_description["size"],
mode=resource_description["mode"],
) for resource_path, resource_description in
resource_descriptions_by_path.items() if generate_key(
resource_description) in already_uploaded_entries
]
_logger.debug("Uploading bits of application")
job = self.client.v2.apps.upload(app["metadata"]["guid"],
resources, temp_file, True)
self._poll_job(job)
finally:
_logger.debug("Skipping remove of zip file")
def _upload_application_directory(self, app, path):
_logger.debug('Uploading application from directory %s', path)
_, temp_file = tempfile.mkstemp()
try:
resource_descriptions_by_path = PushOperation._load_all_resources(
path)
def generate_key(item):
return '%s-%d' % (item["sha1"], item["size"])
already_uploaded_entries = [
generate_key(item) for item in self.client.v2.resources.match([
dict(sha1=item["sha1"], size=item["size"])
for item in resource_descriptions_by_path.values()
])
]
_logger.debug('Already uploaded %d / %d items',
len(already_uploaded_entries),
len(resource_descriptions_by_path))
FileHelper.zip(
temp_file, path,
lambda item: generate_key(resource_descriptions_by_path[item]
) not in already_uploaded_entries)
_logger.debug('Diff zip file built: %s', temp_file)
resources = [
dict(fn=resource_path,
sha1=resource_description["sha1"],
size=resource_description["size"],
mode=resource_description["mode"]) for resource_path,
resource_description in resource_descriptions_by_path.items()
if generate_key(
resource_description) in already_uploaded_entries
]
_logger.debug('Uploading bits of application')
job = self.client.v2.apps.upload(app['metadata']['guid'],
resources, temp_file, True)
self._poll_job(job)
finally:
_logger.debug('Skipping remove of zip file')
def unzip(self):
FileHelper.unzip(os.path.join(self.input_dirpath, 'myzip.zip'),
self.output_dirpath)
