def upload_context(self, context_filename):
gcs_uploader = gcp.GCSUploader()
context_hash = utils.crc(context_filename)
return gcs_uploader.upload_to_bucket(bucket_name=self.gcp_project,
blob_name='fairing_builds/' +
context_hash,
file_to_upload=context_filename)
def context_tar_gz(self, output_file=constants.DEFAULT_CONTEXT_FILENAME):
self.input_files = self.preprocess()
with tarfile.open(output_file, "w:gz") as tar:
for src, dst in self.context_map().items():
tar.add(src, filter=reset_tar_mtime, arcname=dst)
self._context_tar_path = output_file
return output_file, utils.crc(self._context_tar_path)
def upload_context(self, context_filename):
s3_uploader = aws.S3Uploader(self.region)
context_hash = utils.crc(context_filename)
bucket_name = self.bucket_name or 'kubeflow-' + self.aws_account + '-' + self.region
return s3_uploader.upload_to_bucket(bucket_name=bucket_name,
blob_name='fairing_builds/' +
context_hash,
file_to_upload=context_filename)
def upload_context(self, context_filename):
azure_uploader = azure.AzureUploader(self.region)
context_hash = utils.crc(context_filename)
# TODO ME find out what's happening with the return value
return azure_uploader.upload_to_container(
self.region,
self.storage_account_name,
self.container_name,
self.group_name,
blob_name='fairing_builds/' + context_hash,
file_to_upload=context_filename)
def context_tar_gz(self, output_file=None):
if not output_file:
_, output_file = tempfile.mkstemp(prefix="/tmp/fairing_context_")
logging.info("Creating docker context: %s", output_file)
self.input_files = self.preprocess()
with tarfile.open(output_file, "w:gz", dereference=True) as tar:
for dst, src in self.context_map().items():
logging.debug("Context: %s, Adding %s at %s", output_file,
src, dst)
tar.add(src, filter=reset_tar_mtime, arcname=dst, recursive=False)
self._context_tar_path = output_file
return output_file, utils.crc(self._context_tar_path)
def context_tar_gz(self, output_file=constants.DEFAULT_CONTEXT_FILENAME):
logging.info("Creating docker context: %s", output_file)
self.input_files = self.preprocess()
with tarfile.open(output_file, "w:gz", dereference=True) as tar:
for dst, src in self.context_map().items():
logging.info("Context: %s, Adding %s at %s", output_file, src,
dst)
tar.add(src,
filter=reset_tar_mtime,
arcname=dst,
recursive=False)
self._context_tar_path = output_file
return output_file, utils.crc(self._context_tar_path)
def prepare(self, context_filename): # pylint:disable=arguments-differ
self.context_hash = utils.crc(context_filename)
self.context_path = self.upload_context(context_filename)
