def _AlterRuntime(config_filename, runtime):
try:
# 0. Take backup
with tempfile.NamedTemporaryFile(prefix='app.yaml.') as f:
backup_fname = f.name
log.status.Print(
'Copying original config [{0}] to backup location [{1}].'.format(
config_filename, backup_fname))
shutil.copyfile(config_filename, backup_fname)
# 1. Open and parse file using ruamel
with files.FileReader(config_filename) as yaml_file:
encoding = yaml_file.encoding
config = yaml.load(yaml_file, yaml.RoundTripLoader)
# 2. Alter the ruamel in-memory object representing the yaml file
config['runtime'] = runtime
# 3. Create an in-memory file buffer and write yaml file to it
raw_buf = io.BytesIO()
tmp_yaml_buf = io.TextIOWrapper(raw_buf, encoding)
yaml.dump(config, tmp_yaml_buf, Dumper=yaml.RoundTripDumper)
# 4. Overwrite the original app.yaml
with files.BinaryFileWriter(config_filename) as yaml_file:
tmp_yaml_buf.seek(0)
yaml_file.write(raw_buf.getvalue())
except Exception as e:
raise fingerprinter.AlterConfigFileError(e)
def _ungzip_file(file_path):
"""Unzips gzip file."""
temporary_file_path = file_path + '.tmp'
with gzip.open(file_path, 'rb') as gzipped_file:
with files.BinaryFileWriter(temporary_file_path) as ungzipped_file:
shutil.copyfileobj(gzipped_file, ungzipped_file)
shutil.move(temporary_file_path, file_path)
def UploadSource(upload_dir, source_files, object_ref, gen_files=None):
"""Upload a gzipped tarball of the source directory to GCS.
Note: To provide parity with docker's behavior, we must respect .dockerignore.
Args:
upload_dir: the directory to be archived.
source_files: [str], relative paths to upload.
object_ref: storage_util.ObjectReference, the Cloud Storage location to
upload the source tarball to.
gen_files: dict of filename to (str) contents of generated config and
source context files.
"""
gen_files = gen_files or {}
dockerignore_contents = _GetDockerignoreExclusions(upload_dir, gen_files)
included_paths = _GetIncludedPaths(upload_dir, source_files,
dockerignore_contents)
# We can't use tempfile.NamedTemporaryFile here because ... Windows.
# See https://bugs.python.org/issue14243. There are small cleanup races
# during process termination that will leave artifacts on the filesystem.
# eg, CTRL-C on windows leaves both the directory and the file. Unavoidable.
# On Posix, `kill -9` has similar behavior, but CTRL-C allows cleanup.
with files.TemporaryDirectory() as temp_dir:
f = files.BinaryFileWriter(os.path.join(temp_dir, 'src.tgz'))
with gzip.GzipFile(mode='wb', fileobj=f) as gz:
_CreateTar(upload_dir, gen_files, included_paths, gz)
f.close()
storage_client = storage_api.StorageClient()
storage_client.CopyFileToGCS(object_ref.bucket_ref, f.name,
object_ref.name)
def decompress_gzip_if_necessary(source_resource,
gzipped_path,
destination_path,
do_not_decompress_flag=False):
"""Checks if file is elligible for decompression and decompresses if true.
Args:
source_resource (ObjectResource): May contain encoding metadata.
gzipped_path (str): File path to unzip.
destination_path (str): File path to write unzipped file to.
do_not_decompress_flag (bool): User flag that blocks decompression.
Returns:
(bool) True if file was successfully decompressed, else False.
"""
content_encoding = getattr(source_resource.metadata, 'contentEncoding', '')
if (do_not_decompress_flag or content_encoding is None
or 'gzip' not in content_encoding.split(',')):
return False
try:
with gzip.open(gzipped_path, 'rb') as gzipped_file:
with files.BinaryFileWriter(destination_path,
create_path=True) as ungzipped_file:
shutil.copyfileobj(gzipped_file, ungzipped_file)
return True
except OSError:
# May indicate trying to decompress non-gzipped file. Clean up.
os.remove(destination_path)
return False
def _perform_download(self, digesters, progress_callback,
download_strategy, start_byte, end_byte):
"""Prepares file stream, calls API, and validates hash."""
mode = (files.BinaryFileWriterMode.MODIFY
if start_byte else files.BinaryFileWriterMode.TRUNCATE)
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True,
mode=mode) as download_stream:
download_stream.seek(start_byte)
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
self._source_resource,
download_stream,
digesters=digesters,
download_strategy=download_strategy,
progress_callback=progress_callback,
start_byte=start_byte,
end_byte=end_byte)
# TODO(b/172048376): Add crc32c, and make this a loop.
if util.HashAlgorithms.MD5 in digesters:
calculated_digest = util.get_base64_hash_digest_string(
digesters[util.HashAlgorithms.MD5])
util.validate_object_hashes_match(
self._source_resource.storage_url,
self._source_resource.md5_hash, calculated_digest)
def _PossiblyWriteRedactedResponseToOutputFile(value, parsed_args):
"""Helper function for writing redacted contents to an output file."""
if not parsed_args.output_file:
return
with files.BinaryFileWriter(parsed_args.output_file) as outfile:
outfile.write(value)
log.status.Print('The redacted contents can be viewed in [{}]'.format(
parsed_args.output_file))
def execute(self, callback=None):
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True) as download_stream:
provider = self._source_resource.storage_url.scheme
bucket_name = self._source_resource.storage_url.bucket_name
object_name = self._source_resource.storage_url.object_name
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
bucket_name, object_name, download_stream)
def WriteToDisk(self):
"""Overwrite Original Yaml File."""
# Only write if file_path is specified.
if not self.file_path:
raise YamlConfigFileError('Could Not Write To Config File: Path Is Empty')
out_file_buf = io.BytesIO()
tmp_yaml_buf = io.TextIOWrapper(out_file_buf, newline='\n',
encoding='utf-8')
yaml.dump_all_round_trip([x.content for x in self.data],
stream=tmp_yaml_buf)
with files.BinaryFileWriter(self.file_path) as f:
tmp_yaml_buf.seek(0)
f.write(out_file_buf.getvalue())
def execute(self, callback=None):
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True) as download_stream:
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
self._source_resource, download_stream)
with files.BinaryFileReader(self._destination_resource.storage_url.
object_name) as completed_download_stream:
downloaded_file_hash = util.get_hash_digest_from_file_stream(
completed_download_stream, util.HashAlgorithms.MD5)
util.validate_object_hashes_match(
self._source_resource.storage_url,
self._source_resource.md5_hash, downloaded_file_hash)
def create_file_if_needed(source_resource, destination_resource):
"""Creates new file if none exists or one that is too large exists at path.
Args:
source_resource (ObjectResource): Contains size metadata for target file.
destination_resource(FileObjectResource|UnknownResource): Contains path to
create file at.
"""
file_path = destination_resource.storage_url.object_name
if os.path.exists(
file_path) and os.path.getsize(file_path) <= source_resource.size:
return
with files.BinaryFileWriter(file_path,
create_path=True,
mode=files.BinaryFileWriterMode.TRUNCATE):
# Wipe or create file.
pass
def Run(self, args):
holder = base_classes.ComputeApiHolder(self.ReleaseTrack())
instance_ref = self._GetInstanceRef(holder, args)
request = holder.client.messages.ComputeInstancesGetScreenshotRequest(
**instance_ref.AsDict())
response = holder.client.MakeRequests([
(holder.client.apitools_client.instances, 'GetScreenshot', request)
])[0]
self._display_file_output = False
if args.IsSpecified('destination'):
with files.BinaryFileWriter(args.destination) as output:
output.write(base64.b64decode(response.contents))
self._resource_name = instance_ref.instance
self._destination = args.destination
self._display_file_output = True
else:
self._response_contents = response.contents
return
def _perform_download(self, request_config, progress_callback,
download_strategy, start_byte, end_byte, write_mode,
digesters):
"""Prepares file stream, calls API, and validates hash."""
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True,
mode=write_mode) as download_stream:
download_stream.seek(start_byte)
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_download_result = api_factory.get_api(
provider).download_object(
self._source_resource,
download_stream,
request_config,
digesters=digesters,
do_not_decompress=self._do_not_decompress,
download_strategy=download_strategy,
progress_callback=progress_callback,
start_byte=start_byte,
end_byte=end_byte)
if hash_util.HashAlgorithm.MD5 in digesters:
calculated_digest = hash_util.get_base64_hash_digest_string(
digesters[hash_util.HashAlgorithm.MD5])
download_util.validate_download_hash_and_delete_corrupt_files(
self._destination_resource.storage_url.object_name,
self._source_resource.md5_hash, calculated_digest)
# Only for one-shot composite object downloads as final CRC32C validated in
# FinalizeSlicedDownloadTask.
elif (hash_util.HashAlgorithm.CRC32C in digesters
and self._component_number is None):
calculated_digest = crc32c.get_hash(
digesters[hash_util.HashAlgorithm.CRC32C])
download_util.validate_download_hash_and_delete_corrupt_files(
self._destination_resource.storage_url.object_name,
self._source_resource.crc32c_hash, calculated_digest)
return api_download_result
def execute(self, callback=None):
if self._source_resource.md5_hash:
digesters = {util.HashAlgorithms.MD5: util.get_md5_hash()}
else:
digesters = {}
with files.BinaryFileWriter(
self._destination_resource.storage_url.object_name,
create_path=True) as download_stream:
provider = self._source_resource.storage_url.scheme
# TODO(b/162264437): Support all of download_object's parameters.
api_factory.get_api(provider).download_object(
self._source_resource, download_stream, digesters=digesters)
# TODO(b/172048376): Add crc32c, and make this a loop.
if util.HashAlgorithms.MD5 in digesters:
calculated_digest = util.get_base64_hash_digest_string(
digesters[util.HashAlgorithms.MD5])
util.validate_object_hashes_match(
self._source_resource.storage_url,
self._source_resource.md5_hash, calculated_digest)
def EncryptedSSLCredentials(config_path):
"""Generates the encrypted client SSL credentials.
The encrypted client SSL credentials are stored in a file which is returned
along with the password.
Args:
config_path: path to the context aware configuration file.
Raises:
CertProvisionException: if the cert could not be provisioned.
ConfigException: if there is an issue in the context aware config.
Returns:
Tuple[str, bytes]: cert and key file path and passphrase bytes.
"""
try:
(
has_cert,
cert_bytes,
key_bytes,
passphrase_bytes
) = _mtls_helper.get_client_ssl_credentials(
generate_encrypted_key=True,
context_aware_metadata_path=config_path)
if has_cert:
cert_path = os.path.join(
config.Paths().global_config_dir, 'caa_cert.pem')
with files.BinaryFileWriter(cert_path) as f:
f.write(cert_bytes)
f.write(key_bytes)
return cert_path, passphrase_bytes
except google_auth_exceptions.ClientCertError as caught_exc:
new_exc = CertProvisionException(caught_exc)
six.raise_from(new_exc, caught_exc)
except files.Error as e:
log.debug('context aware settings discovery file %s - %s', config_path, e)
raise ConfigException()
def DownloadAndExtractTar(url, download_dir, extract_dir,
progress_callback=None, command_path='unknown'):
"""Download and extract the given tar file.
Args:
url: str, The URL to download.
download_dir: str, The path to put the temporary download file into.
extract_dir: str, The path to extract the tar into.
progress_callback: f(float), A function to call with the fraction of
completeness.
command_path: the command path to include in the User-Agent header if the
URL is HTTP
Returns:
[str], The files that were extracted from the tar file.
Raises:
URLFetchError: If there is a problem fetching the given URL.
"""
for d in [download_dir, extract_dir]:
if not os.path.exists(d):
file_utils.MakeDir(d)
download_file_path = os.path.join(download_dir, os.path.basename(url))
if os.path.exists(download_file_path):
os.remove(download_file_path)
(download_callback, install_callback) = (
console_io.SplitProgressBar(progress_callback, [1, 1]))
try:
req = ComponentInstaller.MakeRequest(url, command_path)
try:
total_size = float(req.info().get('Content-length', '0'))
# pylint: disable=broad-except, We never want progress bars to block an
# update.
except Exception:
total_size = 0
with file_utils.BinaryFileWriter(download_file_path) as fp:
# This is the buffer size that shutil.copyfileobj uses.
buf_size = 16*1024
total_written = 0
while True:
buf = req.read(buf_size)
if not buf:
break
fp.write(buf)
total_written += len(buf)
if total_size:
download_callback(total_written / total_size)
download_callback(1)
except (urllib.error.HTTPError,
urllib.error.URLError,
ssl.SSLError) as e:
raise URLFetchError(e)
with tarfile.open(name=download_file_path) as tar:
members = tar.getmembers()
total_files = len(members)
files = []
for num, member in enumerate(members, start=1):
files.append(member.name + '/' if member.isdir() else member.name)
tar.extract(member, extract_dir)
install_callback(num / total_files)
install_callback(1)
os.remove(download_file_path)
return files
def DownloadAndExtractTar(url,
download_dir,
extract_dir,
progress_callback=None,
command_path='unknown'):
"""Download and extract the given tar file.
Args:
url: str, The URL to download.
download_dir: str, The path to put the temporary download file into.
extract_dir: str, The path to extract the tar into.
progress_callback: f(float), A function to call with the fraction of
completeness.
command_path: the command path to include in the User-Agent header if the
URL is HTTP
Returns:
[str], The files that were extracted from the tar file.
Raises:
URLFetchError: If there is a problem fetching the given URL.
"""
for d in [download_dir, extract_dir]:
if not os.path.exists(d):
file_utils.MakeDir(d)
download_file_path = os.path.join(download_dir, os.path.basename(url))
if os.path.exists(download_file_path):
os.remove(download_file_path)
(download_callback,
install_callback) = (console_io.SplitProgressBar(progress_callback,
[1, 1]))
try:
response = MakeRequest(url, command_path)
with file_utils.BinaryFileWriter(download_file_path) as fp:
total_written = 0
total_size = len(response.content)
for chunk in response.iter_content(chunk_size=WRITE_BUFFER_SIZE):
fp.write(chunk)
total_written += len(chunk)
download_callback(total_written / total_size)
download_callback(1)
except (requests.exceptions.HTTPError, OSError) as e:
raise URLFetchError(e)
with tarfile.open(name=download_file_path) as tar:
members = tar.getmembers()
total_files = len(members)
files = []
for num, member in enumerate(members, start=1):
files.append(member.name + '/' if member.isdir() else member.name)
tar.extract(member, extract_dir)
full_path = os.path.join(extract_dir, member.name)
# Ensure read-and-write permission for all files
if os.path.isfile(full_path) and not os.access(full_path, os.W_OK):
os.chmod(full_path, stat.S_IWUSR | stat.S_IREAD)
install_callback(num / total_files)
install_callback(1)
os.remove(download_file_path)
return files
def Run(self, args):
if args.no_clobber and args.if_generation_match:
raise ValueError(
'Cannot specify both generation precondition and no-clobber.')
encryption_util.initialize_key_store(args)
source_expansion_iterator = name_expansion.NameExpansionIterator(
args.source,
all_versions=args.all_versions,
recursion_requested=args.recursive,
ignore_symlinks=args.ignore_symlinks)
task_status_queue = task_graph_executor.multiprocessing_context.Queue()
raw_destination_url = storage_url.storage_url_from_string(
args.destination)
if (isinstance(raw_destination_url, storage_url.FileUrl)
and args.storage_class):
raise ValueError(
'Cannot specify storage class for a non-cloud destination: {}'.
format(raw_destination_url))
parallelizable = True
shared_stream = None
if (args.all_versions
and (properties.VALUES.storage.process_count.GetInt() != 1
or properties.VALUES.storage.thread_count.GetInt() != 1)):
log.warning(
'Using sequential instead of parallel task execution. This will'
' maintain version ordering when copying all versions of an object.'
)
parallelizable = False
if (isinstance(raw_destination_url, storage_url.FileUrl)
and raw_destination_url.is_pipe):
log.warning('Downloading to a pipe.'
' This command may stall until the pipe is read.')
parallelizable = False
shared_stream = files.BinaryFileWriter(args.destination)
user_request_args = (
user_request_args_factory.get_user_request_args_from_command_args(
args,
metadata_type=user_request_args_factory.MetadataType.OBJECT))
task_iterator = copy_task_iterator.CopyTaskIterator(
source_expansion_iterator,
args.destination,
custom_md5_digest=args.content_md5,
do_not_decompress=args.do_not_decompress,
print_created_message=args.print_created_message,
shared_stream=shared_stream,
skip_unsupported=args.skip_unsupported,
task_status_queue=task_status_queue,
user_request_args=user_request_args,
)
self.exit_code = task_executor.execute_tasks(
task_iterator,
parallelizable=parallelizable,
task_status_queue=task_status_queue,
progress_manager_args=task_status.ProgressManagerArgs(
task_status.IncrementType.FILES_AND_BYTES,
manifest_path=user_request_args.manifest_path,
),
continue_on_error=args.continue_on_error,
)
if shared_stream:
shared_stream.close()
