def _BuildDeploymentManifest(info, source_dir, bucket_ref, tmp_dir):
"""Builds a deployment manifest for use with the App Engine Admin API.
Args:
info: An instance of yaml_parsing.ServiceInfo.
source_dir: str, path to the service's source directory
bucket_ref: The reference to the bucket files will be placed in.
tmp_dir: A temp directory for storing generated files (currently just source
context files).
Returns:
A deployment manifest (dict) for use with the Admin API.
"""
manifest = {}
bucket_url = 'https://storage.googleapis.com/{0}'.format(bucket_ref.bucket)
skip_files_regex = info.parsed.skip_files.regex
runtime = info.parsed.runtime if info.parsed.runtime else ''
source_file_iterator = source_files_util.GetSourceFileIterator(
source_dir, skip_files_regex, info.HasExplicitSkipFiles(), runtime,
info.env)
# Normal application files.
for rel_path in source_file_iterator:
full_path = os.path.join(source_dir, rel_path)
sha1_hash = file_utils.Checksum.HashSingleFile(full_path,
algorithm=hashlib.sha1)
manifest_path = '/'.join([bucket_url, sha1_hash])
manifest[_FormatForManifest(rel_path)] = {
'sourceUrl': manifest_path,
'sha1Sum': sha1_hash
}
# Source context files. These are temporary files which indicate the current
# state of the source repository (git, cloud repo, etc.)
context_files = context_util.CreateContextFiles(tmp_dir,
None,
source_dir=source_dir)
for context_file in context_files:
rel_path = os.path.basename(context_file)
if rel_path in manifest:
# The source context file was explicitly provided by the user.
log.debug(
'Source context already exists. Using the existing file.')
continue
else:
sha1_hash = file_utils.Checksum.HashSingleFile(
context_file, algorithm=hashlib.sha1)
manifest_path = '/'.join([bucket_url, sha1_hash])
manifest[_FormatForManifest(rel_path)] = {
'sourceUrl': manifest_path,
'sha1Sum': sha1_hash,
}
return manifest
def _BuildDeploymentManifest(upload_dir, source_files, bucket_ref, tmp_dir):
"""Builds a deployment manifest for use with the App Engine Admin API.
Args:
upload_dir: str, path to the service's upload directory
source_files: [str], relative paths to upload.
bucket_ref: The reference to the bucket files will be placed in.
tmp_dir: A temp directory for storing generated files (currently just source
context files).
Returns:
A deployment manifest (dict) for use with the Admin API.
"""
manifest = {}
bucket_url = 'https://storage.googleapis.com/{0}'.format(bucket_ref.bucket)
# Normal application files.
for rel_path in source_files:
full_path = os.path.join(upload_dir, rel_path)
sha1_hash = file_utils.Checksum.HashSingleFile(full_path,
algorithm=hashlib.sha1)
manifest_path = '/'.join([bucket_url, sha1_hash])
manifest[_FormatForManifest(rel_path)] = {
'sourceUrl': manifest_path,
'sha1Sum': sha1_hash
}
# Source context files. These are temporary files which indicate the current
# state of the source repository (git, cloud repo, etc.)
context_files = context_util.CreateContextFiles(tmp_dir,
None,
source_dir=upload_dir)
for context_file in context_files:
rel_path = os.path.basename(context_file)
if rel_path in manifest:
# The source context file was explicitly provided by the user.
log.debug(
'Source context already exists. Using the existing file.')
continue
else:
sha1_hash = file_utils.Checksum.HashSingleFile(
context_file, algorithm=hashlib.sha1)
manifest_path = '/'.join([bucket_url, sha1_hash])
manifest[_FormatForManifest(rel_path)] = {
'sourceUrl': manifest_path,
'sha1Sum': sha1_hash,
}
return manifest
def _BuildDeploymentManifest(info, bucket_ref, tmp_dir):
"""Builds a deployment manifest for use with the App Engine Admin API.
Args:
info: An instance of yaml_parsing.ServiceInfo.
bucket_ref: The reference to the bucket files will be placed in.
tmp_dir: A temp directory for storing generated files (currently just source
context files).
Returns:
A deployment manifest (dict) for use with the Admin API.
"""
source_dir = os.path.dirname(info.file)
excluded_files_regex = info.parsed.skip_files.regex
manifest = {}
bucket_url = 'https://storage.googleapis.com/{0}'.format(bucket_ref.bucket)
# Normal application files.
for rel_path in util.FileIterator(source_dir, excluded_files_regex):
full_path = os.path.join(source_dir, rel_path)
sha1_hash = _GetSha1(full_path)
manifest_path = '/'.join([bucket_url, sha1_hash])
manifest[rel_path] = {'sourceUrl': manifest_path, 'sha1Sum': sha1_hash}
# Source context files. These are temporary files which indicate the current
# state of the source repository (git, cloud repo, etc.)
context_files = context_util.CreateContextFiles(tmp_dir,
None,
source_dir=source_dir)
for context_file in context_files:
rel_path = os.path.basename(context_file)
if rel_path in manifest:
# The source context file was explicitly provided by the user.
log.debug(
'Source context already exists. Using the existing file.')
continue
else:
sha1_hash = _GetSha1(context_file)
manifest_path = '/'.join([bucket_url, sha1_hash])
manifest[rel_path] = {
'sourceUrl': manifest_path,
'sha1Sum': sha1_hash,
}
return manifest
def _BuildStagingDirectory(source_dir, staging_dir, bucket_ref,
excluded_regexes):
"""Creates a staging directory to be uploaded to Google Cloud Storage.
The staging directory will contain a symlink for each file in the original
directory. The source is a file whose name is the sha1 hash of the original
file and points to the original file.
Consider the following original structure:
app/
main.py
tools/
foo.py
Assume main.py has SHA1 hash 123 and foo.py has SHA1 hash 456. The resultant
staging directory will look like:
/tmp/staging/
123 -> app/main.py
456 -> app/tools/foo.py
(Note: "->" denotes a symlink)
If the staging directory is then copied to a GCS bucket at
gs://staging-bucket/ then the resulting manifest will be:
{
"app/main.py": {
"sourceUrl": "https://storage.googleapis.com/staging-bucket/123",
"sha1Sum": "123"
},
"app/tools/foo.py": {
"sourceUrl": "https://storage.googleapis.com/staging-bucket/456",
"sha1Sum": "456"
}
}
Args:
source_dir: The original directory containing the application's source
code.
staging_dir: The directory where the staged files will be created.
bucket_ref: A reference to the GCS bucket where the files will be uploaded.
excluded_regexes: List of file patterns to skip while building the staging
directory.
Raises:
LargeFileError: if one of the files to upload exceeds the maximum App Engine
file size.
Returns:
A dictionary which represents the file manifest.
"""
manifest = {}
bucket_url = bucket_ref.ToAppEngineApiReference()
def AddFileToManifest(manifest_path, input_path):
"""Adds the given file to the current manifest.
Args:
manifest_path: The path to the file as it will be stored in the manifest.
input_path: The location of the file to be added to the manifest.
Returns:
If the target was already in the manifest with different contexts,
returns None. In all other cases, returns a target location to which the
caller must copy, move, or link the file.
"""
file_ext = os.path.splitext(input_path)[1]
sha1_hash = file_utils.Checksum().AddFileContents(input_path).HexDigest()
target_filename = sha1_hash + file_ext
target_path = os.path.join(staging_dir, target_filename)
dest_path = '/'.join([bucket_url, target_filename])
old_url = manifest.get(manifest_path, {}).get('sourceUrl', '')
if old_url and old_url != dest_path:
return None
manifest[manifest_path] = {
'sourceUrl': dest_path,
'sha1Sum': sha1_hash,
}
return target_path
for relative_path in util.FileIterator(source_dir, excluded_regexes):
local_path = os.path.join(source_dir, relative_path)
size = os.path.getsize(local_path)
if size > _MAX_FILE_SIZE:
raise LargeFileError(local_path, size, _MAX_FILE_SIZE)
target_path = AddFileToManifest(relative_path, local_path)
if not os.path.exists(target_path):
_CopyOrSymlink(local_path, target_path)
context_files = context_util.CreateContextFiles(
staging_dir, None, overwrite=True, source_dir=source_dir)
for context_file in context_files:
manifest_path = os.path.basename(context_file)
target_path = AddFileToManifest(manifest_path, context_file)
if not target_path:
log.status.Print('Not generating {0} because a user-generated '
'file with the same name exists.'.format(manifest_path))
if not target_path or os.path.exists(target_path):
# If we get here, it probably means that the user already generated the
# context file manually and put it either in the top directory or in some
# subdirectory. The new context file is useless and may confuse later
# stages of the upload (it is in the staging directory with a
# nonconformant name), so delete it. The entry in the manifest will point
# at the existing file.
os.remove(context_file)
else:
# Rename the source-context*.json file (which is in the staging directory)
# to the hash-based name in the same directory.
os.rename(context_file, target_path)
log.debug('Generated deployment manifest: "{0}"'.format(
json.dumps(manifest, indent=2, sort_keys=True)))
return manifest
