def _BuildDeploymentManifest(info, source_dir, bucket_ref, tmp_dir): """Builds a deployment manifest for use with the App Engine Admin API. Args: info: An instance of yaml_parsing.ServiceInfo. source_dir: str, path to the service's source directory bucket_ref: The reference to the bucket files will be placed in. tmp_dir: A temp directory for storing generated files (currently just source context files). Returns: A deployment manifest (dict) for use with the Admin API. """ manifest = {} bucket_url = 'https://storage.googleapis.com/{0}'.format(bucket_ref.bucket) skip_files_regex = info.parsed.skip_files.regex runtime = info.parsed.runtime if info.parsed.runtime else '' source_file_iterator = source_files_util.GetSourceFileIterator( source_dir, skip_files_regex, info.HasExplicitSkipFiles(), runtime, info.env) # Normal application files. for rel_path in source_file_iterator: full_path = os.path.join(source_dir, rel_path) sha1_hash = file_utils.Checksum.HashSingleFile(full_path, algorithm=hashlib.sha1) manifest_path = '/'.join([bucket_url, sha1_hash]) manifest[_FormatForManifest(rel_path)] = { 'sourceUrl': manifest_path, 'sha1Sum': sha1_hash } # Source context files. These are temporary files which indicate the current # state of the source repository (git, cloud repo, etc.) context_files = context_util.CreateContextFiles(tmp_dir, None, source_dir=source_dir) for context_file in context_files: rel_path = os.path.basename(context_file) if rel_path in manifest: # The source context file was explicitly provided by the user. log.debug( 'Source context already exists. Using the existing file.') continue else: sha1_hash = file_utils.Checksum.HashSingleFile( context_file, algorithm=hashlib.sha1) manifest_path = '/'.join([bucket_url, sha1_hash]) manifest[_FormatForManifest(rel_path)] = { 'sourceUrl': manifest_path, 'sha1Sum': sha1_hash, } return manifest
def _BuildDeploymentManifest(upload_dir, source_files, bucket_ref, tmp_dir): """Builds a deployment manifest for use with the App Engine Admin API. Args: upload_dir: str, path to the service's upload directory source_files: [str], relative paths to upload. bucket_ref: The reference to the bucket files will be placed in. tmp_dir: A temp directory for storing generated files (currently just source context files). Returns: A deployment manifest (dict) for use with the Admin API. """ manifest = {} bucket_url = 'https://storage.googleapis.com/{0}'.format(bucket_ref.bucket) # Normal application files. for rel_path in source_files: full_path = os.path.join(upload_dir, rel_path) sha1_hash = file_utils.Checksum.HashSingleFile(full_path, algorithm=hashlib.sha1) manifest_path = '/'.join([bucket_url, sha1_hash]) manifest[_FormatForManifest(rel_path)] = { 'sourceUrl': manifest_path, 'sha1Sum': sha1_hash } # Source context files. These are temporary files which indicate the current # state of the source repository (git, cloud repo, etc.) context_files = context_util.CreateContextFiles(tmp_dir, None, source_dir=upload_dir) for context_file in context_files: rel_path = os.path.basename(context_file) if rel_path in manifest: # The source context file was explicitly provided by the user. log.debug( 'Source context already exists. Using the existing file.') continue else: sha1_hash = file_utils.Checksum.HashSingleFile( context_file, algorithm=hashlib.sha1) manifest_path = '/'.join([bucket_url, sha1_hash]) manifest[_FormatForManifest(rel_path)] = { 'sourceUrl': manifest_path, 'sha1Sum': sha1_hash, } return manifest
def _BuildDeploymentManifest(info, bucket_ref, tmp_dir): """Builds a deployment manifest for use with the App Engine Admin API. Args: info: An instance of yaml_parsing.ServiceInfo. bucket_ref: The reference to the bucket files will be placed in. tmp_dir: A temp directory for storing generated files (currently just source context files). Returns: A deployment manifest (dict) for use with the Admin API. """ source_dir = os.path.dirname(info.file) excluded_files_regex = info.parsed.skip_files.regex manifest = {} bucket_url = 'https://storage.googleapis.com/{0}'.format(bucket_ref.bucket) # Normal application files. for rel_path in util.FileIterator(source_dir, excluded_files_regex): full_path = os.path.join(source_dir, rel_path) sha1_hash = _GetSha1(full_path) manifest_path = '/'.join([bucket_url, sha1_hash]) manifest[rel_path] = {'sourceUrl': manifest_path, 'sha1Sum': sha1_hash} # Source context files. These are temporary files which indicate the current # state of the source repository (git, cloud repo, etc.) context_files = context_util.CreateContextFiles(tmp_dir, None, source_dir=source_dir) for context_file in context_files: rel_path = os.path.basename(context_file) if rel_path in manifest: # The source context file was explicitly provided by the user. log.debug( 'Source context already exists. Using the existing file.') continue else: sha1_hash = _GetSha1(context_file) manifest_path = '/'.join([bucket_url, sha1_hash]) manifest[rel_path] = { 'sourceUrl': manifest_path, 'sha1Sum': sha1_hash, } return manifest
def _BuildStagingDirectory(source_dir, staging_dir, bucket_ref, excluded_regexes): """Creates a staging directory to be uploaded to Google Cloud Storage. The staging directory will contain a symlink for each file in the original directory. The source is a file whose name is the sha1 hash of the original file and points to the original file. Consider the following original structure: app/ main.py tools/ foo.py Assume main.py has SHA1 hash 123 and foo.py has SHA1 hash 456. The resultant staging directory will look like: /tmp/staging/ 123 -> app/main.py 456 -> app/tools/foo.py (Note: "->" denotes a symlink) If the staging directory is then copied to a GCS bucket at gs://staging-bucket/ then the resulting manifest will be: { "app/main.py": { "sourceUrl": "https://storage.googleapis.com/staging-bucket/123", "sha1Sum": "123" }, "app/tools/foo.py": { "sourceUrl": "https://storage.googleapis.com/staging-bucket/456", "sha1Sum": "456" } } Args: source_dir: The original directory containing the application's source code. staging_dir: The directory where the staged files will be created. bucket_ref: A reference to the GCS bucket where the files will be uploaded. excluded_regexes: List of file patterns to skip while building the staging directory. Raises: LargeFileError: if one of the files to upload exceeds the maximum App Engine file size. Returns: A dictionary which represents the file manifest. """ manifest = {} bucket_url = bucket_ref.ToAppEngineApiReference() def AddFileToManifest(manifest_path, input_path): """Adds the given file to the current manifest. Args: manifest_path: The path to the file as it will be stored in the manifest. input_path: The location of the file to be added to the manifest. Returns: If the target was already in the manifest with different contexts, returns None. In all other cases, returns a target location to which the caller must copy, move, or link the file. """ file_ext = os.path.splitext(input_path)[1] sha1_hash = file_utils.Checksum().AddFileContents(input_path).HexDigest() target_filename = sha1_hash + file_ext target_path = os.path.join(staging_dir, target_filename) dest_path = '/'.join([bucket_url, target_filename]) old_url = manifest.get(manifest_path, {}).get('sourceUrl', '') if old_url and old_url != dest_path: return None manifest[manifest_path] = { 'sourceUrl': dest_path, 'sha1Sum': sha1_hash, } return target_path for relative_path in util.FileIterator(source_dir, excluded_regexes): local_path = os.path.join(source_dir, relative_path) size = os.path.getsize(local_path) if size > _MAX_FILE_SIZE: raise LargeFileError(local_path, size, _MAX_FILE_SIZE) target_path = AddFileToManifest(relative_path, local_path) if not os.path.exists(target_path): _CopyOrSymlink(local_path, target_path) context_files = context_util.CreateContextFiles( staging_dir, None, overwrite=True, source_dir=source_dir) for context_file in context_files: manifest_path = os.path.basename(context_file) target_path = AddFileToManifest(manifest_path, context_file) if not target_path: log.status.Print('Not generating {0} because a user-generated ' 'file with the same name exists.'.format(manifest_path)) if not target_path or os.path.exists(target_path): # If we get here, it probably means that the user already generated the # context file manually and put it either in the top directory or in some # subdirectory. The new context file is useless and may confuse later # stages of the upload (it is in the staging directory with a # nonconformant name), so delete it. The entry in the manifest will point # at the existing file. os.remove(context_file) else: # Rename the source-context*.json file (which is in the staging directory) # to the hash-based name in the same directory. os.rename(context_file, target_path) log.debug('Generated deployment manifest: "{0}"'.format( json.dumps(manifest, indent=2, sort_keys=True))) return manifest