def _prod_deployment_helper(config_dir,
package_zip_paths,
deploy_go=True,
deploy_appengine=True):
"""Helper for production deployment."""
config = local_config.Config()
deployment_bucket = config.get('project.deployment.bucket')
gae_config = config.sub_config(local_config.GAE_CONFIG_PATH)
gae_deployment = gae_config.sub_config('deployment')
project = gae_config.get('application_id')
print('Deploying %s to prod.' % project)
yaml_paths = gae_deployment.get_absolute_path('prod')
yaml_paths = appengine.filter_yaml_paths(yaml_paths, deploy_go)
if deploy_appengine:
_update_pubsub_queues(project)
_update_alerts(project)
_update_bigquery(project)
_deploy_app_prod(
project,
deployment_bucket,
yaml_paths,
package_zip_paths,
deploy_appengine=deploy_appengine)
if deploy_appengine:
common.execute('python butler.py run setup --config-dir {config_dir} '
'--non-dry-run'.format(config_dir=config_dir))
print('Production deployment finished.')
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if 'GOOGLE_CLOUDBUILD' in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
elif 'TRAVIS_BRANCH' in os.environ:
_, output = common.execute(
'git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
py_changed_file_paths = [
f for f in output.splitlines() if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [f for f in output.splitlines() if f.endswith('.go')]
for file_path in py_changed_file_paths:
if os.path.exists(file_path):
common.execute('pylint ' + file_path)
common.execute('yapf -d ' + file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if os.path.exists(file_path):
common.execute(golint_path + ' ' + file_path)
_, output = common.execute('gofmt -d ' + file_path)
if output.strip():
sys.exit(1)
def _delete_old_versions(project, service, delete_window):
"""Delete old versions."""
def _to_datetime(entry):
"""Parse datetime entry."""
return datetime.datetime(entry['year'], entry['month'], entry['day'],
entry['hour'], entry['minute'], entry['second'])
_, versions = common.execute('gcloud app versions list --format=json '
'--project=%s --service=%s' % (project, service))
versions = [
Version(version['id'], _to_datetime(version['last_deployed_time']),
version['traffic_split']) for version in json.loads(versions)
]
versions.sort(key=lambda v: v.deploy_time)
assert versions[-1].traffic_split == 1.0
to_delete = _versions_to_delete(versions, delete_window)
if not to_delete:
return
versions = ' '.join(version.id for version in to_delete)
common.execute('gcloud app versions delete --quiet '
'--project=%s --service=%s %s' % (project, service, versions))
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if 'GOOGLE_CLOUDBUILD' in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
elif 'TRAVIS_BRANCH' in os.environ:
_, output = common.execute(
'git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
file_paths = [f for f in output.splitlines() if os.path.exists(f)]
py_changed_file_paths = [
f for f in file_paths if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [f for f in file_paths if f.endswith('.go')]
yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')]
for file_path in py_changed_file_paths:
_execute_command_and_track_error('pylint ' + file_path)
_execute_command_and_track_error('yapf -d ' + file_path)
futurize_excludes = ' '.join(
['-x ' + exception for exception in _FUTURIZE_EXCEPTIONS])
futurize_command = 'futurize -0 {excludes} {file_path}'.format(
excludes=futurize_excludes, file_path=file_path)
futurize_output = _execute_command_and_track_error(futurize_command)
if ('No changes to ' not in futurize_output and
'No files need to be modified' not in futurize_output):
# Futurize doesn't modify its return code depending on the result.
_error('Python 3 compatibility error introduced.')
py_import_order(file_path)
py_test_init_check(file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + ' ' + file_path)
output = _execute_command_and_track_error('gofmt -d ' + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print('Linting failed, see errors above.')
sys.exit(1)
else:
print('Linting passed.')
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv("PYTHONPATH", "")
os.environ["PYTHONPATH"] = appengine.find_sdk_path() + ":" + pythonpath
if "GOOGLE_CLOUDBUILD" in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute("git diff --name-only master FETCH_HEAD")
elif "TRAVIS_BRANCH" in os.environ:
_, output = common.execute(
"git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)")
else:
_, output = common.execute("git diff --name-only FETCH_HEAD")
file_paths = [f for f in output.splitlines() if os.path.exists(f)]
py_changed_file_paths = [
f for f in file_paths
if f.endswith(".py") and not is_auto_generated_file(f)
]
go_changed_file_paths = [f for f in file_paths if f.endswith(".go")]
yaml_changed_file_paths = [f for f in file_paths if f.endswith(".yaml")]
for file_path in py_changed_file_paths:
_execute_command_and_track_error("pylint " + file_path)
_execute_command_and_track_error("yapf -d " + file_path)
futurize_excludes = " ".join("-x " + exception
for exception in _FUTURIZE_EXCEPTIONS)
futurize_command = "futurize -0 {excludes} {file_path}".format(
excludes=futurize_excludes, file_path=file_path)
futurize_output = _execute_command_and_track_error(futurize_command)
if ("No changes to " not in futurize_output
and "No files need to be modified" not in futurize_output):
# Futurize doesn't modify its return code depending on the result.
_error("Python 3 compatibility error introduced.")
py_import_order(file_path)
py_test_init_check(file_path)
golint_path = os.path.join("local", "bin", "golint")
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + " " + file_path)
output = _execute_command_and_track_error("gofmt -d " + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print("Linting failed, see errors above.")
sys.exit(1)
else:
print("Linting passed.")
def is_diff_origin_master():
"""Check if the current state is different from origin/master."""
common.execute('git fetch')
remote_sha = get_remote_sha()
_, local_sha = common.execute('git rev-parse HEAD')
_, diff_output = common.execute('git diff origin/master --stat')
return diff_output.strip() or remote_sha.strip() != local_sha.strip()
def bootstrap():
# Wait for the server to run.
time.sleep(10)
print('Bootstrapping datastore...')
common.execute(
('python butler.py run setup '
'--non-dry-run --local --config-dir={config_dir}').format(
config_dir=constants.TEST_CONFIG_DIR),
exit_on_error=False)
def _deploy_manifest(bucket_name, manifest_path):
"""Deploy source manifest to GCS."""
if sys.version_info.major == 3:
manifest_suffix = '.3'
else:
manifest_suffix = ''
common.execute('gsutil cp -a public-read %s '
'gs://%s/clusterfuzz-source.manifest%s' %
(manifest_path, bucket_name, manifest_suffix))
def execute(_):
"""Lint changed code."""
if "GOOGLE_CLOUDBUILD" in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
py_changed_file_paths = [
f for f in output.splitlines() if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [
f for f in output.splitlines() if f.endswith('.go')
]
for file_path in py_changed_file_paths:
if os.path.exists(file_path):
common.execute('pylint ' + file_path)
common.execute('yapf -d ' + file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if os.path.exists(file_path):
common.execute(golint_path + ' ' + file_path)
_, output = common.execute('gofmt -d ' + file_path)
if output.strip():
sys.exit(1)
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if 'GOOGLE_CLOUDBUILD' in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
file_paths = [
f.decode('utf-8') for f in output.splitlines() if os.path.exists(f)
]
py_changed_file_paths = [
f for f in file_paths
if f.endswith('.py') and not is_auto_generated_file(f)
]
go_changed_file_paths = [f for f in file_paths if f.endswith('.go')]
yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')]
for file_path in py_changed_file_paths:
line_length_override = ''
if '_test.py' in file_path:
line_length_override = '--max-line-length=240'
_execute_command_and_track_error(
f'pylint {line_length_override} {file_path}')
_execute_command_and_track_error(f'yapf -d {file_path}')
_execute_command_and_track_error(
f'{formatter.ISORT_CMD} -c {file_path}')
py_test_init_check(file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + ' ' + file_path)
output = _execute_command_and_track_error('gofmt -d ' + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print('Linting failed, see errors above.')
sys.exit(1)
else:
print('Linting passed.')
def execute(_):
"""Lint changed code."""
_, output = common.execute('git diff --name-only FETCH_HEAD')
py_changed_file_paths = [
f for f in output.splitlines() if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [
f for f in output.splitlines() if f.endswith('.go')
]
for file_path in py_changed_file_paths:
if os.path.exists(file_path):
common.execute('pylint ' + file_path)
common.execute('yapf -d ' + file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if os.path.exists(file_path):
common.execute(golint_path + ' ' + file_path)
_, output = common.execute('gofmt -d ' + file_path)
if output.strip():
sys.exit(1)
def execute(_):
"""Lint changed code."""
pythonpath = os.getenv('PYTHONPATH', '')
os.environ['PYTHONPATH'] = appengine.find_sdk_path() + ':' + pythonpath
if 'GOOGLE_CLOUDBUILD' in os.environ:
# Explicitly compare against master if we're running on the CI
_, output = common.execute('git diff --name-only master FETCH_HEAD')
elif 'TRAVIS_BRANCH' in os.environ:
_, output = common.execute(
'git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD)')
else:
_, output = common.execute('git diff --name-only FETCH_HEAD')
file_paths = [f for f in output.splitlines() if os.path.exists(f)]
py_changed_file_paths = [
f for f in file_paths if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [f for f in file_paths if f.endswith('.go')]
yaml_changed_file_paths = [f for f in file_paths if f.endswith('.yaml')]
for file_path in py_changed_file_paths:
_execute_command_and_track_error('pylint ' + file_path)
_execute_command_and_track_error('yapf -d ' + file_path)
py_import_order(file_path)
py_test_init_check(file_path)
golint_path = os.path.join('local', 'bin', 'golint')
for file_path in go_changed_file_paths:
if not os.path.basename(file_path) in _GOLINT_EXCEPTIONS:
_execute_command_and_track_error(golint_path + ' ' + file_path)
output = _execute_command_and_track_error('gofmt -d ' + file_path)
if output.strip():
_error()
for file_path in yaml_changed_file_paths:
yaml_validate(file_path)
for file_path in file_paths:
license_validate(file_path)
if _error_occurred:
print('Linting failed, see errors above.')
sys.exit(1)
else:
print('Linting passed.')
def execute(_):
"""Format changed code."""
_, output = common.execute('git diff --name-only FETCH_HEAD')
file_paths = [f for f in output.splitlines() if os.path.exists(f)]
py_changed_file_paths = [
f for f in file_paths if f.endswith('.py') and
# Exclude auto-generated files.
not f.endswith('_pb2.py') and not f.endswith('_pb2_grpc.py')
]
go_changed_file_paths = [f for f in file_paths if f.endswith('.go')]
for file_path in py_changed_file_paths:
common.execute('yapf -i ' + file_path)
for file_path in go_changed_file_paths:
common.execute('gofmt -w ' + file_path)
def symlink_dirs():
"""Symlink folders for use on appengine."""
symlink_config_dir()
common.symlink(
src=os.path.join('src', 'protos'),
target=os.path.join(SRC_DIR_PY, 'protos'))
common.symlink(
src=os.path.join('src', 'python'),
target=os.path.join(SRC_DIR_PY, 'python'))
# While importing third party modules, we may call pkg_resources.
# pkg_resources normalizes paths by calling os.path.realpath on them, which is
# incompatible with the App Engine sandbox since the resulting path will no
# longer be under appengine/.
common.copy_dir(
src=os.path.join('src', 'third_party'),
target=os.path.join(SRC_DIR_PY, 'third_party'))
# Remove existing local_gcs symlink (if any). This is important, as otherwise
# we will try deploying the directory in production. This is only needed for
# local development in run_server.
local_gcs_symlink_path = os.path.join(SRC_DIR_PY, 'local_gcs')
if os.path.exists(local_gcs_symlink_path):
os.remove(local_gcs_symlink_path)
_, output = common.execute('bazel run //local:create_gopath', cwd='src')
os.environ['GOPATH'] = output.splitlines()[-1]
def _deploy_appengine(project, yamls, stop_previous_version, version=None):
"""Deploy to appengine using `yamls`."""
stop_previous_version_arg = ("--stop-previous-version"
if stop_previous_version else
"--no-stop-previous-version")
version_arg = "--version=" + version if version else ""
for retry_num in range(DEPLOY_RETRIES + 1):
return_code, _ = common.execute(
"gcloud app deploy %s --quiet "
"--project=%s %s %s" %
(stop_previous_version_arg, project, version_arg, " ".join(yamls)),
exit_on_error=False,
)
if return_code == 0:
break
if retry_num == DEPLOY_RETRIES:
print("Failed to deploy after %d retries." % DEPLOY_RETRIES)
sys.exit(return_code)
print("gcloud deployment failed, retrying...")
time.sleep(RETRY_WAIT_SECONDS)
def _execute_command_and_track_error(command):
"""Executes command, tracks error state."""
returncode, output = common.execute(command, exit_on_error=False)
if returncode != 0:
_error()
return output.decode('utf-8')
def _update_redis(project):
"""Update redis instance."""
_update_deployment_manager(project, 'redis',
os.path.join('redis', 'instance.yaml'))
region = appengine.region(project)
return_code, _ = common.execute(
'gcloud compute networks vpc-access connectors describe '
'connector --region={region} '
'--project={project}'.format(project=project, region=region),
exit_on_error=False)
if return_code:
# Does not exist.
common.execute('gcloud compute networks vpc-access connectors create '
'connector --network=default --region={region} '
'--range=10.8.0.0/28 '
'--project={project}'.format(project=project, region=region))
def region(project):
"""Get the App Engine region."""
return_code, location = common.execute(
'gcloud app describe --project={project} '
'--format="value(locationId)"'.format(project=project))
if return_code:
raise RuntimeError('Could not get App Engine region')
return region_from_location(location.strip().decode('utf-8'))
def execute(_):
"""Run integration tests."""
if sys.version_info.major == 2:
print('Skipping integration_tests on Python 2.')
return
command = 'run_server'
indicator = b'Booting worker'
try:
lines = []
server = common.execute_async(
'python -u butler.py {} --skip-install-deps'.format(command))
test_utils.wait_for_emulator_ready(
server,
command,
indicator,
timeout=RUN_SERVER_TIMEOUT,
output_lines=lines)
# Sleep a small amount of time to ensure the server is definitely ready.
time.sleep(1)
# Call setup ourselves instead of passing --bootstrap since we have no idea
# when that finishes.
# TODO(ochang): Make bootstrap a separate butler command and just call that.
common.execute(
('python butler.py run setup '
'--non-dry-run --local --config-dir={config_dir}'
).format(config_dir=constants.TEST_CONFIG_DIR),
exit_on_error=False)
request = urllib.request.urlopen('http://' + constants.DEV_APPSERVER_HOST)
request.read() # Raises exception on error
except Exception:
print('Error occurred:')
print(b''.join(lines))
raise
finally:
server.terminate()
# TODO(ochang): Test that bot runs, and do a basic fuzzing session to ensure
# things work end to end.
print('All end-to-end integration tests passed.')
def _get_redis_ip(project):
"""Get the redis IP address."""
region = appengine.region(project)
return_code, ip = common.execute(
'gcloud redis instances describe redis-instance '
'--project={project} --region={region} '
'--format="value(host)"'.format(project=project, region=region))
if return_code:
raise RuntimeError('Failed to get redis IP.')
return ip.decode('utf-8').strip()
def _is_nodejs_up_to_date():
"""Check if node is of version MINIMUM_NODEJS_VERSION."""
return_code, output = common.execute('node -v')
if return_code != 0:
return False
m = re.match(r'v([0-9]+)\..+', output.strip())
if not m:
return False
major_version = int(m.group(1))
return major_version >= MIN_SUPPORTED_NODEJS_VERSION
def execute(args):
"""Build and run all tests under src/go."""
go_directory = os.path.join('src', 'go')
common.execute('bazel build //...', cwd=go_directory)
if args.verbose or args.unsuppress_output:
test_output_arg = '--test_output=all'
else:
test_output_arg = '--test_output=errors'
common.execute(
'bazel test --sandbox_writable_path={home} ' # Necessary for gcloud.
'{test_output_arg} '
'--test_env=CONFIG_DIR_OVERRIDE={config_dir_override} '
'--test_env=ROOT_DIR={root_dir} '
'--test_env=INTEGRATION={integration} '
'--test_env=CLUSTERFUZZ_MUTABLE_TEST_BUCKET={test_bucket} //...'.format(
home=os.getenv('HOME'),
test_output_arg=test_output_arg,
config_dir_override=os.path.abspath(os.path.join('configs', 'test')),
root_dir=os.getenv('ROOT_DIR'),
integration=os.getenv('INTEGRATION', '0'),
test_bucket=common.test_bucket_for_user()),
cwd=go_directory)
def _delete_old_versions(project, service, delete_window):
"""Delete old versions."""
def _to_datetime(entry):
"""Parse datetime entry."""
return datetime.datetime(
entry["year"],
entry["month"],
entry["day"],
entry["hour"],
entry["minute"],
entry["second"],
)
_, versions = common.execute("gcloud app versions list --format=json "
"--project=%s --service=%s" %
(project, service))
versions = [
Version(
version["id"],
_to_datetime(version["last_deployed_time"]),
version["traffic_split"],
) for version in json.loads(versions)
]
versions.sort(key=lambda v: v.deploy_time)
if versions[-1].traffic_split != 1.0:
raise AssertionError
to_delete = _versions_to_delete(versions, delete_window)
if not to_delete:
return
versions = " ".join(version.id for version in to_delete)
common.execute("gcloud app versions delete --quiet "
"--project=%s --service=%s %s" %
(project, service, versions))
def find_sdk_path():
"""Find the App Engine SDK path."""
if common.get_platform() == 'windows':
_, gcloud_path = common.execute('where gcloud.cmd', print_output=False)
else:
gcloud_path = spawn.find_executable('gcloud')
if not gcloud_path:
print('Please install the Google Cloud SDK and set up PATH to point to it.')
sys.exit(1)
cloud_sdk_path = os.path.dirname(
os.path.dirname(os.path.realpath(gcloud_path)))
appengine_sdk_path = os.path.join(cloud_sdk_path, 'platform',
'google_appengine')
if not os.path.exists(appengine_sdk_path):
print('App Engine SDK not found. Please run local/install_deps.bash')
sys.exit(1)
return appengine_sdk_path
def symlink_dirs():
"""Symlink folders for use on appengine."""
symlink_config_dir()
common.symlink(src=os.path.join('src', 'protos'),
target=os.path.join(SRC_DIR_PY, 'protos'))
common.symlink(src=os.path.join('src', 'python'),
target=os.path.join(SRC_DIR_PY, 'python'))
common.symlink(src=os.path.join('src', 'third_party'),
target=os.path.join(SRC_DIR_PY, 'third_party'))
# Remove existing local_gcs symlink (if any). This is important, as otherwise
# we will try deploying the directory in production. This is only needed for
# local development in run_server.
local_gcs_symlink_path = os.path.join(SRC_DIR_PY, 'local_gcs')
if os.path.exists(local_gcs_symlink_path):
os.remove(local_gcs_symlink_path)
_, output = common.execute('bazel run //local:create_gopath', cwd='src')
os.environ['GOPATH'] = output.splitlines()[-1]
def _deploy_appengine(project, yamls, stop_previous_version, version=None):
"""Deploy to appengine using `yamls`."""
stop_previous_version_arg = ('--stop-previous-version'
if stop_previous_version else
'--no-stop-previous-version')
version_arg = '--version=' + version if version else ''
for retry_num in range(DEPLOY_RETRIES + 1):
return_code, _ = common.execute(
'gcloud app deploy %s --quiet '
'--project=%s %s %s' % (stop_previous_version_arg, project, version_arg,
' '.join(yamls)),
exit_on_error=False)
if return_code == 0:
break
if retry_num == DEPLOY_RETRIES:
print('Failed to deploy after %d retries.' % DEPLOY_RETRIES)
sys.exit(return_code)
print('gcloud deployment failed, retrying...')
time.sleep(RETRY_WAIT_SECONDS)
def get_remote_sha():
"""Get remote sha of origin/master."""
_, remote_sha_line = common.execute('git ls-remote origin refs/heads/master')
return re.split(r'\s+', remote_sha_line)[0]
def _deploy_manifest(bucket_name, manifest_path):
"""Deploy source manifest to GCS."""
common.execute(
'gsutil cp -a public-read %s '
'gs://%s/clusterfuzz-source.manifest' % (manifest_path, bucket_name))
def _deploy_zip(bucket_name, zip_path):
"""Deploy zip to GCS."""
common.execute('gsutil cp %s gs://%s/%s' % (zip_path, bucket_name,
os.path.basename(zip_path)))
def package(revision,
target_zip_dir=constants.PACKAGE_TARGET_ZIP_DIRECTORY,
target_manifest_path=constants.PACKAGE_TARGET_MANIFEST_PATH,
platform_name=None):
"""Prepare clusterfuzz-source.zip."""
is_ci = os.getenv('TEST_BOT_ENVIRONMENT')
if not is_ci and common.is_git_dirty():
print('Your branch is dirty. Please fix before packaging.')
sys.exit(1)
if not _is_nodejs_up_to_date():
print(
'You do not have nodejs, or your nodejs is not at least version 4.'
)
sys.exit(1)
common.install_dependencies(platform_name=platform_name)
# This needs to be done before packaging step to let src/appengine/config be
# archived for bot.
appengine.symlink_dirs()
_, ls_files_output = common.execute('git -C . ls-files',
print_output=False)
file_paths = ls_files_output.splitlines()
if not os.path.exists(target_zip_dir):
os.makedirs(target_zip_dir)
target_zip_name = constants.LEGACY_ZIP_NAME
if platform_name:
target_zip_name = platform_name + '.zip'
target_zip_path = os.path.join(target_zip_dir, target_zip_name)
_clear_zip(target_zip_path)
output_file = zipfile.ZipFile(target_zip_path, 'w', zipfile.ZIP_DEFLATED)
# Add files from git.
for file_path in file_paths:
if (file_path.startswith('config') or file_path.startswith('local')
or file_path.startswith(os.path.join('src', 'appengine'))
or file_path.startswith(os.path.join('src', 'local')) or
file_path.startswith(os.path.join('src', 'python', 'tests'))):
continue
_add_to_zip(output_file, file_path)
# These are project configuration yamls.
for path in _get_files(os.path.join('src', 'appengine', 'config')):
_add_to_zip(output_file, path)
# These are third party dependencies.
for path in _get_files(os.path.join('src', 'third_party')):
_add_to_zip(output_file, path)
output_file.close()
with open(target_manifest_path, 'w') as f:
f.write('%s\n' % revision)
with zipfile.ZipFile(target_zip_path, 'a', zipfile.ZIP_DEFLATED) as f:
_add_to_zip(f, target_manifest_path,
constants.PACKAGE_TARGET_MANIFEST_PATH)
print('Revision: %s' % revision)
print()
print('%s is ready.' % target_zip_path)
return target_zip_path
