def test_package(self):
"""Test package."""
package.package(
'revision',
self.zip_directory,
self.manifest_filename,
platform_name='linux')
zip_filename = os.path.join(self.zip_directory, 'linux.zip')
with zipfile.ZipFile(zip_filename, 'r') as f:
files = f.namelist()
manifest_file = os.path.join('clusterfuzz', 'src', 'appengine',
'resources', 'clusterfuzz-source.manifest')
self.assertIn(manifest_file, files)
self.assertEqual('revision\n', f.read(manifest_file))
self.assertIn(
os.path.join('clusterfuzz', 'src', 'third_party', 'googleapiclient',
'__init__.py'), files)
self.assertIn(
os.path.join('clusterfuzz', 'src', 'appengine', 'config', 'gae',
'auth.yaml'), files)
self.assertNotIn(
os.path.join('clusterfuzz', 'configs', 'test', 'gae', 'auth.yaml'),
files)
self.assertNotIn(
os.path.join('clusterfuzz', 'local', '__init__.py'), files)
self.assertNotIn(
os.path.join('clusterfuzz', 'src', 'appengine', 'app.yaml'), files)
def stage(self, config_dir):
"""Stage a zip (built by `python butler.py package`)."""
os.environ['CONFIG_DIR_OVERRIDE'] = config_dir
self.restart()
time.sleep(1)
zip_path = package.package(
revision=butler_common.compute_staging_revision(),
platform_name='windows')
remote_zip_path = (
'{clusterfuzz_parent_path}\\{staging_source_filename}'.format(
clusterfuzz_parent_path=self.clusterfuzz_parent_path,
staging_source_filename=self.staging_source_filename))
api.put(zip_path, remote_zip_path)
api.put(EXTRACT_ZIP_PS_LOCAL_PATH, EXTRACT_ZIP_PS_REMOTE_PATH)
self._powershell(EXTRACT_ZIP_PS_REMOTE_PATH, 'file')
self.restart()
def stage(self, config_dir):
"""Stage a zip (built by `python butler.py package`)."""
os.environ['CONFIG_DIR_OVERRIDE'] = config_dir
# Restarting ensures that the target bot is updated to latest revision.
# See crbug.com/674173 for more info.
self.restart()
local_zip_path = package.package(
revision=butler_common.compute_staging_revision(),
platform_name=self.platform)
self._copy_staging_archive_from_local_to_remote(local_zip_path)
self._run(('cd {clusterfuzz_parent_path} && '
'unzip -o -d . {staging_source_filename}').format(
clusterfuzz_parent_path=self.clusterfuzz_parent_path,
staging_source_filename=self.staging_source_filename))
self._run('chown -R {username} {clusterfuzz_parent_path}'.format(
username=self.username,
clusterfuzz_parent_path=self.clusterfuzz_parent_path))
self.restart()
def execute(args):
"""Deploy Clusterfuzz to Appengine."""
os.environ['ROOT_DIR'] = '.'
if not os.path.exists(args.config_dir):
print('Please provide a valid configuration directory.')
sys.exit(1)
os.environ['CONFIG_DIR_OVERRIDE'] = args.config_dir
if not common.has_file_in_path('gcloud'):
print('Please install gcloud.')
sys.exit(1)
is_ci = os.getenv('TEST_BOT_ENVIRONMENT')
if not is_ci and common.is_git_dirty():
print('Your branch is dirty. Please fix before deploying.')
sys.exit(1)
if not common.has_file_in_path('gsutil'):
print('gsutil not found in PATH.')
sys.exit(1)
# Build templates before deployment.
appengine.build_templates()
if not is_ci and not args.staging:
if is_diff_origin_master():
if args.force:
print('You are not on origin/master. --force is used. Continue.')
for _ in range(3):
print('.')
time.sleep(1)
print()
else:
print('You are not on origin/master. Please fix or use --force.')
sys.exit(1)
if args.staging:
revision = common.compute_staging_revision()
platforms = ['linux'] # No other platforms required.
elif args.prod:
revision = common.compute_prod_revision()
platforms = list(constants.PLATFORMS.keys())
else:
print('Please specify either --prod or --staging. For production '
'deployments, you probably want to use deploy.sh from your '
'configs directory instead.')
sys.exit(1)
deploy_zips = 'zips' in args.targets
deploy_appengine = 'appengine' in args.targets
package_zip_paths = []
if deploy_zips:
for platform_name in platforms:
package_zip_paths.append(
package.package(revision, platform_name=platform_name))
else:
# package.package calls these, so only set these up if we're not packaging,
# since they can be fairly slow.
appengine.symlink_dirs()
common.install_dependencies('linux')
with open(constants.PACKAGE_TARGET_MANIFEST_PATH, 'w') as f:
f.write('%s\n' % revision)
too_large_file_path = find_file_exceeding_limit('src/appengine',
APPENGINE_FILESIZE_LIMIT)
if too_large_file_path:
print(("%s is larger than %d bytes. It wouldn't be deployed to appengine."
' Please fix.') % (too_large_file_path, APPENGINE_FILESIZE_LIMIT))
sys.exit(1)
deploy_go = args.with_go
if args.staging:
_staging_deployment_helper(deploy_go)
else:
_prod_deployment_helper(args.config_dir, package_zip_paths, deploy_go,
deploy_appengine)
with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f:
print('Source updated to %s' % f.read())
if platforms[-1] != common.get_platform():
# Make sure the installed dependencies are for the current platform.
common.install_dependencies()
def execute(args):
"""Deploy Clusterfuzz to Appengine."""
# TODO(ochang): Remove once python3 deployment is fixed.
os.environ["CLOUDSDK_PYTHON"] = "python2"
os.environ["ROOT_DIR"] = "."
if not os.path.exists(args.config_dir):
print("Please provide a valid configuration directory.")
sys.exit(1)
os.environ["CONFIG_DIR_OVERRIDE"] = args.config_dir
if not common.has_file_in_path("gcloud"):
print("Please install gcloud.")
sys.exit(1)
is_ci = os.getenv("TEST_BOT_ENVIRONMENT")
if not is_ci and common.is_git_dirty():
print("Your branch is dirty. Please fix before deploying.")
sys.exit(1)
if not common.has_file_in_path("gsutil"):
print("gsutil not found in PATH.")
sys.exit(1)
# Build templates before deployment.
appengine.build_templates()
if not is_ci and not args.staging:
if is_diff_origin_master():
if args.force:
print(
"You are not on origin/master. --force is used. Continue.")
for _ in range(3):
print(".")
time.sleep(1)
print()
else:
print(
"You are not on origin/master. Please fix or use --force.")
sys.exit(1)
if args.staging:
revision = common.compute_staging_revision()
platforms = ["linux"] # No other platforms required.
elif args.prod:
revision = common.compute_prod_revision()
platforms = list(constants.PLATFORMS.keys())
else:
print("Please specify either --prod or --staging. For production "
"deployments, you probably want to use deploy.sh from your "
"configs directory instead.")
sys.exit(1)
deploy_zips = "zips" in args.targets
deploy_appengine = "appengine" in args.targets
package_zip_paths = []
if deploy_zips:
for platform_name in platforms:
package_zip_paths.append(
package.package(revision, platform_name=platform_name))
else:
# package.package calls these, so only set these up if we're not packaging,
# since they can be fairly slow.
appengine.symlink_dirs()
common.install_dependencies("linux")
with open(constants.PACKAGE_TARGET_MANIFEST_PATH, "w") as f:
f.write("%s\n" % revision)
too_large_file_path = find_file_exceeding_limit("src/appengine",
APPENGINE_FILESIZE_LIMIT)
if too_large_file_path:
print((
"%s is larger than %d bytes. It wouldn't be deployed to appengine."
" Please fix.") % (too_large_file_path, APPENGINE_FILESIZE_LIMIT))
sys.exit(1)
deploy_go = args.with_go
if args.staging:
_staging_deployment_helper(deploy_go)
else:
_prod_deployment_helper(args.config_dir, package_zip_paths, deploy_go,
deploy_appengine)
with open(constants.PACKAGE_TARGET_MANIFEST_PATH) as f:
print("Source updated to %s" % f.read())
if platforms[-1] != common.get_platform():
# Make sure the installed dependencies are for the current platform.
common.install_dependencies()
