def test_it_does_not_post_logs_or_status_if_SKIP_LOGGING(
self, mock_del, mock_post, monkeypatch):
monkeypatch.setenv('SKIP_LOGGING', 'true')
post_build_timeout(MOCK_LOG_URL, MOCK_STATUS_URL, MOCK_BUILDER_URL)
mock_post.assert_not_called()
# but the builder DELETE should still be called
mock_del.assert_called_once_with(MOCK_BUILDER_URL)
def test_it_works(self, mock_del, mock_post):
post_build_timeout(MOCK_STATUS_URL)
expected_output = b64string(
'The build did not complete. It may have timed out.')
assert mock_post.call_count == 1
mock_post.assert_any_call(MOCK_STATUS_URL,
json={
'status': STATUS_ERROR,
'message': expected_output
})
def test_it_works(self, mock_del, mock_post):
post_build_timeout(MOCK_LOG_URL, MOCK_STATUS_URL, MOCK_BUILDER_URL)
expected_output = b64string(
'The build did not complete. It may have timed out.')
assert mock_post.call_count == 2
mock_post.assert_any_call(MOCK_LOG_URL,
json={
'source': 'ERROR',
'output': expected_output
})
mock_post.assert_any_call(MOCK_STATUS_URL,
json={
'status': 1,
'message': expected_output
})
mock_del.assert_called_once_with(MOCK_BUILDER_URL)
def main(ctx):
'''
Main task to run a full site build process.
All values needed for the build are loaded from
environment variables.
'''
# (variable naming)
# pylint: disable=C0103
# keep track of total time
start_time = datetime.now()
# Load from .env for development
load_dotenv()
# These environment variables will be set into the environment
# by federalist-builder.
# During development, we can use a `.env` file (loaded above)
# to make it easier to specify variables.
AWS_DEFAULT_REGION = os.environ['AWS_DEFAULT_REGION']
BUCKET = os.environ['BUCKET']
BASEURL = os.environ['BASEURL']
CACHE_CONTROL = os.environ['CACHE_CONTROL']
BRANCH = os.environ['BRANCH']
CONFIG = os.environ['CONFIG']
REPOSITORY = os.environ['REPOSITORY']
OWNER = os.environ['OWNER']
SITE_PREFIX = os.environ['SITE_PREFIX']
GENERATOR = os.environ['GENERATOR']
AWS_ACCESS_KEY_ID = os.environ['AWS_ACCESS_KEY_ID']
AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY']
# List of private strings to be removed from any posted logs
private_values = [AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY]
# Optional environment variables
SOURCE_REPO = os.getenv('SOURCE_REPO', '')
SOURCE_OWNER = os.getenv('SOURCE_OWNER', '')
# GITHUB_TOKEN can be empty if a non-Federalist user
# makes a commit to a repo and thus initiates a build for a
# Federalist site
GITHUB_TOKEN = os.getenv('GITHUB_TOKEN', '')
if GITHUB_TOKEN:
# only include it in list of values to strip from log output
# if it exists
private_values.append(GITHUB_TOKEN)
# Ex: https://federalist-builder.fr.cloud.gov/builds/<token>/callback
FEDERALIST_BUILDER_CALLBACK = os.environ['FEDERALIST_BUILDER_CALLBACK']
# Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/status/<token>
STATUS_CALLBACK = os.environ['STATUS_CALLBACK']
# Ex: https://federalist-staging.18f.gov/v0/build/<build_id>/log/<token>
LOG_CALLBACK = os.environ['LOG_CALLBACK']
try:
# throw a timeout exception after TIMEOUT_SECONDS
with Timeout(TIMEOUT_SECONDS, swallow_exc=False):
LOGGER.info(f'Running build for {OWNER}/{REPOSITORY}/{BRANCH}')
# Unfortunately, pyinvoke doesn't have a great way to call tasks
# from within other tasks. If you call them directly,
# their pre- and post-dependencies are not executed.
#
# Here's the GitHub issue about this:
# https://github.com/pyinvoke/invoke/issues/170
#
# So rather than calling the task functions through python, we'll
# call them instead using `ctx.run('invoke the_task ...')` via the
# helper `run_task` method.
##
# CLONE
#
if SOURCE_OWNER and SOURCE_REPO:
# First clone the source (ie, template) repository
clone_source_flags = {
'--owner': SOURCE_OWNER,
'--repository': SOURCE_REPO,
'--branch': BRANCH,
}
run_task(ctx,
'clone-repo',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=clone_source_flags,
env={'GITHUB_TOKEN': GITHUB_TOKEN})
# Then push the cloned source repo up to the destination repo.
# Note that the dest repo must already exist but be empty.
# The Federalist web app takes care of that operation.
push_repo_flags = {
'--owner': OWNER,
'--repository': REPOSITORY,
'--branch': BRANCH,
}
run_task(ctx,
'push-repo-remote',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=push_repo_flags,
env={'GITHUB_TOKEN': GITHUB_TOKEN})
else:
# Just clone the given repository
clone_flags = {
'--owner': OWNER,
'--repository': REPOSITORY,
'--branch': BRANCH,
'--depth': '--depth 1',
}
run_task(ctx,
'clone-repo',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=clone_flags,
env={'GITHUB_TOKEN': GITHUB_TOKEN})
##
# BUILD
#
build_flags = {
'--branch': BRANCH,
'--owner': OWNER,
'--repository': REPOSITORY,
'--site-prefix': SITE_PREFIX,
'--base-url': BASEURL,
}
# Run the npm `federalist` task (if it is defined)
run_task(ctx,
'run-federalist-script',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=build_flags)
# Run the appropriate build engine based on GENERATOR
if GENERATOR == 'jekyll':
build_flags['--config'] = CONFIG
run_task(ctx,
'build-jekyll',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=build_flags)
elif GENERATOR == 'hugo':
# extra: --hugo-version (not yet used)
run_task(ctx,
'build-hugo',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=build_flags)
elif GENERATOR == 'static':
# no build arguments are needed
run_task(ctx,
'build-static',
log_callback=LOG_CALLBACK,
private_values=private_values)
elif (GENERATOR == 'node.js' or GENERATOR == 'script only'):
LOGGER.info('build already ran in \'npm run federalist\'')
else:
raise ValueError(f'Invalid GENERATOR: {GENERATOR}')
##
# PUBLISH
#
publish_flags = {
'--base-url': BASEURL,
'--site-prefix': SITE_PREFIX,
'--bucket': BUCKET,
'--cache-control': CACHE_CONTROL,
'--aws-region': AWS_DEFAULT_REGION,
}
publish_env = {
'AWS_ACCESS_KEY_ID': AWS_ACCESS_KEY_ID,
'AWS_SECRET_ACCESS_KEY': AWS_SECRET_ACCESS_KEY,
}
run_task(ctx,
'publish',
log_callback=LOG_CALLBACK,
private_values=private_values,
flags_dict=publish_flags,
env=publish_env)
delta_string = delta_to_mins_secs(datetime.now() - start_time)
LOGGER.info(f'Total build time: {delta_string}')
# Finished!
post_build_complete(STATUS_CALLBACK, FEDERALIST_BUILDER_CALLBACK)
except TimeoutException:
LOGGER.info('Build has timed out')
post_build_timeout(LOG_CALLBACK, STATUS_CALLBACK,
FEDERALIST_BUILDER_CALLBACK)
except UnexpectedExit as err:
# Combine the error's stdout and stderr into one string
err_string = format_output(err.result.stdout, err.result.stderr)
# replace any private values that might be in the error message
err_string = replace_private_values(err_string, private_values)
# log the original exception
LOGGER.info(f'Exception raised during build:')
LOGGER.info(err_string)
# replace the message with a custom one, if it exists
err_string = find_custom_error_message(err_string)
post_build_error(LOG_CALLBACK, STATUS_CALLBACK,
FEDERALIST_BUILDER_CALLBACK, err_string)
except Exception as err: # pylint: disable=W0703
# Getting here means something really weird has happened
# since all errors caught during tasks should be caught
# in the previous block as `UnexpectedExit` exceptions.
err_string = str(err)
err_string = replace_private_values(err_string, private_values)
# log the original exception
LOGGER.info('Unexpected exception raised during build:')
LOGGER.info(err_string)
err_message = ('Unexpected error. Please try again and '
'contact federalist-support if it persists.')
post_build_error(LOG_CALLBACK, STATUS_CALLBACK,
FEDERALIST_BUILDER_CALLBACK, err_message)
def build(aws_access_key_id,
aws_default_region,
aws_secret_access_key,
federalist_builder_callback,
status_callback,
baseurl,
branch,
bucket,
build_id,
config,
generator,
github_token,
owner,
repository,
site_prefix,
user_environment_variables=[]):
'''
Main task to run a full site build process.
All values needed for the build are loaded from
environment variables.
'''
# keep track of total time
start_time = datetime.now()
# Make the working directory if it doesn't exist
WORKING_DIR_PATH.mkdir(exist_ok=True)
logger = None
cache_control = os.getenv('CACHE_CONTROL', 'max-age=60')
database_url = os.environ['DATABASE_URL']
user_environment_variable_key = os.environ['USER_ENVIRONMENT_VARIABLE_KEY']
try:
post_build_processing(status_callback)
# throw a timeout exception after TIMEOUT_SECONDS
with Timeout(TIMEOUT_SECONDS, swallow_exc=False):
build_info = f'{owner}/{repository}@id:{build_id}'
decrypted_uevs = decrypt_uevs(user_environment_variable_key,
user_environment_variables)
priv_vals = [uev['value'] for uev in decrypted_uevs]
priv_vals.append(aws_access_key_id)
priv_vals.append(aws_secret_access_key)
if github_token:
priv_vals.append(github_token)
logattrs = {
'branch': branch,
'buildid': build_id,
'owner': owner,
'repository': repository,
}
init_logging(priv_vals, logattrs, database_url)
logger = get_logger('main')
def run_step(returncode, msg):
if returncode != 0:
raise StepException(msg)
logger.info(f'Running build for {owner}/{repository}/{branch}')
if generator not in GENERATORS:
raise ValueError(f'Invalid generator: {generator}')
##
# FETCH
#
run_step(
fetch_repo(owner, repository, branch, github_token),
'There was a problem fetching the repository, see the above logs for details.'
)
##
# BUILD
#
run_step(
setup_node(),
'There was a problem setting up Node, see the above logs for details.'
)
# Run the npm `federalist` task (if it is defined)
run_step(
run_federalist_script(branch, owner, repository, site_prefix,
baseurl, decrypted_uevs),
'There was a problem running the federalist script, see the above logs for details.'
)
# Run the appropriate build engine based on generator
if generator == 'jekyll':
run_step(
setup_ruby(),
'There was a problem setting up Ruby, see the above logs for details.'
)
run_step(
setup_bundler(),
'There was a problem setting up Bundler, see the above logs for details.'
)
run_step(
build_jekyll(branch, owner, repository, site_prefix,
baseurl, config, decrypted_uevs),
'There was a problem running Jekyll, see the above logs for details.'
)
elif generator == 'hugo':
# extra: --hugo-version (not yet used)
run_step(
download_hugo(),
'There was a problem downloading Hugo, see the above logs for details.'
)
run_step(
build_hugo(branch, owner, repository, site_prefix, baseurl,
decrypted_uevs),
'There was a problem running Hugo, see the above logs for details.'
)
elif generator == 'static':
# no build arguments are needed
build_static()
elif (generator == 'node.js' or generator == 'script only'):
logger.info('build already ran in \'npm run federalist\'')
else:
raise ValueError(f'Invalid generator: {generator}')
##
# PUBLISH
#
publish(baseurl, site_prefix, bucket, cache_control,
aws_default_region, aws_access_key_id,
aws_secret_access_key)
delta_string = delta_to_mins_secs(datetime.now() - start_time)
logger.info(f'Total build time: {delta_string}')
# Finished!
post_build_complete(status_callback, federalist_builder_callback)
sys.exit(0)
except StepException as err:
'''
Thrown when a step itself fails, usually because a command exited
with a non-zero return code
'''
logger.error(str(err))
post_build_error(status_callback, federalist_builder_callback,
str(err))
sys.exit(1)
except TimeoutException:
logger.warning(f'Build({build_info}) has timed out')
post_build_timeout(status_callback, federalist_builder_callback)
except Exception as err: # pylint: disable=W0703
# Getting here means something really weird has happened
# since all errors caught during tasks should be caught
# in the previous block as `UnexpectedExit` exceptions.
err_string = str(err)
# log the original exception
msg = f'Unexpected exception raised during build({build_info}): {err_string}'
if logger:
logger.warning(msg)
else:
print(msg)
err_message = (f'Unexpected build({build_info}) error. Please try '
'again and contact federalist-support if it persists.')
post_build_error(status_callback, federalist_builder_callback,
err_message)