def queue_start_demo( github_user, github_repo, github_pr, github_sender=None, github_verify_sender=True, send_github_notification=False, ): demo_url = get_demo_url_pr(github_repo, github_pr) context = get_demo_context( demo_url=demo_url, github_user=github_user, github_repo=github_repo, github_pr=github_pr, ) logger = get_demo_logger(__name__, **context) logger.info( 'Adding demo to queue for %s/%s on PR %s (%s)', github_user, github_repo, github_pr, demo_url, ) tasks = [ start_demo_task.s( context=context, github_sender=github_sender, github_verify_sender=github_verify_sender, **context, ) ] if send_github_notification: tasks.append(notify_github_task.s(context=context, **context)) chain(*tasks).apply_async()
def start_demo_task(self, demo_url, github_user, github_repo, github_pr, context, github_sender=None, github_verify_sender=True, **kwargs): logger = get_demo_logger(__name__, **context) logger.debug('Starting start_demo_task task for %s', demo_url) try: return start_demo( demo_url=demo_url, github_user=github_user, github_repo=github_repo, github_pr=github_pr, github_sender=github_sender, github_verify_sender=github_verify_sender, context=context, ) except Exception as e: logger.error(e) # Retry on failure with a growing cooldown retry_count = self.request.retries seconds_to_wait = 2 * retry_count raise self.retry(exc=e, countdown=seconds_to_wait)
def stop_demo( demo_url, context=None, ): if context: logger = get_demo_logger(__name__, **context) else: logger = logging.getLogger(__name__) logger.info('Stopping demo: %s', demo_url) local_path = os.path.join(settings.DEMO_DIR, demo_url) if not os.path.isdir(local_path): return # Check for the run command to clean run_command_path = os.path.join(local_path, 'run') if os.path.exists(run_command_path): logger.info('Running clean command') p = Popen( ['./run', 'clean'], cwd=local_path, ) p.wait() logger.info('Deleting files for %s', demo_url) shutil.rmtree(local_path)
def notify_github_task( self, message, demo_url, github_user, github_repo, github_pr, context, **kwargs ): logger = get_demo_logger(__name__, **context) logger.debug('Starting notify_github task for %s', demo_url) if not message: logger.debug('No notification message to send for %s', demo_url) return False try: return notify_github_pr( message=message, github_user=github_user, github_repo=github_repo, github_pr=github_pr, context=context, ) except Exception as e: logger.error(e) # If the notification fails, retry with a growing cooldown retry_count = self.request.retries seconds_to_wait = 3 * retry_count + 1 raise self.retry(exc=e, countdown=seconds_to_wait)
def stop_demo_task(self, demo_url, context, **kwargs): logger = get_demo_logger(__name__, **context) logger.debug('Running the stop_demo task for %s', demo_url) try: return stop_demo(demo_url=demo_url, context=context) except Exception as e: logger.error(e) # Retry on failure with a growing cooldown retry_count = self.request.retries seconds_to_wait = 2 * retry_count raise self.retry(exc=e, countdown=seconds_to_wait)
def notify_github_pr( message, github_user, github_repo, github_pr, context=None, ): if context: logger = get_demo_logger(__name__, **context) else: logger = logging.getLogger(__name__) comment = { 'body': message, } api_url = ( 'https://api.github.com' '/repos/{github_user}/{github_repo}/' 'issues/{github_pr}/comments' ).format( github_user=github_user, github_repo=github_repo, github_pr=github_pr, ) logger.debug('GitHub comment API URL: %s', api_url) logger.info('Commenting on pull request: %s', comment['body']) github_token = os.getenv('GITHUB_TOKEN') if not github_token: logger.warning('Missing GITHUB_TOKEN environment variable') return None api_headers = { 'Authorization': 'token {token}'.format(token=github_token) } if settings.DEBUG: logger.debug('Simulating GitHub notification while DEBUG is active') return True session = requests.session() request = session.post(api_url, json.dumps(comment), headers=api_headers) if request.status_code == 201: logger.info('Successfully notified Pull Request') else: logger.warning('Could not notify Pull Request: %s', request.content) raise Exception( 'Failed to notify Github: {content}'.format( content=request.content ) )
def queue_stop_demo( github_user, github_repo, github_pr, ): demo_url = get_demo_url_pr(github_repo, github_pr) context = get_demo_context( demo_url=demo_url, github_user=github_user, github_repo=github_repo, github_pr=github_pr, ) logger = get_demo_logger(__name__, **context) logger.info( 'Adding demo removal to queue for %s/%s on PR %s (%s)', github_user, github_repo, github_pr, demo_url, ) stop_demo_task.delay(context=context, **context)
def start_demo( demo_url, github_user, github_repo, github_pr, github_sender=None, github_verify_sender=True, context=None, ): if context: logger = get_demo_logger(__name__, **context) else: logger = logging.getLogger(__name__) if github_verify_sender: logger.debug('Verifying if user is collaborator of repo', ) if not github_sender: logger.error('GitHub webhook sender is not set for verification') return None logger.debug( 'Verifying %s is a collaborator of %s/%s', github_sender, github_user, github_repo, ) try: if not _is_repo_collaborator(github_user, github_repo, github_sender): logger.info("%s is not a collaborator of this repo", github_sender) return ("User is not a collaborator of this repo. " "Please start demo manually.") except GitHubError as ge: # If user does not have permission to check repo collaborators # an exception with the 403 code is expected. if 403 == ge.code: gh = login('-', password=settings.GITHUB_TOKEN) bot = gh.user().login return ("User {bot} does not have enough permissions " "to perform necessary checks. " "Please review user permissions for this repository." ).format(bot=bot) return "There was a GitHub API error." logger.info('User is a collaborator of the repo') logger.info('Preparing demo: %s', demo_url) os.makedirs(settings.DEMO_DIR, exist_ok=True) local_path = os.path.join(settings.DEMO_DIR, demo_url) run_command_path = os.path.join(local_path, 'run') # Clone repo and update PR if not os.path.isdir(local_path): clone_url = GITHUB_CLONE_URL.format( github_user=github_user, github_repo=github_repo, ) logger.info('Cloning git repo: %s', clone_url) p = Popen(['git', 'clone', clone_url, local_path]) return_code = p.wait() if return_code > 0: logger.error('Error while cloning %s', clone_url) return False elif os.path.exists(run_command_path): logger.info('Cleaning previous run script') p = Popen( ['./run', 'clean'], cwd=local_path, ) p.wait() if github_pr: logger.info('Pulling PR branch for %s', github_pr) p = Popen( ['git', 'pr', str(github_pr)], cwd=local_path, ) return_code = p.wait() if return_code > 0: logger.error('Error while pulling PR %s branch', github_pr) return False p = Popen(['git', 'reset', '--hard', 'HEAD'], cwd=local_path) p.wait() # Check for the run command to continue if not os.path.exists(run_command_path): message = 'No ./run found. Unable to start demo.' logger.info(message) return message # Check the project has the minimum required version of ./run run_script_version = (check_output( ["./run", "--version"], cwd=local_path).decode("utf-8").rstrip().split("@")[-1]) if StrictVersion(run_script_version) < StrictVersion( MIN_RUNSCRIPT_VERSION): message = ( "Unable to start demo. Minimum required " "version of ./run script is {}").format(MIN_RUNSCRIPT_VERSION) logger.info(message) return message # Stop bower complaining about running as root... # This actually updates the run command for now and resets on rerun run_file_contents = open(run_command_path).read() bower_string = 'bower install' bower_string_for_root = 'bower install --allow-root' if bower_string_for_root not in run_file_contents: for line in fileinput.input(run_command_path, inplace=True): print(line.replace(bower_string, bower_string_for_root), end='') # Set the docker name if not created docker_project_path = os.path.join(local_path, '.docker-project') if not os.path.exists(docker_project_path): with open(docker_project_path, "w") as project_file: project_file.write(demo_url) demo_url_path = '' # Check for Jekyll base paths jekyll_config_name = r'^_config\.ya?ml$' jekyll_config_path = False for _file in os.listdir(local_path): # if fnmatch.fnmatch(_file, jekyll_config_name): if re.search(jekyll_config_name, _file): jekyll_config_path = os.path.join(local_path, _file) break if jekyll_config_path: logger.info('Found Jekyll config, looking for baseurl') with open(jekyll_config_path, 'r') as stream: try: jekyll_config = yaml.load(stream) except yaml.YAMLError as e: logger.error('Error parsing Jekyll config YAML: %s', e) demo_url_path = jekyll_config.get('baseurl', '').strip('/') logger.info('Setting demo path to %s', demo_url_path) # Start ./run server with extra options demo_url_full = ''.join(['http://', demo_url, '/']) if demo_url_path: demo_url_full = ''.join([demo_url_full, demo_url_path, '/']) logger.info('Starting demo: %s', demo_url_full) port = _get_open_port() docker_options = '' docker_labels = { 'traefik.enable': 'true', 'traefik.frontend.rule': 'Host:{url}'.format(url=demo_url), 'traefik.port': port, 'run.demo': True, 'run.demo.url': demo_url, 'run.demo.url_full': demo_url_full, 'run.demo.github_user': github_user, 'run.demo.github_repo': github_repo, 'run.demo.github_pr': github_pr, } for key, value in docker_labels.items(): docker_options += " -l {key}={value}".format(key=key, value=value) logger.debug('Docker options: %s', docker_options) # We are going to inject all env var items beginning with DEMO_OPT_ # This allows us to quickly set up global options and secrets for apps. docker_env_opts = { k: v for k, v in os.environ.items() if k.startswith('DEMO_OPT_') } for key, value in docker_env_opts.items(): docker_options += " -e {key}={value}".format(key=key[9:], value=value) run_env = os.environ.copy() run_env["CANONICAL_WEBTEAM_RUN_SERVE_DOCKER_OPTS"] = docker_options serve_args = '' if 'tutorials' in github_repo: serve_args = './tutorials/*/' p = Popen( ['./run', 'serve', '--detach', '--port', str(port), serve_args], cwd=local_path, env=run_env, ) return_code = p.wait() if return_code > 0: raise Exception('Error starting ./run') message = 'Starting demo at: {demo_url}'.format(demo_url=demo_url_full) return message