def rollback(version, app): """ Rollback release to a previous release. """ cli.user() if version and version[0] == 'v': version = version[1:] else: click.echo('Invalid version given.') sys.exit(1) if not version: click.echo(f'Getting latest release for app {app}... ', nl=False) with click_spinner.spinner(): res = api.Releases.get(app=app) version = int(res[0]['id']) - 1 click.echo(click.style('√', fg='green')) if int(version) == 0: click.echo('Unable to rollback a release before v1.') sys.exit(1) click.echo(f'Rolling back to v{version}... ', nl=False) with click_spinner.spinner(): res = api.Releases.rollback(version=version, app=app) click.echo(click.style('√', fg='green')) click.echo(f'Deployed new release... ' + click.style(f'v{res["id"]}', bold=True, fg='magenta'))
def wait_until_online(self): if self.dry_run: logging.info( f"Would wait for '{self['name']}' to come back online") else: logging.info(f"Waiting for '{self['name']}' to come back online", self.log_to_slack) with click_spinner.spinner(): while True: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: s.settimeout(5) result = s.connect_ex((self['name'], 22)) if result == 0: break if self.dry_run: logging.info(f"Would wait for libvirt on '{self['name']}'") else: logging.info(f"Waiting for libvirt on '{self['name']}'", self.log_to_slack) with click_spinner.spinner(): while True: try: if self.execute('virsh list').return_code == 0: break except (ConnectionResetError, UnexpectedExit, CommandTimedOut): pass time.sleep(5)
def deploy(app, message): """ Deploy your app instantly to the Asyncy Cloud """ cli.user() payload = compile_app(app, False) # Also adds a spinner. if payload is None: sys.exit(1) # Error already printed by compile_app. click.echo(f'Deploying app {app}... ', nl=False) with click_spinner.spinner(): config = Config.get(app) release = Releases.create(config, payload, app, message) url = f'https://{app}.asyncyapp.com' click.echo() click.echo( click.style('√', fg='green') + f' Version {release["id"]} of your app has ' f'been queued for deployment\n') click.echo('Waiting for deployment to complete... ', nl=False) with click_spinner.spinner(): if Apps.maintenance(app, maintenance=None): click.echo() click.echo() click.echo('Your app is in maintenance mode.\n' 'Run the following to turn off it off:') cli.print_command('asyncy maintenance off') click.echo() click.echo('Once maintenance mode is turned off, ' 'your app will be deployed immediately.') return state = 'QUEUED' while state in ['DEPLOYING', 'QUEUED']: state = Releases.get(app)[0]['state'] sleep(0.5) click.echo() if state == 'DEPLOYED': click.echo(click.style('√', fg='green') + ' Deployed successfully!') click.echo(f'If your story listens to HTTP requests, visit {url}') elif state == 'FAILED': click.echo(click.style('X', fg='red') + ' Deployment failed!', err=True) click.echo( 'Please use the following command to view your app\'s logs:', err=True) cli.print_command('asyncy logs') else: click.echo( f'An unhandled state of your app has been encountered - {state}', err=True) click.echo(f'Please shoot an email to [email protected]')
def audit_elections( data_opened: str, data_pre_election: str, data_tally: str, bb: str, tally_all: str, group_id: Optional[str] = typer.Argument(None), ): data_pre_election = _load_pre_election(data_pre_election) data_tally = _load_tally_data(data_tally) print("Auditing pre_election - data_tally...") with click_spinner.spinner(): valid = audit_pre_election_tables(data_pre_election, data_tally) print("valid" if valid else "not valid!") del data_pre_election data_opened = _load_opened_data(data_opened) print("Auditing data_tally - data_opened...") with click_spinner.spinner(): valid = audit_tables(data_opened, data_tally) print("valid" if valid else "not valid") del data_tally bb = _load_bb(bb) if group_id is not None: print( f"Filtering bulletin board is enabled, only votes for group {group_id} will be taken into account" ) bb_filtered = {} for vote_id, bb_entry in bb.items(): if bb_entry.group_id == group_id: bb_filtered[vote_id] = bb_entry bb = bb_filtered print("Auditing data_opened - bb...") with click_spinner.spinner(): valid = audit_bb(bb, data_opened) print("valid" if valid else "not valid") del bb tally_all = _load_tally(tally_all) print("Auditing data_opened - tally with all votes...") with click_spinner.spinner(): valid = audit_tables_tally(tally_all, data_opened) print("valid" if valid else "not valid")
def query( self, query: str, query_variables: dict = None, ) -> Union[dict, None]: try: query = gql(query) with click_spinner.spinner(beep=False, disable=False, force=False, stream=sys.stdout): data = self.client.execute( document=query, variable_values=query_variables, ) except TransportServerError: # refresh token response = self.authentication.refresh() if not response["success"]: console.exit_login_required() self.access_token = response["response"]["access_token"] self.client = self._client() raise RetryException("retry") return data
def bulk_insert(self, musics): if not musics: logger.info("no musics to insert") return None if config.debug: with tqdm(total=len(musics), desc="inserting music one by one") as pbar: for music in musics: logger.debug("inserting %s", music) self.upsert_music(music) pbar.update(1) return None j = json.dumps([m.to_dict() for m in musics]) b64 = j.encode('utf-8') data = base64.b64encode(b64) query = f''' mutation {{ bulkInsert(input: {{data: "{data.decode()}"}}) {{ clientMutationId }} }}''' with click_spinner.spinner(disable=config.quiet): return self._post(query)
def init_db(self, erase: bool = False, exclude_schema=None): """ Initializes a database from scratch, creating tables and needed resources. Note that this does not create the database per se. If executing this directly, remember to use an app_context. Resources can hook functions that will be called when this method executes, by subclassing :meth:`teal.resource. Resource.load_resource`. """ assert _app_ctx_stack.top, 'Use an app context.' print('Initializing database...'.ljust(30), end='') with click_spinner.spinner(): if erase: if exclude_schema: # Using then a schema teal sqlalchemy assert isinstance(self.db, SchemaSQLAlchemy) self.db.drop_schema() else: # using regular flask sqlalchemy self.db.drop_all() self._init_db(exclude_schema) self._init_resources() self.db.session.commit() print('done.')
def delete(ctx, opts, owner_repo_package, yes): """ Delete a package from a repository. - OWNER/REPO/PACKAGE: Specify the OWNER namespace (i.e. user or org), the REPO name where the package is stored, and the PACKAGE name (slug) of the package itself. All separated by a slash. Example: 'your-org/awesome-repo/better-pkg'. """ owner, repo, slug = owner_repo_package args = { 'owner': click.style(owner, bold=True), 'repo': click.style(repo, bold=True), 'package': click.style(slug, bold=True) } if not yes and not click.confirm( 'Are you sure you want to delete %(package)s from ' '%(owner)s/%(repo)s?' % args): click.secho('OK! Phew, close call. :-)', fg='yellow') return click.echo('Deleting %(package)s from %(owner)s/%(repo)s ... ' % args, nl=False) context_msg = 'Failed to delete the package!' with handle_api_exceptions(ctx, opts=opts, context_msg=context_msg): with spinner(): delete_package(owner=owner, repo=repo, slug=slug) click.secho('OK', fg='green')
def parse_version(ctx, param, version) -> str: if not version: return version if param.name == "allennlp_version": package = "allennlp" else: package = "allennlp-models" if version.startswith("git@"): git_url = f"https://github.com/allenai/{package}" if version == "git@master": # Get the latest commit from the git repo. click.secho("Checking for latest commit...", fg="yellow") with click_spinner.spinner(): latest_commits = list( shell_out_command(["git", "ls-remote", git_url + ".git"]) ) latest = latest_commits[0].split("\t")[0] version = f"git+{git_url}.git@{latest}" else: version = f"git+{git_url}.{version}" elif version.startswith("git+"): pass else: version = f"{package}=={version}" click.echo("Using " + click.style(f"{version}", fg="green")) return version
def push( docker_registry, docker_repo, docker_username, docker_password, docker_tag, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): secrets = cli_common.taskcluster.get_secrets( taskcluster_secret, None, required=[ 'DOCKER_USERNAME', 'DOCKER_PASSWORD', ], taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) docker_username = docker_username or secrets['DOCKER_USERNAME'] docker_password = docker_password or secrets['DOCKER_PASSWORD'] image_reference = f'docker-daemon:{docker_repo}:{docker_tag}' click.echo(' => Pushing the image to the registry ... ', nl=False) with click_spinner.spinner(): please_cli.utils.push_docker_image( registry=docker_registry, username=docker_username, password=docker_password, image=image_reference, repo=docker_repo, tag=docker_tag, )
def run(self): with click_spinner.spinner(): click.secho('Installing dependencies...', fg='green') delegator.run('yarn') click.secho('Building the app...', fg='green') delegator.run('yarn run build') self.update_index_file()
def create_package(ctx, opts, owner, repo, package_type, skip_errors, **kwargs): """Create a new package via the API.""" click.echo('Creating a new %(package_type)s package ... ' % {'package_type': click.style(package_type, bold=True)}, nl=False) context_msg = 'Failed to create package!' with handle_api_exceptions(ctx, opts=opts, context_msg=context_msg, reraise_on_error=skip_errors): with spinner(): slug_perm, slug = api_create_package(package_format=package_type, owner=owner, repo=repo, **kwargs) click.secho('OK', fg='green') click.echo( 'Created: %(owner)s/%(repo)s/%(slug)s (%(slug_perm)s)' % { 'owner': click.style(owner, fg='magenta'), 'repo': click.style(repo, fg='blue'), 'slug': click.style(slug, fg='green'), 'slug_perm': click.style(slug_perm, bold=True) }) return slug_perm, slug
def login(user, token): """Save your GitHub access token.""" app_dir = click.get_app_dir(QQQ) config_path = f'{app_dir}/{CONFIG_FILE}' # Verify user with click_spinner.spinner(): if not GitHub.verify_token(user, token): click.echo( click.style('Invalid GitHub username or token!', fg='red')) raise click.Abort # Check if file already exists if Path(config_path).is_file(): # File exists, prompt to overwrite click.confirm( f'{click.format_filename(config_path)} already exists, update?', abort=True) # Create config object cp = configparser.ConfigParser() cp['auth'] = {'user': user, 'token': token} # Make sure the qqq dir exists if not Path(app_dir).is_dir(): click.echo(f'Creating directory {click.format_filename(app_dir)}...') Path(app_dir).mkdir(parents=True, exist_ok=True) # Write to config file with open(config_path, 'w') as config_file: cp.write(config_file) click.echo( f'Updated config file located at:\t{click.format_filename(config_path)}' )
def run( name: str, parameter: Tuple[str], cache: bool, wait: bool, owner: str, show_output: bool, ): """Run a report""" params = process_cmd_param_vals(parameter) log.info(f"Running script with parameters {params}") script = api.Script.get(name, owner=owner) with api_error_handler("Error running script"): r = script.run(parameters=params, cache=cache) if wait: with click_spinner.spinner(): while not r.is_complete(): time.sleep(2) r.refresh() log.debug(f"Run completed with status {r.status}") if show_output: click.echo(r.output) if r.status == "SUCCESS": if r.result: success_msg(f"Script result - '{r.result}'") if r.report: report = api.Report.by_id(r.report) success_msg(f"Report generated at {report.web_url}") else: failure_msg( f"Script run failed/cancelled\n{r.error_msg}: {r.error_detail}" ) else: success_msg(f"Script run started, view at {script.web_url}")
def logs(follow, last, service, service_name, app, level): """ Fetch logs for your app """ cli.user() click.echo(f'Retrieving logs for {app}... ', nl=False) with click_spinner.spinner(): app_id = Apps.get_uuid_from_hostname(app) click.echo() cli.track( 'App Logs Requested', { 'App name': app, 'Follow': 'Yes' if follow else 'No', 'Last N': last, 'Source': 'Runtime' if service is False else 'Service', 'Service Name': service_name if service else 'N/A', 'Minimum level': level.capitalize(), }, ) asyncio.get_event_loop().run_until_complete( connect_and_listen_with_retry( app_id, last, follow, service is False, service, service_name, level, ))
def wait( cluster_id: str, superuser_username: str, superuser_password: str, transport: Transport, skip_http_checks: bool, ) -> None: """ Wait for DC/OS to start. """ urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) message = ( 'A cluster may take some time to be ready.\n' 'The amount of time it takes to start a cluster depends on a variety ' 'of factors.\n' 'If you are concerned that this is hanging, try "dcos-docker doctor" ' 'to diagnose common issues.') click.echo(message) cluster_containers = ClusterContainers( cluster_id=cluster_id, transport=transport, ) http_checks = not skip_http_checks with click_spinner.spinner(): if cluster_containers.is_enterprise: cluster_containers.cluster.wait_for_dcos_ee( superuser_username=superuser_username, superuser_password=superuser_password, http_checks=http_checks, ) return cluster_containers.cluster.wait_for_dcos_oss(http_checks=http_checks)
def _process_log_file(self) -> bool: tail = self.tail_file() loops_since_check = 0 processing = True done_match = False with click_spinner.spinner(): while processing: for line in tail.readlines(): self.logger.debug(f"log: {line}") done_match = (re.search(r"Done \((\d+\.\d+)s\)! For help,", line) is not None) if done_match: processing = False elif "agree to the EULA" in line: self.logger.info("") raise click.ClickException( f"You must agree to Mojang's EULA. " "Please read {EULA_URL} and restart server " "with --accept_eula") if loops_since_check < 5: loops_since_check += 1 elif self.is_running(): loops_since_check = 0 else: self.logger.error(f"{self.server_name} failed to start") processing = False time.sleep(1) self.delete_offset() return done_match
def cmd( ctx, project, nix_shell, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): checks = please_cli.config.PROJECTS_CONFIG.get(project, {}).get('checks') if not checks: raise click.ClickException( 'No checks found for `{}` project.'.format(project)) for check_title, check_command in checks: click.echo(' => {}: '.format(check_title), nl=False) with click_spinner.spinner(): returncode, output, error = ctx.invoke( please_cli.shell.cmd, project=project, quiet=True, command=check_command, nix_shell=nix_shell, taskcluster_secret=taskcluster_secret, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) please_cli.utils.check_result(returncode, output, raise_exception=False)
def generic_wait_command(manager_name, attrs, ctx, uuid, refresh_period): """Performs a generic wait command for task instances. Args: manager_name: A string containing the name of the saltant.client.Client's manager to use. For example, "executable_task_instances". attrs: An iterable containing the attributes of the object to use when displaying it. ctx: A click.core.Context object containing information about the Click session. uuid: A string containing the uuid of the task instance to wait for. refresh_period: A float specifying how many seconds to wait in between checking the task's status. """ # Get the client from the context client = ctx.obj["client"] # Terminate the task instance try: manager = getattr(client, manager_name) # Wait for the task instance to finish with click_spinner.spinner(): object = manager.wait_until_finished(uuid, refresh_period) # Output a list display of the task instance output = generate_list_display(object, attrs) except BadHttpRequestError: # Bad request output = "task instance %s not found" % uuid click.echo(output)
def start(foreground): """ Start the daemon """ client = DaemonClient() click.echo('Starting the daemon... ', nl=False) if foreground: command = [ 'verdi', '-p', client.profile_name, 'daemon', '_start_circus', '--foreground' ] else: command = [ 'verdi', '-p', client.profile_name, 'daemon', '_start_circus' ] try: currenv = get_env_with_venv_bin() subprocess.check_output(command, env=currenv) except subprocess.CalledProcessError as exception: click.echo('failed: {}'.format(exception)) sys.exit(1) # We add a small timeout to give the pid-file a chance to be created with spinner(): time.sleep(1) response = client.get_status() print_client_response_status(response)
def get_ssh_conn(host, username, password=None, ssh_key=None): """Connect to a remote system by SSH port 22. By default it will retry 60 times until it establishes an ssh connection. :param host: Remote systems IP address :type host: str :param username: Username :type username: str :param password: Password :type password: str :param ssh_key: SSH private key for authentication :type ssh_key: str """ with spinner(): try: ssh = SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(AutoAddPolicy()) if ssh_key: ssh.connect(hostname=host, username=username, key_filename=ssh_key, timeout=5) if password: ssh.connect(hostname=host, username=username, password=password, timeout=5) ssh.close() LOG.info("Successfully established SSH connection to %s", host) except (error, SSHException, timeout): raise SSHError('Port 22 is unreachable.')
def exec_cmd_by_ssh(host, username, cmd, password=None, ssh_key=None, fire_forget=False): """Connect to a remote system by SSH port 22 and run a command. By default it will retry 60 times until it establishes an ssh connection. :param host: Remote systems IP address :type host: str :param cmd: Command to execute :type cmd: str :param username: Username :type username: str :param password: Password :type password: str :param ssh_key: SSH private key for authentication :type ssh_key: str :param fire_forget: fire and forget the command :type fire_forget: bool """ with spinner(): try: ssh = SSHClient() ssh.load_system_host_keys() ssh.set_missing_host_key_policy(AutoAddPolicy()) if ssh_key: ssh.connect(hostname=host, username=username, key_filename=ssh_key, timeout=30) if password: ssh.connect(hostname=host, username=username, password=password, timeout=30) if fire_forget: # fire and forget the command given, will return no output _timeout = 0.5 channel = ssh.get_transport().open_session(timeout=_timeout) channel.settimeout(_timeout) channel.exec_command(cmd) LOG.info('Successfully executed command: %s' % cmd) else: results = ssh.exec_command(cmd) return_code = results[1].channel.recv_exit_status() if return_code: LOG.error('Command %s failed to execute.' % cmd) LOG.error(results[2].read()) ssh.close() raise SystemExit(return_code) else: LOG.debug(results[1].read().strip()) LOG.info("Successfully executed command: %s", cmd) ssh.close() except (error, SSHException, timeout): raise SSHError('Port 22 is unreachable.')
def push(docker_registry, docker_repo, docker_username, docker_password, docker_tag, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): secrets = cli_common.taskcluster.get_secrets( taskcluster_secret, None, required=[ 'DOCKER_USERNAME', 'DOCKER_PASSWORD', ], taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) docker_username = docker_username or secrets['DOCKER_USERNAME'] docker_password = docker_password or secrets['DOCKER_PASSWORD'] image_reference = f'docker-daemon:{docker_repo}:{docker_tag}' click.echo(' => Pushing the image to the registry ... ', nl=False) with click_spinner.spinner(): please_cli.utils.push_docker_image( registry=docker_registry, username=docker_username, password=docker_password, image=image_reference, repo=docker_repo, tag=docker_tag, )
def start(foreground, number): """Start the daemon with NUMBER workers. If the NUMBER of desired workers is not specified, the default is used, which is determined by the configuration option `daemon.default_workers`, which if not explicitly changed defaults to 1. """ from aiida.engine.daemon.client import get_daemon_client client = get_daemon_client() echo.echo('Starting the daemon... ', nl=False) if foreground: command = ['verdi', '-p', client.profile.name, 'daemon', _START_CIRCUS_COMMAND, '--foreground', str(number)] else: command = ['verdi', '-p', client.profile.name, 'daemon', _START_CIRCUS_COMMAND, str(number)] try: currenv = get_env_with_venv_bin() subprocess.check_output(command, env=currenv, stderr=subprocess.STDOUT) # pylint: disable=unexpected-keyword-arg except subprocess.CalledProcessError as exception: click.secho('FAILED', fg='red', bold=True) echo.echo_critical(str(exception)) # We add a small timeout to give the pid-file a chance to be created with spinner(): time.sleep(1) response = client.get_status() print_client_response_status(response)
def init_db(self, name: str, org_name: str, org_id: str, tag_url: boltons.urlutils.URL, tag_token: uuid.UUID, erase: bool, common: bool): """Creates an inventory. This creates the database and adds the inventory to the inventory tables with the passed-in settings, and does nothing if the inventory already exists. After you create the inventory you might want to create an user executing *dh user add*. """ assert _app_ctx_stack.top, 'Use an app context.' print('Initializing database...'.ljust(30), end='') with click_spinner.spinner(): if erase: self.db.drop_all(common_schema=common) assert not db.has_schema( self.id), 'Schema {} already exists.'.format(self.id) exclude_schema = 'common' if not common else None self._init_db(exclude_schema=exclude_schema) InventoryDef.set_inventory_config(name, org_name, org_id, tag_url, tag_token) DeviceSearch.set_all_devices_tokens_if_empty(self.db.session) self._init_resources(exclude_schema=exclude_schema) self.db.session.commit() print('done.')
def wait_for_job(self, job_id, retries=10): job_status = 0 with click_spinner.spinner(): while True: if retries <= 0: break try: job_status = self.cs.queryAsyncJobResult(jobid=job_id).get('jobstatus', 0) except CloudStackException as e: if 'multiple JSON fields named jobstatus' not in str(e): raise e logging.debug(e) retries -= 1 except ConnectionError as e: if 'Connection aborted' not in str(e): raise e logging.debug(e) retries -= 1 if int(job_status) == 1: return True elif int(job_status) == 2: break time.sleep(1) return False
def token(username, password): """ Request a token for using Pindo API. """ click.echo('token...') with click_spinner.spinner(): click.echo(Token(username, password))
async def fullscan(db, folders=None, concurrency=1, crawl=False): if folders is None: folders = await db.folders() folders = [f['name'] for f in folders] with click_spinner.spinner(): files = [ f for f in find_files(list(folders)) if f[1].endswith(tuple(supported_formats)) ] size = len(files) * 2 if crawl else len(files) with tqdm(total=size, file=sys.stdout, desc="Loading music", leave=True, position=0, disable=config.quiet) as bar: async def insert(semaphore, f): async with semaphore: try: m = File(f[1], f[0]) if crawl: await m.find_youtube() bar.update(1) debug(m.to_list()) await db.upsert(m) bar.update(1) except asyncpg.exceptions.CheckViolationError as e: warning("Violation: {}".format(e)) semaphore = asyncio.BoundedSemaphore(concurrency) tasks = [asyncio.ensure_future(insert(semaphore, f)) for f in files] await asyncio.gather(*tasks) await db.refresh()
def run(*groups, users: int = 5, duration: int = 0, logger=None, show_progress_bar: bool = False, halt_and_catch_fire: bool = False) -> Report: cm = contextlib.nullcontext() if show_progress_bar and not logger: cm = click_spinner.spinner() with cm: started = time.time() executions = [] if duration: while time.time() - started <= duration: res = _run(*groups, users=users, logger=logger, halt_and_catch_fire=halt_and_catch_fire) executions.extend(res) else: res = _run(*groups, users=users, logger=logger, halt_and_catch_fire=halt_and_catch_fire) executions.extend(res) return Report(executions)
def compile_app(app_name_for_analytics, debug) -> dict: """ Compiles, prints pretty info, and returns the compiled tree. :return: The compiled tree """ from storyscript.App import App click.echo(click.style('Compiling Stories...', bold=True)) with click_spinner.spinner(): try: stories = json.loads(App.compile(os.getcwd())) except BaseException: import traceback traceback.print_exc() click.echo('Failed', err=True) stories = None result = 'Success' count = 0 if stories is None: result = 'Failed' else: count = len(stories.get('stories', {})) cli.track('App Compiled', { 'App name': app_name_for_analytics, 'Result': result, 'Stories': count }) return stories
def refresh_token(username, password): """ Refresh a Token. """ click.echo('token...') with click_spinner.spinner(): click.echo(RefreshToken(username, password))
def genfiles(folders): with click_spinner.spinner(disable=config.quiet): count = 0 directories = [os.path.abspath(f) for f in folders] for d in directories: count += filecount(d, supported_formats) logger.info("File count: %s", count) files = [] with tqdm(total=count, desc="Music listing", leave=False, disable=config.quiet) as pbar: file_list = find_files(folders, supported_formats) music_files = list(file_list) for f in music_files: m = File(f[1], f[0]) files.append(m) pbar.update(1) return files
def main(filename): if not filename.endswith('.md'): raise click.BadParameter("Please provide a Markdown file ending with .md") input_filename = filename output_filename = filename.replace('md', 'pdf') event_handler = PandocHandler(input_filename, output_filename) observer = Observer() observer.schedule(event_handler, path='.') observer.start() try: with spinner(): while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join()
def bulk_insert(self, musics): if not musics: logger.info("no musics to insert") return None j = json.dumps([m.to_dict() for m in musics]) b64 = j.encode('utf-8') data = base64.b64encode(b64) query = ''' mutation {{ bulkInsert(input: {{data: "{}"}}) {{ clientMutationId }} }}'''.format(data.decode()) with click_spinner.spinner(disable=config.quiet): return self._post(query)
def unlinked_references(doi_file_name, arxiv_file_name): """Find often cited literature that is not on INSPIRE. It generates two files with a list of DOI/arxiv ids respectively, in which each line has the respective identifier, folowed by two numbers, representing the amount of times the literature has been cited by a core and a non-core article respectively. The lists are ordered by an internal measure of relevance.""" with click_spinner.spinner(): click.echo('Looking up unlinked references...') result_doi, result_arxiv = check_unlinked_references() click.echo('Done!') click.echo(u'Output written to "{}" and "{}"'.format(doi_file_name.name, arxiv_file_name.name)) for item in result_doi: doi_file_name.write(u'{i[0]}: {i[1]}\n'.format(i=item)) for item in result_arxiv: arxiv_file_name.write(u'{i[0]}: {i[1]}\n'.format(i=item))
def cmd(ctx, project, nix_build, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): with click_spinner.spinner(): click.echo(f' => Testing project {project} ...', nl=False) outputs = ctx.invoke(please_cli.build.cmd, project=project, nix_path_attributes=[project], interactive=False, nix_build=nix_build, taskcluster_secret=taskcluster_secret, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) if len(outputs) == 1: click.secho('DONE', fg='green') else: click.secho('ERROR', fg='red')
def run_update(project, nix_shell, root_dir, interactive): logger.info('Running dependency update.') command = [ nix_shell, please_cli.config.ROOT_DIR + '/nix/update.nix', '--argstr', 'pkg', project ] click.echo('Updating dependencies of {}: '.format(project), nl=False) with click_spinner.spinner(): returncode, output, error = cli_common.command.run( command, stream=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=root_dir, ) please_cli.utils.check_result( returncode, output, raise_exception=True, ask_for_details=interactive, )
def cmd(ctx, project, quiet, nix_shell, interactive, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): project_config = please_cli.config.PROJECTS_CONFIG.get(project, {}) run_type = project_config.get('run') run_options = project_config.get('run_options', {}) if not run_type: raise click.ClickException('Application `{}` is not configured to be runnable.'.format(project)) host = run_options.get('host', os.environ.get('HOST', '127.0.0.1')) port = str(run_options.get('port', 8000)) schema = 'https://' project_name = project.replace('-', '_') ca_cert_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'ca.crt') server_cert_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'server.crt') server_key_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'server.key') os.environ['DEBUG'] = 'true' os.environ['PROJECT_NAME'] = project_name pg_host = please_cli.config.PROJECTS_CONFIG['postgresql']['run_options'].get('host', host) pg_port = str(please_cli.config.PROJECTS_CONFIG['postgresql']['run_options']['port']) redis_host = please_cli.config.PROJECTS_CONFIG['redis']['run_options'].get('host', host) redis_port = str(please_cli.config.PROJECTS_CONFIG['redis']['run_options']['port']) if 'postgresql' in project_config.get('requires', []): dbname = 'services' click.echo(' => Checking if database `{}` exists ... '.format(dbname), nl=False) with click_spinner.spinner(): result, output, error = ctx.invoke( please_cli.shell.cmd, project=project, quiet=True, command=' '.join([ 'psql', '-lqt', '-h', pg_host, '-p', pg_port, ]), nix_shell=nix_shell, ) if result != 0 and 'psql: could not connect to server' in output: click.secho('ERROR', fg='red') raise click.UsageError( 'Could not connect to the database.\n\n' 'Please run:\n\n' ' ./please run postgresql\n\n' 'in a separate terminal.' ) please_cli.utils.check_result(result, output, ask_for_details=interactive) database_exists = False for line in output.split('\n'): column1 = line.split('|')[0].strip() if column1 == dbname: database_exists = True break if not database_exists: click.echo(' => Creating `{}` database ` ... '.format(dbname), nl=False) with click_spinner.spinner(): result, output, error = ctx.invoke( please_cli.shell.cmd, project=project, command=' '.join([ 'createdb', '-h', pg_host, '-p', pg_port, dbname, ]), nix_shell=nix_shell, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) os.environ['DATABASE_URL'] = 'postgresql://{}:{}/{}'.format( pg_host, pg_port, dbname ) if 'redis' in project_config.get('requires', []): # Check redis is running click.echo(' => Checking if redis is running... ', nl=False) with click_spinner.spinner(): result, output, error = ctx.invoke( please_cli.shell.cmd, project=project, quiet=True, command='redis-cli -h {} -p {} ping'.format(redis_host, redis_port), nix_shell=nix_shell, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) # Setup config for client application os.environ['REDIS_URL'] = 'redis://{}:{}'.format(redis_host, redis_port) if run_type == 'POSTGRESQL': data_dir = run_options.get('data_dir', os.path.join(please_cli.config.TMP_DIR, 'postgresql')) if not os.path.isdir(data_dir): click.echo(' => Initialize database folder `{}` ... '.format(data_dir), nl=False) with click_spinner.spinner(): result, output, error = ctx.invoke(please_cli.shell.cmd, project=project, command='initdb -D {} --auth=trust'.format(data_dir), nix_shell=nix_shell, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) schema = '' command = [ 'postgres', '-D', data_dir, '-h', host, '-p', port, ] elif run_type == 'REDIS': data_dir = run_options.get('data_dir', os.path.join(please_cli.config.TMP_DIR, 'redis')) if not os.path.isdir(data_dir): os.makedirs(data_dir) command = [ 'redis-server', '--dir', data_dir, '--bind', host, '--port', port, ] elif run_type == 'FLASK': for env_name, env_value in run_options.get('envs', {}).items(): env_name = please_cli.utils.normalize_name(env_name).upper() os.environ[env_name] = env_value if not os.path.exists(ca_cert_file) or \ not os.path.exists(server_cert_file) or \ not os.path.exists(server_key_file): ctx.invoke(please_cli.create_certs.cmd, certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'), ) project_cache_dir = os.path.join(please_cli.config.TMP_DIR, 'cache', project_name) if not os.path.isdir(project_cache_dir): os.makedirs(project_cache_dir) os.environ['CACHE_TYPE'] = 'filesystem' os.environ['CACHE_DIR'] = project_cache_dir os.environ['APP_SETTINGS'] = os.path.join( please_cli.config.ROOT_DIR, 'src', project_name, 'settings.py') os.environ['APP_URL'] = '{}{}:{}'.format(schema, host, port) os.environ['CORS_ORIGINS'] = '*' command = [ 'gunicorn', please_cli.utils.normalize_name(project_name) + '.flask:app', '--bind', '{}:{}'.format(host, port), '--ca-certs={}'.format(ca_cert_file), '--certfile={}'.format(server_cert_file), '--keyfile={}'.format(server_key_file), '--workers', '2', '--timeout', '3600', '--reload', '--reload-engine=poll', '--log-file', '-', ] elif run_type == 'SPHINX': schema = 'http://' command = [ 'HOST=' + host, 'PORT=' + port, 'python', 'run.py', ] elif run_type == 'ELM': if not os.path.exists(ca_cert_file) or \ not os.path.exists(server_cert_file) or \ not os.path.exists(server_key_file): ctx.invoke(please_cli.create_certs.cmd, certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'), ) os.environ['WEBPACK_RELEASE_VERSION'] = please_cli.config.VERSION os.environ['WEBPACK_RELEASE_CHANNEL'] = 'development' os.environ['SSL_CACERT'] = ca_cert_file os.environ['SSL_CERT'] = server_cert_file os.environ['SSL_KEY'] = server_key_file os.environ['HOST'] = host os.environ['PORT'] = port for env_name, env_value in run_options.get('envs', {}).items(): env_name = 'WEBPACK_' + please_cli.utils.normalize_name(env_name).upper() os.environ[env_name] = env_value # XXX: once we move please_cli.config.PROJECTS to nix we wont need this for require in project_config.get('requires', []): env_name = 'WEBPACK_{}_URL'.format(please_cli.utils.normalize_name(require).upper()) env_value = '{}://{}:{}'.format( please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('schema', 'https'), please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('host', host), please_cli.config.PROJECTS_CONFIG[require]['run_options']['port'], ) os.environ[env_name] = env_value command = [ 'webpack-dev-server', '--host', host, '--port', port, '--config', os.path.join(please_cli.config.ROOT_DIR, 'src', project_name, 'webpack.config.js'), ] elif run_type == 'NEUTRINO': if not os.path.exists(ca_cert_file) or \ not os.path.exists(server_cert_file) or \ not os.path.exists(server_key_file): ctx.invoke(please_cli.create_certs.cmd, certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'), ) envs = dict( SSL_CACERT=ca_cert_file, SSL_CERT=server_cert_file, SSL_KEY=server_key_file, HOST=host, PORT=port, RELEASE_VERSION=please_cli.config.VERSION, RELEASE_CHANNEL='development', ) for require in project_config.get('requires', []): env_name = '{}_URL'.format(please_cli.utils.normalize_name(require).upper()) env_value = '{}://{}:{}'.format( please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('schema', 'https'), please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('host', host), please_cli.config.PROJECTS_CONFIG[require]['run_options']['port'], ) envs[env_name] = env_value for env_name, env_value in run_options.get('envs', {}).items(): env_name = please_cli.utils.normalize_name(env_name).upper() envs[env_name] = env_value for env_name, env_value in envs.items(): os.environ[env_name] = env_value command = ['yarn', 'start'] click.echo(' => Running {} on {}{}:{} ...'.format(project, schema, host, port)) returncode, output, error = ctx.invoke(please_cli.shell.cmd, project=project, quiet=quiet, command=' '.join(command), nix_shell=nix_shell, taskcluster_secret=taskcluster_secret, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) sys.exit(returncode)
def build(docker, docker_repo, docker_tag, nix_cache_public_keys, nix_cache_public_urls, interactive, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, ): secrets = cli_common.taskcluster.get_secrets( taskcluster_secret, None, required=[ 'NIX_CACHE_PUBLIC_KEYS', 'NIX_CACHE_PUBLIC_URLS', ], taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) nix_cache_public_keys = nix_cache_public_keys or secrets['NIX_CACHE_PUBLIC_KEYS'] nix_cache_public_urls = nix_cache_public_urls or secrets['NIX_CACHE_PUBLIC_URLS'] docker = please_cli.utils.which(docker) assert docker is not None, \ 'Docker is not available in your environment. Please use: nix-env -f /etc/nix/nixpkgs -iA docker' docker_file = os.path.join(please_cli.config.ROOT_DIR, 'nix', 'docker', 'Dockerfile') nixpkgs_json_file = os.path.join(please_cli.config.ROOT_DIR, 'nix', 'nixpkgs.json') nix_json_file = os.path.join(please_cli.config.ROOT_DIR, 'nix', 'nix.json') temp_docker_file = os.path.join(please_cli.config.ROOT_DIR, 'Dockerfile') with open(nixpkgs_json_file) as f: nixpkgs_json = json.load(f) if 'rev' not in nixpkgs_json or \ 'owner' not in nixpkgs_json or \ 'repo' not in nixpkgs_json or \ 'sha256_tarball' not in nixpkgs_json: raise click.ClickException('`nix/nixpkgs.json` is not of correct format.') with open(nix_json_file) as f: nix_json = json.load(f) if 'version' not in nix_json or \ 'sha256' not in nix_json: raise click.ClickException('`nix/nix.json` is not of correct format.') try: shutil.copyfile(docker_file, temp_docker_file) click.echo(' => Building base docker image ... ', nl=False) with click_spinner.spinner(): result, output, error = cli_common.command.run( [ docker, 'build', '--no-cache', '--pull', '--force-rm', '--build-arg', 'NIXPKGS_OWNER=' + nixpkgs_json['owner'], '--build-arg', 'NIXPKGS_REPO=' + nixpkgs_json['repo'], '--build-arg', 'NIXPKGS_REV=' + nixpkgs_json['rev'], '--build-arg', 'NIXPKGS_SHA256=' + nixpkgs_json['sha256_tarball'], '--build-arg', 'NIX_VERSION=' + nix_json['version'], '--build-arg', 'NIX_SHA256=' + nix_json['sha256'], '--build-arg', 'NIX_CACHE_PUBLIC_KEYS=' + ' '.join(nix_cache_public_keys), '--build-arg', 'NIX_CACHE_PUBLIC_URLS=' + ' '.join(nix_cache_public_urls), '-t', f'{docker_repo}:{docker_tag}', please_cli.config.ROOT_DIR, ], stream=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) finally: if os.path.exists(temp_docker_file): os.unlink(temp_docker_file)
def cmd_docker(project, nix_build, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, docker, interactive, load_image, ): image_path = please_cli.config.TMP_DIR + '/result-docker-{project}'.format(project=project) # Build docker image for project click.echo(' => Building docker image for {}'.format(project)) with click_spinner.spinner(): command = [ nix_build, please_cli.config.ROOT_DIR + '/nix/default.nix', '-A', '{}.docker'.format(project), '-o', image_path ] result, output, error = cli_common.command.run( command, stream=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) please_cli.utils.check_result( result, output, ask_for_details=interactive, ) if not load_image: real_image_path = os.path.realpath(image_path) click.echo(' => Copying docker image {} to {}'.format(real_image_path, please_cli.config.TMP_DIR)) with click_spinner.spinner(): command = ['cp', real_image_path, please_cli.config.TMP_DIR] result, output, error = cli_common.command.run( command, stream=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) click.echo('You can load the image with this command: \n$ {docker} load -i {real_image_path}'.format( docker=docker, real_image_path=real_image_path, )) return # Loading docker image click.echo(' => Importing docker image from {}'.format(image_path)) with click_spinner.spinner(): command = [ docker, 'load', '-i', image_path, ] result, output, error = cli_common.command.run( command, stream=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) please_cli.utils.check_result( result, output, ask_for_details=interactive, ) click.echo(' => Image loaded')
def cmd(ctx, project, github_commit, task_group_id, nix_path_attributes, nix_build, nix, cache_bucket, cache_region, taskcluster_secret, taskcluster_client_id, taskcluster_access_token, interactive, ): required_secrets = [ 'NIX_CACHE_SECRET_KEYS', ] if cache_bucket and cache_region: required_secrets += [ 'CACHE_ACCESS_KEY_ID', 'CACHE_SECRET_ACCESS_KEY', ] secrets = cli_common.taskcluster.get_secrets(taskcluster_secret, project, required=required_secrets, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) temp_files = [] nix_cache_secret_keys = [] for secret_key in secrets['NIX_CACHE_SECRET_KEYS']: fd, temp_file = tempfile.mkstemp(text=True) with open(temp_file, 'w') as f: f.write(secret_key) os.close(fd) nix_cache_secret_keys += [ '--option', 'secret-key-files', temp_file, ] temp_files.append(temp_file) outputs = [] trace = ctx.obj['verbose'] > 1 and ['--show-trace'] or [] for nix_path_attribute in nix_path_attributes: click.echo(' => Building {} ... '.format(nix_path_attribute), nl=False) argstrs = [] if github_commit: argstrs.extend(['--argstr', 'githubCommit', github_commit]) if task_group_id: argstrs.extend(['--argstr', 'taskGroupId', task_group_id]) output = os.path.join( please_cli.config.TMP_DIR, 'result-build-{}'.format(nix_path_attribute.replace('.', '-').replace('_', '-')), ) outputs.append(output) with click_spinner.spinner(): result, output, error = cli_common.command.run( [ nix_build, please_cli.config.ROOT_DIR + '/nix/default.nix', '-A', nix_path_attribute, '-o', output, ] + nix_cache_secret_keys + argstrs + trace, stream=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, ) please_cli.utils.check_result( result, output, ask_for_details=interactive, ) if result != 0: break if cache_bucket and cache_region: tmp_cache_dir = os.path.join(please_cli.config.TMP_DIR, 'cache') if not os.path.exists(tmp_cache_dir): os.makedirs(tmp_cache_dir) os.environ['AWS_ACCESS_KEY_ID'] = secrets['CACHE_ACCESS_KEY_ID'] os.environ['AWS_SECRET_ACCESS_KEY'] = secrets['CACHE_SECRET_ACCESS_KEY'] command = [ nix, 'copy', '--to', 's3://{}?region={}'.format(cache_bucket, cache_region), '-vvvv', ] + outputs click.echo(' => Creating cache artifacts for {} project ... '.format(project), nl=False) with click_spinner.spinner(): result, output, error = cli_common.command.run( command, stream=True, stderr=subprocess.STDOUT, ) please_cli.utils.check_result( result, output, ask_for_details=interactive, ) for temp_file in temp_files: os.remove(temp_file) return outputs
def cmd(certificates_dir, openssl, c_rehash, openssl_config, interactive=True): if not os.path.isdir(certificates_dir): click.echo(' => Creating certificates directory ... ') with click_spinner.spinner(): os.makedirs(certificates_dir) please_cli.utils.check_result(0, '', ask_for_details=interactive) ca_key_file = os.path.join(certificates_dir, 'ca.key') ca_cert_file = os.path.join(certificates_dir, 'ca.crt') if os.path.exists(ca_key_file) or os.path.exists(ca_cert_file): click.echo(' => Removing existing certificates ... ') with click_spinner.spinner(): if os.path.exists(ca_key_file): os.unlink(ca_key_file) if os.path.exists(ca_cert_file): os.unlink(ca_cert_file) please_cli.utils.check_result(0, '', ask_for_details=interactive) click.echo(' => Building CA certificate key ... ', nl=False) with click_spinner.spinner(): result, output, error = cli_common.command.run( [ openssl, 'genrsa', '-out', ca_key_file, '2048', ], stream=True, stderr=subprocess.STDOUT, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) click.echo(' => Self signing CA certificate ... ', nl=False) with click_spinner.spinner(): result, output, error = cli_common.command.run( [ openssl, 'req', '-x509', '-new', '-nodes', '-key', ca_key_file, '-days', '1024', '-out', ca_cert_file, '-subj', '/C=FR/ST=France/L=Paris/O=Mozilla/OU=Dev/CN=RelEngServices', ], stream=True, stderr=subprocess.STDOUT, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) server_key_file = os.path.join(certificates_dir, 'server.key') server_cert_file = os.path.join(certificates_dir, 'server.crt') server_csr_file = os.path.join(certificates_dir, 'server.csr') server_cnf_file = os.path.join(certificates_dir, 'server.cnf') if os.path.exists(server_key_file): os.unlink(server_key_file) if os.path.exists(server_cert_file): os.unlink(server_cert_file) if os.path.exists(server_csr_file): os.unlink(server_csr_file) if os.path.exists(server_cnf_file): os.unlink(server_cnf_file) click.echo(' => Building backend private certificate key ... ', nl=False) with click_spinner.spinner(): result, output, error = cli_common.command.run( [ openssl, 'genrsa', '-out', server_key_file, '2048', ], stream=True, stderr=subprocess.STDOUT, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) click.echo(' => Creating openssh configuration ... ', nl=False) with open(openssl_config, 'r') as f: openssl_config_content = f.read() with open(server_cnf_file, 'w+') as f: f.write('{}\n[SAN]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1'.format(openssl_config_content)) please_cli.utils.check_result(0, '', ask_for_details=interactive) click.echo(' => Building backend csr certificate with mandatory subjectAltName ... ', nl=False) with click_spinner.spinner(): result, output, error = cli_common.command.run( [ openssl, 'req', '-sha256', '-new', '-key', server_key_file, '-out', server_csr_file, '-subj', '/C=FR/ST=France/L=Paris/O=Mozilla/OU=Dev/CN=localhost', '-reqexts', 'SAN', '-config', server_cnf_file, ], stream=True, stderr=subprocess.STDOUT, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) click.echo(' => Signing server certificate with CA certificate ... ', nl=False) with click_spinner.spinner(): result, output, error = cli_common.command.run( [ openssl, 'x509', '-req', '-in', server_csr_file, '-CA', ca_cert_file, '-CAkey', ca_key_file, '-CAcreateserial', '-out', server_cert_file, '-days', '500', '-extensions', 'SAN', '-extfile', server_cnf_file, ], stream=True, stderr=subprocess.STDOUT, ) please_cli.utils.check_result(result, output, ask_for_details=interactive) click.echo(' => Hash certificates directory ... ', nl=False) with click_spinner.spinner(): os.unlink(server_csr_file) result, output, error = cli_common.command.run( [c_rehash, certificates_dir], stream=True, stderr=subprocess.STDOUT, ) please_cli.utils.check_result(result, output, ask_for_details=interactive)
def cmd(project, cache_urls, nix_instantiate, channel, indent=0, interactive=True, ): '''Command to check if project is already in cache. ''' indent = ' ' * indent channel_derivations = dict() nix_path_attributes = [project] deploys = please_cli.config.PROJECTS_CONFIG.get(project, dict()).get('deploys', []) for deploy in deploys: for _channel, options in deploy.get('options', dict()).items(): if _channel in please_cli.config.DEPLOY_CHANNELS: nix_path_attribute = options.get('nix_path_attribute') if nix_path_attribute: nix_path_attributes.append(project + '.' + nix_path_attribute) else: nix_path_attributes.append(project) nix_path_attributes = list(set(nix_path_attributes)) for nix_path_attribute in nix_path_attributes: project_exists = False click.echo('{} => Calculating `{}` hash ... '.format(indent, nix_path_attribute), nl=False) command = [ nix_instantiate, os.path.join(please_cli.config.ROOT_DIR, 'nix/default.nix'), '-A', nix_path_attribute, ] if interactive: with click_spinner.spinner(): result, output, error = cli_common.command.run( command, stream=True, stderr=subprocess.STDOUT, ) else: result, output, error = cli_common.command.run( command, stream=True, stderr=subprocess.STDOUT, ) try: drv = output.split('\n')[-1].strip() with open(drv) as f: channel_derivations[nix_path_attribute] = eval(f.read()) except Exception as e: log.exception(e) raise click.ClickException('Something went wrong when reading derivation file for `{}` project.'.format(nix_path_attribute)) click.echo('{} found.'.format(channel_derivations[nix_path_attribute].nix_hash)) click.echo('{} => Checking cache if build artifacts exists for `{}` ... '.format(indent, nix_path_attribute), nl=False) with click_spinner.spinner(): project_exists = False for cache_url in cache_urls: response = requests.get( '%s/%s.narinfo' % (cache_url, channel_derivations[nix_path_attribute].nix_hash), ) project_exists = response.status_code == 200 if project_exists: break result = 1 if project_exists: result = 0 please_cli.utils.check_result( result, success_message='EXISTS', error_message='NOT EXISTS', raise_exception=False, ask_for_details=interactive, ) return project_exists, channel_derivations[project].nix_hash
def cmd(ctx, github_commit, channel, owner, pull_request, task_id, cache_urls, nix_instantiate, taskcluster_client_id, taskcluster_access_token, dry_run, ): '''A tool to be ran on each commit. ''' taskcluster_secret = 'repo:github.com/mozilla-releng/services:branch:' + channel if pull_request is not None: taskcluster_secret = 'repo:github.com/mozilla-releng/services:pull-request' taskcluster_queue = cli_common.taskcluster.get_service('queue') taskcluster_notify = cli_common.taskcluster.get_service('notify') click.echo(' => Retriving taskGroupId ... ', nl=False) with click_spinner.spinner(): task = taskcluster_queue.task(task_id) if 'taskGroupId' not in task: please_cli.utils.check_result(1, 'taskGroupId does not exists in task: {}'.format(json.dumps(task))) task_group_id = task['taskGroupId'] please_cli.utils.check_result(0, '') click.echo(' taskGroupId: ' + task_group_id) if channel in please_cli.config.DEPLOY_CHANNELS: taskcluster_notify.irc(dict(channel='#release-services', message=f'New deployment on {channel} is about to start: https://tools.taskcluster.net/groups/{task_group_id}')) message = ('release-services team is about to release a new version of mozilla/release-services ' '(*.mozilla-releng.net, *.moz.tools). Any alerts coming up soon will be best directed ' 'to #release-services IRC channel. Automated message (such as this) will be send ' 'once deployment is done. Thank you.') '''This message will only be sent when channel is production. ''' if channel is 'production': for msgChannel in ['#ci', '#moc']: taskcluster_notify.irc(dict(channel=msgChannel, message=message)) click.echo(' => Checking cache which project needs to be rebuilt') build_projects = [] project_hashes = dict() for project in sorted(PROJECTS): click.echo(' => ' + project) project_exists_in_cache, project_hash = ctx.invoke( please_cli.check_cache.cmd, project=project, cache_urls=cache_urls, nix_instantiate=nix_instantiate, channel=channel, indent=8, interactive=False, ) project_hashes[project] = project_hash if not project_exists_in_cache: build_projects.append(project) projects_to_deploy = [] if channel in please_cli.config.DEPLOY_CHANNELS: click.echo(' => Checking which project needs to be redeployed') # TODO: get status for our index branch deployed_projects = {} for project_name in sorted(PROJECTS): deployed_projects.get(project_name) # update hook for each project if please_cli.config.PROJECTS_CONFIG[project_name]['update'] is True: if channel == 'production': update_hook_nix_path_atttribute = f'updateHook.{channel}.scheduled' else: update_hook_nix_path_atttribute = f'updateHook.{channel}.notScheduled' projects_to_deploy.append(( project_name, [], 'TASKCLUSTER_HOOK', { 'enable': True, 'docker_registry': 'index.docker.io', 'docker_repo': 'mozillareleng/services', 'name-suffix': '-update-dependencies', 'nix_path_attribute': update_hook_nix_path_atttribute, }, )) if deployed_projects == project_hashes[project_name]: continue if 'deploys' not in please_cli.config.PROJECTS_CONFIG[project_name]: continue for deploy in please_cli.config.PROJECTS_CONFIG[project_name]['deploys']: for deploy_channel in deploy['options']: if channel == deploy_channel: projects_to_deploy.append(( project_name, please_cli.config.PROJECTS_CONFIG[project_name].get('requires', []), deploy['target'], deploy['options'][channel], )) click.echo(' => Creating taskcluster tasks definitions') tasks = [] # 1. build tasks build_tasks = {} for index, project in enumerate(sorted(build_projects)): project_uuid = slugid.nice().decode('utf-8') required = [] if pull_request is not None: required += [ 'CACHE_BUCKET', 'CACHE_REGION', ] secrets = cli_common.taskcluster.get_secrets( taskcluster_secret, project, required=required, taskcluster_client_id=taskcluster_client_id, taskcluster_access_token=taskcluster_access_token, ) build_tasks[project_uuid] = get_build_task( index, project, task_group_id, task_id, github_commit, owner, channel, taskcluster_secret, pull_request is None and secrets.get('CACHE_BUCKET') or None, pull_request is None and secrets.get('CACHE_REGION') or None, ) tasks.append((project_uuid, build_tasks[project_uuid])) if projects_to_deploy: # 2. maintanance on task maintanance_on_uuid = slugid.nice().decode('utf-8') if len(build_tasks.keys()) == 0: maintanance_on_dependencies = [task_id] else: maintanance_on_dependencies = [i for i in build_tasks.keys()] maintanance_on_task = get_task( task_group_id, maintanance_on_dependencies, github_commit, channel, taskcluster_secret, './please -vv tools maintanance:on ' + ' '.join(list(set([i[0] for i in projects_to_deploy]))), { 'name': '2. Maintanance ON', 'description': '', 'owner': owner, 'source': 'https://github.com/mozilla/release-services/tree/' + channel, }, ) tasks.append((maintanance_on_uuid, maintanance_on_task)) # 3. deploy tasks (if on production/staging) deploy_tasks = {} for index, (project, project_requires, deploy_target, deploy_options) in \ enumerate(sorted(projects_to_deploy, key=lambda x: x[0])): try: enable = deploy_options['enable'] except KeyError: raise click.ClickException(f'Missing {enable} in project {project} and channel {channel} deploy options') if not enable: continue project_uuid = slugid.nice().decode('utf-8') project_task = get_deploy_task( index, project, project_requires, deploy_target, deploy_options, task_group_id, maintanance_on_uuid, github_commit, owner, channel, taskcluster_secret, ) if project_task: deploy_tasks[project_uuid] = project_task tasks.append((project_uuid, deploy_tasks[project_uuid])) # 4. maintanance off task maintanance_off_uuid = slugid.nice().decode('utf-8') maintanance_off_task = get_task( task_group_id, [i for i in deploy_tasks.keys()], github_commit, channel, taskcluster_secret, './please -vv tools maintanance:off ' + ' '.join(list(set([i[0] for i in projects_to_deploy]))), { 'name': '4. Maintanance OFF', 'description': '', 'owner': owner, 'source': 'https://github.com/mozilla/release-services/tree/' + channel, }, ) maintanance_off_task['requires'] = 'all-resolved' tasks.append((maintanance_off_uuid, maintanance_off_task)) click.echo(' => Submitting taskcluster definitions to taskcluster') if dry_run: tasks2 = {task_id: task for task_id, task in tasks} for task_id, task in tasks: click.echo(' => %s [taskId: %s]' % (task['metadata']['name'], task_id)) click.echo(' dependencies:') deps = [] for dep in task['dependencies']: depName = '0. Decision task' if dep in tasks2: depName = tasks2[dep]['metadata']['name'] deps.append(' - %s [taskId: %s]' % (depName, dep)) for dep in sorted(deps): click.echo(dep) else: for task_id, task in tasks: taskcluster_queue.createTask(task_id, task)