def rollback(version, app):
"""
Rollback release to a previous release.
"""
cli.user()
if version and version[0] == 'v':
version = version[1:]
else:
click.echo('Invalid version given.')
sys.exit(1)
if not version:
click.echo(f'Getting latest release for app {app}... ', nl=False)
with click_spinner.spinner():
res = api.Releases.get(app=app)
version = int(res[0]['id']) - 1
click.echo(click.style('√', fg='green'))
if int(version) == 0:
click.echo('Unable to rollback a release before v1.')
sys.exit(1)
click.echo(f'Rolling back to v{version}... ', nl=False)
with click_spinner.spinner():
res = api.Releases.rollback(version=version, app=app)
click.echo(click.style('√', fg='green'))
click.echo(f'Deployed new release... ' +
click.style(f'v{res["id"]}', bold=True, fg='magenta'))
def wait_until_online(self):
if self.dry_run:
logging.info(
f"Would wait for '{self['name']}' to come back online")
else:
logging.info(f"Waiting for '{self['name']}' to come back online",
self.log_to_slack)
with click_spinner.spinner():
while True:
with socket.socket(socket.AF_INET,
socket.SOCK_STREAM) as s:
s.settimeout(5)
result = s.connect_ex((self['name'], 22))
if result == 0:
break
if self.dry_run:
logging.info(f"Would wait for libvirt on '{self['name']}'")
else:
logging.info(f"Waiting for libvirt on '{self['name']}'",
self.log_to_slack)
with click_spinner.spinner():
while True:
try:
if self.execute('virsh list').return_code == 0:
break
except (ConnectionResetError, UnexpectedExit,
CommandTimedOut):
pass
time.sleep(5)
def deploy(app, message):
"""
Deploy your app instantly to the Asyncy Cloud
"""
cli.user()
payload = compile_app(app, False) # Also adds a spinner.
if payload is None:
sys.exit(1) # Error already printed by compile_app.
click.echo(f'Deploying app {app}... ', nl=False)
with click_spinner.spinner():
config = Config.get(app)
release = Releases.create(config, payload, app, message)
url = f'https://{app}.asyncyapp.com'
click.echo()
click.echo(
click.style('√', fg='green') +
f' Version {release["id"]} of your app has '
f'been queued for deployment\n')
click.echo('Waiting for deployment to complete... ', nl=False)
with click_spinner.spinner():
if Apps.maintenance(app, maintenance=None):
click.echo()
click.echo()
click.echo('Your app is in maintenance mode.\n'
'Run the following to turn off it off:')
cli.print_command('asyncy maintenance off')
click.echo()
click.echo('Once maintenance mode is turned off, '
'your app will be deployed immediately.')
return
state = 'QUEUED'
while state in ['DEPLOYING', 'QUEUED']:
state = Releases.get(app)[0]['state']
sleep(0.5)
click.echo()
if state == 'DEPLOYED':
click.echo(click.style('√', fg='green') + ' Deployed successfully!')
click.echo(f'If your story listens to HTTP requests, visit {url}')
elif state == 'FAILED':
click.echo(click.style('X', fg='red') + ' Deployment failed!',
err=True)
click.echo(
'Please use the following command to view your app\'s logs:',
err=True)
cli.print_command('asyncy logs')
else:
click.echo(
f'An unhandled state of your app has been encountered - {state}',
err=True)
click.echo(f'Please shoot an email to [email protected]')
def audit_elections(
data_opened: str,
data_pre_election: str,
data_tally: str,
bb: str,
tally_all: str,
group_id: Optional[str] = typer.Argument(None),
):
data_pre_election = _load_pre_election(data_pre_election)
data_tally = _load_tally_data(data_tally)
print("Auditing pre_election - data_tally...")
with click_spinner.spinner():
valid = audit_pre_election_tables(data_pre_election, data_tally)
print("valid" if valid else "not valid!")
del data_pre_election
data_opened = _load_opened_data(data_opened)
print("Auditing data_tally - data_opened...")
with click_spinner.spinner():
valid = audit_tables(data_opened, data_tally)
print("valid" if valid else "not valid")
del data_tally
bb = _load_bb(bb)
if group_id is not None:
print(
f"Filtering bulletin board is enabled, only votes for group {group_id} will be taken into account"
)
bb_filtered = {}
for vote_id, bb_entry in bb.items():
if bb_entry.group_id == group_id:
bb_filtered[vote_id] = bb_entry
bb = bb_filtered
print("Auditing data_opened - bb...")
with click_spinner.spinner():
valid = audit_bb(bb, data_opened)
print("valid" if valid else "not valid")
del bb
tally_all = _load_tally(tally_all)
print("Auditing data_opened - tally with all votes...")
with click_spinner.spinner():
valid = audit_tables_tally(tally_all, data_opened)
print("valid" if valid else "not valid")
def query(
self,
query: str,
query_variables: dict = None,
) -> Union[dict, None]:
try:
query = gql(query)
with click_spinner.spinner(beep=False,
disable=False,
force=False,
stream=sys.stdout):
data = self.client.execute(
document=query,
variable_values=query_variables,
)
except TransportServerError:
# refresh token
response = self.authentication.refresh()
if not response["success"]:
console.exit_login_required()
self.access_token = response["response"]["access_token"]
self.client = self._client()
raise RetryException("retry")
return data
def bulk_insert(self, musics):
if not musics:
logger.info("no musics to insert")
return None
if config.debug:
with tqdm(total=len(musics),
desc="inserting music one by one") as pbar:
for music in musics:
logger.debug("inserting %s", music)
self.upsert_music(music)
pbar.update(1)
return None
j = json.dumps([m.to_dict() for m in musics])
b64 = j.encode('utf-8')
data = base64.b64encode(b64)
query = f'''
mutation
{{
bulkInsert(input: {{data: "{data.decode()}"}})
{{
clientMutationId
}}
}}'''
with click_spinner.spinner(disable=config.quiet):
return self._post(query)
def init_db(self, erase: bool = False, exclude_schema=None):
"""
Initializes a database from scratch,
creating tables and needed resources.
Note that this does not create the database per se.
If executing this directly, remember to use an app_context.
Resources can hook functions that will be called when this
method executes, by subclassing :meth:`teal.resource.
Resource.load_resource`.
"""
assert _app_ctx_stack.top, 'Use an app context.'
print('Initializing database...'.ljust(30), end='')
with click_spinner.spinner():
if erase:
if exclude_schema: # Using then a schema teal sqlalchemy
assert isinstance(self.db, SchemaSQLAlchemy)
self.db.drop_schema()
else: # using regular flask sqlalchemy
self.db.drop_all()
self._init_db(exclude_schema)
self._init_resources()
self.db.session.commit()
print('done.')
def delete(ctx, opts, owner_repo_package, yes):
"""
Delete a package from a repository.
- OWNER/REPO/PACKAGE: Specify the OWNER namespace (i.e. user or org), the
REPO name where the package is stored, and the PACKAGE name (slug) of the
package itself. All separated by a slash.
Example: 'your-org/awesome-repo/better-pkg'.
"""
owner, repo, slug = owner_repo_package
args = {
'owner': click.style(owner, bold=True),
'repo': click.style(repo, bold=True),
'package': click.style(slug, bold=True)
}
if not yes and not click.confirm(
'Are you sure you want to delete %(package)s from '
'%(owner)s/%(repo)s?' % args):
click.secho('OK! Phew, close call. :-)', fg='yellow')
return
click.echo('Deleting %(package)s from %(owner)s/%(repo)s ... ' % args,
nl=False)
context_msg = 'Failed to delete the package!'
with handle_api_exceptions(ctx, opts=opts, context_msg=context_msg):
with spinner():
delete_package(owner=owner, repo=repo, slug=slug)
click.secho('OK', fg='green')
def parse_version(ctx, param, version) -> str:
if not version:
return version
if param.name == "allennlp_version":
package = "allennlp"
else:
package = "allennlp-models"
if version.startswith("git@"):
git_url = f"https://github.com/allenai/{package}"
if version == "git@master":
# Get the latest commit from the git repo.
click.secho("Checking for latest commit...", fg="yellow")
with click_spinner.spinner():
latest_commits = list(
shell_out_command(["git", "ls-remote", git_url + ".git"])
)
latest = latest_commits[0].split("\t")[0]
version = f"git+{git_url}.git@{latest}"
else:
version = f"git+{git_url}.{version}"
elif version.startswith("git+"):
pass
else:
version = f"{package}=={version}"
click.echo("Using " + click.style(f"{version}", fg="green"))
return version
def push(
docker_registry,
docker_repo,
docker_username,
docker_password,
docker_tag,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
):
secrets = cli_common.taskcluster.get_secrets(
taskcluster_secret,
None,
required=[
'DOCKER_USERNAME',
'DOCKER_PASSWORD',
],
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
docker_username = docker_username or secrets['DOCKER_USERNAME']
docker_password = docker_password or secrets['DOCKER_PASSWORD']
image_reference = f'docker-daemon:{docker_repo}:{docker_tag}'
click.echo(' => Pushing the image to the registry ... ', nl=False)
with click_spinner.spinner():
please_cli.utils.push_docker_image(
registry=docker_registry,
username=docker_username,
password=docker_password,
image=image_reference,
repo=docker_repo,
tag=docker_tag,
)
def run(self):
with click_spinner.spinner():
click.secho('Installing dependencies...', fg='green')
delegator.run('yarn')
click.secho('Building the app...', fg='green')
delegator.run('yarn run build')
self.update_index_file()
def create_package(ctx, opts, owner, repo, package_type, skip_errors,
**kwargs):
"""Create a new package via the API."""
click.echo('Creating a new %(package_type)s package ... ' %
{'package_type': click.style(package_type, bold=True)},
nl=False)
context_msg = 'Failed to create package!'
with handle_api_exceptions(ctx,
opts=opts,
context_msg=context_msg,
reraise_on_error=skip_errors):
with spinner():
slug_perm, slug = api_create_package(package_format=package_type,
owner=owner,
repo=repo,
**kwargs)
click.secho('OK', fg='green')
click.echo(
'Created: %(owner)s/%(repo)s/%(slug)s (%(slug_perm)s)' % {
'owner': click.style(owner, fg='magenta'),
'repo': click.style(repo, fg='blue'),
'slug': click.style(slug, fg='green'),
'slug_perm': click.style(slug_perm, bold=True)
})
return slug_perm, slug
def login(user, token):
"""Save your GitHub access token."""
app_dir = click.get_app_dir(QQQ)
config_path = f'{app_dir}/{CONFIG_FILE}'
# Verify user
with click_spinner.spinner():
if not GitHub.verify_token(user, token):
click.echo(
click.style('Invalid GitHub username or token!', fg='red'))
raise click.Abort
# Check if file already exists
if Path(config_path).is_file():
# File exists, prompt to overwrite
click.confirm(
f'{click.format_filename(config_path)} already exists, update?',
abort=True)
# Create config object
cp = configparser.ConfigParser()
cp['auth'] = {'user': user, 'token': token}
# Make sure the qqq dir exists
if not Path(app_dir).is_dir():
click.echo(f'Creating directory {click.format_filename(app_dir)}...')
Path(app_dir).mkdir(parents=True, exist_ok=True)
# Write to config file
with open(config_path, 'w') as config_file:
cp.write(config_file)
click.echo(
f'Updated config file located at:\t{click.format_filename(config_path)}'
)
def run(
name: str,
parameter: Tuple[str],
cache: bool,
wait: bool,
owner: str,
show_output: bool,
):
"""Run a report"""
params = process_cmd_param_vals(parameter)
log.info(f"Running script with parameters {params}")
script = api.Script.get(name, owner=owner)
with api_error_handler("Error running script"):
r = script.run(parameters=params, cache=cache)
if wait:
with click_spinner.spinner():
while not r.is_complete():
time.sleep(2)
r.refresh()
log.debug(f"Run completed with status {r.status}")
if show_output:
click.echo(r.output)
if r.status == "SUCCESS":
if r.result:
success_msg(f"Script result - '{r.result}'")
if r.report:
report = api.Report.by_id(r.report)
success_msg(f"Report generated at {report.web_url}")
else:
failure_msg(
f"Script run failed/cancelled\n{r.error_msg}: {r.error_detail}"
)
else:
success_msg(f"Script run started, view at {script.web_url}")
def logs(follow, last, service, service_name, app, level):
"""
Fetch logs for your app
"""
cli.user()
click.echo(f'Retrieving logs for {app}... ', nl=False)
with click_spinner.spinner():
app_id = Apps.get_uuid_from_hostname(app)
click.echo()
cli.track(
'App Logs Requested',
{
'App name': app,
'Follow': 'Yes' if follow else 'No',
'Last N': last,
'Source': 'Runtime' if service is False else 'Service',
'Service Name': service_name if service else 'N/A',
'Minimum level': level.capitalize(),
},
)
asyncio.get_event_loop().run_until_complete(
connect_and_listen_with_retry(
app_id,
last,
follow,
service is False,
service,
service_name,
level,
))
def wait(
cluster_id: str,
superuser_username: str,
superuser_password: str,
transport: Transport,
skip_http_checks: bool,
) -> None:
"""
Wait for DC/OS to start.
"""
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
message = (
'A cluster may take some time to be ready.\n'
'The amount of time it takes to start a cluster depends on a variety '
'of factors.\n'
'If you are concerned that this is hanging, try "dcos-docker doctor" '
'to diagnose common issues.')
click.echo(message)
cluster_containers = ClusterContainers(
cluster_id=cluster_id,
transport=transport,
)
http_checks = not skip_http_checks
with click_spinner.spinner():
if cluster_containers.is_enterprise:
cluster_containers.cluster.wait_for_dcos_ee(
superuser_username=superuser_username,
superuser_password=superuser_password,
http_checks=http_checks,
)
return
cluster_containers.cluster.wait_for_dcos_oss(http_checks=http_checks)
def _process_log_file(self) -> bool:
tail = self.tail_file()
loops_since_check = 0
processing = True
done_match = False
with click_spinner.spinner():
while processing:
for line in tail.readlines():
self.logger.debug(f"log: {line}")
done_match = (re.search(r"Done \((\d+\.\d+)s\)! For help,",
line) is not None)
if done_match:
processing = False
elif "agree to the EULA" in line:
self.logger.info("")
raise click.ClickException(
f"You must agree to Mojang's EULA. "
"Please read {EULA_URL} and restart server "
"with --accept_eula")
if loops_since_check < 5:
loops_since_check += 1
elif self.is_running():
loops_since_check = 0
else:
self.logger.error(f"{self.server_name} failed to start")
processing = False
time.sleep(1)
self.delete_offset()
return done_match
def cmd(
ctx,
project,
nix_shell,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
):
checks = please_cli.config.PROJECTS_CONFIG.get(project, {}).get('checks')
if not checks:
raise click.ClickException(
'No checks found for `{}` project.'.format(project))
for check_title, check_command in checks:
click.echo(' => {}: '.format(check_title), nl=False)
with click_spinner.spinner():
returncode, output, error = ctx.invoke(
please_cli.shell.cmd,
project=project,
quiet=True,
command=check_command,
nix_shell=nix_shell,
taskcluster_secret=taskcluster_secret,
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
please_cli.utils.check_result(returncode,
output,
raise_exception=False)
def generic_wait_command(manager_name, attrs, ctx, uuid, refresh_period):
"""Performs a generic wait command for task instances.
Args:
manager_name: A string containing the name of the
saltant.client.Client's manager to use. For example,
"executable_task_instances".
attrs: An iterable containing the attributes of the object to
use when displaying it.
ctx: A click.core.Context object containing information about
the Click session.
uuid: A string containing the uuid of the task instance to
wait for.
refresh_period: A float specifying how many seconds to wait in
between checking the task's status.
"""
# Get the client from the context
client = ctx.obj["client"]
# Terminate the task instance
try:
manager = getattr(client, manager_name)
# Wait for the task instance to finish
with click_spinner.spinner():
object = manager.wait_until_finished(uuid, refresh_period)
# Output a list display of the task instance
output = generate_list_display(object, attrs)
except BadHttpRequestError:
# Bad request
output = "task instance %s not found" % uuid
click.echo(output)
def start(foreground):
"""
Start the daemon
"""
client = DaemonClient()
click.echo('Starting the daemon... ', nl=False)
if foreground:
command = [
'verdi', '-p', client.profile_name, 'daemon', '_start_circus',
'--foreground'
]
else:
command = [
'verdi', '-p', client.profile_name, 'daemon', '_start_circus'
]
try:
currenv = get_env_with_venv_bin()
subprocess.check_output(command, env=currenv)
except subprocess.CalledProcessError as exception:
click.echo('failed: {}'.format(exception))
sys.exit(1)
# We add a small timeout to give the pid-file a chance to be created
with spinner():
time.sleep(1)
response = client.get_status()
print_client_response_status(response)
def get_ssh_conn(host, username, password=None, ssh_key=None):
"""Connect to a remote system by SSH port 22.
By default it will retry 60 times until it establishes an ssh connection.
:param host: Remote systems IP address
:type host: str
:param username: Username
:type username: str
:param password: Password
:type password: str
:param ssh_key: SSH private key for authentication
:type ssh_key: str
"""
with spinner():
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
if ssh_key:
ssh.connect(hostname=host,
username=username,
key_filename=ssh_key,
timeout=5)
if password:
ssh.connect(hostname=host,
username=username,
password=password,
timeout=5)
ssh.close()
LOG.info("Successfully established SSH connection to %s", host)
except (error, SSHException, timeout):
raise SSHError('Port 22 is unreachable.')
def exec_cmd_by_ssh(host,
username,
cmd,
password=None,
ssh_key=None,
fire_forget=False):
"""Connect to a remote system by SSH port 22 and run a command.
By default it will retry 60 times until it establishes an ssh connection.
:param host: Remote systems IP address
:type host: str
:param cmd: Command to execute
:type cmd: str
:param username: Username
:type username: str
:param password: Password
:type password: str
:param ssh_key: SSH private key for authentication
:type ssh_key: str
:param fire_forget: fire and forget the command
:type fire_forget: bool
"""
with spinner():
try:
ssh = SSHClient()
ssh.load_system_host_keys()
ssh.set_missing_host_key_policy(AutoAddPolicy())
if ssh_key:
ssh.connect(hostname=host,
username=username,
key_filename=ssh_key,
timeout=30)
if password:
ssh.connect(hostname=host,
username=username,
password=password,
timeout=30)
if fire_forget:
# fire and forget the command given, will return no output
_timeout = 0.5
channel = ssh.get_transport().open_session(timeout=_timeout)
channel.settimeout(_timeout)
channel.exec_command(cmd)
LOG.info('Successfully executed command: %s' % cmd)
else:
results = ssh.exec_command(cmd)
return_code = results[1].channel.recv_exit_status()
if return_code:
LOG.error('Command %s failed to execute.' % cmd)
LOG.error(results[2].read())
ssh.close()
raise SystemExit(return_code)
else:
LOG.debug(results[1].read().strip())
LOG.info("Successfully executed command: %s", cmd)
ssh.close()
except (error, SSHException, timeout):
raise SSHError('Port 22 is unreachable.')
def push(docker_registry,
docker_repo,
docker_username,
docker_password,
docker_tag,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
):
secrets = cli_common.taskcluster.get_secrets(
taskcluster_secret,
None,
required=[
'DOCKER_USERNAME',
'DOCKER_PASSWORD',
],
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
docker_username = docker_username or secrets['DOCKER_USERNAME']
docker_password = docker_password or secrets['DOCKER_PASSWORD']
image_reference = f'docker-daemon:{docker_repo}:{docker_tag}'
click.echo(' => Pushing the image to the registry ... ', nl=False)
with click_spinner.spinner():
please_cli.utils.push_docker_image(
registry=docker_registry,
username=docker_username,
password=docker_password,
image=image_reference,
repo=docker_repo,
tag=docker_tag,
)
def start(foreground, number):
"""Start the daemon with NUMBER workers.
If the NUMBER of desired workers is not specified, the default is used, which is determined by the configuration
option `daemon.default_workers`, which if not explicitly changed defaults to 1.
"""
from aiida.engine.daemon.client import get_daemon_client
client = get_daemon_client()
echo.echo('Starting the daemon... ', nl=False)
if foreground:
command = ['verdi', '-p', client.profile.name, 'daemon', _START_CIRCUS_COMMAND, '--foreground', str(number)]
else:
command = ['verdi', '-p', client.profile.name, 'daemon', _START_CIRCUS_COMMAND, str(number)]
try:
currenv = get_env_with_venv_bin()
subprocess.check_output(command, env=currenv, stderr=subprocess.STDOUT) # pylint: disable=unexpected-keyword-arg
except subprocess.CalledProcessError as exception:
click.secho('FAILED', fg='red', bold=True)
echo.echo_critical(str(exception))
# We add a small timeout to give the pid-file a chance to be created
with spinner():
time.sleep(1)
response = client.get_status()
print_client_response_status(response)
def init_db(self, name: str, org_name: str, org_id: str,
tag_url: boltons.urlutils.URL, tag_token: uuid.UUID,
erase: bool, common: bool):
"""Creates an inventory.
This creates the database and adds the inventory to the
inventory tables with the passed-in settings, and does nothing if the
inventory already exists.
After you create the inventory you might want to create an user
executing *dh user add*.
"""
assert _app_ctx_stack.top, 'Use an app context.'
print('Initializing database...'.ljust(30), end='')
with click_spinner.spinner():
if erase:
self.db.drop_all(common_schema=common)
assert not db.has_schema(
self.id), 'Schema {} already exists.'.format(self.id)
exclude_schema = 'common' if not common else None
self._init_db(exclude_schema=exclude_schema)
InventoryDef.set_inventory_config(name, org_name, org_id, tag_url,
tag_token)
DeviceSearch.set_all_devices_tokens_if_empty(self.db.session)
self._init_resources(exclude_schema=exclude_schema)
self.db.session.commit()
print('done.')
def wait_for_job(self, job_id, retries=10):
job_status = 0
with click_spinner.spinner():
while True:
if retries <= 0:
break
try:
job_status = self.cs.queryAsyncJobResult(jobid=job_id).get('jobstatus', 0)
except CloudStackException as e:
if 'multiple JSON fields named jobstatus' not in str(e):
raise e
logging.debug(e)
retries -= 1
except ConnectionError as e:
if 'Connection aborted' not in str(e):
raise e
logging.debug(e)
retries -= 1
if int(job_status) == 1:
return True
elif int(job_status) == 2:
break
time.sleep(1)
return False
def token(username, password):
"""
Request a token for using Pindo API.
"""
click.echo('token...')
with click_spinner.spinner():
click.echo(Token(username, password))
async def fullscan(db, folders=None, concurrency=1, crawl=False):
if folders is None:
folders = await db.folders()
folders = [f['name'] for f in folders]
with click_spinner.spinner():
files = [
f for f in find_files(list(folders))
if f[1].endswith(tuple(supported_formats))
]
size = len(files) * 2 if crawl else len(files)
with tqdm(total=size,
file=sys.stdout,
desc="Loading music",
leave=True,
position=0,
disable=config.quiet) as bar:
async def insert(semaphore, f):
async with semaphore:
try:
m = File(f[1], f[0])
if crawl:
await m.find_youtube()
bar.update(1)
debug(m.to_list())
await db.upsert(m)
bar.update(1)
except asyncpg.exceptions.CheckViolationError as e:
warning("Violation: {}".format(e))
semaphore = asyncio.BoundedSemaphore(concurrency)
tasks = [asyncio.ensure_future(insert(semaphore, f)) for f in files]
await asyncio.gather(*tasks)
await db.refresh()
def run(*groups,
users: int = 5,
duration: int = 0,
logger=None,
show_progress_bar: bool = False,
halt_and_catch_fire: bool = False) -> Report:
cm = contextlib.nullcontext()
if show_progress_bar and not logger:
cm = click_spinner.spinner()
with cm:
started = time.time()
executions = []
if duration:
while time.time() - started <= duration:
res = _run(*groups,
users=users,
logger=logger,
halt_and_catch_fire=halt_and_catch_fire)
executions.extend(res)
else:
res = _run(*groups,
users=users,
logger=logger,
halt_and_catch_fire=halt_and_catch_fire)
executions.extend(res)
return Report(executions)
def compile_app(app_name_for_analytics, debug) -> dict:
"""
Compiles, prints pretty info, and returns the compiled tree.
:return: The compiled tree
"""
from storyscript.App import App
click.echo(click.style('Compiling Stories...', bold=True))
with click_spinner.spinner():
try:
stories = json.loads(App.compile(os.getcwd()))
except BaseException:
import traceback
traceback.print_exc()
click.echo('Failed', err=True)
stories = None
result = 'Success'
count = 0
if stories is None:
result = 'Failed'
else:
count = len(stories.get('stories', {}))
cli.track('App Compiled', {
'App name': app_name_for_analytics,
'Result': result,
'Stories': count
})
return stories
def refresh_token(username, password):
"""
Refresh a Token.
"""
click.echo('token...')
with click_spinner.spinner():
click.echo(RefreshToken(username, password))
def genfiles(folders):
with click_spinner.spinner(disable=config.quiet):
count = 0
directories = [os.path.abspath(f) for f in folders]
for d in directories:
count += filecount(d, supported_formats)
logger.info("File count: %s", count)
files = []
with tqdm(total=count, desc="Music listing", leave=False, disable=config.quiet) as pbar:
file_list = find_files(folders, supported_formats)
music_files = list(file_list)
for f in music_files:
m = File(f[1], f[0])
files.append(m)
pbar.update(1)
return files
def main(filename):
if not filename.endswith('.md'):
raise click.BadParameter("Please provide a Markdown file ending with .md")
input_filename = filename
output_filename = filename.replace('md', 'pdf')
event_handler = PandocHandler(input_filename, output_filename)
observer = Observer()
observer.schedule(event_handler, path='.')
observer.start()
try:
with spinner():
while True:
time.sleep(1)
except KeyboardInterrupt:
observer.stop()
observer.join()
def bulk_insert(self, musics):
if not musics:
logger.info("no musics to insert")
return None
j = json.dumps([m.to_dict() for m in musics])
b64 = j.encode('utf-8')
data = base64.b64encode(b64)
query = '''
mutation
{{
bulkInsert(input: {{data: "{}"}})
{{
clientMutationId
}}
}}'''.format(data.decode())
with click_spinner.spinner(disable=config.quiet):
return self._post(query)
def unlinked_references(doi_file_name, arxiv_file_name):
"""Find often cited literature that is not on INSPIRE.
It generates two files with a list of DOI/arxiv ids respectively,
in which each line has the respective identifier, folowed by two numbers,
representing the amount of times the literature has been cited
by a core and a non-core article respectively.
The lists are ordered by an internal measure of relevance."""
with click_spinner.spinner():
click.echo('Looking up unlinked references...')
result_doi, result_arxiv = check_unlinked_references()
click.echo('Done!')
click.echo(u'Output written to "{}" and "{}"'.format(doi_file_name.name, arxiv_file_name.name))
for item in result_doi:
doi_file_name.write(u'{i[0]}: {i[1]}\n'.format(i=item))
for item in result_arxiv:
arxiv_file_name.write(u'{i[0]}: {i[1]}\n'.format(i=item))
def cmd(ctx,
project,
nix_build,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
):
with click_spinner.spinner():
click.echo(f' => Testing project {project} ...', nl=False)
outputs = ctx.invoke(please_cli.build.cmd,
project=project,
nix_path_attributes=[project],
interactive=False,
nix_build=nix_build,
taskcluster_secret=taskcluster_secret,
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
if len(outputs) == 1:
click.secho('DONE', fg='green')
else:
click.secho('ERROR', fg='red')
def run_update(project, nix_shell, root_dir, interactive):
logger.info('Running dependency update.')
command = [
nix_shell,
please_cli.config.ROOT_DIR + '/nix/update.nix',
'--argstr', 'pkg',
project
]
click.echo('Updating dependencies of {}: '.format(project), nl=False)
with click_spinner.spinner():
returncode, output, error = cli_common.command.run(
command,
stream=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=root_dir,
)
please_cli.utils.check_result(
returncode,
output,
raise_exception=True,
ask_for_details=interactive,
)
def cmd(ctx,
project,
quiet,
nix_shell,
interactive,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
):
project_config = please_cli.config.PROJECTS_CONFIG.get(project, {})
run_type = project_config.get('run')
run_options = project_config.get('run_options', {})
if not run_type:
raise click.ClickException('Application `{}` is not configured to be runnable.'.format(project))
host = run_options.get('host', os.environ.get('HOST', '127.0.0.1'))
port = str(run_options.get('port', 8000))
schema = 'https://'
project_name = project.replace('-', '_')
ca_cert_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'ca.crt')
server_cert_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'server.crt')
server_key_file = os.path.join(please_cli.config.TMP_DIR, 'certs', 'server.key')
os.environ['DEBUG'] = 'true'
os.environ['PROJECT_NAME'] = project_name
pg_host = please_cli.config.PROJECTS_CONFIG['postgresql']['run_options'].get('host', host)
pg_port = str(please_cli.config.PROJECTS_CONFIG['postgresql']['run_options']['port'])
redis_host = please_cli.config.PROJECTS_CONFIG['redis']['run_options'].get('host', host)
redis_port = str(please_cli.config.PROJECTS_CONFIG['redis']['run_options']['port'])
if 'postgresql' in project_config.get('requires', []):
dbname = 'services'
click.echo(' => Checking if database `{}` exists ... '.format(dbname), nl=False)
with click_spinner.spinner():
result, output, error = ctx.invoke(
please_cli.shell.cmd,
project=project,
quiet=True,
command=' '.join([
'psql',
'-lqt',
'-h', pg_host,
'-p', pg_port,
]),
nix_shell=nix_shell,
)
if result != 0 and 'psql: could not connect to server' in output:
click.secho('ERROR', fg='red')
raise click.UsageError(
'Could not connect to the database.\n\n'
'Please run:\n\n'
' ./please run postgresql\n\n'
'in a separate terminal.'
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
database_exists = False
for line in output.split('\n'):
column1 = line.split('|')[0].strip()
if column1 == dbname:
database_exists = True
break
if not database_exists:
click.echo(' => Creating `{}` database ` ... '.format(dbname), nl=False)
with click_spinner.spinner():
result, output, error = ctx.invoke(
please_cli.shell.cmd,
project=project,
command=' '.join([
'createdb',
'-h', pg_host,
'-p', pg_port,
dbname,
]),
nix_shell=nix_shell,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
os.environ['DATABASE_URL'] = 'postgresql://{}:{}/{}'.format(
pg_host, pg_port, dbname
)
if 'redis' in project_config.get('requires', []):
# Check redis is running
click.echo(' => Checking if redis is running... ', nl=False)
with click_spinner.spinner():
result, output, error = ctx.invoke(
please_cli.shell.cmd,
project=project,
quiet=True,
command='redis-cli -h {} -p {} ping'.format(redis_host, redis_port),
nix_shell=nix_shell,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
# Setup config for client application
os.environ['REDIS_URL'] = 'redis://{}:{}'.format(redis_host, redis_port)
if run_type == 'POSTGRESQL':
data_dir = run_options.get('data_dir', os.path.join(please_cli.config.TMP_DIR, 'postgresql'))
if not os.path.isdir(data_dir):
click.echo(' => Initialize database folder `{}` ... '.format(data_dir), nl=False)
with click_spinner.spinner():
result, output, error = ctx.invoke(please_cli.shell.cmd,
project=project,
command='initdb -D {} --auth=trust'.format(data_dir),
nix_shell=nix_shell,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
schema = ''
command = [
'postgres',
'-D', data_dir,
'-h', host,
'-p', port,
]
elif run_type == 'REDIS':
data_dir = run_options.get('data_dir', os.path.join(please_cli.config.TMP_DIR, 'redis'))
if not os.path.isdir(data_dir):
os.makedirs(data_dir)
command = [
'redis-server',
'--dir', data_dir,
'--bind', host,
'--port', port,
]
elif run_type == 'FLASK':
for env_name, env_value in run_options.get('envs', {}).items():
env_name = please_cli.utils.normalize_name(env_name).upper()
os.environ[env_name] = env_value
if not os.path.exists(ca_cert_file) or \
not os.path.exists(server_cert_file) or \
not os.path.exists(server_key_file):
ctx.invoke(please_cli.create_certs.cmd,
certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'),
)
project_cache_dir = os.path.join(please_cli.config.TMP_DIR, 'cache', project_name)
if not os.path.isdir(project_cache_dir):
os.makedirs(project_cache_dir)
os.environ['CACHE_TYPE'] = 'filesystem'
os.environ['CACHE_DIR'] = project_cache_dir
os.environ['APP_SETTINGS'] = os.path.join(
please_cli.config.ROOT_DIR, 'src', project_name, 'settings.py')
os.environ['APP_URL'] = '{}{}:{}'.format(schema, host, port)
os.environ['CORS_ORIGINS'] = '*'
command = [
'gunicorn',
please_cli.utils.normalize_name(project_name) + '.flask:app',
'--bind', '{}:{}'.format(host, port),
'--ca-certs={}'.format(ca_cert_file),
'--certfile={}'.format(server_cert_file),
'--keyfile={}'.format(server_key_file),
'--workers', '2',
'--timeout', '3600',
'--reload',
'--reload-engine=poll',
'--log-file', '-',
]
elif run_type == 'SPHINX':
schema = 'http://'
command = [
'HOST=' + host,
'PORT=' + port,
'python', 'run.py',
]
elif run_type == 'ELM':
if not os.path.exists(ca_cert_file) or \
not os.path.exists(server_cert_file) or \
not os.path.exists(server_key_file):
ctx.invoke(please_cli.create_certs.cmd,
certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'),
)
os.environ['WEBPACK_RELEASE_VERSION'] = please_cli.config.VERSION
os.environ['WEBPACK_RELEASE_CHANNEL'] = 'development'
os.environ['SSL_CACERT'] = ca_cert_file
os.environ['SSL_CERT'] = server_cert_file
os.environ['SSL_KEY'] = server_key_file
os.environ['HOST'] = host
os.environ['PORT'] = port
for env_name, env_value in run_options.get('envs', {}).items():
env_name = 'WEBPACK_' + please_cli.utils.normalize_name(env_name).upper()
os.environ[env_name] = env_value
# XXX: once we move please_cli.config.PROJECTS to nix we wont need this
for require in project_config.get('requires', []):
env_name = 'WEBPACK_{}_URL'.format(please_cli.utils.normalize_name(require).upper())
env_value = '{}://{}:{}'.format(
please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('schema', 'https'),
please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('host', host),
please_cli.config.PROJECTS_CONFIG[require]['run_options']['port'],
)
os.environ[env_name] = env_value
command = [
'webpack-dev-server',
'--host', host,
'--port', port,
'--config', os.path.join(please_cli.config.ROOT_DIR, 'src', project_name, 'webpack.config.js'),
]
elif run_type == 'NEUTRINO':
if not os.path.exists(ca_cert_file) or \
not os.path.exists(server_cert_file) or \
not os.path.exists(server_key_file):
ctx.invoke(please_cli.create_certs.cmd,
certificates_dir=os.path.join(please_cli.config.TMP_DIR, 'certs'),
)
envs = dict(
SSL_CACERT=ca_cert_file,
SSL_CERT=server_cert_file,
SSL_KEY=server_key_file,
HOST=host,
PORT=port,
RELEASE_VERSION=please_cli.config.VERSION,
RELEASE_CHANNEL='development',
)
for require in project_config.get('requires', []):
env_name = '{}_URL'.format(please_cli.utils.normalize_name(require).upper())
env_value = '{}://{}:{}'.format(
please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('schema', 'https'),
please_cli.config.PROJECTS_CONFIG[require]['run_options'].get('host', host),
please_cli.config.PROJECTS_CONFIG[require]['run_options']['port'],
)
envs[env_name] = env_value
for env_name, env_value in run_options.get('envs', {}).items():
env_name = please_cli.utils.normalize_name(env_name).upper()
envs[env_name] = env_value
for env_name, env_value in envs.items():
os.environ[env_name] = env_value
command = ['yarn', 'start']
click.echo(' => Running {} on {}{}:{} ...'.format(project, schema, host, port))
returncode, output, error = ctx.invoke(please_cli.shell.cmd,
project=project,
quiet=quiet,
command=' '.join(command),
nix_shell=nix_shell,
taskcluster_secret=taskcluster_secret,
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
sys.exit(returncode)
def build(docker,
docker_repo,
docker_tag,
nix_cache_public_keys,
nix_cache_public_urls,
interactive,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
):
secrets = cli_common.taskcluster.get_secrets(
taskcluster_secret,
None,
required=[
'NIX_CACHE_PUBLIC_KEYS',
'NIX_CACHE_PUBLIC_URLS',
],
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
nix_cache_public_keys = nix_cache_public_keys or secrets['NIX_CACHE_PUBLIC_KEYS']
nix_cache_public_urls = nix_cache_public_urls or secrets['NIX_CACHE_PUBLIC_URLS']
docker = please_cli.utils.which(docker)
assert docker is not None, \
'Docker is not available in your environment. Please use: nix-env -f /etc/nix/nixpkgs -iA docker'
docker_file = os.path.join(please_cli.config.ROOT_DIR, 'nix', 'docker', 'Dockerfile')
nixpkgs_json_file = os.path.join(please_cli.config.ROOT_DIR, 'nix', 'nixpkgs.json')
nix_json_file = os.path.join(please_cli.config.ROOT_DIR, 'nix', 'nix.json')
temp_docker_file = os.path.join(please_cli.config.ROOT_DIR, 'Dockerfile')
with open(nixpkgs_json_file) as f:
nixpkgs_json = json.load(f)
if 'rev' not in nixpkgs_json or \
'owner' not in nixpkgs_json or \
'repo' not in nixpkgs_json or \
'sha256_tarball' not in nixpkgs_json:
raise click.ClickException('`nix/nixpkgs.json` is not of correct format.')
with open(nix_json_file) as f:
nix_json = json.load(f)
if 'version' not in nix_json or \
'sha256' not in nix_json:
raise click.ClickException('`nix/nix.json` is not of correct format.')
try:
shutil.copyfile(docker_file, temp_docker_file)
click.echo(' => Building base docker image ... ', nl=False)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
[
docker,
'build',
'--no-cache',
'--pull',
'--force-rm',
'--build-arg', 'NIXPKGS_OWNER=' + nixpkgs_json['owner'],
'--build-arg', 'NIXPKGS_REPO=' + nixpkgs_json['repo'],
'--build-arg', 'NIXPKGS_REV=' + nixpkgs_json['rev'],
'--build-arg', 'NIXPKGS_SHA256=' + nixpkgs_json['sha256_tarball'],
'--build-arg', 'NIX_VERSION=' + nix_json['version'],
'--build-arg', 'NIX_SHA256=' + nix_json['sha256'],
'--build-arg', 'NIX_CACHE_PUBLIC_KEYS=' + ' '.join(nix_cache_public_keys),
'--build-arg', 'NIX_CACHE_PUBLIC_URLS=' + ' '.join(nix_cache_public_urls),
'-t', f'{docker_repo}:{docker_tag}',
please_cli.config.ROOT_DIR,
],
stream=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
finally:
if os.path.exists(temp_docker_file):
os.unlink(temp_docker_file)
def cmd_docker(project,
nix_build,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
docker,
interactive,
load_image,
):
image_path = please_cli.config.TMP_DIR + '/result-docker-{project}'.format(project=project)
# Build docker image for project
click.echo(' => Building docker image for {}'.format(project))
with click_spinner.spinner():
command = [
nix_build,
please_cli.config.ROOT_DIR + '/nix/default.nix',
'-A', '{}.docker'.format(project),
'-o', image_path
]
result, output, error = cli_common.command.run(
command,
stream=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(
result,
output,
ask_for_details=interactive,
)
if not load_image:
real_image_path = os.path.realpath(image_path)
click.echo(' => Copying docker image {} to {}'.format(real_image_path, please_cli.config.TMP_DIR))
with click_spinner.spinner():
command = ['cp', real_image_path, please_cli.config.TMP_DIR]
result, output, error = cli_common.command.run(
command,
stream=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
click.echo('You can load the image with this command: \n$ {docker} load -i {real_image_path}'.format(
docker=docker,
real_image_path=real_image_path,
))
return
# Loading docker image
click.echo(' => Importing docker image from {}'.format(image_path))
with click_spinner.spinner():
command = [
docker,
'load',
'-i', image_path,
]
result, output, error = cli_common.command.run(
command,
stream=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(
result,
output,
ask_for_details=interactive,
)
click.echo(' => Image loaded')
def cmd(ctx,
project,
github_commit,
task_group_id,
nix_path_attributes,
nix_build,
nix,
cache_bucket,
cache_region,
taskcluster_secret,
taskcluster_client_id,
taskcluster_access_token,
interactive,
):
required_secrets = [
'NIX_CACHE_SECRET_KEYS',
]
if cache_bucket and cache_region:
required_secrets += [
'CACHE_ACCESS_KEY_ID',
'CACHE_SECRET_ACCESS_KEY',
]
secrets = cli_common.taskcluster.get_secrets(taskcluster_secret,
project,
required=required_secrets,
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
temp_files = []
nix_cache_secret_keys = []
for secret_key in secrets['NIX_CACHE_SECRET_KEYS']:
fd, temp_file = tempfile.mkstemp(text=True)
with open(temp_file, 'w') as f:
f.write(secret_key)
os.close(fd)
nix_cache_secret_keys += [
'--option',
'secret-key-files',
temp_file,
]
temp_files.append(temp_file)
outputs = []
trace = ctx.obj['verbose'] > 1 and ['--show-trace'] or []
for nix_path_attribute in nix_path_attributes:
click.echo(' => Building {} ... '.format(nix_path_attribute), nl=False)
argstrs = []
if github_commit:
argstrs.extend(['--argstr', 'githubCommit', github_commit])
if task_group_id:
argstrs.extend(['--argstr', 'taskGroupId', task_group_id])
output = os.path.join(
please_cli.config.TMP_DIR,
'result-build-{}'.format(nix_path_attribute.replace('.', '-').replace('_', '-')),
)
outputs.append(output)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
[
nix_build,
please_cli.config.ROOT_DIR + '/nix/default.nix',
'-A', nix_path_attribute,
'-o', output,
] + nix_cache_secret_keys + argstrs + trace,
stream=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(
result,
output,
ask_for_details=interactive,
)
if result != 0:
break
if cache_bucket and cache_region:
tmp_cache_dir = os.path.join(please_cli.config.TMP_DIR, 'cache')
if not os.path.exists(tmp_cache_dir):
os.makedirs(tmp_cache_dir)
os.environ['AWS_ACCESS_KEY_ID'] = secrets['CACHE_ACCESS_KEY_ID']
os.environ['AWS_SECRET_ACCESS_KEY'] = secrets['CACHE_SECRET_ACCESS_KEY']
command = [
nix, 'copy',
'--to', 's3://{}?region={}'.format(cache_bucket, cache_region),
'-vvvv',
] + outputs
click.echo(' => Creating cache artifacts for {} project ... '.format(project), nl=False)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
command,
stream=True,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(
result,
output,
ask_for_details=interactive,
)
for temp_file in temp_files:
os.remove(temp_file)
return outputs
def cmd(certificates_dir, openssl, c_rehash, openssl_config, interactive=True):
if not os.path.isdir(certificates_dir):
click.echo(' => Creating certificates directory ... ')
with click_spinner.spinner():
os.makedirs(certificates_dir)
please_cli.utils.check_result(0, '', ask_for_details=interactive)
ca_key_file = os.path.join(certificates_dir, 'ca.key')
ca_cert_file = os.path.join(certificates_dir, 'ca.crt')
if os.path.exists(ca_key_file) or os.path.exists(ca_cert_file):
click.echo(' => Removing existing certificates ... ')
with click_spinner.spinner():
if os.path.exists(ca_key_file):
os.unlink(ca_key_file)
if os.path.exists(ca_cert_file):
os.unlink(ca_cert_file)
please_cli.utils.check_result(0, '', ask_for_details=interactive)
click.echo(' => Building CA certificate key ... ', nl=False)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
[
openssl,
'genrsa',
'-out', ca_key_file,
'2048',
],
stream=True,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
click.echo(' => Self signing CA certificate ... ', nl=False)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
[
openssl,
'req', '-x509', '-new', '-nodes',
'-key', ca_key_file,
'-days', '1024',
'-out', ca_cert_file,
'-subj', '/C=FR/ST=France/L=Paris/O=Mozilla/OU=Dev/CN=RelEngServices',
],
stream=True,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
server_key_file = os.path.join(certificates_dir, 'server.key')
server_cert_file = os.path.join(certificates_dir, 'server.crt')
server_csr_file = os.path.join(certificates_dir, 'server.csr')
server_cnf_file = os.path.join(certificates_dir, 'server.cnf')
if os.path.exists(server_key_file):
os.unlink(server_key_file)
if os.path.exists(server_cert_file):
os.unlink(server_cert_file)
if os.path.exists(server_csr_file):
os.unlink(server_csr_file)
if os.path.exists(server_cnf_file):
os.unlink(server_cnf_file)
click.echo(' => Building backend private certificate key ... ', nl=False)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
[
openssl,
'genrsa',
'-out', server_key_file,
'2048',
],
stream=True,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
click.echo(' => Creating openssh configuration ... ', nl=False)
with open(openssl_config, 'r') as f:
openssl_config_content = f.read()
with open(server_cnf_file, 'w+') as f:
f.write('{}\n[SAN]\nsubjectAltName=DNS:localhost,DNS:127.0.0.1'.format(openssl_config_content))
please_cli.utils.check_result(0, '', ask_for_details=interactive)
click.echo(' => Building backend csr certificate with mandatory subjectAltName ... ', nl=False)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
[
openssl,
'req', '-sha256', '-new',
'-key', server_key_file,
'-out', server_csr_file,
'-subj', '/C=FR/ST=France/L=Paris/O=Mozilla/OU=Dev/CN=localhost',
'-reqexts', 'SAN',
'-config', server_cnf_file,
],
stream=True,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
click.echo(' => Signing server certificate with CA certificate ... ', nl=False)
with click_spinner.spinner():
result, output, error = cli_common.command.run(
[
openssl,
'x509', '-req',
'-in', server_csr_file,
'-CA', ca_cert_file,
'-CAkey', ca_key_file,
'-CAcreateserial',
'-out', server_cert_file,
'-days', '500',
'-extensions', 'SAN',
'-extfile', server_cnf_file,
],
stream=True,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
click.echo(' => Hash certificates directory ... ', nl=False)
with click_spinner.spinner():
os.unlink(server_csr_file)
result, output, error = cli_common.command.run(
[c_rehash, certificates_dir],
stream=True,
stderr=subprocess.STDOUT,
)
please_cli.utils.check_result(result, output, ask_for_details=interactive)
def cmd(project,
cache_urls,
nix_instantiate,
channel,
indent=0,
interactive=True,
):
'''Command to check if project is already in cache.
'''
indent = ' ' * indent
channel_derivations = dict()
nix_path_attributes = [project]
deploys = please_cli.config.PROJECTS_CONFIG.get(project, dict()).get('deploys', [])
for deploy in deploys:
for _channel, options in deploy.get('options', dict()).items():
if _channel in please_cli.config.DEPLOY_CHANNELS:
nix_path_attribute = options.get('nix_path_attribute')
if nix_path_attribute:
nix_path_attributes.append(project + '.' + nix_path_attribute)
else:
nix_path_attributes.append(project)
nix_path_attributes = list(set(nix_path_attributes))
for nix_path_attribute in nix_path_attributes:
project_exists = False
click.echo('{} => Calculating `{}` hash ... '.format(indent, nix_path_attribute), nl=False)
command = [
nix_instantiate,
os.path.join(please_cli.config.ROOT_DIR, 'nix/default.nix'),
'-A', nix_path_attribute,
]
if interactive:
with click_spinner.spinner():
result, output, error = cli_common.command.run(
command,
stream=True,
stderr=subprocess.STDOUT,
)
else:
result, output, error = cli_common.command.run(
command,
stream=True,
stderr=subprocess.STDOUT,
)
try:
drv = output.split('\n')[-1].strip()
with open(drv) as f:
channel_derivations[nix_path_attribute] = eval(f.read())
except Exception as e:
log.exception(e)
raise click.ClickException('Something went wrong when reading derivation file for `{}` project.'.format(nix_path_attribute))
click.echo('{} found.'.format(channel_derivations[nix_path_attribute].nix_hash))
click.echo('{} => Checking cache if build artifacts exists for `{}` ... '.format(indent, nix_path_attribute), nl=False)
with click_spinner.spinner():
project_exists = False
for cache_url in cache_urls:
response = requests.get(
'%s/%s.narinfo' % (cache_url, channel_derivations[nix_path_attribute].nix_hash),
)
project_exists = response.status_code == 200
if project_exists:
break
result = 1
if project_exists:
result = 0
please_cli.utils.check_result(
result,
success_message='EXISTS',
error_message='NOT EXISTS',
raise_exception=False,
ask_for_details=interactive,
)
return project_exists, channel_derivations[project].nix_hash
def cmd(ctx,
github_commit,
channel,
owner,
pull_request,
task_id,
cache_urls,
nix_instantiate,
taskcluster_client_id,
taskcluster_access_token,
dry_run,
):
'''A tool to be ran on each commit.
'''
taskcluster_secret = 'repo:github.com/mozilla-releng/services:branch:' + channel
if pull_request is not None:
taskcluster_secret = 'repo:github.com/mozilla-releng/services:pull-request'
taskcluster_queue = cli_common.taskcluster.get_service('queue')
taskcluster_notify = cli_common.taskcluster.get_service('notify')
click.echo(' => Retriving taskGroupId ... ', nl=False)
with click_spinner.spinner():
task = taskcluster_queue.task(task_id)
if 'taskGroupId' not in task:
please_cli.utils.check_result(1, 'taskGroupId does not exists in task: {}'.format(json.dumps(task)))
task_group_id = task['taskGroupId']
please_cli.utils.check_result(0, '')
click.echo(' taskGroupId: ' + task_group_id)
if channel in please_cli.config.DEPLOY_CHANNELS:
taskcluster_notify.irc(dict(channel='#release-services',
message=f'New deployment on {channel} is about to start: https://tools.taskcluster.net/groups/{task_group_id}'))
message = ('release-services team is about to release a new version of mozilla/release-services '
'(*.mozilla-releng.net, *.moz.tools). Any alerts coming up soon will be best directed '
'to #release-services IRC channel. Automated message (such as this) will be send '
'once deployment is done. Thank you.')
'''This message will only be sent when channel is production.
'''
if channel is 'production':
for msgChannel in ['#ci', '#moc']:
taskcluster_notify.irc(dict(channel=msgChannel, message=message))
click.echo(' => Checking cache which project needs to be rebuilt')
build_projects = []
project_hashes = dict()
for project in sorted(PROJECTS):
click.echo(' => ' + project)
project_exists_in_cache, project_hash = ctx.invoke(
please_cli.check_cache.cmd,
project=project,
cache_urls=cache_urls,
nix_instantiate=nix_instantiate,
channel=channel,
indent=8,
interactive=False,
)
project_hashes[project] = project_hash
if not project_exists_in_cache:
build_projects.append(project)
projects_to_deploy = []
if channel in please_cli.config.DEPLOY_CHANNELS:
click.echo(' => Checking which project needs to be redeployed')
# TODO: get status for our index branch
deployed_projects = {}
for project_name in sorted(PROJECTS):
deployed_projects.get(project_name)
# update hook for each project
if please_cli.config.PROJECTS_CONFIG[project_name]['update'] is True:
if channel == 'production':
update_hook_nix_path_atttribute = f'updateHook.{channel}.scheduled'
else:
update_hook_nix_path_atttribute = f'updateHook.{channel}.notScheduled'
projects_to_deploy.append((
project_name,
[],
'TASKCLUSTER_HOOK',
{
'enable': True,
'docker_registry': 'index.docker.io',
'docker_repo': 'mozillareleng/services',
'name-suffix': '-update-dependencies',
'nix_path_attribute': update_hook_nix_path_atttribute,
},
))
if deployed_projects == project_hashes[project_name]:
continue
if 'deploys' not in please_cli.config.PROJECTS_CONFIG[project_name]:
continue
for deploy in please_cli.config.PROJECTS_CONFIG[project_name]['deploys']:
for deploy_channel in deploy['options']:
if channel == deploy_channel:
projects_to_deploy.append((
project_name,
please_cli.config.PROJECTS_CONFIG[project_name].get('requires', []),
deploy['target'],
deploy['options'][channel],
))
click.echo(' => Creating taskcluster tasks definitions')
tasks = []
# 1. build tasks
build_tasks = {}
for index, project in enumerate(sorted(build_projects)):
project_uuid = slugid.nice().decode('utf-8')
required = []
if pull_request is not None:
required += [
'CACHE_BUCKET',
'CACHE_REGION',
]
secrets = cli_common.taskcluster.get_secrets(
taskcluster_secret,
project,
required=required,
taskcluster_client_id=taskcluster_client_id,
taskcluster_access_token=taskcluster_access_token,
)
build_tasks[project_uuid] = get_build_task(
index,
project,
task_group_id,
task_id,
github_commit,
owner,
channel,
taskcluster_secret,
pull_request is None and secrets.get('CACHE_BUCKET') or None,
pull_request is None and secrets.get('CACHE_REGION') or None,
)
tasks.append((project_uuid, build_tasks[project_uuid]))
if projects_to_deploy:
# 2. maintanance on task
maintanance_on_uuid = slugid.nice().decode('utf-8')
if len(build_tasks.keys()) == 0:
maintanance_on_dependencies = [task_id]
else:
maintanance_on_dependencies = [i for i in build_tasks.keys()]
maintanance_on_task = get_task(
task_group_id,
maintanance_on_dependencies,
github_commit,
channel,
taskcluster_secret,
'./please -vv tools maintanance:on ' + ' '.join(list(set([i[0] for i in projects_to_deploy]))),
{
'name': '2. Maintanance ON',
'description': '',
'owner': owner,
'source': 'https://github.com/mozilla/release-services/tree/' + channel,
},
)
tasks.append((maintanance_on_uuid, maintanance_on_task))
# 3. deploy tasks (if on production/staging)
deploy_tasks = {}
for index, (project, project_requires, deploy_target, deploy_options) in \
enumerate(sorted(projects_to_deploy, key=lambda x: x[0])):
try:
enable = deploy_options['enable']
except KeyError:
raise click.ClickException(f'Missing {enable} in project {project} and channel {channel} deploy options')
if not enable:
continue
project_uuid = slugid.nice().decode('utf-8')
project_task = get_deploy_task(
index,
project,
project_requires,
deploy_target,
deploy_options,
task_group_id,
maintanance_on_uuid,
github_commit,
owner,
channel,
taskcluster_secret,
)
if project_task:
deploy_tasks[project_uuid] = project_task
tasks.append((project_uuid, deploy_tasks[project_uuid]))
# 4. maintanance off task
maintanance_off_uuid = slugid.nice().decode('utf-8')
maintanance_off_task = get_task(
task_group_id,
[i for i in deploy_tasks.keys()],
github_commit,
channel,
taskcluster_secret,
'./please -vv tools maintanance:off ' + ' '.join(list(set([i[0] for i in projects_to_deploy]))),
{
'name': '4. Maintanance OFF',
'description': '',
'owner': owner,
'source': 'https://github.com/mozilla/release-services/tree/' + channel,
},
)
maintanance_off_task['requires'] = 'all-resolved'
tasks.append((maintanance_off_uuid, maintanance_off_task))
click.echo(' => Submitting taskcluster definitions to taskcluster')
if dry_run:
tasks2 = {task_id: task for task_id, task in tasks}
for task_id, task in tasks:
click.echo(' => %s [taskId: %s]' % (task['metadata']['name'], task_id))
click.echo(' dependencies:')
deps = []
for dep in task['dependencies']:
depName = '0. Decision task'
if dep in tasks2:
depName = tasks2[dep]['metadata']['name']
deps.append(' - %s [taskId: %s]' % (depName, dep))
for dep in sorted(deps):
click.echo(dep)
else:
for task_id, task in tasks:
taskcluster_queue.createTask(task_id, task)
