def __init__(self, current_version, wanted_version):
ClickException.__init__(
self, 'Cannot migrate from version {0} to version '
'{1}'.format(current_version, wanted_version)
)
self.current_version = current_version
self.wanted_version = wanted_version
def test_invalid_pull_from_pypi(self, fake_project_cli, mocker, tmp_path,
fake_metadata):
"""
Test for pulling package from pypi, and it cannot be found.
"""
pypi_error_message = (
"ERROR: Could not find a version that satisfies the requirement")
python_call_mock = mocker.patch(
"kedro.framework.cli.pipeline.python_call",
side_effect=ClickException(pypi_error_message),
)
mocker.patch(
"kedro.framework.cli.pipeline.tempfile.TemporaryDirectory",
return_value=tmp_path,
)
invalid_pypi_name = "non_existent"
result = CliRunner().invoke(fake_project_cli,
["pipeline", "pull", invalid_pypi_name],
obj=fake_metadata)
assert result.exit_code
python_call_mock.assert_called_once_with("pip", [
"download", "--no-deps", "--dest",
str(tmp_path), invalid_pypi_name
])
assert pypi_error_message in result.stdout
def delete(config, tree_names, all, force):
"""Delete indices and their catalog entries.
This deletes the indices that have the format version of the copy of DXR
this runs under.
"""
es = ElasticSearch(config.es_hosts)
if all:
echo('Deleting catalog...')
es.delete_index(config.es_catalog_index)
# TODO: Delete tree indices as well.
else:
for tree_name in tree_names:
frozen_id = '%s/%s' % (FORMAT, tree_name)
try:
frozen = es.get(config.es_catalog_index, TREE, frozen_id)
except ElasticHttpNotFoundError:
raise ClickException('No tree "%s" in catalog.' % tree_name)
# Delete the index first. That way, if that fails, we can still
# try again; we won't have lost the catalog entry. Refresh is
# infrequent enough that we wouldn't avoid a race around a
# catalogued but deleted instance the other way around.
try:
es.delete_index(frozen['_source']['es_alias'])
except ElasticHttpNotFoundError:
# It's already gone. Fine. Just remove the catalog entry.
pass
es.delete(config.es_catalog_index, TREE, frozen_id)
def get_eth_network(eth_network: Optional[str]) -> NetworkConfig:
"""
Parse the `eth_network` parameter to extract a URL. If `eth_network` does
not contain a URL, try interpreting it as a network name, otherwise
interpret it as a file to load the network config from. Fall back to a
default network config filename, and finally the default network name.
"""
if eth_network is None:
if exists(ETH_NETWORK_FILE_DEFAULT):
eth_network = ETH_NETWORK_FILE_DEFAULT
else:
eth_network = ETH_NETWORK_DEFAULT
if eth_network.startswith("http"):
# When given only a url, assume the default network name
return NetworkConfig(ETH_NETWORK_DEFAULT, eth_network)
# Try loading from a file
if exists(eth_network):
with open(eth_network) as network_f:
return NetworkConfig.from_json(network_f.read())
# Assume a network name
try:
endpoint = ETH_RPC_ENDPOINT_DEFAULTS[eth_network]
return NetworkConfig(eth_network, endpoint)
except KeyError as ex:
raise ClickException(
f"invalid network name / url: {eth_network}") from ex
def get_remote_url(cwd=None, remote='origin'):
"""拿cwd的remote的url.
发现envoy.run的command只能是bytes, 不能是unicode.
"""
ctx = click.get_current_context()
r = delegator.run('git remote get-url %s' % str(remote), cwd=cwd)
if r.return_code:
raise ClickException(r.err)
remote = r.out.strip()
# 对gitlab ci需要特殊处理一下
# 丫有个特殊的格式, 不太好支持...
match = _GITLAB_CI_REMOTE_URL_PATTERN.match(remote)
if match:
host = match.group(2)
group = match.group(3)
project = match.group(4)
return 'git@{host}:{group}/{project}.git'.format(host=host,
group=group,
project=project)
if ctx.obj['debug']:
click.echo(debug_log('get_remote_url: %s', remote))
return remote
def get_renderer_by_id(report, renderer_id):
try:
return next(filter(lambda r: r.id == renderer_id, report.renderers))
except StopIteration:
raise ClickException(
f'The output format `{renderer_id}` is not available for report {report.local_id}.'
)
def app_show(context, application_id, format, export):
try:
app = context.apps.retrieve(application_id)
except ApiClientException as e:
raise ClickException(e) from e
if format == 'yaml':
dump = yaml.safe_dump(app,
indent=4,
allow_unicode=True,
default_flow_style=False)
else:
dump = json.dumps(app, sort_keys=True, indent=4)
click.echo(dump)
if export is not None:
del app['_created'], app['_etag'], app['_updated'], app[
'pending_changes'], app['user']
if format == 'yaml':
dump = yaml.safe_dump(app,
indent=4,
allow_unicode=True,
default_flow_style=False)
else:
dump = json.dumps(app, sort_keys=True, indent=4)
export.write(dump)
def deploy(context, application_id, module, all_modules, strategy,
safe_deploy_strategy, live_logs, output, no_color):
# TODO find a "clicker" way to do this parameter validation
if not module and not all_modules:
raise MissingParameter(
'You must have one (and only one) from --module and --all-modules parameters',
param_hint='module',
param_type='parameter')
if module and all_modules:
raise BadParameter(
'You must have only one from --module and --all-modules parameters',
param_hint='module')
if all_modules:
app = context.apps.retrieve(application_id)
module = [m["name"] for m in app["modules"]]
modules = []
for m in module:
name, rev = m.split(':') if ':' in m else (m, None)
mod = {"name": name}
if rev:
mod["rev"] = rev
modules.append(mod)
try:
job_id = context.jobs.command_deploy(application_id, modules, strategy,
safe_deploy_strategy)
handle_job_creation(context, job_id, live_logs, output, no_color)
except ApiClientException as e:
raise ClickException(e) from e
def run_service_task(cluster, service, command, success_string, timeout, region, container=None, ecs_client=None):
ecs_client = ecs_client if ecs_client else _get_ecs_client(region)
container = service if not container else container
task_definition = get_task_definition_for_service(
cluster=cluster, service=service, region=region, ecs_client=ecs_client)
latest_task_definition = task_definition[:task_definition.rfind(':')]
latest_task_definition_name = latest_task_definition[latest_task_definition.rfind('/') + 1:]
resp = ecs_client.describe_task_definition(taskDefinition=latest_task_definition)
container_definitions = resp['taskDefinition']['containerDefinitions']
container_definition = [d for d in container_definitions if d["name"] == container]
if len(container_definition) != 1:
raise ClickException(
('Container "{}" not found in task.\n'
'Containers available in task: {}'.format(container, [d["name"] for d in container_definitions])))
container_name = container_definition[0]['name']
run_task_and_wait_for_success(
cluster=cluster,
task_definition=latest_task_definition_name,
command=command,
name=container_name,
success_string=success_string,
timeout=timeout,
region=region,
ecs_client=ecs_client,
)
def update_service_to_new_task_definition(cluster, service, task_definition, region, force=True, ecs_client=None):
ecs_client = ecs_client if ecs_client else _get_ecs_client(region)
try:
response = ecs_client.update_service(
cluster=cluster,
service=service,
taskDefinition=task_definition,
forceNewDeployment=force,
)
except ecs_client.exceptions.ClusterNotFoundException:
raise ClickException("Cluster not found: '{}'".format(cluster))
except ecs_client.exceptions.ServiceNotFoundException:
raise ClickException("Service not found: '{}'".format(service))
except ClientError as ex:
raise ClickException(ex)
def next_stage(current_stage, upgrade_stage):
"""
Determine the VM stage that occurs after the given stage.
:param current_stage: current VM stage
:return: next VM stage
"""
if upgrade_stage is not None:
if upgrade_stage is "current":
return current_stage
if upgrade_stage is "fork":
return Stage.fork
if upgrade_stage is "base":
return Stage.base
if upgrade_stage is "crio":
return Stage.crio
if current_stage == Stage.bare:
return Stage.base
elif current_stage == Stage.base:
return Stage.build
elif current_stage == Stage.build:
return Stage.install
elif current_stage == Stage.install:
echo(
'Warning: No next stage exists past the "{}" stage. Overwriting current stage instead.'
.format(Stage.install))
return Stage.install
else:
raise ClickException(
'The current stage of the VM, "{}", has no next stage specified.'.
format(current_stage))
def register(username: str, email: str, password: str,
password_confirmation: str) -> str:
"""
Register new Registry account and automatically login if successful.
:param username: str username.
:param email: str email.
:param password: str password.
:param password_confirmation: str password confirmation.
:return: str auth token.
"""
data = {
"username": username,
"email": email,
"password1": password,
"password2": password_confirmation,
}
resp_json, status_code = request_api(
"POST",
"/rest-auth/registration/",
data=data,
handle_400=False,
return_code=True,
)
if status_code == 400:
errors: List[str] = []
for key in ("username", "email", "password1", "password2"):
param_errors = resp_json.get(key)
if param_errors:
errors.extend(param_errors)
raise ClickException("Errors occured during registration.\n" +
"\n".join(errors))
return resp_json["key"]
def _add_transaction(self, transaction):
"""Adds a transaction and updates the calculated values."""
if self._share_balance == 0:
# to prevent divide by 0 error
old_acb_per_share = 0
else:
old_acb_per_share = self._total_acb / self._share_balance
proceeds = (transaction.qty * transaction.price
) * transaction.exchange_rate # noqa: E501
if transaction.action == 'SELL':
self._share_balance -= transaction.qty
acb = old_acb_per_share * transaction.qty
capital_gain = proceeds - transaction.expenses - acb
self._total_acb -= acb
else:
self._share_balance += transaction.qty
acb = proceeds + transaction.expenses
capital_gain = Decimal(0.0)
self._total_acb += acb
if self._share_balance < 0:
raise ClickException("Transaction caused negative share balance")
transaction.share_balance = self._share_balance
transaction.proceeds = proceeds
transaction.capital_gain = capital_gain
transaction.acb = acb
def _dump_image(image_location, image_name, media_path):
image = requests.get(image_location)
if image.status_code == 200:
with open(os.path.join(media_path, image_name), 'wb') as f:
f.write(image.content)
else:
raise ClickException(f"Error obtaining image from {image_location}")
def update_configuration_option(container, option, value, write_func):
"""
Update an option in a configuration file.
:param container: configuration file to work on
:param option: name of the option to update
:param value: value to update to
:param write_func: func to serialize the updated config
"""
if option not in container:
raise ClickException(message='Option {} not found in configuration.'.format(option))
# try to interpret the most common values
if value.isdigit():
value = int(value)
elif value.lower() in ['true', 'yes']:
value = True
elif value.lower() in ['false', 'no']:
value = False
elif ',' in value:
value = value.split(',')
setattr(container, option, value)
write_func()
echo('Option {} updated to be {}.'.format(option, value))
def copy_local_tracks_to_folder(tracklist: List[str], dest: str) -> None:
"""Copy local files from list to a new destination."""
destination: Path = Path(dest)
missing_files: List[str] = []
file_destination: Path
if not destination.is_dir():
destination = destination.parent
with click.progressbar(
tracklist,
label="Copying from playlist:",
) as bar:
for abs_path in bar:
if not Path(abs_path).exists():
missing_files.append(abs_path)
else:
name_only = Path(abs_path).name
file_destination = destination / name_only
if not file_destination.exists():
try:
shutil.copy2(Path(abs_path), destination)
except (OSError) as error:
message = str(error)
raise ClickException(message)
if missing_files:
click.echo("Missing files from playlist were NOT copied:")
click.echo("\n".join(missing_files))
def swapbluegreen(context, application_id, strategy, live_logs, output,
no_color):
try:
job_id = context.jobs.command_swapbluegreen(application_id, strategy)
handle_job_creation(context, job_id, live_logs, output, no_color)
except ApiClientException as e:
raise ClickException(e) from e
def update_service_to_latest_task_definition(cluster, service, region):
ecs_client = _get_ecs_client(region)
old_task_definition = get_task_definition_for_service(cluster=cluster,
service=service,
region=region)
LOGGER.info('Current task definition ARN: {}'.format(old_task_definition))
try:
new_task_definition = ecs_client.describe_task_definition(
taskDefinition=old_task_definition[:old_task_definition.rfind(':'
)])
except ClientError as ex:
raise ClickException(ex)
new_task_definition_arn = new_task_definition['taskDefinition'][
'taskDefinitionArn']
LOGGER.info('New task definition ARN: {}'.format(new_task_definition_arn))
update_service_to_new_task_definition(
cluster=cluster,
service=service,
task_definition=new_task_definition_arn,
region=region,
)
def apps_list(context, nb, page, name, env, role):
try:
apps, max_results, total, cur_page = context.apps.list(nb=nb,
page=page,
name=name,
env=env,
role=role)
except ApiClientException as e:
raise ClickException(e) from e
if max_results >= total:
click.echo('Showing all the {} applications'.format(total))
else:
click.echo('Showing {} on {} applications - Page {}'.format(
max_results, total, cur_page))
click.echo(
tabulate([[
app['_id'],
app['name'],
app['env'],
app['role'],
'{} ({})'.format(
app['blue_green']['color'],
'Online' if app['blue_green']['is_online'] else 'Offline') if
app.get('blue_green', {}).get('enable_blue_green', False) else '',
app.get('description', ''),
] for app in apps],
headers=[
'ID', 'Name', 'Environment', 'Role', 'Color',
'Description'
]))
def migrate_service(cluster, service, command, success_string, timeout,
region):
ecs_client = _get_ecs_client(region)
task_definition = get_task_definition_for_service(cluster=cluster,
service=service,
region=region)
latest_task_definition = task_definition[:task_definition.rfind(':')]
latest_task_definition_name = latest_task_definition[latest_task_definition
.rfind('/') + 1:]
resp = ecs_client.describe_task_definition(
taskDefinition=latest_task_definition)
container_definitions = resp['taskDefinition']['containerDefinitions']
if len(container_definitions) != 1:
raise ClickException(
('Exactly one container is allowed to be specified in service.\n'
'Number of containers specified: {}'.format(
len(container_definitions))))
container_name = container_definitions[0]['name']
run_task_and_wait_for_success(
cluster=cluster,
task_definition=latest_task_definition_name,
command=command,
name=container_name,
success_string=success_string,
timeout=timeout,
region=region,
)
def fetch_report_from_gads_client_customer_obj(self, report_definition,
client_customer_id):
if not self.valid_client_customer_id(client_customer_id):
raise ClickException(
f"Invalid format: {client_customer_id}. Client customer ID should respect the following format 123-456-7890."
)
else:
try:
adwords_client = self.init_adwords_client(client_customer_id)
report_downloader = adwords_client.GetReportDownloader()
customer_report = report_downloader.DownloadReportAsStream(
report_definition,
client_customer_id=client_customer_id,
include_zero_impressions=self.include_zero_impressions,
skip_report_header=True,
skip_column_header=True,
skip_report_summary=True,
)
return customer_report
except AdWordsReportBadRequestError as e:
if e.type == "AuthorizationError.CUSTOMER_NOT_ACTIVE":
logger.warning(
f"Skipping clientCustomerId {client_customer_id} (inactive)."
)
else:
raise Exception(f"Wrong request. Error type: {e.type}")
def token_approve(
ctx: Context,
value: str,
eth_addr: str,
eth_private_key: str,
wait: bool,
instance_file: str) -> None:
"""
Approve the mixer to spend some amount of ERC20/223 tokens
"""
approve_value = EtherValue(value)
eth_addr = load_eth_address(eth_addr)
eth_private_key_data = load_eth_private_key(eth_private_key)
web3 = open_web3_from_network(get_eth_network(ctx.obj["eth_network"]))
mixer_desc = load_mixer_description(instance_file)
if not mixer_desc.token:
raise ClickException("no token for mixer {mixer_desc.mixer.address}")
token_instance = mixer_desc.token.instantiate(web3)
approve_call = token_instance.functions.approve(
mixer_desc.mixer.address,
approve_value.wei)
tx_hash = send_contract_call(
web3, approve_call, eth_addr, eth_private_key_data)
if wait:
web3.eth.waitForTransactionReceipt(tx_hash) # pylint: disable=no-member
else:
print(tx_hash.hex())
def install(package, parallel):
"""
Install one or more packages.
"""
console.print(f"resolving packages: {package}")
installers: Dict[str, Callable] = InstallerManager().get_installers()
for pkg in package:
if pkg not in installers:
raise ClickException(
f"unable to locate installer for package {pkg}")
if not parallel:
# install one by one
for pkg in package:
console.print(f"installing [bold]{pkg}[/bold]")
with console.status("installing..."):
try:
installers[pkg]()
except Exception as e:
console.print(f"[red]error[/red]: {e}")
else:
console.print(f"install {parallel} packages in parallel:")
# collect installers and install in parallel:
with Pool(processes=parallel) as pool:
pool.map(_do_install, package)
def validate_ad_insights_level(self):
if self.ad_insights:
if self.level == "creative" or self.object_type == "creative":
raise ClickException(
f"Wrong query. The 'creative' level is not available in Ad Insights queries.\
Accepted levels: {FACEBOOK_OBJECTS[1:]}")
def jobs_list(context, nb, page, application, env, role, command, status,
user):
try:
job_list, max_results, total, cur_page = context.jobs.list(
nb=nb,
page=page,
application=application,
env=env,
role=role,
command=command,
status=status,
user=user)
except ApiClientException as e:
raise ClickException(e) from e
if max_results >= total:
click.echo('Showing all the {} jobs'.format(total))
else:
click.echo('Showing {} on {} jobs - Page {}'.format(
max_results, total, cur_page))
click.echo(
tabulate([[
job['_id'], job['app_id']['name'] if job.get('app_id') else '',
job['command'], job['status'], job['user'], job['_created']
] for job in job_list],
headers=[
'ID', 'Application name', 'Command', 'Status', 'User',
'Date'
]))
def check_object_id(ctx, param, values):
try:
[int(value) for value in values]
return values
except ValueError:
raise ClickException(
"Wrong format. Ad object IDs should only contains digits.")
def eth_gen_address(eth_addr_file: Optional[str],
eth_private_key_file: Optional[str],
use_private_key: Optional[str]) -> None:
"""
Locally generate a new Ethereum private key and address file, and write
them to the current directory.
"""
sys.stderr.write(
"*** WARNING: this address should not be used in production ***\n")
eth_addr_file = eth_addr_file or ETH_ADDRESS_DEFAULT
eth_private_key_file = eth_private_key_file or ETH_PRIVATE_KEY_FILE_DEFAULT
if use_private_key:
eth_private_key = bytes.fromhex(use_private_key)
else:
eth_private_key = gen_eth_private_key()
if len(eth_private_key) != 32:
raise ClickException("invalid private key length")
write_eth_private_key(eth_private_key, eth_private_key_file)
eth_address = eth_address_from_private_key(eth_private_key)
write_eth_address(eth_address, eth_addr_file)
print(f"{eth_address}: written to {eth_addr_file}, "
f"private key to {eth_private_key_file}")
def check_for_local_server():
"""
Raise an error if no local server is defined in the global configuration
"""
if not config["local_server"]["url"]:
raise ClickException(
"No local server selected. Run `openag db init` to select one")
def migrate(**kwargs):
try:
click.echo("make sure source file is python 3 compatible")
click.echo(
migration_tool.make_sure_source_code_is_python3_compatible(
kwargs["in_file"])
)
data = open(kwargs["in_file"], "r").read()
emulate_progress_bar("check for unsupported modules", 5)
migration_tool.check_for_unsupported_modules(data)
emulate_progress_bar("make sure all required methods are implemented")
data = migration_tool.add_missing_base_methods(data)
emulate_progress_bar("remove unsupported imports", 5)
data = migration_tool.remove_quantopian_imports(data)
data = migration_tool.remove_commission(data)
emulate_progress_bar("define a logger", 5)
data = migration_tool.define_logger(data)
emulate_progress_bar("checking if using pipeline", 5)
data = migration_tool.add_pipelinelive_imports(data)
emulate_progress_bar("adding pylivetrader imports", 5)
data = migration_tool.add_pylivetrader_imports(data)
emulate_progress_bar("Finalizing")
data = migration_tool.cleanup(data)
with open(kwargs["out_file"], 'w') as f:
f.write(data)
except Exception as e:
raise ClickException(e)
def check_for_cloud_server():
"""
Raise an error if no cloud server is defined in the global configuration
"""
if not config["cloud_server"]["url"]:
raise ClickException(
"No cloud server selected. Run `openag cloud init` to select one")
def update(recid):
"""Update metadata for record with given recid in DataCite."""
uuid = PersistentIdentifier.get('recid', recid).object_uuid
record = Record.get_record(uuid)
doi = record['doi']
try:
provider = DataCiteProviderWrapper.get(pid_value=doi,
pid_type='doi')
except PIDDoesNotExistError:
raise ClickException('Record with DOI {} not registered in DataCite.'
.format(doi))
# serialize record to schema40
doc = DataCiteSerializer().dump(record).data
schema40.validate(doc)
doc = schema40.tostring(doc)
landing_page = os.path.join(
current_app.config.get('PIDSTORE_LANDING_BASE_URL'),
recid)
provider.update(url=landing_page,
doc=doc)
db.session.commit()
click.echo('Record with DOI {} updated in DataCite'.format(doi))
def __init__(self, message):
if PY2 and message is not None:
message = message.encode('utf-8')
ClickException.__init__(self, message)
self.message = message
