def submit_context_for_building(user: str, project: str,
controller: Controller,
build_context: BinaryIO,
quiet_build: bool) -> str:
metadata = {
'user': user,
'project': project,
}
status_json_strings = controller.create_snapshot(metadata, build_context)
errors = []
snapshot_id: str = None
for json_str in status_json_strings:
data = json.loads(json_str)
if 'stream' in data:
if not quiet_build:
print(data['stream'], end='', flush=True)
if 'error' in data:
errors.append(data['error'].rstrip())
if 'id' in data:
snapshot_id = data['id']
if errors or not snapshot_id:
log_error('The snapshot was not successfully created.')
pull_access_denied = False
for error in errors:
if error.startswith('pull access denied'):
pull_access_denied = True
print(error)
exc_message = 'We did not receive a snapshot ID.'
if pull_access_denied:
raise CLIException(exc_message) \
from PullAccessDeniedException()
else:
raise CLIException(exc_message)
return snapshot_id
def print(self, configuration):
log_error(self.message)
cause = self.__cause__
if cause:
print(cause)
if configuration.debug:
traceback.print_exception(type(cause), cause,
cause.__traceback__)
def run(self):
user = self.configuration.user
if self.all_of_them_plz:
if self.instance_ids is not None:
raise CLIException('Can\'t specify both a list of instances '
'and --all-of-them-plz')
user_in_message = 'all users' if self.ignore_ownership else user
log_warning(
f'Killing all instances running jobs of {user_in_message} '
'for all projects')
if not self.oh_yeah:
answer = input('Are you sure? (yeah/Nope): ')
if answer != 'yeah':
raise CLIException('Cancelled by user')
else:
if self.instance_ids is None or len(self.instance_ids) == 0:
raise CLIException(
'You must specify a list of instance IDs with the -i '
'option. Use `plz list` to get instance IDs')
if self.including_idle:
raise CLIException(
'Option --including-idle only makes sense together with '
'--all-of-them-plz')
# The way the API likes it in this case
self.including_idle = None
log_info('Killing instances: ' + ' '.join(self.instance_ids))
if not self.all_of_them_plz and not self.instance_ids:
raise CLIException('No instance IDs specified')
try:
were_there_instances_to_kill = self.controller.kill_instances(
instance_ids=self.instance_ids,
force_if_not_idle=self.force_if_not_idle,
ignore_ownership=self.ignore_ownership,
including_idle=self.including_idle,
user=user)
except ProviderKillingInstancesException as e:
fails = e.failed_instance_ids_to_messages
log_error('Error terminating instances: \n' + ''.join([
f'{instance_id}: {message}\n'
for instance_id, message in fails.items()
]))
raise CLIException(
'Couldn\'t terminate all instances. You can use '
'--force-if-not-idle for non-idle instances')
if not were_there_instances_to_kill:
log_warning(
'Request to kill all instances, yet no instances were found.')
if not self.including_idle:
log_warning('Maybe you forgot --including-idle ?')
log_info('It was a clean job')
def get_execution_id_from_start_response(
response_dicts: Iterator[dict]) -> Tuple[str, bool]:
execution_id: Optional[str] = None
ok = True
for data in response_dicts:
if 'id' in data:
execution_id = data['id']
elif 'status' in data:
print('Instance status:', data['status'].rstrip())
elif 'error' in data:
ok = False
log_error(data['error'].rstrip())
if not execution_id:
raise CLIException('We did not receive an execution ID.')
return execution_id, ok
def run(self):
response_dict = self.controller.ping(self.ping_timeout)
if response_dict.get('plz', None) == 'pong':
if not self.silent_on_success:
log_info('Backend is reachable')
controller_build_timestamp = response_dict['build_timestamp']
if controller_build_timestamp != self.build_timestamp:
log_error(
'Version mismatch! The controller is running version\n'
f'{controller_build_timestamp}\n'
'while the cli is running\n'
f'{self.build_timestamp}\n'
'You can install the controller version with\n'
'pip install https://s3-eu-west-1.amazonaws.com/'
f'plz.prodo.ai/plz_cli-0.1.{controller_build_timestamp}'
'-py3-none-any.whl')
raise ExitWithStatusCodeException(1)
else:
log_error('Backend is unreachable')
raise ExitWithStatusCodeException(1)
def run(self):
if self.all_of_them_plz:
log_warning('Killing all instances for all users and projects')
if not self.oh_yeah:
answer = input('Are you sure? (yeah/Nope): ')
if answer != 'yeah':
raise CLIException('Cancelled by user')
instance_ids_for_controller = None
else:
if self.instance_ids is None or len(self.instance_ids) == 0:
raise CLIException(
'You must specify a list of instance IDs with the -i '
'option. Use `plz list` to get instance IDs')
log_info('Killing instances: ' + ' '.join(self.instance_ids))
instance_ids_for_controller = self.instance_ids
if not self.all_of_them_plz and not self.instance_ids:
raise CLIException('No instance IDs specified')
try:
were_there_instances_to_kill = self.controller.kill_instances(
instance_ids=instance_ids_for_controller,
force_if_not_idle=self.force_if_not_idle)
except ProviderKillingInstancesException as e:
fails = e.failed_instance_ids_to_messages
log_error('Error terminating instances: \n' + ''.join([
f'{instance_id}: {message}\n'
for instance_id, message in fails.items()
]))
raise CLIException(
'Couldn\'t terminate all instances. You can use '
'--force-if-not-idle for non-idle instances')
if not were_there_instances_to_kill:
log_warning(
'Request to kill all instances, yet no instances were found.')
log_info('It was a clean job')
def main(args=sys.argv[1:]):
parser = argparse.ArgumentParser()
parser.add_argument('-t', '--ping-timeout', type=int, default=5)
parser.add_argument('-c', '--configuration-path', type=str, default=None)
subparsers = parser.add_subparsers(title='operations',
dest='operation_name')
for operation in OPERATIONS:
subparser = subparsers.add_parser(operation.name(),
help=operation.__doc__)
operation.prepare_argument_parser(subparser, args)
options = parser.parse_args(args)
operation_name = options.operation_name
if operation_name is None:
parser.print_help()
sys.exit(2)
try:
configuration_path = options.configuration_path \
or os.environ.get('PLZ_CONFIGURATION_PATH', None)
configuration = Configuration.load(configuration_path, operation_name)
except ValidationException as e:
e.print()
sys.exit(2)
setup_logger(configuration)
option_dict = options.__dict__
del option_dict['operation_name']
ping_timeout = options.ping_timeout
if operation_name != 'ping-backend':
del option_dict['ping_timeout']
else:
option_dict['build_timestamp'] = _build_timestamp
del option_dict['configuration_path']
operation_classes = [o for o in OPERATIONS if o.name() == operation_name]
if len(operation_classes) == 0:
log_error('Internal error: couldn\'t find operation: '
f'{operation_name}')
sys.exit(os.EX_SOFTWARE)
if len(operation_classes) > 1:
log_error('Internal error: more than one operation with name: '
f'{operation_name}')
sys.exit(os.EX_SOFTWARE)
operation = operation_classes[0](configuration=configuration,
**option_dict)
try:
if operation_name != 'ping-backend':
# Ping the backend anyway as to avoid wasting user's time when the
# backend is down
PingBackendOperation(configuration,
silent_on_success=True,
ping_timeout=ping_timeout,
build_timestamp=_build_timestamp).run()
operation.run()
except KeyboardInterrupt:
log_error('Interrupted by the user.')
sys.exit(1)
except CLIException as e:
e.print(configuration)
sys.exit(e.exit_code)
except ExitWithStatusCodeException as e:
sys.exit(e.exit_code)