def wrapped(*args, **kwargs):
try:
return f(*args, **kwargs)
except Exception as cause:
cause_message = getattr(cause, 'message', '')
cause_message = f': {cause_message}' if cause_message else ''
raise CLIException(message + cause_message) from cause
def capture_build_context(image: str, image_extensions: [str], command: str,
context_path: [str], excluded_paths: [str],
included_paths: [str],
exclude_gitignored_files) -> BinaryIO:
dockerfile_path = os.path.join(context_path, 'plz.Dockerfile')
dockerfile_created = False
try:
with open(dockerfile_path, mode='x') as dockerfile:
dockerfile_created = True
dockerfile.write(f'FROM {image}\n')
for step in image_extensions:
dockerfile.write(step)
dockerfile.write('\n')
dockerfile.write(f'WORKDIR /src\n'
f'COPY . ./\n'
f'CMD {command}\n')
os.chmod(dockerfile_path, 0o644)
matching_excluded_paths = get_matching_excluded_paths(
context_path=context_path,
excluded_paths=excluded_paths,
included_paths=included_paths,
exclude_gitignored_files=exclude_gitignored_files)
build_context = docker.utils.build.tar(
path=context_path,
exclude=matching_excluded_paths,
gzip=True,
)
except FileExistsError as e:
raise CLIException(
'The directory cannot have a plz.Dockerfile.') from e
finally:
if dockerfile_created:
os.remove(dockerfile_path)
return build_context
def parse_file(path) -> Parameters:
if path is None:
return {}
try:
with open(path) as f:
return parse_io(f, path)
except FileNotFoundError as e:
raise CLIException(
f'The parameters file "{path}" does not exist.') from e
def _get_top_level_config(config_file_name, config_set_explicitly: bool):
config = Configuration.from_file(config_file_name,
Configuration.PROPERTIES,
fail_on_read_error=True)
# The user provided a configuration file explicitly, but we couldn't
# read a configuration from it
if config_set_explicitly and config is None:
raise CLIException(
f'Couldn\'t read a configuration from {config_file_name}')
return config
def request(self, method: str, *path_segments: str,
codes_with_exceptions: Optional[Set[int]] = None, **kwargs) \
-> Response:
codes_with_exceptions = codes_with_exceptions or set()
try:
url = self.prefix + '/' + '/'.join(path_segments)
session = requests.session()
if self.schema == ssh_session.PLZ_SSH_SCHEMA:
add_ssh_channel_adapter(session, self.connection_info)
response = session.request(method, url, **kwargs)
self._maybe_raise_exception(response, codes_with_exceptions)
return response
except (ConnectionError, requests.ConnectionError,
urllib3.exceptions.NewConnectionError) as e:
raise CLIException(
f'We couldn\'t establish a connection to the server.') from e
except (TimeoutError, requests.Timeout) as e:
raise CLIException(
'Our connection to the server timed out.') from e
def harvest(self):
try:
self.controller.delete_execution(
execution_id=self.get_execution_id(),
fail_if_running=True,
fail_if_deleted=False)
except InstanceStillRunningException:
if self.force_if_running:
log_info('Process is still running')
return
else:
raise CLIException(
'Process is still running, run `plz stop` if you want to '
'terminate it, \nor use --force-if-running (discouraged)')
def _is_there_a_head_commit(context_path: str) -> bool:
# We could be doing `git rev-list -n 1 --all`, and check that the output
# is non-empty, but Ubuntu ships with ridiculous versions of git
result = subprocess.run(['git', '-C', context_path, 'show-ref', '--head'],
input=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding='utf-8')
if result.returncode not in {0, 1} or result.stderr != '':
raise CLIException('Error finding if there are commits. \n'
f'Return code: {result.returncode}. \n'
f'Stdout: {result.stdout}. \n'
f'Stderr: [{result.stderr}]. \n')
return result.returncode == 0 and ' HEAD\n' in result.stdout
def run(self):
if self.all_of_them_plz:
log_warning('Killing all instances for all users and projects')
if not self.oh_yeah:
answer = input('Are you sure? (yeah/Nope): ')
if answer != 'yeah':
raise CLIException('Cancelled by user')
instance_ids_for_controller = None
else:
if self.instance_ids is None or len(self.instance_ids) == 0:
raise CLIException(
'You must specify a list of instance IDs with the -i '
'option. Use `plz list` to get instance IDs')
log_info('Killing instances: ' + ' '.join(self.instance_ids))
instance_ids_for_controller = self.instance_ids
if not self.all_of_them_plz and not self.instance_ids:
raise CLIException('No instance IDs specified')
try:
were_there_instances_to_kill = self.controller.kill_instances(
instance_ids=instance_ids_for_controller,
force_if_not_idle=self.force_if_not_idle)
except ProviderKillingInstancesException as e:
fails = e.failed_instance_ids_to_messages
log_error('Error terminating instances: \n' + ''.join([
f'{instance_id}: {message}\n'
for instance_id, message in fails.items()
]))
raise CLIException(
'Couldn\'t terminate all instances. You can use '
'--force-if-not-idle for non-idle instances')
if not were_there_instances_to_kill:
log_warning(
'Request to kill all instances, yet no instances were found.')
log_info('It was a clean job')
def get_execution_id_from_start_response(
response_dicts: Iterator[dict]) -> Tuple[str, bool]:
execution_id: Optional[str] = None
ok = True
for data in response_dicts:
if 'id' in data:
execution_id = data['id']
elif 'status' in data:
print('Instance status:', data['status'].rstrip())
elif 'error' in data:
ok = False
log_error(data['error'].rstrip())
if not execution_id:
raise CLIException('We did not receive an execution ID.')
return execution_id, ok
def _get_head_commit(context_path: str) -> Optional[str]:
if not _is_there_a_head_commit(context_path):
return None
result = subprocess.run(['git', '-C', context_path, 'rev-parse', 'HEAD'],
input=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
encoding='utf-8')
commit = result.stdout.strip()
if result.returncode != 0 or result.stderr != '' or len(commit) == 0:
raise CLIException('Couldn\'t get HEAD commit. \n'
f'Return code: {result.returncode}. \n'
f'Stdout: {result.stdout}. \n'
f'Stderr: [{result.stderr}]. \n')
return commit
def from_configuration(configuration: Configuration,
controller: Controller):
if not configuration.input:
return NoInputData()
if configuration.input.startswith('file://'):
path = configuration.input[len('file://'):]
return LocalInputData(controller=controller,
user=configuration.user,
project=configuration.project,
path=path)
elif configuration.input.startswith('input_id://'):
input_id = configuration.input[len('input_id://'):]
return LocalInputData(controller=controller,
user=configuration.user,
project=configuration.project,
input_id=input_id)
raise CLIException('Could not parse the configured input.')
def put_input(self, input_id: str, input_metadata: InputMetadata,
input_data_stream: BinaryIO) -> None:
response = self.server.put(
'data',
'input',
input_id,
data=input_data_stream,
stream=True,
params={
'user': input_metadata.user,
'project': input_metadata.project,
'path': input_metadata.path,
'timestamp_millis': input_metadata.timestamp_millis
})
_check_status(response, requests.codes.ok)
if input_id != response.json()['id']:
raise CLIException('Got wrong input id back from the server')
def run(self):
user = self.configuration.user
if self.all_of_them_plz:
if self.instance_ids is not None:
raise CLIException('Can\'t specify both a list of instances '
'and --all-of-them-plz')
user_in_message = 'all users' if self.ignore_ownership else user
log_warning(
f'Killing all instances running jobs of {user_in_message} '
'for all projects')
if not self.oh_yeah:
answer = input('Are you sure? (yeah/Nope): ')
if answer != 'yeah':
raise CLIException('Cancelled by user')
else:
if self.instance_ids is None or len(self.instance_ids) == 0:
raise CLIException(
'You must specify a list of instance IDs with the -i '
'option. Use `plz list` to get instance IDs')
if self.including_idle:
raise CLIException(
'Option --including-idle only makes sense together with '
'--all-of-them-plz')
# The way the API likes it in this case
self.including_idle = None
log_info('Killing instances: ' + ' '.join(self.instance_ids))
if not self.all_of_them_plz and not self.instance_ids:
raise CLIException('No instance IDs specified')
try:
were_there_instances_to_kill = self.controller.kill_instances(
instance_ids=self.instance_ids,
force_if_not_idle=self.force_if_not_idle,
ignore_ownership=self.ignore_ownership,
including_idle=self.including_idle,
user=user)
except ProviderKillingInstancesException as e:
fails = e.failed_instance_ids_to_messages
log_error('Error terminating instances: \n' + ''.join([
f'{instance_id}: {message}\n'
for instance_id, message in fails.items()
]))
raise CLIException(
'Couldn\'t terminate all instances. You can use '
'--force-if-not-idle for non-idle instances')
if not were_there_instances_to_kill:
log_warning(
'Request to kill all instances, yet no instances were found.')
if not self.including_idle:
log_warning('Maybe you forgot --including-idle ?')
log_info('It was a clean job')
def harvest(self,
atomic_execution_id: Optional[str] = None,
composition_path: Optional[List[Tuple[str, Any]]] = None):
if atomic_execution_id is None:
atomic_execution_id = self.get_execution_id()
if composition_path is None:
composition_path = []
try:
self.controller.delete_execution(execution_id=atomic_execution_id,
fail_if_running=True,
fail_if_deleted=False)
except InstanceStillRunningException:
if self.force_if_running or len(composition_path) > 0:
log_info('Process is still running')
return
else:
raise CLIException(
'Process is still running, run `plz stop` if you want to '
'terminate it, \nor use --force-if-running (discouraged)')
def retrieve_output(self):
execution_id = self.get_execution_id()
output_tarball_bytes = self.controller.get_output_files(
self.get_execution_id(), path=self.path)
formatted_output_dir = self.output_dir.replace('%e', execution_id)
formatted_output_dir = os.path.join(
formatted_output_dir, self.path if self.path is not None else '')
try:
os.makedirs(formatted_output_dir)
except FileExistsError:
if self.force_if_running:
log_info('Removing existing output directory')
shutil.rmtree(formatted_output_dir)
os.makedirs(formatted_output_dir)
else:
raise CLIException(
f'The output directory "{formatted_output_dir}" '
'already exists.')
for path in untar(output_tarball_bytes, formatted_output_dir):
print(path)
def retrieve_output(
self,
atomic_execution_id: Optional[str] = None,
composition_path: Optional[List[Tuple[str, Any]]] = None):
if atomic_execution_id is None:
atomic_execution_id = self.get_execution_id()
if composition_path is None:
composition_path = []
if len(composition_path) > 0:
index = int(composition_path[-1][1])
else:
index = None
output_tarball_bytes = self.controller.get_output_files(
atomic_execution_id, path=self.path, index=index)
formatted_output_dir = \
self.output_dir.replace('%e', self.get_execution_id())
formatted_output_dir = os.path.join(
formatted_output_dir,
*('-'.join(node) for node in composition_path),
self.path if self.path is not None else '')
try:
os.makedirs(formatted_output_dir)
except FileExistsError:
if len(composition_path) > 0 and not self.rewrite_subexecutions:
log_info('Output directory already present')
return
if self.force_if_running or len(composition_path) > 0:
log_info('Removing existing output directory')
shutil.rmtree(formatted_output_dir)
os.makedirs(formatted_output_dir)
else:
raise CLIException(
f'The output directory "{formatted_output_dir}" '
'already exists.')
for path in untar(output_tarball_bytes, formatted_output_dir):
print(path)
def run(self):
execution_id = self.get_execution_id()
if execution_id is None or execution_id == '':
raise CLIException('No execution ID for this user!')
print(execution_id, end='\n' if sys.stdout.isatty() else '')
def run(self):
if not self.command:
raise CLIException('No command specified! Use --command or '
'include a `command` entry in plz.config.json')
if not self.configuration.image:
raise CLIException('No image specified! Include an `image` entry '
'in plz.config.json')
if os.path.exists(self.output_dir):
raise CLIException(
f'The output directory "{self.output_dir}" already exists.')
params = parameters.parse_file(self.parameters_file)
exclude_gitignored_files = \
self.configuration.exclude_gitignored_files
context_path = self.configuration.context_path
def build_context_suboperation():
return capture_build_context(
image=self.configuration.image,
image_extensions=self.configuration.image_extensions,
command=self.configuration.command,
context_path=context_path,
excluded_paths=self.configuration.excluded_paths,
included_paths=self.configuration.included_paths,
exclude_gitignored_files=exclude_gitignored_files,
)
retries = self.configuration.workarounds['docker_build_retries']
while retries + 1 > 0:
with self.suboperation(
f'Capturing the files in {os.path.abspath(context_path)}',
build_context_suboperation) as build_context:
try:
snapshot_id = self.suboperation(
'Building the program snapshot',
lambda: submit_context_for_building(
user=self.configuration.user,
project=self.configuration.project,
controller=self.controller,
build_context=build_context,
quiet_build=self.configuration.quiet_build))
break
except CLIException as e:
if type(e.__cause__) == PullAccessDeniedException \
and retries > 0:
log_warning(str(e))
log_warning(
'This might be a transient error. Retrying')
retries -= 1
time.sleep(7)
else:
raise e
input_id = self.suboperation('Capturing the input',
self.capture_input,
if_set=self.configuration.input)
execution_id, was_start_ok = self.suboperation(
'Sending request to start execution', lambda: self.start_execution(
snapshot_id, params, input_id, context_path))
self.execution_id = execution_id
self.follow_execution(was_start_ok)