def check_status(response: Response,
project: Project,
step: ProjectStep,
force: bool = False) -> str:
"""
:param response:
:param project:
:param step:
:param force:
:return:
"""
path = step.source_path
if step.is_muted:
environ.log('[{}]: Muted (skipped)'.format(step.definition.name))
return SKIP_STATUS
if not os.path.exists(path):
response.fail(code='MISSING_SOURCE_FILE',
message='Source file not found "{}"'.format(path),
id=step.definition.name,
path=path).console('[{id}]: Not found "{path}"'.format(
id=step.definition.name, path=path))
return ERROR_STATUS
if not force and not step.is_dirty():
environ.log('[{}]: Nothing to update'.format(step.definition.name))
return SKIP_STATUS
return OK_STATUS
def autocomplete(command_name: str, prefix: str, line: str, begin_index: int,
end_index: int):
"""
:param command_name:
:param prefix:
:param line:
:param begin_index:
:param end_index:
:return:
"""
cmds = fetch()
if command_name not in cmds:
return []
parts = parse.explode_line(line)[1:]
if line.endswith(' '):
parts.append('')
try:
module = cmds[command_name]
if hasattr(module, 'autocomplete'):
out = getattr(module, 'autocomplete')(prefix, line, parts)
if out is not None:
return out
except Exception as err:
environ.log(message='[ERROR] Autocomplete Failed: "{}"'.format(err),
whitespace=1)
return []
def execute_remote(context: cli.CommandContext, **kwargs) -> Response:
""" """
sync_response = sync_command.execute(cli.make_command_context(
name=sync_command.NAME,
remote_connection=context.remote_connection
))
context.response.consume(sync_response)
if sync_response.failed:
return context.response
environ.log('[STARTED]: Remote run execution', whitespace=1)
thread = sync.send_remote_command(
command=context.name,
raw_args=context.raw_args,
asynchronous=True,
show_logs=True
)
thread.join()
response = thread.responses[-1]
return context.response.consume(response)
def execute(
context: cli.CommandContext,
force: bool = False,
all_projects: bool = False
) -> Response:
"""
:param context:
:param force:
:param all_projects:
"""
response = context.response
environ.log_header('REMOVE RESULTS', level=2)
environ.log(
ALL_MESSAGE if all_projects else PROJECT_MESSAGE,
whitespace=1
)
do_it = force
if not force:
do_it = query.confirm(
'Are you sure you want to continue',
default=False
)
if not do_it:
return response.notify(
kind='ABORTED',
code='NO_PURGE',
message='No files were deleted'
).console(
whitespace=1
).response
if context.remote_connection.active:
return remote_purge(context)
path = environ.configs.fetch('results_directory')
path = path if path else environ.paths.user('results')
if environ.systems.remove(path):
response.notify(
kind='SUCCESS',
code='RESULTS_PURGED',
message='All results have been removed'
).console(
whitespace=1
)
else:
response.fail(
code='PURGE_FAILURE',
message='Failed to remove results'
).console(
whitespace=1
)
return response
def get_source_path() -> typing.Union[str, None]:
source_path = get_version_one_path()
if source_path is None:
environ.log(PLOTLY_WARNING)
return None
elif not os.path.exists(source_path):
source_path = get_version_two_path()
return source_path
def get_source_path() -> typing.Union[str, None]:
source_path = get_version_one_path()
if source_path is None:
environ.log(PLOTLY_WARNING)
return None
elif not os.path.exists(source_path):
source_path = get_version_two_path()
return source_path
def echo_all():
"""
:return:
"""
out = ['Current Configuration:']
for key, value in environ.configs.fetch_all().items():
out.append(' * {key}: {value}'.format(key=key, value=value))
environ.log('\n'.join(out))
def do_synchronize(
context: cli.CommandContext,
source_directory: str,
newer_than: float
) -> Response:
""" """
synchronized = []
def on_progress(message: ResponseMessage):
if message.kind == 'SKIP':
return
if len(synchronized) < 1:
environ.log_header(
text='SYNCHRONIZING',
level=2,
whitespace=1
)
if message.code == 'STARTED':
synchronized.append(message)
chunk_count = message.data.get('chunk_count', 0)
if message.code == 'DONE' and chunk_count < 2:
return
message.console()
sync_response = sync.files.send_all_in(
directory=source_directory,
remote_connection=context.remote_connection,
newer_than=newer_than,
progress_callback=on_progress
)
context.response.consume(sync_response)
context.response.update(synchronized_count=len(synchronized))
if len(synchronized) < 1:
return context.response
touch_response = sync.comm.send_request(
endpoint='/sync-touch',
method='GET',
remote_connection=context.remote_connection
)
context.response.consume(touch_response)
if not context.response.failed:
environ.log('Synchronization Complete', whitespace=1)
return context.response
def echo_key(key: str):
"""
:param key:
:return:
"""
value = environ.configs.fetch(key)
if value is None:
environ.log('[MISSING]: No "{}" key was found'.format(key))
return
environ.log('[VALUE]: "{}" = {}'.format(key, value))
def print_path_group(header, paths):
if not paths:
return
environ.log_header(header, level=6, indent_by=2)
entries = []
for p in paths:
parts = p.rstrip(os.sep).split(os.sep)
name = parts[-1]
if name.startswith('@'):
name = name.split(':', 1)[-1]
entries.append('* "{name}" -> {path}'.format(name=name, path=p))
environ.log('\n'.join(entries), indent_by=4)
def complete(response: Response,
project: typing.Union[Project, None],
starting: ProjectStep = None,
force: bool = False,
limit: int = -1) -> list:
"""
Runs the entire project, writes the results files, and returns the URL to
the report file
:param response:
:param project:
:param starting:
:param force:
:param limit:
:return:
Local URL to the report path
"""
if project is None:
project = cauldron.project.get_internal_project()
starting_index = 0
if starting:
starting_index = project.steps.index(starting)
count = 0
steps_run = []
for ps in project.steps:
if 0 < limit <= count:
break
if ps.index < starting_index:
continue
if not force and not ps.is_dirty():
if limit < 1:
environ.log('[{}]: Nothing to update'.format(
ps.definition.name))
continue
count += 1
steps_run.append(ps)
success = source.run_step(response, project, ps, force=True)
if not success or project.stop_condition.halt:
return steps_run
return steps_run
def choice(
title: str,
prompt: str,
choices: list,
default_index: int = None
) -> typing.Tuple[int, str]:
"""
:param title:
:param prompt:
:param choices:
:param default_index:
:return:
"""
entries = []
for index in range(len(choices)):
entries.append('[{index}]: {value}'.format(
index=index + 1,
value=choices[index]
))
entries.insert(0, '')
entries.insert(
0,
'{bar}\n{title}\n{bar}'.format(title=title, bar='-' * len(title))
)
environ.log(entries, whitespace=1)
default = ''
if default_index is not None:
default = ' [{}]'.format(1 + max(0, min(len(choices), default_index)))
while True:
result = input('{question}{default}:'.format(
question=prompt,
default=default
))
result = re.compile('[^0-9]*').sub('', result)
if len(result) == 0:
if default_index is None:
continue
result = default_index
else:
result = max(0, min(int(result) - 1, len(choices)))
return result, choices[result]
def remove_key(key: str, persists: bool = True):
"""
Removes the specified key from the cauldron configs if the key exists
:param key:
The key in the cauldron configs object to remove
:param persists:
"""
environ.configs.remove(key, include_persists=persists)
environ.configs.save()
environ.log(
'[REMOVED]: "{}" from configuration settings'.format(key)
)
def print_path_group(header, paths):
if not paths:
return
environ.log_header(header, level=6, indent_by=2)
entries = []
for p in paths:
parts = p.rstrip(os.sep).split(os.sep)
name = parts[-1]
if name.startswith('@'):
name = name.split(':', 1)[-1]
entries.append(
'* "{name}" -> {path}'.format(name=name, path=p)
)
environ.log('\n'.join(entries), indent_by=4)
def list_snapshots(project: Project):
"""
:param project:
:return:
"""
snapshots = get_snapshot_listing(project)
if not snapshots:
environ.log('No snapshots found')
return None
entries = []
for item in snapshots:
entries.append('* {}'.format(item['name']))
environ.log_header('EXISTING SNAPSHOTS')
environ.log(entries, whitespace_bottom=1, indent_by=3)
def create(project: 'projects.Project') -> COMPONENT:
"""
:return:
"""
try:
from bokeh.resources import Resources as BokehResources
bokeh_resources = BokehResources(mode='absolute')
except Exception:
bokeh_resources = None
if bokeh_resources is None:
environ.log(BOKEH_WARNING)
return COMPONENT([], [])
return definitions.merge_components(
_assemble_component(project, 'bokeh-css', ['bokeh', 'bokeh.css'],
bokeh_resources.css_files),
_assemble_component(project, 'bokeh-js', ['bokeh', 'bokeh.js'],
bokeh_resources.js_files))
def set_key(key: str, value: typing.List[str], persists: bool = True):
"""
Removes the specified key from the cauldron configs if the key exists
:param key:
The key in the cauldron configs object to remove
:param value:
:param persists:
"""
if key.endswith('_path') or key.endswith('_paths'):
for index in range(len(value)):
value[index] = environ.paths.clean(value[index])
if len(value) == 1:
value = value[0]
environ.configs.put(**{key: value}, persists=persists)
environ.configs.save()
environ.log('[SET]: "{}" to "{}"'.format(key, value))
def choice(title: str,
prompt: str,
choices: list,
default_index: int = None) -> typing.Tuple[int, str]:
"""
:param title:
:param prompt:
:param choices:
:param default_index:
:return:
"""
entries = []
for index in range(len(choices)):
entries.append('[{index}]: {value}'.format(index=index + 1,
value=choices[index]))
entries.insert(0, '')
entries.insert(
0, '{bar}\n{title}\n{bar}'.format(title=title, bar='-' * len(title)))
environ.log(entries, whitespace=1)
default = ''
if default_index is not None:
default = ' [{}]'.format(1 + max(0, min(len(choices), default_index)))
while True:
result = input('{question}{default}:'.format(question=prompt,
default=default))
result = re.compile('[^0-9]*').sub('', result)
if len(result) == 0:
if default_index is None:
continue
result = default_index
else:
result = max(0, min(int(result) - 1, len(choices)))
return result, choices[result]
def check_status(
response: Response,
project: Project,
step: ProjectStep,
force: bool = False
) -> str:
"""
:param response:
:param project:
:param step:
:param force:
:return:
"""
path = step.source_path
if step.is_muted:
environ.log('[{}]: Muted (skipped)'.format(step.definition.name))
return SKIP_STATUS
if not os.path.exists(path):
response.fail(
code='MISSING_SOURCE_FILE',
message='Source file not found "{}"'.format(path),
id=step.definition.name,
path=path
).console(
'[{id}]: Not found "{path}"'.format(
id=step.definition.name,
path=path
)
)
return ERROR_STATUS
if not force and not step.is_dirty():
environ.log('[{}]: Nothing to update'.format(step.definition.name))
return SKIP_STATUS
return OK_STATUS
def execute_remote(context: cli.CommandContext, **kwargs) -> Response:
""" """
sync_response = sync_command.execute(
cli.make_command_context(name=sync_command.NAME,
remote_connection=context.remote_connection))
context.response.consume(sync_response)
if sync_response.failed:
return context.response
environ.log('[STARTED]: Remote run execution', whitespace=1)
thread = sync.send_remote_command(command=context.name,
raw_args=context.raw_args,
asynchronous=True,
show_logs=True)
thread.join()
response = thread.responses[-1]
return context.response.consume(response)
def execute(
port: int = 5010,
debug: bool = False,
public: bool = False,
host=None,
authentication_code: str = '',
**kwargs
):
"""
:param port:
:param debug:
:param public:
:param host:
:param authentication_code:
:return:
"""
if kwargs.get('version'):
environ.log('VERSION: {}'.format(environ.version))
sys.exit(0)
if host is None and public:
host = '0.0.0.0'
server_data['host'] = host
server_data['port'] = port
server_data['debug'] = debug
server_data['id'] = environ.start_time.isoformat()
authorization['code'] = authentication_code if authentication_code else ''
if not debug:
log = logging.getLogger('werkzeug')
log.setLevel(logging.ERROR)
environ.modes.add(environ.modes.INTERACTIVE)
APPLICATION.run(port=port, debug=debug, host=host)
environ.modes.remove(environ.modes.INTERACTIVE)
def create_snapshot(
project: Project,
*args: typing.List[str],
show: bool = True
):
"""
:param project:
:param show:
:return:
"""
if len(args) < 1:
snapshot_name = datetime.now().strftime('%Y%b%d-%H-%M-%S')
else:
snapshot_name = args[0]
snapshot_directory = project.snapshot_path()
if not os.path.exists(snapshot_directory):
os.makedirs(snapshot_directory)
snapshot_name = snapshot_name.replace(' ', '-')
snapshot_directory = project.snapshot_path(snapshot_name)
environ.systems.remove(snapshot_directory)
shutil.copytree(project.output_directory, snapshot_directory)
url = project.snapshot_url(snapshot_name)
environ.log_header('Snapshot URL', 5)
environ.log(
'* {}'.format(url),
whitespace_bottom=1,
indent_by=2
)
if show:
webbrowser.open(url)
def remove_snapshot(
project: Project,
*args: typing.List[str]
):
"""
:param project:
:param args:
:return:
"""
if len(args) < 1 or not args[0].strip():
environ.log(
"""
Are you sure you want to remove all snapshots in this project?
""")
if not query.confirm('Confirm Delete All', False):
environ.log(
'[ABORTED]: No snapshots were deleted',
whitespace=1
)
return
if not environ.systems.remove(project.snapshot_path()):
environ.log(
'[ERROR]: Failed to delete snapshots',
whitespace=1
)
return
environ.log(
'[SUCCESS]: All snapshots have been removed',
whitespace=1
)
return
snapshot_name = args[0]
environ.log(
"""
Are you sure you want to remove the snapshot "{}"?
""".format(snapshot_name),
whitespace=1
)
if not query.confirm('Confirm Deletion', False):
environ.log(
"""
[ABORTED]: "{}" was not removed
""".format(snapshot_name),
whitespace=1
)
return
if not environ.systems.remove(project.snapshot_path(snapshot_name)):
environ.log(
"""
[ERROR]: Unable to delete snapshot "{}" at this time
""".format(snapshot_name),
whitespace=1
)
return
environ.log(
"""
[SUCCESS]: Snapshot "{}" was removed
""".format(snapshot_name),
whitespace=1
)
return
def run_step(
response: Response,
project: Project,
step: typing.Union[ProjectStep, str],
force: bool = False
) -> bool:
"""
:param response:
:param project:
:param step:
:param force:
:return:
"""
step = get_step(project, step)
if step is None:
return False
status = check_status(response, project, step, force)
if status == ERROR_STATUS:
return False
step.error = None
if status == SKIP_STATUS:
return True
os.chdir(os.path.dirname(step.source_path))
project.current_step = step
step.report.clear()
step.dom = None
step.is_running = True
step.progress_message = None
step.progress = 0
# Set the top-level display and cache values to the current project values
# before running the step for availability within the step scripts
cauldron.shared = cauldron.project.shared
redirection.enable(step)
try:
result = _execute_step(project, step)
except Exception as error:
result = dict(
success=False,
message='{}'.format(error),
html_message='<pre>{}</pre>'.format(error)
)
os.chdir(os.path.expanduser('~'))
step.mark_dirty(not result['success'])
step.error = result.get('html_message')
step.last_modified = time.time() if result['success'] else 0.0
step.is_running = False
step.progress = 0
step.progress_message = None
step.dumps()
# Make sure this is called prior to printing response information to the
# console or that will come along for the ride
redirection.disable(step)
if result['success']:
environ.log('[{}]: Updated'.format(step.definition.name))
else:
response.fail(
message='Step execution error',
code='EXECUTION_ERROR',
project=project.kernel_serialize(),
step_name=step.definition.name
).console_raw(result['message'])
return result['success']
def execute(
context: cli.CommandContext,
action: str,
arguments: list,
no_show: bool = False
) -> Response:
"""
:param context:
:param action:
:param arguments:
:param no_show:
:return:
"""
show = not no_show
if not action:
return context.response.fail(
code='NO_ACTION_ARG',
message='An action is required for the snapshot command'
).console(
whitespace=1
).response
action = action.strip().lower()
project = cauldron.project.internal_project
if not project:
return context.response.fail(
code='NO_OPEN_PROJECT',
message='No open project'
).console(
'[ERROR]: No open project. Use the "open" command to open one.'
)
if action == 'remove':
return actions.remove_snapshot(project, *arguments)
if action == 'add':
return actions.create_snapshot(project, *arguments, show=show)
if action == 'list':
return actions.list_snapshots(project)
if action == 'open':
name = arguments[0]
result = actions.open_snapshot(project, name)
if result is None:
environ.log('[ERROR]: No snapshot found named "{}"'.format(name))
return context.response
environ.log_header('SNAPSHOT: {}'.format(name))
environ.log(
"""
URL: {url}
LAST MODIFIED: {modified}
""".format(
url=result['url'],
modified=datetime.fromtimestamp(
result['last_modified']
).strftime('%H:%M %b %d, %Y')
),
whitespace=1
)
if show:
webbrowser.open(result['url'])
def execute(
context: cli.CommandContext,
command: str = None,
name: str = None,
path: str = None,
temporary: bool = False
):
"""
:return:
"""
response = context.response
if name:
name = name.replace(' ', '_').strip('"').strip()
if path:
path = environ.paths.clean(path.strip('"'))
if not os.path.isdir(path):
path = os.path.dirname(path)
environ.log(
"""
[WARNING]: The specified path was not a directory. Aliases must
be directories, so the directory containing the specified
file will be used instead:
{}
""".format(path),
whitespace=1
)
environ.configs.load()
temporary_aliases = environ.configs.session.get('folder_aliases', {})
persistent_aliases = environ.configs.persistent.get('folder_aliases', {})
if not name and command in ['add', 'remove']:
return response.fail(
code='MISSING_ARG',
message='You need to specify the name of the alias'
).console(
whitespace=1
).response
if command == 'list':
items = []
aliases = dict(
list(temporary_aliases.items()) +
list(persistent_aliases.items())
)
for k, v in aliases.items():
items.append('{}\n {}'.format(k, v['path']))
environ.log_header('EXISTING ALIASES')
environ.log(items)
return response
aliases = temporary_aliases if temporary else persistent_aliases
if command == 'add':
aliases[name] = dict(
path=path
)
environ.configs.put(
persists=not bool(temporary),
folder_aliases=aliases
)
return response.notify(
kind='ADDED',
code='ALIAS_ADDED',
message='The alias "{}" has been saved'.format(name)
).console(
whitespace=1
).response
if command == 'remove':
if name in aliases:
del aliases[name]
environ.configs.put(
persists=not bool(temporary),
folder_aliases=aliases
)
return response.notify(
kind='REMOVED',
code='ALIAS_REMOVED',
message='The alias "{}" has been removed'.format(name)
).console(
whitespace=1
).response
return response.fail(
code='UNKNOWN_COMMAND',
message='Unrecognized alias command "{}"'.format(command)
).console(
whitespace=1
).response
def run_step(
response: Response,
project: Project,
step: typing.Union[ProjectStep, str],
force: bool = False
) -> bool:
"""
:param response:
:param project:
:param step:
:param force:
:return:
"""
step = get_step(project, step)
if step is None:
return False
status = check_status(response, project, step, force)
if status == ERROR_STATUS:
return False
step.error = None
if status == SKIP_STATUS:
return True
os.chdir(os.path.dirname(step.source_path))
project.current_step = step
step.report.clear()
step.dom = None
step.is_visible = True
step.is_running = True
step.progress_message = None
step.progress = 0
step.sub_progress_message = None
step.sub_progress = 0
step.start_time = datetime.utcnow()
step.end_time = None
# Set the top-level display and cache values to the current project values
# before running the step for availability within the step scripts
cauldron.shared = cauldron.project.shared
redirection.enable(step)
try:
result = _execute_step(project, step)
except Exception as error:
result = dict(
success=False,
message='{}'.format(error),
html_message='<pre>{}</pre>'.format(error)
)
step.end_time = datetime.utcnow()
os.chdir(os.path.expanduser('~'))
step.mark_dirty(not result['success'])
step.error = result.get('html_message')
step.last_modified = time.time() if result['success'] else 0.0
step.is_running = False
step.progress = 0
step.progress_message = None
step.dumps()
# Make sure this is called prior to printing response information to the
# console or that will come along for the ride
redirection.disable(step)
step.project.stop_condition = result.get(
'stop_condition',
StopCondition(False, False)
)
if result['success']:
environ.log('[{}]: Updated in {}'.format(
step.definition.name,
step.get_elapsed_timestamp()
))
else:
response.fail(
message='Step execution error',
code='EXECUTION_ERROR',
project=project.kernel_serialize(),
step_name=step.definition.name
).console_raw(result['message'])
return result['success']
