def check_status(response: Response, project: Project, step: ProjectStep, force: bool = False) -> str: """ :param response: :param project: :param step: :param force: :return: """ path = step.source_path if step.is_muted: environ.log('[{}]: Muted (skipped)'.format(step.definition.name)) return SKIP_STATUS if not os.path.exists(path): response.fail(code='MISSING_SOURCE_FILE', message='Source file not found "{}"'.format(path), id=step.definition.name, path=path).console('[{id}]: Not found "{path}"'.format( id=step.definition.name, path=path)) return ERROR_STATUS if not force and not step.is_dirty(): environ.log('[{}]: Nothing to update'.format(step.definition.name)) return SKIP_STATUS return OK_STATUS
def autocomplete(command_name: str, prefix: str, line: str, begin_index: int, end_index: int): """ :param command_name: :param prefix: :param line: :param begin_index: :param end_index: :return: """ cmds = fetch() if command_name not in cmds: return [] parts = parse.explode_line(line)[1:] if line.endswith(' '): parts.append('') try: module = cmds[command_name] if hasattr(module, 'autocomplete'): out = getattr(module, 'autocomplete')(prefix, line, parts) if out is not None: return out except Exception as err: environ.log(message='[ERROR] Autocomplete Failed: "{}"'.format(err), whitespace=1) return []
def execute_remote(context: cli.CommandContext, **kwargs) -> Response: """ """ sync_response = sync_command.execute(cli.make_command_context( name=sync_command.NAME, remote_connection=context.remote_connection )) context.response.consume(sync_response) if sync_response.failed: return context.response environ.log('[STARTED]: Remote run execution', whitespace=1) thread = sync.send_remote_command( command=context.name, raw_args=context.raw_args, asynchronous=True, show_logs=True ) thread.join() response = thread.responses[-1] return context.response.consume(response)
def execute( context: cli.CommandContext, force: bool = False, all_projects: bool = False ) -> Response: """ :param context: :param force: :param all_projects: """ response = context.response environ.log_header('REMOVE RESULTS', level=2) environ.log( ALL_MESSAGE if all_projects else PROJECT_MESSAGE, whitespace=1 ) do_it = force if not force: do_it = query.confirm( 'Are you sure you want to continue', default=False ) if not do_it: return response.notify( kind='ABORTED', code='NO_PURGE', message='No files were deleted' ).console( whitespace=1 ).response if context.remote_connection.active: return remote_purge(context) path = environ.configs.fetch('results_directory') path = path if path else environ.paths.user('results') if environ.systems.remove(path): response.notify( kind='SUCCESS', code='RESULTS_PURGED', message='All results have been removed' ).console( whitespace=1 ) else: response.fail( code='PURGE_FAILURE', message='Failed to remove results' ).console( whitespace=1 ) return response
def get_source_path() -> typing.Union[str, None]: source_path = get_version_one_path() if source_path is None: environ.log(PLOTLY_WARNING) return None elif not os.path.exists(source_path): source_path = get_version_two_path() return source_path
def echo_all(): """ :return: """ out = ['Current Configuration:'] for key, value in environ.configs.fetch_all().items(): out.append(' * {key}: {value}'.format(key=key, value=value)) environ.log('\n'.join(out))
def do_synchronize( context: cli.CommandContext, source_directory: str, newer_than: float ) -> Response: """ """ synchronized = [] def on_progress(message: ResponseMessage): if message.kind == 'SKIP': return if len(synchronized) < 1: environ.log_header( text='SYNCHRONIZING', level=2, whitespace=1 ) if message.code == 'STARTED': synchronized.append(message) chunk_count = message.data.get('chunk_count', 0) if message.code == 'DONE' and chunk_count < 2: return message.console() sync_response = sync.files.send_all_in( directory=source_directory, remote_connection=context.remote_connection, newer_than=newer_than, progress_callback=on_progress ) context.response.consume(sync_response) context.response.update(synchronized_count=len(synchronized)) if len(synchronized) < 1: return context.response touch_response = sync.comm.send_request( endpoint='/sync-touch', method='GET', remote_connection=context.remote_connection ) context.response.consume(touch_response) if not context.response.failed: environ.log('Synchronization Complete', whitespace=1) return context.response
def echo_key(key: str): """ :param key: :return: """ value = environ.configs.fetch(key) if value is None: environ.log('[MISSING]: No "{}" key was found'.format(key)) return environ.log('[VALUE]: "{}" = {}'.format(key, value))
def print_path_group(header, paths): if not paths: return environ.log_header(header, level=6, indent_by=2) entries = [] for p in paths: parts = p.rstrip(os.sep).split(os.sep) name = parts[-1] if name.startswith('@'): name = name.split(':', 1)[-1] entries.append('* "{name}" -> {path}'.format(name=name, path=p)) environ.log('\n'.join(entries), indent_by=4)
def complete(response: Response, project: typing.Union[Project, None], starting: ProjectStep = None, force: bool = False, limit: int = -1) -> list: """ Runs the entire project, writes the results files, and returns the URL to the report file :param response: :param project: :param starting: :param force: :param limit: :return: Local URL to the report path """ if project is None: project = cauldron.project.get_internal_project() starting_index = 0 if starting: starting_index = project.steps.index(starting) count = 0 steps_run = [] for ps in project.steps: if 0 < limit <= count: break if ps.index < starting_index: continue if not force and not ps.is_dirty(): if limit < 1: environ.log('[{}]: Nothing to update'.format( ps.definition.name)) continue count += 1 steps_run.append(ps) success = source.run_step(response, project, ps, force=True) if not success or project.stop_condition.halt: return steps_run return steps_run
def choice( title: str, prompt: str, choices: list, default_index: int = None ) -> typing.Tuple[int, str]: """ :param title: :param prompt: :param choices: :param default_index: :return: """ entries = [] for index in range(len(choices)): entries.append('[{index}]: {value}'.format( index=index + 1, value=choices[index] )) entries.insert(0, '') entries.insert( 0, '{bar}\n{title}\n{bar}'.format(title=title, bar='-' * len(title)) ) environ.log(entries, whitespace=1) default = '' if default_index is not None: default = ' [{}]'.format(1 + max(0, min(len(choices), default_index))) while True: result = input('{question}{default}:'.format( question=prompt, default=default )) result = re.compile('[^0-9]*').sub('', result) if len(result) == 0: if default_index is None: continue result = default_index else: result = max(0, min(int(result) - 1, len(choices))) return result, choices[result]
def remove_key(key: str, persists: bool = True): """ Removes the specified key from the cauldron configs if the key exists :param key: The key in the cauldron configs object to remove :param persists: """ environ.configs.remove(key, include_persists=persists) environ.configs.save() environ.log( '[REMOVED]: "{}" from configuration settings'.format(key) )
def print_path_group(header, paths): if not paths: return environ.log_header(header, level=6, indent_by=2) entries = [] for p in paths: parts = p.rstrip(os.sep).split(os.sep) name = parts[-1] if name.startswith('@'): name = name.split(':', 1)[-1] entries.append( '* "{name}" -> {path}'.format(name=name, path=p) ) environ.log('\n'.join(entries), indent_by=4)
def list_snapshots(project: Project): """ :param project: :return: """ snapshots = get_snapshot_listing(project) if not snapshots: environ.log('No snapshots found') return None entries = [] for item in snapshots: entries.append('* {}'.format(item['name'])) environ.log_header('EXISTING SNAPSHOTS') environ.log(entries, whitespace_bottom=1, indent_by=3)
def create(project: 'projects.Project') -> COMPONENT: """ :return: """ try: from bokeh.resources import Resources as BokehResources bokeh_resources = BokehResources(mode='absolute') except Exception: bokeh_resources = None if bokeh_resources is None: environ.log(BOKEH_WARNING) return COMPONENT([], []) return definitions.merge_components( _assemble_component(project, 'bokeh-css', ['bokeh', 'bokeh.css'], bokeh_resources.css_files), _assemble_component(project, 'bokeh-js', ['bokeh', 'bokeh.js'], bokeh_resources.js_files))
def set_key(key: str, value: typing.List[str], persists: bool = True): """ Removes the specified key from the cauldron configs if the key exists :param key: The key in the cauldron configs object to remove :param value: :param persists: """ if key.endswith('_path') or key.endswith('_paths'): for index in range(len(value)): value[index] = environ.paths.clean(value[index]) if len(value) == 1: value = value[0] environ.configs.put(**{key: value}, persists=persists) environ.configs.save() environ.log('[SET]: "{}" to "{}"'.format(key, value))
def choice(title: str, prompt: str, choices: list, default_index: int = None) -> typing.Tuple[int, str]: """ :param title: :param prompt: :param choices: :param default_index: :return: """ entries = [] for index in range(len(choices)): entries.append('[{index}]: {value}'.format(index=index + 1, value=choices[index])) entries.insert(0, '') entries.insert( 0, '{bar}\n{title}\n{bar}'.format(title=title, bar='-' * len(title))) environ.log(entries, whitespace=1) default = '' if default_index is not None: default = ' [{}]'.format(1 + max(0, min(len(choices), default_index))) while True: result = input('{question}{default}:'.format(question=prompt, default=default)) result = re.compile('[^0-9]*').sub('', result) if len(result) == 0: if default_index is None: continue result = default_index else: result = max(0, min(int(result) - 1, len(choices))) return result, choices[result]
def check_status( response: Response, project: Project, step: ProjectStep, force: bool = False ) -> str: """ :param response: :param project: :param step: :param force: :return: """ path = step.source_path if step.is_muted: environ.log('[{}]: Muted (skipped)'.format(step.definition.name)) return SKIP_STATUS if not os.path.exists(path): response.fail( code='MISSING_SOURCE_FILE', message='Source file not found "{}"'.format(path), id=step.definition.name, path=path ).console( '[{id}]: Not found "{path}"'.format( id=step.definition.name, path=path ) ) return ERROR_STATUS if not force and not step.is_dirty(): environ.log('[{}]: Nothing to update'.format(step.definition.name)) return SKIP_STATUS return OK_STATUS
def execute_remote(context: cli.CommandContext, **kwargs) -> Response: """ """ sync_response = sync_command.execute( cli.make_command_context(name=sync_command.NAME, remote_connection=context.remote_connection)) context.response.consume(sync_response) if sync_response.failed: return context.response environ.log('[STARTED]: Remote run execution', whitespace=1) thread = sync.send_remote_command(command=context.name, raw_args=context.raw_args, asynchronous=True, show_logs=True) thread.join() response = thread.responses[-1] return context.response.consume(response)
def execute( port: int = 5010, debug: bool = False, public: bool = False, host=None, authentication_code: str = '', **kwargs ): """ :param port: :param debug: :param public: :param host: :param authentication_code: :return: """ if kwargs.get('version'): environ.log('VERSION: {}'.format(environ.version)) sys.exit(0) if host is None and public: host = '0.0.0.0' server_data['host'] = host server_data['port'] = port server_data['debug'] = debug server_data['id'] = environ.start_time.isoformat() authorization['code'] = authentication_code if authentication_code else '' if not debug: log = logging.getLogger('werkzeug') log.setLevel(logging.ERROR) environ.modes.add(environ.modes.INTERACTIVE) APPLICATION.run(port=port, debug=debug, host=host) environ.modes.remove(environ.modes.INTERACTIVE)
def create_snapshot( project: Project, *args: typing.List[str], show: bool = True ): """ :param project: :param show: :return: """ if len(args) < 1: snapshot_name = datetime.now().strftime('%Y%b%d-%H-%M-%S') else: snapshot_name = args[0] snapshot_directory = project.snapshot_path() if not os.path.exists(snapshot_directory): os.makedirs(snapshot_directory) snapshot_name = snapshot_name.replace(' ', '-') snapshot_directory = project.snapshot_path(snapshot_name) environ.systems.remove(snapshot_directory) shutil.copytree(project.output_directory, snapshot_directory) url = project.snapshot_url(snapshot_name) environ.log_header('Snapshot URL', 5) environ.log( '* {}'.format(url), whitespace_bottom=1, indent_by=2 ) if show: webbrowser.open(url)
def remove_snapshot( project: Project, *args: typing.List[str] ): """ :param project: :param args: :return: """ if len(args) < 1 or not args[0].strip(): environ.log( """ Are you sure you want to remove all snapshots in this project? """) if not query.confirm('Confirm Delete All', False): environ.log( '[ABORTED]: No snapshots were deleted', whitespace=1 ) return if not environ.systems.remove(project.snapshot_path()): environ.log( '[ERROR]: Failed to delete snapshots', whitespace=1 ) return environ.log( '[SUCCESS]: All snapshots have been removed', whitespace=1 ) return snapshot_name = args[0] environ.log( """ Are you sure you want to remove the snapshot "{}"? """.format(snapshot_name), whitespace=1 ) if not query.confirm('Confirm Deletion', False): environ.log( """ [ABORTED]: "{}" was not removed """.format(snapshot_name), whitespace=1 ) return if not environ.systems.remove(project.snapshot_path(snapshot_name)): environ.log( """ [ERROR]: Unable to delete snapshot "{}" at this time """.format(snapshot_name), whitespace=1 ) return environ.log( """ [SUCCESS]: Snapshot "{}" was removed """.format(snapshot_name), whitespace=1 ) return
def run_step( response: Response, project: Project, step: typing.Union[ProjectStep, str], force: bool = False ) -> bool: """ :param response: :param project: :param step: :param force: :return: """ step = get_step(project, step) if step is None: return False status = check_status(response, project, step, force) if status == ERROR_STATUS: return False step.error = None if status == SKIP_STATUS: return True os.chdir(os.path.dirname(step.source_path)) project.current_step = step step.report.clear() step.dom = None step.is_running = True step.progress_message = None step.progress = 0 # Set the top-level display and cache values to the current project values # before running the step for availability within the step scripts cauldron.shared = cauldron.project.shared redirection.enable(step) try: result = _execute_step(project, step) except Exception as error: result = dict( success=False, message='{}'.format(error), html_message='<pre>{}</pre>'.format(error) ) os.chdir(os.path.expanduser('~')) step.mark_dirty(not result['success']) step.error = result.get('html_message') step.last_modified = time.time() if result['success'] else 0.0 step.is_running = False step.progress = 0 step.progress_message = None step.dumps() # Make sure this is called prior to printing response information to the # console or that will come along for the ride redirection.disable(step) if result['success']: environ.log('[{}]: Updated'.format(step.definition.name)) else: response.fail( message='Step execution error', code='EXECUTION_ERROR', project=project.kernel_serialize(), step_name=step.definition.name ).console_raw(result['message']) return result['success']
def execute( context: cli.CommandContext, action: str, arguments: list, no_show: bool = False ) -> Response: """ :param context: :param action: :param arguments: :param no_show: :return: """ show = not no_show if not action: return context.response.fail( code='NO_ACTION_ARG', message='An action is required for the snapshot command' ).console( whitespace=1 ).response action = action.strip().lower() project = cauldron.project.internal_project if not project: return context.response.fail( code='NO_OPEN_PROJECT', message='No open project' ).console( '[ERROR]: No open project. Use the "open" command to open one.' ) if action == 'remove': return actions.remove_snapshot(project, *arguments) if action == 'add': return actions.create_snapshot(project, *arguments, show=show) if action == 'list': return actions.list_snapshots(project) if action == 'open': name = arguments[0] result = actions.open_snapshot(project, name) if result is None: environ.log('[ERROR]: No snapshot found named "{}"'.format(name)) return context.response environ.log_header('SNAPSHOT: {}'.format(name)) environ.log( """ URL: {url} LAST MODIFIED: {modified} """.format( url=result['url'], modified=datetime.fromtimestamp( result['last_modified'] ).strftime('%H:%M %b %d, %Y') ), whitespace=1 ) if show: webbrowser.open(result['url'])
def execute( context: cli.CommandContext, command: str = None, name: str = None, path: str = None, temporary: bool = False ): """ :return: """ response = context.response if name: name = name.replace(' ', '_').strip('"').strip() if path: path = environ.paths.clean(path.strip('"')) if not os.path.isdir(path): path = os.path.dirname(path) environ.log( """ [WARNING]: The specified path was not a directory. Aliases must be directories, so the directory containing the specified file will be used instead: {} """.format(path), whitespace=1 ) environ.configs.load() temporary_aliases = environ.configs.session.get('folder_aliases', {}) persistent_aliases = environ.configs.persistent.get('folder_aliases', {}) if not name and command in ['add', 'remove']: return response.fail( code='MISSING_ARG', message='You need to specify the name of the alias' ).console( whitespace=1 ).response if command == 'list': items = [] aliases = dict( list(temporary_aliases.items()) + list(persistent_aliases.items()) ) for k, v in aliases.items(): items.append('{}\n {}'.format(k, v['path'])) environ.log_header('EXISTING ALIASES') environ.log(items) return response aliases = temporary_aliases if temporary else persistent_aliases if command == 'add': aliases[name] = dict( path=path ) environ.configs.put( persists=not bool(temporary), folder_aliases=aliases ) return response.notify( kind='ADDED', code='ALIAS_ADDED', message='The alias "{}" has been saved'.format(name) ).console( whitespace=1 ).response if command == 'remove': if name in aliases: del aliases[name] environ.configs.put( persists=not bool(temporary), folder_aliases=aliases ) return response.notify( kind='REMOVED', code='ALIAS_REMOVED', message='The alias "{}" has been removed'.format(name) ).console( whitespace=1 ).response return response.fail( code='UNKNOWN_COMMAND', message='Unrecognized alias command "{}"'.format(command) ).console( whitespace=1 ).response
def run_step( response: Response, project: Project, step: typing.Union[ProjectStep, str], force: bool = False ) -> bool: """ :param response: :param project: :param step: :param force: :return: """ step = get_step(project, step) if step is None: return False status = check_status(response, project, step, force) if status == ERROR_STATUS: return False step.error = None if status == SKIP_STATUS: return True os.chdir(os.path.dirname(step.source_path)) project.current_step = step step.report.clear() step.dom = None step.is_visible = True step.is_running = True step.progress_message = None step.progress = 0 step.sub_progress_message = None step.sub_progress = 0 step.start_time = datetime.utcnow() step.end_time = None # Set the top-level display and cache values to the current project values # before running the step for availability within the step scripts cauldron.shared = cauldron.project.shared redirection.enable(step) try: result = _execute_step(project, step) except Exception as error: result = dict( success=False, message='{}'.format(error), html_message='<pre>{}</pre>'.format(error) ) step.end_time = datetime.utcnow() os.chdir(os.path.expanduser('~')) step.mark_dirty(not result['success']) step.error = result.get('html_message') step.last_modified = time.time() if result['success'] else 0.0 step.is_running = False step.progress = 0 step.progress_message = None step.dumps() # Make sure this is called prior to printing response information to the # console or that will come along for the ride redirection.disable(step) step.project.stop_condition = result.get( 'stop_condition', StopCondition(False, False) ) if result['success']: environ.log('[{}]: Updated in {}'.format( step.definition.name, step.get_elapsed_timestamp() )) else: response.fail( message='Step execution error', code='EXECUTION_ERROR', project=project.kernel_serialize(), step_name=step.definition.name ).console_raw(result['message']) return result['success']