コード例 #1
0
    def show_response(cls, response, callback=None):

        if callable(callback):
            response = callback(response)

        if response is not None:
            LOG.echo(response)
コード例 #2
0
    def new(self,
            name: str = None,
            model: str = None,
            path: str = None,
            files: dict = None,
            skip_upload=False,
            lightweight=False,
            **kwargs):

        model = Model.find_one(name=model)

        if lightweight:
            model.assert_datasets_can_be_lightweight()

        barrel = None
        ds = super().new(name=name,
                         model=model,
                         lightweight=lightweight,
                         **kwargs,
                         _duplicate_filter=dict(name=name, model=model))

        try:
            if not skip_upload:
                barrel = self._store(ds, path, files)
        except Exception as e:
            LOG.warn("Reverting creation of dataset '{}'".format(ds.name))
            ds.delete()
            if barrel is not None:
                barrel.purge(ignore=True)
            raise e
        else:
            return ds
コード例 #3
0
ファイル: shipyard.py プロジェクト: noronha-dataops/noronha
    def build(self, nocache: bool = False):

        work_path = None

        try:
            LOG.info("Building {} from {}".format(self.img_spec.target,
                                                  self.repo.address))
            work_path = self.make_work_path()
            build_path = self.deploy_source(work_path)

            logs = self.docker.build(path=build_path,
                                     tag=self.img_spec.target,
                                     nocache=nocache,
                                     rm=True)

            self.print_logs(logs)
            self.image = self.docker.images(self.img_spec.target)[0]
        except Exception as e:
            raise NhaDockerError("Failed to build from {}".format(
                self.repo)) from e
        else:
            self.tag_image()
            self.push_image()
            return self.image_id
        finally:
            if work_path is not None:
                work_path.dispose()
コード例 #4
0
ファイル: movers.py プロジェクト: Telres/noronha
    def new(self,
            name: str = None,
            model: str = None,
            train: str = None,
            ds: str = None,
            path: str = None,
            pretrained: str = None,
            skip_upload=False,
            lightweight=False,
            **kwargs):

        if path is None:
            raise NhaAPIError(
                "Cannot publish model version if path to model files is not provided"
            )

        model = Model.find_one(name=model)

        if lightweight:
            model.assert_movers_can_be_lightweight()

        if ds is not None:
            kwargs['ds'] = Dataset.find_one(name=ds, model=model).to_embedded()

        if train is not None:
            if self.proj is None:
                raise NhaAPIError(
                    "Cannot determine parent training if no working project is set"
                )
            else:
                kwargs['train'] = Training.find_one(
                    name=train, proj=self.proj.name).to_embedded()

        if pretrained is not None:
            kwargs['pretrained'] = ModelVersion.find_by_pk(
                pretrained).to_embedded()
            LOG.info(
                "Model version used pre-trained model '{}'".format(pretrained))

        mv: ModelVersion = super().new(name=name,
                                       model=model,
                                       lightweight=lightweight,
                                       **kwargs,
                                       _duplicate_filter=dict(name=name,
                                                              model=model))

        barrel = None

        try:
            if not skip_upload:
                barrel = self._store(mv, path)
        except Exception as e:
            LOG.warn("Reverting creation of model version '{}'".format(
                mv.name))
            mv.delete()
            if barrel is not None:
                barrel.purge(ignore=True)
            raise e

        return mv
コード例 #5
0
    def echo_outputs(self, cell):

        LOG.echo('-' * self.LINE_WIDTH)

        if not hasattr(cell, 'outputs') or len(cell.outputs) == 0:
            return

        [self._print_cell_output(out) for out in cell.outputs]
        LOG.echo('-' * self.LINE_WIDTH)
コード例 #6
0
ファイル: shipyard.py プロジェクト: noronha-dataops/noronha
    def untag(self):

        try:
            DockerCompass().get_api().remove_image(self.target)
        except Exception as e:
            LOG.error(e)
            return False
        else:
            return True
コード例 #7
0
ファイル: shipyard.py プロジェクト: noronha-dataops/noronha
    def build(self, nocache: bool = False):

        source = '{}:{}'.format(self.repo.address, self.img_spec.tag)
        LOG.info("Moving pre-built image from {} to {}".format(
            source, self.img_spec.target))
        self.docker.pull(self.repo.address, tag=self.img_spec.tag)
        self.image = self.docker.images(source)[0]
        self.tag_image()
        self.push_image()
        return self.image_id
コード例 #8
0
    def _print_cell_output(self, out):

        dyct = out.dict()

        if dyct.get('output_type') == 'error':
            ename, evalue = dyct.get('ename', ''), dyct.get('evalue', '')
            LOG.error('{}: {}'.format(ename, evalue))
        else:
            text = dyct.get('text', '')
            LOG.echo(text.strip())
コード例 #9
0
ファイル: movers.py プロジェクト: Telres/noronha
    def _store(self, mv: ModelVersion, path: str = None):

        barrel = MoversBarrel(mv)

        if barrel.schema is None:
            LOG.warn(
                "Publishing model version '{}' without a strict file definition"
                .format(mv.get_pk()))

        barrel.store_from_path(path)
        return barrel
コード例 #10
0
ファイル: model.py プロジェクト: noronha-dataops/noronha
    def new(self, **kwargs):

        if not kwargs.get('model_files'):
            LOG.warn(
                "Creating model without a strict model persistence definition")

        if not kwargs.get('data_files'):
            LOG.warn(
                "Creating model without a strict dataset files definition")

        return super().new(**kwargs)
コード例 #11
0
    def show_exception(cls, exception, callback=None):

        if isinstance(exception, PrettyError):
            exc = exception.pretty()
        else:
            exc = PrettyError.parse_exc(exception)

        if callable(callback):
            detail = callback(exception)
            LOG.info(detail)

        LOG.error(exc)
コード例 #12
0
ファイル: shipyard.py プロジェクト: noronha-dataops/noronha
    def push_image(self):

        if self.img_spec.pushable:
            LOG.info("Pushing {}".format(self.img_spec.target))
            log = self.docker.push(self.img_spec.repo, tag=self.img_spec.tag)
            outcome = json.loads(Regex.LINE_BREAK.split(log.strip())[-1])

            if 'error' in outcome:
                raise NhaDockerError(
                    "Failed to push image '{}'. Error: {}".format(
                        self.img_spec.target, outcome.get('errorDetail')))
        else:
            LOG.warn("Docker registry is not configured. Skipping image push.")
コード例 #13
0
    def _save_output(self, note_path, output_path):

        try:
            LOG.info("Saving output notebook: {}".format(output_path))
            # TODO: convert to pdf (find a light-weight lib for that)
            NotebookBarrel(proj=self.proj,
                           notebook=note_path,
                           file_name=output_path).store_from_path(os.getcwd())
        except Exception as e:
            err = NhaStorageError(
                "Failed to save output notebook '{}'".format(output_path))
            e.__cause__ = e
            LOG.error(err)
コード例 #14
0
ファイル: shipyard.py プロジェクト: noronha-dataops/noronha
    def print_logs(self, logs):

        for line in logs:
            dyct = assert_dict(line)

            if 'stream' in dyct:
                message = dyct['stream'].strip()

                if message:
                    LOG.debug(message)

            if 'error' in dyct:
                raise NhaDockerError(dyct['error'].strip())
コード例 #15
0
def version():
    """Framework's version"""

    LOG.echo("Noronha Dataops v%s" % FrameworkConst.FW_VERSION)
    pkg = pkg_resources.require(FrameworkConst.FW_NAME)[0]

    try:
        meta = pkg.get_metadata_lines('METADATA')
    except FileNotFoundError:
        meta = pkg.get_metadata_lines('PKG-INFO')

    for line in meta:
        if not line.startswith('Requires'):
            LOG.info(line)
コード例 #16
0
ファイル: utils.py プロジェクト: noronha-dataops/noronha
    def __call__(self,
                 ref_to_proj: str = None,
                 resolvers: list = (),
                 ignore: bool = False):

        proj = None

        for res in resolvers or self.ALL:
            assert res in self.ALL
            method = getattr(self, 'resolve_{}'.format(res))
            proj = method(ref_to_proj)

            if proj is not None:
                LOG.info("Working project is '{}'".format(proj.name))
                LOG.debug("Project resolution method was '{}'".format(res))
                break
        else:
            message = """Could not determine working project from reference '{}'""".format(
                ref_to_proj)
            details = """Resolvers used: {}""".format(resolvers)

            if ignore:
                LOG.info(message)
                LOG.debug(details)
            else:
                raise ResolutionError(message, details)

        return proj
コード例 #17
0
ファイル: publish.py プロジェクト: noronha-dataops/noronha
    def __call__(self,
                 src_path: str = Paths.TMP,
                 details: dict = None,
                 version_name: str = None,
                 model_name: str = None,
                 uses_dataset: bool = True,
                 dataset_name: str = None,
                 uses_pretrained: bool = False,
                 pretrained_with: str = None,
                 lightweight: bool = False):

        version_name = version_name or self.train.name
        model_name = self._infer_parent_model(model_name)
        ds = self._infer_dataset(model_name, uses_dataset, dataset_name)
        mv = None
        err = None

        try:
            mv = self.mv_api.new(name=version_name,
                                 model=model_name,
                                 ds=ds.name if ds else None,
                                 train=self.train.name,
                                 path=src_path,
                                 details=details or {},
                                 pretrained=self._infer_pretrained(
                                     uses_pretrained, pretrained_with),
                                 lightweight=lightweight,
                                 _replace=True)
        except Exception as e:
            LOG.warn("Model version {}:{} publish failed".format(
                model_name, version_name))
            err = e

        if self.train.name:
            self.train.update(mover=mv, ds=ds)

        if err:
            raise err

        if get_purpose() == DockerConst.Section.IDE:
            LOG.info(
                "For testing purposes, model files will be moved to the deployed model path"
            )
            MetaCargo(docs=[mv], section=DockerConst.Section.IDE).deploy()
            MoversCargo(mv, local=True,
                        section=DockerConst.Section.IDE).move(src_path)

        return mv
コード例 #18
0
    def clean(self):

        if not isinstance(self, EmbeddedDocument):
            self.modified = datetime.now()

        if hasattr(self, 'name') and self.name is None:
            name = random_name.generate_name(separator='-')
            setattr(self, 'name', name)
            LOG.warn("{} is anonymous. Using random name: {}".format(
                self.__class__.__name__, name))

        for key, val in self._fields.items():
            if isinstance(val, ReferenceField):
                getattr(self, key)  # assure that referenced document exists

        self._id = self.get_pk()
コード例 #19
0
ファイル: model.py プロジェクト: noronha-dataops/noronha
    def rm(self, name):

        LOG.warn(
            "All datasets and model versions for the model '{}' will be deleted"
            .format(name))
        self._decide("Would you like to proceed?",
                     interrupt=True,
                     default=True)
        report = {'Removed Datasets': [], 'Removed ModelVersions': []}

        for key, api in zip(report.keys(), [DatasetAPI(), ModelVersionAPI()]):
            for obj in api.lyst(model=name):
                api.rm(model=name, name=obj.name)
                report[key].append(obj.name)

        report.update(super().rm(name=name))
        return report
コード例 #20
0
    def __call__(self, objs: List[SmartBaseDoc]):

        if len(objs) == 0:
            LOG.echo("No {}(s) found".format(self.obj_title))
        else:
            LOG.echo("Listing {}(s):".format(self.obj_title))

            for obj in objs:
                if self.expand:
                    LOG.echo(obj.pretty())
                else:
                    LOG.echo(obj.show())
コード例 #21
0
ファイル: note.py プロジェクト: noronha-dataops/noronha
    def __call__(self,
                 tag: str = DockerConst.LATEST,
                 port: int = NoteConst.HOST_PORT,
                 movers: list = None,
                 datasets: list = None,
                 **kwargs):

        LOG.info("Notebook IDE will be mapped to port {}".format(port))
        return NotebookExp(
            port=port,
            proj=self.proj,
            tag=tag,
            movers=[
                ModelVersion.find_by_pk(mv).to_embedded()
                for mv in movers or []
            ],
            datasets=[Dataset.find_by_pk(ds) for ds in datasets or []],
            resource_profile=kwargs.pop('resource_profile',
                                        None)).launch(**kwargs)
コード例 #22
0
ファイル: model.py プロジェクト: noronha-dataops/noronha
    def update(self, name, **kwargs):

        if kwargs.get('model_files'):
            LOG.warn(
                "If 'model_files' definition is changed, old model versions may become unusable"
            )
            self._decide("Do you want to proceed?",
                         default=True,
                         interrupt=True)

        if kwargs.get('data_files'):
            LOG.warn(
                "If 'data_files' definition is changed, old datasets may become unusable"
            )
            self._decide("Do you want to proceed?",
                         default=True,
                         interrupt=True)

        return super().update(filter_kwargs=dict(name=name),
                              update_kwargs=kwargs)
コード例 #23
0
    def rm(self, name, model):

        ds = self.doc().find_one(name=name, model=model)
        # TODO: check if dataset is not being used in a training right now
        ds.delete()

        if ds.stored:
            LOG.info("Purging dataset '{}' from the file manager".format(
                ds.show()))
            file_status = 'purged' if DatasetBarrel(ds).purge(
                ignore=True) else 'not_found'
        else:
            LOG.info("Dataset '{}' is not stored. Skipping purge".format(
                ds.show()))
            file_status = 'not_stored'

        return dict(name=name,
                    model=model,
                    record='removed',
                    files=file_status)
コード例 #24
0
 def launch(self, tasks=1, skip_build=False, just_build=False, **_):
     
     if not just_build:
         assert self.scallable or tasks == 1, MisusageError(
             "Plugin '{}' is not scallable".format(self.alias)
         )
         
         assert self.isle_compass.native, MisusageError(
             "There is no point in setting up the plugin '{}' because it's configured in 'foreign mode'"
             .format(self.alias)
         )
     
     if not skip_build:
         self.builder.build()
     
     if not just_build:
         super().launch(tasks=tasks)
         
         if self.isle_compass.port is not None:
             LOG.info("Mapping service '{}' to port {}".format(self.make_name(), self.isle_compass.port))
コード例 #25
0
    def new(self,
            repo=None,
            models: list = None,
            home_dir: str = None,
            **kwargs):

        if models:
            finder = Model().find_one
            models = [finder(name=model_name) for model_name in models]
        else:
            models = []
            LOG.warn(
                "No models specified for the new project. "
                "When publishing a model version this project must specify its model name."
            )

        if home_dir is None:
            LOG.warn("No home directory was specified for the new project.")
            if self._decide(
                    "Would you like to use the current working directory?",
                    default=False):
                home_dir = os.getcwd()
                LOG.info("Using as home directory: {}".format(home_dir))

        return super().new(home_dir=None if home_dir is None else
                           LocalRepository(home_dir).address,
                           models=models,
                           **kwargs)
コード例 #26
0
    def _predict_route(self):

        out, err, code = {}, None, None
        kwargs = self.make_request_kwargs()

        try:
            out = self.make_result(**kwargs)
            code = OnlineConst.ReturnCode.OK
        except Exception as e:
            if isinstance(e, NhaDataError):
                err = e.pretty()
                code = OnlineConst.ReturnCode.BAD_REQUEST
            elif isinstance(e, (PrettyError, ServingError)):
                err = e.pretty()
                code = OnlineConst.ReturnCode.SERVER_ERROR
                self._health = False
            else:
                err = repr(e)
                code = OnlineConst.ReturnCode.NOT_IMPLEMENTED

            LOG.error(err)
        finally:
            if self._enrich:
                response = self._cleaner({
                    'result': out,
                    'err': err,
                    'metadata': self.make_metadata(**kwargs)
                })
            else:
                response = out or err

            if isinstance(response, (dict, list)):
                response = assert_json(response, encode=True, encoding=OnlineConst.DEFAULT_CHARSET)
            else:
                response = assert_str(response)

            return self.application.make_response(code, response)
コード例 #27
0
    def _store(self, ds: Dataset, path: str = None, files: dict = None):

        if path or files:  # either is not None
            barrel = DatasetBarrel(ds)

            if barrel.schema is None:
                LOG.warn(
                    "Publishing dataset '{}' without a strict file definition".
                    format(ds.get_pk()))

            if path:
                barrel.store_from_path(path)
            elif files:
                barrel.store_from_dict(files)
            else:
                raise NotImplementedError()

            return barrel
        else:
            LOG.warn(
                "Dataset '{}' for model '{}' is not being stored by the framework"
                .format(ds.name, ds.model.name))
            ds.update(stored=False)
            return None
コード例 #28
0
    def _run(self, **kwargs):

        try:
            LOG.debug("Notebook parameters:")
            LOG.debug(kwargs.get('parameters', {}))
            self.proc_mon.set_state(Task.State.RUNNING)
            pm.execute_notebook(**kwargs)
        except Exception as e:
            self._handle_exc(e)
            return False
        else:
            LOG.info("Notebook execution succeeded!")
            self.proc_mon.set_state(Task.State.FINISHED)
            return True
コード例 #29
0
    def cell_start(self, cell, *args, **kwargs):

        self.index += 1
        LOG.echo(self.format_source(cell))
        super().cell_start(cell, *args, **kwargs)
コード例 #30
0
    def _print_exc(self, e: Exception):

        LOG.error("Notebook execution failed:")
        LOG.error(e)
        self.proc_mon.set_state(Task.State.FAILED)