def load(self): if self.path is not None and os.path.exists(self.path): with open(self.path, 'r', encoding='utf8') as f: logger.debug('Loading metadata from %s', self.path) self.data = pyjackson.load(f, _LocalContainer) else: self.data = _LocalContainer()
def _build_image(self, context_dir, env: DockerEnv) -> docker.models.images.Image: tag = self.params.uri logger.debug('Building docker image %s from %s...', tag, context_dir) with env.daemon.client() as client: self.params.registry.login(client) if not self.force_overwrite: if self.params.exists(client): raise ValueError( f'Image {tag} already exists at {self.params.registry}. ' f'Change name or set force_overwrite=True.') else: self.params.delete(client) # to avoid spawning dangling images try: image, logs = client.images.build(path=context_dir, tag=tag, rm=True) logger.info('Built docker image %s', tag) _print_docker_logs(logs) self.params.registry.push(client, tag) return image except errors.BuildError as e: _print_docker_logs(e.build_log, logging.ERROR) raise
def _build_image(self, context_dir): tag = '{}:{}'.format(self.name, self.tag) logger.debug('Building docker image %s from %s...', tag, context_dir) with create_docker_client() as client: if not self.force_overwrite: try: client.images.get(tag) raise ValueError( f'Image {tag} already exists. Change name or set force_overwrite=True.' ) except errors.ImageNotFound: pass else: try: client.images.remove( tag) # to avoid spawning dangling images except errors.ImageNotFound: pass try: _, logs = client.images.build(path=context_dir, tag=tag) logger.info('Build successful') _print_docker_logs(logs) except errors.BuildError as e: _print_docker_logs(e.build_log) raise
def run(self, service_instance: DockerServiceInstance, rm=True, detach=True): if not isinstance(service_instance, DockerServiceInstance): raise TypeError( 'ServiceInstance should be of type DockerServiceInstance instead of {}' .format(type(service_instance))) with create_docker_client( service_instance.target_host.get_host()) as client: if isinstance(service_instance.docker_image.registry, RemoteDockerRegistry): client.login( registry=service_instance.docker_image.registry.host, username=service_instance.docker_image.registry.username, password=service_instance.docker_image.registry.password) import docker.errors # FIXME try: # always detach from container and just stream logs if detach=False container = client.containers.run( service_instance.docker_image.get_image_uri(), name=service_instance.name, auto_remove=rm, ports=service_instance.ports_mapping, detach=True) if not detach: try: # infinite loop of logs while container running or if everything ok for log in self.logs(container, stream=True): logger.debug(log) if not self._is_service_running( client, service_instance): raise DockerRunnerException( "The container died unexpectedly.") except KeyboardInterrupt: logger.info('Interrupted. Stopping the container') container.stop() else: if not self._is_service_running(client, service_instance): if not rm: for log in self.logs(container, stdout=False, stderr=True): raise DockerRunnerException( "The container died unexpectedly.", log) else: # Can't get logs from removed container raise DockerRunnerException( "The container died unexpectedly. Try to run the container " "with detach=False or rm=False args to get more info." ) return container except docker.errors.ContainerError as e: if e.stderr: print(e.stderr.decode(), file=sys.stderr) raise
def _delete_object(self, object_type: Type[Attaching], obj, error_type): with self._session() as s: p = s.query(object_type).filter(object_type.id == obj.id).first() if p is None: raise error_type(obj) logger.debug('Deleting object %s', p) s.delete(p)
def run(self, instance: DockerContainer, image: DockerImage, env: DockerEnv, rm=True, detach=True, **kwargs): if not (isinstance(instance, DockerContainer) and isinstance( image, DockerImage) and isinstance(env, DockerEnv)): raise TypeError( 'DockerRunner works with DockerContainer, DockerImage and DockerHost only' ) with env.daemon.client() as client: image.registry.login(client) try: # always detach from container and just stream logs if detach=False container = client.containers.run(image.uri, name=instance.name, auto_remove=rm, ports=instance.port_mapping, detach=True, **instance.params, **kwargs) instance.container_id = container.id if not detach: try: # infinite loop of logs while container running or if everything ok for log in self._logs(container, stream=True): logger.debug(log) self._sleep() if not self._is_service_running(client, instance.name): raise DockerRunnerException( "The container died unexpectedly.") except KeyboardInterrupt: logger.info('Interrupted. Stopping the container') container.stop() else: self._sleep(.5) if not self._is_service_running(client, instance.name): if not rm: for log in self._logs(container, stdout=False, stderr=True): raise DockerRunnerException( "The container died unexpectedly.", log) else: # Can't get logs from removed container raise DockerRunnerException( "The container died unexpectedly. Try to run the container " "with detach=False or rm=False args to get more info." ) except docker.errors.ContainerError as e: if e.stderr: print(e.stderr.decode(), file=sys.stderr) raise
def _s3_res(self): logger.debug('Creating s3 resource with endpoint %s', self.endpoint) return boto3.resource('s3', endpoint_url=self.endpoint, aws_access_key_id=S3Config.ACCESS_KEY, aws_secret_access_key=S3Config.SECRET_KEY, region_name=self.region)
def _get_sql_object_by_id(self, object_type: Type[Attaching], id: str): with self._session() as s: logger.debug('Getting %s[%s]', object_type.__name__, id) obj = s.query(object_type).filter(object_type.id == id).first() if obj is None: return return obj
def delete_artifact(self, artifact_type, artifact_id: str): artifact_id = f'{artifact_type}/{artifact_id}' path = os.path.join(self.path, artifact_id) if not os.path.exists(path): raise NoSuchArtifactError(artifact_id, self) logger.debug('Removing artifact %s', path) shutil.rmtree(path)
def _write_distribution(self, target_dir): super()._write_distribution(target_dir) logger.debug('Putting Dockerfile to distribution...') with open(os.path.join(target_dir, 'Dockerfile'), 'w') as df: dockerfile = self.dockerfile_gen.generate() df.write(dockerfile)
def __call__(self, *args, **kwargs): if args and kwargs: raise ValueError( 'Parameters should be passed either in positional or in keyword fashion, not both' ) if len(args) > len(self.method.args) or len(kwargs) > len( self.method.args): raise ValueError( f'Too much parameters given, expected: {len(self.method.args)}' ) data = {} for i, arg in enumerate(self.method.args): obj = None if len(args) > i: obj = args[i] if arg.name in kwargs: obj = kwargs[arg.name] if obj is None: raise ValueError( f'Parameter with name "{arg.name}" (position {i}) should be passed' ) data[arg.name] = serialize(obj, arg.type) logger.debug('Calling server method "%s", args: %s ...', self.method.name, data) out = self.call_method(self.method.name, data) logger.debug('Server call returned %s', out) return deserialize(out, self.method.out_type)
def _write_binaries(self, path): """ Writes binaries to dir :param path: target directory to write binaries """ logger.debug('Putting model artifacts to distribution...') a = self.provider.get_artifacts() a.materialize(path)
def _bootstrap_method(method: InterfaceMethodDescriptor): logger.debug('Bootstraping server method "%s" with %s argument(s)...', method.name, len(method.args)) args = [] for arg_name, arg_type in method.args.items(): args.append(_Argument(arg_name, arg_type)) return _Method(method.name, args, method.out_type)
def _delete_artifact(self, model_id: str): if not self._bucket_exists(): raise NoSuchArtifactError(model_id, self) keys = list(self._list_blobs(model_id).keys()) if len(keys) == 0: raise NoSuchArtifactError(model_id, self) else: logger.debug('Deleting %s from %s/%s', model_id, self.endpoint, self.bucket_name) self._s3.delete_objects(Bucket=self.bucket_name, Delete={'Objects': [{'Key': k} for k in keys]})
def _write_distribution(self, target_dir): super()._write_distribution(target_dir) logger.debug('Putting Dockerfile to distribution...') env = self.provider.get_env() logger.debug('Determined environment for running model: %s.', env) with open(os.path.join(target_dir, 'Dockerfile'), 'w') as df: dockerfile = self.dockerfile_gen.generate(env) df.write(dockerfile)
def _write_distribution(self, target_dir): """ Writes full distribution to dir :param target_dir: target directory to write distribution """ logger.debug('Writing model distribution to "%s"...', target_dir) self._write_sources(target_dir) self._write_binaries(target_dir) self._write_requirements(target_dir) self._write_run_script(target_dir)
def save(self, obj, save_persistent_id=True): if id(obj) in self.seen: return self.seen.add(id(obj)) self._add_requirement(obj) try: return super(EbonitePickler, self).save(obj, save_persistent_id) except (ValueError, TypeError, PickleError) as e: # if object cannot be serialized, it's probably a C object and we don't need to go deeper logger.debug('Skipping dependency analysis for %s because of %s: %s', obj, type(e).__name__, e)
def load(cls, extension: Union[str, Extension]): """ Load single extension :param extension: str of :class:`Extension` instance to load """ if isinstance(extension, str): extension = Extension(extension, [], force=True) if extension not in cls._loaded_extensions and not module_imported(extension.module): logger.debug('Importing extension module %s', extension.module) cls._loaded_extensions[extension] = import_module(extension.module)
def _write_distribution(self, target_dir): super()._write_distribution(target_dir) logger.debug('Putting Dockerfile to distribution...') env = self.provider.get_env() logger.debug('Determined environment for running model: %s.', env) with open(os.path.join(target_dir, 'Dockerfile'), 'w') as df: unix_packages = self.provider.get_requirements().of_type( UnixPackageRequirement) dockerfile = self.dockerfile_gen.generate(env, unix_packages) df.write(dockerfile)
def _delete_object(self, object_type: Type[Attaching], obj, ne_error_type, ie_error_type): with self._session() as s: p = s.query(object_type).filter(object_type.id == obj.id).first() if p is None: raise ne_error_type(obj) logger.debug('Deleting object %s', p) try: s.delete(p) s.commit() except IntegrityError: raise ie_error_type(obj)
def _create_object(self, object_type: Type[Attaching], obj: T, error_type) -> T: with self._session(False) as s: p = object_type.from_obj(obj, new=True) s.add(p) try: logger.debug('Inserting object %s', p) s.commit() except IntegrityError: raise error_type(obj) obj._id = str(p.id) return obj
def _get_objects(self, object_type: Type[Attaching], add_filter=None) -> List: with self._session() as s: if add_filter is None: logger.debug('Getting %ss', object_type.__name__) else: logger.debug('Getting %ss with filter %s', object_type.__name__, add_filter) q = s.query(object_type) if add_filter: q = q.filter(add_filter) return [o.to_obj() for o in q.all()]
def _session(self, commit=True) -> Session: if self._active_session is None: logger.debug('Creating session for %s', self.db_uri) self._active_session = self._Session() new_session = True else: new_session = False yield self._active_session if commit: self._active_session.commit() if new_session: self._active_session.close() self._active_session = None
def _push_artifact(self, model_id: str, blobs: typing.Dict[str, Blob]) -> ArtifactCollection: path = os.path.join(self.path, model_id) if os.path.exists(path): raise ArtifactExistsError(model_id, self) os.makedirs(path, exist_ok=True) result = {} for filepath, blob in blobs.items(): join = os.path.join(path, filepath) os.makedirs(os.path.dirname(join), exist_ok=True) logger.debug('Writing artifact %s to %s', blob, join) blob.materialize(join) result[filepath] = LocalFileBlob(join) return Blobs(result)
def _delete_object(self, object_type: Type[Attaching], obj, ne_error_type, ie_error_type): with self._session() as s: p = s.query(object_type).filter(object_type.id == obj.id).first() if p is None: raise ne_error_type(obj) logger.debug('Deleting object %s', p) try: s.delete(p) s.commit() except IntegrityError: s.rollback() if p.to_obj().bind_meta_repo(self).has_children(): raise ie_error_type(obj) else: raise UnknownMetadataError
def _add_requirement(self, obj_or_module): if not isinstance(obj_or_module, ModuleType): try: module = get_object_module(obj_or_module) except AttributeError as e: # Some internal Tensorflow 2.x object crashes `inspect` module on Python 3.6 logger.debug('Skipping dependency analysis for %s because of %s: %s', obj_or_module, type(e).__name__, e) return else: module = obj_or_module if module is not None and not self._should_ignore(module): self._modules.add(module) if is_local_module(module): # add imports of this module for local_req in get_local_module_reqs(module): self._add_requirement(local_req)
def _get_object_by_name(self, object_type: Type[Attaching], name, add_filter=None): with self._session() as s: if add_filter is None: logger.debug('Getting %s with name %s', object_type.__name__, name) else: logger.debug('Getting %s with name %s with filter %s', object_type.__name__, name, add_filter) q = s.query(object_type).filter(object_type.name == name) if add_filter is not None: q = q.filter(add_filter) obj = q.first() if obj is None: return return obj.to_obj()
def __init_subclass__(cls, **kwargs): if hasattr(cls, '__init__'): init = getattr(cls, '__init__') argspec = inspect.getfullargspec(init) if len(argspec.args) > 1: raise ValueError('Hook type [{}] cannot have __init__ with arguments'.format(cls.__name__)) if not is_abstract_method(cls.process): for b in reversed(cls.__bases__): analyzer = getattr(b, ANALYZER_FIELD, None) if analyzer is not None: analyzer.hooks.append(cls()) logger.debug('Registering %s to %s', cls.__name__, analyzer.__name__) break else: raise ValueError( '{} defines process method, but dont have any parents with attached Analyzer'.format(cls)) super(Hook, cls).__init_subclass__(**kwargs)
def _push_artifact(self, model_id: str, blobs: typing.Dict[str, Blob]) -> ArtifactCollection: self._ensure_bucket() if len(self._list_blobs(model_id)) > 0: raise ArtifactExistsError(model_id, self) if len(blobs) == 0: self._s3.upload_fileobj(io.BytesIO(b''), self.bucket_name, model_id) return Blobs({}) result = {} for filepath, blob in blobs.items(): join = os.path.join(model_id, filepath) with blob.bytestream() as b: logger.debug('Uploading %s to s3 %s/%s', blob, self.endpoint, self.bucket_name) self._s3.upload_fileobj(b, self.bucket_name, join) result[filepath] = S3Blob(join, self.bucket_name, self.endpoint) return Blobs(result)
def _write_sources(self, target_dir): """ Writes sources to dir :param target_dir: target directory to write sources """ for name, content in self.provider.get_sources().items(): logger.debug('Putting model source "%s" to distribution...', name) path = os.path.join(target_dir, name) os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, 'w', encoding='utf8') as src: src.write(content) if not ebonite_from_pip(): logger.debug( 'Putting Ebonite sources to distribution as local installation is employed...' ) main_module_path = get_lib_path('.') shutil.copytree(main_module_path, os.path.join(target_dir, 'ebonite'))