def load(self):
if self.path is not None and os.path.exists(self.path):
with open(self.path, 'r', encoding='utf8') as f:
logger.debug('Loading metadata from %s', self.path)
self.data = pyjackson.load(f, _LocalContainer)
else:
self.data = _LocalContainer()
def _build_image(self, context_dir,
env: DockerEnv) -> docker.models.images.Image:
tag = self.params.uri
logger.debug('Building docker image %s from %s...', tag, context_dir)
with env.daemon.client() as client:
self.params.registry.login(client)
if not self.force_overwrite:
if self.params.exists(client):
raise ValueError(
f'Image {tag} already exists at {self.params.registry}. '
f'Change name or set force_overwrite=True.')
else:
self.params.delete(client) # to avoid spawning dangling images
try:
image, logs = client.images.build(path=context_dir,
tag=tag,
rm=True)
logger.info('Built docker image %s', tag)
_print_docker_logs(logs)
self.params.registry.push(client, tag)
return image
except errors.BuildError as e:
_print_docker_logs(e.build_log, logging.ERROR)
raise
def _build_image(self, context_dir):
tag = '{}:{}'.format(self.name, self.tag)
logger.debug('Building docker image %s from %s...', tag, context_dir)
with create_docker_client() as client:
if not self.force_overwrite:
try:
client.images.get(tag)
raise ValueError(
f'Image {tag} already exists. Change name or set force_overwrite=True.'
)
except errors.ImageNotFound:
pass
else:
try:
client.images.remove(
tag) # to avoid spawning dangling images
except errors.ImageNotFound:
pass
try:
_, logs = client.images.build(path=context_dir, tag=tag)
logger.info('Build successful')
_print_docker_logs(logs)
except errors.BuildError as e:
_print_docker_logs(e.build_log)
raise
def run(self,
service_instance: DockerServiceInstance,
rm=True,
detach=True):
if not isinstance(service_instance, DockerServiceInstance):
raise TypeError(
'ServiceInstance should be of type DockerServiceInstance instead of {}'
.format(type(service_instance)))
with create_docker_client(
service_instance.target_host.get_host()) as client:
if isinstance(service_instance.docker_image.registry,
RemoteDockerRegistry):
client.login(
registry=service_instance.docker_image.registry.host,
username=service_instance.docker_image.registry.username,
password=service_instance.docker_image.registry.password)
import docker.errors # FIXME
try:
# always detach from container and just stream logs if detach=False
container = client.containers.run(
service_instance.docker_image.get_image_uri(),
name=service_instance.name,
auto_remove=rm,
ports=service_instance.ports_mapping,
detach=True)
if not detach:
try:
# infinite loop of logs while container running or if everything ok
for log in self.logs(container, stream=True):
logger.debug(log)
if not self._is_service_running(
client, service_instance):
raise DockerRunnerException(
"The container died unexpectedly.")
except KeyboardInterrupt:
logger.info('Interrupted. Stopping the container')
container.stop()
else:
if not self._is_service_running(client, service_instance):
if not rm:
for log in self.logs(container,
stdout=False,
stderr=True):
raise DockerRunnerException(
"The container died unexpectedly.", log)
else:
# Can't get logs from removed container
raise DockerRunnerException(
"The container died unexpectedly. Try to run the container "
"with detach=False or rm=False args to get more info."
)
return container
except docker.errors.ContainerError as e:
if e.stderr:
print(e.stderr.decode(), file=sys.stderr)
raise
def _delete_object(self, object_type: Type[Attaching], obj, error_type):
with self._session() as s:
p = s.query(object_type).filter(object_type.id == obj.id).first()
if p is None:
raise error_type(obj)
logger.debug('Deleting object %s', p)
s.delete(p)
def run(self,
instance: DockerContainer,
image: DockerImage,
env: DockerEnv,
rm=True,
detach=True,
**kwargs):
if not (isinstance(instance, DockerContainer) and isinstance(
image, DockerImage) and isinstance(env, DockerEnv)):
raise TypeError(
'DockerRunner works with DockerContainer, DockerImage and DockerHost only'
)
with env.daemon.client() as client:
image.registry.login(client)
try:
# always detach from container and just stream logs if detach=False
container = client.containers.run(image.uri,
name=instance.name,
auto_remove=rm,
ports=instance.port_mapping,
detach=True,
**instance.params,
**kwargs)
instance.container_id = container.id
if not detach:
try:
# infinite loop of logs while container running or if everything ok
for log in self._logs(container, stream=True):
logger.debug(log)
self._sleep()
if not self._is_service_running(client, instance.name):
raise DockerRunnerException(
"The container died unexpectedly.")
except KeyboardInterrupt:
logger.info('Interrupted. Stopping the container')
container.stop()
else:
self._sleep(.5)
if not self._is_service_running(client, instance.name):
if not rm:
for log in self._logs(container,
stdout=False,
stderr=True):
raise DockerRunnerException(
"The container died unexpectedly.", log)
else:
# Can't get logs from removed container
raise DockerRunnerException(
"The container died unexpectedly. Try to run the container "
"with detach=False or rm=False args to get more info."
)
except docker.errors.ContainerError as e:
if e.stderr:
print(e.stderr.decode(), file=sys.stderr)
raise
def _s3_res(self):
logger.debug('Creating s3 resource with endpoint %s', self.endpoint)
return boto3.resource('s3',
endpoint_url=self.endpoint,
aws_access_key_id=S3Config.ACCESS_KEY,
aws_secret_access_key=S3Config.SECRET_KEY,
region_name=self.region)
def _get_sql_object_by_id(self, object_type: Type[Attaching], id: str):
with self._session() as s:
logger.debug('Getting %s[%s]', object_type.__name__, id)
obj = s.query(object_type).filter(object_type.id == id).first()
if obj is None:
return
return obj
def delete_artifact(self, artifact_type, artifact_id: str):
artifact_id = f'{artifact_type}/{artifact_id}'
path = os.path.join(self.path, artifact_id)
if not os.path.exists(path):
raise NoSuchArtifactError(artifact_id, self)
logger.debug('Removing artifact %s', path)
shutil.rmtree(path)
def _write_distribution(self, target_dir):
super()._write_distribution(target_dir)
logger.debug('Putting Dockerfile to distribution...')
with open(os.path.join(target_dir, 'Dockerfile'), 'w') as df:
dockerfile = self.dockerfile_gen.generate()
df.write(dockerfile)
def __call__(self, *args, **kwargs):
if args and kwargs:
raise ValueError(
'Parameters should be passed either in positional or in keyword fashion, not both'
)
if len(args) > len(self.method.args) or len(kwargs) > len(
self.method.args):
raise ValueError(
f'Too much parameters given, expected: {len(self.method.args)}'
)
data = {}
for i, arg in enumerate(self.method.args):
obj = None
if len(args) > i:
obj = args[i]
if arg.name in kwargs:
obj = kwargs[arg.name]
if obj is None:
raise ValueError(
f'Parameter with name "{arg.name}" (position {i}) should be passed'
)
data[arg.name] = serialize(obj, arg.type)
logger.debug('Calling server method "%s", args: %s ...',
self.method.name, data)
out = self.call_method(self.method.name, data)
logger.debug('Server call returned %s', out)
return deserialize(out, self.method.out_type)
def _write_binaries(self, path):
"""
Writes binaries to dir
:param path: target directory to write binaries
"""
logger.debug('Putting model artifacts to distribution...')
a = self.provider.get_artifacts()
a.materialize(path)
def _bootstrap_method(method: InterfaceMethodDescriptor):
logger.debug('Bootstraping server method "%s" with %s argument(s)...',
method.name, len(method.args))
args = []
for arg_name, arg_type in method.args.items():
args.append(_Argument(arg_name, arg_type))
return _Method(method.name, args, method.out_type)
def _delete_artifact(self, model_id: str):
if not self._bucket_exists():
raise NoSuchArtifactError(model_id, self)
keys = list(self._list_blobs(model_id).keys())
if len(keys) == 0:
raise NoSuchArtifactError(model_id, self)
else:
logger.debug('Deleting %s from %s/%s', model_id, self.endpoint, self.bucket_name)
self._s3.delete_objects(Bucket=self.bucket_name, Delete={'Objects': [{'Key': k} for k in keys]})
def _write_distribution(self, target_dir):
super()._write_distribution(target_dir)
logger.debug('Putting Dockerfile to distribution...')
env = self.provider.get_env()
logger.debug('Determined environment for running model: %s.', env)
with open(os.path.join(target_dir, 'Dockerfile'), 'w') as df:
dockerfile = self.dockerfile_gen.generate(env)
df.write(dockerfile)
def _write_distribution(self, target_dir):
"""
Writes full distribution to dir
:param target_dir: target directory to write distribution
"""
logger.debug('Writing model distribution to "%s"...', target_dir)
self._write_sources(target_dir)
self._write_binaries(target_dir)
self._write_requirements(target_dir)
self._write_run_script(target_dir)
def save(self, obj, save_persistent_id=True):
if id(obj) in self.seen:
return
self.seen.add(id(obj))
self._add_requirement(obj)
try:
return super(EbonitePickler, self).save(obj, save_persistent_id)
except (ValueError, TypeError, PickleError) as e:
# if object cannot be serialized, it's probably a C object and we don't need to go deeper
logger.debug('Skipping dependency analysis for %s because of %s: %s', obj, type(e).__name__, e)
def load(cls, extension: Union[str, Extension]):
"""
Load single extension
:param extension: str of :class:`Extension` instance to load
"""
if isinstance(extension, str):
extension = Extension(extension, [], force=True)
if extension not in cls._loaded_extensions and not module_imported(extension.module):
logger.debug('Importing extension module %s', extension.module)
cls._loaded_extensions[extension] = import_module(extension.module)
def _write_distribution(self, target_dir):
super()._write_distribution(target_dir)
logger.debug('Putting Dockerfile to distribution...')
env = self.provider.get_env()
logger.debug('Determined environment for running model: %s.', env)
with open(os.path.join(target_dir, 'Dockerfile'), 'w') as df:
unix_packages = self.provider.get_requirements().of_type(
UnixPackageRequirement)
dockerfile = self.dockerfile_gen.generate(env, unix_packages)
df.write(dockerfile)
def _delete_object(self, object_type: Type[Attaching], obj, ne_error_type,
ie_error_type):
with self._session() as s:
p = s.query(object_type).filter(object_type.id == obj.id).first()
if p is None:
raise ne_error_type(obj)
logger.debug('Deleting object %s', p)
try:
s.delete(p)
s.commit()
except IntegrityError:
raise ie_error_type(obj)
def _create_object(self, object_type: Type[Attaching], obj: T,
error_type) -> T:
with self._session(False) as s:
p = object_type.from_obj(obj, new=True)
s.add(p)
try:
logger.debug('Inserting object %s', p)
s.commit()
except IntegrityError:
raise error_type(obj)
obj._id = str(p.id)
return obj
def _get_objects(self,
object_type: Type[Attaching],
add_filter=None) -> List:
with self._session() as s:
if add_filter is None:
logger.debug('Getting %ss', object_type.__name__)
else:
logger.debug('Getting %ss with filter %s',
object_type.__name__, add_filter)
q = s.query(object_type)
if add_filter:
q = q.filter(add_filter)
return [o.to_obj() for o in q.all()]
def _session(self, commit=True) -> Session:
if self._active_session is None:
logger.debug('Creating session for %s', self.db_uri)
self._active_session = self._Session()
new_session = True
else:
new_session = False
yield self._active_session
if commit:
self._active_session.commit()
if new_session:
self._active_session.close()
self._active_session = None
def _push_artifact(self, model_id: str, blobs: typing.Dict[str, Blob]) -> ArtifactCollection:
path = os.path.join(self.path, model_id)
if os.path.exists(path):
raise ArtifactExistsError(model_id, self)
os.makedirs(path, exist_ok=True)
result = {}
for filepath, blob in blobs.items():
join = os.path.join(path, filepath)
os.makedirs(os.path.dirname(join), exist_ok=True)
logger.debug('Writing artifact %s to %s', blob, join)
blob.materialize(join)
result[filepath] = LocalFileBlob(join)
return Blobs(result)
def _delete_object(self, object_type: Type[Attaching], obj, ne_error_type,
ie_error_type):
with self._session() as s:
p = s.query(object_type).filter(object_type.id == obj.id).first()
if p is None:
raise ne_error_type(obj)
logger.debug('Deleting object %s', p)
try:
s.delete(p)
s.commit()
except IntegrityError:
s.rollback()
if p.to_obj().bind_meta_repo(self).has_children():
raise ie_error_type(obj)
else:
raise UnknownMetadataError
def _add_requirement(self, obj_or_module):
if not isinstance(obj_or_module, ModuleType):
try:
module = get_object_module(obj_or_module)
except AttributeError as e:
# Some internal Tensorflow 2.x object crashes `inspect` module on Python 3.6
logger.debug('Skipping dependency analysis for %s because of %s: %s', obj_or_module, type(e).__name__, e)
return
else:
module = obj_or_module
if module is not None and not self._should_ignore(module):
self._modules.add(module)
if is_local_module(module):
# add imports of this module
for local_req in get_local_module_reqs(module):
self._add_requirement(local_req)
def _get_object_by_name(self,
object_type: Type[Attaching],
name,
add_filter=None):
with self._session() as s:
if add_filter is None:
logger.debug('Getting %s with name %s', object_type.__name__,
name)
else:
logger.debug('Getting %s with name %s with filter %s',
object_type.__name__, name, add_filter)
q = s.query(object_type).filter(object_type.name == name)
if add_filter is not None:
q = q.filter(add_filter)
obj = q.first()
if obj is None:
return
return obj.to_obj()
def __init_subclass__(cls, **kwargs):
if hasattr(cls, '__init__'):
init = getattr(cls, '__init__')
argspec = inspect.getfullargspec(init)
if len(argspec.args) > 1:
raise ValueError('Hook type [{}] cannot have __init__ with arguments'.format(cls.__name__))
if not is_abstract_method(cls.process):
for b in reversed(cls.__bases__):
analyzer = getattr(b, ANALYZER_FIELD, None)
if analyzer is not None:
analyzer.hooks.append(cls())
logger.debug('Registering %s to %s', cls.__name__, analyzer.__name__)
break
else:
raise ValueError(
'{} defines process method, but dont have any parents with attached Analyzer'.format(cls))
super(Hook, cls).__init_subclass__(**kwargs)
def _push_artifact(self, model_id: str, blobs: typing.Dict[str, Blob]) -> ArtifactCollection:
self._ensure_bucket()
if len(self._list_blobs(model_id)) > 0:
raise ArtifactExistsError(model_id, self)
if len(blobs) == 0:
self._s3.upload_fileobj(io.BytesIO(b''), self.bucket_name, model_id)
return Blobs({})
result = {}
for filepath, blob in blobs.items():
join = os.path.join(model_id, filepath)
with blob.bytestream() as b:
logger.debug('Uploading %s to s3 %s/%s', blob, self.endpoint, self.bucket_name)
self._s3.upload_fileobj(b, self.bucket_name, join)
result[filepath] = S3Blob(join, self.bucket_name, self.endpoint)
return Blobs(result)
def _write_sources(self, target_dir):
"""
Writes sources to dir
:param target_dir: target directory to write sources
"""
for name, content in self.provider.get_sources().items():
logger.debug('Putting model source "%s" to distribution...', name)
path = os.path.join(target_dir, name)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, 'w', encoding='utf8') as src:
src.write(content)
if not ebonite_from_pip():
logger.debug(
'Putting Ebonite sources to distribution as local installation is employed...'
)
main_module_path = get_lib_path('.')
shutil.copytree(main_module_path,
os.path.join(target_dir, 'ebonite'))
