def _resources(self):
resources_requirements = self._filter_requirements(['pip'],
include=False)
output = {(name, self.requirements[name]['to'])
for name in resources_requirements}
PESTO_LOG.info('resources requirements : {}'.format(output))
return output
def test(build_config_path, profiles, nvidia=False, network=None):
build_config = BuildConfig.from_path(path=build_config_path, profiles=profiles, network=network)
PESTO_LOG.info('build configuration : {}'.format(build_config))
pesto_path = Path(build_config.algorithm_path) / 'pesto' / 'tests' / 'resources'
TestRunner(docker_image_name=build_config.docker_image_name, network=network, nvidia=nvidia).run_all(pesto_path)
def _pip_requirements(self):
pip_requirements = self._filter_requirements(['pip'])
output = [
os.path.basename(self.requirements[_]['from'])
for _ in pip_requirements
]
PESTO_LOG.info('pip requirements : {}'.format(output))
return output
def copy_pesto_whl(self) -> None:
PESTO_LOG.info('********** copy local pesto wheel if any **********')
source_dir = os.path.join(Path.home(), ".pesto/dist")
dest_dir = os.path.join(self.workspace, 'dist/')
os.makedirs(dest_dir, exist_ok=True)
for filename in glob.glob(os.path.join(source_dir, '*.*')):
PESTO_LOG.info('********** copy {} in {} **********'.format(
filename, dest_dir))
shutil.copy(filename, dest_dir)
def conf_validation(self) -> None:
PESTO_LOG.info('********** validate requirements.json **********')
requirements_schema = load_json(Builder.SCHEMA_PATH,
'requirements_schema.json')
validate_json(self.configs[PestoFiles.requirements],
requirements_schema)
PESTO_LOG.info('********** validate config.json **********')
validate_json(self.configs[PestoFiles.config],
self.configs[PestoFiles.config_schema])
def copy_requirement(from_uri: str, target_path: str) -> None:
mkdir(target_path)
if from_uri.startswith("gs://"):
PESTO_LOG.info('copy to : {}'.format(target_path))
subprocess.call(
shlex.split('gsutil cp {0} {1}'.format(
from_uri.rstrip(os.path.sep), target_path)))
elif from_uri.startswith("file://") and os.path.exists(
from_uri.replace("file://", "")):
shutil.copyfile(from_uri.replace("file://", ""), target_path)
else:
wget.download(url=from_uri, out=target_path)
def copy_factory_files(self) -> None:
PESTO_LOG.info('********** copy factory files **********')
if os.path.exists(self.workspace):
shutil.rmtree(self.workspace)
os.makedirs(self.workspace, exist_ok=True)
PESTO_LOG.info('workspace created : {}'.format(self.workspace))
# copy pesto required resources (api_geo_process_v1.0.yaml)
PESTO_LOG.debug('COPY pesto resources to workspace from: {}'.format(
PROCESSING_FACTORY_PATH))
# shutil.copytree(os.path.join(PROCESSING_FACTORY_PATH, 'pesto/cli'), os.path.join(self.workspace, 'pesto/cli'))
os.makedirs(os.path.join(self.workspace, "pesto"), exist_ok=True)
shutil.copyfile(
os.path.join(PROCESSING_FACTORY_PATH,
'pesto/cli/resources/doc/api_geo_process_v1.0.yaml'),
os.path.join(self.workspace, 'pesto/api_geo_process_v1.0.yaml'))
# copy algorithm
target_path = os.path.join(self.workspace, self.build_config.name)
PESTO_LOG.debug('COPY algorithm to workspace from: {} to: {}'.format(
self.build_config.algorithm_path, target_path))
shutil.copytree(self.build_config.algorithm_path, target_path)
# copy/update config files
shutil.rmtree(os.path.join(target_path, 'pesto', 'api'))
shutil.rmtree(os.path.join(target_path, 'pesto', 'build'))
os.makedirs(os.path.join(target_path, 'pesto', 'api'))
os.makedirs(os.path.join(target_path, 'pesto', 'build'))
for item in self.configs:
with open(os.path.join(target_path, 'pesto', item.value),
'w') as _:
json.dump(self.configs[item], _, indent=4, sort_keys=True)
def __init__(self, build_config: BuildConfig):
PESTO_LOG.info('***** init packaging *****')
self.build_config = build_config
self.configs = ConfigLoader.load(build_config)
PESTO_LOG.info('********** build parameters **********')
PESTO_LOG.info('{}'.format(json.dumps(build_config.__dict__,
indent=4)))
PESTO_LOG.info(
'processing factory path : {}'.format(PROCESSING_FACTORY_PATH))
def init(target, template):
cmd = "cookiecutter {} --output-dir {}".format(template, target)
PESTO_LOG.info(cmd)
PESTO_LOG.info(
"\nPlease fill necessary information to initialize your template\n")
res = cookiecutter(template, output_dir=target)
PESTO_LOG.info("Service generated at {}".format(res))
def search_build_config(args: Any) -> str:
build_config_path = str(args.build_config)
PESTO_LOG.debug('search build config ...')
if ':' in build_config_path:
PESTO_LOG.info('search {} in PESTO build repository : {}'.format(build_config_path, PESTO_WORKSPACE))
name, version = build_config_path.split(':')
build_config_path = os.path.join(PESTO_WORKSPACE, name, version, name)
if not build_config_path.endswith('.json'):
PESTO_LOG.warning('build parameter {} is not a json ...'.format(build_config_path))
build_config_path = os.path.join(build_config_path, 'pesto', 'build', 'build.json')
PESTO_LOG.warning('search for build.json in {}'.format(build_config_path))
if not os.path.exists(build_config_path):
raise ValueError('build.json not found at {}'.format(build_config_path))
return build_config_path
def push(build_config_path, registry_path, profiles: List[str]):
PESTO_LOG.info('Push docker image to registry : {}'.format(registry_path))
build_config = BuildConfig.from_path(path=build_config_path,
profiles=profiles)
PESTO_LOG.info('build configuration : {}'.format(build_config))
algo_name = build_config.name
algo_version = build_config.version
local_image_name = '{}:{}'.format(algo_name, algo_version)
remote_image_name = '{}/{}'.format(registry_path, local_image_name)
print(local_image_name)
print(remote_image_name)
PESTO_LOG.info('tag image {} as {}'.format(local_image_name,
remote_image_name))
subprocess.call(
shlex.split('docker tag {} {}'.format(local_image_name,
remote_image_name)))
PESTO_LOG.info('push to gcloud : {}'.format(remote_image_name))
subprocess.call(
shlex.split('gcloud docker -- push {}'.format(remote_image_name)))
def copy_requirements(self) -> None:
PESTO_LOG.info('********** copy requirements **********')
requirements = self.configs[PestoFiles.requirements]['requirements']
for name in requirements.keys():
from_uri = requirements[name]['from']
temporary_path = os.path.join(self.workspace, 'requirements',
os.path.basename(from_uri))
target_path = os.path.join(self.workspace, name)
PESTO_LOG.info('COPY from {} to {}'.format(from_uri,
temporary_path))
copy_requirement(from_uri, temporary_path)
if from_uri.endswith('tar.gz'):
PESTO_LOG.info('EXTRACT from {} to {}'.format(
temporary_path, target_path))
with tarfile.open(temporary_path, 'r:gz') as file:
file.extractall(path=target_path)
def display_banner() -> None:
processing_factory_banner = resource_string('pesto.cli.resources', 'banner.txt') \
.decode('utf-8') \
.replace(' ---------------', ': {:10s}----'.format(PESTO_VERSION))
PESTO_LOG.info(processing_factory_banner)
def build_docker_image(self) -> None:
PESTO_LOG.info('********** build docker image **********')
DockerBuilder(self.configs[PestoFiles.requirements],
self.build_config).build(self.workspace)
