def spawn_crawler(self, container_name, container_image, id, server_url, server_password, metadata_namespace, envs): pool = spawning_pool.SpawningPool() project_name = container_name + '-' + id self._log.debug('spawn_crawler: ' + project_name) pool.setServerUrl(server_url) pool.setServerPassword(server_password) pool.setApiVersion('v1') pool.setKind('Pod') pool.setMetadataName(project_name) pool.setMetadataNamespace(metadata_namespace) pool.addMetadataLabel('name', project_name) pool.addMetadataLabel('group', container_name) pool.addMetadataLabel('SPAWN_ID', id) container = pool.createContainer() pool.setContainerName(container, project_name) for key, value in envs.items(): pool.addContainerEnv(container, key, value) pool.setContainerImage(container, container_image) pool.setContainerImagePullPolicy(container, 'Always') pool.addContainer(container) pool.setRestartPolicy('Never') pool.spawn()
def spawn_cropper(uuid): pool = spawning_pool.SpawningPool() project_name = 'bl-cropper-' + uuid print('spawn_cropper: ' + project_name) pool.setServerUrl(REDIS_SERVER) pool.setApiVersion('v1') pool.setKind('Pod') pool.setMetadataName(project_name) pool.setMetadataNamespace('index') pool.addMetadataLabel('name', project_name) pool.addMetadataLabel('SPAWN_ID', uuid) container = pool.createContainer() pool.setContainerName(container, project_name) pool.addContainerEnv(container, 'AWS_ACCESS_KEY', AWS_ACCESS_KEY) pool.addContainerEnv(container, 'AWS_SECRET_ACCESS_KEY', AWS_SECRET_ACCESS_KEY) pool.addContainerEnv(container, 'REDIS_SERVER', REDIS_SERVER) pool.addContainerEnv(container, 'SPAWN_ID', uuid) pool.setContainerImage(container, 'bluelens/bl-cropper:latest') pool.addContainer(container) pool.setRestartPolicy('Never') pool.spawn()
def spawn_crawler(host_code, version_id): pool = spawning_pool.SpawningPool() id = host_code.lower() project_name = 'bl-crawler-' + id log.debug('spawn_crawler: ' + project_name) pool.setServerUrl(REDIS_SERVER) pool.setServerPassword(REDIS_PASSWORD) pool.setApiVersion('v1') pool.setKind('Pod') pool.setMetadataName(project_name) pool.setMetadataNamespace(RELEASE_MODE) pool.addMetadataLabel('name', project_name) pool.addMetadataLabel('group', 'bl-crawler') pool.addMetadataLabel('SPAWN_ID', id) container = pool.createContainer() pool.setContainerName(container, project_name) pool.addContainerEnv(container, 'REDIS_SERVER', REDIS_SERVER) pool.addContainerEnv(container, 'REDIS_PASSWORD', REDIS_PASSWORD) pool.addContainerEnv(container, 'SPAWN_ID', id) pool.addContainerEnv(container, 'HOST_CODE', host_code) pool.addContainerEnv(container, 'VERSION_ID', version_id) pool.addContainerEnv(container, 'RELEASE_MODE', RELEASE_MODE) pool.addContainerEnv(container, 'DB_PRODUCT_HOST', DB_PRODUCT_HOST) pool.addContainerEnv(container, 'DB_PRODUCT_PORT', DB_PRODUCT_PORT) pool.addContainerEnv(container, 'DB_PRODUCT_USER', DB_PRODUCT_USER) pool.addContainerEnv(container, 'DB_PRODUCT_PASSWORD', DB_PRODUCT_PASSWORD) pool.addContainerEnv(container, 'DB_PRODUCT_NAME', DB_PRODUCT_NAME) pool.setContainerImage(container, 'bluelens/bl-crawler:' + RELEASE_MODE) pool.setContainerImagePullPolicy(container, 'Always') pool.addContainer(container) pool.setRestartPolicy('Never') pool.spawn()
def exit(): print('exit: ' + SPAWN_ID) logging.debug('exit: ' + SPAWN_ID) data = {} data['namespace'] = 'index' data['id'] = SPAWN_ID spawn = spawning_pool.SpawningPool() spawn.setServerUrl(REDIS_SERVER) spawn.delete(data)
def remove_prev_pods(): pool = spawning_pool.SpawningPool() pool.setServerUrl(REDIS_SERVER) pool.setServerPassword(REDIS_PASSWORD) data = {} data['key'] = 'group' data['value'] = 'bl-object-indexer' pool.delete(data) time.sleep(60)
def delete_pod(): log.info('delete_pod: ' + SPAWN_ID) data = {} data['namespace'] = RELEASE_MODE data['id'] = SPAWN_ID spawn = spawning_pool.SpawningPool() spawn.setServerUrl(REDIS_SERVER) spawn.setServerPassword(REDIS_PASSWORD) spawn.delete(data)
def remove_prev_pods(): pool = spawning_pool.SpawningPool() pool.setServerUrl(REDIS_SERVER) pool.setServerPassword(REDIS_PASSWORD) pool.setMetadataNamespace(RELEASE_MODE) data = {} data['key'] = 'group' data['value'] = 'bl-text-classification-modeler' pool.delete(data) time.sleep(60)
def delete_pod(self, spawn_id, release_mode): self.log('delete_pod: ' + spawn_id) data = {} data['namespace'] = release_mode data['key'] = 'SPAWN_ID' data['value'] = spawn_id spawn = spawning_pool.SpawningPool() spawn.setServerUrl(self._redis_server) spawn.setServerPassword(self._redis_password) spawn.delete(data)
def remove_prev_pods(): pool = spawning_pool.SpawningPool() pool.setServerUrl(REDIS_SERVER) pool.setServerPassword(REDIS_PASSWORD) pool.setMetadataNamespace(RELEASE_MODE) data = {} data['namespace'] = RELEASE_MODE data['key'] = 'group' data['value'] = 'bl-image-processor' pool.delete(data) time.sleep(60)
def spawn_classifier(uuid): pool = spawning_pool.SpawningPool() project_name = 'bl-object-classifier-' + uuid log.debug('spawn_classifier: ' + project_name) pool.setServerUrl(REDIS_SERVER) pool.setServerPassword(REDIS_PASSWORD) pool.setApiVersion('v1') pool.setKind('Pod') pool.setMetadataName(project_name) pool.setMetadataNamespace(RELEASE_MODE) pool.addMetadataLabel('name', project_name) pool.addMetadataLabel('group', 'bl-object-classifier') pool.addMetadataLabel('SPAWN_ID', uuid) container = pool.createContainer() pool.setContainerName(container, project_name) pool.addContainerEnv(container, 'AWS_ACCESS_KEY', AWS_ACCESS_KEY) pool.addContainerEnv(container, 'AWS_SECRET_ACCESS_KEY', AWS_SECRET_ACCESS_KEY) pool.addContainerEnv(container, 'REDIS_SERVER', REDIS_SERVER) pool.addContainerEnv(container, 'REDIS_PASSWORD', REDIS_PASSWORD) pool.addContainerEnv(container, 'SPAWN_ID', uuid) pool.addContainerEnv(container, 'MAX_PROCESS_NUM', str(MAX_PROCESS_NUM)) pool.addContainerEnv(container, 'RELEASE_MODE', RELEASE_MODE) pool.addContainerEnv(container, 'OD_HOST', OD_HOST) pool.addContainerEnv(container, 'OD_PORT', OD_PORT) pool.addContainerEnv(container, 'DB_PRODUCT_HOST', DB_PRODUCT_HOST) pool.addContainerEnv(container, 'DB_PRODUCT_PORT', DB_PRODUCT_PORT) pool.addContainerEnv(container, 'DB_PRODUCT_USER', DB_PRODUCT_USER) pool.addContainerEnv(container, 'DB_PRODUCT_PASSWORD', DB_PRODUCT_PASSWORD) pool.addContainerEnv(container, 'DB_PRODUCT_NAME', DB_PRODUCT_NAME) pool.addContainerEnv(container, 'DB_OBJECT_HOST', DB_OBJECT_HOST) pool.addContainerEnv(container, 'DB_OBJECT_PORT', DB_OBJECT_PORT) pool.addContainerEnv(container, 'DB_OBJECT_USER', DB_OBJECT_USER) pool.addContainerEnv(container, 'DB_OBJECT_PASSWORD', DB_OBJECT_PASSWORD) pool.addContainerEnv(container, 'DB_OBJECT_NAME', DB_OBJECT_NAME) pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_HOST', DB_OBJECT_FEATURE_HOST) pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_PORT', DB_OBJECT_FEATURE_PORT) pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_USER', DB_OBJECT_FEATURE_USER) pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_PASSWORD', DB_OBJECT_FEATURE_PASSWORD) pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_NAME', DB_OBJECT_FEATURE_NAME) pool.addContainerEnv(container, 'DB_IMAGE_HOST', DB_IMAGE_HOST) pool.addContainerEnv(container, 'DB_IMAGE_PORT', DB_IMAGE_PORT) pool.addContainerEnv(container, 'DB_IMAGE_USER', DB_IMAGE_USER) pool.addContainerEnv(container, 'DB_IMAGE_PASSWORD', DB_IMAGE_PASSWORD) pool.addContainerEnv(container, 'DB_IMAGE_NAME', DB_IMAGE_NAME) pool.setContainerImage(container, 'bluelens/bl-object-classifier:' + RELEASE_MODE) pool.setContainerImagePullPolicy(container, 'Always') pool.addContainer(container) pool.setRestartPolicy('Never') pool.spawn()
from __future__ import print_function from bluelens_spawning_pool import spawning_pool pool = spawning_pool.SpawningPool() project_name = 'bl-cropper' pool.setServerUrl('35.187.244.252') pool.setApiVersion('v1') pool.setKind('pod') pool.setMetadataName(project_name) pool.setMetadataNamespace('index') pool.addMetadataLabel('name', project_name) container = pool.createContainer() pool.setContainerName(container, project_name) pool.addContainerEnv(container, 'key1', 'xxxxx') pool.addContainerEnv(container, 'key2', 'yyyyy') pool.setContainerImage(container, 'bluelens/bl-cropper:latest') pool.addContainer(container) pool.setRestartPolicy('Never') pool.spawn()