def spawn_crawler(self, container_name, container_image, id, server_url,
server_password, metadata_namespace, envs):
pool = spawning_pool.SpawningPool()
project_name = container_name + '-' + id
self._log.debug('spawn_crawler: ' + project_name)
pool.setServerUrl(server_url)
pool.setServerPassword(server_password)
pool.setApiVersion('v1')
pool.setKind('Pod')
pool.setMetadataName(project_name)
pool.setMetadataNamespace(metadata_namespace)
pool.addMetadataLabel('name', project_name)
pool.addMetadataLabel('group', container_name)
pool.addMetadataLabel('SPAWN_ID', id)
container = pool.createContainer()
pool.setContainerName(container, project_name)
for key, value in envs.items():
pool.addContainerEnv(container, key, value)
pool.setContainerImage(container, container_image)
pool.setContainerImagePullPolicy(container, 'Always')
pool.addContainer(container)
pool.setRestartPolicy('Never')
pool.spawn()
def spawn_cropper(uuid):
pool = spawning_pool.SpawningPool()
project_name = 'bl-cropper-' + uuid
print('spawn_cropper: ' + project_name)
pool.setServerUrl(REDIS_SERVER)
pool.setApiVersion('v1')
pool.setKind('Pod')
pool.setMetadataName(project_name)
pool.setMetadataNamespace('index')
pool.addMetadataLabel('name', project_name)
pool.addMetadataLabel('SPAWN_ID', uuid)
container = pool.createContainer()
pool.setContainerName(container, project_name)
pool.addContainerEnv(container, 'AWS_ACCESS_KEY', AWS_ACCESS_KEY)
pool.addContainerEnv(container, 'AWS_SECRET_ACCESS_KEY',
AWS_SECRET_ACCESS_KEY)
pool.addContainerEnv(container, 'REDIS_SERVER', REDIS_SERVER)
pool.addContainerEnv(container, 'SPAWN_ID', uuid)
pool.setContainerImage(container, 'bluelens/bl-cropper:latest')
pool.addContainer(container)
pool.setRestartPolicy('Never')
pool.spawn()
def spawn_crawler(host_code, version_id):
pool = spawning_pool.SpawningPool()
id = host_code.lower()
project_name = 'bl-crawler-' + id
log.debug('spawn_crawler: ' + project_name)
pool.setServerUrl(REDIS_SERVER)
pool.setServerPassword(REDIS_PASSWORD)
pool.setApiVersion('v1')
pool.setKind('Pod')
pool.setMetadataName(project_name)
pool.setMetadataNamespace(RELEASE_MODE)
pool.addMetadataLabel('name', project_name)
pool.addMetadataLabel('group', 'bl-crawler')
pool.addMetadataLabel('SPAWN_ID', id)
container = pool.createContainer()
pool.setContainerName(container, project_name)
pool.addContainerEnv(container, 'REDIS_SERVER', REDIS_SERVER)
pool.addContainerEnv(container, 'REDIS_PASSWORD', REDIS_PASSWORD)
pool.addContainerEnv(container, 'SPAWN_ID', id)
pool.addContainerEnv(container, 'HOST_CODE', host_code)
pool.addContainerEnv(container, 'VERSION_ID', version_id)
pool.addContainerEnv(container, 'RELEASE_MODE', RELEASE_MODE)
pool.addContainerEnv(container, 'DB_PRODUCT_HOST', DB_PRODUCT_HOST)
pool.addContainerEnv(container, 'DB_PRODUCT_PORT', DB_PRODUCT_PORT)
pool.addContainerEnv(container, 'DB_PRODUCT_USER', DB_PRODUCT_USER)
pool.addContainerEnv(container, 'DB_PRODUCT_PASSWORD', DB_PRODUCT_PASSWORD)
pool.addContainerEnv(container, 'DB_PRODUCT_NAME', DB_PRODUCT_NAME)
pool.setContainerImage(container, 'bluelens/bl-crawler:' + RELEASE_MODE)
pool.setContainerImagePullPolicy(container, 'Always')
pool.addContainer(container)
pool.setRestartPolicy('Never')
pool.spawn()
def exit():
print('exit: ' + SPAWN_ID)
logging.debug('exit: ' + SPAWN_ID)
data = {}
data['namespace'] = 'index'
data['id'] = SPAWN_ID
spawn = spawning_pool.SpawningPool()
spawn.setServerUrl(REDIS_SERVER)
spawn.delete(data)
def remove_prev_pods():
pool = spawning_pool.SpawningPool()
pool.setServerUrl(REDIS_SERVER)
pool.setServerPassword(REDIS_PASSWORD)
data = {}
data['key'] = 'group'
data['value'] = 'bl-object-indexer'
pool.delete(data)
time.sleep(60)
def delete_pod():
log.info('delete_pod: ' + SPAWN_ID)
data = {}
data['namespace'] = RELEASE_MODE
data['id'] = SPAWN_ID
spawn = spawning_pool.SpawningPool()
spawn.setServerUrl(REDIS_SERVER)
spawn.setServerPassword(REDIS_PASSWORD)
spawn.delete(data)
def remove_prev_pods():
pool = spawning_pool.SpawningPool()
pool.setServerUrl(REDIS_SERVER)
pool.setServerPassword(REDIS_PASSWORD)
pool.setMetadataNamespace(RELEASE_MODE)
data = {}
data['key'] = 'group'
data['value'] = 'bl-text-classification-modeler'
pool.delete(data)
time.sleep(60)
def delete_pod(self, spawn_id, release_mode):
self.log('delete_pod: ' + spawn_id)
data = {}
data['namespace'] = release_mode
data['key'] = 'SPAWN_ID'
data['value'] = spawn_id
spawn = spawning_pool.SpawningPool()
spawn.setServerUrl(self._redis_server)
spawn.setServerPassword(self._redis_password)
spawn.delete(data)
def remove_prev_pods():
pool = spawning_pool.SpawningPool()
pool.setServerUrl(REDIS_SERVER)
pool.setServerPassword(REDIS_PASSWORD)
pool.setMetadataNamespace(RELEASE_MODE)
data = {}
data['namespace'] = RELEASE_MODE
data['key'] = 'group'
data['value'] = 'bl-image-processor'
pool.delete(data)
time.sleep(60)
def spawn_classifier(uuid):
pool = spawning_pool.SpawningPool()
project_name = 'bl-object-classifier-' + uuid
log.debug('spawn_classifier: ' + project_name)
pool.setServerUrl(REDIS_SERVER)
pool.setServerPassword(REDIS_PASSWORD)
pool.setApiVersion('v1')
pool.setKind('Pod')
pool.setMetadataName(project_name)
pool.setMetadataNamespace(RELEASE_MODE)
pool.addMetadataLabel('name', project_name)
pool.addMetadataLabel('group', 'bl-object-classifier')
pool.addMetadataLabel('SPAWN_ID', uuid)
container = pool.createContainer()
pool.setContainerName(container, project_name)
pool.addContainerEnv(container, 'AWS_ACCESS_KEY', AWS_ACCESS_KEY)
pool.addContainerEnv(container, 'AWS_SECRET_ACCESS_KEY', AWS_SECRET_ACCESS_KEY)
pool.addContainerEnv(container, 'REDIS_SERVER', REDIS_SERVER)
pool.addContainerEnv(container, 'REDIS_PASSWORD', REDIS_PASSWORD)
pool.addContainerEnv(container, 'SPAWN_ID', uuid)
pool.addContainerEnv(container, 'MAX_PROCESS_NUM', str(MAX_PROCESS_NUM))
pool.addContainerEnv(container, 'RELEASE_MODE', RELEASE_MODE)
pool.addContainerEnv(container, 'OD_HOST', OD_HOST)
pool.addContainerEnv(container, 'OD_PORT', OD_PORT)
pool.addContainerEnv(container, 'DB_PRODUCT_HOST', DB_PRODUCT_HOST)
pool.addContainerEnv(container, 'DB_PRODUCT_PORT', DB_PRODUCT_PORT)
pool.addContainerEnv(container, 'DB_PRODUCT_USER', DB_PRODUCT_USER)
pool.addContainerEnv(container, 'DB_PRODUCT_PASSWORD', DB_PRODUCT_PASSWORD)
pool.addContainerEnv(container, 'DB_PRODUCT_NAME', DB_PRODUCT_NAME)
pool.addContainerEnv(container, 'DB_OBJECT_HOST', DB_OBJECT_HOST)
pool.addContainerEnv(container, 'DB_OBJECT_PORT', DB_OBJECT_PORT)
pool.addContainerEnv(container, 'DB_OBJECT_USER', DB_OBJECT_USER)
pool.addContainerEnv(container, 'DB_OBJECT_PASSWORD', DB_OBJECT_PASSWORD)
pool.addContainerEnv(container, 'DB_OBJECT_NAME', DB_OBJECT_NAME)
pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_HOST', DB_OBJECT_FEATURE_HOST)
pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_PORT', DB_OBJECT_FEATURE_PORT)
pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_USER', DB_OBJECT_FEATURE_USER)
pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_PASSWORD', DB_OBJECT_FEATURE_PASSWORD)
pool.addContainerEnv(container, 'DB_OBJECT_FEATURE_NAME', DB_OBJECT_FEATURE_NAME)
pool.addContainerEnv(container, 'DB_IMAGE_HOST', DB_IMAGE_HOST)
pool.addContainerEnv(container, 'DB_IMAGE_PORT', DB_IMAGE_PORT)
pool.addContainerEnv(container, 'DB_IMAGE_USER', DB_IMAGE_USER)
pool.addContainerEnv(container, 'DB_IMAGE_PASSWORD', DB_IMAGE_PASSWORD)
pool.addContainerEnv(container, 'DB_IMAGE_NAME', DB_IMAGE_NAME)
pool.setContainerImage(container, 'bluelens/bl-object-classifier:' + RELEASE_MODE)
pool.setContainerImagePullPolicy(container, 'Always')
pool.addContainer(container)
pool.setRestartPolicy('Never')
pool.spawn()
from __future__ import print_function
from bluelens_spawning_pool import spawning_pool
pool = spawning_pool.SpawningPool()
project_name = 'bl-cropper'
pool.setServerUrl('35.187.244.252')
pool.setApiVersion('v1')
pool.setKind('pod')
pool.setMetadataName(project_name)
pool.setMetadataNamespace('index')
pool.addMetadataLabel('name', project_name)
container = pool.createContainer()
pool.setContainerName(container, project_name)
pool.addContainerEnv(container, 'key1', 'xxxxx')
pool.addContainerEnv(container, 'key2', 'yyyyy')
pool.setContainerImage(container, 'bluelens/bl-cropper:latest')
pool.addContainer(container)
pool.setRestartPolicy('Never')
pool.spawn()
