def __init__(self, shockurl, arasturl, config, threads, queue, kill_queue,
job_list, ctrl_conf):
self.parser = SafeConfigParser()
self.parser.read(config)
self.job_list = job_list
# Load plugins
self.pmanager = ModuleManager(threads, kill_queue, job_list)
# Set up environment
self.shockurl = shockurl
self.arasturl = arasturl
self.datapath = self.parser.get('compute', 'datapath')
if queue:
self.queue = queue
print('Using queue:{}'.format(self.queue))
else:
self.queue = self.parser.get('rabbitmq', 'default_routing_key')
self.min_free_space = float(
self.parser.get('compute', 'min_free_space'))
m = ctrl_conf['meta']
a = ctrl_conf['assembly']
self.metadata = meta.MetadataConnection(arasturl, int(a['mongo_port']),
m['mongo.db'],
m['mongo.collection'],
m['mongo.collection.auth'])
self.gc_lock = multiprocessing.Lock()
def start(config_file,
mongo_host=None,
mongo_port=None,
rabbit_host=None,
rabbit_port=None):
global parser, metadata
logging.basicConfig(level=logging.DEBUG)
parser = SafeConfigParser()
parser.read(config_file)
metadata = meta.MetadataConnection(
parser.get('assembly', 'mongo_host'),
int(parser.get('assembly', 'mongo_port')),
parser.get('meta', 'mongo.db'), parser.get('meta', 'mongo.collection'),
parser.get('meta', 'mongo.collection.auth'))
##### CherryPy ######
conf = {
'global': {
'server.socket_host': '0.0.0.0',
'server.socket_port': 8000,
'log.screen': True,
'ar_shock_url': parser.get('shock', 'host'),
},
}
static_root = parser.get('web_serve', 'root')
root = Root()
root.user = UserResource()
root.module = ModuleResource()
root.shock = ShockResource({"shockurl": get_upload_url()})
root.static = StaticResource(static_root)
#### Admin Routes ####
rmq_host = parser.get('assembly', 'rabbitmq_host')
rmq_mp = parser.get('rabbitmq', 'management_port')
rmq_user = parser.get('rabbitmq', 'management_user')
rmq_pass = parser.get('rabbitmq', 'management_pass')
root.admin = SystemResource(rmq_host, rmq_mp, rmq_user, rmq_pass)
#cherrypy.request.hooks.attach('before_request_body', authenticate_request)
cherrypy.request.hooks.attach('before_finalize', CORS)
cherrypy.quickstart(root, '/', conf)
def __init__(self, shockurl, rmq_host, rmq_port, mongo_host, mongo_port,
config, threads, queues, kill_list, kill_list_lock, job_list,
job_list_lock, ctrl_conf, datapath, binpath, modulebin):
self.parser = SafeConfigParser()
self.parser.read(config)
self.kill_list = kill_list
self.kill_list_lock = kill_list_lock
self.job_list = job_list
self.job_list_lock = job_list_lock
# Load plugins
self.threads = threads
self.binpath = binpath
self.modulebin = modulebin
self.pmanager = ModuleManager(threads, kill_list, kill_list_lock,
job_list, binpath, modulebin)
# Set up environment
self.shockurl = shockurl
self.datapath = datapath
self.rmq_host = rmq_host
self.rmq_port = rmq_port
self.mongo_host = mongo_host
self.mongo_port = mongo_port
self.queues = queues
self.min_free_space = float(
self.parser.get('compute', 'min_free_space'))
self.data_expiration_days = float(
self.parser.get('compute', 'data_expiration_days'))
m = ctrl_conf['meta']
a = ctrl_conf['assembly']
collections = {
'jobs': m.get('mongo.collection', 'jobs'),
'auth': m.get('mongo.collection.auth', 'auth'),
'data': m.get('mongo.collection.data', 'data'),
'running': m.get('mongo.collection.running', 'running_jobs')
}
###### TODO Use REST API
self.metadata = meta.MetadataConnection(self.mongo_host,
self.mongo_port, m['mongo.db'],
collections)
self.gc_lock = multiprocessing.Lock()
def __init__(self, os_user, os_pass, os_tenant, os_auth_url, config_file):
super(CloudMonitor, self).__init__(os_user,
os_pass,
os_tenant,
os_auth_url,
service_type='compute',
insecure=True)
p = SafeConfigParser()
p.read(config_file)
self.default_flavor = p.get('cloud', 'flavor.default')
self.default_keypair = p.get('cloud', 'keypair.default')
self.default_image = p.get('cloud', 'image.kbase_compute')
self.compute_init = p.get('cloud', 'compute_init')
self.secgroup = p.get('cloud', 'sec_group')
self.active_nodes = p.get('cloud', 'active.collection')
self.all_nodes = p.get('cloud', 'history.collection')
self.mongo = p.get('meta', 'mongo.host')
self.metadata = metadata.MetadataConnection(config_file, self.mongo)
def start(config_file,
shock_url=None,
mongo_host=None,
mongo_port=None,
rabbit_host=None,
rabbit_port=None):
global parser, metadata, rjobmon
# logging.basicConfig(level=logging.DEBUG)
parser = SafeConfigParser()
parser.read(config_file)
collections = {
'jobs': parser.get('meta', 'mongo.collection'),
'auth': parser.get('meta', 'mongo.collection.auth'),
'data': parser.get('meta', 'mongo.collection.data'),
'running': parser.get('meta', 'mongo.collection.running')
}
# Config precedence: args > config file
if shock_url:
parser.set('shock', 'host', shock.verify_shock_url(shock_url))
if mongo_host:
parser.set('assembly', 'mongo_host', mongo_host)
if mongo_port:
parser.set('assembly', 'mongo_port', str(mongo_port))
if rabbit_host:
parser.set('assembly', 'rabbitmq_host', rabbit_host)
if rabbit_port:
parser.set('assembly', 'rabbitmq_port', str(rabbit_port))
metadata = meta.MetadataConnection(
parser.get('assembly', 'mongo_host'),
int(parser.get('assembly', 'mongo_port')),
parser.get('meta', 'mongo.db'), collections)
##### Running Job Monitor #####
rjobmon = RunningJobsMonitor(metadata)
cherrypy.process.plugins.Monitor(
cherrypy.engine,
rjobmon.purge,
frequency=int(parser.get('monitor', 'running_job_freq'))).subscribe()
##### CherryPy ######
conf = {
'global': {
'server.socket_host': '0.0.0.0',
'server.socket_port': int(parser.get('assembly', 'cherrypy_port')),
'log.screen': True,
'ar_shock_url': parser.get('shock', 'host'),
'environment': 'production'
},
}
static_root = parser.get('web_serve', 'root')
root = Root()
root.user = UserResource()
root.module = ModuleResource()
root.recipe = RecipeResource()
root.shock = ShockResource({"shockurl": get_upload_url()})
root.static = StaticResource(static_root)
#### Admin Routes ####
rmq_host = parser.get('assembly', 'rabbitmq_host')
rmq_mp = parser.get('rabbitmq', 'management_port')
rmq_user = parser.get('rabbitmq', 'management_user')
rmq_pass = parser.get('rabbitmq', 'management_pass')
root.admin = SystemResource(rmq_host, rmq_mp, rmq_user, rmq_pass)
cherrypy.request.hooks.attach('before_finalize', CORS)
cherrypy.quickstart(root, '/', conf)