def _import_manager(self, MANAGERS, manager):
"""
Auxiliar function to return module imported from MANAGERS, been referencied by manager.
"""
if manager in MANAGERS:
output = "Importing manager '%s' from '%s'" % (manager,
MANAGERS[manager])
logger.debug(output)
return import_module(MANAGERS[manager])
else:
output = "Importing manager '%s'" % (manager)
logger.debug(output)
return import_module(manager)
def make_communicators(self):
"""
Make communicator objects corresponding to the configured resources.
Return a dictionary, mapping the resource name into the corresponding objects.
"""
communicators = dict()
for name, resdict in list(self.resources.items()):
try:
communicator = import_module(
COMMUNICATORS[resdict['communicator']])
com_object = getattr(communicator, 'Communicator')()
com_object.username = resdict.get('username')
com_object.frontend = resdict.get('frontend')
com_object.private_key = resdict.get('private_key')
com_object.public_key = resdict.get('public_key')
com_object.work_directory = resdict.get(
'scratch', REMOTE_JOBS_DIR)
communicators[name] = com_object
logger.debug(
"Communicator of resource '%s' is defined by: %s.", name,
', '.join([("%s=%s" % (k, v)) for k, v in sorted(
communicators[name].__dict__.items())]))
except Exception as err:
output = "Failed creating communicator for resource '%s' : %s" % (
name, str(err))
logger.warning(output, exc_info=1)
return communicators
def manage_instances(args, resource_name, config):
if args == "start":
try:
hdpackage = import_module(RESOURCE_MANAGERS[config['lrms']] +
".%s" % config['lrms'])
except Exception as err:
raise Exception("The infrastructure selected does not exist. " +
str(err))
threads = []
handlers = []
try:
instance = eval("hdpackage.Instance( config )")
except KeyError as err:
logger.error( "Either you have defined an incorrect value in your configuration file 'resources.conf'" \
" or there's a value that doesn't correspond with any of the keys in your cloud setup file 'cloudsetup.json':" )
raise
except Exception as err:
logger.error(
"An error ocurred while trying to create a VM instance.")
raise
for number_of_th in range(int(config['instances'])):
th = threading.Thread(target=start_instance,
args=(
instance,
resource_name,
))
th.start()
threads.append(th)
[th.join() for th in threads]
elif args == "stop":
instances = []
if not exists(pickled_file + "_" + resource_name):
logger.error(
"There are no available VMs to be deleted for the resource %s"
% resource_name)
else:
with open(pickled_file + "_" + resource_name, "r") as pf:
while True:
try:
instances.append(pickle.load(pf))
except EOFError:
break
if not instances:
logger.error(
"There are no VMs defined in '%s' or the file is not well formed."
% (pickled_file + "_" + resource_name))
exit(1)
threads = []
for instance in instances:
th = threading.Thread(target=stop_instance,
args=(instance, resource_name))
th.start()
threads.append(th)
[th.join() for th in threads]
else:
logger.error("Invalid option")
exit(1)
def __setstate__(self, dict):
self.__dict__.update(dict)
communicator = import_module(COMMUNICATORS[self.data['communicator']])
com_obj = getattr(communicator, 'Communicator')()
com_obj.username = self.data['username']
com_obj.frontend = self.data['frontend']
com_obj.private_key = self.private_key
com_obj.public_key = self.data.get('public_key',
self.private_key + '.pub')
com_obj.work_directory = self.data.get('scratch', REMOTE_JOBS_DIR)
self.com_object = com_obj
def make_resources(self):
"""
Make manager objects corresponding to the configured resources.
Return a dictionary, mapping the resource name into the corresponding objects.
"""
resources = dict()
for name, resdict in list(self.resources.items()):
try:
resources[name] = dict()
manager = import_module(RESOURCE_MANAGERS[resdict['lrms']])
resource_object = getattr(manager, 'Resource')()
resource_object.name = name
resource_object.features = resdict
job_object = getattr(manager, 'Job')()
job_object.resfeatures = resdict
resources[name]['Resource'] = resource_object
resources[name]['Job'] = job_object
except Exception as err:
output = "Failed creating objects for resource '%s' of type : %s" % (
name, str(err))
logger.warning(output, exc_info=1)
return resources
def __init__(self, basic_data):
self.data = basic_data
self.id = None
self.id_volume = None
self.id_link = None
self.int_ip = None
self.ext_ip = None
self.volume = int(basic_data['volume'])
self.myproxy_server = basic_data.get('myproxy_server', '')
self.private_key = expanduser(basic_data['private_key'])
self.context_file = basename(self.private_key) + ".login"
self.vm_user = basic_data.get('vm_user', 'drm4g_admin')
self.cloud_contextualisation_file = basic_data.get(
'vm_config', join(DRM4G_DIR, "etc", "cloud_config.conf"))
self.comm = basic_data['communicator']
self.max_jobs_running = basic_data['max_jobs_running']
self.vm_comm = basic_data.get('vm_communicator', self.comm)
if self.vm_comm == 'local':
self.vm_comm = 'pk_ssh'
pub = read_key(self.private_key + ".pub")
try:
cloud_setup = {}
for name, features in load_json(cloud_setup_file).items():
cloud_setup[name] = CloudSetup(name, features)
except Exception as err:
logger.error("Error reading the cloud setup file: " + str(err))
infra_cfg = cloud_setup[basic_data['lrms']]
cloud_cfg = infra_cfg.cloud_providers[basic_data['cloud_provider']]
self.vo = infra_cfg.vo
self.endpoint = cloud_cfg["endpoint"]
self.flavour = cloud_cfg["flavours"][basic_data['flavour']]
self.app_name = basic_data['virtual_image']
self.app = cloud_cfg["apps"][self.app_name]
communicator = import_module(COMMUNICATORS[basic_data['communicator']])
com_obj = getattr(communicator, 'Communicator')()
com_obj.username = basic_data['username']
com_obj.frontend = basic_data['frontend']
com_obj.private_key = self.private_key
com_obj.public_key = basic_data.get('public_key',
self.private_key + '.pub')
com_obj.work_directory = basic_data.get('scratch', REMOTE_JOBS_DIR)
self.com_object = com_obj
self.proxy_file = join(REMOTE_VOS_DIR, "x509up.%s") % self.vo
'''
commented so that the context file is created everytime
just in case the user changed something in the contextualisation file
#cmd = "ls %s" % self.context_file #to check if it exists
out,err = self.com_object.execCommand( cmd )
if err:
'''
with open(self.cloud_contextualisation_file, "r") as contex_file:
cloud_config = contex_file.read()
if 'vm_config' not in basic_data.keys():
content = cloud_config % (self.vm_user, self.vm_user, pub)
else:
content = cloud_config
logger.debug("Your contextualisation file %s :\n%s\n" %
(self.cloud_contextualisation_file, content))
#content = generic_cloud_cfg % (self.vm_user, self.vm_user, pub)
cmd = "echo '%s' > %s" % (content, self.context_file)
out, err = self.com_object.execCommand(cmd)
if err:
raise Exception("Wasn't able to create the context file %s." %
self.context_file + err)
cmd = "ls %s" % self.proxy_file #to check if it exists
out, err = self.com_object.execCommand(cmd)
if err:
self._renew_voms_proxy()