def init_conf(configuration, hosturl='', hostidentifier=''):
"""Pull out a full configuration for specified resource with all missing
fields set to default values.
"""
conf = {}
if not hosturl:
return conf
resource_id = '%s.%s' % (hosturl, hostidentifier)
conf_file = os.path.join(configuration.resource_home, resource_id,
'config.MiG')
if not os.path.isfile(conf_file):
return conf
(status, msg, conf) = get_resource_config_dict(configuration, conf_file)
if not status:
return conf
# Fill in frontendhome if not already set
if not conf.get('frontendhome', None):
conf['frontendhome'] = ''
if conf.get('RESOURCEHOME', None):
home = conf['RESOURCEHOME']
base_index = home.find('/MiG/mig_frontend/')
if base_index != -1:
home = home[:base_index]
conf['frontendhome'] = home
if conf.get('HOSTKEY', None):
if re.match('^[a-zA-Z0-9.-]+,[0-9.]+$', conf['HOSTKEY'].split()[0]):
conf['HOSTKEY'] = conf['HOSTKEY'].split(None, 1)[1]
# Fill in all_X if not already set
if not conf.get('all_exes', None):
all = {}
all['execution_user'] = ''
all['nodecount'] = 1
all['cputime'] = 3600
all['execution_precondition'] = ' '
all['prepend_execute'] = 'nice -19'
all['exehostlog'] = ''
all['joblog'] = ''
all['start_command'] = 'default'
all['status_command'] = 'default'
all['stop_command'] = 'default'
all['clean_command'] = 'default'
all['continuous'] = True
all['shared_fs'] = True
all['vgrid_list'] = []
all['executionnodes'] = []
all['executionhome'] = ''
if conf.get('EXECONFIG', None):
all['executionnodes'] = generate_execution_node_string(
conf['EXECONFIG'])
first = conf['EXECONFIG'][0]
home = str(first['execution_dir'])
base_index = home.find('/MiG/mig_exe/')
if base_index != -1:
home = home[:base_index]
all['executionhome'] = home
all['execution_user'] = str(first['execution_user'])
all['nodecount'] = first['nodecount']
all['cputime'] = first['cputime']
all['execution_precondition'] = str(
first['execution_precondition']).strip("'")
all['prepend_execute'] = str(first['prepend_execute']).strip('"')
all['start_command'] = str(first['start_command'])
default_start_command = default_exe_start_command(first)
if all['start_command'].strip().replace('\\', '') == \
default_start_command.strip().replace('\\', ''):
all['start_command'] = 'default'
local_start_command = local_exe_start_command(first)
if all['start_command'].strip().replace('\\', '') == \
local_start_command.strip().replace('\\', ''):
all['start_command'] = 'local'
local_leader_start_command = local_exe_start_command(
first, 'leader')
if all['start_command'].strip().replace('\\', '') == \
local_leader_start_command.strip().replace('\\', ''):
all['start_command'] = 'local'
local_dummy_start_command = local_exe_start_command(first, 'dummy')
if all['start_command'].strip().replace('\\', '') == \
local_dummy_start_command.strip().replace('\\', ''):
all['start_command'] = 'local'
all['status_command'] = str(first['status_command'])
default_status_command = default_exe_status_command(first)
if all['status_command'].strip().replace('\\', '') == \
default_status_command.strip().replace('\\', ''):
all['status_command'] = 'default'
local_status_command = local_exe_status_command(first)
if all['status_command'].strip().replace('\\', '') == \
local_status_command.strip().replace('\\', ''):
all['status_command'] = 'local'
local_leader_status_command = local_exe_status_command(
first, 'leader')
if all['status_command'].strip().replace('\\', '') == \
local_leader_status_command.strip().replace('\\', ''):
all['status_command'] = 'local'
local_dummy_status_command = local_exe_status_command(
first, 'dummy')
if all['status_command'].strip().replace('\\', '') == \
local_dummy_status_command.strip().replace('\\', ''):
all['status_command'] = 'local'
all['stop_command'] = str(first['stop_command'])
default_stop_command = default_exe_stop_command(first)
if all['stop_command'].strip().replace('\\', '') == \
default_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'default'
local_stop_command = local_exe_stop_command(first)
if all['stop_command'].strip().replace('\\', '') == \
local_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'local'
local_leader_stop_command = local_exe_stop_command(first, 'leader')
if all['stop_command'].strip().replace('\\', '') == \
local_leader_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'local'
local_dummy_stop_command = local_exe_stop_command(first, 'dummy')
if all['stop_command'].strip().replace('\\', '') == \
local_dummy_stop_command.strip().replace('\\', ''):
conf['stop_command'] = 'local'
all['clean_command'] = str(first['clean_command'])
default_clean_command = default_exe_clean_command(first)
if all['clean_command'].strip().replace('\\', '') == \
default_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'default'
local_clean_command = local_exe_clean_command(first)
if all['clean_command'].strip().replace('\\', '') == \
local_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'local'
local_leader_clean_command = local_exe_clean_command(
first, 'leader')
if all['clean_command'].strip().replace('\\', '') == \
local_leader_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'local'
local_dummy_clean_command = local_exe_clean_command(first, 'dummy')
if all['clean_command'].strip().replace('\\', '') == \
local_dummy_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'local'
# Handle old typo gracefully
if first.has_key('continuous'):
all['continuous'] = first['continuous']
else:
all['continuous'] = first['continious']
all['shared_fs'] = first['shared_fs']
# Read in list value
all['vgrid'] = first['vgrid']
conf['all_exes'] = all
if not conf.get('all_stores', None):
all = {}
all['storage_disk'] = 10
all['storage_protocol'] = 'sftp'
all['storage_port'] = 22
all['storage_user'] = '******'
all['storage_dir'] = '/home/miguser/MiG-storage'
all['start_command'] = 'default'
all['status_command'] = 'default'
all['stop_command'] = 'default'
all['clean_command'] = 'default'
all['shared_fs'] = True
all['vgrid'] = []
all['storagenodes'] = ''
all['storagehome'] = ''
if conf.get('STORECONFIG', None):
all['storagenodes'] = generate_storage_node_string(
conf['STORECONFIG'])
first = conf['STORECONFIG'][0]
home = str(first['storage_dir'])
base_index = home.find('/MiG/mig_store/')
if base_index != -1:
home = home[:base_index]
all['storagehome'] = home
all['storage_disk'] = first['storage_disk']
all['storage_protocol'] = str(first['storage_protocol'])
all['storage_port'] = first['storage_port']
all['storage_user'] = str(first['storage_user'])
all['storage_dir'] = str(first['storage_dir'])
all['start_command'] = str(first['start_command'])
default_start_command = default_store_start_command(first)
if all['start_command'].strip().replace('\\', '') == \
default_start_command.strip().replace('\\', ''):
all['start_command'] = 'default'
local_start_command = local_store_start_command(first)
if all['start_command'].strip().replace('\\', '') == \
local_start_command.strip().replace('\\', ''):
all['start_command'] = 'local'
all['status_command'] = str(first['status_command'])
default_status_command = default_store_status_command(first)
if all['status_command'].strip().replace('\\', '') == \
default_status_command.strip().replace('\\', ''):
all['status_command'] = 'default'
local_status_command = local_store_status_command(first)
if all['status_command'].strip().replace('\\', '') == \
local_status_command.strip().replace('\\', ''):
all['status_command'] = 'local'
all['stop_command'] = str(first['stop_command'])
default_stop_command = default_store_stop_command(first)
if all['stop_command'].strip().replace('\\', '') == \
default_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'default'
local_stop_command = local_store_stop_command(first)
if all['stop_command'].strip().replace('\\', '') == \
local_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'local'
all['clean_command'] = str(conf['STORECONFIG'][0]['clean_command'])
default_clean_command = default_store_clean_command(first)
if all['clean_command'].strip().replace('\\', '') == \
default_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'default'
local_clean_command = local_store_clean_command(first)
if all['clean_command'].strip().replace('\\', '') == \
local_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'local'
all['shared_fs'] = first['shared_fs']
# Read in list value
all['vgrid'] = first['vgrid']
conf['all_stores'] = all
return conf
def create_resource_conf(
configuration,
client_id,
resource_name,
resource_identifier,
resource_configfile,
new_resource=True
):
"""Create a resource from conf in pending file. If pending_file is a
relative path it will prefixed with the resource_pending dir of the
client_id.
"""
if new_resource:
msg = """
Trying to create configuration for new resource: '%s.%s' from file '%s':
""" % (resource_name, str(resource_identifier),
resource_configfile)
else:
msg = """
Trying to update configuration for existing resource '%s.%s':
""" % (resource_name, str(resource_identifier))
client_dir = client_id_dir(client_id)
if os.path.isabs(resource_configfile):
pending_file = resource_configfile
else:
pending_file = os.path.join(configuration.resource_pending, client_dir,
resource_configfile)
tmpfile = pending_file + '.tmp'
new_configfile = os.path.join(configuration.resource_home, '%s.%s' % \
(resource_name, resource_identifier),
'config.MiG')
if not os.path.exists(pending_file):
msg += """
Failure:
File: '%s' doesn't exist."""\
% pending_file
return (False, msg)
(status, conf_msg, config_dict) = get_resource_config_dict(pending_file)
if not status:
msg += '\n%s' % conf_msg
return (False, msg)
if config_dict['HOSTURL'] != resource_name:
msg += \
"""
Failure:
resource_name: '%s'
does'nt match hosturl: '%s'
in configfile: '%s'"""\
% (resource_name, config_dict['HOSTURL'], pending_file)
return (False, msg)
if not new_resource and \
config_dict['HOSTIDENTIFIER'] != resource_identifier:
msg += \
"""
Failure:
resource_identifier: '%s'
does'nt match hostidentifier: '%s'
in configfile: '%s'"""\
% (resource_identifier, config_dict['HOSTIDENTIFIER'],
pending_file)
return (False, msg)
try:
fr = open(pending_file, 'r')
fw = open(tmpfile, 'w')
readline = fr.readline()
while len(readline) > 0:
fw.write(readline.replace('$HOSTIDENTIFIER',
str(resource_identifier)))
readline = fr.readline()
fw.close()
fr.close()
except Exception, err:
msg += \
'Failed to apply hostidentifier to configfile. Failure: %s'\
% err
return (False, msg)
def create_resource_conf(configuration,
client_id,
resource_name,
resource_identifier,
resource_configfile,
new_resource=True):
"""Create a resource from conf in pending file. If pending_file is a
relative path it will prefixed with the resource_pending dir of the
client_id.
"""
if new_resource:
msg = """
Trying to create configuration for new resource: '%s.%s' from file '%s':
""" % (resource_name, str(resource_identifier), resource_configfile)
else:
msg = """
Trying to update configuration for existing resource '%s.%s':
""" % (resource_name, str(resource_identifier))
client_dir = client_id_dir(client_id)
if os.path.isabs(resource_configfile):
pending_file = resource_configfile
else:
pending_file = os.path.join(configuration.resource_pending, client_dir,
resource_configfile)
tmpfile = pending_file + '.tmp'
new_configfile = os.path.join(configuration.resource_home, '%s.%s' % \
(resource_name, resource_identifier),
'config.MiG')
if not os.path.exists(pending_file):
msg += """
Failure:
File: '%s' doesn't exist."""\
% pending_file
return (False, msg)
(status, conf_msg,
config_dict) = get_resource_config_dict(configuration, pending_file)
if not status:
msg += '\n%s' % conf_msg
return (False, msg)
if config_dict['HOSTURL'] != resource_name:
msg += \
"""
Failure:
resource_name: '%s'
does'nt match hosturl: '%s'
in configfile: '%s'"""\
% (resource_name, config_dict['HOSTURL'], pending_file)
return (False, msg)
if not new_resource and \
config_dict['HOSTIDENTIFIER'] != resource_identifier:
msg += \
"""
Failure:
resource_identifier: '%s'
does'nt match hostidentifier: '%s'
in configfile: '%s'"""\
% (resource_identifier, config_dict['HOSTIDENTIFIER'],
pending_file)
return (False, msg)
try:
fr = open(pending_file, 'r')
fw = open(tmpfile, 'w')
readline = fr.readline()
while len(readline) > 0:
fw.write(readline.replace(keyword_auto, str(resource_identifier)))
readline = fr.readline()
fw.close()
fr.close()
except Exception, err:
msg += \
'Failed to apply hostidentifier to configfile. Failure: %s'\
% err
return (False, msg)
def init_conf(configuration, hosturl='', hostidentifier=''):
"""Pull out a full configuration for specified resource with all missing
fields set to default values.
"""
conf = {}
if not hosturl:
return conf
resource_id = '%s.%s' % (hosturl, hostidentifier)
conf_file = os.path.join(configuration.resource_home, resource_id,
'config.MiG')
if not os.path.isfile(conf_file):
return conf
(status, msg, conf) = \
get_resource_config_dict(conf_file)
if not status:
return conf
# Fill in frontendhome if not already set
if not conf.get('frontendhome', None):
conf['frontendhome'] = ''
if conf.get('RESOURCEHOME', None):
home = conf['RESOURCEHOME']
base_index = home.find('/MiG/mig_frontend/')
if base_index != -1:
home = home[:base_index]
conf['frontendhome'] = home
if conf.get('HOSTKEY', None):
if conf['HOSTKEY'].split()[1].startswith('ssh-'):
conf['HOSTKEY'] = conf['HOSTKEY'].split(None, 1)[1]
# Fill in all_X if not already set
if not conf.get('all_exes', None):
all = {}
all['execution_user'] = ''
all['nodecount'] = 1
all['cputime'] = 3600
all['execution_precondition'] = ' '
all['prepend_execute'] = 'nice -19'
all['exehostlog'] = ''
all['joblog'] = ''
all['start_command'] = 'default'
all['status_command'] = 'default'
all['stop_command'] = 'default'
all['clean_command'] = 'default'
all['continuous'] = True
all['shared_fs'] = True
all['vgrid_list'] = []
all['executionnodes'] = []
all['executionhome'] = ''
if conf.get('EXECONFIG', None):
all['executionnodes'] = generate_execution_node_string(conf['EXECONFIG'])
first = conf['EXECONFIG'][0]
home = str(first['execution_dir'])
base_index = home.find('/MiG/mig_exe/')
if base_index != -1:
home = home[:base_index]
all['executionhome'] = home
all['execution_user'] = str(first['execution_user'])
all['nodecount'] = first['nodecount']
all['cputime'] = first['cputime']
all['execution_precondition'] = str(first['execution_precondition']).strip("'")
all['prepend_execute'] = str(first['prepend_execute']).strip('"')
all['start_command'] = str(first['start_command'])
default_start_command = \
default_exe_start_command(str(first['execution_dir']), str(first['name']))
if all['start_command'].strip().replace('\\', '')\
== default_start_command.strip().replace('\\', ''):
all['start_command'] = 'default'
local_start_command = \
local_exe_start_command(str(first['execution_dir']), str(first['name']))
if all['start_command'].strip().replace('\\', '')\
== local_start_command.strip().replace('\\', ''):
all['start_command'] = 'local'
local_leader_start_command = \
local_exe_start_command(str(first['execution_dir']), str(first['name']), 'leader')
if all['start_command'].strip().replace('\\', '')\
== local_leader_start_command.strip().replace('\\',
''):
all['start_command'] = 'local'
local_dummy_start_command = \
local_exe_start_command(str(first['execution_dir']), str(first['name']), 'dummy')
if all['start_command'].strip().replace('\\', '')\
== local_dummy_start_command.strip().replace('\\', ''
):
all['start_command'] = 'local'
all['status_command'] = str(first['status_command'])
default_status_command = \
default_exe_status_command(str(first['execution_dir']), str(first['name']))
if all['status_command'].strip().replace('\\', '')\
== default_status_command.strip().replace('\\', ''):
all['status_command'] = 'default'
local_status_command = \
local_exe_status_command(str(first['execution_dir']), str(first['name']))
if all['status_command'].strip().replace('\\', '')\
== local_status_command.strip().replace('\\', ''):
all['status_command'] = 'local'
local_leader_status_command = \
local_exe_status_command(str(first['execution_dir']), str(first['name']), 'leader')
if all['status_command'].strip().replace('\\', '')\
== local_leader_status_command.strip().replace('\\', ''
):
all['status_command'] = 'local'
local_dummy_status_command = \
local_exe_status_command(str(first['execution_dir']), str(first['name']), 'dummy')
if all['status_command'].strip().replace('\\', '')\
== local_dummy_status_command.strip().replace('\\', ''
):
all['status_command'] = 'local'
all['stop_command'] = str(first['stop_command'
])
default_stop_command = \
default_exe_stop_command(str(first['execution_dir']), str(first['name']))
if all['stop_command'].strip().replace('\\', '')\
== default_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'default'
local_stop_command = \
local_exe_stop_command(str(first['execution_dir']), str(first['name']))
if all['stop_command'].strip().replace('\\', '')\
== local_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'local'
local_leader_stop_command = \
local_exe_stop_command(str(first['execution_dir']), str(first['name']), 'leader')
if all['stop_command'].strip().replace('\\', '')\
== local_leader_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'local'
local_dummy_stop_command = \
local_exe_stop_command(str(first['execution_dir']), str(first['name']), 'dummy')
if all['stop_command'].strip().replace('\\', '')\
== local_dummy_stop_command.strip().replace('\\', ''):
conf['stop_command'] = 'local'
all['clean_command'] = str(first['clean_command'])
default_clean_command = \
default_exe_clean_command(str(first['execution_dir']), str(first['name']))
if all['clean_command'].strip().replace('\\', '')\
== default_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'default'
local_clean_command = \
local_exe_clean_command(str(first['execution_dir']), str(first['name']))
if all['clean_command'].strip().replace('\\', '')\
== local_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'local'
local_leader_clean_command = \
local_exe_clean_command(str(first['execution_dir']), str(first['name']), 'leader')
if all['clean_command'].strip().replace('\\', '')\
== local_leader_clean_command.strip().replace('\\', ''
):
all['clean_command'] = 'local'
local_dummy_clean_command = \
local_exe_clean_command(str(first['execution_dir']), str(first['name']), 'dummy')
if all['clean_command'].strip().replace('\\', '')\
== local_dummy_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'local'
# Handle old typo gracefully
if first.has_key('continuous'):
all['continuous'] = first['continuous']
else:
all['continuous'] = first['continious']
all['shared_fs'] = first['shared_fs']
# Read in list value
all['vgrid'] = first['vgrid']
conf['all_exes'] = all
if not conf.get('all_stores', None):
all = {}
all['storage_disk'] = 10
all['storage_protocol'] = 'sftp'
all['storage_port'] = 22
all['storage_user'] = '******'
all['storage_dir'] = '/home/miguser/MiG-storage'
all['start_command'] = 'default'
all['status_command'] = 'default'
all['stop_command'] = 'default'
all['clean_command'] = 'default'
all['shared_fs'] = True
all['vgrid'] = []
all['storagenodes'] = ''
all['storagehome'] = ''
if conf.get('STORECONFIG', None):
all['storagenodes'] = generate_storage_node_string(conf['STORECONFIG'])
first = conf['STORECONFIG'][0]
home = str(first['storage_dir'])
base_index = home.find('/MiG/mig_store/')
if base_index != -1:
home = home[:base_index]
all['storagehome'] = home
all['storage_disk'] = first['storage_disk']
all['storage_protocol'] = str(first['storage_protocol'])
all['storage_port'] = first['storage_port']
all['storage_user'] = str(first['storage_user'])
all['storage_dir'] = str(first['storage_dir'])
all['start_command'] = str(first['start_command'])
default_start_command = \
default_store_start_command(str(first['storage_dir']), str(first['name']))
if all['start_command'].strip().replace('\\', '')\
== default_start_command.strip().replace('\\', ''):
all['start_command'] = 'default'
local_start_command = \
local_store_start_command(str(first['storage_dir']), str(first['name']))
if all['start_command'].strip().replace('\\', '')\
== local_start_command.strip().replace('\\', ''):
all['start_command'] = 'local'
all['status_command'] = str(first['status_command'])
default_status_command = \
default_store_status_command(str(first['storage_dir']), str(first['name']))
if all['status_command'].strip().replace('\\', '')\
== default_status_command.strip().replace('\\', ''):
all['status_command'] = 'default'
local_status_command = \
local_store_status_command(str(first['storage_dir']), str(first['name']))
if all['status_command'].strip().replace('\\', '')\
== local_status_command.strip().replace('\\', ''):
all['status_command'] = 'local'
all['stop_command'] = str(first['stop_command'
])
default_stop_command = \
default_store_stop_command(str(first['storage_dir']), str(first['name']))
if all['stop_command'].strip().replace('\\', '')\
== default_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'default'
local_stop_command = \
local_store_stop_command(str(first['storage_dir']), str(first['name']))
if all['stop_command'].strip().replace('\\', '')\
== local_stop_command.strip().replace('\\', ''):
all['stop_command'] = 'local'
all['clean_command'] = str(conf['STORECONFIG'
][0]['clean_command'])
default_clean_command = \
default_store_clean_command(str(first['storage_dir']), str(first['name']))
if all['clean_command'].strip().replace('\\', '')\
== default_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'default'
local_clean_command = \
local_store_clean_command(str(first['storage_dir']), str(first['name']))
if all['clean_command'].strip().replace('\\', '')\
== local_clean_command.strip().replace('\\', ''):
all['clean_command'] = 'local'
all['shared_fs'] = first['shared_fs']
# Read in list value
all['vgrid'] = first['vgrid']
conf['all_stores'] = all
return conf
