def init_conf(configuration, hosturl='', hostidentifier=''): """Pull out a full configuration for specified resource with all missing fields set to default values. """ conf = {} if not hosturl: return conf resource_id = '%s.%s' % (hosturl, hostidentifier) conf_file = os.path.join(configuration.resource_home, resource_id, 'config.MiG') if not os.path.isfile(conf_file): return conf (status, msg, conf) = get_resource_config_dict(configuration, conf_file) if not status: return conf # Fill in frontendhome if not already set if not conf.get('frontendhome', None): conf['frontendhome'] = '' if conf.get('RESOURCEHOME', None): home = conf['RESOURCEHOME'] base_index = home.find('/MiG/mig_frontend/') if base_index != -1: home = home[:base_index] conf['frontendhome'] = home if conf.get('HOSTKEY', None): if re.match('^[a-zA-Z0-9.-]+,[0-9.]+$', conf['HOSTKEY'].split()[0]): conf['HOSTKEY'] = conf['HOSTKEY'].split(None, 1)[1] # Fill in all_X if not already set if not conf.get('all_exes', None): all = {} all['execution_user'] = '' all['nodecount'] = 1 all['cputime'] = 3600 all['execution_precondition'] = ' ' all['prepend_execute'] = 'nice -19' all['exehostlog'] = '' all['joblog'] = '' all['start_command'] = 'default' all['status_command'] = 'default' all['stop_command'] = 'default' all['clean_command'] = 'default' all['continuous'] = True all['shared_fs'] = True all['vgrid_list'] = [] all['executionnodes'] = [] all['executionhome'] = '' if conf.get('EXECONFIG', None): all['executionnodes'] = generate_execution_node_string( conf['EXECONFIG']) first = conf['EXECONFIG'][0] home = str(first['execution_dir']) base_index = home.find('/MiG/mig_exe/') if base_index != -1: home = home[:base_index] all['executionhome'] = home all['execution_user'] = str(first['execution_user']) all['nodecount'] = first['nodecount'] all['cputime'] = first['cputime'] all['execution_precondition'] = str( first['execution_precondition']).strip("'") all['prepend_execute'] = str(first['prepend_execute']).strip('"') all['start_command'] = str(first['start_command']) default_start_command = default_exe_start_command(first) if all['start_command'].strip().replace('\\', '') == \ default_start_command.strip().replace('\\', ''): all['start_command'] = 'default' local_start_command = local_exe_start_command(first) if all['start_command'].strip().replace('\\', '') == \ local_start_command.strip().replace('\\', ''): all['start_command'] = 'local' local_leader_start_command = local_exe_start_command( first, 'leader') if all['start_command'].strip().replace('\\', '') == \ local_leader_start_command.strip().replace('\\', ''): all['start_command'] = 'local' local_dummy_start_command = local_exe_start_command(first, 'dummy') if all['start_command'].strip().replace('\\', '') == \ local_dummy_start_command.strip().replace('\\', ''): all['start_command'] = 'local' all['status_command'] = str(first['status_command']) default_status_command = default_exe_status_command(first) if all['status_command'].strip().replace('\\', '') == \ default_status_command.strip().replace('\\', ''): all['status_command'] = 'default' local_status_command = local_exe_status_command(first) if all['status_command'].strip().replace('\\', '') == \ local_status_command.strip().replace('\\', ''): all['status_command'] = 'local' local_leader_status_command = local_exe_status_command( first, 'leader') if all['status_command'].strip().replace('\\', '') == \ local_leader_status_command.strip().replace('\\', ''): all['status_command'] = 'local' local_dummy_status_command = local_exe_status_command( first, 'dummy') if all['status_command'].strip().replace('\\', '') == \ local_dummy_status_command.strip().replace('\\', ''): all['status_command'] = 'local' all['stop_command'] = str(first['stop_command']) default_stop_command = default_exe_stop_command(first) if all['stop_command'].strip().replace('\\', '') == \ default_stop_command.strip().replace('\\', ''): all['stop_command'] = 'default' local_stop_command = local_exe_stop_command(first) if all['stop_command'].strip().replace('\\', '') == \ local_stop_command.strip().replace('\\', ''): all['stop_command'] = 'local' local_leader_stop_command = local_exe_stop_command(first, 'leader') if all['stop_command'].strip().replace('\\', '') == \ local_leader_stop_command.strip().replace('\\', ''): all['stop_command'] = 'local' local_dummy_stop_command = local_exe_stop_command(first, 'dummy') if all['stop_command'].strip().replace('\\', '') == \ local_dummy_stop_command.strip().replace('\\', ''): conf['stop_command'] = 'local' all['clean_command'] = str(first['clean_command']) default_clean_command = default_exe_clean_command(first) if all['clean_command'].strip().replace('\\', '') == \ default_clean_command.strip().replace('\\', ''): all['clean_command'] = 'default' local_clean_command = local_exe_clean_command(first) if all['clean_command'].strip().replace('\\', '') == \ local_clean_command.strip().replace('\\', ''): all['clean_command'] = 'local' local_leader_clean_command = local_exe_clean_command( first, 'leader') if all['clean_command'].strip().replace('\\', '') == \ local_leader_clean_command.strip().replace('\\', ''): all['clean_command'] = 'local' local_dummy_clean_command = local_exe_clean_command(first, 'dummy') if all['clean_command'].strip().replace('\\', '') == \ local_dummy_clean_command.strip().replace('\\', ''): all['clean_command'] = 'local' # Handle old typo gracefully if first.has_key('continuous'): all['continuous'] = first['continuous'] else: all['continuous'] = first['continious'] all['shared_fs'] = first['shared_fs'] # Read in list value all['vgrid'] = first['vgrid'] conf['all_exes'] = all if not conf.get('all_stores', None): all = {} all['storage_disk'] = 10 all['storage_protocol'] = 'sftp' all['storage_port'] = 22 all['storage_user'] = '******' all['storage_dir'] = '/home/miguser/MiG-storage' all['start_command'] = 'default' all['status_command'] = 'default' all['stop_command'] = 'default' all['clean_command'] = 'default' all['shared_fs'] = True all['vgrid'] = [] all['storagenodes'] = '' all['storagehome'] = '' if conf.get('STORECONFIG', None): all['storagenodes'] = generate_storage_node_string( conf['STORECONFIG']) first = conf['STORECONFIG'][0] home = str(first['storage_dir']) base_index = home.find('/MiG/mig_store/') if base_index != -1: home = home[:base_index] all['storagehome'] = home all['storage_disk'] = first['storage_disk'] all['storage_protocol'] = str(first['storage_protocol']) all['storage_port'] = first['storage_port'] all['storage_user'] = str(first['storage_user']) all['storage_dir'] = str(first['storage_dir']) all['start_command'] = str(first['start_command']) default_start_command = default_store_start_command(first) if all['start_command'].strip().replace('\\', '') == \ default_start_command.strip().replace('\\', ''): all['start_command'] = 'default' local_start_command = local_store_start_command(first) if all['start_command'].strip().replace('\\', '') == \ local_start_command.strip().replace('\\', ''): all['start_command'] = 'local' all['status_command'] = str(first['status_command']) default_status_command = default_store_status_command(first) if all['status_command'].strip().replace('\\', '') == \ default_status_command.strip().replace('\\', ''): all['status_command'] = 'default' local_status_command = local_store_status_command(first) if all['status_command'].strip().replace('\\', '') == \ local_status_command.strip().replace('\\', ''): all['status_command'] = 'local' all['stop_command'] = str(first['stop_command']) default_stop_command = default_store_stop_command(first) if all['stop_command'].strip().replace('\\', '') == \ default_stop_command.strip().replace('\\', ''): all['stop_command'] = 'default' local_stop_command = local_store_stop_command(first) if all['stop_command'].strip().replace('\\', '') == \ local_stop_command.strip().replace('\\', ''): all['stop_command'] = 'local' all['clean_command'] = str(conf['STORECONFIG'][0]['clean_command']) default_clean_command = default_store_clean_command(first) if all['clean_command'].strip().replace('\\', '') == \ default_clean_command.strip().replace('\\', ''): all['clean_command'] = 'default' local_clean_command = local_store_clean_command(first) if all['clean_command'].strip().replace('\\', '') == \ local_clean_command.strip().replace('\\', ''): all['clean_command'] = 'local' all['shared_fs'] = first['shared_fs'] # Read in list value all['vgrid'] = first['vgrid'] conf['all_stores'] = all return conf
def create_resource_conf( configuration, client_id, resource_name, resource_identifier, resource_configfile, new_resource=True ): """Create a resource from conf in pending file. If pending_file is a relative path it will prefixed with the resource_pending dir of the client_id. """ if new_resource: msg = """ Trying to create configuration for new resource: '%s.%s' from file '%s': """ % (resource_name, str(resource_identifier), resource_configfile) else: msg = """ Trying to update configuration for existing resource '%s.%s': """ % (resource_name, str(resource_identifier)) client_dir = client_id_dir(client_id) if os.path.isabs(resource_configfile): pending_file = resource_configfile else: pending_file = os.path.join(configuration.resource_pending, client_dir, resource_configfile) tmpfile = pending_file + '.tmp' new_configfile = os.path.join(configuration.resource_home, '%s.%s' % \ (resource_name, resource_identifier), 'config.MiG') if not os.path.exists(pending_file): msg += """ Failure: File: '%s' doesn't exist."""\ % pending_file return (False, msg) (status, conf_msg, config_dict) = get_resource_config_dict(pending_file) if not status: msg += '\n%s' % conf_msg return (False, msg) if config_dict['HOSTURL'] != resource_name: msg += \ """ Failure: resource_name: '%s' does'nt match hosturl: '%s' in configfile: '%s'"""\ % (resource_name, config_dict['HOSTURL'], pending_file) return (False, msg) if not new_resource and \ config_dict['HOSTIDENTIFIER'] != resource_identifier: msg += \ """ Failure: resource_identifier: '%s' does'nt match hostidentifier: '%s' in configfile: '%s'"""\ % (resource_identifier, config_dict['HOSTIDENTIFIER'], pending_file) return (False, msg) try: fr = open(pending_file, 'r') fw = open(tmpfile, 'w') readline = fr.readline() while len(readline) > 0: fw.write(readline.replace('$HOSTIDENTIFIER', str(resource_identifier))) readline = fr.readline() fw.close() fr.close() except Exception, err: msg += \ 'Failed to apply hostidentifier to configfile. Failure: %s'\ % err return (False, msg)
def create_resource_conf(configuration, client_id, resource_name, resource_identifier, resource_configfile, new_resource=True): """Create a resource from conf in pending file. If pending_file is a relative path it will prefixed with the resource_pending dir of the client_id. """ if new_resource: msg = """ Trying to create configuration for new resource: '%s.%s' from file '%s': """ % (resource_name, str(resource_identifier), resource_configfile) else: msg = """ Trying to update configuration for existing resource '%s.%s': """ % (resource_name, str(resource_identifier)) client_dir = client_id_dir(client_id) if os.path.isabs(resource_configfile): pending_file = resource_configfile else: pending_file = os.path.join(configuration.resource_pending, client_dir, resource_configfile) tmpfile = pending_file + '.tmp' new_configfile = os.path.join(configuration.resource_home, '%s.%s' % \ (resource_name, resource_identifier), 'config.MiG') if not os.path.exists(pending_file): msg += """ Failure: File: '%s' doesn't exist."""\ % pending_file return (False, msg) (status, conf_msg, config_dict) = get_resource_config_dict(configuration, pending_file) if not status: msg += '\n%s' % conf_msg return (False, msg) if config_dict['HOSTURL'] != resource_name: msg += \ """ Failure: resource_name: '%s' does'nt match hosturl: '%s' in configfile: '%s'"""\ % (resource_name, config_dict['HOSTURL'], pending_file) return (False, msg) if not new_resource and \ config_dict['HOSTIDENTIFIER'] != resource_identifier: msg += \ """ Failure: resource_identifier: '%s' does'nt match hostidentifier: '%s' in configfile: '%s'"""\ % (resource_identifier, config_dict['HOSTIDENTIFIER'], pending_file) return (False, msg) try: fr = open(pending_file, 'r') fw = open(tmpfile, 'w') readline = fr.readline() while len(readline) > 0: fw.write(readline.replace(keyword_auto, str(resource_identifier))) readline = fr.readline() fw.close() fr.close() except Exception, err: msg += \ 'Failed to apply hostidentifier to configfile. Failure: %s'\ % err return (False, msg)
def init_conf(configuration, hosturl='', hostidentifier=''): """Pull out a full configuration for specified resource with all missing fields set to default values. """ conf = {} if not hosturl: return conf resource_id = '%s.%s' % (hosturl, hostidentifier) conf_file = os.path.join(configuration.resource_home, resource_id, 'config.MiG') if not os.path.isfile(conf_file): return conf (status, msg, conf) = \ get_resource_config_dict(conf_file) if not status: return conf # Fill in frontendhome if not already set if not conf.get('frontendhome', None): conf['frontendhome'] = '' if conf.get('RESOURCEHOME', None): home = conf['RESOURCEHOME'] base_index = home.find('/MiG/mig_frontend/') if base_index != -1: home = home[:base_index] conf['frontendhome'] = home if conf.get('HOSTKEY', None): if conf['HOSTKEY'].split()[1].startswith('ssh-'): conf['HOSTKEY'] = conf['HOSTKEY'].split(None, 1)[1] # Fill in all_X if not already set if not conf.get('all_exes', None): all = {} all['execution_user'] = '' all['nodecount'] = 1 all['cputime'] = 3600 all['execution_precondition'] = ' ' all['prepend_execute'] = 'nice -19' all['exehostlog'] = '' all['joblog'] = '' all['start_command'] = 'default' all['status_command'] = 'default' all['stop_command'] = 'default' all['clean_command'] = 'default' all['continuous'] = True all['shared_fs'] = True all['vgrid_list'] = [] all['executionnodes'] = [] all['executionhome'] = '' if conf.get('EXECONFIG', None): all['executionnodes'] = generate_execution_node_string(conf['EXECONFIG']) first = conf['EXECONFIG'][0] home = str(first['execution_dir']) base_index = home.find('/MiG/mig_exe/') if base_index != -1: home = home[:base_index] all['executionhome'] = home all['execution_user'] = str(first['execution_user']) all['nodecount'] = first['nodecount'] all['cputime'] = first['cputime'] all['execution_precondition'] = str(first['execution_precondition']).strip("'") all['prepend_execute'] = str(first['prepend_execute']).strip('"') all['start_command'] = str(first['start_command']) default_start_command = \ default_exe_start_command(str(first['execution_dir']), str(first['name'])) if all['start_command'].strip().replace('\\', '')\ == default_start_command.strip().replace('\\', ''): all['start_command'] = 'default' local_start_command = \ local_exe_start_command(str(first['execution_dir']), str(first['name'])) if all['start_command'].strip().replace('\\', '')\ == local_start_command.strip().replace('\\', ''): all['start_command'] = 'local' local_leader_start_command = \ local_exe_start_command(str(first['execution_dir']), str(first['name']), 'leader') if all['start_command'].strip().replace('\\', '')\ == local_leader_start_command.strip().replace('\\', ''): all['start_command'] = 'local' local_dummy_start_command = \ local_exe_start_command(str(first['execution_dir']), str(first['name']), 'dummy') if all['start_command'].strip().replace('\\', '')\ == local_dummy_start_command.strip().replace('\\', '' ): all['start_command'] = 'local' all['status_command'] = str(first['status_command']) default_status_command = \ default_exe_status_command(str(first['execution_dir']), str(first['name'])) if all['status_command'].strip().replace('\\', '')\ == default_status_command.strip().replace('\\', ''): all['status_command'] = 'default' local_status_command = \ local_exe_status_command(str(first['execution_dir']), str(first['name'])) if all['status_command'].strip().replace('\\', '')\ == local_status_command.strip().replace('\\', ''): all['status_command'] = 'local' local_leader_status_command = \ local_exe_status_command(str(first['execution_dir']), str(first['name']), 'leader') if all['status_command'].strip().replace('\\', '')\ == local_leader_status_command.strip().replace('\\', '' ): all['status_command'] = 'local' local_dummy_status_command = \ local_exe_status_command(str(first['execution_dir']), str(first['name']), 'dummy') if all['status_command'].strip().replace('\\', '')\ == local_dummy_status_command.strip().replace('\\', '' ): all['status_command'] = 'local' all['stop_command'] = str(first['stop_command' ]) default_stop_command = \ default_exe_stop_command(str(first['execution_dir']), str(first['name'])) if all['stop_command'].strip().replace('\\', '')\ == default_stop_command.strip().replace('\\', ''): all['stop_command'] = 'default' local_stop_command = \ local_exe_stop_command(str(first['execution_dir']), str(first['name'])) if all['stop_command'].strip().replace('\\', '')\ == local_stop_command.strip().replace('\\', ''): all['stop_command'] = 'local' local_leader_stop_command = \ local_exe_stop_command(str(first['execution_dir']), str(first['name']), 'leader') if all['stop_command'].strip().replace('\\', '')\ == local_leader_stop_command.strip().replace('\\', ''): all['stop_command'] = 'local' local_dummy_stop_command = \ local_exe_stop_command(str(first['execution_dir']), str(first['name']), 'dummy') if all['stop_command'].strip().replace('\\', '')\ == local_dummy_stop_command.strip().replace('\\', ''): conf['stop_command'] = 'local' all['clean_command'] = str(first['clean_command']) default_clean_command = \ default_exe_clean_command(str(first['execution_dir']), str(first['name'])) if all['clean_command'].strip().replace('\\', '')\ == default_clean_command.strip().replace('\\', ''): all['clean_command'] = 'default' local_clean_command = \ local_exe_clean_command(str(first['execution_dir']), str(first['name'])) if all['clean_command'].strip().replace('\\', '')\ == local_clean_command.strip().replace('\\', ''): all['clean_command'] = 'local' local_leader_clean_command = \ local_exe_clean_command(str(first['execution_dir']), str(first['name']), 'leader') if all['clean_command'].strip().replace('\\', '')\ == local_leader_clean_command.strip().replace('\\', '' ): all['clean_command'] = 'local' local_dummy_clean_command = \ local_exe_clean_command(str(first['execution_dir']), str(first['name']), 'dummy') if all['clean_command'].strip().replace('\\', '')\ == local_dummy_clean_command.strip().replace('\\', ''): all['clean_command'] = 'local' # Handle old typo gracefully if first.has_key('continuous'): all['continuous'] = first['continuous'] else: all['continuous'] = first['continious'] all['shared_fs'] = first['shared_fs'] # Read in list value all['vgrid'] = first['vgrid'] conf['all_exes'] = all if not conf.get('all_stores', None): all = {} all['storage_disk'] = 10 all['storage_protocol'] = 'sftp' all['storage_port'] = 22 all['storage_user'] = '******' all['storage_dir'] = '/home/miguser/MiG-storage' all['start_command'] = 'default' all['status_command'] = 'default' all['stop_command'] = 'default' all['clean_command'] = 'default' all['shared_fs'] = True all['vgrid'] = [] all['storagenodes'] = '' all['storagehome'] = '' if conf.get('STORECONFIG', None): all['storagenodes'] = generate_storage_node_string(conf['STORECONFIG']) first = conf['STORECONFIG'][0] home = str(first['storage_dir']) base_index = home.find('/MiG/mig_store/') if base_index != -1: home = home[:base_index] all['storagehome'] = home all['storage_disk'] = first['storage_disk'] all['storage_protocol'] = str(first['storage_protocol']) all['storage_port'] = first['storage_port'] all['storage_user'] = str(first['storage_user']) all['storage_dir'] = str(first['storage_dir']) all['start_command'] = str(first['start_command']) default_start_command = \ default_store_start_command(str(first['storage_dir']), str(first['name'])) if all['start_command'].strip().replace('\\', '')\ == default_start_command.strip().replace('\\', ''): all['start_command'] = 'default' local_start_command = \ local_store_start_command(str(first['storage_dir']), str(first['name'])) if all['start_command'].strip().replace('\\', '')\ == local_start_command.strip().replace('\\', ''): all['start_command'] = 'local' all['status_command'] = str(first['status_command']) default_status_command = \ default_store_status_command(str(first['storage_dir']), str(first['name'])) if all['status_command'].strip().replace('\\', '')\ == default_status_command.strip().replace('\\', ''): all['status_command'] = 'default' local_status_command = \ local_store_status_command(str(first['storage_dir']), str(first['name'])) if all['status_command'].strip().replace('\\', '')\ == local_status_command.strip().replace('\\', ''): all['status_command'] = 'local' all['stop_command'] = str(first['stop_command' ]) default_stop_command = \ default_store_stop_command(str(first['storage_dir']), str(first['name'])) if all['stop_command'].strip().replace('\\', '')\ == default_stop_command.strip().replace('\\', ''): all['stop_command'] = 'default' local_stop_command = \ local_store_stop_command(str(first['storage_dir']), str(first['name'])) if all['stop_command'].strip().replace('\\', '')\ == local_stop_command.strip().replace('\\', ''): all['stop_command'] = 'local' all['clean_command'] = str(conf['STORECONFIG' ][0]['clean_command']) default_clean_command = \ default_store_clean_command(str(first['storage_dir']), str(first['name'])) if all['clean_command'].strip().replace('\\', '')\ == default_clean_command.strip().replace('\\', ''): all['clean_command'] = 'default' local_clean_command = \ local_store_clean_command(str(first['storage_dir']), str(first['name'])) if all['clean_command'].strip().replace('\\', '')\ == local_clean_command.strip().replace('\\', ''): all['clean_command'] = 'local' all['shared_fs'] = first['shared_fs'] # Read in list value all['vgrid'] = first['vgrid'] conf['all_stores'] = all return conf