def plugin_main(args, **kwargs):
"""
Takes in mapfiles and change host names to allow for efficient MPI reduction
Parameters
----------
mapfiles : list of strs
List of the names of the input mapfiles. WILL BE MODIFIED!
mapfile_dir : str
Name of the directory containing the mapfile
head_node_only : str
String: Either True or False. Describes whether to use just the head node or not.
Returns
-------
result : empty dictionary
"""
result = {}
mapfiles = (kwargs['mapfiles'][1:-1]).split(',') # read in list of mapfiles from string (separated by commas)
mapfile_dir = kwargs['mapfile_dir']
head_node_only = (kwargs['head_node_only'] in ['True','true','T','t','1'])
fn_list=[]
for mf in mapfiles:
fn_list.append( os.path.join(mapfile_dir,mf) )
# caution: remember to reload the compute node iterable for every mapfile to ensure corresponding entries have the same node set as host
for fn in fn_list:
if(head_node_only):
cn_cycle = it.cycle( get_head_node( ClusterDesc(str(os.environ['cluster_desc_file'])) ) ) # Read in head node. Set up iterator (unnessary with just one node, but better to have less code!)
else:
cn_cycle = it.cycle( get_compute_nodes( ClusterDesc(str(os.environ['cluster_desc_file'])) ) ) # Read in list of compute nodes. Set up iterator to cyclically iterate over them.
data = DataMap.load(fn) # read in current data map file (probably with all host values set to "localhost")
iterator = DataMap.SkipIterator(data) # set up iterator for all values in mapfile
for value in iterator:
value.host = cn_cycle.next() # iterate through map file, assigning each entry a host from the available compute nodes in a cyclical fashion
data.save(fn) # overwrite original file
return result
def plugin_main(args, **kwargs):
"""
Find the measurement set closest to a given solution table, suitable for reading station names.
Parameters
----------
mapfile_ms : str
Mapfile of the measurement sets
mapfile_grpd : str
Mapfile of the (grouped) calibration tables
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
mapfile_in = kwargs['mapfile_ms']
mapfile_grpd = kwargs['mapfile_grpd']
filename = kwargs['filename']
data = DataMap.load(mapfile_in) # these are actual MS files
groups = DataMap.load(mapfile_grpd) # these are probably parmdbs
name_stem = ((data[0].file.split('SB'))[0]) + 'SB'
name_end = '_'.join(((data[0].file.split('SB'))[1]).split('_'))[3:]
iterator = DataMap.SkipIterator(data)
num_list = []
for value in iterator:
name = value.file
name = (name.split('SB'))[1]
name = (name.split('_'))[0]
name = (name.split('.'))[0]
num_list.append(int(name))
iterator = DataMap.SkipIterator(groups)
map_out = DataMap([])
for value in iterator:
name = value.file
name = (name.split('SB'))[1]
name = (name.split('_'))[0]
name = (name.split('.'))[0]
if( len(name.split('gr')) > 1 ):
name = (name.split('gr'))[-1] # in case of grouping!
num = int((name.split('-'))[0]) * int((name.split('-'))[1])
else:
num = int(name)
if num in num_list:
map_out.data.append(DataProduct(value.host, name_stem + str(num).zfill(3) + name_end, value.skip ))
else:
num = find_nearest(num_list, num) # if not an exact match, use the nearest
map_out.data.append(DataProduct(value.host, name_stem + str(num).zfill(3) + name_end, value.skip ))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Takes in mapfile_in, containing many files, and returns only one
Parameters
----------
mapfile_in : str
Parmdbs containing phase solutions
mapfile_dir : str
mapfile directory
filename : str
output filename
n: str
Size to expand to
mapfile_comp : str
target MSs
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
mapfile_in = kwargs['mapfile_in']
mapfile_comp = kwargs['mapfile_comp']
filename = kwargs['filename']
n = int(kwargs['n_expand'])
tables_data = DataMap.load(
mapfile_in) # these are the phase solution tables
ms_data = DataMap.load(mapfile_comp) # these are actual MS files
iterator = DataMap.SkipIterator(ms_data)
ms_list = []
for value in iterator:
name = value.file
name = (name.split('SB'))[1]
name = name[0:3]
ms_list.append(int(name))
iterator = DataMap.SkipIterator(tables_data)
sol_list = []
for value in iterator:
name = value.file
try:
name = (name.split('SBgr'))[1] # try for SB groups first
except:
name = (name.split('SB'))[1]
name = name[0:3]
sol_list.append(int(name))
ms_list = np.array(ms_list)
sol_list = np.array(sol_list)
if ((len(ms_list) == 0) or (len(sol_list) == 0)):
print('ERROR: ExpandSingle: 0 solution files or MS files detected.')
print('ERROR: ExpandSingle: # MS files = ' + str(len(ms_list)))
print('ERROR: ExpandSingle: # Solution files = ' + str(len(sol_list)))
return (1)
diff_list = np.diff(sol_list)
if (len(diff_list) > 0):
n_internal = np.min()
if (n_internal != n):
print(
'WARNING: Internal estimate of n differs from user-provided value.'
)
print('WARNING: Internal estimate will be used.')
else:
n_internal = n
# typically the sol name
map_out = DataMap([])
nsol = 0
nms = 0
iterator = DataMap.SkipIterator(tables_data)
for value in iterator:
for i in range(n_internal):
if (ms_list[nms] - sol_list[nsol] > n_internal):
print('Error in expanding phase calibration mapfile.')
return (1)
else:
map_out.data.append(
DataProduct(value.host, value.file, value.skip))
nms = nms + 1
if (nms == len(ms_list)):
break
nsol = nsol + 1
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
