def _load_mapfiles(self):
"""
Load data map file, instrument map file, and sky map file.
Update the 'skip' fields in these map files: if 'skip' is True in any
of the maps, then 'skip' must be set to True in all maps.
"""
self.logger.debug("Loading map files:"
"\n\tdata map: %s\n\tinstrument map: %s\n\tsky map: %s" % (
self.inputs['args'][0],
self.inputs['instrument_mapfile'],
self.inputs['sky_mapfile']
)
)
self.data_map = DataMap.load(self.inputs['args'][0])
self.inst_map = DataMap.load(self.inputs['instrument_mapfile'])
self.sky_map = DataMap.load(self.inputs['sky_mapfile'])
if not validate_data_maps(self.data_map, self.inst_map, self.sky_map):
self.logger.error("Validation of input data mapfiles failed")
return False
# Update the skip fields of the three maps. If 'skip' is True in any of
# these maps, then 'skip' must be set to True in all maps.
for x, y, z in zip(self.data_map, self.inst_map, self.sky_map):
x.skip = y.skip = z.skip = (x.skip or y.skip or z.skip)
return True
def plugin_main(args, **kwargs):
"""
Matchs the hosts in one datamap with those in another
Parameters
----------
mapfile_in : str, optional
Filename of datamap to adjust
mapfile_to_match : str, optional
Filename of datamap to match
"""
mapfile_in = kwargs['mapfile_in']
mapfile_to_match = kwargs['mapfile_to_match']
map_in = DataMap.load(mapfile_in)
map_in.iterator = DataMap.SkipIterator
map_to_match = DataMap.load(mapfile_to_match)
map_to_match.iterator = DataMap.SkipIterator
hosts_to_match = []
for item in map_to_match:
hosts_to_match.append(item.host)
for item, host in zip(map_in, hosts_to_match):
item.host = host
map_in.save(mapfile_in)
def go(self):
super(imager_create_dbs, self).go()
# get assoc_theta, convert from empty string if needed
assoc_theta = self.inputs["assoc_theta"]
if assoc_theta == "":
assoc_theta = None
# Load mapfile data from files
self.logger.info(self.inputs["slice_paths_mapfile"])
slice_paths_map = MultiDataMap.load(self.inputs["slice_paths_mapfile"])
input_map = DataMap.load(self.inputs['args'][0])
source_list_map = DataMap.load(self.inputs['source_list_map_path'])
if self._validate_input_data(input_map, slice_paths_map):
return 1
# Run the nodes with now collected inputs
jobs, output_map = self._run_create_dbs_node(
input_map, slice_paths_map, assoc_theta,
source_list_map)
# Collect the output of the node scripts write to (map) files
return self._collect_and_assign_outputs(jobs, output_map,
slice_paths_map)
def _load_mapfiles(self):
"""
Load data map file, instrument map file, and sky map file.
Update the 'skip' fields in these map files: if 'skip' is True in any
of the maps, then 'skip' must be set to True in all maps.
"""
self.logger.debug(
"Loading map files:"
"\n\tdata map: %s\n\tinstrument map: %s\n\tsky map: %s" %
(self.inputs['args'][0], self.inputs['instrument_mapfile'],
self.inputs['sky_mapfile']))
self.data_map = DataMap.load(self.inputs['args'][0])
self.inst_map = DataMap.load(self.inputs['instrument_mapfile'])
self.sky_map = DataMap.load(self.inputs['sky_mapfile'])
if not validate_data_maps(self.data_map, self.inst_map, self.sky_map):
self.logger.error("Validation of input data mapfiles failed")
return False
# Update the skip fields of the three maps. If 'skip' is True in any of
# these maps, then 'skip' must be set to True in all maps.
for x, y, z in zip(self.data_map, self.inst_map, self.sky_map):
x.skip = y.skip = z.skip = (x.skip or y.skip or z.skip)
return True
def go(self):
super(imager_create_dbs, self).go()
# get assoc_theta, convert from empty string if needed
assoc_theta = self.inputs["assoc_theta"]
if assoc_theta == "":
assoc_theta = None
# Load mapfile data from files
self.logger.info(self.inputs["slice_paths_mapfile"])
slice_paths_map = MultiDataMap.load(self.inputs["slice_paths_mapfile"])
input_map = DataMap.load(self.inputs['args'][0])
source_list_map = DataMap.load(self.inputs['source_list_map_path'])
if self._validate_input_data(input_map, slice_paths_map):
return 1
# Run the nodes with now collected inputs
jobs, output_map = self._run_create_dbs_node(
input_map, slice_paths_map, assoc_theta,
source_list_map)
# Collect the output of the node scripts write to (map) files
return self._collect_and_assign_outputs(jobs, output_map,
slice_paths_map)
def plugin_main(args, **kwargs):
"""
Takes in list of targets and an h5parm solution set and returns a list of stations
in the target data which mismatch the calibrator solutions antenna table
Parameters
----------
mapfile_in : str
Mapfile for input measurement sets
h5parmdb: str
Location of the solution h5parm set
solset_name: str
Name of the solution set of the corresponding h5parm set to compare with
filter: str
Default filter constrains for the ndppp_prep_target step (usually removing International Baselines)
Returns
-------
result : dict
Output station names to filter
"""
mapfile_in = kwargs['mapfile_in']
h5parmdb = kwargs['h5parmdb']
solset_name = kwargs['solset_name']
filter = kwargs['filter']
data = DataMap.load(mapfile_in)
mslist = [data[i].file for i in xrange(len(data))]
#mslist = MSfiles.lstrip('[').rstrip(']').replace(' ','').replace("'","").split(',')
if len(mslist) == 0:
raise ValueError(
"Did not find any existing directory in input MS list!")
pass
else:
MS = mslist[0]
pass
## reading ANTENNA table of MS
antennaFile = MS + "/ANTENNA"
logging.info('Collecting information from the ANTENNA table.')
antennaTable = pt.table(antennaFile, ack=False)
antennaNames = antennaTable.getcol('NAME')
## reading ANTENNA information of h5parm
data = h5parm(h5parmdb, readonly=True)
solset = data.getSolset(solset_name)
station_names = solset.getAnt().keys()
## check whether there are more stations in the target than in the calibrator solutions
missing_stations = list(set(antennaNames) - set(station_names))
for missing_station in missing_stations:
filter += ';!' + missing_station + '*'
pass
## return results
result = {'filter': str(filter)}
return result
pass
def plugin_main(args, **kwargs):
"""
Prunes entries from a mapfile
Parameters
----------
mapfile_in : str
Filename of datamap to trim
prune_str : str
Entries starting with this string will be removed.
Returns
-------
result : dict
New datamap filename
"""
mapfile_in = kwargs['mapfile_in']
prune_str = kwargs['prune_str'].lower()
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
prunelen = len(prune_str)
map_out = DataMap([])
map_in = DataMap.load(mapfile_in)
for i, item in enumerate(map_in):
if item.file[:prunelen].lower() != prune_str:
map_out.data.append(DataProduct(item.host, item.file, item.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Takes in list of targets and an h5parm solution set and returns a list of stations
in the target data which mismatch the calibrator solutions antenna table
Parameters
----------
mapfile_in : str
Mapfile for input measurement sets
filter: str
Default filter constrains for the ndppp_prep_target step (usually removing International Baselines)
Returns
-------
result : dict
Output station names to filter
"""
mapfile_in = kwargs['mapfile_in']
data = DataMap.load(mapfile_in)
mslist = [data[i].file for i in xrange(len(data))]
msfile = mslist[0]
observationTable = pyrap.tables.table(msfile + '::OBSERVATION')
targetName = observationTable.getcol('LOFAR_TARGET')['array'][0]
## return results
result = {'targetName': targetName}
return result
pass
def plugin_main(args, **kwargs):
"""
Appends a string to filenames in a mapfile
Parameters
----------
mapfile_in : str
Filename of datamap to append to
append : str
String to append
append_index : bool
If True, append a unique index to each file
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
New datamap filename
"""
mapfile_in = kwargs['mapfile_in']
if 'append_index' in kwargs:
append_index = kwargs['append_index']
if type(append_index) is str:
if append_index.lower() == 'true':
append_index = True
else:
append_index = False
else:
append_index = False
append_str = kwargs['append']
if append_str == 'None':
append_str = ''
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
map_out = DataMap([])
map_in = DataMap.load(mapfile_in)
for i, item in enumerate(map_in):
if append_index:
map_out.data.append(
DataProduct(item.host,
item.file + append_str + '_{}'.format(i),
item.skip))
else:
map_out.data.append(
DataProduct(item.host, item.file + append_str, item.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Copies each entry of mapfile_in as often as the the length of the corresponding
group into a new mapfile
Parameters
----------
mapfile_in : str
Name of the input mapfile to be expanded. (E.g. with the skymodels for the
different groups.)
mapfile_groups : str
Name of the multi-mapfile with the given groups. Number of groups need
to be the same as the number of files in mapfile_in.
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
ignore_dummies: str (optional)
If true, do not count dummy entries when expanding
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
try:
ignore_dummies = str(kwargs['ignore_dummies']) # if the user has defined a dummy preference, follow it, otherwise count dummies as usual
ignore_dummies = ignore_dummies in ['true', 'True', '1', 'T', 't']
except:
ignore_dummies = False
inmap = DataMap.load(kwargs['mapfile_in'])
groupmap = MultiDataMap.load(kwargs['mapfile_groups'])
if len(inmap) != len(groupmap):
raise ValueError('PipelineStep_mapfileSingleToGroup: length of {0} and {1} differ'.format(kwargs['mapfile_in'],kwargs['mapfile_groups']))
map_out = DataMap([])
inindex = 0
if ignore_dummies:
for groupID in xrange(len(groupmap)):
for fileID in xrange(len(groupmap[groupID].file)):
if (groupmap[groupID].file)[fileID] != 'dummy_entry':
map_out.data.append(DataProduct(inmap[groupID].host, inmap[groupID].file, (inmap[groupID].skip or groupmap[groupID].skip) ))
else:
for groupID in xrange(len(groupmap)):
for fileID in xrange(len(groupmap[groupID].file)):
map_out.data.append(DataProduct(inmap[groupID].host, inmap[groupID].file, (inmap[groupID].skip or groupmap[groupID].skip) ))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
fileid = kwargs['mapfile_in']
datamap = DataMap.load(fileid)
hdf5File = os.path.join(kwargs['hdf5_dir'],kwargs['hdf5file'])
if kwargs.has_key('instrument'):
instrument = kwargs['instrument']
else:
instrument = '/instrument'
if kwargs.has_key('compression'):
compression = int(kwargs['compression'])
else:
compression = 5
if kwargs.has_key('solset'):
solsetName = kwargs['solset']
else:
solsetName = None
# Check is all the necessary files are available
antennaFile = os.path.join(datamap[0].file,'ANTENNA')
if not os.path.isdir(antennaFile):
logging.critical('Missing ANTENNA table.')
sys.exit(1)
fieldFile = os.path.join(datamap[0].file,'FIELD')
if not os.path.isdir(fieldFile):
logging.critical('Missing FIELD table.')
sys.exit(1)
skydbFile = os.path.join(datamap[0].file,'sky')
if not os.path.isdir(skydbFile):
logging.critical('Missing sky table.')
sys.exit(1)
#generate list of parmDB-filenames
parmDBnames = [ MS.file+instrument for MS in datamap ]
#create and fill the hdf5-file:
solset = parmDBs2h5parm(hdf5File, parmDBnames, antennaFile, fieldFile, skydbFile, compression=compression, solsetName=solsetName)
# Add CREATE entry to history
h5parmDB = h5parm(hdf5File, readonly = False)
soltabs = h5parmDB.getSoltabs(solset=solset)
for st in soltabs:
sw = solWriter(soltabs[st])
sw.addHistory('CREATE (by PipelineStep_losotoImporter from %s / %s - %s)' % (os.path.abspath(''),
os.path.basename(parmDBnames[0]), os.path.basename(parmDBnames[-1]) ) )
h5parmDB.close()
#generate mapfile and wrap up
mapfileentry = {}
mapfileentry['host'] = 'localhost'
mapfileentry['file'] = hdf5File
mapfileentry['skip'] = False
outfileid = os.path.join(kwargs['mapfile_dir'], kwargs['filename'])
outmap = open(outfileid, 'w')
outmap.write(repr([mapfileentry]))
outmap.close()
result = {}
result['mapfile'] = outfileid
return result
def plugin_main(args, **kwargs):
"""
Checks a "check" mapfile for values of 'None' and, if found, changes the
input mapfile "file" to "empty".
Note: the check and input mapfiles must have the same length
Parameters
----------
mapfile_in : str
Name of the input mapfile from which to select files.
mapfile_check : str
Name of the mapfile to check for None
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
inmap = DataMap.load(kwargs['mapfile_in'])
checkmap = DataMap.load(kwargs['mapfile_check'])
if len(inmap) != len(checkmap):
raise ValueError('Input and check mapfiles must have the same length')
map_out = DataMap([])
for checkitem, item in zip(checkmap, inmap):
if checkitem.file.lower() == 'none':
map_out.data.append(DataProduct(item.host, 'empty', item.skip))
else:
map_out.append(item)
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def finalize(self):
"""
Finalize this operation
"""
# Add output datamaps to direction object for later reference
self.direction.input_files_single_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'input_files_single.mapfile')
self.direction.verify_subtract_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'verify_subtract.break.mapfile')
self.direction.dir_dep_parmdb_mapfile = os.path.join(
self.pipeline_mapfile_dir,
'merge_normalized_selfcal_parmdbs.mapfile')
self.direction.converted_parmdb_mapfile = os.path.join(
self.pipeline_mapfile_dir,
'convert_normalized_merged_selfcal_parmdbs.mapfile')
self.direction.dir_indep_skymodels_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'full_skymodels.mapfile')
self.direction.selfcal_plots_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'make_selfcal_plots.mapfile')
if self.direction.create_preapply_h5parm:
self.direction.preapply_parmdb_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'create_preapply_h5parm.mapfile')
self.direction.sourcedb_new_facet_sources = os.path.join(
self.pipeline_mapfile_dir,
'make_sourcedb_new_facet_sources_for_facet_imaging.mapfile')
self.direction.diff_models_field_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'predict_and_difference_models.mapfile')
# Store results of verify_subtract check. This will work if the verification
# was done using multiple bands although we use only one at the moment
if (os.path.exists(self.direction.verify_subtract_mapfile) and
not self.parset['calibration_specific']['skip_selfcal_check']):
ok_mapfile = DataMap.load(self.direction.verify_subtract_mapfile)
ok_flags = [ast.literal_eval(item.file) for item in ok_mapfile]
if all(ok_flags):
self.direction.selfcal_ok = True
else:
self.direction.selfcal_ok = False
elif self.parset['calibration_specific']['skip_selfcal_check']:
self.direction.selfcal_ok = True
else:
self.direction.selfcal_ok = False
# Delete temp data
self.direction.cleanup_mapfiles = [
os.path.join(self.pipeline_mapfile_dir, 'shift_cal.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'concat_data.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'apply_dir_dep.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'average_pre.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'average_post.mapfile'),
os.path.join(self.pipeline_mapfile_dir,
'sorted_groups.mapfile_groups')
]
self.log.debug('Cleaning up files (direction: {})'.format(
self.direction.name))
self.direction.cleanup()
self.cleanup()
def plugin_main(args, **kwargs):
"""
Selects those files from mapfile_in that have the same filename-base as the one in
mapfile_reference.
Parameters
----------
mapfile_in : str
Name of the input mapfile from which to select files.
mapfile_reference : str
Name of the reference mapfile
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
inmap = DataMap.load(kwargs['mapfile_in'])
refmap = DataMap.load(kwargs['mapfile_reference'])
map_out = DataMap([])
basenames = [
os.path.splitext(os.path.basename(item.file))[0] for item in inmap
]
for refitem in refmap:
refbase = os.path.splitext(os.path.basename(refitem.file))[0]
idx = basenames.index(refbase)
map_out.append(inmap[idx])
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Appends a string to filenames in a mapfile
Parameters
----------
mapfile_in : str
Filename of datamap to append to
append : str
String to append
append_index : bool
If True, append a unique index to each file
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
New datamap filename
"""
mapfile_in = kwargs['mapfile_in']
if 'append_index' in kwargs:
append_index = kwargs['append_index']
if type(append_index) is str:
if append_index.lower() == 'true':
append_index = True
else:
append_index = False
else:
append_index = False
append_str = kwargs['append']
if append_str == 'None':
append_str = ''
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
map_out = DataMap([])
map_in = DataMap.load(mapfile_in)
for i, item in enumerate(map_in):
if append_index:
map_out.data.append(DataProduct(item.host, item.file+append_str+'_{}'.format(i), item.skip))
else:
map_out.data.append(DataProduct(item.host, item.file+append_str, item.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Makes a mapfile for list of files
Parameters
----------
files : list or str
List of files or mapfile with such a list as the only entry. May be
given as a list of strings or as a string (e.g.,
'[s1.skymodel, s2.skymodel]'
hosts : list or str
List of hosts/nodes. May be given as a list or as a string (e.g.,
'[host1, host2]'
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
if type(kwargs['files']) is str:
try:
# Check if input is mapfile containing list as a string
map_in = DataMap.load(kwargs['files'])
in_files = [item.file for item in map_in]
files = []
for f in in_files:
files += f.strip('[]').split(',')
except:
files = kwargs['files']
files = files.strip('[]').split(',')
files = [f.strip() for f in files]
if type(kwargs['hosts']) is str:
hosts = kwargs['hosts'].strip('[]').split(',')
hosts = [h.strip() for h in hosts]
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
for i in range(len(files)-len(hosts)):
hosts.append(hosts[i])
map_out = DataMap([])
for h, f in zip(hosts, files):
map_out.data.append(DataProduct(h, f, False))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
mapfile_in = kwargs['mapfile_in']
station_filter = kwargs['station_filter']
data = DataMap.load(mapfile_in)
mslist = [data[i].file for i in range(len(data))]
## derive the fraction of flagged data of the entire observation
print('Reading data.')
logging.info('Reading data.')
pool = multiprocessing.Pool(processes=multiprocessing.cpu_count())
flagged_fraction_dict = pool.map(find_flagged_fraction, mslist)
print('Apply station filter ' + str(station_filter))
logging.info('Apply station filter ' + str(station_filter))
flagged_fraction_data = {}
for entry in flagged_fraction_dict:
antennas = entry.keys()
selected_stations = [
station_name for station_name in antennas
if re.match(station_filter, station_name)
]
if len(selected_stations) == 0:
logging.error('No stations left after filtering.')
return (1)
for antenna in selected_stations:
try:
flagged_fraction_data[antenna].append(float(entry[antenna]))
except KeyError:
flagged_fraction_data[antenna] = [float(entry[antenna])]
flagged_fraction_list = []
sorted_stations = sorted(flagged_fraction_data.keys())
for antenna in sorted_stations:
flagged_fraction = sum(flagged_fraction_data[antenna]) / len(
flagged_fraction_data[antenna])
flagged_fraction_list.append(flagged_fraction)
try:
flagged_fraction_data[flagged_fraction].append(antenna)
except KeyError:
flagged_fraction_data[flagged_fraction] = [antenna]
min_flagged_fraction = min(flagged_fraction_list)
refant = flagged_fraction_data[min_flagged_fraction][0]
logging.info('Selected station ' + str(refant) +
' as reference antenna. Fraction of flagged data is ' +
'{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
print('Selected station ' + str(refant) +
' as reference antenna. Fraction of flagged data is ' +
'{:>3}'.format('{:.1f}'.format(min_flagged_fraction) + '%'))
## return results
result = {'refant': str(refant)}
return (result)
def go(self):
# TODO: Remove dependency on mapfile_dir
self.logger.info("Starting copier run")
super(copier, self).go()
# Load data from mapfiles
self.source_map = DataMap.load(self.inputs['mapfile_source'])
self.target_map = DataMap.load(self.inputs['mapfile_target'])
# validate data in mapfiles
if not self._validate_mapfiles(self.inputs['allow_rename']):
return 1
# Run the compute nodes with the node specific mapfiles
for source, target in zip(self.source_map, self.target_map):
args = [source.host, source.file, target.file]
self.append_job(target.host, args)
# start the jobs, return the exit status.
return self.run_jobs()
def _create_mapfile_ato(inmap):
maps = DataMap([])
mapsin = DataMap.load(inmap)
mapsin.iterator = DataMap.SkipIterator
newlist = ''
for i, item in enumerate(mapsin):
newlist = newlist + item.file + ','
newlist = newlist.rstrip(',')
newlist = '[' + newlist + ']'
maps.data.append(DataProduct('localhost', newlist, False))
return maps
def plugin_main(args, **kwargs):
"""
Makes a mapfile by compressing input mapfile items into one item
Parameters
----------
mapfile_in : str
Filename of datamap containing MS files
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
list_format : bool, optional
If True, the compreseed item will use a Python list format (e.g.,
'[file1, file2, ...]'. If False, it will be a space-separated list (e.g.,
'file1 file2 ...'
Returns
-------
result : dict
New parmdb datamap filename
"""
mapfile_in = kwargs['mapfile_in']
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
if 'list_format' in kwargs:
list_format = kwargs['list_format']
else:
list_format = True
if type(list_format) is str:
if list_format.lower() == 'true':
list_format = True
else:
list_format = False
map_in = DataMap.load(mapfile_in)
map_out = DataMap([])
map_in.iterator = DataMap.SkipIterator
file_list = [item.file for item in map_in]
if list_format:
newlist = '[{0}]'.format(','.join(file_list))
else:
newlist = '{0}'.format(' '.join(file_list))
# Just assign host of first file to compressed file
hosts = [item.host for item in map_in]
map_out.data.append(DataProduct(hosts[0], newlist, False))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Makes a mapfile for list of files
Parameters
----------
files : list or str
List of files or mapfile with such a list as the only entry. May be
given as a list of strings or as a string (e.g.,
'[s1.skymodel, s2.skymodel]'
hosts : list or str
List of hosts/nodes. May be given as a list or as a string (e.g.,
'[host1, host2]'
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
if type(kwargs['files']) is str:
try:
# Check if input is mapfile containing list as a string
map_in = DataMap.load(kwargs['files'])
in_files = [item.file for item in map_in]
files = []
for f in in_files:
files += f.strip('[]').split(',')
except:
files = kwargs['files']
files = files.strip('[]').split(',')
files = [f.strip() for f in files]
if type(kwargs['hosts']) is str:
hosts = kwargs['hosts'].strip('[]').split(',')
hosts = [h.strip() for h in hosts]
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
for i in range(len(files) - len(hosts)):
hosts.append(hosts[i])
map_out = DataMap([])
for h, f in zip(hosts, files):
map_out.data.append(DataProduct(h, f, False))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def update_state(dir_input):
"""
Updates the paths in mapfiles or state files
Parameters
----------
dir_input : str
Directory containing files to update
"""
file_list = glob.glob(os.path.join(dir_input, '*'))
if dir_input.endswith('mapfiles'):
# Assume path is a pipeline mapfiles directory. In this case, we can
# simply substitute the new working_dir for the old one in each of the
# mapfiles
working_dir = dir_input.split('results/')[0]
for f in file_list:
map = DataMap.load(f)
for item in map:
if '/' in item.file:
old_working_dir = item.file.split('results/')[0]
item.file = item.file.replace(old_working_dir, working_dir)
map.save(f)
elif dir_input.endswith('state'):
# Assume path is the Factor state directory. In this case, we can try to
# load files as pickled state files and look for paths inside. If found,
# substitute new working_dir for the old one
working_dir = os.path.dirname(dir_input)
for f in file_list:
try:
with open(f, "rb") as fp:
d = pickle.load(fp)
for k, v in d.iteritems():
if type(v) is str:
if k == 'working_dir':
d[k] = working_dir
if '/' in v:
for infix in ['results/', 'state/', 'chunks/']:
parts = v.split(infix)
if len(parts) > 1:
d[k] = os.path.join(working_dir, infix, parts[-1])
elif type(v) is list:
for i, l in enumerate(v):
if '/' in l:
for infix in ['results/', 'state/', 'chunks/']:
parts = l.split(infix)
if len(parts) > 1:
v[i] = os.path.join(working_dir, infix, parts[-1])
d[k] = v
with open(f, "w") as fp:
pickle.dump(d, fp)
except:
pass
def _create_mapfile_ato(inmap):
maps = DataMap([])
mapsin = DataMap.load(inmap)
mapsin.iterator = DataMap.SkipIterator
newlist = ''
for i, item in enumerate(mapsin):
newlist = newlist + item.file + ','
newlist = newlist.rstrip(',')
newlist = '[' + newlist + ']'
maps.data.append(DataProduct('localhost', newlist, False))
return maps
def plugin_main(args, **kwargs):
"""
Selects those files from mapfile_in that have the same filename-base as the one in
mapfile_reference.
Parameters
----------
mapfile_in : str
Name of the input mapfile from which to select files.
mapfile_reference : str
Name of the reference mapfile
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
inmap = DataMap.load(kwargs['mapfile_in'])
refmap = DataMap.load(kwargs['mapfile_reference'])
map_out = DataMap([])
basenames = [ os.path.splitext(os.path.basename(item.file))[0] for item in inmap]
for refitem in refmap:
refbase = os.path.splitext(os.path.basename(refitem.file))[0]
idx = basenames.index(refbase)
map_out.append(inmap[idx])
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Trims a string from filenames in a mapfile
Note that everything from the last instance of the matching string to the
end is trimmed.
Parameters
----------
mapfile_in : str
Filename of datamap to trim
trim_str : str
String to remove
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
counter : int
If counter is greater than 0, replace "image32" with "image42". This is
a special argument for facetselfcal looping only
Returns
-------
result : dict
New datamap filename
"""
mapfile_in = kwargs['mapfile_in']
trim_str = kwargs['trim']
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
if 'counter' in kwargs:
counter = int(kwargs['counter'])
else:
counter = 0
map_out = DataMap([])
map_in = DataMap.load(mapfile_in)
for i, item in enumerate(map_in):
index = item.file.rfind(trim_str)
if index >= 0:
item_trim = item.file[:index]
if counter > 0:
item_trim = item_trim.replace('image32', 'image42')
map_out.data.append(DataProduct(item.host, item_trim,
item.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Takes in mapfile_in, containing many files, and returns only one
Parameters
----------
mapfile_in : str
Parmdbs containing phase solutions
mapfile_dir : str
mapfile directory
filename : str
output filename
mapfile_comp : str
target MSs
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
mapfile_in = kwargs['mapfile_in']
mapfile_comp = kwargs['mapfile_comp']
filename = kwargs['filename']
value = DataMap.load(mapfile_in)[
0] # this the the single mapfile to be expanded
n = len(DataMap.load(mapfile_comp)) # these are actual MS files
map_out = DataMap([])
for i in range(n):
map_out.data.append(DataProduct(value.host, value.file, value.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Makes a mapfile by expanding single input mapfile item into many items
Parameters
----------
mapfile_in : str
Filename of datamap containing single item
mapfile_to_match : str
Filename of datamap containing multiple items
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
New parmdb datamap filename
"""
mapfile_in = kwargs['mapfile_in']
mapfile_to_match = kwargs['mapfile_to_match']
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
map_in = DataMap.load(mapfile_in)
map_match = DataMap.load(mapfile_to_match)
map_out = DataMap([])
map_match.iterator = DataMap.SkipIterator
for item in map_match:
map_out.data.append(DataProduct(item.host, map_in[0].file, item.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Takes in mapfile_in, containing many files, and returns only one
Parameters
----------
mapfile_in : str
Parmdbs containing phase solutions
mapfile_dir : str
mapfile directory
filename : str
output filename
mapfile_comp : str
target MSs
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
mapfile_in = kwargs['mapfile_in']
mapfile_comp = kwargs['mapfile_comp']
filename = kwargs['filename']
value = DataMap.load(mapfile_in)[0] # this the the single mapfile to be expanded
n = len(DataMap.load(mapfile_comp)) # these are actual MS files
map_out = DataMap([])
for i in range(n):
map_out.data.append(DataProduct(value.host,value.file, value.skip ))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def _bbs(self,
timeslice_map_path,
parmdbs_map_path,
sourcedb_map_path,
skip=False):
"""
Perform a calibration step. First with a set of sources from the
gsm and in later iterations also on the found sources
"""
# create parset for bbs run
parset = self.parset.makeSubset("BBS.")
parset_path = self._write_parset_to_file(
parset, "bbs", "Parset for calibration with a local sky model")
# create the output file path
output_mapfile = self._write_datamap_to_file(
None, "bbs_output", "Mapfile with calibrated measurement sets.")
converted_sourcedb_map_path = self._write_datamap_to_file(
None, "source_db", "correctly shaped mapfile for input sourcedbs")
if skip:
return output_mapfile
# The create db step produces a mapfile with a single sourcelist for
# the different timeslices. Generate a mapfile with copies of the
# sourcelist location: This allows validation of maps in combination
# get the original map data
sourcedb_map = DataMap.load(sourcedb_map_path)
parmdbs_map = MultiDataMap.load(parmdbs_map_path)
converted_sourcedb_map = []
# sanity check for correcy output from previous recipes
if not validate_data_maps(sourcedb_map, parmdbs_map):
self.logger.error("The input files for bbs do not contain "
"matching host names for each entry content:")
self.logger.error(repr(sourcedb_map))
self.logger.error(repr(parmdbs_map))
raise PipelineException("Invalid input data for imager_bbs recipe")
self.run_task("imager_bbs",
timeslice_map_path,
parset=parset_path,
instrument_mapfile=parmdbs_map_path,
sourcedb_mapfile=sourcedb_map_path,
mapfile=output_mapfile,
working_directory=self.scratch_directory)
return output_mapfile
def _combine_local_map(inmap):
map_out = DataMap([])
map_in = DataMap.load(inmap)
map_in.iterator = DataMap.SkipIterator
local_files = {}
for item in map_in:
if item.host in local_files:
local_files[item.host] += item.file + ','
else:
local_files[item.host] = item.file + ','
for k, v in local_files.iteritems():
v = v.rstrip(',')
v = '[' + v + ']'
map_out.data.append(DataProduct(k, v, False))
return map_out
def _combine_local_map(inmap):
map_out = DataMap([])
map_in = DataMap.load(inmap)
map_in.iterator = DataMap.SkipIterator
local_files = {}
for item in map_in:
if item.host in local_files:
local_files[item.host] += item.file + ','
else:
local_files[item.host] = item.file + ','
for k, v in local_files.iteritems():
v = v.rstrip(',')
v = '[' + v + ']'
map_out.data.append(DataProduct(k, v, False))
return map_out
def verify_subtract(direction):
"""
Checks selfcal success
"""
verify_subtract_mapfile = os.path.join(direction.working_dir, 'results', 'facetselfcal',
direction.name, 'mapfiles', 'verify_subtract.break.mapfile')
if os.path.exists(verify_subtract_mapfile):
ok_mapfile = DataMap.load(verify_subtract_mapfile)
ok_flags = [ast.literal_eval(item.file) for item in ok_mapfile]
if all(ok_flags):
return True
else:
return False
else:
return False
def plugin_main(args, **kwargs):
"""
Copies each entry of mapfile_in as often as the the length of the corresponding
group into a new mapfile
Parameters
----------
mapfile_in : str
Name of the input mapfile to be expanded. (E.g. with the skymodels for the
different groups.)
mapfile_groups : str
Name of the multi-mapfile with the given groups. Number of groups need
to be the same as the number of files in mapfile_in.
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
inmap = DataMap.load(kwargs['mapfile_in'])
groupmap = MultiDataMap.load(kwargs['mapfile_groups'])
if len(inmap) != len(groupmap):
raise ValueError(
'PipelineStep_mapfileSingleToGroup: length of {0} and {1} differ'.
format(kwargs['mapfile_in'], kwargs['mapfile_groups']))
map_out = DataMap([])
inindex = 0
for groupID in xrange(len(groupmap)):
for fileID in xrange(len(groupmap[groupID].file)):
map_out.data.append(
DataProduct(inmap[groupID].host, inmap[groupID].file,
(inmap[groupID].skip or groupmap[groupID].skip)))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def _bbs(self, timeslice_map_path, parmdbs_map_path, sourcedb_map_path,
skip = False):
"""
Perform a calibration step. First with a set of sources from the
gsm and in later iterations also on the found sources
"""
# create parset for bbs run
parset = self.parset.makeSubset("BBS.")
parset_path = self._write_parset_to_file(parset, "bbs",
"Parset for calibration with a local sky model")
# create the output file path
output_mapfile = self._write_datamap_to_file(None, "bbs_output",
"Mapfile with calibrated measurement sets.")
converted_sourcedb_map_path = self._write_datamap_to_file(None,
"source_db", "correctly shaped mapfile for input sourcedbs")
if skip:
return output_mapfile
# The create db step produces a mapfile with a single sourcelist for
# the different timeslices. Generate a mapfile with copies of the
# sourcelist location: This allows validation of maps in combination
# get the original map data
sourcedb_map = DataMap.load(sourcedb_map_path)
parmdbs_map = MultiDataMap.load(parmdbs_map_path)
converted_sourcedb_map = []
# sanity check for correcy output from previous recipes
if not validate_data_maps(sourcedb_map, parmdbs_map):
self.logger.error("The input files for bbs do not contain "
"matching host names for each entry content:")
self.logger.error(repr(sourcedb_map))
self.logger.error(repr(parmdbs_map))
raise PipelineException("Invalid input data for imager_bbs recipe")
self.run_task("imager_bbs",
timeslice_map_path,
parset = parset_path,
instrument_mapfile = parmdbs_map_path,
sourcedb_mapfile = sourcedb_map_path,
mapfile = output_mapfile,
working_directory = self.scratch_directory)
return output_mapfile
def verify_subtract(direction):
"""
Checks selfcal success
"""
verify_subtract_mapfile = os.path.join(direction.working_dir, 'results',
'facetselfcal', direction.name,
'mapfiles',
'verify_subtract.break.mapfile')
if os.path.exists(verify_subtract_mapfile):
ok_mapfile = DataMap.load(verify_subtract_mapfile)
ok_flags = [ast.literal_eval(item.file) for item in ok_mapfile]
if all(ok_flags):
return True
else:
return False
else:
return False
def plugin_main(args, **kwargs):
"""
Makes a mapfile by uncompressing input mapfile list item into separate items
Parameters
----------
mapfile_in : str
Filename of datamap containing list of MS files
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
hosts : str
List of hosts/nodes. May be given as a list or as a string
(e.g., '[host1, host2]'
Returns
-------
result : dict
New parmdb datamap filename
"""
mapfile_in = kwargs['mapfile_in']
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
if type(kwargs['hosts']) is str:
hosts = kwargs['hosts'].strip('[]').split(',')
hosts = [h.strip() for h in hosts]
map_in = DataMap.load(mapfile_in)
map_out = DataMap([])
files = map_in[0].file.strip('[]').split(',')
files = [f.strip() for f in files]
for i in range(len(files) - len(hosts)):
hosts.append(hosts[i])
for file, host in zip(files, hosts):
map_out.data.append(DataProduct(host, file, False))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Makes a mapfile by uncompressing input mapfile list item into separate items
Parameters
----------
mapfile_in : str
Filename of datamap containing list of MS files
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
hosts : str
List of hosts/nodes. May be given as a list or as a string
(e.g., '[host1, host2]'
Returns
-------
result : dict
New parmdb datamap filename
"""
mapfile_in = kwargs['mapfile_in']
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
if type(kwargs['hosts']) is str:
hosts = kwargs['hosts'].strip('[]').split(',')
hosts = [h.strip() for h in hosts]
map_in = DataMap.load(mapfile_in)
map_out = DataMap([])
files = map_in[0].file.strip('[]').split(',')
files = [f.strip() for f in files]
for i in range(len(files)-len(hosts)):
hosts.append(hosts[i])
for file, host in zip(files, hosts):
map_out.data.append(DataProduct(host, file, False))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def finalize(self):
"""
Finalize this operation
"""
# Add output datamaps to direction object for later reference
self.direction.input_files_single_mapfile = os.path.join(self.pipeline_mapfile_dir,
'input_files_single.mapfile')
self.direction.verify_subtract_mapfile = os.path.join(self.pipeline_mapfile_dir,
'verify_subtract.break.mapfile')
self.direction.dir_dep_parmdb_mapfile = os.path.join(self.pipeline_mapfile_dir,
'merge_normalized_selfcal_parmdbs.mapfile')
self.direction.dir_indep_skymodels_mapfile = os.path.join(self.pipeline_mapfile_dir,
'full_skymodels.mapfile')
self.direction.selfcal_plots_mapfile = os.path.join(self.pipeline_mapfile_dir,
'make_selfcal_plots.mapfile')
# Store results of verify_subtract check. This will work if the verification
# was done using multiple bands although we use only one at the moment
if (os.path.exists(self.direction.verify_subtract_mapfile) and not
self.parset['calibration_specific']['skip_selfcal_check']):
ok_mapfile = DataMap.load(self.direction.verify_subtract_mapfile)
ok_flags = [ast.literal_eval(item.file) for item in ok_mapfile]
if all(ok_flags):
self.direction.selfcal_ok = True
else:
self.direction.selfcal_ok = False
elif self.parset['calibration_specific']['skip_selfcal_check']:
self.direction.selfcal_ok = True
else:
self.direction.selfcal_ok = False
# Delete temp data
self.direction.cleanup_mapfiles = [
os.path.join(self.pipeline_mapfile_dir, 'add_all_facet_sources.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'shift_and_average.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'concat_data.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'concat_blavg_data.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'predict_outlier_model.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'corrupt_outlier_model.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'average_pre.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'average_post.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'sorted_groups.mapfile_groups')]
self.log.debug('Cleaning up files (direction: {})'.format(self.direction.name))
self.direction.cleanup()
def plugin_main(args, **kwargs):
"""
Copies each entry of mapfile_in as often as the the length of the corresponding
group into a new mapfile
Parameters
----------
mapfile_in : str
Name of the input mapfile to be expanded. (E.g. with the skymodels for the
different groups.)
mapfile_groups : str
Name of the multi-mapfile with the given groups. Number of groups need
to be the same as the number of files in mapfile_in.
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
Output datamap filename
"""
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
inmap = DataMap.load(kwargs['mapfile_in'])
groupmap = MultiDataMap.load(kwargs['mapfile_groups'])
if len(inmap) != len(groupmap):
raise ValueError('PipelineStep_mapfileSingleToGroup: length of {0} and {1} differ'.format(kwargs['mapfile_in'],kwargs['mapfile_groups']))
map_out = DataMap([])
inindex = 0
for groupID in xrange(len(groupmap)):
for fileID in xrange(len(groupmap[groupID].file)):
map_out.data.append(DataProduct(inmap[groupID].host, inmap[groupID].file, (inmap[groupID].skip or groupmap[groupID].skip) ))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def finalize(self):
"""
Finalize this operation
"""
# Add output datamaps to direction object for later reference
self.direction.input_files_single_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'input_files_single.mapfile')
self.direction.verify_subtract_mapfile = os.path.join(
self.pipeline_mapfile_dir, 'verify_subtract.break.mapfile')
# Store results of verify_subtract check. This will work if the verification
# was done using multiple bands although we use only one at the moment
if os.path.exists(self.direction.verify_subtract_mapfile
) and not self.parset['skip_selfcal_check']:
ok_mapfile = DataMap.load(self.direction.verify_subtract_mapfile)
ok_flags = [ast.literal_eval(item.file) for item in ok_mapfile]
if all(ok_flags):
self.direction.selfcal_ok = True
else:
self.direction.selfcal_ok = False
elif self.parset['skip_selfcal_check']:
self.direction.selfcal_ok = True
else:
self.direction.selfcal_ok = False
# Delete temp data
self.direction.cleanup_mapfiles = [
os.path.join(self.pipeline_mapfile_dir,
'add_all_facet_sources.mapfile'),
os.path.join(self.pipeline_mapfile_dir,
'shift_and_average.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'concat_data.mapfile'),
os.path.join(self.pipeline_mapfile_dir,
'concat_blavg_data.mapfile'),
os.path.join(self.pipeline_mapfile_dir,
'predict_outlier_model.mapfile'),
os.path.join(self.pipeline_mapfile_dir,
'corrupt_outlier_model.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'average_pre.mapfile'),
os.path.join(self.pipeline_mapfile_dir, 'average_post.mapfile')
]
self.log.debug('Cleaning up files (direction: {})'.format(
self.direction.name))
self.direction.cleanup()
def plugin_main(args, **kwargs):
print 'PLUGIN KWARG: ', kwargs
result = {}
datamap = None
fileid = kwargs['mapfile_in']
datamap = DataMap.load(fileid)
#if kwargs['change_files']:
# for item in datamap:
# item.file = kwargs['change_files']
if kwargs['join_files']:
for item in datamap:
item.file = os.path.join(item.file,kwargs['join_files'])
if kwargs['newname']:
fileid = os.path.join(os.path.dirname(fileid), kwargs['newname'])
if datamap:
print 'Wrinting mapfile: ',fileid
datamap.save(fileid)
result['mapfile'] = fileid
return result
def plugin_main(args, **kwargs):
"""
Makes a mapfile by repeating max size in input mapfile items
Parameters
----------
mapfile_in : str
Filename of datamap containing MS files
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
New datamap filename
"""
mapfile_in = kwargs['mapfile_in']
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
map_in = DataMap.load(mapfile_in)
map_out = DataMap([])
# Find max size in x and y
xsize_list = []
ysize_list = []
for item in map_in:
xsize, ysize = [int(s) for s in item.file.split(' ')]
xsize_list.append(xsize)
ysize_list.append(ysize)
maxsize = '{0} {1}'.format(max(xsize_list), max(ysize_list))
for item in map_in:
map_out.data.append(DataProduct(item.host, maxsize, item.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
print 'PLUGIN KWARG: ', kwargs
result = {}
datamap = None
fileid = kwargs['mapfile_in']
datamap = DataMap.load(fileid)
#if kwargs['change_files']:
# for item in datamap:
# item.file = kwargs['change_files']
if kwargs['join_files']:
for item in datamap:
item.file = os.path.join(item.file, kwargs['join_files'])
if kwargs['newname']:
fileid = os.path.join(os.path.dirname(fileid), kwargs['newname'])
if datamap:
print 'Wrinting mapfile: ', fileid
datamap.save(fileid)
result['mapfile'] = fileid
return result
def plugin_main(args, **kwargs):
"""
Takes in mapfiles and change host names to allow for efficient MPI reduction
Parameters
----------
mapfiles : list of strs
List of the names of the input mapfiles. WILL BE MODIFIED!
mapfile_dir : str
Name of the directory containing the mapfile
head_node_only : str
String: Either True or False. Describes whether to use just the head node or not.
Returns
-------
result : empty dictionary
"""
result = {}
mapfiles = (kwargs['mapfiles'][1:-1]).split(',') # read in list of mapfiles from string (separated by commas)
mapfile_dir = kwargs['mapfile_dir']
head_node_only = (kwargs['head_node_only'] in ['True','true','T','t','1'])
fn_list=[]
for mf in mapfiles:
fn_list.append( os.path.join(mapfile_dir,mf) )
# caution: remember to reload the compute node iterable for every mapfile to ensure corresponding entries have the same node set as host
for fn in fn_list:
if(head_node_only):
cn_cycle = it.cycle( get_head_node( ClusterDesc(str(os.environ['cluster_desc_file'])) ) ) # Read in head node. Set up iterator (unnessary with just one node, but better to have less code!)
else:
cn_cycle = it.cycle( get_compute_nodes( ClusterDesc(str(os.environ['cluster_desc_file'])) ) ) # Read in list of compute nodes. Set up iterator to cyclically iterate over them.
data = DataMap.load(fn) # read in current data map file (probably with all host values set to "localhost")
iterator = DataMap.SkipIterator(data) # set up iterator for all values in mapfile
for value in iterator:
value.host = cn_cycle.next() # iterate through map file, assigning each entry a host from the available compute nodes in a cyclical fashion
data.save(fn) # overwrite original file
return result
def plugin_main(args, **kwargs):
"""
Updates the hosts in an input datamap
Parameters
----------
mapfile_in : str, optional
Filename of datamap
mapfile_dir: str, optional
Directory containing mapfiles. All mapfiles in this directory will be
updated
hosts : str
List of hosts/nodes. May be given as a list or as a string
(e.g., '[host1, host2]'
"""
if 'mapfile_dir' in kwargs:
mapfiles_in = glob.glob(os.path.join(kwargs['mapfile_dir'], '*'))
else:
mapfiles_in = [kwargs['mapfile_in']]
if len(mapfiles_in) == 0:
return
if type(kwargs['hosts']) is str:
hosts = kwargs['hosts'].strip('[]').split(',')
hosts = [h.strip() for h in hosts]
for mapfile_in in mapfiles_in:
try:
map = DataMap.load(mapfile_in)
for i in range(len(map) - len(hosts)):
hosts.append(hosts[i])
for item, host in zip(map, hosts):
item.host = host
map.save(mapfile_in)
except:
print(
'File {} does not appear to be a mapfile. Skipping it.'.format(
mapfile_in))
def input2strlist(invar):
str_list = None
if type(invar) is str:
if invar.startswith('[') and invar.endswith(']'):
str_list = [f.strip(' \'\"') for f in invar.strip('[]').split(',')]
else:
map_in = DataMap.load(invar)
map_in.iterator = DataMap.SkipIterator
str_list = []
for fname in map_in:
if fname.startswith('[') and fname.endswith(']'):
for f in fname.strip('[]').split(','):
str_list.append(f.strip(' \'\"'))
else:
str_list.append(fname.strip(' \'\"'))
elif type(invar) is list:
str_list = [str(f).strip(' \'\"') for f in invar]
else:
raise TypeError('input2strlist: Type '+str(type(invar))+' unknown!')
return str_list
def cleanup(self):
"""
Cleans up unneeded data
"""
from lofarpipe.support.data_map import DataMap
for mapfile in self.cleanup_mapfiles:
try:
datamap = DataMap.load(mapfile)
for item in datamap:
# Handle case in which item.file is a Python list
if item.file[0] == '[' and item.file[-1] == ']':
files = item.file.strip('[]').split(',')
else:
files = [item.file]
for f in files:
if os.path.exists(f):
os.system('rm -rf {0}'.format(f))
except IOError:
pass
def plugin_main(args, **kwargs):
"""
Makes a mapfile by filtering input mapfile items into one item (the middle
one)
Parameters
----------
mapfile_in : str
Filename of datamap containing MS files
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
New parmdb datamap filename
"""
mapfile_in = kwargs['mapfile_in']
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
map_in = DataMap.load(mapfile_in)
map_out = DataMap([])
map_in.iterator = DataMap.SkipIterator
files = [item.file for item in map_in]
hosts = [item.host for item in map_in]
if 'index' in kwargs:
index = int(kwargs['index'])
else:
index = len(files)/2
map_out.data.append(DataProduct(hosts[index], files[index], False))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def plugin_main(args, **kwargs):
"""
Updates the hosts in an input datamap
Parameters
----------
mapfile_in : str, optional
Filename of datamap
mapfile_dir: str, optional
Directory containing mapfiles. All mapfiles in this directory will be
updated
hosts : str
List of hosts/nodes. May be given as a list or as a string
(e.g., '[host1, host2]'
"""
if 'mapfile_dir' in kwargs:
mapfiles_in = glob.glob(os.path.join(kwargs['mapfile_dir'], '*'))
else:
mapfiles_in = [kwargs['mapfile_in']]
if len(mapfiles_in) == 0:
return
if type(kwargs['hosts']) is str:
hosts = kwargs['hosts'].strip('[]').split(',')
hosts = [h.strip() for h in hosts]
for mapfile_in in mapfiles_in:
try:
map = DataMap.load(mapfile_in)
for i in range(len(map)-len(hosts)):
hosts.append(hosts[i])
for item, host in zip(map, hosts):
item.host = host
map.save(mapfile_in)
except:
print('File {} does not appear to be a mapfile. Skipping it.'.format(mapfile_in))
def plugin_main(args, **kwargs):
"""
Appends a string to filenames in a mapfile
Parameters
----------
mapfile_in : str
Filename of datamap to append to
append_str : str
String to append
mapfile_dir : str
Directory for output mapfile
filename: str
Name of output mapfile
Returns
-------
result : dict
New datamap filename
"""
mapfile_in = kwargs['mapfile_in']
append_str = kwargs['append']
if append_str == 'None':
append_str = ''
mapfile_dir = kwargs['mapfile_dir']
filename = kwargs['filename']
map_out = DataMap([])
map_in = DataMap.load(mapfile_in)
for i, item in enumerate(map_in):
map_out.data.append(DataProduct(item.host, item.file+append_str, item.skip))
fileid = os.path.join(mapfile_dir, filename)
map_out.save(fileid)
result = {'mapfile': fileid}
return result
def check_existing_files(mapfile):
"""
Checks if files in input mapfile exist
Parameters
----------
mapfile : str
Filename of mapfile to check
Returns
-------
file : list
List of files
"""
all_exist = True
all_files = []
log.info('Checking for existing files...')
try:
datamap = DataMap.load(mapfile)
for item in datamap:
# Handle case in which item.file is a Python list
if item.file[0] == '[' and item.file[-1] == ']':
files = item.file.strip('[]').split(',')
else:
files = [item.file]
for f in files:
if not os.path.exists(f):
all_exist = False
all_files.extend(files)
if all_exist:
log.info('...all files exist')
else:
log.warning('...one or more files not found')
return all_files
except IOError:
return []
def check_existing_files(self, mapfile):
"""
Checks if files in input mapfile exist
Parameters
----------
mapfile : str
Filename of mapfile to check
Returns
-------
all_exist : bool
True if all files in mapfile exist, False if not
"""
from lofarpipe.support.data_map import DataMap
all_exist = True
self.log.debug('Checking for existing files...')
try:
datamap = DataMap.load(mapfile)
for item in datamap:
# Handle case in which item.file is a Python list
if item.file[0] == '[' and item.file[-1] == ']':
files = item.file.strip('[]').split(',')
else:
files = [item.file]
for f in files:
if not os.path.exists(f):
all_exist = False
if all_exist:
self.log.debug('...all files exist')
else:
self.log.debug('...one or more files not found')
return all_exist
except IOError:
self.log.debug('Could not read mapfile {}. Skipping it'.format(mapfile))
return False
