def print_HAcov(self, png=None): """ some info on the MSs """ telescope = self.mssListObj[0].getTelescope() if telescope == 'LOFAR': telescope_coords = EarthLocation(lat=52.90889*u.deg, lon=6.86889*u.deg, height=0*u.m) elif telescope == 'GMRT': telescope_coords = EarthLocation(lat=19.0948*u.deg, lon=74.0493*u.deg, height=0*u.m) else: raise('Unknown Telescope.') has = []; elevs = [] for ms in self.mssListObj: time = np.mean(ms.getTimeRange()) time = Time( time/86400, format='mjd') time.delta_ut1_utc = 0. # no need to download precise table for leap seconds logger.info('%s (%s): Hour angle: %.1f hrs - Elev: %.2f (Sun distance: %.0f)' % (ms.nameMS,time.iso,ms.ha.deg/15.,ms.elev.deg,ms.sun_dist.deg)) has.append(ms.ha.deg/15.) elevs.append(ms.elev.deg) if png is not None: import matplotlib.pyplot as pl pl.figure(figsize=(6,6)) ax1 = pl.gca() ax1.plot(has, elevs, 'ko') ax1.set_xlabel('HA [hrs]') ax1.set_ylabel('elevs [deg]') logger.debug('Save plot: %s' % png) pl.savefig(png)
def blank_image_fits(filename, maskname, outfile = None, inverse = False, blankval = 0.): """ Set to "blankval" all the pixels inside the given region if inverse=True, set to "blankval" pixels outside region. filename: fits file region: ds9 region outfile: output name inverse: reverse region mask blankval: pixel value to set """ if (outfile == None): outfile = filename with pyfits.open(maskname) as fits: mask = fits[0].data if (inverse): mask = ~(mask.astype(bool)) with pyfits.open(filename) as fits: data = fits[0].data assert mask.shape == data.shape # mask and data should be same shape sum_before = np.sum(data) data[mask] = blankval logger.debug("%s: Blanking (%s): sum of values: %f -> %f" % (filename, maskname, sum_before, np.sum(data))) fits.writeto(outfile, overwrite=True)
def makeBeamReg(self, outfile, pb_cut=None, to_null=False, freq='mid'): """ Create a ds9 region of the beam outfile : str output file pb_cut : float, optional diameter of the beam to_null : bool, optional arrive to the first null, not the FWHM freq: min,max,med which frequency to use to estimate the beam size """ logger.debug('Making PB region: '+outfile) ra, dec = self.getPhaseCentre() if pb_cut is None: radius = self.getFWHM(freq=freq)/2. else: radius = pb_cut/2. if to_null: radius *= 2 # rough estimation s = Shape('circle', None) s.coord_format = 'fk5' s.coord_list = [ ra, dec, radius ] # ra, dec, radius s.coord_format = 'fk5' s.attr = ([], {'width': '2', 'point': 'cross', 'font': '"helvetica 16 normal roman"'}) s.comment = 'color=red text="beam"' regions = pyregion.ShapeList([s]) lib_util.check_rm(outfile) regions.write(outfile)
def run_losoto(s, c, h5s, parsets, plots_dir=None) -> object: """ s : scheduler c : cycle name, e.g. "final" h5s : lists of H5parm files or string of 1 h5parm parsets : lists of parsets to execute """ logger.info("Running LoSoTo...") h5out = 'cal-' + c + '.h5' if type(h5s) is str: h5s = [h5s] # convert from killMS for i, h5 in enumerate(h5s): if h5[-3:] == 'npz': newh5 = h5.replace('.npz', '.h5') s.add('killMS2H5parm.py -V --nofulljones %s %s ' % (newh5, h5), log='losoto-' + c + '.log', commandType="python", processors='max') s.run(check=True) h5s[i] = newh5 # concat/move if len(h5s) > 1: check_rm(h5out) s.add('H5parm_collector.py -V -s sol000 -o ' + h5out + ' ' + ' '.join(h5s), log='losoto-' + c + '.log', commandType="python", processors='max') s.run(check=True) else: os.system('cp -r %s %s' % (h5s[0], h5out)) check_rm('plots') os.makedirs('plots') for parset in parsets: logger.debug('-- executing ' + parset + '...') s.add('losoto -V ' + h5out + ' ' + parset, log='losoto-' + c + '.log', logAppend=True, commandType="python", processors='max') s.run(check=True) if plots_dir is None: check_rm('plots-' + c) os.system('mv plots plots-' + c) else: if not os.path.exists(plots_dir): os.system('mkdir ' + plots_dir) os.system('mv plots/* ' + plots_dir) check_rm('plots')
def getNchan(self): """ Find number of channels """ with tables.table(self.pathMS + "/SPECTRAL_WINDOW", ack=False) as t: nchan = t.getcol("NUM_CHAN") assert (nchan[0] == nchan).all() # all SpWs have same channels? logger.debug("%s: channel number: %i", self.pathMS, nchan[0]) return nchan[0]
def getTimeInt(self): """ Get time interval in seconds """ with tables.table(self.pathMS, ack=False) as t: nTimes = len(set(t.getcol("TIME"))) t_init, t_end = self.getTimeRange() deltaT = (t_end - t_init) / nTimes logger.debug("%s: time interval (seconds): %f", self.pathMS, deltaT) return deltaT
def getChanband(self): """ Find bandwidth of a channel in Hz """ with tables.table(self.pathMS + "/SPECTRAL_WINDOW", ack=False) as t: chan_w = t.getcol("CHAN_WIDTH")[0] assert all(x == chan_w[0] for x in chan_w) # all chans have same width logger.debug("%s: channel width (MHz): %f", self.pathMS, chan_w[0] / 1.e6) return chan_w[0]
def add(self, cmd='', log='', logAppend=True, commandType='', processors=None): """ Add a command to the scheduler list cmd: the command to run log: log file name that can be checked at the end logAppend: if True append, otherwise replace commandType: can be a list of known command types as "BBS", "DPPP", ... processors: number of processors to use, can be "max" to automatically use max number of processors per node """ if (log != ''): log = self.log_dir + '/' + log if (logAppend): cmd += " >> " else: cmd += " > " cmd += log + " 2>&1" # if running wsclean add the string if commandType == 'wsclean': logger.debug('Running wsclean: %s' % cmd) elif commandType == 'DPPP': logger.debug('Running DPPP: %s' % cmd) if (processors != None and processors == 'max'): processors = self.max_processors if self.qsub: # if number of processors not specified, try to find automatically if (processors == None): processors = 1 # default use single CPU if ("calibrate-stand-alone" == cmd[:21]): processors = 1 if ("DPPP" == cmd[:5]): processors = 1 if ("wsclean" == cmd[:7]): processors = self.max_processors if ("awimager" == cmd[:8]): processors = self.max_processors if (processors > self.max_processors): processors = self.max_processors self.action_list.append([str(processors), '\'' + cmd + '\'']) else: self.action_list.append(cmd) if (log != ""): self.log_list.append((log, commandType))
def __exit__(self, type, value, traceback): """ Catch "Skip" errors, if not skipped, write to file after exited without exceptions. """ if type is None: with open(self.filename, "a") as f: f.write(self.__step__ + '\n') logger.debug('Done: {}'.format(self.__step__)) return # No exception if issubclass(type, Skip): logger.warning('SKIP: {}'.format(self.__step__)) return True # Suppress special SkipWithBlock exception
def move(self, pathMSNew, overwrite=False, keepOrig=False): """ Move (or rename) the MS to another locus in the file system. """ logger.debug('Move: '+self.pathMS+' -> '+pathMSNew) if overwrite == True: lib_util.check_rm(pathMSNew) if not os.path.exists(pathMSNew): if keepOrig: shutil.copytree(self.pathMS, pathMSNew) else: shutil.move(self.pathMS, pathMSNew) self.setPathVariables(pathMSNew)
def blank_image_reg(filename, region, outfile=None, inverse=False, blankval=0., op="AND"): """ Set to "blankval" all the pixels inside the given region if inverse=True, set to "blankval" pixels outside region. If a list of region is provided the operation is applied to each region one after the other filename: fits file region: ds9 region or list of regions outfile: output name inverse: reverse final *combined* mask blankval: pixel value to set op: how to combine multiple regions with AND or OR """ if outfile == None: outfile = filename if not type(region) is list: region = [region] # open fits with pyfits.open(filename) as fits: origshape = fits[0].data.shape header, data = flatten(fits) sum_before = np.sum(data) if (op == 'AND'): total_mask = np.ones(shape=data.shape).astype(bool) if (op == 'OR'): total_mask = np.zeros(shape=data.shape).astype(bool) for this_region in region: # extract mask r = pyregion.open(this_region) mask = r.get_mask(header=header, shape=data.shape) if (op == 'AND'): total_mask = total_mask & mask if (op == 'OR'): total_mask = total_mask | mask if (inverse): total_mask = ~total_mask data[total_mask] = blankval # save fits fits[0].data = data.reshape(origshape) fits.writeto(outfile, overwrite=True) logger.debug("%s: Blanking (%s): sum of values: %f -> %f" % (filename, region, sum_before, np.sum(data)))
def __init__(self, coords, fluxes, kernel_size=0.2, look_distance=0.3, grouping_distance=0.03): """ coords: x,y coordinates for source positions fluxes: total flux for each source kernel_size: attenuate attraction, it this the flux times a gaussian of the distance with this as sigma [deg] look_distance: max distance to look for nearby sources [deg] grouping_distance: [deg] """ self.coords = np.array(coords) self.fluxes = fluxes self.kernel_size = kernel_size # deg self.look_distance = look_distance # deg self.grouping_distance = grouping_distance # deg orig: 0.01 self.past_coords = [np.copy(self.coords)] self.n_iterations = 100 self.clusters = [] logger.debug("Grouper: kernel_size=%.1f; look_distance=%.1f; grouping_distance=%.2f" % (kernel_size,look_distance,grouping_distance) )
def add(self, cmd='', log='', logAppend=True, commandType='', processors=None): """ Add a command to the scheduler list cmd: the command to run log: log file name that can be checked at the end logAppend: if True append, otherwise replace commandType: can be a list of known command types as "BBS", "DPPP", ... processors: number of processors to use, can be "max" to automatically use max number of processors per node """ if (log != ''): log = self.log_dir + '/' + log if (logAppend): cmd += " >> " else: cmd += " > " cmd += log + " 2>&1" # if running wsclean add the string if commandType == 'wsclean': logger.debug('Running wsclean: %s' % cmd) elif commandType == 'DPPP': logger.debug('Running DPPP: %s' % cmd) elif commandType == 'singularity': cmd = 'SINGULARITY_TMPDIR=/dev/shm singularity exec -B /tmp,/dev/shm,/localwork,/localwork.ssd,/home /home/fdg/node31/opt/src/lofar_sksp_ddf.simg ' + cmd logger.debug('Running singularity: %s' % cmd) elif commandType == 'python': logger.debug('Running python: %s' % cmd) if (processors != None and processors == 'max'): processors = self.max_processors if self.qsub: # if number of processors not specified, try to find automatically if (processors == None): processors = 1 # default use single CPU if ("DPPP" == cmd[:4]): processors = 1 if ("wsclean" == cmd[:7]): processors = self.max_processors if (processors > self.max_processors): processors = self.max_processors self.action_list.append([str(processors), '\'' + cmd + '\'']) else: self.action_list.append(cmd) if (log != ""): self.log_list.append((log, commandType))
def run_losoto(s, c, h5s, parsets): """ s : scheduler c : cycle name, e.g. "final" h5s : lists of H5parm files or string of 1 h5parm parsets : lists of parsets to execute """ logger.info("Running LoSoTo...") h5 = 'cal-' + c + '.h5' if type(h5s) is str: h5s = [h5s] # concat/move if len(h5s) > 1: check_rm("cal-" + c + ".h5") s.add('H5parm_collector.py -V -s sol000 -o ' + h5 + ' ' + ' '.join(h5s), log='losoto-' + c + '.log', commandType="python", processors='max') s.run(check=True) else: os.system('cp -r %s %s' % (h5s[0], h5)) check_rm('plots') os.makedirs('plots') for parset in parsets: logger.debug('-- executing ' + parset + '...') s.add('losoto -V ' + h5 + ' ' + parset, log='losoto-' + c + '.log', logAppend=True, commandType="python", processors='max') s.run(check=True) check_rm('plots-' + c) os.system('mv plots plots-' + c)
def columnAddSimilar(pathMS, columnNameNew, columnNameSimilar, dataManagerInfoNameNew, overwrite = False, fillWithOnes = True, comment = "", verbose = False): # more to lib_ms """ Add a column to a MS that is similar to a pre-existing column (in shape, but not in values). pathMS: path of the MS columnNameNew: name of the column to be added columnNameSimilar: name of the column from which properties are copied (e.g. "DATA") dataManagerInfoNameNew: string value for the data manager info (DMI) keyword "NAME" (should be unique in the MS) overwrite: whether or not to overwrite column 'columnNameNew' if it already exists fillWithOnes: whether or not to fill the newly-made column with ones verbose: whether or not to produce abundant output """ t = tables.table(pathMS, readonly = False) if (columnExists(t, columnNameNew) and not overwrite): logger.warning("Attempt to add column '" + columnNameNew + "' aborted, as it already exists and 'overwrite = False' in columnAddSimilar(...).") else: # Either the column does not exist yet, or it does but overwriting is allowed. # Remove column if necessary. if (columnExists(t, columnNameNew)): logger.info("Removing column '" + columnNameNew + "'...") t.removecols(columnNameNew) # Add column. columnDescription = t.getcoldesc(columnNameSimilar) dataManagerInfo = t.getdminfo(columnNameSimilar) if (verbose): logger.debug("columnDescription:") logger.debug(columnDescription) logger.debug("dataManagerInfo:") logger.debug(dataManagerInfo) columnDescription["comment"] = "" # What about adding something here like: #columnDescription["dataManagerGroup"] = ...? dataManagerInfo["NAME"] = dataManagerInfoNameNew if (verbose): logger.debug("columnDescription (updated):") logger.debug(columnDescription) logger.debug("dataManagerInfo (updated):") logger.debug(dataManagerInfo) logger.info("Adding column '" + columnNameNew + "'...") t.addcols(tables.makecoldesc(columnNameNew, columnDescription), dataManagerInfo) # Fill with ones if desired. if (fillWithOnes): logger.info("Filling column '" + columnNameNew + "' with ones...") columnDataSimilar = t.getcol(columnNameSimilar) t.putcol(columnNameNew, np.ones_like(columnDataSimilar)) # Close the table to avoid that it is locked for further use. t.close()
def make_voronoi_reg(directions, fitsfile, outdir_reg='regions', out_mask='facet.fits', png=None): """ Take a list of coordinates and an image and voronoi tesselate the sky. It saves ds9 regions + fits mask of the facets directions : array of Direction objects firsfile : mask fits file to tassellate (used for coordinates and as template for the out_mask) outdir_reg : dir where to save regions out_mask : output mask with different numbers in each facet png : output png file that shows the tassellation """ def closest_node(node, nodes): """ Return closest values to node from nodes """ nodes = np.asarray(nodes) dist_2 = np.sum((nodes - node)**2, axis=1) return np.argmin(dist_2) logger.debug("Image used for tasselation reference: " + fitsfile) fits = pyfits.open(fitsfile) hdr, data = lib_img.flatten(fits) w = pywcs.WCS(hdr) pixsize = np.abs(hdr['CDELT1']) # Get facets central pixels ras = np.array([d.position_cal[0] for d in directions]) decs = np.array([d.position_cal[1] for d in directions]) x_fs, y_fs = w.all_world2pix(ras, decs, 0, ra_dec_order=True) # keep trak of numbers in the direction names to name correctly patches in the fits files # in this way Isl_patch_12 will have "12" into the fits for that patch. nums = [d.isl_num for d in directions] x_c = data.shape[0] / 2. y_c = data.shape[1] / 2. # Check if dir is in img, otherwise drop idx_for_facet = [] for i, direction in enumerate(directions): x, y = w.all_world2pix(ras[i], decs[i], 0, ra_dec_order=True) if x < 0 or x > data.shape[0] or y < 0 or y > data.shape[1]: logger.info( 'Direction %s is outside the primary beam and will not have a facet (it will still be a calibrator).' % direction.name) else: idx_for_facet.append(i) # convert to pixel space (voronoi must be in eucledian space) x1 = 0 y1 = 0 x2 = data.shape[1] # note that y is before x in fits.data y2 = data.shape[0] # do tasselization vor = Voronoi( np.array((x_fs[idx_for_facet], y_fs[idx_for_facet])).transpose()) box = np.array([[x1, y1], [x2, y2]]) impoly = voronoi_finite_polygons_2d_box(vor, box) # create fits mask (each region one number) x, y = np.meshgrid(np.arange(x2), np.arange(y2)) # make a canvas with coordinates x, y = x.flatten(), y.flatten() pixels = np.vstack((x, y)).T data_facet = np.zeros(shape=data.shape) for num, poly in zip(nums, impoly): p = Path(poly) pixels_region = p.contains_points(pixels) data_facet[pixels_region.reshape(y2, x2)] = num # put all values in each island equal to the closest region struct = generate_binary_structure(2, 2) data = binary_dilation(data, structure=struct, iterations=3).astype(data.dtype) # expand masks blobs, number_of_blobs = label(data.astype(int).squeeze(), structure=[[1, 1, 1], [1, 1, 1], [1, 1, 1]]) center_of_masses = center_of_mass(data, blobs, list(range(number_of_blobs + 1))) for blob in range(1, number_of_blobs + 1): # get closer facet facet_num = closest_node( center_of_masses[blob], np.array([y_fs[idx_for_facet], x_fs[idx_for_facet]]).T) # put all pixel of that mask to that facet value data_facet[blobs == blob] = nums[facet_num] # save fits mask pyfits.writeto(out_mask, data_facet, hdr, overwrite=True) # save regions if not os.path.isdir(outdir_reg): os.makedirs(outdir_reg) all_s = [] for i, poly in enumerate(impoly): ra, dec = w.all_pix2world(poly[:, 0], poly[:, 1], 0, ra_dec_order=True) coords = np.array([ra, dec]).T.flatten() s = Shape('Polygon', None) s.coord_format = 'fk5' s.coord_list = coords # ra, dec, radius s.coord_format = 'fk5' s.attr = ([], { 'width': '2', 'point': 'cross', 'font': '"helvetica 16 normal roman"' }) s.comment = 'color=red' all_s.append(s) regions = pyregion.ShapeList([s]) regionfile = outdir_reg + '/' + directions[ idx_for_facet[i]].name + '.reg' regions.write(regionfile) # add names for all.reg for d in directions: s = Shape('circle', None) s.coord_format = 'fk5' s.coord_list = [d.position_cal[0], d.position_cal[1], 0.01] # ra, dec, radius s.coord_format = 'fk5' s.attr = ([], { 'width': '1', 'point': 'cross', 'font': '"helvetica 16 normal roman"' }) s.comment = 'color=white text="%s"' % d.name all_s.append(s) regions = pyregion.ShapeList(all_s) regionfile = outdir_reg + '/all.reg' regions.write(regionfile) logger.debug( 'There are %i regions within the PB and %i outside (no facet).' % (len(idx_for_facet), len(directions) - len(idx_for_facet))) # plot tesselization if png is not None: import matplotlib.pyplot as pl pl.figure(figsize=(8, 8)) ax1 = pl.gca() voronoi_plot_2d(vor, ax1, show_vertices=True, line_colors='black', line_width=2, point_size=4) for i, d in enumerate(directions): ax1.text(x_fs[i], y_fs[i], d.name, fontsize=15) ax1.plot([x1, x1, x2, x2, x1], [y1, y2, y2, y1, y1]) ax1.set_xlabel('RA (pixel)') ax1.set_ylabel('Dec (pixel)') ax1.set_xlim(x1, x2) ax1.set_ylim(y1, y2) logger.debug('Save plot: %s' % png) pl.savefig(png)