def login(self, user=None, password=None, credentials_file=None, verbose=False): """Performs a login. User and password can be used or a file that contains user name and password (2 lines: one for user name and the following one for the password) Parameters ---------- user : str, mandatory if 'file' is not provided, default None login name password : str, mandatory if 'file' is not provided, default None user password credentials_file : str, mandatory if no 'user' & 'password' are provided file containing user and password in two lines verbose : bool, optional, default 'False' flag to display information about the process """ if credentials_file is not None: # read file: get user & password with open(credentials_file, "r") as ins: user = ins.readline().strip() password = ins.readline().strip() if user is None: log.info("Invalid user name") return if password is None: log.info("Invalid password") return self.__user = user self.__pwd = password self.__dologin(verbose)
def remove_jobs(self, jobs_list, verbose=False): """Removes the specified jobs Parameters ---------- jobs_list : str, mandatory jobs identifiers to be removed verbose : bool, optional, default 'False' flag to display information about the process """ if jobs_list is None: return jobsIds = None if isinstance(jobs_list, str): jobsIds = jobs_list elif isinstance(jobs_list, list): jobsIds = ','.join(jobs_list) else: raise Exception("Invalid object type") if verbose: print("Jobs to be removed: " + str(jobsIds)) data = "JOB_IDS=" + jobsIds subContext = "deletejobs" connHandler = self.__getconnhandler() response = connHandler.execute_post(subContext, data) if verbose: print(response.status, response.reason) print(response.getheaders()) isError = connHandler.check_launch_response_status( response, verbose, 200) if isError: log.info(response.reason) raise requests.exceptions.HTTPError(response.reason)
def read_out_noise(image, amount, gain=1.0): ''' This function provides the errors introduced with the readout of the CCD Parameters ---------- image : numpy ndarray #can be changed with the only shape The image of the CCD, used by the function to extract the shape amount : float, int The central value of the distribution of the errors, it should be related with the background (the sqrt) gain : float, optional The value of the gain used by the CCD Output ------ noise : numpy ndarray The image of the CCD with the readout noise generated ''' log.info(hist()) shape = image.shape noise = np.random.normal(scale=amount / gain, size=shape) return noise
def list_async_jobs(self, verbose=False): """Returns all the asynchronous jobs Parameters ---------- verbose : bool, optional, default 'False' flag to display information about the process Returns ------- A list of Job objects """ subContext = "async" response = self.__connHandler.execute_get(subContext) if verbose: print(response.status, response.reason) print(response.getheaders()) isError = self.__connHandler.check_launch_response_status( response, verbose, 200) if isError: log.info(response.reason) raise requests.exceptions.HTTPError(response.reason) return None # parse jobs jsp = JobListSaxParser(async_job=True) jobs = jsp.parseData(response) if jobs is not None: for j in jobs: j.connHandler = self.__connHandler return jobs
def _convolve_model_dir_2(model_dir, filters, overwrite=False): for f in filters: if f.name is None: raise Exception("filter name needs to be set") if f.central_wavelength is None: raise Exception("filter central wavelength needs to be set") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_cube = SEDCube.read(os.path.join(model_dir, 'flux.fits'), order='nu') par_table = load_parameter_table(model_dir) if not np.all(par_table['MODEL_NAME'] == sed_cube.names): raise ValueError( "Model names in SED cube and parameter file do not match") log.info("{0} SEDs found in {1}".format(sed_cube.n_models, model_dir)) # Set up convolved fluxes fluxes = [ ConvolvedFluxes(model_names=sed_cube.names, apertures=sed_cube.apertures, initialize_arrays=True) for i in range(len(filters)) ] # Set up list of binned filters binned_filters = [f.rebin(sed_cube.nu) for f in filters] # We do the unit conversion - if needed - at the last minute val_factor = sed_cube.val.unit.to(u.mJy) unc_factor = sed_cube.unc.unit.to(u.mJy) # Loop over apertures for i_ap in ProgressBar(range(sed_cube.n_ap)): sed_val = sed_cube.val[i_ap].transpose() sed_unc = sed_cube.val[i_ap].transpose() for i, f in enumerate(binned_filters): response = f.response.astype(sed_val.dtype) fluxes[i].flux[:, i_ap] = np.sum(sed_val * response, axis=1) * val_factor fluxes[i].error[:, i_ap] = np.sqrt( np.sum((sed_unc * response)**2, axis=1)) * unc_factor for i, f in enumerate(binned_filters): fluxes[i].central_wavelength = f.central_wavelength fluxes[i].apertures = sed_cube.apertures fluxes[i].model_names = sed_cube.names fluxes[i].write(model_dir + '/convolved/' + f.name + '.fits', overwrite=overwrite)
def saturation_controll(image): ''' The value of a pixel can't overcome the maximum value of a 16bit memory (65535) and it is called saturated. This function will controll that no pixel would surpass this value and, if it does, this function will reset its value to 65535 Parameter --------- image : numpy ndarray It is the image to controll Output ------ image : numpy ndarray The image controlled and adjusted ''' log.info(hist()) size = image.shape for i in range(size[0]): for j in range(size[1]): if image[i, j] >= 65535: image[i, j] = 65535 return image
def save_fits(data, defocus_distance, center, choose): ''' Save a fits with a simple header. Parameters ---------- data : numpy array Image to save defocus_distance : float Information about the defocus distance to insert in the header center : tuple, array Invormation about the coordinate to insert in the header choose : int is a variable that determines the star used ''' log.info(hist()) name = ['Kolmogorov', 'Speckle_Sum', 'Simple_seeing'] choose = choose-1 hdul = fits.PrimaryHDU(data) #save the file hdr = hdul.header hdr['RA'] = (center[0], "Right Ascension in decimal hours" ) hdr['DEC'] = (center[1], "Declination in decimal degrees") hdr['IMGTYPE'] = 'object' hdr['IMAGETYP'] = 'object' hdr['BZERO'] = 32768 hdr['DEFOCUS'] = (defocus_distance, "[mm] Distance from focal plane") hdul.scale('int16', bzero=32768) hdul.writeto(f'try_{name[choose]}.fits', overwrite = True)
def remove_jobs(self, jobs_list, verbose=False): """Removes the specified jobs Parameters ---------- jobs_list : str, mandatory jobs identifiers to be removed verbose : bool, optional, default 'False' flag to display information about the process """ if jobs_list is None: return jobsIds = None if isinstance(jobs_list, str): jobsIds = jobs_list elif isinstance(jobs_list, list): jobsIds = ','.join(jobs_list) else: raise Exception("Invalid object type") if verbose: print("Jobs to be removed: " + str(jobsIds)) data = "JOB_IDS=" + jobsIds subContext = "deletejobs" connHandler = self.__getconnhandler() response = connHandler.execute_post(subContext, data) if verbose: print(response.status, response.reason) print(response.getheaders()) isError = connHandler.check_launch_response_status(response, verbose, 200) if isError: log.info(response.reason) raise requests.exceptions.HTTPError(response.reason)
def validate_schema(filename, version='1.1'): """ Validates the given file against the appropriate VOTable schema. Parameters ---------- filename : str The path to the XML file to validate version : str, optional The VOTABLE version to check, which must be a string \"1.0\", \"1.1\", \"1.2\" or \"1.3\". If it is not one of these, version \"1.1\" is assumed. For version \"1.0\", it is checked against a DTD, since that version did not have an XML Schema. Returns ------- returncode, stdout, stderr : int, str, str Returns the returncode from xmllint and the stdout and stderr as strings """ if version not in ('1.0', '1.1', '1.2', '1.3'): log.info(f'{filename} has version {version}, using schema 1.1') version = '1.1' if version in ('1.1', '1.2', '1.3'): schema_path = data.get_pkg_data_filename( f'data/VOTable.v{version}.xsd') else: schema_path = data.get_pkg_data_filename( 'data/VOTable.dtd') return validate.validate_schema(filename, schema_path)
def defocus(pupil, defocus_distance, r, wavelenght): ''' Takes the image and created the FT at some distance Parameters ---------- pupil : numpy ndarray The image of the aperture to transform defocus_distance : float The distance of the defocus in mm in range(-2.5,2.5) r : numpy ndarray The radial variable on the pupil wavelenght : float The center wavelength in mm Output ------ image : numpy ndarray The FT of the pupil at defocus_distance from the focus plane, AKA the PSF ''' log.info(hist()) phaseAngle = 1j*defocus_distance*10*np.sqrt((2*np.pi/wavelenght)**2-r**2+0j) #unnecessary 0j but keeping it for complex reasons kernel = np.exp(phaseAngle) defocusPupil = pupil * kernel defocusPSFA = fft.fftshift(fft.fft2(defocusPupil)) image = np.abs(defocusPSFA) return image
def intensity_aberration(aperture, scale, modes, pupil): '''Creates the amplitude atmospheric aberration. Parameters ---------- aperture : int The aperture of the telescope in mm scale : int The scale of the image generated modes : int Number of zernikle terms to use pupil : numpy ndarray The image of the aperture Output ------ zernike : numpy ndarray The image of the aberration overlapped with the image of the aperture ''' log.info(hist()) nx_size= aperture//scale zernike = [] zernike_array = ZA(modes, nx_size) for i in range(modes): zernike.append(reshape_aber(zernike_array[i], scale, pupil)) return zernike
def reshape_aber(image, scale, pupil): ''' Adapt the aberration to the pupil size Parameters ---------- image : int The aperture of the telescope in mm scale : int The scale of the image generated pupil : numpy ndarray The image of the aperture Output ------ image : numpy ndarray The image of the aberration with the right shape to be combinated with the aperture ''' log.info(hist()) image = np.repeat(np.repeat(image, scale, axis = 0), scale, axis = 1) units= int((np.shape(pupil)[0]-np.shape(image)[0])/2) image = np.pad(image,(units), mode = 'constant') return image
def physical_CCD(CCD_structure): ''' Generates a CCD with a given structure Parameters ---------- CCD_structure : array The varies entries must have the measure on the CCD [0] : int, float; the x length of the CCD in mm [1] : int, float; the y length of the CCD in mm [2] : float ; the length of a pixel in mm Outputs ------- CCD : numpy ndarray The image of the CCD x : numpy ndarray The x variable on the CCD y : numpy ndarray The y variable on the CCD ''' log.info(hist()) len_y = int(CCD_structure[0]/CCD_structure[2])//2 len_x = int(CCD_structure[1]/CCD_structure[2])//2 x, y = np.mgrid[-len_x:len_x,-len_y:len_y] r = np.sqrt(x**2+y**2) CCD = np.piecewise(r, [r], [0]) return CCD
def list_async_jobs(self, verbose=False): """Returns all the asynchronous jobs Parameters ---------- verbose : bool, optional, default 'False' flag to display information about the process Returns ------- A list of Job objects """ subContext = "async" response = self.__connHandler.execute_get(subContext) if verbose: print(response.status, response.reason) print(response.getheaders()) isError = self.__connHandler.check_launch_response_status(response, verbose, 200) if isError: log.info(response.reason) raise requests.exceptions.HTTPError(response.reason) return None # parse jobs jsp = JobListSaxParser(async_job=True) jobs = jsp.parseData(response) if jobs is not None: for j in jobs: j.connHandler = self.__connHandler return jobs
def converter_to_pixel(CCD_resolution, focal_lenght, quantity_to_convert): #TO REVISIONATE '''Convert a quantity on the aperture to pixels on CCD ''' log.info(hist()) converter_rad_arc = 180*3600/np.pi quantity_to_convert = quantity_to_convert quantity_angle = np.arctan(quantity_to_convert/focal_lenght) * converter_rad_arc Q_on_CCD = quantity_angle / CCD_resolution return Q_on_CCD
def _convolve_model_dir_2(model_dir, filters, overwrite=False, memmap=True): for f in filters: if f.name is None: raise Exception("filter name needs to be set") if f.central_wavelength is None: raise Exception("filter central wavelength needs to be set") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_cube = SEDCube.read(os.path.join(model_dir, 'flux.fits'), order='nu', memmap=memmap) par_table = load_parameter_table(model_dir) if not np.all(par_table['MODEL_NAME'] == sed_cube.names): raise ValueError("Model names in SED cube and parameter file do not match") log.info("{0} SEDs found in {1}".format(sed_cube.n_models, model_dir)) # Set up convolved fluxes fluxes = [ConvolvedFluxes(model_names=sed_cube.names, apertures=sed_cube.apertures, initialize_arrays=True) for i in range(len(filters))] # Set up list of binned filters binned_filters = [f.rebin(sed_cube.nu) for f in filters] # We do the unit conversion - if needed - at the last minute val_factor = sed_cube.val.unit.to(u.mJy) unc_factor = sed_cube.unc.unit.to(u.mJy) # Loop over apertures for i_ap in ProgressBar(range(sed_cube.n_ap)): sed_val = sed_cube.val[:, i_ap, :] sed_unc = sed_cube.val[:, i_ap, :] for i, f in enumerate(binned_filters): response = f.response.astype(sed_val.dtype) fluxes[i].flux[:, i_ap] = np.sum(sed_val * response, axis=1) * val_factor fluxes[i].error[:, i_ap] = np.sqrt(np.sum((sed_unc * response) ** 2, axis=1)) * unc_factor for i, f in enumerate(binned_filters): fluxes[i].central_wavelength = f.central_wavelength fluxes[i].apertures = sed_cube.apertures fluxes[i].model_names = sed_cube.names fluxes[i].write(model_dir + '/convolved/' + f.name + '.fits', overwrite=overwrite)
def load_measure(): ''' Load the basic data of the telescope and the CCD from a JSON file Outputs ------- telescope_structure : list a list that contains the information about the telescope telescope_structure[0] = f_l : int It is the focal length of the telescope in mm telescope_structure[1] = ape : int It is the aperture of the telescope in mm telescope_structure[2] = obs : int It is the central obstruction of the telescope in mm telescope_structure[3] = wav : float It is the central wavelength of the sensible spectrum of the CCD in mm ccd_structure : array a array that contains the information about the CCD ccd_structure[0] = C_x : float It is the length of the CCD along the x axis in mm ccd_structure[0] = C_y : float It is the length of the CCD along the y axis in mm ccd_structure[0] = pix : float It is the lenght of a single pixel in mm ccd_data : list a list that cointains information about the errors generated in the CCD ccd_data[0] = gain : float the gain of the CCD ccd_data[1] = read_out_electrons : float the electrons that generate the read out noise ''' log.info(hist()) #filename = "Antola_data.json" #filename = "San_Pedro_data.json" filename = "San_Pedro_data_CCD2.json" f = open(filename, "r") data = json.load(f) Telescope = data['Telescope'] f_l = Telescope['focal_lenght'] ape = Telescope['aperture'] obs = Telescope['obstruction'] wav = Telescope['wavelenght'] CCD = data['CCD'] C_x = CCD['CCD_x'] C_y = CCD['CCD_y'] pix = CCD['pixels'] gain = CCD['gain'] read_out_electrons = CCD["read_out_electrons"] f.close() telescope_structure = (f_l, ape, obs, wav) ccd_structure = [C_x, C_y, pix] ccd_data = (gain, read_out_electrons) return telescope_structure, ccd_structure, ccd_data
def sensitivity_variations(image, vignetting=True, dust=True): ''' The sensivity isn't constant, but can vary over the CCD, tipically with a gaussian trend and can be worsen by the presence of the dust. This function provides for this, creating the flat frame Parameters ---------- image : numpy ndarray #can be changed with the only shape nect The image of the CCD, used by the function to extract the shape vignetting : bool, optional If True, the gaussian figure is created on a image with the shape of the CCD dust : bool, optional If True, there will be added the donut generated by the dust on the CCD image Output ------ sensitivity :numpy ndarray The image of the CCD with a big gaussuan curve to simulate the variability of the sensivity ''' log.info(hist()) sensitivity = np.zeros_like(image) + 1.0 shape = np.array(sensitivity.shape) if dust or vignetting: # I don't know why, but y,x not x,y y, x = np.indices(sensitivity.shape) if vignetting: #TODO, centro gaussiana da spostare # Generate very wide gaussian centered on the center of the image, # multiply the sensitivity by it. #narrowing = np.random.randint(5,10) vign_model = Gaussian2D(amplitude=1, x_mean=shape[0] / 2, y_mean=shape[1] / 2, x_stddev=2 * (shape.max()), y_stddev=2 * (shape.max())) vign_im = vign_model(x, y) sensitivity *= vign_im if dust: dust_im = add_donuts(image, number=20) dust_im = dust_im / dust_im.max() sensitivity *= dust_im return sensitivity
def create_config_file(pkg, rootname='astropy', overwrite=False): """ Create the default configuration file for the specified package. If the file already exists, it is updated only if it has not been modified. Otherwise the ``overwrite`` flag is needed to overwrite it. Parameters ---------- pkg : str The package to be updated. rootname : str Name of the root configuration directory. overwrite : bool Force updating the file if it already exists. Returns ------- updated : bool If the profile was updated, `True`, otherwise `False`. """ # local import to prevent using the logger before it is configured from astropy.logger import log cfgfn = get_config_filename(pkg, rootname=rootname) # generate the default config template template_content = io.StringIO() generate_config(pkg, template_content) template_content.seek(0) template_content = template_content.read() doupdate = True # if the file already exists, check that it has not been modified if cfgfn is not None and path.exists(cfgfn): with open(cfgfn, 'rt', encoding='latin-1') as fd: content = fd.read() doupdate = is_unedited_config_file(content, template_content) if doupdate or overwrite: with open(cfgfn, 'wt', encoding='latin-1') as fw: fw.write(template_content) log.info('The configuration file has been successfully written ' f'to {cfgfn}') return True elif not doupdate: log.warning('The configuration file already exists and seems to ' 'have been customized, so it has not been updated. ' 'Use overwrite=True if you really want to update it.') return False
def login(self, token=None, store_token=False, reenter_token=False): """ Log session into the MAST portal. Parameters ---------- token : string, optional Default is None. The token to authenticate the user. This can be generated at https://auth.mast.stsci.edu/token?suggested_name=Astroquery&suggested_scope=mast:exclusive_access. If not supplied, it will be prompted for if not in the keyring or set via $MAST_API_TOKEN store_token : bool, optional Default False. If true, MAST token will be stored securely in your keyring. reenter_token : bool, optional Default False. Asks for the token even if it is already stored in the keyring or $MAST_API_TOKEN environment variable. This is the way to overwrite an already stored password on the keyring. """ if token is None and "MAST_API_TOKEN" in os.environ: token = os.environ["MAST_API_TOKEN"] if token is None: token = keyring.get_password("astroquery:mast.stsci.edu.token", "masttoken") if token is None or reenter_token: info_msg = "If you do not have an API token already, visit the following link to create one: " log.info(info_msg + self.AUTH_URL) token = getpass("Enter MAST API Token: ") # store token if desired if store_token: keyring.set_password("astroquery:mast.stsci.edu.token", "masttoken", token) self.session.headers["Accept"] = "application/json" self.session.cookies["mast_token"] = token info = self.session_info() if not info["anon"]: log.info("MAST API token accepted, welcome {}".format( info["attrib"].get("display_name"))) else: warn_msg = ( "MAST API token invalid!\n" "To make create a new API token visit to following link: " + self.AUTH_URL) warnings.warn(warn_msg, AuthenticationWarning) return not info["anon"]
def sky_background_aperture(focal_lenght, aperture, obstruction, trellis=True, atmosphere=False): ''' Calls telescope, intensity_aberration and phase_aberration to create the optical figure at the aperure Parameters ---------- focal_lenght : int The focal length of the telescope in mm aperture : int The aperture of the telescope in mm obstruction : int The central obstraction of the telescope in mm trellis : bool, optional If trellis == True the structure that hold on the obstruction is drawn, else the structure is not drawn atmosphere : bool, optional If atmosphere == True intensity_aberration and phase_aberration are called, else they are not Output ------ pupil : numpy ndarray The image of the aperture with all the aberrations r : numpy ndarray The radial variable on the pupil ''' log.info(hist()) pupil, r = telescope(focal_lenght, aperture, obstruction, trellis) if atmosphere: #zer = intensity_aberration(aperture, scale, modes, pupil) #zernike = zer[0] #for i in range(modes-1): # a = np.random.random() # zernike += (a/5)*zer[i] phase = phase_aberration(aperture, scale, D, r0, L0, pupil) #Small aperture, long exposure, the kolmogorov turbulance on small scale are no #phase = np.sqrt(np.abs(phase))**2 kernel = pupil * phase pupil = pupil * np.exp(1j*kernel+0j) return pupil, r
def Coordinator(coord, center, CCD_structure, catalog): ''' The function takes the coordinates of a star and uses the WCS keywords to give back the position on the CCD in pixel Parameters ---------- coord : SkyCoord It is the coordinates of the star already elaborated by astropy center : array It must contain the position of the center of the CCD and it is used to obtain the distance of the star from it CCD_res : float It is the resolution of the CCD in arcsec/pixel Outputs ------- x : float It is the position of the star on the grid of the CCD along the axis x y : float It is the position of the star on the grid of the CCD along the axis y ''' log.info(hist()) if catalog == 'Simbad': offset_x = CCD_structure[0]/CCD_structure[2] offset_y = CCD_structure[1]/CCD_structure[2] w = wcs.WCS(naxis=2) w.wcs.crpix = [1, 1] w.wcs.crval = [center[0], center[1]] w.wcs.ctype = ["RA", "DEC"] x,y = wcs.utils.skycoord_to_pixel(coord, w) x = ((x*3600)/CCD_structure[3]) + offset_x/2 #(deg*arc/sec*deg)*arc/sec*pixel+offset y = ((y*3600)/CCD_structure[3]) + offset_y/2 elif catalog == 'Gaia': offset_x = CCD_structure[0]/CCD_structure[2] offset_y = CCD_structure[1]/CCD_structure[2] w = wcs.WCS(naxis=2) w.wcs.crpix = [1, 1] w.wcs.crval = [center[0], center[1]] w.wcs.ctype = ["RA", "DEC"] x,y = wcs.utils.skycoord_to_pixel(coord, w) x = ((x*3600)/CCD_structure[3]) + offset_x/2 #(deg*arc/sec*deg)*arc/sec*pixel+offset y = ((y*3600)/CCD_structure[3]) + offset_y/2 return y, x
def make_cosmic_rays(image, number, strength=10000): ''' It can appens that during an acquisition of an image some pixel are "saturated" by the cosmic rays, this function provides for a CCD image with this effect Parameters ---------- image : numpy ndarray #can be changed with the only shape The image of the CCD, used by the function to extract the shape number : int This number is the number of cosmic ray within a single image strenght : int, optional This is the intensity of a cosmic ray on a pixel Output ------ cosmic_image :numpy ndarray The image of the CCD with the pixel overflowed by the cosmic rays generated ''' log.info(hist()) cosmic_image = np.zeros_like(image) # Yes, the order below is correct. The x axis is the column, which # is the second index. max_y, max_x = cosmic_image.shape # Get the smallest dimension to ensure the cosmic rays are within the image maximum_pos = np.min(cosmic_image.shape) # These will be center points of the cosmic rays, which we place away from # the edges to ensure they are visible. xy_cosmic = np.random.randint(0.1 * maximum_pos, 0.9 * maximum_pos, size=[number, 2]) cosmic_length = 5 # pixels, a little big cosmic_width = 2 theta_cosmic = 2 * np.pi * np.random.rand() apertures = EllipticalAperture(xy_cosmic, cosmic_length, cosmic_width, theta_cosmic) masks = apertures.to_mask(method='center') for mask in masks: cosmic_image += strength * mask.to_image(shape=cosmic_image.shape) return cosmic_image
def load_table(self, table, verbose=False): """Loads the specified table Parameters ---------- table : str, mandatory full qualified table name (i.e. schema name + table name) verbose : bool, optional, default 'False' flag to display information about the process Returns ------- A table object """ log.info("Retrieving table '"+str(table)+"'") connHandler = self.__getconnhandler() response = connHandler.execute_get("tables?tables="+table) if verbose: print(response.status, response.reason) isError = connHandler.check_launch_response_status(response, verbose, 200) if isError: log.info("{} {}".format(response.status, response.reason)) raise requests.exceptions.HTTPError(response.reason) return None log.info("Parsing table '"+str(table)+"'...") tsp = TableSaxParser() tsp.parseData(response) log.info("Done.") return tsp.get_table()
def load_table(self, table, verbose=False): """Loads the specified table Parameters ---------- table : str, mandatory full qualified table name (i.e. schema name + table name) verbose : bool, optional, default 'False' flag to display information about the process Returns ------- A table object """ log.info("Retrieving table '" + str(table) + "'") connHandler = self.__getconnhandler() response = connHandler.execute_get("tables?tables=" + table) if verbose: print(response.status, response.reason) isError = connHandler.check_launch_response_status( response, verbose, 200) if isError: log.info("{} {}".format(response.status, response.reason)) raise requests.exceptions.HTTPError(response.reason) return None log.info("Parsing table '" + str(table) + "'...") tsp = TableSaxParser() tsp.parseData(response) log.info("Done.") return tsp.get_table()
def dark_current(image, current, exposure_time, gain=1.0, hot_pixels=False): ''' This function creates a matrix with the shape of the CCD with the errors introduced by the dark current with a poissonian distribution. It also provide for the presence of hot pixels in the CCD Parameters ---------- image : numpy ndarray #can be changed with the only shape The image of the CCD, used by the function to extract the shape current : float This is the value of the dark current for 1 second exposure_time : int The number of second used for obtainig the CCD's image gain : float, optional The value of the gain used in the CCD, more the gain, more the errors hot_pixel : bool, optional This flag allows to choose if there will by the hot pixels or not if True there will be hot pixels Output ------ dark_bias : numpy ndarray The image of the CCD with the dark current noise generated ''' log.info(hist()) base_current = current * exposure_time / gain dark_bias = np.random.poisson(base_current, size=image.shape) if hot_pixels: '''set the probability of 0.01% of a pixel to be hot''' y_max, x_max = dark_bias.shape numb_hot = int(0.0001 * x_max * y_max) hot_x = np.random.randint(0, x_max, size=numb_hot) hot_y = np.random.randint(0, y_max, size=numb_hot) hot_current = 10000 * current dark_bias[[hot_y, hot_x]] = hot_current * exposure_time #/gain return dark_bias
def telescope_on_CCD(CCD_resolution, binning, telescope_structure, defocus_distance, trellis=True, atmosphere=False): ''' This function calls sky_backgroud_aperture, defocus and image_processing to create the sample on the PSF on the CCD Parameters ---------- CCD_resolution : float The resolution on the CCD in arcsec/pixel telescope_structure : list telescope_structure[0] = focal_lenght : int The focal length of the telescope in mm telescope_structure[1] = aperture : int The aperture of the telescope in mm telescope_structure[2] = obstruction : int The central obstraction of the telescope in mm telescope_structure[3] = wavelenght : float The center wavelength in mm defocus_distance : float The distance of the defocus in mm in range(-2.5,2.5) trellis : bool, optional If trellis == True the structure that hold on the obstruction is drawn, else the structure is not drawn atmosphere : bool, optional If atmosphere == True intensity_aberration and phase_aberration are called, else they are not Output ------ image : numpy ndarray The image of the PSF ''' log.info(hist()) #units = converter_to_pixel(CCD_resolution, focal_lenght, 1) #convert 1mm on the aperture in pixel on CCD units = CCD_resolution//0.12 #empiric value image, r = sky_background_aperture(telescope_structure[0], telescope_structure[1], telescope_structure[2], trellis, atmosphere) #creates the aperture image = defocus(image, defocus_distance, r, telescope_structure[3]) #creates the image on the screen image = image_processing(image, binning, telescope_structure[1], atmosphere) return image
def image_processing(image, m, aperture, atmosphere): ''' This function takes an image and some parameters to obtain the same image cut and strechetd to fit the CCD. Also, this function takes only the essentioal information thus reducing the total weight of the final image Parameters ---------- image : numpy ndarray The image, usually the PSF m : int The binnig, used to re-sum the PSF pixels aperture : int The aperture of the telescope in mm atmosphere : bool To the long exposure Output ------ new_image : numpy ndarray The image cleaned with only the good parts and with the right measure ''' log.info(hist()) image = (np.abs(image))**2 size = image.shape if atmosphere: m = m*4 new_size = (size[0]//m, size[1]//m) new_image = np.zeros(new_size) if m==1: new_image = image elif m>=2: for i in range(new_size[0]): for j in range(new_size[1]): new_image[[i],[j]] = sum_image(image, i, j, m) else: print('binning problem, PSF binnig ignored') new_image = image new_image = (new_image/(np.sum(new_image)))*0.2 new_image = rotate(new_image, angle=45) return new_image
def __dologin(self, verbose=False): self.__isLoggedIn = False response = self.__execLogin(self.__user, self.__pwd, verbose) # check response connHandler = self.__getconnhandler() isError = connHandler.check_launch_response_status(response, verbose, 200) if isError: log.info("Login error: " + str(response.reason)) raise requests.exceptions.HTTPError("Login error: " + str(response.reason)) else: # extract cookie cookie = self._Tap__findCookieInHeader(response.getheaders()) if cookie is not None: self.__isLoggedIn = True connHandler.set_cookie(cookie)
def _build_temp_install(self): """ Install the package and to a temporary directory for the purposes of testing. This allows us to test the install command, include the entry points, and also avoids creating pyc and __pycache__ directories inside the build directory """ # On OSX the default path for temp files is under /var, but in most # cases on OSX /var is actually a symlink to /private/var; ensure we # dereference that link, because pytest is very sensitive to relative # paths... tmp_dir = tempfile.mkdtemp(prefix=self.package_name + '-test-', dir=self.temp_root) self.tmp_dir = os.path.realpath(tmp_dir) log.info(f'installing to temporary directory: {self.tmp_dir}') # We now install the package to the temporary directory. We do this # rather than build and copy because this will ensure that e.g. entry # points work. self.reinitialize_command('install') install_cmd = self.distribution.get_command_obj('install') install_cmd.prefix = self.tmp_dir if self.verbose_install: self.run_command('install') else: with _suppress_stdout(): self.run_command('install') # We now get the path to the site-packages directory that was created # inside self.tmp_dir install_cmd = self.get_finalized_command('install') self.testing_path = install_cmd.install_lib # Ideally, docs_path is set properly in run(), but if it is still # not set here, do not pretend it is, otherwise bad things happen. # See astropy/package-template#157 if self.docs_path is not None: new_docs_path = os.path.join(self.testing_path, os.path.basename(self.docs_path)) shutil.copytree(self.docs_path, new_docs_path) self.docs_path = new_docs_path shutil.copy('setup.cfg', self.testing_path)
def __dologin(self, verbose=False): self.__isLoggedIn = False response = self.__execLogin(self.__user, self.__pwd, verbose) # check response connHandler = self.__getconnhandler() isError = connHandler.check_launch_response_status( response, verbose, 200) if isError: log.info("Login error: " + str(response.reason)) raise requests.exceptions.HTTPError("Login error: " + str(response.reason)) else: # extract cookie cookie = self._Tap__findCookieInHeader(response.getheaders()) if cookie is not None: self.__isLoggedIn = True connHandler.set_cookie(cookie)
def phase_aberration(aperture, scale, D, r0, L0, pupil): ''' Creates an image with the Kolmogorov algorithm with the phase atmospheric aberration. This function works but usually is not used because the telescope is limited by the seeing. Parameters ---------- aperture : int The aperture of the telescope in mm scale : int The scale of the image generated D : int #inutile e rindondante con aperture, da cambiare e togliere r0 : float The Fried parameter of the 'seeing' L0 : float, int The outer scale of the 'seeing' pupil : numpy ndarray The image of the aperture Output ------ phase_screen : numpy ndarray The image of the turbolence overlapped with the image of the aperture ''' log.info(hist()) nx_size= aperture//scale plx_scale = D/nx_size phase_screen = PhaseScreenKolmogorov(nx_size, plx_scale, r0, L0) phase_screen.add_row() phase_screen = phase_screen.scrn phase_screen = reshape_aber(phase_screen, scale, pupil) return phase_screen
def sky_brightness(plate_scale, x_pix, y_pix, photo_filters, exposure_time, moon_phase=3): log.info(hist()) ''' moon_phase = int, optional moon_phase can go to 0 (new moon) to 3 (full moon) with intermedian phases 1 (7 day from new moon) and 2 (10 day from new moon) ''' sub = DATA["Sky_brightness"] for n, i in enumerate(photo_filters): if i == "g": photo_filters[n] = "V" sky_brightness = [sub[k] for k in photo_filters if k in sub] x_arcsec = plate_scale * x_pix y_arcsec = plate_scale * y_pix Area = x_arcsec * y_arcsec magnitudo_ab_sky = [] magnitudo_ab_sky.append(sky_brightness[0][moon_phase] - 2.5*np.log10(Area)) photons = magnitudo_to_photons(magnitudo_ab_sky, photo_filters) electrons = photons_to_electrons(photons, photo_filters) tot = sum(electrons)*exposure_time return tot
def search_async_jobs(self, jobfilter=None, verbose=False): """Searches for jobs applying the specified filter Parameters ---------- jobfilter : JobFilter, optional, default None job filter verbose : bool, optional, default 'False' flag to display information about the process Returns ------- A list of Job objects """ # jobs/list?[&session=][&limit=][&offset=][&order=][&metadata_only=true|false] subContext = "jobs/async" if jobfilter is not None: data = jobfilter.createUrlRequest() if data is not None: subContext = subContext + '?' + self.__appendData(data) connHandler = self.__getconnhandler() response = connHandler.execute_get(subContext) if verbose: print(response.status, response.reason) print(response.getheaders()) isError = connHandler.check_launch_response_status(response, verbose, 200) if isError: log.info(response.reason) raise requests.exceptions.HTTPError(response.reason) return None # parse jobs jsp = JobSaxParser(async_job=True) jobs = jsp.parseData(response) if jobs is not None: for j in jobs: j.set_connhandler(connHandler) return jobs
def _format_output(self, raw_output): if self.FORMAT == 'csv': split_output = raw_output.splitlines() columns = list( csv.reader([split_output[0]], delimiter=',', quotechar='"'))[0] rows = split_output[1:] # Quick test to see if API returned a valid csv file # If not, try to return JSON-compliant dictionary. test_row = list(csv.reader([rows[0]], delimiter=',', quotechar='"'))[0] if (len(columns) != len(test_row)): log.info("The API did not return a valid CSV output! \n" "Outputing JSON-compliant dictionary instead.") output = json.loads(raw_output) return output # Initialize and populate dictionary output_dict = {key: [] for key in columns} for row in rows: split_row = list( csv.reader([row], delimiter=',', quotechar='"'))[0] for ct, key in enumerate(columns): output_dict[key].append(split_row[ct]) # Convert dictionary to Astropy Table. output = Table(output_dict, names=columns) else: # Server response is JSON compliant. Simply # convert from raw text to dictionary. output = json.loads(raw_output) return output
def telescope(focal_lenght, aperture, obstruction, trellis=True): ''' Creates the figures of the telescope Parameters ---------- focal_lenght : int The focal length of the telescope in mm aperture : int The aperture of the telescope in mm obstruction : int The central obstraction of the telescope in mm trellis : bool, optional If trellis == True the structure that hold on the obstruction is drawn, else the structure is not drawn Output ------ pupil : numpy ndarray The image of the aperture r : numpy ndarray The radial variable on the pupil ''' log.info(hist()) x,y = np.mgrid[-aperture/2:aperture/2, -aperture/2:aperture/2] # creates the 2D grid for the 2D function *4 r = np.sqrt(x**2+y**2) pupil = np.piecewise(r, [r < aperture/2, r > aperture/2, r < obstruction/2], [1, 0, 0]) #creates the aperture if trellis: structure_x = np.piecewise(x, [x, x>1, x<0], [0,1,1]) #creates the structure that keep the obstruction structure_y = np.piecewise(y, [y, y>1, y<0], [0,1,1]) pupil = pupil*(structure_x*structure_y) return pupil, r
def find_radius_cumul(self, fraction): """ Find for each model the radius containing a fraction of the flux. Parameters ---------- fraction: float The fraction to use when determining the radius """ log.info("Calculating radii containing %g%s of the flux" % (fraction * 100., '%')) radius = np.zeros(self.n_models, dtype=self.flux.dtype) * u.au if self.apertures is None: return radius else: required = fraction * self.flux[:, -1] # Linear interpolation - need to loop over apertures for vectorization for ia in range(len(self.apertures) - 1): calc = (required >= self.flux[:, ia]) & (required < self.flux[:, ia + 1]) radius[calc] = (required[calc] - self.flux[calc, ia]) / \ (self.flux[calc, ia + 1] - self.flux[calc, ia]) * \ (self.apertures[ia + 1] - self.apertures[ia]) + \ self.apertures[ia] calc = (required < self.flux[:, 0]) radius[calc] = self.apertures[0] calc = (required >= self.flux[:, -1]) radius[calc] = self.apertures[-1] return radius
def __load_tables(self, only_names=False, include_shared_tables=False, verbose=False): """Loads all public tables Parameters ---------- only_names : bool, TAP+ only, optional, default 'False' True to load table names only include_shared_tables : bool, TAP+, optional, default 'False' True to include shared tables verbose : bool, optional, default 'False' flag to display information about the process Returns ------- A list of table objects """ # share_info=true&share_accessible=true&only_tables=true flags = "" addedItem = False if only_names: flags = "only_tables=true" addedItem = True if include_shared_tables: if addedItem: flags += "&" flags += "share_accessible=true" addedItem = True log.info("Retrieving tables...") if flags != "": response = self.__connHandler.execute_get("tables?"+flags) else: response = self.__connHandler.execute_get("tables") if verbose: print(response.status, response.reason) isError = self.__connHandler.check_launch_response_status(response, verbose, 200) if isError: log.info("{} {}".format(response.status, response.reason)) raise requests.exceptions.HTTPError(response.reason) return None log.info("Parsing tables...") tsp = TableSaxParser() tsp.parseData(response) log.info("Done.") return tsp.get_tables()
def find_radius_sigma(self, fraction): """ Find for each model a fractional surface brightness radius This is the outermost radius where the surface brightness is larger than a fraction of the peak surface brightness. Parameters ---------- fraction: float The fraction to use when determining the radius """ log.info("Calculating %g%s peak surface brightness radii" % (fraction * 100., '%')) sigma = np.zeros(self.flux.shape, dtype=self.flux.dtype) sigma[:, 0] = self.flux[:, 0] / self.apertures[0] ** 2 sigma[:, 1:] = (self.flux[:, 1:] - self.flux[:, :-1]) / \ (self.apertures[1:] ** 2 - self.apertures[:-1] ** 2) maximum = np.max(sigma, axis=1) radius = np.zeros(self.n_models, dtype=self.flux.dtype) * u.au # Linear interpolation - need to loop over apertures backwards for vectorization for ia in range(len(self.apertures) - 2, -1, -1): calc = (sigma[:, ia] > fraction * maximum) & (radius == 0.) radius[calc] = (sigma[calc, ia] - fraction * maximum[calc]) / \ (sigma[calc, ia] - sigma[calc, ia + 1]) * \ (self.apertures[ia + 1] - self.apertures[ia]) + \ self.apertures[ia] calc = sigma[:, -1] > fraction * maximum radius[calc] = self.apertures[-1] return radius
def validate_schema(filename, version='1.1'): """ Validates the given file against the appropriate VOTable schema. Parameters ---------- filename : str The path to the XML file to validate version : str, optional The VOTABLE version to check, which must be a string \"1.0\", \"1.1\", \"1.2\" or \"1.3\". If it is not one of these, version \"1.1\" is assumed. For version \"1.0\", it is checked against a DTD, since that version did not have an XML Schema. Returns ------- returncode, stdout, stderr : int, str, str Returns the returncode from xmllint and the stdout and stderr as strings """ if version not in ('1.0', '1.1', '1.2', '1.3'): log.info('{0} has version {1}, using schema 1.1'.format( filename, version)) version = '1.1' if version in ('1.1', '1.2', '1.3'): schema_path = data.get_pkg_data_filename( 'data/VOTable.v{0}.xsd'.format(version)) else: schema_path = data.get_pkg_data_filename( 'data/VOTable.dtd') return validate.validate_schema(filename, schema_path)
def load_async_job(self, jobid=None, name=None, verbose=False): """Loads an asynchronous job Parameters ---------- jobid : str, mandatory if no name is provided, default None job identifier name : str, mandatory if no jobid is provided, default None job name verbose : bool, optional, default 'False' flag to display information about the process Returns ------- A Job object """ if name is not None: jobfilter = Filter() jobfilter.add_filter('name', name) jobs = self.search_async_jobs(jobfilter) if jobs is None or len(jobs) < 1: log.info("No job found for name '"+str(name)+"'") return None jobid = jobs[0].get_jobid() if jobid is None: log.info("No job identifier found") return None subContext = "async/" + str(jobid) response = self.__connHandler.execute_get(subContext) if verbose: print(response.status, response.reason) print(response.getheaders()) isError = self.__connHandler.check_launch_response_status(response, verbose, 200) if isError: log.info(response.reason) raise requests.exceptions.HTTPError(response.reason) return None # parse job jsp = JobSaxParser(async_job=True) job = jsp.parseData(response)[0] job.set_connhandler(self.__connHandler) # load resulst job.get_results() return job
def launch_job_async(self, query, name=None, output_file=None, output_format="votable", verbose=False, dump_to_file=False, background=False, upload_resource=None, upload_table_name=None): """Launches an asynchronous job Parameters ---------- query : str, mandatory query to be executed output_file : str, optional, default None file name where the results are saved if dumpToFile is True. If this parameter is not provided, the jobid is used instead output_format : str, optional, default 'votable' results format verbose : bool, optional, default 'False' flag to display information about the process dump_to_file : bool, optional, default 'False' if True, the results are saved in a file instead of using memory background : bool, optional, default 'False' when the job is executed in asynchronous mode, this flag specifies whether the execution will wait until results are available upload_resource: str, optional, default None resource to be uploaded to UPLOAD_SCHEMA upload_table_name: str, required if uploadResource is provided, default None resource temporary table name associated to the uploaded resource Returns ------- A Job object """ if verbose: print("Launched query: '"+str(query)+"'") if upload_resource is not None: if upload_table_name is None: raise ValueError( "Table name is required when a resource is uploaded") response = self.__launchJobMultipart(query, upload_resource, upload_table_name, output_format, "async", verbose, name) else: response = self.__launchJob(query, output_format, "async", verbose, name) isError = self.__connHandler.check_launch_response_status(response, verbose, 303) job = Job(async_job=True, query=query, connhandler=self.__connHandler) suitableOutputFile = self.__getSuitableOutputFile(True, output_file, response.getheaders(), isError, output_format) job.outputFile = suitableOutputFile job.set_response_status(response.status, response.reason) job.parameters['format'] = output_format if isError: job.set_failed(True) if dump_to_file: self.__connHandler.dump_to_file(suitableOutputFile, response) raise requests.exceptions.HTTPError(response.reason) else: location = self.__connHandler.find_header( response.getheaders(), "location") jobid = self.__getJobId(location) if verbose: print("job " + str(jobid) + ", at: " + str(location)) job.jobid = jobid job.remoteLocation = location if not background: if verbose: print("Retrieving async. results...") # saveResults or getResults will block (not background) if dump_to_file: job.save_results(verbose) else: job.get_results() log.info("Query finished.") return job
def _download_file(self, url, local_filepath, timeout=None, auth=None, continuation=True, cache=False, method="GET", head_safe=False, **kwargs): """ Download a file. Resembles `astropy.utils.data.download_file` but uses the local ``_session`` """ if head_safe: response = self._session.request("HEAD", url, timeout=timeout, stream=True, auth=auth, **kwargs) else: response = self._session.request(method, url, timeout=timeout, stream=True, auth=auth, **kwargs) response.raise_for_status() if 'content-length' in response.headers: length = int(response.headers['content-length']) else: length = None if ((os.path.exists(local_filepath) and ('Accept-Ranges' in response.headers) and continuation)): open_mode = 'ab' existing_file_length = os.stat(local_filepath).st_size if length is not None and existing_file_length >= length: # all done! log.info("Found cached file {0} with expected size {1}." .format(local_filepath, existing_file_length)) return elif existing_file_length == 0: open_mode = 'wb' else: log.info("Continuing download of file {0}, with {1} bytes to " "go ({2}%)".format(local_filepath, length - existing_file_length, (length-existing_file_length)/length*100)) # bytes are indexed from 0: # https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#range-request-header end = "{0}".format(length-1) if length is not None else "" self._session.headers['Range'] = "bytes={0}-{1}".format(existing_file_length, end) response = self._session.request(method, url, timeout=timeout, stream=True, auth=auth, **kwargs) response.raise_for_status() elif cache and os.path.exists(local_filepath): if length is not None: statinfo = os.stat(local_filepath) if statinfo.st_size != length: log.warning("Found cached file {0} with size {1} that is " "different from expected size {2}" .format(local_filepath, statinfo.st_size, length)) open_mode = 'wb' else: log.info("Found cached file {0} with expected size {1}." .format(local_filepath, statinfo.st_size)) response.close() return else: log.info("Found cached file {0}.".format(local_filepath)) response.close() return else: open_mode = 'wb' if head_safe: response = self._session.request(method, url, timeout=timeout, stream=True, auth=auth, **kwargs) response.raise_for_status() blocksize = astropy.utils.data.conf.download_block_size bytes_read = 0 # Only show progress bar if logging level is INFO or lower. if log.getEffectiveLevel() <= 20: progress_stream = None # Astropy default else: progress_stream = io.StringIO() with ProgressBarOrSpinner( length, ('Downloading URL {0} to {1} ...' .format(url, local_filepath)), file=progress_stream) as pb: with open(local_filepath, open_mode) as f: for block in response.iter_content(blocksize): f.write(block) bytes_read += blocksize if length is not None: pb.update(bytes_read if bytes_read <= length else length) else: pb.update(bytes_read) response.close() return response
def convolve_model_dir(model_dir, filters, overwrite=False): """ Convolve all the model SEDs in a model directory Parameters ---------- model_dir : str The path to the model directory filters : list A list of :class:`~sedfitter.filter.Filter` objects to use for the convolution overwrite : bool, optional Whether to overwrite the output files """ for f in filters: if f.name is None: raise Exception("filter name needs to be set") if f.central_wavelength is None: raise Exception("filter central wavelength needs to be set") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_files = (glob.glob(model_dir + '/seds/*.fits.gz') + glob.glob(model_dir + '/seds/*/*.fits.gz') + glob.glob(model_dir + '/seds/*.fits') + glob.glob(model_dir + '/seds/*/*.fits')) par_table = load_parameter_table(model_dir) if len(sed_files) == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(len(sed_files), model_dir)) # Find out apertures first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures # Set up convolved fluxes fluxes = [ConvolvedFluxes(model_names=np.zeros(len(sed_files), dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(len(filters))] # Set up list of binned filters binned_filters = [] binned_nu = None # Loop over SEDs b = ProgressBar(len(sed_files)) for im, sed_file in enumerate(sed_files): log.debug('Convolving {0}'.format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu') # Check if filters need to be re-binned try: assert binned_nu is not None np.testing.assert_array_almost_equal_nulp(s.nu.value, binned_nu.value, 100) except AssertionError: log.info('Rebinning filters') binned_filters = [f.rebin(s.nu) for f in filters] binned_nu = s.nu b.update() # Convolve for i, f in enumerate(binned_filters): fluxes[i].central_wavelength = f.central_wavelength fluxes[i].apertures = apertures fluxes[i].model_names[im] = s.name if n_ap == 1: fluxes[i].flux[im] = np.sum(s.flux * f.response) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2)) else: fluxes[i].flux[im, :] = np.sum(s.flux * f.response, axis=1) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2, axis=1)) for i, f in enumerate(binned_filters): fluxes[i].sort_to_match(par_table['MODEL_NAME']) fluxes[i].write(model_dir + '/convolved/' + f.name + '.fits', overwrite=overwrite)
def check_conesearch_sites(destdir=os.curdir, verbose=True, parallel=True, url_list='default'): """ Validate Cone Search Services. .. note:: URLs are unescaped prior to validation. Only check queries with ``<testQuery>`` parameters. Does not perform meta-data and erroneous queries. Parameters ---------- destdir : str, optional Directory to store output files. Will be created if does not exist. Existing files with these names will be deleted or replaced: * conesearch_good.json * conesearch_warn.json * conesearch_exception.json * conesearch_error.json verbose : bool, optional Print extra info to log. parallel : bool, optional Enable multiprocessing. url_list : list of string, optional Only check these access URLs against ``astroquery.vo_conesearch.validator.conf.conesearch_master_list`` and ignore the others, which will not appear in output files. By default, check those in ``astroquery.vo_conesearch.validator.conf.conesearch_urls``. If `None`, check everything. Raises ------ IOError Invalid destination directory. timeout URL request timed out. ValidationMultiprocessingError Multiprocessing failed. """ if url_list == 'default': url_list = conf.conesearch_urls if (not isinstance(destdir, six.string_types) or len(destdir) == 0 or os.path.exists(destdir) and not os.path.isdir(destdir)): raise IOError('Invalid destination directory') # pragma: no cover if not os.path.exists(destdir): os.mkdir(destdir) # Output dir created by votable.validator out_dir = os.path.join(destdir, 'results') if not os.path.exists(out_dir): os.mkdir(out_dir) # Output files db_file = OrderedDict() db_file['good'] = os.path.join(destdir, 'conesearch_good.json') db_file['warn'] = os.path.join(destdir, 'conesearch_warn.json') db_file['excp'] = os.path.join(destdir, 'conesearch_exception.json') db_file['nerr'] = os.path.join(destdir, 'conesearch_error.json') # JSON dictionaries for output files js_tree = {} for key in db_file: js_tree[key] = VOSDatabase.create_empty() # Delete existing files, if any, to be on the safe side. # Else can cause confusion if program exited prior to # new files being written but old files are still there. if os.path.exists(db_file[key]): # pragma: no cover os.remove(db_file[key]) if verbose: log.info('Existing file {0} deleted'.format(db_file[key])) # Master VO database from registry. Silence all the warnings. with warnings.catch_warnings(): warnings.simplefilter('ignore') js_mstr = VOSDatabase.from_registry( conf.conesearch_master_list, encoding='binary', show_progress=verbose) # Validate only a subset of the services. if url_list is not None: # Make sure URL is unique and fixed. url_list = set(map( unescape_all, [cur_url.encode('utf-8') if isinstance(cur_url, str) else cur_url for cur_url in url_list])) uniq_rows = len(url_list) url_list_processed = [] # To track if given URL is valid in registry if verbose: log.info('Only {0}/{1} site(s) are validated'.format(uniq_rows, len(js_mstr))) # Validate all services. else: uniq_rows = len(js_mstr) key_lookup_by_url = {} # Process each catalog in the registry. for cur_key, cur_cat in js_mstr.get_catalogs(): cur_url = cur_cat['url'].encode('utf-8') # Skip if: # a. not a Cone Search service # b. not in given subset, if any if ((cur_cat['cap_type'] != b'conesearch') or (url_list is not None and cur_url not in url_list)): continue # Use testQuery to return non-empty VO table with max verbosity. testquery_pars = parse_cs(cur_cat['ivoid'], cur_cat['cap_index']) cs_pars_arr = ['{}={}'.format(key, testquery_pars[key]).encode('utf-8') for key in testquery_pars] cs_pars_arr += [b'VERB=3'] # Track the service. key_lookup_by_url[cur_url + b'&'.join(cs_pars_arr)] = cur_key if url_list is not None: url_list_processed.append(cur_url) # Give warning if any of the user given subset is not in the registry. if url_list is not None: url_list_skipped = url_list - set(url_list_processed) n_skipped = len(url_list_skipped) if n_skipped > 0: warn_str = '{0} not found in registry! Skipped:\n'.format( n_skipped) for cur_url in url_list_skipped: warn_str += '\t{0}\n'.format(cur_url) warnings.warn(warn_str, AstropyUserWarning) all_urls = list(key_lookup_by_url) timeout = data.conf.remote_timeout map_args = [(out_dir, url, timeout) for url in all_urls] # Validate URLs if parallel: pool = multiprocessing.Pool() try: mp_list = pool.map(_do_validation, map_args) except Exception as exc: # pragma: no cover raise ValidationMultiprocessingError( 'An exception occurred during parallel processing ' 'of validation results: {0}'.format(exc)) else: mp_list = map(_do_validation, map_args) # Categorize validation results for r in mp_list: db_key = r['out_db_name'] cat_key = key_lookup_by_url[r.url] cur_cat = js_mstr.get_catalog(cat_key) _copy_r_to_cat(r, cur_cat) js_tree[db_key].add_catalog(cat_key, cur_cat) # Write to HTML html_subsets = result.get_result_subsets(mp_list, out_dir) html.write_index(html_subsets, all_urls, out_dir) if parallel: html_subindex_args = [(out_dir, html_subset, uniq_rows) for html_subset in html_subsets] pool.map(_html_subindex, html_subindex_args) else: for html_subset in html_subsets: _html_subindex((out_dir, html_subset, uniq_rows)) # Write to JSON n = {} n_tot = 0 for key in db_file: n[key] = len(js_tree[key]) n_tot += n[key] js_tree[key].to_json(db_file[key], overwrite=True) if verbose: log.info('{0}: {1} catalog(s)'.format(key, n[key])) # Checksum if verbose: log.info('total: {0} out of {1} catalog(s)'.format(n_tot, uniq_rows)) if n['good'] == 0: # pragma: no cover warnings.warn( 'No good sites available for Cone Search.', AstropyUserWarning)
def convolve_model_dir_monochromatic(model_dir, overwrite=False, max_ram=8, wav_min=-np.inf * u.micron, wav_max=np.inf * u.micron): """ Convolve all the model SEDs in a model directory Parameters ---------- model_dir : str The path to the model directory overwrite : bool, optional Whether to overwrite the output files max_ram : float, optional The maximum amount of RAM that can be used (in Gb) wav_min : float, optional The minimum wavelength to consider. Only wavelengths above this value will be output. wav_max : float, optional The maximum wavelength to consider. Only wavelengths below this value will be output. """ modpar = parfile.read(os.path.join(model_dir, 'models.conf'), 'conf') if modpar.get('version', 1) > 1: raise ValueError("monochromatic filters are no longer used for new-style model directories") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_files = (glob.glob(model_dir + '/seds/*.fits.gz') + glob.glob(model_dir + '/seds/*/*.fits.gz') + glob.glob(model_dir + '/seds/*.fits') + glob.glob(model_dir + '/seds/*/*.fits')) par_table = load_parameter_table(model_dir) # Find number of models n_models = len(sed_files) if n_models == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(n_models, model_dir)) # Find out apertures and wavelengths first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures n_wav = first_sed.n_wav wavelengths = first_sed.wav # For model grids that are very large, it is not possible to compute all # fluxes in one go, so we need to process in chunks in wavelength space. chunk_size = min(n_wav, int(np.floor(max_ram * 1024. ** 3 / (4. * 2. * n_models * n_ap)))) if chunk_size == n_wav: log.info("Producing all monochromatic files in one go") else: log.info("Producing monochromatic files in chunks of {0}".format(chunk_size)) filters = Table() filters['wav'] = wavelengths filters['filter'] = np.zeros(wavelengths.shape, dtype='S10') # Figure out range of wavelength indices to use # (wavelengths array is sorted in reverse order) jlo = n_wav - 1 - (wavelengths[::-1].searchsorted(wav_max) - 1) jhi = n_wav - 1 - wavelengths[::-1].searchsorted(wav_min) chunk_size = min(chunk_size, jhi - jlo + 1) # Loop over wavelength chunks for jmin in range(jlo, jhi, chunk_size): # Find upper wavelength to compute jmax = min(jmin + chunk_size - 1, jhi) log.info('Processing wavelengths {0} to {1}'.format(jmin, jmax)) # Set up convolved fluxes fluxes = [ConvolvedFluxes(model_names=np.zeros(n_models, dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(chunk_size)] b = ProgressBar(len(sed_files)) # Loop over SEDs for im, sed_file in enumerate(sed_files): b.update() log.debug('Processing {0}'.format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu') # Convolve for j in range(chunk_size): fluxes[j].central_wavelength = wavelengths[j + jmin] fluxes[j].apertures = apertures fluxes[j].model_names[im] = s.name if n_ap == 1: fluxes[j].flux[im] = s.flux[0, j + jmin] fluxes[j].error[im] = s.error[0, j + jmin] else: fluxes[j].flux[im, :] = s.flux[:, j + jmin] fluxes[j].error[im, :] = s.error[:, j + jmin] for j in range(chunk_size): fluxes[j].sort_to_match(par_table['MODEL_NAME']) fluxes[j].write('{0:s}/convolved/MO{1:03d}.fits'.format(model_dir, j + jmin + 1), overwrite=overwrite) filters['filter'][j + jmin] = "MO{0:03d}".format(j + jmin + 1) return filters
def _convolve_model_dir_1(model_dir, filters, overwrite=False): for f in filters: if f.name is None: raise Exception("filter name needs to be set") if f.central_wavelength is None: raise Exception("filter central wavelength needs to be set") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + "/convolved"): os.mkdir(model_dir + "/convolved") # Find all SED files to convolve sed_files = ( glob.glob(model_dir + "/seds/*.fits.gz") + glob.glob(model_dir + "/seds/*/*.fits.gz") + glob.glob(model_dir + "/seds/*.fits") + glob.glob(model_dir + "/seds/*/*.fits") ) par_table = load_parameter_table(model_dir) if len(sed_files) == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(len(sed_files), model_dir)) # Find out apertures first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures # Set up convolved fluxes fluxes = [ ConvolvedFluxes( model_names=np.zeros(len(sed_files), dtype="U30" if six.PY3 else "S30"), apertures=apertures, initialize_arrays=True, ) for i in range(len(filters)) ] # Set up list of binned filters binned_filters = [] binned_nu = None # Loop over SEDs b = ProgressBar(len(sed_files)) for im, sed_file in enumerate(sed_files): log.debug("Convolving {0}".format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order="nu") # Check if filters need to be re-binned try: assert binned_nu is not None np.testing.assert_array_almost_equal_nulp(s.nu.value, binned_nu.value, 100) except AssertionError: log.info("Rebinning filters") binned_filters = [f.rebin(s.nu) for f in filters] binned_nu = s.nu b.update() # Convolve for i, f in enumerate(binned_filters): fluxes[i].central_wavelength = f.central_wavelength fluxes[i].apertures = apertures fluxes[i].model_names[im] = s.name if n_ap == 1: fluxes[i].flux[im] = np.sum(s.flux * f.response) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2)) else: fluxes[i].flux[im, :] = np.sum(s.flux * f.response, axis=1) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2, axis=1)) for i, f in enumerate(binned_filters): fluxes[i].sort_to_match(par_table["MODEL_NAME"]) fluxes[i].write(model_dir + "/convolved/" + f.name + ".fits", overwrite=overwrite)