def get_besancon_model_file(filename, verbose=True, save=True, savename=None, overwrite=True): """ Download a Besancon model from the website Parameters ---------- filename : string The besancon filename, with format ##########.######.resu verbose : bool Print details about the download process save : bool Save the table after acquiring it? savename : None or string If not specified, defaults to the .resu table name overwrite : bool Overwrite the file if it exists? Defaults to True because the .resu tables should have unique names by default, so there's little risk of accidentally overwriting important information """ url = url_download+filename elapsed_time = 0 t0 = time.time() sys.stdout.write("\n") while 1: sys.stdout.write(u"\r") try: U = urllib2.urlopen(url,timeout=5) if verbose: print "" print "Loading page..." results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: results = page.read() break except urllib2.URLError: sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) time.sleep(30) elapsed_time += 30 continue except socket.timeout: sys.stdout.write(u"Waiting 30s for model to finish (elapsed wait time %is, total %i)\r" % (elapsed_time,time.time()-t0)) time.sleep(30) elapsed_time += 30 continue if save: if savename is None: savename = filename if not overwrite and os.path.exists(savename): raise IOError("File %s already exists." % savename) outf = open(savename,'w') print >>outf,results outf.close() return parse_besancon_model_string(results)
def get_nrao_image(lon, lat, system='galactic', epoch='J2000', size=1.0, max_rms=1e4, band="", verbose=True, savename=None, save=True, overwrite=False, directory='./', get_uvfits=False): """ Search for and download Parameters ---------- lon : float lat : float Right ascension and declination or glon/glat system : ['celestial','galactic'] System of lon/lat. Can be any valid coordinate system supported by the coords package epoch : string Epoch of the coordinate system (e.g., B1950, J2000) savename : None or string filename to save fits file as. If None, will become G###.###p###.###_(survey).fits size : float Size of search radius (arcminutes) max_rms : float Maximum allowable noise level in the image (mJy) verbose : bool Print out extra error messages? save : bool Save FITS file? overwrite : bool Overwrite if file already exists? directory : string Directory to store file in. Defaults to './'. get_uvfits : bool Get the UVfits files instead of the IMfits files? Examples -------- >>> fitsfile = get_nrao_image(49.489,-0.37) """ if band not in valid_bands: raise ValueError("Invalid band. Valid bands are: %s" % valid_bands) ra,dec = coords.Position([lon,lat],system=system,equinox=epoch).j2000() radecstr = coords.Position([ra,dec],system='celestial',equinox='J2000').hmsdms().replace(":"," ") glon,glat = coords.Position([ra,dec],system='celestial',equinox='J2000').galactic() # Construct request request = {} request["nvas_pos"] = radecstr request["nvas_rad"] = size request["nvas_rms"] = max_rms request["nvas_scl"] = size request["submit"] = "Search" request["nvas_bnd"] = band # create the request header data request = urllib.urlencode(request) # load the URL as text U = urllib.urlopen(request_URL, request) # read results with progressbar results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) if get_uvfits: links = uvfits_re.findall(results) else: links = imfits_re.findall(results) configurations = config_re.findall(results) if len(links) == 0: if verbose: print "No matches found at ra,dec = %s." % (radecstr) return [] if verbose > 1: print "Configurations: " print "\n".join(["%40s: %20s" % (L,C) for L,C in zip(links,configurations)]) if save and not os.path.exists(directory): os.mkdir(directory) if save: opener = urllib2.build_opener() if verbose: print "Found %i imfits files" % len(links) images = [] for link,config in zip(links,configurations): # Get image filename basename = os.path.basename(link) # Get the file U = opener.open(link) if verbose: print "Downloading image from %s" % link results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: results = U.read() S = StringIO.StringIO(results) try: fitsfile = pyfits.open(S,ignore_missing_end=True) except IOError: S.seek(0) G = gzip.GzipFile(fileobj=S) fitsfile = pyfits.open(G,ignore_missing_end=True) # Get Multiframe ID from the header images.append(fitsfile) if save: h0 = fitsfile[0].header freq_ghz = h0['CRVAL3'] / 1e9 for bn, bandlimits in band_freqs.iteritems(): if freq_ghz < bandlimits[1] and freq_ghz > bandlimits[0]: bandname = bn obj = str(h0['OBJECT']).strip() program = h0['OBSERVER'].strip() h0['CONFIG'] = config if savename is None: if get_uvfits: filename = "VLA_%s_G%07.3f%+08.3f_%s_%s.uvfits" % (bandname,glon,glat,obj,program) else: filename = "VLA_%s_G%07.3f%+08.3f_%s_%s.fits" % (bandname,glon,glat,obj,program) else: filename = savename # Set final directory and file names final_file = directory + '/' + filename if verbose: print "Saving file %s" % final_file fitsfile.writeto(final_file, clobber=overwrite) return images
def get_catalog_gal( self, glon, glat, directory=None, radius=1, save=False, verbose=True, savename=None, overwrite=False ): """ Get all sources in the catalog within some radius Parameters ---------- glon : float glat : float Galactic latitude and longitude at the center directory : None or string Directory to download files into. Defaults to self.directory radius : float Radius in which to search for catalog entries in arcminutes savename : string or None The file name to save the catalog to. If unspecified, will save as UKIDSS_catalog_G###.###-###.###_r###.fits.gz, where the #'s indicate galactic lon/lat and radius Returns ------- List of pyfits.primaryHDU instances containing FITS tables Example ------- >>> R = UKIDSSQuery() >>> data = R.get_catalog_gal(10.625,-0.38,radius=0.1) >>> bintable = data[0][1] """ # Construct request self.request = {} self.request["database"] = self.database self.request["programmeID"] = verify_programme_id(self.programmeID, querytype="catalog") self.request["from"] = "source" self.request["formaction"] = "region" self.request["ra"] = glon self.request["dec"] = glat self.request["sys"] = "G" self.request["radius"] = radius self.request["xSize"] = "" self.request["ySize"] = "" self.request["boxAlignment"] = "RADec" self.request["emailAddress"] = "" self.request["format"] = "FITS" self.request["compress"] = "GZIP" self.request["rows"] = 1 self.request["select"] = "*" self.request["where"] = "" self.query_str = url_getcatalog + urllib.urlencode(self.request) if directory is None: directory = self.directory # Retrieve page page = self.opener.open(url_getcatalog + urllib.urlencode(self.request)) if verbose: print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) else: results = page.read() # Parse results for links format = formatter.NullFormatter() # create default formatter htmlparser = LinksExtractor(format) # create new parser object htmlparser.feed(results) htmlparser.close() links = list(set(htmlparser.get_links())) # Loop through links and retrieve FITS tables c = 0 data = [] for link in links: if not "8080" in link: c = c + 1 if not os.path.exists(directory): os.mkdir(directory) if save: if savename is None: savename = "UKIDSS_catalog_G%07.3f%+08.3f_r%03i.fits.gz" % (glon, glat, radius) filename = directory + "/" + savename U = self.opener.open(link) if verbose: print "Downloading catalog %s" % link results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: results = U.read() S = StringIO.StringIO(results) try: fitsfile = pyfits.open(S, ignore_missing_end=True) except IOError: S.seek(0) G = gzip.GzipFile(fileobj=S) fitsfile = pyfits.open(G, ignore_missing_end=True) data.append(fitsfile) if save: fitsfile.writeto(filename.rstrip(".gz"), clobber=overwrite) return data
def get_images_radius( self, ra, dec, radius, filter="all", frametype="stack", directory=None, n_concurrent=1, save=True, verbose=True, overwrite=False, ): """ Get all images within some radius of a specified RA/Dec Parameters ---------- ra : float dec : float ra/dec center to search around radius : float Radius of circle to search within filter : ['all','J','H','K'] The color filter to download. frametype : ['stack', 'normal', 'interleave', 'deep%stack', 'confidence', 'difference', 'leavstack', 'all'] The type of image directory : None or string Directory to download files into. Defaults to self.directory verbose : bool Print out extra error messages? save : bool Save FITS file? overwrite : bool Overwrite if file already exists? n_concurrent : int Number of concurrent download threads to start Examples -------- >>> R = UKIDSSQuery() >>> fitsfile = R.get_image_gal(10.5,0.0) """ # Check for validity of requested frame_type if frametype not in frame_types: raise ValueError("Invalide frame type. Valid frame types are: %s" % frame_types) if filter not in self.filters: raise ValueError("Invalide filter. Valid filters are: %s" % self.filters.keys()) if directory is None: directory = self.directory # Construct self.request self.request = {} self.request["database"] = self.database self.request["programmeID"] = verify_programme_id(self.programmeID, querytype="image") self.request["userSelect"] = "default" self.request["obsType"] = "object" self.request["frameType"] = frametype self.request["filterID"] = self.filters[filter] self.request["minRA"] = str(round(ra - radius / cos(radians(dec)), 2)) self.request["maxRA"] = str(round(ra + radius / cos(radians(dec)), 2)) self.request["formatRA"] = "degrees" self.request["minDec"] = str(dec - radius) self.request["maxDec"] = str(dec + radius) self.request["formatDec"] = "degrees" self.request["startDay"] = 0 self.request["startMonth"] = 0 self.request["startYear"] = 0 self.request["endDay"] = 0 self.request["endMonth"] = 0 self.request["endYear"] = 0 self.request["dep"] = 0 self.request["mfid"] = "" self.request["lmfid"] = "" self.request["fsid"] = "" self.request["rows"] = 1000 self.query_str = url_getimages + urllib.urlencode(self.request) # Retrieve page page = self.opener.open(url_getimages, urllib.urlencode(self.request)) if verbose: print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) else: results = page.read() # Parse results for links format = formatter.NullFormatter() htmlparser = LinksExtractor(format) htmlparser.feed(results) htmlparser.close() links = htmlparser.get_links() # Loop through links and retrieve FITS images for link in links: if not os.path.exists(directory): os.mkdir(directory) if not os.path.exists(directory + "/" + frametype): os.mkdir(directory + "/" + frametype) if "fits_download" in link and "_cat.fits" not in link and "_two.fit" not in link: # Get image filename basename = os.path.basename(link.split("&")[0]) temp_file = directory + "/" + frametype + "/" + basename if verbose: print "Downloading %s..." % basename p = mp.Process(target=progressbar.retrieve, args=(link, temp_file, self.opener)) else: p = mp.Process(target=urllib.urlretrieve, args=(link, temp_file)) p.start() while True: if len(mp.active_children()) < n_concurrent: break time.sleep(0.1)
def get_image_gal( self, glon, glat, filter="all", frametype="stack", directory=None, size=1.0, verbose=True, save=True, savename=None, overwrite=False, ): """ Get an image at a specified glon/glat. Size can be specified Parameters ---------- glon : float glat : float Galactic latitude and longitude at the center filter : ['all','J','H','K','H2','Z','Y','Br'] The color filter to download. frametype : ['stack','normal','interleave','deep%stack','confidence','difference','leavstack','all'] The type of image directory : None or string Directory to download files into. Defaults to self.directory size : float Size of cutout (symmetric) in arcminutes verbose : bool Print out extra error messages? save : bool Save FITS file? savename : string or None The file name to save the catalog to. If unspecified, will save as UKIDSS_[band]_G###.###-###.###_[obj].fits.gz, where the #'s indicate galactic lon/lat and [band] and [obj] refer to the filter and the object name overwrite : bool Overwrite if file already exists? Examples -------- >>> R = UKIDSSQuery() >>> fitsfile = R.get_image_gal(10.5,0.0) # get UWISH2 data (as per http://astro.kent.ac.uk/uwish2/main.html) >>> R.database='U09B8v20120403' >>> R.login(username='******',password='******',community='nonSurvey') >>> R.get_image_gal(49.489,-0.27,frametype='leavstack',size=20,filter='H2') """ # Check for validity of requested frame_type if frametype not in frame_types: raise ValueError("Invalide frame type. Valid frame types are: %s" % frame_types) if filter not in self.filters: raise ValueError("Invalide filter. Valid filters are: %s" % self.filters.keys()) # Construct request self.request = {} self.request["database"] = self.database self.request["programmeID"] = verify_programme_id(self.programmeID, querytype="image") self.request["ra"] = glon self.request["dec"] = glat self.request["sys"] = "G" self.request["filterID"] = self.filters[filter] self.request["xsize"] = size self.request["ysize"] = size self.request["obsType"] = "object" self.request["frameType"] = frametype self.request["mfid"] = "" self.query_str = url_getimage + urllib.urlencode(self.request) if directory is None: directory = self.directory # Retrieve page page = self.opener.open(url_getimage, urllib.urlencode(self.request)) if verbose: print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) if verbose == "debug": print url_getimage, urllib.urlencode(self.request) else: results = page.read() # Parse results for links format = formatter.NullFormatter() htmlparser = LinksExtractor(format) htmlparser.feed(results) htmlparser.close() links = htmlparser.get_links() if verbose: print "Found %i targets" % (len(links)) # Loop through links and retrieve FITS images images = [] for link in links: if not os.path.exists(directory): os.mkdir(directory) # Get image filename basename = os.path.basename(link.split("&")[0]).replace(".fit", ".fits.gz") # Get the file U = self.opener.open(link.replace("getImage", "getFImage")) if verbose: print "Downloading image from %s" % link results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: results = U.read() S = StringIO.StringIO(results) try: fitsfile = pyfits.open(S, ignore_missing_end=True) except IOError: S.seek(0) G = gzip.GzipFile(fileobj=S) fitsfile = pyfits.open(G, ignore_missing_end=True) # Get Multiframe ID from the header images.append(fitsfile) if save: h0 = fitsfile[0].header filt = str(h0["FILTER"]).strip() obj = filt + "_" + str(h0["OBJECT"]).strip().replace(":", ".") if savename is None: filename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt, glon, glat, obj) else: filename = savename # Set final directory and file names final_file = directory + "/" + filename if verbose: print "Saving file %s" % final_file fitsfile.writeto(final_file, clobber=overwrite) return images
def get_image_gal(self, glon, glat, filter='all', frametype='stack', directory=None, size=1.0, verbose=True, save=True, savename=None, overwrite=False): """ Get an image at a specified glon/glat. Size can be specified Parameters ---------- glon : float glat : float Galactic latitude and longitude at the center filter : ['all','J','H','K'] The color filter to download. frametype : ['stack','normal','interleave','deep%stack','confidence','difference','all'] The type of image directory : None or string Directory to download files into. Defaults to self.directory size : float Size of cutout (symmetric) in arcminutes verbose : bool Print out extra error messages? save : bool Save FITS file? savename : string or None The file name to save the catalog to. If unspecified, will save as UKIDSS_[band]_G###.###-###.###_[obj].fits.gz, where the #'s indicate galactic lon/lat and [band] and [obj] refer to the filter and the object name overwrite : bool Overwrite if file already exists? Examples -------- >>> R = Query() >>> fitsfile = R.get_image_gal(10.5,0.0) """ # Check for validity of requested frame_type if frametype not in frame_types: raise ValueError("Invalide frame type. Valid frame types are: %s" % frame_types) if filter not in self.filters: raise ValueError("Invalide filter. Valid filters are: %s" % self.filters.keys()) # Construct request request = {} request['database'] = self.database request['programmeID'] = self.programmeID request['ra'] = glon request['dec'] = glat request['sys'] = 'G' request['filterID'] = self.filters[filter] request['xsize'] = size request['ysize'] = size request['obsType'] = 'object' request['frameType'] = frametype request['mfid'] = '' if directory is None: directory = self.directory # Retrieve page page = self.opener.open(url_getimage, urllib.urlencode(request)) if verbose: print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) else: results = page.read() # Parse results for links format = formatter.NullFormatter() htmlparser = LinksExtractor(format) htmlparser.feed(results) htmlparser.close() links = htmlparser.get_links() if verbose: print "Found %i targets" % (len(links)) # Loop through links and retrieve FITS images images = [] for link in links: if not os.path.exists(directory): os.mkdir(directory) # Get image filename basename = os.path.basename( link.split("&")[0]).replace('.fit', '.fits.gz') # Get the file U = self.opener.open(link.replace("getImage", "getFImage")) if verbose: print "Downloading image from %s" % link results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) else: results = U.read() S = StringIO.StringIO(results) try: fitsfile = pyfits.open(S,ignore_missing_end=True) except IOError: S.seek(0) G = gzip.GzipFile(fileobj=S) fitsfile = pyfits.open(G,ignore_missing_end=True) # Get Multiframe ID from the header images.append(fitsfile) if save: h0 = fitsfile[0].header filt = str(h0['FILTER']).strip() obj = filt + "_" + str(h0['OBJECT']).strip().replace(":", ".") if savename is None: filename = "UKIDSS_%s_G%07.3f%+08.3f_%s.fits" % (filt,glon,glat,obj) else: filename = savename # Set final directory and file names final_file = directory + '/' + filename if verbose: print "Saving file %s" % final_file fitsfile.writeto(final_file, clobber=overwrite) return images
def get_images_radius(self, ra, dec, radius, filter='all', frametype='stack', directory=None, n_concurrent=1, save=True, verbose=True, overwrite=False): """ Get all images within some radius of a specified RA/Dec Parameters ---------- ra : float dec : float ra/dec center to search around radius : float Radius of circle to search within filter : ['all','J','H','K'] The color filter to download. frametype : ['stack','normal','interleave','deep%stack','confidence','difference','all'] The type of image directory : None or string Directory to download files into. Defaults to self.directory verbose : bool Print out extra error messages? save : bool Save FITS file? overwrite : bool Overwrite if file already exists? n_concurrent : int Number of concurrent download threads to start Examples -------- >>> R = Query() >>> fitsfile = R.get_image_gal(10.5,0.0) """ # Check for validity of requested frame_type if frametype not in frame_types: raise ValueError("Invalide frame type. Valid frame types are: %s" % frame_types) if filter not in self.filters: raise ValueError("Invalide filter. Valid filters are: %s" % self.filters.keys()) # Construct request request = {} request['database'] = self.database request['programmeID'] = self.programmeID request['userSelect'] = 'default' request['obsType'] = 'object' request['frameType'] = frametype request['filterID'] = self.filters[filter] request['minRA'] = str(round(ra - radius / cos(radians(dec)),2)) request['maxRA'] = str(round(ra + radius / cos(radians(dec)),2)) request['formatRA'] = 'degrees' request['minDec'] = str(dec - radius) request['maxDec'] = str(dec + radius) request['formatDec'] = 'degrees' request['startDay'] = 0 request['startMonth'] = 0 request['startYear'] = 0 request['endDay'] = 0 request['endMonth'] = 0 request['endYear'] = 0 request['dep'] = 0 request['mfid'] = '' request['lmfid'] = '' request['fsid'] = '' request['rows'] = 1000 # Retrieve page page = self.opener.open(url_getimages, urllib.urlencode(request)) if verbose: print "Loading page..." results = progressbar.chunk_read(page, report_hook=progressbar.chunk_report) else: results = page.read() # Parse results for links format = formatter.NullFormatter() htmlparser = LinksExtractor(format) htmlparser.feed(results) htmlparser.close() links = htmlparser.get_links() # Loop through links and retrieve FITS images for link in links: if not os.path.exists(directory): os.mkdir(directory) if not os.path.exists(directory + '/' + frametype): os.mkdir(directory + '/' + frametype) if 'fits_download' in link and '_cat.fits' not in link and '_two.fit' not in link: # Get image filename basename = os.path.basename(link.split("&")[0]) temp_file = directory + '/' + frametype + '/' + basename if verbose: print "Downloading %s..." % basename p = mp.Process( target=urllib.urlretrieve, args=(link, temp_file)) p.start() while True: if len(mp.active_children()) < n_concurrent: break time.sleep(0.1)
def get_magpis_image_gal(glon, glat, survey='bolocam', size=1.0, verbose=False, savename=None, save=True, overwrite=False, directory='./'): """ Get an image at a specified glon/glat. Size can be specified WARNING: MAGPIS has a maxmimum image size of about 2048x2048 Parameters ---------- glon : float glat : float Galactic latitude and longitude at the center survey : string Which MAGPIS survey do you want to cut out? frametype : ['stack','normal','interleave','deep%stack','confidence','difference','all'] The type of image savename : None or string filename to save fits file as. If None, will become G###.###p###.###_(survey).fits size : float Size of cutout (symmetric) in arcminutes verbose : bool Print out extra error messages? save : bool Save FITS file? overwrite : bool Overwrite if file already exists? directory : string Directory to store file in. Defaults to './'. Examples -------- >>> fitsfile = get_magpis_image_gal(10.5,0.0) """ if survey not in surveys: raise ValueError("Invalide survey. Valid surveys are: %s" % surveys) # Construct request request = {} request["Survey"] = survey # NOTE: RA is passed as a 2-part string, DEC is not used. Whoops! request["RA"] = "%s %s" % (glon,glat) # request["Dec"] = request["Equinox"] = "Galactic" request["ImageSize"] = size request["ImageType"] = "FITS" # these options are not used # optional request["MaxInt"] = 10 # optional request["Epochs"] = # optional request["Fieldname"] = # create the request header data request = urllib.urlencode(request) # load the URL as text U = urllib.urlopen(url_gpscutout, request) # read results with progressbar results = progressbar.chunk_read(U, report_hook=progressbar.chunk_report) # turn the text into a StringIO object for FITS reading S = StringIO.StringIO(results) fitsfile = pyfits.open(S) if save: if savename is None: savename = "G%08.4f%+09.4f_%s.fits" % (glon,glat,survey) if directory[-1] != '/': directory += '/' fitsfile.writeto(directory+savename, clobber=overwrite) return fitsfile