def test_response_block_properties(client): res = client.search(a.Time('2020/3/4', '2020/3/6'), a.Instrument('aia'), a.Wavelength(171 * u.angstrom), a.Sample(10 * u.minute), response_format="legacy") properties = res.response_block_properties() assert len(properties) == 0
def test_vso_fetch_hmi(tmpdir): start_time = "2017-01-25" end_time = "2017-01-25T23:59:59" results = Fido.search(a.Time(start_time, end_time), a.Instrument.hmi & a.Physobs.los_magnetic_field, a.Sample(1 * u.minute)) files = Fido.fetch(results[0, 0], path=tmpdir) assert len(files) == 1
def __init__(self, times=("2020/05/25", "2020/05/27"), cadence=1 * u.hour, instrument=None, path=PATH): self.attrsTime = a.Time(TimeRange(times)) self.cadence = a.Sample(cadence) self.instrument = a.Instrument(instrument) self.path = path print(f"Using {self.path}") self.dfiles = None
def download_files(start_time, end_time, series_name, segs, email_for_notification, interval, download_path): segVal=a.jsoc.Segment(segs[0]) for i in range(1, len(segs)): segVal&=a.jsoc.Segment(segs[i]) result = Fido.search( a.Time(start_time, end_time), a.jsoc.Series(series_name), a.jsoc.Notify(email_for_notification), a.Sample(interval), segVal) downloaded_files=Fido.fetch(result, path=os.path.join(download_path, '{file}')) print('Files downloaded.')
def grab(dirPath, wavelength, date, time, download=False): """ Return a previously downloaded AIA file according to search criteria. Simple method to return a specific AIA file that you've already downloaded, or download a file matching your search criteria if it doesn't exist in the specified directory. Inputs: ----------- dirpath: path to search for file (string, e.g. './AIAfiles') wavelength: wavelength in angstroms (string, e.g. '94') date: string, format yyyy_mm_dd time: string, format hh_mm[_ss] - second can be specified if known download: boolean, whether to download an image if it is not found Outputs: ----------- A string containing the path to the file found """ searchPath = dirPath + '/aia_lev1_' + wavelength + 'a_' + date + 't' \ + time + '*.fits' foundFiles = glob.glob(searchPath) if len(foundFiles) == 0: if not download: raise Exception('No files found that match input!') else: date = date.replace('_', '-') time = time.replace('_', ':') start = Time(date + 'T' + time) end = start + 59*u.second wv = int(wavelength) result = Fido.search(a.Time(start, end), a.Instrument.aia, a.Wavelength(wv*u.angstrom), a.Sample(1*u.minute)) path = dirPath + '/{file}' downd = Fido.fetch(result, path=path) return downd elif len(foundFiles) > 1: print('Multiple files match input! Returning the first file.') return foundFiles[0]
def vso_search(trange=None, instrument=None, wavelength=None, sample=None, provider=None, source=None): try: from sunpy.net import Fido, attrs as a except: print('VSO_SEARCH: Error, requires sunpy. Sunpy not installed?') return None import astropy.units as u if not trange or not instrument: print( 'VSO_SEARCH: Error, must specify a valid time range and instrument' ) return None try: times = a.Time(Time(trange[0]).iso, Time(trange[1]).iso) except: print('VSO_SEARCH: Error, invalid time range') return None inst = a.Instrument(instrument) paras = [times, inst] if wavelength: wave = a.Wavelength(wavelength * u.angstrom) paras = paras + [wave] if sample is not None: sample = a.Sample(sample * u.second) # in units of seconds paras = paras + [sample] if provider is not None: provider = a.Provider(provider) paras = paras + [provider] if source is not None: source = a.Source(source) paras = paras + [source] qr = Fido.search(*paras) return qr
def combined_aia(fixed_map, time): atime = a.Sample(24 * u.hour) & a.Time(time, time + astropy.time.TimeDelta(1e-3)) aia = a.Instrument.aia & a.Wavelength(17 * u.nm, 18 * u.nm) res = Fido.search(atime, aia) files = Fido.fetch(res[:, 0]) map_aia = sunpy.map.Map(files).resample((1024, 1024) * u.pix) output, footprint = reproject_interp(fixed_map, map_aia.wcs, map_aia.data.shape) out_fixed = sunpy.map.Map(output, map_aia.wcs) fig = plt.figure(figsize=(20, 15)) ax1 = fig.add_subplot(1, 1, 1, projection=map_aia) map_aia.plot(axes=ax1) out_fixed.plot(axes=ax1, alpha=0.5) plt.show() plt.plot(map_aia.data.flatten(), out_fixed.data.flatten(), ',')
def aia_pch_data_download(savepath='', waves=[171, 193, 211, 304]): warnings.simplefilter('ignore', category=AstropyDeprecationWarning) warnings.simplefilter('ignore', category=ErfaWarning) blank_results = Fido.search(a.jsoc.Time('2030/1/1', '2030/1/2'), a.jsoc.Series('aia.lev1_euv_12s')) base = parse_time('2019/01/01 00:00:00') ending = parse_time('2020/10/01 00:00:00') date_list = [ dt for dt in rrule( MONTHLY, dtstart=base.to_datetime(), until=ending.to_datetime()) ] for wave in waves: for ii in range(0, len(date_list) - 1): if not savepath: save_dir = os.path.abspath(os.path.curdir) else: save_dir = os.path.abspath(savepath) directories = np.str(wave) + date_list[ii].strftime("/%Y/%m") save_path = os.path.join(save_dir, directories) os.makedirs(save_path, exist_ok=True) results = blank_results while results.file_num == 0: results = Fido.search( a.jsoc.Time(date_list[ii].strftime('%Y/%m/%d'), date_list[ii + 1].strftime('%Y/%m/%d')), a.jsoc.Notify('*****@*****.**'), a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Wavelength(wave * u.angstrom), a.Sample(3 * u.hour)) expected_number = 224. # 8 images per day, 28 days min. neededfileindex = [] if (results.file_num / expected_number) > 0.70: downloaded_file = Fido.fetch(results, path=save_path + '/{file}') num_missing = len(downloaded_file.errors) retry_count = 0 print('Validating Downloads...') partial = [ try_fits_file(qfile) for qfile in downloaded_file.data ] num_missing += np.sum(partial) while num_missing > 0: downloaded_file = Fido.fetch(results, path=save_path + '/{file}') num_missing = len(downloaded_file.errors) print('Validating Downloads...') partial = [ try_fits_file(qfile) for qfile in downloaded_file.data ] num_missing += np.sum(partial) retry_count += 1 print(f'Retry number {retry_count} to catch errors... ') if retry_count >= 20: num_missing = 0 print('Error in ' + date_list[ii].strftime('%Y/%m/%d') + ' ' + np.str(wave)) neededfileindex += [date_list[ii]] if results.file_num == len( fnmatch.filter(os.listdir(save_path), '*.image_lev1.fits')): print('Download between ' + date_list[ii].strftime('%Y/%m/%d') + ' and ' + date_list[ii + 1].strftime('%Y/%m/%d') + ' successful') print('Removing spikes files...') for s in downloaded_file.data: if "spikes" in s: os.remove(s) else: print('Expected ' + np.str(results.file_num) + ' files.') print('Got ' + np.str( len( fnmatch.filter(os.listdir(save_path), '*.image_lev1.fits'))) + ' files.') with open( save_dir + '/' + np.str(wave) + 'NeededFileIndex_' + date_list[ii].strftime('%Y') + '.txt', 'w') as f: for item in neededfileindex: f.write("%s\n" % item)
import astropy.units as u from astropy.coordinates import SkyCoord import sunpy.coordinates.wcs_utils import sunpy.map from sunpy.net import Fido from sunpy.net import attrs as a ############################################################################## # The first step is to download some data, we are going to get an image from # early 2011 when the STEREO spacecraft were roughly 90 deg separated from the # Earth. stereo = (a.Source('STEREO_B') & a.Instrument("EUVI") & a.Time('2011-01-01', '2011-01-01T00:10:00')) aia = (a.Instrument.aia & a.Sample(24 * u.hour) & a.Time('2011-01-01', '2011-01-02')) wave = a.Wavelength(30 * u.nm, 31 * u.nm) result = Fido.search(wave, aia | stereo) ############################################################################### # Let's inspect the result print(result) ############################################################################## # and download the files downloaded_files = Fido.fetch(result) print(downloaded_files) ##############################################################################
def chortle(cr): # Read configuration file config = configparser.ConfigParser() config.read('config.cfg') # Specify directory structures from configuration file datdir = config['paths']['datdir'] magdir = config['paths']['magdir'] outdir = config['paths']['outdir'] # Specify timing parameters nsday = 1 # Grab the start and end dates for this rotation t0 = sunpy.coordinates.sun.carrington_rotation_time(cr) t0.format = 'datetime' t0 = t0.value t1 = sunpy.coordinates.sun.carrington_rotation_time(cr + 1) t1.format = 'datetime' t1 = t1.value # For now, round these to the nearest day if t0.hour > 12: t0 = t0 + datetime.timedelta(days=1) t0 = datetime.datetime(t0.year, t0.month, t0.day) if t1.hour > 12: t1 = t1 + datetime.timedelta(days=1) t1 = datetime.datetime(t1.year, t1.month, t1.day) # Download appropriate data search_aia = (a.Instrument('AIA') & a.Sample(nsday * u.day) & a.Time(t0, t1)) res_aia = Fido.search(a.Wavelength(19.3 * u.nm), search_aia) search_sta = (a.vso.Source('STEREO_A') & a.Instrument('SECCHI') & a.Detector('EUVI') & a.Sample(nsday * u.day) & a.Time(t0, t1)) res_sta = Fido.search(a.Wavelength(19.5 * u.nm), search_sta) search_stb = (a.vso.Source('STEREO_B') & a.Instrument('SECCHI') & a.Detector('EUVI') & a.Sample(nsday * u.day) & a.Time(t0, t1)) res_stb = Fido.search(a.Wavelength(19.5 * u.nm), search_stb) # Running into some errors with automatically fetching HMI data # Manually download and load for now... #c = drms.Client() #c.pkeys('hmi_synoptic_mr_polfil_720s') #res_hmi = Fido.search(a.jsoc.Series('hmi_synoptic_mr_polfil_720s') & a.jsoc.PrimeKey('crnum', '2193')) files_aia = Fido.fetch(res_aia, path=datdir + 'aia/') files_aia.sort() files_sta = Fido.fetch(res_sta, path=datdir + 'sta/') files_sta.sort() files_stb = Fido.fetch(res_stb, path=datdir + 'stb/') files_stb.sort() skip_aia = skip_sta = skip_stb = False if len(files_aia) == 0: skip_aia = True if len(files_sta) == 0: skip_sta = True if len(files_stb) == 0: skip_stb = True ## Generate some blank storage arrays oshape = [720, 1440] chmap = np.zeros(oshape, dtype=np.double) chmap[:, :] = np.nan ## Grab a synoptic magnetogram br0 = sunpy.io.fits.read(magdir + 'hmi.synoptic_mr_polfil_720s.' + str(cr) + '.Mr_polfil.fits')[1].data br = sunpy.image.resample.resample(br0, oshape, method='linear') ## Iterate through this data, reprojecting as you go for file in files_aia: # Read in data map_aia = sunpy.map.Map(file) # Check for bad files if map_aia.exposure_time == 0: continue # Construct an output header header = sunpy.map.make_fitswcs_header( np.empty(oshape), SkyCoord(0, 0, unit=u.deg, frame="heliographic_carrington", obstime=map_aia.date), scale=[180 / oshape[0], 360 / oshape[1]] * u.deg / u.pix, projection_code="CAR") #header['crval2'] = 0 out_wcs = WCS(header) # Reproject array, footprint = reproject_interp((map_aia.data, map_aia.wcs), out_wcs, shape_out=oshape) omap_aia = sunpy.map.Map((array, header)) omap_aia.plot_settings = map_aia.plot_settings # Normalize data to exposure time omap_aia_data = (omap_aia.data / (map_aia.exposure_time / u.second)).value # Condense the reprojected map minimums into chmap chmap = np.fmin(chmap, omap_aia_data) chmap_aia = np.copy(chmap) ## Generate some blank storage arrays oshape = [720, 1440] chmap = np.zeros(oshape, dtype=np.double) chmap[:, :] = np.nan ## Iterate through this data, reprojecting as you go for file in files_sta: # Read in data map_sta = sunpy.map.Map(file) # Check for bad files if map_sta.exposure_time == 0: continue # Construct an output header header = sunpy.map.make_fitswcs_header( np.empty(oshape), SkyCoord( 0, 0, unit=u.deg, frame="heliographic_carrington", #frame="heliographic_stonyhurst", obstime=map_sta.date), #reference_pixel=[0,(oshape[0] - 1)/2.]*u.pix, scale=[180 / oshape[0], 360 / oshape[1]] * u.deg / u.pix, projection_code="CAR") #header['crval2'] = 0 out_wcs = WCS(header) # Reproject array, footprint = reproject_interp((map_sta.data, map_sta.wcs), out_wcs, shape_out=oshape) omap_sta = sunpy.map.Map((array, header)) omap_sta.plot_settings = map_sta.plot_settings # Normalize data to exposure time omap_sta_data = (omap_sta.data / (map_sta.exposure_time / u.second)).value # Condense the reprojected map minimums into chmap chmap = np.fmin(chmap, omap_sta_data) chmap_sta = np.copy(chmap) ## Generate some blank storage arrays oshape = [720, 1440] chmap = np.zeros(oshape, dtype=np.double) chmap[:, :] = np.nan ## Iterate through this data, reprojecting as you go for file in files_stb: # Read in data map_stb = sunpy.map.Map(file) # Check for bad files if map_stb.exposure_time == 0: continue # Construct an output header header = sunpy.map.make_fitswcs_header( np.empty(oshape), SkyCoord( 0, 0, unit=u.deg, frame="heliographic_carrington", #frame="heliographic_stonyhurst", obstime=map_stb.date), #reference_pixel=[0,(oshape[0] - 1)/2.]*u.pix, scale=[180 / oshape[0], 360 / oshape[1]] * u.deg / u.pix, projection_code="CAR") #header['crval2'] = 0 out_wcs = WCS(header) # Reproject array, footprint = reproject_interp((map_stb.data, map_stb.wcs), out_wcs, shape_out=oshape) omap_stb = sunpy.map.Map((array, header)) omap_stb.plot_settings = map_stb.plot_settings # Normalize data to exposure time omap_stb_data = (omap_stb.data / (map_stb.exposure_time / u.second)).value # Condense the reprojected map minimums into chmap chmap = np.fmin(chmap, omap_stb_data) chmap_stb = np.copy(chmap) # CL - Make sure to align output maps with longitude coordinates, notably shift the zero point from the center to the edge of the data frame #coords = sunpy.map.all_coordinates_from_map(omap_aia) # CL - Create a secondary map weighted by longevity of CH over rotation when observed? # Shift things and generate coordinates # CL - Temporary fix for weird start NaN column and row # CL - Might also need to shift coordinates to sine latitude... chmap_aia[:, 0] = (chmap_aia[:, -1] + chmap_aia[:, 1]) / 2 chmap_aia[0, :] = np.nan chmap_aia = np.roll(chmap_aia, [0, int(oshape[1] / 2)]) chmap_sta[:, 0] = (chmap_sta[:, -1] + chmap_sta[:, 1]) / 2 chmap_sta[0, :] = np.nan chmap_sta = np.roll(chmap_sta, [0, int(oshape[1] / 2)]) chmap_stb[:, 0] = (chmap_stb[:, -1] + chmap_stb[:, 1]) / 2 chmap_stb[0, :] = np.nan chmap_stb = np.roll(chmap_stb, [0, int(oshape[1] / 2)]) lats = np.linspace(-90, 90, oshape[0]) lons = np.linspace(0, 360, oshape[1]) pscale = [oshape[0] / 180, oshape[1] / 360] # Generate threshold values for AIA qs = np.nanmedian(chmap_aia) if np.isfinite(qs): thrsh = np.array([]) [dlat, dlon] = [60, 60] for ilat in np.arange(0, 180 / dlat): for ilon in np.arange(0, 360 / dlon): plat0 = int(ilat * dlat * pscale[0]) plat1 = int((ilat + 1) * dlat * pscale[0]) plon0 = int(ilon * dlon * pscale[1]) plon1 = int((ilon + 1) * dlon * pscale[1]) sarr = chmap_aia[plat0:plat1, plon0:plon1] #sarr_hist = histogram(sarr[where(np.isfinite(sarr))].flatten(), bins=100, range=[np.nanmin(sarr),qs]) sarr_hist = np.histogram(sarr[np.where( np.isfinite(sarr))].flatten(), bins=100, range=[0, qs]) #sarr_dist = scipy.stats.rv_histogram(sarr_hist) sh_x = sarr_hist[1][0:-1] sh_y = sarr_hist[0] sh_y2 = scipy.signal.convolve( sh_y, scipy.signal.hann(20), mode='same') / sum( scipy.signal.hann(20)) pks = scipy.signal.find_peaks_cwt(sh_y2, np.arange(1, 20)) if len(pks) >= 2: sh_x2 = sh_x[pks[0]:pks[-1] - 1] sh_y3 = sh_y2[pks[0]:pks[-1] - 1] else: minval = int(len(sh_x) / 4) maxval = int(len(sh_x) * 0.9) sh_x2 = sh_x[minval:maxval] sh_y3 = sh_y2[minval:maxval] pks2 = scipy.signal.find_peaks_cwt(-1 * sh_y3, np.arange(1, 20)) if len(pks2) != 0: thrsh = np.append(thrsh, sh_x2[pks2[0]]) else: thrsh = np.append(thrsh, np.nan) chval_aia = np.nanmean(thrsh) # Generate threshold values for STA qs = np.nanmedian(chmap_sta) if np.isfinite(qs): thrsh = np.array([]) [dlat, dlon] = [60, 60] for ilat in np.arange(0, 180 / dlat): for ilon in np.arange(0, 360 / dlon): plat0 = int(ilat * dlat * pscale[0]) plat1 = int((ilat + 1) * dlat * pscale[0]) plon0 = int(ilon * dlon * pscale[1]) plon1 = int((ilon + 1) * dlon * pscale[1]) sarr = chmap_sta[plat0:plat1, plon0:plon1] #sarr_hist = histogram(sarr[where(np.isfinite(sarr))].flatten(), bins=100, range=[np.nanmin(sarr),qs]) sarr_hist = np.histogram(sarr[np.where( np.isfinite(sarr))].flatten(), bins=100, range=[0, qs]) #sarr_dist = scipy.stats.rv_histogram(sarr_hist) sh_x = sarr_hist[1][0:-1] sh_y = sarr_hist[0] sh_y2 = scipy.signal.convolve( sh_y, scipy.signal.hann(20), mode='same') / sum( scipy.signal.hann(20)) pks = scipy.signal.find_peaks_cwt(sh_y2, np.arange(1, 20)) if len(pks) >= 2: sh_x2 = sh_x[pks[0]:pks[-1] - 1] sh_y3 = sh_y2[pks[0]:pks[-1] - 1] else: minval = int(len(sh_x) / 4) maxval = int(len(sh_x) * 0.9) sh_x2 = sh_x[minval:maxval] sh_y3 = sh_y2[minval:maxval] pks2 = scipy.signal.find_peaks_cwt(-1 * sh_y3, np.arange(1, 20)) if len(pks2) != 0: thrsh = np.append(thrsh, sh_x2[pks2[0]]) else: thrsh = np.append(thrsh, np.nan) chval_sta = np.nanmean(thrsh) # Generate threshold values for STB qs = np.nanmedian(chmap_stb) if np.isfinite(qs): thrsh = np.array([]) [dlat, dlon] = [60, 60] for ilat in np.arange(0, 180 / dlat): for ilon in np.arange(0, 360 / dlon): plat0 = int(ilat * dlat * pscale[0]) plat1 = int((ilat + 1) * dlat * pscale[0]) plon0 = int(ilon * dlon * pscale[1]) plon1 = int((ilon + 1) * dlon * pscale[1]) sarr = chmap_stb[plat0:plat1, plon0:plon1] #sarr_hist = histogram(sarr[where(np.isfinite(sarr))].flatten(), bins=100, range=[np.nanmin(sarr),qs]) sarr_hist = np.histogram(sarr[np.where( np.isfinite(sarr))].flatten(), bins=100, range=[0, qs]) #sarr_dist = scipy.stats.rv_histogram(sarr_hist) sh_x = sarr_hist[1][0:-1] sh_y = sarr_hist[0] sh_y2 = scipy.signal.convolve( sh_y, scipy.signal.hann(20), mode='same') / sum( scipy.signal.hann(20)) pks = scipy.signal.find_peaks_cwt(sh_y2, np.arange(1, 20)) if len(pks) >= 2: sh_x2 = sh_x[pks[0]:pks[-1] - 1] sh_y3 = sh_y2[pks[0]:pks[-1] - 1] else: minval = int(len(sh_x) / 4) maxval = int(len(sh_x) * 0.9) sh_x2 = sh_x[minval:maxval] sh_y3 = sh_y2[minval:maxval] pks2 = scipy.signal.find_peaks_cwt(-1 * sh_y3, np.arange(1, 20)) if len(pks2) != 0: thrsh = np.append(thrsh, sh_x2[pks2[0]]) else: thrsh = np.append(thrsh, np.nan) chval_stb = np.nanmean(thrsh) chmap0_aia = np.copy(chmap_aia) if not skip_aia: chmap0_aia[np.where(chmap_aia > chval_aia)] = 0 chmap0_sta = np.copy(chmap_sta) if not skip_sta: chmap0_sta[np.where(chmap_sta > chval_sta)] = 0 chmap0_stb = np.copy(chmap_stb) if not skip_stb: chmap0_stb[np.where(chmap_stb > chval_stb)] = 0 # Generate a merged chmap # CL - make changes here to restore to original behavior of measuring CH depth. Might need to create a normalized merged of AIA / STA data to fill the gaps nodat = (np.logical_and(~np.isfinite(chmap0_aia), ~np.isfinite(chmap0_sta))) chmap0 = (np.nansum(np.dstack( (chmap0_aia, chmap0_sta, chmap0_stb)), 2) != 0) # chmap0 = chmap_aia * (np.nansum(np.dstack((chmap0_aia, chmap0_sta)),2) != 0) ocstruct = np.ones([3, 3]) chmap1 = scipy.ndimage.binary_opening(scipy.ndimage.binary_closing( chmap0, structure=ocstruct, iterations=1), structure=ocstruct, iterations=1) * chmap0 # Label each dark region for comparison labstruct = np.ones([3, 3]) lchmap = (scipy.ndimage.label(chmap1, structure=labstruct))[0] for i in np.arange(2, lchmap.max() + 1): chmag = br[np.where(lchmap == i)] if len(chmag) < 10: chmap1[np.where(lchmap == i)] = 0 continue if abs(scipy.stats.skew(chmag)) < 0.5: chmap1[np.where(lchmap == i)] = 0 continue chmap = chmap1 # For visualization, scale the images to created a merged view im_aia = np.copy(chmap_aia) im_sta = np.copy(chmap_sta) im_stb = np.copy(chmap_stb) im_aia = im_aia - np.nanmin(im_aia) im_aia = im_aia / np.nanmax(im_aia) im_sta = im_sta - np.nanmin(im_sta) im_sta = im_sta / np.nanmax(im_sta) im_stb = im_stb - np.nanmin(im_stb) im_stb = im_stb / np.nanmax(im_stb) chim = np.fmin(im_sta, im_aia) chim = np.fmin(chim, im_stb) # Save everything out to file fname = outdir + 'chmap/chmap-' + str(cr) + '.fits' sunpy.io.write_file(fname, chmap * chim, header, overwrite=True) fname = outdir + 'chmap/chmap-' + str(cr) + '-chim.fits' sunpy.io.write_file(fname, chim, header, overwrite=True)
import astropy.units as u import sunpy.map from sunpy.net import Fido from sunpy.net import attrs as a ###################################################################### # In this example we are going to make a lot of side by side figures, so # let's change the default figure size. plt.rcParams['figure.figsize'] = (16, 8) ###################################################################### # We are going to download one AIA and one HMI magnetogram image. time = (a.Sample(24 * u.hour) & a.Time('2010-08-19', '2010-08-19T00:10:00', '2010-08-19') & a.vso.Extent(0, 0, 0, 0, "FULLDISK")) aia = a.Instrument('AIA') & a.Wavelength(17 * u.nm, 18 * u.nm) hmi = a.Instrument('HMI') & a.Physobs("LOS_magnetic_field") res = Fido.search(time, aia | hmi) files = Fido.fetch(res[:, 0]) ###################################################################### # We create a map for each image and resample each one just to # reduce the computation time. map_aia, map_hmi = [ m.resample((1024, 1024) * u.pix) for m in sunpy.map.Map(sorted(files))
def test_no_match(): with pytest.raises(DrmsQueryError): Fido.search(a.Time("2016/10/01", "2016/10/02"), a.jsoc.Series("bob"), a.Sample(10 * u.s))
elif SERIES == "hmi.Ic_noLimbDark_720s": SEGMENT = "continuum" PRINTER.info_text("Data segment set to 'continuum'") ######################### PRINTER.line() PRINTER.info_text("Searching JSOC database") if INSTRUMENT == "aia": results = Fido.search( a.jsoc.Time("%s" % START_TIME.replace(microsecond=0).isoformat(), "%s" % END_TIME.replace(microsecond=0).isoformat()), a.jsoc.Notify(EMAIL), a.jsoc.Series(SERIES), a.jsoc.Wavelength(WAVELENGTH * u.angstrom), a.Sample(CADENCE * u.second)) elif INSTRUMENT == "hmi": results = Fido.search( a.jsoc.Time("%s" % START_TIME.replace(microsecond=0).isoformat(), "%s" % END_TIME.replace(microsecond=0).isoformat()), a.jsoc.Notify(EMAIL), a.jsoc.Series(SERIES), a.jsoc.Segment(SEGMENT), a.Sample(CADENCE * u.second)) PRINTER.info_text("Search complete. Displaying:") print "\n" + results ######################### PRINTER.line() PRINTER.input_text("Press 'enter' to download")
from astropy.time import TimeDelta import sunpy.coordinates import sunpy.map from sunpy.io.special import srs from sunpy.net import Fido from sunpy.net import attrs as a from sunpy.time import parse_time ############################################################################## # For this example, we will search for and download a single HMI using Fido. start_time = parse_time("2017-01-25") end_time = start_time + TimeDelta(23 * u.hour + 59 * u.minute + 59 * u.second) results = Fido.search(a.Time(start_time, end_time), a.Instrument.hmi & a.Physobs.los_magnetic_field, a.Sample(60 * u.second)) ############################################################################## # Let's select only the first file, download it and create a map. result = results[0, 0] file_name = Fido.fetch(result) smap = sunpy.map.Map(file_name) ############################################################################## # Download the SRS file. srs_results = Fido.search(a.Time(start_time, end_time), a.Instrument.srs_table) srs_downloaded_files = Fido.fetch(srs_results) ############################################################################## # We get one SRS file per day. To read this file, we pass the filename into # the SRS reader. So now `srs_table` contains an astropy table.
# See `this page <http://jsoc.stanford.edu/ajax/register_email.html>`_ # for more details. jsoc_email = os.environ["JSOC_EMAIL"] ##################################################### # Now we are ready to construct the query. Note that all of this is # the same for a full-frame image except for the # cutout component. We will download images from a 12 hour interval # centered on the time of the above cutout. # We request one image every 2 hours. query = Fido.search( a.Time(cutouts.date - 6*u.h, cutouts.date + 6*u.h), a.Wavelength(cutouts.wavelength), a.Sample(2*u.h), a.jsoc.Series.aia_lev1_euv_12s, a.jsoc.Notify(jsoc_email), a.jsoc.Segment.image, cutout, ) ##################################################### # Submit the export request and download the data. files = Fido.fetch(query) files.sort() ##################################################### # Now that we've downloaded the files, we can create # a `~sunpy.map.MapSequence` from them.
from astropy.coordinates import SkyCoord from astropy.wcs import WCS import sunpy.map import sunpy.sun from sunpy.coordinates import get_body_heliographic_stonyhurst from sunpy.net import Fido from sunpy.net import attrs as a ###################################################################### # To get started, let's download the data: stereo = (a.Instrument("EUVI") & a.Time('2011-11-01', '2011-11-01T00:10:00')) aia = (a.Instrument.aia & a.Sample(24 * u.hour) & a.Time('2011-11-01', '2011-11-02')) wave = a.Wavelength(19.5 * u.nm, 19.5 * u.nm) res = Fido.search(wave, aia | stereo) files = Fido.fetch(res) ###################################################################### # Next we create a sunpy map for each of the files. maps = sunpy.map.Map(sorted(files)) ###################################################################### # To reduce memory consumption we also downsample these maps before continuing, # you can disable this. maps = [m.resample((1024, 1024)*u.pix) for m in maps]
os.makedirs(path) if not os.path.exists(path) else None start_date = pd.Timestamp('2011/01/01 00:00:14Z') end_date = pd.Timestamp('2017/01/01 00:00:00Z') batch_span = 30*24 # download files in groups spanning 30 days batch_start = start_date batch_end = batch_start + pd.Timedelta(hours=batch_span) while batch_end <= end_date: res_aia = Fido.search(a.Time(batch_start, batch_end), a.jsoc.Notify(email), a.jsoc.Series('aia.lev1_uv_24s'), a.jsoc.Segment('image'), a.jsoc.Wavelength(1700*u.AA), a.Sample(12*u.hour) ) print(res_aia) # download files try: downloaded_files = Fido.fetch(res_aia, path=path) except Exception as e: print("couldn't download files between " + str(batch_start) + " and " + str(batch_end)) print(str(e)) # end of current batch: end_of_batch = str(res_aia.get_response(0)['T_REC'][-1]) # start and end dates of next batch
How to download an HMI magnetogram data with Fido and make a plot. """ import astropy.units as u import matplotlib.pyplot as plt import sunpy.map from sunpy.net import Fido, attrs as a ############################################################################### # To download the required data, we use # `Fido <sunpy.net.fido_factory.UnifiedDownloaderFactory>`, a downloader client. # First define the search variables, a timerange, the instrument, # the observation type, # and a cadence of images spaced every 720 seconds. result = Fido.search(a.Time('2011/11/09 17:40:00', '2011/11/09 17:55:00'), a.Instrument('hmi'), a.Sample(720 * u.s), a.vso.Physobs('LOS_magnetic_field')) ############################################################################### # Now we can see what results we obtained from our search. # Notice we have two files. print(result) ############################################################################### # The following shows how to download the results. If we # don't provide a path it will download the file into the sunpy data directory. # The output provides the path of the downloaded files. # The result can be from several data clients, so we have to index the client first # client and then index the file. # Slice the first record returned by the first client.
HMI = args.HMI HMI_path = args.HMI_path STEREO = args.STEREO STEREO_path = args.STEREO_path cadence = args.cadence * u.hour # take images every 12 hours wavelength = args.wavelength * u.AA # wavelength in angstroms stereo_cadence = args.STEREO_cadence # use 1 in every n images # time intervals to avoid missing data missing = [AIA_start, AIA_end] if AIA: res_aia = Fido.search(a.Time(AIA_start, AIA_end), a.jsoc.Notify(email), a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Segment('image'), a.jsoc.Wavelength(wavelength), a.Sample(cadence)) print(res_aia) # save the the query details to file with open('data_query_aia.pkl', 'wb') as f: pickle.dump([AIA_start, AIA_end, res_aia], f) # add dates of missing data res_table = res_aia.get_response(0) for i in range(len(res_table)): if str(res_table['CAR_ROT'][i]) == "--": time = (str(res_table['T_REC'][i][:-6]) + '00').replace('_', ' ') missing.append(time.replace('.', '/')) if HMI:
client = jsoc.JSOCClient() years = [i + 2013 for i in range(7)] months = [i + 2 for i in range(11)] def remove_glob(pathname, recursive=True): for p in glob.glob(pathname, recursive=recursive): if os.path.isfile(p): os.remove(p) for year in years: for month in months: days = [day + 1 for day in range(calendar.monthrange(year, month)[1])] for day in days: start = datetime.datetime(year, month, day, 00, 00, 00) end = start + datetime.timedelta(days=1) print(start, end) response = client.search( a.Time(start, end), a.jsoc.Series('hmi.Mharp_720s'), a.Sample(1 * u.hour), a.jsoc.Notify("*****@*****.**" )) #開始時間のところに等号を入れられなかったので調整 path = "/media/akito/Data/Mharp/bitmap/" + str(year) + "/" + str( year) + str(month).zfill(2) res = response.client.fetch(response, path=path) rm_path1 = path + "/" + "*.magnetogram.fits" rm_path2 = path + "/" + "*.mask.fits" remove_glob(rm_path1) remove_glob(rm_path2)
download_chunk = timedelta( days=10 ) #avoid download chunks greater than 1 month in order to not download too much at once #breaks the download into pieces and downloads current_time = start while (current_time < end): if (end - current_time > download_chunk): next_time = current_time + download_chunk else: next_time = end response = Fido.search(attrs.jsoc.Time(current_time, next_time), attrs.jsoc.Notify('*****@*****.**'), attrs.jsoc.Series('hmi.Sharp_720s'), attrs.jsoc.Segment('bitmap'), attrs.Sample(time_interval.total_seconds() * u.s)) response res = Fido.fetch(response, path=sharp_dir + '/{file}.fits') current_time = next_time warnings.simplefilter( "ignore" ) #.verify('fix') produces many warnings which will lag the jupyter notebook #extracts relevant keywords in the given order keywords = [ 'HARPNUM', 'T_REC', 'NAXIS1', 'NAXIS2', 'CDELT1', 'CDELT2', 'IMCRPIX1', 'IMCRPIX2', 'LAT_FWT', 'LON_FWT', 'NPIX' ] #Keywords in order to be saved filenames = os.listdir(sharp_dir) filename = 'data.txt'
# email first. Please replace this with your email # address once you have registered. # See `this page <http://jsoc.stanford.edu/ajax/register_email.html>`_ # for more details. jsoc_email = os.environ["JSOC_EMAIL"] ##################################################### # Now we are ready to construct the query. Note that all of this is # the same for a full-frame image except for the # cutout component. We will download images from a 12 hour interval # centered on the time of the above cutout. # We request one image every 12 minutes. q = Fido.search( a.Time(m_cutout.date - 6 * u.h, m_cutout.date + 6 * u.h), a.Wavelength(m_cutout.wavelength), a.Sample(12 * u.min), a.jsoc.Series.aia_lev1_euv_12s, a.jsoc.Notify(jsoc_email), a.jsoc.Segment.image, cutout, ) ##################################################### # Submit the export request and download the data. We # set the number of parallel downloads to 2 so as not # to overwhelm the JSOC export service with our number # of download requests. files = Fido.fetch(q, max_conn=2) files.sort() #####################################################
def test_query(client): Jresp = client.search( a.Time('2020/1/1T00:00:00', '2020/1/1T00:01:30'), a.jsoc.Series('hmi.M_45s'), a.Sample(90 * u.second)) assert isinstance(Jresp, JSOCResponse) assert len(Jresp) == 2
p.add_argument('-cadence', type=int, default=8, help='sampling time for data') args = p.parse_args() print( r'---------------------- Fido download for EIT,EUVI-A, EUVI-B ----------------------' ) eit = (a.Instrument("eit") & a.Time(args.startdate, args.enddate)) euvi = (a.vso.Source('STEREO_A') & a.Instrument("EUVI") & a.Time(args.startdate, args.enddate)) euvib = (a.vso.Source('STEREO_B') & a.Instrument("EUVI") & a.Time(args.startdate, args.enddate)) samp = a.Sample(args.cadence * u.minute) wave = a.Wavelength(args.wavelengths[0] * u.AA, args.wavelengths[1] * u.AA) print(args.startdate, args.enddate, wave, samp) fethched_results = Fido.search(wave, eit | euvi | euvib, samp) print( r'---------------------- Fido fetched files for EIT,EUVI-A, EUVI-B ----------------------' ) print(fethched_results) files = Fido.fetch(fethched_results, path=args.data_path) while not files.errors: time.sleep(30) files = Fido.fetch(files) '''
from sunpy.net.attr import AttrAnd, AttrOr ta = Time(time_now, format='datetime', scale='tai') ta1 = Time(time_now - datetime.timedelta(days=2), format='datetime', scale='tai') ta2 = Time(time_now - datetime.timedelta(days=1), format='datetime', scale='tai') segments = ['inclination', 'azimuth', 'disambig', 'field'] series = 'hmi.B_720s' interval = 10 * u.min email = '*****@*****.**' response = Fido.search( a.jsoc.Time(ta1, ta2), a.jsoc.Series(series), a.jsoc.Notify(email), a.Sample(interval), AttrAnd(list(map(a.jsoc.Segment, segments))) # i.e. attrs.jsoc.Segment('...') & attrs.jsoc.Segment('...') ... ) response Fido.search(a.jsoc.Time('2014-01-01T00:00:00', '2014-01-01T01:00:00'), a.jsoc.Series('hmi.sharp_720s'), a.jsoc.Notify('*****@*****.**'), a.jsoc.Segment('image')) Fido.search(a.jsoc.Time('2019-01-15T00:00:00', '2019-01-16T01:00:00'), a.jsoc.Series('hmi.M_720s_nrt'), a.jsoc.Notify('*****@*****.**'), a.jsoc.Segment('image'))
import astropy.units as u import sunpy.map from sunpy.net import Fido, attrs as a ############################################################################### # Now we will download some data with `sunpy.net.Fido`. # A `Fido.search` requires us to specify a `~sunpy.net.attr.Time`, # `~sunpy.net.attr.Sample`, `~sunpy.net.attr.Instrument` # and the `~sunpy.net.attr.vso.Physobs`. # We set a time range from ``2015/11/04 12:00:00`` to ``2015/11/04 12:10:00`` # for HMI ``LOS_magnetic_field`` with the images spaced every 720 seconds. result = Fido.search(a.Time('2015/11/04 12:00:00', '2015/11/04 12:10:00'), a.Instrument('hmi'), a.Sample(720*u.s), a.vso.Physobs('LOS_magnetic_field')) ############################################################################### # Now we can see what results we obtained from our search. # Notice we have two files. One is the full disk image we plan to display # and the other is a synoptic version of said image. print(result) ############################################################################### # Once we are happy with the results obtained from the search. # We can download the data with `Fido.fetch` . # In this case we only want one file so we can index the result. # A `Fido` result can be from several clients, so we have to index the first # client and then index the first result.
import astropy.units as u from sunpy.net import Fido from sunpy.net import attrs as a stereo = (a.vso.Source('STEREO_B') & a.Instrument('EUVI') & a.Time('2011-01-01', '2011-01-01T00:10:00')) aia = (a.Instrument('AIA') & a.Sample(24 * u.hour) & a.Time('2011-01-01', '2011-01-02')) wave = a.Wavelength(30 * u.nm, 31 * u.nm) results = Fido.search(aia | stereo, wave) res = Fido.fetch(results, path='./data/{file}')