def test_response_block_properties(client):
res = client.search(a.Time('2020/3/4', '2020/3/6'),
a.Instrument('aia'),
a.Wavelength(171 * u.angstrom),
a.Sample(10 * u.minute),
response_format="legacy")
properties = res.response_block_properties()
assert len(properties) == 0
def test_vso_fetch_hmi(tmpdir):
start_time = "2017-01-25"
end_time = "2017-01-25T23:59:59"
results = Fido.search(a.Time(start_time, end_time),
a.Instrument.hmi & a.Physobs.los_magnetic_field,
a.Sample(1 * u.minute))
files = Fido.fetch(results[0, 0], path=tmpdir)
assert len(files) == 1
def __init__(self,
times=("2020/05/25", "2020/05/27"),
cadence=1 * u.hour,
instrument=None,
path=PATH):
self.attrsTime = a.Time(TimeRange(times))
self.cadence = a.Sample(cadence)
self.instrument = a.Instrument(instrument)
self.path = path
print(f"Using {self.path}")
self.dfiles = None
def download_files(start_time, end_time, series_name, segs, email_for_notification, interval, download_path):
segVal=a.jsoc.Segment(segs[0])
for i in range(1, len(segs)):
segVal&=a.jsoc.Segment(segs[i])
result = Fido.search(
a.Time(start_time, end_time),
a.jsoc.Series(series_name),
a.jsoc.Notify(email_for_notification),
a.Sample(interval),
segVal)
downloaded_files=Fido.fetch(result, path=os.path.join(download_path, '{file}'))
print('Files downloaded.')
def grab(dirPath, wavelength, date, time, download=False):
"""
Return a previously downloaded AIA file according to search criteria.
Simple method to return a specific AIA file that you've already downloaded,
or download a file matching your search criteria if it doesn't exist in the
specified directory.
Inputs:
-----------
dirpath: path to search for file (string, e.g. './AIAfiles')
wavelength: wavelength in angstroms (string, e.g. '94')
date: string, format yyyy_mm_dd
time: string, format hh_mm[_ss] - second can be specified if known
download: boolean, whether to download an image if it is not found
Outputs:
-----------
A string containing the path to the file found
"""
searchPath = dirPath + '/aia_lev1_' + wavelength + 'a_' + date + 't' \
+ time + '*.fits'
foundFiles = glob.glob(searchPath)
if len(foundFiles) == 0:
if not download:
raise Exception('No files found that match input!')
else:
date = date.replace('_', '-')
time = time.replace('_', ':')
start = Time(date + 'T' + time)
end = start + 59*u.second
wv = int(wavelength)
result = Fido.search(a.Time(start, end),
a.Instrument.aia,
a.Wavelength(wv*u.angstrom),
a.Sample(1*u.minute))
path = dirPath + '/{file}'
downd = Fido.fetch(result, path=path)
return downd
elif len(foundFiles) > 1:
print('Multiple files match input! Returning the first file.')
return foundFiles[0]
def vso_search(trange=None,
instrument=None,
wavelength=None,
sample=None,
provider=None,
source=None):
try:
from sunpy.net import Fido, attrs as a
except:
print('VSO_SEARCH: Error, requires sunpy. Sunpy not installed?')
return None
import astropy.units as u
if not trange or not instrument:
print(
'VSO_SEARCH: Error, must specify a valid time range and instrument'
)
return None
try:
times = a.Time(Time(trange[0]).iso, Time(trange[1]).iso)
except:
print('VSO_SEARCH: Error, invalid time range')
return None
inst = a.Instrument(instrument)
paras = [times, inst]
if wavelength:
wave = a.Wavelength(wavelength * u.angstrom)
paras = paras + [wave]
if sample is not None:
sample = a.Sample(sample * u.second) # in units of seconds
paras = paras + [sample]
if provider is not None:
provider = a.Provider(provider)
paras = paras + [provider]
if source is not None:
source = a.Source(source)
paras = paras + [source]
qr = Fido.search(*paras)
return qr
def combined_aia(fixed_map, time):
atime = a.Sample(24 * u.hour) & a.Time(time,
time + astropy.time.TimeDelta(1e-3))
aia = a.Instrument.aia & a.Wavelength(17 * u.nm, 18 * u.nm)
res = Fido.search(atime, aia)
files = Fido.fetch(res[:, 0])
map_aia = sunpy.map.Map(files).resample((1024, 1024) * u.pix)
output, footprint = reproject_interp(fixed_map, map_aia.wcs,
map_aia.data.shape)
out_fixed = sunpy.map.Map(output, map_aia.wcs)
fig = plt.figure(figsize=(20, 15))
ax1 = fig.add_subplot(1, 1, 1, projection=map_aia)
map_aia.plot(axes=ax1)
out_fixed.plot(axes=ax1, alpha=0.5)
plt.show()
plt.plot(map_aia.data.flatten(), out_fixed.data.flatten(), ',')
def aia_pch_data_download(savepath='', waves=[171, 193, 211, 304]):
warnings.simplefilter('ignore', category=AstropyDeprecationWarning)
warnings.simplefilter('ignore', category=ErfaWarning)
blank_results = Fido.search(a.jsoc.Time('2030/1/1', '2030/1/2'),
a.jsoc.Series('aia.lev1_euv_12s'))
base = parse_time('2019/01/01 00:00:00')
ending = parse_time('2020/10/01 00:00:00')
date_list = [
dt for dt in rrule(
MONTHLY, dtstart=base.to_datetime(), until=ending.to_datetime())
]
for wave in waves:
for ii in range(0, len(date_list) - 1):
if not savepath:
save_dir = os.path.abspath(os.path.curdir)
else:
save_dir = os.path.abspath(savepath)
directories = np.str(wave) + date_list[ii].strftime("/%Y/%m")
save_path = os.path.join(save_dir, directories)
os.makedirs(save_path, exist_ok=True)
results = blank_results
while results.file_num == 0:
results = Fido.search(
a.jsoc.Time(date_list[ii].strftime('%Y/%m/%d'),
date_list[ii + 1].strftime('%Y/%m/%d')),
a.jsoc.Notify('*****@*****.**'),
a.jsoc.Series('aia.lev1_euv_12s'),
a.jsoc.Wavelength(wave * u.angstrom), a.Sample(3 * u.hour))
expected_number = 224. # 8 images per day, 28 days min.
neededfileindex = []
if (results.file_num / expected_number) > 0.70:
downloaded_file = Fido.fetch(results,
path=save_path + '/{file}')
num_missing = len(downloaded_file.errors)
retry_count = 0
print('Validating Downloads...')
partial = [
try_fits_file(qfile) for qfile in downloaded_file.data
]
num_missing += np.sum(partial)
while num_missing > 0:
downloaded_file = Fido.fetch(results,
path=save_path + '/{file}')
num_missing = len(downloaded_file.errors)
print('Validating Downloads...')
partial = [
try_fits_file(qfile) for qfile in downloaded_file.data
]
num_missing += np.sum(partial)
retry_count += 1
print(f'Retry number {retry_count} to catch errors... ')
if retry_count >= 20:
num_missing = 0
print('Error in ' +
date_list[ii].strftime('%Y/%m/%d') + ' ' +
np.str(wave))
neededfileindex += [date_list[ii]]
if results.file_num == len(
fnmatch.filter(os.listdir(save_path),
'*.image_lev1.fits')):
print('Download between ' +
date_list[ii].strftime('%Y/%m/%d') + ' and ' +
date_list[ii + 1].strftime('%Y/%m/%d') + ' successful')
print('Removing spikes files...')
for s in downloaded_file.data:
if "spikes" in s:
os.remove(s)
else:
print('Expected ' + np.str(results.file_num) + ' files.')
print('Got ' + np.str(
len(
fnmatch.filter(os.listdir(save_path),
'*.image_lev1.fits'))) + ' files.')
with open(
save_dir + '/' + np.str(wave) + 'NeededFileIndex_' +
date_list[ii].strftime('%Y') + '.txt', 'w') as f:
for item in neededfileindex:
f.write("%s\n" % item)
import astropy.units as u
from astropy.coordinates import SkyCoord
import sunpy.coordinates.wcs_utils
import sunpy.map
from sunpy.net import Fido
from sunpy.net import attrs as a
##############################################################################
# The first step is to download some data, we are going to get an image from
# early 2011 when the STEREO spacecraft were roughly 90 deg separated from the
# Earth.
stereo = (a.Source('STEREO_B') & a.Instrument("EUVI")
& a.Time('2011-01-01', '2011-01-01T00:10:00'))
aia = (a.Instrument.aia & a.Sample(24 * u.hour)
& a.Time('2011-01-01', '2011-01-02'))
wave = a.Wavelength(30 * u.nm, 31 * u.nm)
result = Fido.search(wave, aia | stereo)
###############################################################################
# Let's inspect the result
print(result)
##############################################################################
# and download the files
downloaded_files = Fido.fetch(result)
print(downloaded_files)
##############################################################################
def chortle(cr):
# Read configuration file
config = configparser.ConfigParser()
config.read('config.cfg')
# Specify directory structures from configuration file
datdir = config['paths']['datdir']
magdir = config['paths']['magdir']
outdir = config['paths']['outdir']
# Specify timing parameters
nsday = 1
# Grab the start and end dates for this rotation
t0 = sunpy.coordinates.sun.carrington_rotation_time(cr)
t0.format = 'datetime'
t0 = t0.value
t1 = sunpy.coordinates.sun.carrington_rotation_time(cr + 1)
t1.format = 'datetime'
t1 = t1.value
# For now, round these to the nearest day
if t0.hour > 12:
t0 = t0 + datetime.timedelta(days=1)
t0 = datetime.datetime(t0.year, t0.month, t0.day)
if t1.hour > 12:
t1 = t1 + datetime.timedelta(days=1)
t1 = datetime.datetime(t1.year, t1.month, t1.day)
# Download appropriate data
search_aia = (a.Instrument('AIA') & a.Sample(nsday * u.day)
& a.Time(t0, t1))
res_aia = Fido.search(a.Wavelength(19.3 * u.nm), search_aia)
search_sta = (a.vso.Source('STEREO_A') & a.Instrument('SECCHI')
& a.Detector('EUVI') & a.Sample(nsday * u.day)
& a.Time(t0, t1))
res_sta = Fido.search(a.Wavelength(19.5 * u.nm), search_sta)
search_stb = (a.vso.Source('STEREO_B') & a.Instrument('SECCHI')
& a.Detector('EUVI') & a.Sample(nsday * u.day)
& a.Time(t0, t1))
res_stb = Fido.search(a.Wavelength(19.5 * u.nm), search_stb)
# Running into some errors with automatically fetching HMI data
# Manually download and load for now...
#c = drms.Client()
#c.pkeys('hmi_synoptic_mr_polfil_720s')
#res_hmi = Fido.search(a.jsoc.Series('hmi_synoptic_mr_polfil_720s') & a.jsoc.PrimeKey('crnum', '2193'))
files_aia = Fido.fetch(res_aia, path=datdir + 'aia/')
files_aia.sort()
files_sta = Fido.fetch(res_sta, path=datdir + 'sta/')
files_sta.sort()
files_stb = Fido.fetch(res_stb, path=datdir + 'stb/')
files_stb.sort()
skip_aia = skip_sta = skip_stb = False
if len(files_aia) == 0:
skip_aia = True
if len(files_sta) == 0:
skip_sta = True
if len(files_stb) == 0:
skip_stb = True
## Generate some blank storage arrays
oshape = [720, 1440]
chmap = np.zeros(oshape, dtype=np.double)
chmap[:, :] = np.nan
## Grab a synoptic magnetogram
br0 = sunpy.io.fits.read(magdir + 'hmi.synoptic_mr_polfil_720s.' +
str(cr) + '.Mr_polfil.fits')[1].data
br = sunpy.image.resample.resample(br0, oshape, method='linear')
## Iterate through this data, reprojecting as you go
for file in files_aia:
# Read in data
map_aia = sunpy.map.Map(file)
# Check for bad files
if map_aia.exposure_time == 0: continue
# Construct an output header
header = sunpy.map.make_fitswcs_header(
np.empty(oshape),
SkyCoord(0,
0,
unit=u.deg,
frame="heliographic_carrington",
obstime=map_aia.date),
scale=[180 / oshape[0], 360 / oshape[1]] * u.deg / u.pix,
projection_code="CAR")
#header['crval2'] = 0
out_wcs = WCS(header)
# Reproject
array, footprint = reproject_interp((map_aia.data, map_aia.wcs),
out_wcs,
shape_out=oshape)
omap_aia = sunpy.map.Map((array, header))
omap_aia.plot_settings = map_aia.plot_settings
# Normalize data to exposure time
omap_aia_data = (omap_aia.data /
(map_aia.exposure_time / u.second)).value
# Condense the reprojected map minimums into chmap
chmap = np.fmin(chmap, omap_aia_data)
chmap_aia = np.copy(chmap)
## Generate some blank storage arrays
oshape = [720, 1440]
chmap = np.zeros(oshape, dtype=np.double)
chmap[:, :] = np.nan
## Iterate through this data, reprojecting as you go
for file in files_sta:
# Read in data
map_sta = sunpy.map.Map(file)
# Check for bad files
if map_sta.exposure_time == 0: continue
# Construct an output header
header = sunpy.map.make_fitswcs_header(
np.empty(oshape),
SkyCoord(
0,
0,
unit=u.deg,
frame="heliographic_carrington",
#frame="heliographic_stonyhurst",
obstime=map_sta.date),
#reference_pixel=[0,(oshape[0] - 1)/2.]*u.pix,
scale=[180 / oshape[0], 360 / oshape[1]] * u.deg / u.pix,
projection_code="CAR")
#header['crval2'] = 0
out_wcs = WCS(header)
# Reproject
array, footprint = reproject_interp((map_sta.data, map_sta.wcs),
out_wcs,
shape_out=oshape)
omap_sta = sunpy.map.Map((array, header))
omap_sta.plot_settings = map_sta.plot_settings
# Normalize data to exposure time
omap_sta_data = (omap_sta.data /
(map_sta.exposure_time / u.second)).value
# Condense the reprojected map minimums into chmap
chmap = np.fmin(chmap, omap_sta_data)
chmap_sta = np.copy(chmap)
## Generate some blank storage arrays
oshape = [720, 1440]
chmap = np.zeros(oshape, dtype=np.double)
chmap[:, :] = np.nan
## Iterate through this data, reprojecting as you go
for file in files_stb:
# Read in data
map_stb = sunpy.map.Map(file)
# Check for bad files
if map_stb.exposure_time == 0: continue
# Construct an output header
header = sunpy.map.make_fitswcs_header(
np.empty(oshape),
SkyCoord(
0,
0,
unit=u.deg,
frame="heliographic_carrington",
#frame="heliographic_stonyhurst",
obstime=map_stb.date),
#reference_pixel=[0,(oshape[0] - 1)/2.]*u.pix,
scale=[180 / oshape[0], 360 / oshape[1]] * u.deg / u.pix,
projection_code="CAR")
#header['crval2'] = 0
out_wcs = WCS(header)
# Reproject
array, footprint = reproject_interp((map_stb.data, map_stb.wcs),
out_wcs,
shape_out=oshape)
omap_stb = sunpy.map.Map((array, header))
omap_stb.plot_settings = map_stb.plot_settings
# Normalize data to exposure time
omap_stb_data = (omap_stb.data /
(map_stb.exposure_time / u.second)).value
# Condense the reprojected map minimums into chmap
chmap = np.fmin(chmap, omap_stb_data)
chmap_stb = np.copy(chmap)
# CL - Make sure to align output maps with longitude coordinates, notably shift the zero point from the center to the edge of the data frame
#coords = sunpy.map.all_coordinates_from_map(omap_aia)
# CL - Create a secondary map weighted by longevity of CH over rotation when observed?
# Shift things and generate coordinates
# CL - Temporary fix for weird start NaN column and row
# CL - Might also need to shift coordinates to sine latitude...
chmap_aia[:, 0] = (chmap_aia[:, -1] + chmap_aia[:, 1]) / 2
chmap_aia[0, :] = np.nan
chmap_aia = np.roll(chmap_aia, [0, int(oshape[1] / 2)])
chmap_sta[:, 0] = (chmap_sta[:, -1] + chmap_sta[:, 1]) / 2
chmap_sta[0, :] = np.nan
chmap_sta = np.roll(chmap_sta, [0, int(oshape[1] / 2)])
chmap_stb[:, 0] = (chmap_stb[:, -1] + chmap_stb[:, 1]) / 2
chmap_stb[0, :] = np.nan
chmap_stb = np.roll(chmap_stb, [0, int(oshape[1] / 2)])
lats = np.linspace(-90, 90, oshape[0])
lons = np.linspace(0, 360, oshape[1])
pscale = [oshape[0] / 180, oshape[1] / 360]
# Generate threshold values for AIA
qs = np.nanmedian(chmap_aia)
if np.isfinite(qs):
thrsh = np.array([])
[dlat, dlon] = [60, 60]
for ilat in np.arange(0, 180 / dlat):
for ilon in np.arange(0, 360 / dlon):
plat0 = int(ilat * dlat * pscale[0])
plat1 = int((ilat + 1) * dlat * pscale[0])
plon0 = int(ilon * dlon * pscale[1])
plon1 = int((ilon + 1) * dlon * pscale[1])
sarr = chmap_aia[plat0:plat1, plon0:plon1]
#sarr_hist = histogram(sarr[where(np.isfinite(sarr))].flatten(), bins=100, range=[np.nanmin(sarr),qs])
sarr_hist = np.histogram(sarr[np.where(
np.isfinite(sarr))].flatten(),
bins=100,
range=[0, qs])
#sarr_dist = scipy.stats.rv_histogram(sarr_hist)
sh_x = sarr_hist[1][0:-1]
sh_y = sarr_hist[0]
sh_y2 = scipy.signal.convolve(
sh_y, scipy.signal.hann(20), mode='same') / sum(
scipy.signal.hann(20))
pks = scipy.signal.find_peaks_cwt(sh_y2, np.arange(1, 20))
if len(pks) >= 2:
sh_x2 = sh_x[pks[0]:pks[-1] - 1]
sh_y3 = sh_y2[pks[0]:pks[-1] - 1]
else:
minval = int(len(sh_x) / 4)
maxval = int(len(sh_x) * 0.9)
sh_x2 = sh_x[minval:maxval]
sh_y3 = sh_y2[minval:maxval]
pks2 = scipy.signal.find_peaks_cwt(-1 * sh_y3,
np.arange(1, 20))
if len(pks2) != 0:
thrsh = np.append(thrsh, sh_x2[pks2[0]])
else:
thrsh = np.append(thrsh, np.nan)
chval_aia = np.nanmean(thrsh)
# Generate threshold values for STA
qs = np.nanmedian(chmap_sta)
if np.isfinite(qs):
thrsh = np.array([])
[dlat, dlon] = [60, 60]
for ilat in np.arange(0, 180 / dlat):
for ilon in np.arange(0, 360 / dlon):
plat0 = int(ilat * dlat * pscale[0])
plat1 = int((ilat + 1) * dlat * pscale[0])
plon0 = int(ilon * dlon * pscale[1])
plon1 = int((ilon + 1) * dlon * pscale[1])
sarr = chmap_sta[plat0:plat1, plon0:plon1]
#sarr_hist = histogram(sarr[where(np.isfinite(sarr))].flatten(), bins=100, range=[np.nanmin(sarr),qs])
sarr_hist = np.histogram(sarr[np.where(
np.isfinite(sarr))].flatten(),
bins=100,
range=[0, qs])
#sarr_dist = scipy.stats.rv_histogram(sarr_hist)
sh_x = sarr_hist[1][0:-1]
sh_y = sarr_hist[0]
sh_y2 = scipy.signal.convolve(
sh_y, scipy.signal.hann(20), mode='same') / sum(
scipy.signal.hann(20))
pks = scipy.signal.find_peaks_cwt(sh_y2, np.arange(1, 20))
if len(pks) >= 2:
sh_x2 = sh_x[pks[0]:pks[-1] - 1]
sh_y3 = sh_y2[pks[0]:pks[-1] - 1]
else:
minval = int(len(sh_x) / 4)
maxval = int(len(sh_x) * 0.9)
sh_x2 = sh_x[minval:maxval]
sh_y3 = sh_y2[minval:maxval]
pks2 = scipy.signal.find_peaks_cwt(-1 * sh_y3,
np.arange(1, 20))
if len(pks2) != 0:
thrsh = np.append(thrsh, sh_x2[pks2[0]])
else:
thrsh = np.append(thrsh, np.nan)
chval_sta = np.nanmean(thrsh)
# Generate threshold values for STB
qs = np.nanmedian(chmap_stb)
if np.isfinite(qs):
thrsh = np.array([])
[dlat, dlon] = [60, 60]
for ilat in np.arange(0, 180 / dlat):
for ilon in np.arange(0, 360 / dlon):
plat0 = int(ilat * dlat * pscale[0])
plat1 = int((ilat + 1) * dlat * pscale[0])
plon0 = int(ilon * dlon * pscale[1])
plon1 = int((ilon + 1) * dlon * pscale[1])
sarr = chmap_stb[plat0:plat1, plon0:plon1]
#sarr_hist = histogram(sarr[where(np.isfinite(sarr))].flatten(), bins=100, range=[np.nanmin(sarr),qs])
sarr_hist = np.histogram(sarr[np.where(
np.isfinite(sarr))].flatten(),
bins=100,
range=[0, qs])
#sarr_dist = scipy.stats.rv_histogram(sarr_hist)
sh_x = sarr_hist[1][0:-1]
sh_y = sarr_hist[0]
sh_y2 = scipy.signal.convolve(
sh_y, scipy.signal.hann(20), mode='same') / sum(
scipy.signal.hann(20))
pks = scipy.signal.find_peaks_cwt(sh_y2, np.arange(1, 20))
if len(pks) >= 2:
sh_x2 = sh_x[pks[0]:pks[-1] - 1]
sh_y3 = sh_y2[pks[0]:pks[-1] - 1]
else:
minval = int(len(sh_x) / 4)
maxval = int(len(sh_x) * 0.9)
sh_x2 = sh_x[minval:maxval]
sh_y3 = sh_y2[minval:maxval]
pks2 = scipy.signal.find_peaks_cwt(-1 * sh_y3,
np.arange(1, 20))
if len(pks2) != 0:
thrsh = np.append(thrsh, sh_x2[pks2[0]])
else:
thrsh = np.append(thrsh, np.nan)
chval_stb = np.nanmean(thrsh)
chmap0_aia = np.copy(chmap_aia)
if not skip_aia:
chmap0_aia[np.where(chmap_aia > chval_aia)] = 0
chmap0_sta = np.copy(chmap_sta)
if not skip_sta:
chmap0_sta[np.where(chmap_sta > chval_sta)] = 0
chmap0_stb = np.copy(chmap_stb)
if not skip_stb:
chmap0_stb[np.where(chmap_stb > chval_stb)] = 0
# Generate a merged chmap
# CL - make changes here to restore to original behavior of measuring CH depth. Might need to create a normalized merged of AIA / STA data to fill the gaps
nodat = (np.logical_and(~np.isfinite(chmap0_aia),
~np.isfinite(chmap0_sta)))
chmap0 = (np.nansum(np.dstack(
(chmap0_aia, chmap0_sta, chmap0_stb)), 2) != 0)
# chmap0 = chmap_aia * (np.nansum(np.dstack((chmap0_aia, chmap0_sta)),2) != 0)
ocstruct = np.ones([3, 3])
chmap1 = scipy.ndimage.binary_opening(scipy.ndimage.binary_closing(
chmap0, structure=ocstruct, iterations=1),
structure=ocstruct,
iterations=1) * chmap0
# Label each dark region for comparison
labstruct = np.ones([3, 3])
lchmap = (scipy.ndimage.label(chmap1, structure=labstruct))[0]
for i in np.arange(2, lchmap.max() + 1):
chmag = br[np.where(lchmap == i)]
if len(chmag) < 10:
chmap1[np.where(lchmap == i)] = 0
continue
if abs(scipy.stats.skew(chmag)) < 0.5:
chmap1[np.where(lchmap == i)] = 0
continue
chmap = chmap1
# For visualization, scale the images to created a merged view
im_aia = np.copy(chmap_aia)
im_sta = np.copy(chmap_sta)
im_stb = np.copy(chmap_stb)
im_aia = im_aia - np.nanmin(im_aia)
im_aia = im_aia / np.nanmax(im_aia)
im_sta = im_sta - np.nanmin(im_sta)
im_sta = im_sta / np.nanmax(im_sta)
im_stb = im_stb - np.nanmin(im_stb)
im_stb = im_stb / np.nanmax(im_stb)
chim = np.fmin(im_sta, im_aia)
chim = np.fmin(chim, im_stb)
# Save everything out to file
fname = outdir + 'chmap/chmap-' + str(cr) + '.fits'
sunpy.io.write_file(fname, chmap * chim, header, overwrite=True)
fname = outdir + 'chmap/chmap-' + str(cr) + '-chim.fits'
sunpy.io.write_file(fname, chim, header, overwrite=True)
import astropy.units as u
import sunpy.map
from sunpy.net import Fido
from sunpy.net import attrs as a
######################################################################
# In this example we are going to make a lot of side by side figures, so
# let's change the default figure size.
plt.rcParams['figure.figsize'] = (16, 8)
######################################################################
# We are going to download one AIA and one HMI magnetogram image.
time = (a.Sample(24 * u.hour)
& a.Time('2010-08-19', '2010-08-19T00:10:00', '2010-08-19')
& a.vso.Extent(0, 0, 0, 0, "FULLDISK"))
aia = a.Instrument('AIA') & a.Wavelength(17 * u.nm, 18 * u.nm)
hmi = a.Instrument('HMI') & a.Physobs("LOS_magnetic_field")
res = Fido.search(time, aia | hmi)
files = Fido.fetch(res[:, 0])
######################################################################
# We create a map for each image and resample each one just to
# reduce the computation time.
map_aia, map_hmi = [
m.resample((1024, 1024) * u.pix) for m in sunpy.map.Map(sorted(files))
def test_no_match():
with pytest.raises(DrmsQueryError):
Fido.search(a.Time("2016/10/01", "2016/10/02"), a.jsoc.Series("bob"),
a.Sample(10 * u.s))
elif SERIES == "hmi.Ic_noLimbDark_720s":
SEGMENT = "continuum"
PRINTER.info_text("Data segment set to 'continuum'")
#########################
PRINTER.line()
PRINTER.info_text("Searching JSOC database")
if INSTRUMENT == "aia":
results = Fido.search(
a.jsoc.Time("%s" % START_TIME.replace(microsecond=0).isoformat(),
"%s" % END_TIME.replace(microsecond=0).isoformat()),
a.jsoc.Notify(EMAIL), a.jsoc.Series(SERIES),
a.jsoc.Wavelength(WAVELENGTH * u.angstrom),
a.Sample(CADENCE * u.second))
elif INSTRUMENT == "hmi":
results = Fido.search(
a.jsoc.Time("%s" % START_TIME.replace(microsecond=0).isoformat(),
"%s" % END_TIME.replace(microsecond=0).isoformat()),
a.jsoc.Notify(EMAIL), a.jsoc.Series(SERIES), a.jsoc.Segment(SEGMENT),
a.Sample(CADENCE * u.second))
PRINTER.info_text("Search complete. Displaying:")
print "\n" + results
#########################
PRINTER.line()
PRINTER.input_text("Press 'enter' to download")
from astropy.time import TimeDelta
import sunpy.coordinates
import sunpy.map
from sunpy.io.special import srs
from sunpy.net import Fido
from sunpy.net import attrs as a
from sunpy.time import parse_time
##############################################################################
# For this example, we will search for and download a single HMI using Fido.
start_time = parse_time("2017-01-25")
end_time = start_time + TimeDelta(23 * u.hour + 59 * u.minute + 59 * u.second)
results = Fido.search(a.Time(start_time, end_time),
a.Instrument.hmi & a.Physobs.los_magnetic_field,
a.Sample(60 * u.second))
##############################################################################
# Let's select only the first file, download it and create a map.
result = results[0, 0]
file_name = Fido.fetch(result)
smap = sunpy.map.Map(file_name)
##############################################################################
# Download the SRS file.
srs_results = Fido.search(a.Time(start_time, end_time), a.Instrument.srs_table)
srs_downloaded_files = Fido.fetch(srs_results)
##############################################################################
# We get one SRS file per day. To read this file, we pass the filename into
# the SRS reader. So now `srs_table` contains an astropy table.
# See `this page <http://jsoc.stanford.edu/ajax/register_email.html>`_
# for more details.
jsoc_email = os.environ["JSOC_EMAIL"]
#####################################################
# Now we are ready to construct the query. Note that all of this is
# the same for a full-frame image except for the
# cutout component. We will download images from a 12 hour interval
# centered on the time of the above cutout.
# We request one image every 2 hours.
query = Fido.search(
a.Time(cutouts.date - 6*u.h, cutouts.date + 6*u.h),
a.Wavelength(cutouts.wavelength),
a.Sample(2*u.h),
a.jsoc.Series.aia_lev1_euv_12s,
a.jsoc.Notify(jsoc_email),
a.jsoc.Segment.image,
cutout,
)
#####################################################
# Submit the export request and download the data.
files = Fido.fetch(query)
files.sort()
#####################################################
# Now that we've downloaded the files, we can create
# a `~sunpy.map.MapSequence` from them.
from astropy.coordinates import SkyCoord
from astropy.wcs import WCS
import sunpy.map
import sunpy.sun
from sunpy.coordinates import get_body_heliographic_stonyhurst
from sunpy.net import Fido
from sunpy.net import attrs as a
######################################################################
# To get started, let's download the data:
stereo = (a.Instrument("EUVI") &
a.Time('2011-11-01', '2011-11-01T00:10:00'))
aia = (a.Instrument.aia &
a.Sample(24 * u.hour) &
a.Time('2011-11-01', '2011-11-02'))
wave = a.Wavelength(19.5 * u.nm, 19.5 * u.nm)
res = Fido.search(wave, aia | stereo)
files = Fido.fetch(res)
######################################################################
# Next we create a sunpy map for each of the files.
maps = sunpy.map.Map(sorted(files))
######################################################################
# To reduce memory consumption we also downsample these maps before continuing,
# you can disable this.
maps = [m.resample((1024, 1024)*u.pix) for m in maps]
os.makedirs(path) if not os.path.exists(path) else None
start_date = pd.Timestamp('2011/01/01 00:00:14Z')
end_date = pd.Timestamp('2017/01/01 00:00:00Z')
batch_span = 30*24 # download files in groups spanning 30 days
batch_start = start_date
batch_end = batch_start + pd.Timedelta(hours=batch_span)
while batch_end <= end_date:
res_aia = Fido.search(a.Time(batch_start, batch_end),
a.jsoc.Notify(email),
a.jsoc.Series('aia.lev1_uv_24s'),
a.jsoc.Segment('image'),
a.jsoc.Wavelength(1700*u.AA),
a.Sample(12*u.hour)
)
print(res_aia)
# download files
try:
downloaded_files = Fido.fetch(res_aia, path=path)
except Exception as e:
print("couldn't download files between " +
str(batch_start) + " and " + str(batch_end))
print(str(e))
# end of current batch:
end_of_batch = str(res_aia.get_response(0)['T_REC'][-1])
# start and end dates of next batch
How to download an HMI magnetogram data with Fido and make a plot.
"""
import astropy.units as u
import matplotlib.pyplot as plt
import sunpy.map
from sunpy.net import Fido, attrs as a
###############################################################################
# To download the required data, we use
# `Fido <sunpy.net.fido_factory.UnifiedDownloaderFactory>`, a downloader client.
# First define the search variables, a timerange, the instrument,
# the observation type,
# and a cadence of images spaced every 720 seconds.
result = Fido.search(a.Time('2011/11/09 17:40:00', '2011/11/09 17:55:00'),
a.Instrument('hmi'), a.Sample(720 * u.s),
a.vso.Physobs('LOS_magnetic_field'))
###############################################################################
# Now we can see what results we obtained from our search.
# Notice we have two files.
print(result)
###############################################################################
# The following shows how to download the results. If we
# don't provide a path it will download the file into the sunpy data directory.
# The output provides the path of the downloaded files.
# The result can be from several data clients, so we have to index the client first
# client and then index the file.
# Slice the first record returned by the first client.
HMI = args.HMI
HMI_path = args.HMI_path
STEREO = args.STEREO
STEREO_path = args.STEREO_path
cadence = args.cadence * u.hour # take images every 12 hours
wavelength = args.wavelength * u.AA # wavelength in angstroms
stereo_cadence = args.STEREO_cadence # use 1 in every n images
# time intervals to avoid missing data
missing = [AIA_start, AIA_end]
if AIA:
res_aia = Fido.search(a.Time(AIA_start, AIA_end), a.jsoc.Notify(email),
a.jsoc.Series('aia.lev1_euv_12s'),
a.jsoc.Segment('image'),
a.jsoc.Wavelength(wavelength), a.Sample(cadence))
print(res_aia)
# save the the query details to file
with open('data_query_aia.pkl', 'wb') as f:
pickle.dump([AIA_start, AIA_end, res_aia], f)
# add dates of missing data
res_table = res_aia.get_response(0)
for i in range(len(res_table)):
if str(res_table['CAR_ROT'][i]) == "--":
time = (str(res_table['T_REC'][i][:-6]) + '00').replace('_', ' ')
missing.append(time.replace('.', '/'))
if HMI:
client = jsoc.JSOCClient()
years = [i + 2013 for i in range(7)]
months = [i + 2 for i in range(11)]
def remove_glob(pathname, recursive=True):
for p in glob.glob(pathname, recursive=recursive):
if os.path.isfile(p):
os.remove(p)
for year in years:
for month in months:
days = [day + 1 for day in range(calendar.monthrange(year, month)[1])]
for day in days:
start = datetime.datetime(year, month, day, 00, 00, 00)
end = start + datetime.timedelta(days=1)
print(start, end)
response = client.search(
a.Time(start, end), a.jsoc.Series('hmi.Mharp_720s'),
a.Sample(1 * u.hour),
a.jsoc.Notify("*****@*****.**"
)) #開始時間のところに等号を入れられなかったので調整
path = "/media/akito/Data/Mharp/bitmap/" + str(year) + "/" + str(
year) + str(month).zfill(2)
res = response.client.fetch(response, path=path)
rm_path1 = path + "/" + "*.magnetogram.fits"
rm_path2 = path + "/" + "*.mask.fits"
remove_glob(rm_path1)
remove_glob(rm_path2)
download_chunk = timedelta(
days=10
) #avoid download chunks greater than 1 month in order to not download too much at once
#breaks the download into pieces and downloads
current_time = start
while (current_time < end):
if (end - current_time > download_chunk):
next_time = current_time + download_chunk
else:
next_time = end
response = Fido.search(attrs.jsoc.Time(current_time, next_time),
attrs.jsoc.Notify('*****@*****.**'),
attrs.jsoc.Series('hmi.Sharp_720s'),
attrs.jsoc.Segment('bitmap'),
attrs.Sample(time_interval.total_seconds() * u.s))
response
res = Fido.fetch(response, path=sharp_dir + '/{file}.fits')
current_time = next_time
warnings.simplefilter(
"ignore"
) #.verify('fix') produces many warnings which will lag the jupyter notebook
#extracts relevant keywords in the given order
keywords = [
'HARPNUM', 'T_REC', 'NAXIS1', 'NAXIS2', 'CDELT1', 'CDELT2', 'IMCRPIX1',
'IMCRPIX2', 'LAT_FWT', 'LON_FWT', 'NPIX'
] #Keywords in order to be saved
filenames = os.listdir(sharp_dir)
filename = 'data.txt'
# email first. Please replace this with your email
# address once you have registered.
# See `this page <http://jsoc.stanford.edu/ajax/register_email.html>`_
# for more details.
jsoc_email = os.environ["JSOC_EMAIL"]
#####################################################
# Now we are ready to construct the query. Note that all of this is
# the same for a full-frame image except for the
# cutout component. We will download images from a 12 hour interval
# centered on the time of the above cutout.
# We request one image every 12 minutes.
q = Fido.search(
a.Time(m_cutout.date - 6 * u.h, m_cutout.date + 6 * u.h),
a.Wavelength(m_cutout.wavelength),
a.Sample(12 * u.min),
a.jsoc.Series.aia_lev1_euv_12s,
a.jsoc.Notify(jsoc_email),
a.jsoc.Segment.image,
cutout,
)
#####################################################
# Submit the export request and download the data. We
# set the number of parallel downloads to 2 so as not
# to overwhelm the JSOC export service with our number
# of download requests.
files = Fido.fetch(q, max_conn=2)
files.sort()
#####################################################
def test_query(client):
Jresp = client.search(
a.Time('2020/1/1T00:00:00', '2020/1/1T00:01:30'),
a.jsoc.Series('hmi.M_45s'), a.Sample(90 * u.second))
assert isinstance(Jresp, JSOCResponse)
assert len(Jresp) == 2
p.add_argument('-cadence', type=int, default=8, help='sampling time for data')
args = p.parse_args()
print(
r'---------------------- Fido download for EIT,EUVI-A, EUVI-B ----------------------'
)
eit = (a.Instrument("eit") & a.Time(args.startdate, args.enddate))
euvi = (a.vso.Source('STEREO_A') & a.Instrument("EUVI")
& a.Time(args.startdate, args.enddate))
euvib = (a.vso.Source('STEREO_B') & a.Instrument("EUVI")
& a.Time(args.startdate, args.enddate))
samp = a.Sample(args.cadence * u.minute)
wave = a.Wavelength(args.wavelengths[0] * u.AA, args.wavelengths[1] * u.AA)
print(args.startdate, args.enddate, wave, samp)
fethched_results = Fido.search(wave, eit | euvi | euvib, samp)
print(
r'---------------------- Fido fetched files for EIT,EUVI-A, EUVI-B ----------------------'
)
print(fethched_results)
files = Fido.fetch(fethched_results, path=args.data_path)
while not files.errors:
time.sleep(30)
files = Fido.fetch(files)
'''
from sunpy.net.attr import AttrAnd, AttrOr
ta = Time(time_now, format='datetime', scale='tai')
ta1 = Time(time_now - datetime.timedelta(days=2),
format='datetime',
scale='tai')
ta2 = Time(time_now - datetime.timedelta(days=1),
format='datetime',
scale='tai')
segments = ['inclination', 'azimuth', 'disambig', 'field']
series = 'hmi.B_720s'
interval = 10 * u.min
email = '*****@*****.**'
response = Fido.search(
a.jsoc.Time(ta1, ta2), a.jsoc.Series(series), a.jsoc.Notify(email),
a.Sample(interval),
AttrAnd(list(map(a.jsoc.Segment, segments)))
# i.e. attrs.jsoc.Segment('...') & attrs.jsoc.Segment('...') ...
)
response
Fido.search(a.jsoc.Time('2014-01-01T00:00:00', '2014-01-01T01:00:00'),
a.jsoc.Series('hmi.sharp_720s'), a.jsoc.Notify('*****@*****.**'),
a.jsoc.Segment('image'))
Fido.search(a.jsoc.Time('2019-01-15T00:00:00', '2019-01-16T01:00:00'),
a.jsoc.Series('hmi.M_720s_nrt'), a.jsoc.Notify('*****@*****.**'),
a.jsoc.Segment('image'))
import astropy.units as u
import sunpy.map
from sunpy.net import Fido, attrs as a
###############################################################################
# Now we will download some data with `sunpy.net.Fido`.
# A `Fido.search` requires us to specify a `~sunpy.net.attr.Time`,
# `~sunpy.net.attr.Sample`, `~sunpy.net.attr.Instrument`
# and the `~sunpy.net.attr.vso.Physobs`.
# We set a time range from ``2015/11/04 12:00:00`` to ``2015/11/04 12:10:00``
# for HMI ``LOS_magnetic_field`` with the images spaced every 720 seconds.
result = Fido.search(a.Time('2015/11/04 12:00:00', '2015/11/04 12:10:00'),
a.Instrument('hmi'),
a.Sample(720*u.s),
a.vso.Physobs('LOS_magnetic_field'))
###############################################################################
# Now we can see what results we obtained from our search.
# Notice we have two files. One is the full disk image we plan to display
# and the other is a synoptic version of said image.
print(result)
###############################################################################
# Once we are happy with the results obtained from the search.
# We can download the data with `Fido.fetch` .
# In this case we only want one file so we can index the result.
# A `Fido` result can be from several clients, so we have to index the first
# client and then index the first result.
import astropy.units as u
from sunpy.net import Fido
from sunpy.net import attrs as a
stereo = (a.vso.Source('STEREO_B') & a.Instrument('EUVI')
& a.Time('2011-01-01', '2011-01-01T00:10:00'))
aia = (a.Instrument('AIA') & a.Sample(24 * u.hour)
& a.Time('2011-01-01', '2011-01-02'))
wave = a.Wavelength(30 * u.nm, 31 * u.nm)
results = Fido.search(aia | stereo, wave)
res = Fido.fetch(results, path='./data/{file}')
