def createHandler(self):
if self.received:
params = self.params
self.received = False
client = vso.VSOClient()
timestamp = params['timestamp']
start = params['start']
end = params['end']
cadence = params['cadence']
layers = list()
cutoutEnabled = bool(params['cutout.set'])
if (cutoutEnabled):
cutout = CutoutInfo(cutoutEnabled, float(params['cutout.x0']),
float(params['cutout.y0']),
float(params['cutout.w']),
float(params['cutout.h']))
else:
cutout = CutoutInfo(cutoutEnabled)
for layerInfo in params['layers']:
observatory = layerInfo['observatory']
instrument = layerInfo['instrument']
detector = layerInfo['detector']
measurement = layerInfo['measurement']
layerTimeStamp = layerInfo['timestamp']
layers.append(
LayerHandler(client, layerTimeStamp, observatory,
instrument, detector, measurement, cutout))
handler = VSOHandler(client, start, end, timestamp, cadence,
layers)
return handler
def downloadData(q_provider, q_source, q_instrument, q_physobs):
client = vso.VSOClient()
qr = client.search(
vso.attrs.Time('2016-05-07 00:00:00', '2016-05-08 00:00:00'))
print(qr)
database.add_from_vso_query_result(qr)
database.commit()
def qr_block_with_missing_physobs():
return vso.VSOClient().search(
net_attrs.Time('20130805T120000', '20130805T121000'),
net_attrs.Instrument('SWAVES'), net_attrs.Source('STEREO_A'),
net_attrs.Provider('SSC'), net_attrs.Wavelength(
10 * u.kHz, 160 * u.kHz),
response_format="legacy")[0]
def download_euvi_files(eucme, craft_vso, cme_root):
"""
Downloads euvi images of events.
"""
# Loop over the events, make event directory, download closest fits file
# and several pngs.
files = []
for i in range(len(eucme)):
print i
# Now load in the most appropriate EUVI image for each event.
vsoclient = vso.VSOClient()
# Create folder for images to be stored in
cme_dir = cme_root + r'\cme_' + '{0:02d}'.format(i + 1)
print cme_dir
if not os.path.exists(cme_dir):
os.makedirs(cme_dir)
# If there is no EUVI data?
if pd.isnull(eucme['tm'][i]):
files.append(None)
# Skip rest of go
continue
# Search for COR1 data in +/- 30min window around event tm1
dt = pd.Timedelta(30, 'm')
print dt
tl = [(eucme['tm'][i] - dt).isoformat(),
(eucme['tm'][i] + dt).isoformat()]
print tl
# Query VSO to find what files available
qr = vsoclient.query(vso.attrs.Time(tl[0], tl[1]),
vso.attrs.Instrument('euvi'),
vso.attrs.Source(craft_vso))
# #Take out values that are not from the 171Angstrom Channel
# q=0
# while q <= len(qr)-1:
# wve = qr[q]['wave']['wavemin']
# if wve=="171":
# q = q+1
# else:
# qr.pop(q)
print len(qr)
if len(qr) > 1:
# Find closest mathced in time
# Get datetimes of the query resutls
times = np.array([pd.to_datetime(x['time']['start']) for x in qr])
# Remove querys before the closest match
q = 0
while q < np.argmin(np.abs(times - eucme['tm'][i])):
qr.pop(0)
q = q + 1
# Remove querys after the closest match
while len(qr) > 1:
qr.pop(1)
# Download best matched file
fn = vsoclient.get(qr, path=cme_dir + r"\{file}").wait()
files.append(fn[0])
else:
files.append(None)
print(files)
def create_tempmap(date,
n_params=1,
data_dir=home + 'SDO_data/',
maps_dir=home + 'temperature_maps/'):
wlens = ['94', '131', '171', '193', '211', '335']
t0 = 5.6
images = []
#imdates = {}
print 'Finding data for {}.'.format(date.date())
# Loop through wavelengths
for wl, wlen in enumerate(wlens):
#print 'Finding {}A data...'.format(wlen),
fits_dir = data_dir + '{}/{:%Y/%m/%d}/'.format(wlen, date)
filename = fits_dir + 'aia*{0}*{1:%Y?%m?%d}?{1:%H?%M}*lev1?fits'.format(
wlen, date)
temp_im = Map(filename)
# Download data if not enough found
client = vso.VSOClient()
if temp_im == []:
print 'File not found. Downloading from VSO...'
# Wavelength value for query needs to be an astropy Quantity
wquant = u.Quantity(value=int(wlen), unit='Angstrom')
qr = client.query(
vso.attrs.Time(
date, # - dt.timedelta(seconds=6),
date + dt.timedelta(seconds=12)), #6)),
vso.attrs.Wave(wquant, wquant),
vso.attrs.Instrument('aia'),
vso.attrs.Provider('JSOC'))
res = client.get(qr, path=fits_dir + '{file}', site='NSO').wait()
temp_im = Map(res)
if temp_im == []:
print 'Downloading failed.'
print res, len(qr), qr
return np.zeros((512, 512)), None, None
if isinstance(temp_im, list):
temp_im = temp_im[0]
# TODO: save out level 1.5 data so it can be loaded quickly.
temp_im = aiaprep(temp_im)
temp_im.data = temp_im.data / temp_im.exposure_time # Can probably increase speed a bit by making this * (1.0/exp_time)
images.append(temp_im)
#imdates[wlen] = temp_im.date
normim = images[2].data.copy()
# Normalise images to 171A
print 'Normalising images'
for i in range(len(wlens)):
images[i].data = images[i].data / normim
# Produce temperature map
if n_params == 1:
tempmap = find_temp(images, t0) #, force_temp_scan=True)
else:
#tempmap = find_temp_3params(images, t0)
pass
return tempmap
def fetch_aia(self, product, correct=True, median_kernel=5):
"""
pull halpha from that time from Virtual Solar Observatory GONG archive
:param product: aia-[REQUESTED CHANNEL IN ANGSTROMS], e.g. aia-131 gets the 131 angstrom image
:param correct: remove nans and negatives
:return: tuple of product name, fits header, and data object
the header and data object will be None if the request failed
"""
if self.verbose:
print("Requesting {}".format(product))
wavelength = product.split("-")[1]
def time_interval(time):
""" get a window of three minutes around the requested time to ensure an image at GONG cadence"""
return time - timedelta(minutes=15), time + timedelta(minutes=15)
# setup the query for an halpha image and fetch, saving the image in the current directory
client = vso.VSOClient()
wave, source = Quantity(wavelength, "angstrom"), "aia"
query = client.search(vso.attrs.Time(*time_interval(self.date)),
vso.attrs.Instrument(source),
vso.attrs.Wavelength(wave))
if self.verbose:
print("Query length for {} is {}".format(product, len(query)))
if not query: # failed to get a result
return product, None, None
else:
query = query[0]
result = client.fetch([query], path="./").wait(progress=False)
# open the image and remove the file
with fits.open(result[0]) as hdu:
hdu.verify('fix')
head = hdu[1].header
data = hdu[1].data
os.remove(result[0])
data, head = self.align_solar_fov(head, data, 2.5, 1280, rotate=False)
data = resize(data, (1280, 1280))
head['NAXIS1'] = 1280
head['NAXIS2'] = 1280
head['CRPIX1'] = 640
head['CRPIX2'] = 640
head['CDELT1'] = 2.5
head['CDELT2'] = 2.5
if correct:
data[np.isnan(data)] = 0
data[data < 0] = 0
if median_kernel:
data = medfilt(data, median_kernel)
return product, head, data
def vso_session():
client = vso.VSOClient()
qr = client.search(vso.attrs.Time((2011, 9, 20, 1), (2011, 9, 20, 2)),
vso.attrs.Instrument('RHESSI'))
entries = tables.entries_from_query_result(qr)
database = Database('sqlite:///:memory:')
for entry in entries:
database.add(entry)
database.commit()
return database.session
def vso_session():
client = vso.VSOClient()
qr = client.search(a.Time((2011, 9, 20, 1), (2011, 9, 20, 2)),
a.Instrument.rhessi,
response_format="legacy")
entries = tables.entries_from_query_result(qr)
database = Database('sqlite:///:memory:')
for entry in entries:
database.add(entry)
database.commit()
return database.session
def __init__(self, verbose=False):
self.base_url = 'https://stereo-ssc.nascom.nasa.gov/data/ins_data'
# initialize the vso client
self.client = vso.VSOClient()
# initialize the VSO detector field
self.detector = 'EUVI'
if verbose:
print('### Initialized VSO client for ' + self.detector)
def fetch_halpha(self, correct=True, median_kernel=5):
"""
pull halpha from that time from Virtual Solar Observatory GONG archive
:param correct: remove nans and negatives
:return: tuple of "halpha", a fits header, and data object for the GONG image at that time
header and data will be None if request failed
"""
if self.verbose:
print("Requesting halpha")
def time_interval(time):
""" get a window of three minutes around the requested time to ensure an image at GONG cadence"""
return time - timedelta(minutes=3), time + timedelta(minutes=3)
# setup the query for an halpha image and fetch, saving the image in the current directory
client = vso.VSOClient()
halpha, source = Quantity(656.28, "nm"), "gong"
query = client.search(vso.attrs.Time(*time_interval(self.date)),
vso.attrs.Source(source),
vso.attrs.Wavelength(halpha))
if not query: # no images found in query
return "halpha", None, None
else:
query = query[0]
result = client.fetch([query], path="./").wait(progress=False)
# open the image and remove the file
with fits.open(result[0]) as hdu:
head = hdu[1].header
data = hdu[1].data
os.remove(result[0])
# scale halpha to suvi
scale = 2.35
tform = AffineTransform(
scale=(scale, scale)) # , translation=(-1280/2,-1280/2))
data = warp(data, tform, output_shape=(1280, 1280))
tform = AffineTransform(translation=(
-(640 - 1024 / scale),
-(640 - 1024 / scale))) # , translation=(-1280/2,-1280/2))
data = warp(data, tform)
if correct:
data[np.isnan(data)] = 0
data[data < 0] = 0
if median_kernel:
data = medfilt(data, median_kernel)
return "halpha", head, data
def download_aiafits(init_time=None,
time_interval=None,
wave1=None,
wave2=None,
instrument=None,
path=None):
client = vso.VSOClient()
init_t = Time(init_time, format='isot')
timed = TimeDelta(time_interval, format='sec')
end_t = init_t + timed
qr = client.query(vso.attrs.Time(init_t.iso, end_t.iso),
vso.attrs.Instrument(instrument),
vso.attrs.Wave(wave1 * u.AA, wave2 * u.AA))
res = client.get(qr, path=path)
def acquire_fits(directory,
time_range,
observatory='SDO',
instrument='AIA',
detector='AIA',
measurement='211',
verbose=True):
"""Acquire FITS files within the specified time range."""
client = vso.VSOClient()
tstart = time_range.t1.strftime('%Y/%m/%d %H:%M')
tend = time_range.t2.strftime('%Y/%m/%d %H:%M')
#check if any files are already in the directory
current_files = [
f for f in os.listdir(os.path.expanduser(directory))
if f.endswith('.fits')
]
#search VSO for FITS files within the time range, searching for AIA 211A only at a 36s cadence
print 'Querying VSO to find FITS files...'
qr = client.query(vso.attrs.Time(tstart, tend),
vso.attrs.Instrument('aia'), vso.attrs.Wave(211, 211),
vso.attrs.Sample(36))
dir = os.path.expanduser(directory)
print 'Downloading ' + str(len(qr)) + ' files from VSO to ' + dir
for q in qr:
filetimestring=q.time.start[0:4] + '_' + q.time.start[4:6] + '_' + q.time.start[6:8] + 't' \
+ q.time.start[8:10] + '_' +q.time.start[10:12] + '_' + q.time.start[12:14]
exists = []
for c in current_files:
if filetimestring in c:
exists.append(True)
else:
exists.append(False)
if not any(exists) == True:
res = client.get([q], path=os.path.join(dir, '{file}.fits')).wait()
else:
print 'File at time ' + filetimestring + ' already exists. Skipping'
fits_list = [
os.path.join(dir, f) for f in os.listdir(dir) if f.endswith('.fits')
]
return fits_list
def __init__(self, src_name, inst_name, index_file = None, t_start = None, t_end = None, wavl_min = None, wavl_max = None, data_dir =''):
# Set instrument properties
self.source = src_name
self.instrument = inst_name
# check the keyword parameters
if index_file != None: # Check if the file_list parameter is specified
# Read index file into a list with each line as a separate element/filename
with open(index_file) as f:
file_list = f. readlines()
file_list = [x.strip() for x in file_list] # Stip off newlines, and trailing and leading whitespace
self.import_data(file_list) # If so, we are free to import data
elif ((t_start != None) and (t_end != None) and (wavl_min != None) and (wavl_max != None)): # If not, we need to grab the data from the VSO
# Find the available files for download using the Virtual Solar Observatory
c = vso.VSOClient() # Initialize Sunpy VSO client
#qr = c.query(vso.vso.attrs.Time(t_start, t_end), vso.vso.attrs.Instrument(self.instrument), vso.vso.attrs.Wave(wavl_min * u.AA, wavl_max * u.AA))
#qr = c.query(vso.vso.attrs.Time(t_start, t_end), vso.vso.attrs.Instrument(self.instrument))
qr = c.query_legacy(tstart=t_start, tend=t_end, instrument=self.instrument, min_wave=wavl_min, max_wave=wavl_max, unit_wave='Angstrom') # Query the VSO for files
print(qr) # Print the query
# Download the files returned by the query
dw = Downloader() # Initialize custom downloader class
r = c.get(qr, path = data_dir + '/{source}/{instrument}/{file}').wait()
print(r)
# Import the data
# self.import_data(file_list)
else: # Invalid keyword combination
print('Incorrect keyword specification')
def get_aia_vso(
output_path='/Users/admin/Documents/solarmonitor_2_0/sol_mon/fits_tests/{instrument}/'
):
client = vso.VSOClient()
#search_time = a.Time( '2019-01-12 00:00', '2019-01-15 00:00' )
#get utc time rounded down to the previous hour
time_now = datetime.datetime.utcnow().replace(microsecond=0,
second=0,
minute=0)
#time to search for available fits files
search_time = a.Time(
(time_now - datetime.timedelta(hours=10)).strftime('%Y/%m/%d %H:%M'),
time_now.strftime('%Y/%m/%d %H:%M'))
#using the VSO client rather than FIDO
result_aia = client.search(search_time, a.vso.Instrument('AIA'),
a.vso.Sample(1 * u.hour))
download_aia = client.fetch(result_aia[-10:], path=output_path)
return download_aia
def qr_block_with_missing_physobs():
return vso.VSOClient().query(
vso.attrs.Time('20130805T120000', '20130805T121000'),
vso.attrs.Instrument('SWAVES'), vso.attrs.Source('STEREO_A'),
vso.attrs.Provider('SSC'), vso.attrs.Wave(10 * u.kHz, 160 * u.kHz))[0]
def qr_with_none_waves():
return vso.VSOClient().query(
vso.attrs.Time('20121224T120049.8', '20121224T120049.8'),
vso.attrs.Provider('SDAC'), vso.attrs.Instrument('VIRGO'))
def query_result():
client = vso.VSOClient()
return client.query_legacy('2001/1/1', '2001/1/2', instrument='EIT')
import os
import glob
import PIL
## Setup your directory location for data storage
datadir = '/Users/clowder/data/'
## Define search parameters
ndays = 4
tsample = 0.5 * u.hour
curtime = datetime.datetime.utcnow()
starttime = curtime - datetime.timedelta(days=ndays + 1)
endtime = curtime - datetime.timedelta(days=1)
## Initialize the VSO search
client = vso.VSOClient()
## Search for matching files
## Merge these two queries at some point...
qr = client.query(
vso.attrs.Time(starttime, endtime), vso.attrs.Instrument('AIA'),
vso.attrs.Resolution(0.25), vso.attrs.Sample(tsample),
vso.attrs.Wave(193 * u.AA, 193 * u.AA)
| vso.attrs.Wave(304 * u.AA, 304 * u.AA)
| vso.attrs.Wave(171 * u.AA, 171 * u.AA)
| vso.attrs.Wave(211 * u.AA, 211 * u.AA)
| vso.attrs.Wave(131 * u.AA, 131 * u.AA)
| vso.attrs.Wave(335 * u.AA, 335 * u.AA)
| vso.attrs.Wave(94 * u.AA, 94 * u.AA)
| vso.attrs.Wave(1600 * u.AA, 1600 * u.AA))
qr2 = client.query(vso.attrs.Time(starttime, endtime),
def pytest_funcarg__client(request):
return vso.VSOClient()
def query_result():
client = vso.VSOClient()
return client.search(net_attrs.Time('2001/1/1', '2001/1/2'),
net_attrs.Instrument('EIT'))
def qr_with_none_waves():
return vso.VSOClient().search(net_attrs.Time('20121224T120049.8',
'20121224T120049.8'),
net_attrs.Provider('SDAC'),
net_attrs.Instrument('VIRGO'),
response_format="legacy")
def download_qr():
return vso.VSOClient().query(vso.attrs.Time('2012-03-29', '2012-03-29'),
vso.attrs.Instrument('AIA'))
def query_result():
return vso.VSOClient().query(
vso.attrs.Time('20130801T200000', '20130801T200030'),
vso.attrs.Instrument('PLASTIC'))
def vso_client():
vso.VSOClient()
def __init__(self):
self.hek_client = hek.HEKClient()
self.hek_results = ''
self.vso_client = vso.VSOClient()
self.vso_results = []
self.num_of_records = 0
def calculate_temperatures(date,
n_params=1,
data_dir=home + 'SDO_data/',
maps_dir=home + 'temperature_maps/',
n_procs=4):
wlens = ['94', '131', '171', '193', '211', '335']
client = vso.VSOClient()
print 'Finding data for {}.'.format(date.date())
# Loop through wavelengths
for wl, wlen in enumerate(wlens):
#print 'Finding {}A data...'.format(wlen),
fits_dir = data_dir + '{}/{:%Y/%m/%d}/'.format(wlen, date)
filename = fits_dir + 'aia*{0}*{1:%Y?%m?%d}?{1:%H?%M}*lev1?fits'.format(
wlen, date)
temp_im = Map(filename)
# Download data if not enough found
if temp_im == []:
print 'File not found. Downloading from VSO...'
qr = client.query(
vso.attrs.Time(
date, # - dt.timedelta(seconds=6),
date + dt.timedelta(seconds=12)), #6)),
vso.attrs.Wave(wlen, wlen),
vso.attrs.Instrument('aia'),
vso.attrs.Provider('JSOC'))
res = client.get(qr,
path=fits_dir + '{file}',
site='NSO',
methods=['URL_FILE_Rice']).wait()
n_wlens = len(wlens)
temp = np.arange(5.6, 7.01, 0.01)
n_temps = len(temp)
if n_params == 1:
wid = [0.1]
hei = [1.0]
else:
wid = np.arange(0.1, 1.1, 0.1) # Just copying Aschwanden's range here
hei = np.arange(15, 31)
n_widths = len(wid)
n_heights = len(hei)
n_vals = n_temps * n_widths * n_heights
try:
#raise IOError
model = np.memmap(filename=home +
'synth_emiss_{}pars'.format(n_params),
dtype='float32',
mode='r',
shape=(n_vals, n_wlens))
params = np.loadtxt(home + 'gaussian_parameters_{}'.format(n_params))
except IOError:
resp = load_temp_responses()
logt = np.arange(0, 15.05, 0.05)
delta_t = logt[1] - logt[0]
model = np.memmap(filename=home +
'synth_emiss_{}pars'.format(n_params),
dtype='float32',
mode='w+',
shape=(n_vals, n_wlens))
params = np.zeros((n_vals, n_params))
# Define linear index in model for this set of parameters
i = 0
# For given gaussian width, height and centre:
for width in wid:
for height in hei:
for meantemp in temp:
# For each channel k:
# intensity(k, params) <- sum(temperature_response(k, params) * DEM(params)) * dT
dem = gaussian(logt, float(meantemp), width, height)
model[i, :] = np.sum(resp * dem, axis=1) * delta_t
normmod = model[i, 2]
model[i, :] = model[i, :] / normmod
# Store these parameters for use later
params[i, :] = [meantemp, width, height]
i += 1
model.flush()
np.savetxt(home + 'gaussian_parameters_{}'.format(n_params), params)
sys('mpirun -np {} -host localhost python parallel_ext.py {}T{} {} {}'.
format(n_procs, date.date(), date.time(), n_params, n_vals))
return
def qr_block_with_kev_unit():
return vso.VSOClient().query(
vso.attrs.Time((2011, 9, 20, 1), (2011, 9, 20, 2)),
vso.attrs.Instrument('RHESSI'))[0]
def client(request):
return vso.VSOClient()
def qr_block_with_kev_unit():
return vso.VSOClient().search(net_attrs.Time((2011, 9, 20, 1),
(2011, 9, 20, 2)),
net_attrs.Instrument('RHESSI'),
response_format="legacy")[0]
def query_result():
client = vso.VSOClient()
return client.search(net_attrs.Time('2001/1/1', '2001/1/2'),
net_attrs.Instrument('EIT'),
response_format="legacy")
