def test_metadata_search(self):
try:
media_metadata_search()
except ValueError:
pass
except AssertionError:
self.fail("Exception")
def testUser1(self):
print("####User Test media_search & metadata_search #############################")
results_list = md.media_search(dates=[md.datetime(2016, 1, 1, 0, 0, 0), md.datetime(2016, 1, 1, 1, 0, 0)],
waves=['193'])
rnlist = [int(a.recnum) for a in results_list]
print(results_list[0:3])
m = md.media_metadata_search(keywords=['date__obs', 'quality'], recnum_list=rnlist, series='aia.lev1')
print(m[0:3])
self.assertEqual(len(m), 60)
def testMetadaSearch(self):
print("\n####Test " + sitools2_url + " meta-data-search #######################")
print("\n####hmi.sharp_cea_720s_nrt #################################")
print("Test media_metadata_search")
recnum_list = [2075898, 2075899, 2075900, 2075902, 2075903, 2075904, 2075905, 2075940,
2075938, 2075939]
print(recnum_list)
meta = media_metadata_search(keywords=['recnum', 'sunum', 'date__obs', 'quality', 'cdelt1', 'cdelt2', 'crval1'],
series="hmi.sharp_cea_720s_nrt", recnum_list=recnum_list, server=sitools2_url)
print(meta)
self.assertEqual(len(meta), 10)
def testUser1(self):
print(
"####User Test media_search & metadata_search #############################"
)
results_list = md.media_search(dates=[
md.datetime(2016, 1, 1, 0, 0, 0),
md.datetime(2016, 1, 1, 1, 0, 0)
],
waves=['193'])
rnlist = [int(a.recnum) for a in results_list]
print(results_list[0:3])
m = md.media_metadata_search(keywords=['date__obs', 'quality'],
recnum_list=rnlist,
series='aia.lev1')
print(m[0:3])
self.assertEqual(len(m), 60)
def testMetadaSearch(self):
print("\n####Test " + sitools2_url +
" meta-data-search #######################")
print("\n####hmi.sharp_cea_720s_nrt #################################")
print("Test media_metadata_search")
recnum_list = [
2075898, 2075899, 2075900, 2075902, 2075903, 2075904, 2075905,
2075940, 2075938, 2075939
]
print(recnum_list)
meta = media_metadata_search(keywords=[
'recnum', 'sunum', 'date__obs', 'quality', 'cdelt1', 'cdelt2',
'crval1'
],
series="hmi.sharp_cea_720s_nrt",
recnum_list=recnum_list,
server=sitools2_url)
print(meta)
self.assertEqual(len(meta), 10)
import datetime as dt
import sitools2.clients.sdo_client_medoc as md
server = 'http://medoc-sdo.ias.u-psud.fr'
ls = md.media_search(
server=server, series='aia.lev1', waves=['193'], cadence=['10m'],
dates=[dt.datetime(2018, 1, 1, 0, 0, 0), dt.datetime(2018, 1, 1, 1, 0, 0)])
kw = md.media_metadata_search(media_data_list=ls, keywords=['recnum', 'cdelt1', 'cdelt2'], server=server)
assert(len(ls) == len(kw))
print(kw[0])
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses
import datetime as dt
import sitools2.clients.sdo_client_medoc as md
server = 'http://medoc-sdo.ias.u-psud.fr'
ls = md.media_search(
server=server, series='aia.lev1', waves=['193'], cadence=['10m'],
dates=[dt.datetime(2018, 1, 1, 0, 0, 0), dt.datetime(2018, 1, 1, 1, 0, 0)])
kw = md.media_metadata_search(media_data_list=ls, keywords=['recnum', 'cdelt1', 'cdelt2'], server=server)
assert(len(ls) == len(kw))
print(kw[0])
server = 'http://idoc-medoc.ias.u-psud.fr'
ls = md.media_search(
server=server, series='hmi.m_720s', cadence=['12m'],
dates=[dt.datetime(2018, 1, 1, 0, 0, 0), dt.datetime(2018, 1, 1, 1, 0, 0)])
kw = md.media_metadata_search(media_data_list=ls, keywords=['recnum', 'cdelt1', 'cdelt2'], server=server)
assert(len(ls) == len(kw))
print(kw[0])
# Build Recnum list [optional]
recnum_list = []
for result in sdo_data_list:
print(result.sunum, result.recnum, result.date_obs, result.wave)
recnum_list.append(result.recnum)
# Test media_metada_search
# print "Exemple media_metadata_search()"
# my_meta_search = media_metadata_search(KEYWORDS=['date__obs','quality',
# 'cdelt1','cdelt2','crval1', 'sunum', 'recnum'], recnum_list=recnum_list,
# series='hmi.sharp_cea_720s_nrt')
my_meta_search = media_metadata_search(
KEYWORDS=[
'date__obs', 'quality', 'cdelt1', 'cdelt2', 'crval1', 'sunum', 'recnum'
],
MEDIA_DATA_LIST=sdo_data_list)
i = 0
for result in my_meta_search:
print("%s) %s " % (i + 1, result))
i += 1
# #print my_meta_search
# #To limit the results sent by the server set nb_res_max
# #sdo_data_list = search( DATES=[d1,d2], WAVES=['335','304'], nb_res_max= 5,
# #CADENCE=['1 min'] )
# #The fastest way to retrieve data
# #PS : The directory 'results' will be created if it does not exist
"""
from sitools2.clients.sdo_client_medoc import media_search, media_metadata_search
from datetime import datetime
from sitools2.clients import constants
sitools_url = constants.SITOOLS2_URL
d1 = datetime(2016, 1, 1, 0, 0, 0)
d2 = datetime(2016, 1, 2, 0, 0, 0)
sdo_hmi_data_list = media_search(DATES=[d1, d2], SERIES='hmi.sharp_cea_720s_nrt', CADENCE=['1h'], nb_res_max=10,
server=sitools_url)
print(sdo_hmi_data_list[0:3])
# Metadata info
meta = media_metadata_search(
KEYWORDS=['date__obs', 'quality', 'cdelt1', 'cdelt2', 'crval1'],
SERIES='hmi.sharp_cea_720s_nrt',
MEDIA_DATA_LIST=sdo_hmi_data_list,
server=sitools_url)
for data in meta:
print(data)
# Download data
for data in sdo_hmi_data_list:
data.get_file(target_dir='results', segment=["continuum"])
# data.get_file(target_dir='results')
nb_res_max=2)
# Build Recnum list [optional]
recnum_list = []
for result in sdo_data_list:
print(result.sunum, result.recnum, result.date_obs, result.wave)
recnum_list.append(result.recnum)
# Test media_metada_search
# print "Exemple media_metadata_search()"
# my_meta_search = media_metadata_search(KEYWORDS=['date__obs','quality',
# 'cdelt1','cdelt2','crval1', 'sunum', 'recnum'], recnum_list=recnum_list,
# series='hmi.sharp_cea_720s_nrt')
my_meta_search = media_metadata_search(KEYWORDS=[
'date__obs', 'quality', 'cdelt1', 'cdelt2', 'crval1', 'sunum', 'recnum'
],
MEDIA_DATA_LIST=sdo_data_list)
i = 0
for result in my_meta_search:
print("%s) %s " % (i + 1, result))
i += 1
# #print my_meta_search
# #To limit the results sent by the server set nb_res_max
# #sdo_data_list = search( DATES=[d1,d2], WAVES=['335','304'], nb_res_max= 5,
# #CADENCE=['1 min'] )
# #The fastest way to retrieve data
# #PS : The directory 'results' will be created if it does not exist
for data in sdo_data_list:
def query_aia_data(dates,
wl0,
nb_res_max=-1,
cadence='1 min',
increase_date_range=True,
keywords='default'):
''' Get a list of AIA data within given time limits.
Parameters
==========
dates : 2-tuple of datetime.datetime
Interval of dates within which to search for data.
wl0 : str
The AIA channel to use.
nb_res_max : int (default: -1)
Maximum number of results to return.
If set to -1, do not limit the number of results (?). This behaviour
depends on sitools2.clients.sdo_client_medoc.media_search().
cadence : str (default: '1 min')
Data cadence, passed to sitools2.clients.sdo_client_medoc.media_search.
increase_date_range : bool (default: True)
If True, increase interval of dates from (d1, d2) to
(d1 - cadence, d2 + cadence)
keywords : list of str, str, or None (default: 'default')
The metadata keywords to return for each AIA frame.
Either a list of str containing AIA metadata keywords (which can be
lowercase); None to skip the keyword query; or 'default' to query the
following keywords:
date__obs, exptime, int_time,
ctype1, cunit1, crpix1, crval1, cdelt1,
ctype2, cunit2, crpix2, crval2, cdelt2,
crota2, r_sun, x0_mp, y0_mp,
crln_obs, crlt_obs, car_rot.
Returns
=======
aia_frames : list
A list of md.Sdo_data objects that represent the returned AIA frames.
metadata : dict
A dict of lists, where each key is a metadata keyword, and each list
contains metadata retrieved from the Medoc database for each search
result.
'''
if increase_date_range:
reg = re.compile('(?:(?P<days>\d+)\s*(?:d|day|days))?\s*'
'(?:(?P<hours>\d+)\s*(?:h|hour|hours))?\s*'
'(?:(?P<minutes>\d+)\s*(?:m|min|minute|minutes))?\s*'
'(?:(?P<seconds>\d+)\s*(?:s|second|seconds))?\s*')
m = reg.match(cadence)
if m:
m = m.groupdict()
m = {k: float(v) if v else 0 for k, v in m.items()}
cadence_timedelta = datetime.timedelta(**m)
else:
raise ValueError('could not parse cadence')
d1, d2 = dates
dates = [d1 - cadence_timedelta, d2 + cadence_timedelta]
# Get a list of all AIA data
aia_frames = md.media_search(
dates=dates,
waves=[wl0],
cadence=[cadence],
nb_res_max=nb_res_max,
)
msg = 'Reached AIA maximum results number.'
assert (nb_res_max == -1) or (len(aia_frames) < nb_res_max), msg
# remove frames with exposure times that are too short or too long
exptime = np.array([sr.exptime for sr in aia_frames])
if np.std(exptime) > 0.1: # [s]
min_exptime = np.median(exptime) - np.std(exptime)
max_exptime = np.median(exptime) + np.std(exptime)
exptime_mask = (min_exptime < exptime) & (exptime < max_exptime)
aia_frames = list(compress(aia_frames, exptime_mask))
aia_frames = np.array(aia_frames)
if len(aia_frames) == 0:
msg = 'No AIA {} images found between {} and {}'
msg = msg.format(wl0, dates[0], dates[1])
raise ValueError(msg)
# retrieve metadata
if keywords is None:
metadata = {}
else:
if keywords is 'default':
keywords = [
'date__obs',
'exptime',
'int_time',
'ctype1',
'cunit1',
'crpix1',
'crval1',
'cdelt1',
'ctype2',
'cunit2',
'crpix2',
'crval2',
'cdelt2',
'crota2',
'r_sun',
'x0_mp',
'y0_mp',
'crln_obs',
'crlt_obs',
'car_rot',
]
# query metadata for each item in aia_frames, handling possible
# duplicates in this list
metadata = md.media_metadata_search(
keywords=keywords + ['recnum'],
media_data_list=aia_frames,
)
# index metadata with their recnum
metadata = {meta['recnum']: meta for meta in metadata}
metadata = [metadata[af.recnum] for af in aia_frames]
# reshape metadata
# (this drops the recnum field if it was not selected by the user)
metadata = {
kw: [meta_dict[kw] for meta_dict in metadata]
for kw in keywords
}
if 'date__obs' in keywords:
metadata['date__obs'] = [
sdotime_to_utc(t) for t in metadata['date__obs']
]
metadata = {k: np.array(v) for k, v in metadata.items()}
return aia_frames, metadata
def get_map(obs_date, wavelength, ias_location_prefix='/',
search_window=1, discard_bad_quality=False):
''' Get a map containing the closest AIA image found in Medoc for the given
date and wavelength.
Parameters
==========
obs_date : str
The date for which to search AIA data, in ISO format.
wavelength : int or str
The wavelength of the desired AIA channel.
ias_location_prefix : str (default: '/')
The local path from which to retrieve the data.
eg. to use '~/sshfs' to retrieve data from ~/sshfs/SUM02/foo
instead the default /SUM02/foo.
search_window : float (default: 1)
The window (in hours), within which to search for the data.
discard_bad_quality : bool (default: False)
If set to True, discard data for which quality is not 0.
Raises
======
- ValueError when no data matching the requested parameters were found.
'''
# AIA data: get closest AIA corresponding to EIS
# dates
d_eis = dateutil.parser.parse(obs_date)
d1 = d_eis - dt.timedelta(hours=search_window / 2)
d2 = d_eis + dt.timedelta(hours=search_window / 2)
# get data from Medoc
aia_res = md.media_search(
dates=[d1, d2],
waves=[str(wavelength)],
cadence=['1m'],
)
# discard images with bad quality
try:
if discard_bad_quality:
quality = md.media_metadata_search(
media_data_list=aia_res,
keywords=['quality'],
)
quality = np.array([q['quality'] for q in quality])
aia_res = list(np.array(aia_res)[quality == 0])
if len(aia_res) == 0:
raise ValueError
except ValueError:
no_data_msg = 'Could not find data matching the requested parameters.'
raise ValueError(no_data_msg)
# get closest image in date
aia_dates_obs = np.array([res.date_obs for res in aia_res])
aia_closest = np.argmin(np.abs(d_eis - aia_dates_obs))
aia_res = aia_res[aia_closest]
fits_location = get_fits(
aia_res,
ias_location_prefix=ias_location_prefix,
)
aia_map = sunpy.map.Map(fits_location)
aia_map = aia_map.rotate(rmatrix=aia_map.rotation_matrix)
return aia_map
data.get_file(target_dir='results', SEGMENT=['Br'])
sdo_data_list = media_search(DATES=[d1, d2],
WAVES=['335', '193'],
CADENCE=['12s'],
nb_res_max=2)
recnum_list = []
for result in sdo_data_list:
print(result.sunum, result.recnum, result.date_obs)
recnum_list.append(result.recnum)
# Test media_metada_search
print("Exemple media_metadata_search()")
my_meta_search = media_metadata_search(KEYWORDS=[
'date__obs', 'quality', 'cdelt1', 'cdelt2', 'crval1', 'sunum', 'recnum'
],
recnum_list=recnum_list,
series='aia.lev1')
i = 0
for result in my_meta_search:
print(i + 1, ")", result, "\n")
i += 1
# print my_meta_search
# To limit the results sent by the server set nb_res_max
# sdo_data_list = search( DATES=[d1,d2], WAVES=['335','304'], nb_res_max= 5 ,CADENCE=['1 min'] )
# The fastest way to retrieve data
# PS : The directory 'results' will be created if it does not exist
# Need to get a tar ball or zip file :
nb_res_max=10)
for data in sdo_hmi_data_list:
print(data)
data.get_file(target_dir='results', SEGMENT=['Br'])
sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '193'], CADENCE=['12s'], nb_res_max=2)
recnum_list = []
for result in sdo_data_list:
print(result.sunum, result.recnum, result.date_obs)
recnum_list.append(result.recnum)
# Test media_metada_search
print("Exemple media_metadata_search()")
my_meta_search = media_metadata_search(KEYWORDS=['date__obs', 'quality', 'cdelt1', 'cdelt2', 'crval1', 'sunum',
'recnum'], recnum_list=recnum_list, series='aia.lev1'
)
i = 0
for result in my_meta_search:
print(i+1, ")", result, "\n")
i += 1
# print my_meta_search
# To limit the results sent by the server set nb_res_max
# sdo_data_list = search( DATES=[d1,d2], WAVES=['335','304'], nb_res_max= 5 ,CADENCE=['1 min'] )
# The fastest way to retrieve data
# PS : The directory 'results' will be created if it does not exist
# Need to get a tar ball or zip file :
# A bit slower than the previous one