def test_media_search(self): try: # self.assertRaises(ValueError("Nothing to download"), sdo_client_medoc.media_get()) media_search() self.fail("No exception raised") except ValueError: pass except AssertionError: self.fail("exception")
def testSearchMedia(self): print ("####Test media_search #############################") d1 = datetime(2012,8,10,0,0,0) d2 = d1 + timedelta(days=1) sdo_data_list = media_search( DATES=[d1,d2], WAVES=['335','304'], CADENCE=['10 min'] ) self.assertEqual( len(sdo_data_list), 288)
def testSearchMedia(self): print("####Test media_search #############################") d1 = datetime(2012, 8, 10, 0, 0, 0) d2 = d1 + timedelta(days=1) sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '304'], CADENCE=['10 min']) self.assertEqual(len(sdo_data_list), 288)
def testSearch_sharp_cea_720s(self): print("####hmi.sharp_cea_720s #############################") sdo_data_list = media_search(server=sitools2_url, dates=[d1, d2], series='hmi.sharp_cea_720s', cadence=['12 min']) print(sdo_data_list[0:3]) self.assertEqual(len(sdo_data_list), 119)
def testSearch_m_720s_nrt(self): print("####hmi.m_720s_nrt #########################") sdo_data_list = media_search(server=sitools2_url, dates=[d1, d2], series='hmi.m_720s_nrt', cadence=['12 min']) print(sdo_data_list[0:3]) self.assertEqual(len(sdo_data_list), 120)
def testGetMedia(self): print("####Test media_get #############################") d1 = datetime(2012, 8, 10, 0, 0, 0) d2 = d1 + timedelta(days=1) sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '304'], CADENCE=['10 min'], nb_res_max=2) try: media_get(MEDIA_DATA_LIST=sdo_data_list, TARGET_DIR='results') except Exception: raise ValueError("Failed donwloading media data")
def testSearchMedia(self): print("####Test media_search #############################") d1 = datetime(2012, 8, 10, 0, 0, 0) d2 = d1 + timedelta(days=1) sdo_data_list = media_search(server=sitools2_url, dates=[d1, d2], waves=['335', '304'], cadence=['10 min']) self.assertEqual(len(sdo_data_list), 288)
def testUser1(self): print("####User Test media_search & metadata_search #############################") results_list = md.media_search(dates=[md.datetime(2016, 1, 1, 0, 0, 0), md.datetime(2016, 1, 1, 1, 0, 0)], waves=['193']) rnlist = [int(a.recnum) for a in results_list] print(results_list[0:3]) m = md.media_metadata_search(keywords=['date__obs', 'quality'], recnum_list=rnlist, series='aia.lev1') print(m[0:3]) self.assertEqual(len(m), 60)
def testSearchMedia(self): print ("####Test media_search #############################") d1 = datetime(2012,8,10,0,0,0) d2 = d1 + timedelta(days=1) sdo_data_list = media_search( server=sitools2_url, dates=[d1,d2], waves=['335','304'], cadence=['10 min'] ) self.assertEqual( len(sdo_data_list), 288)
def testMediaGetFile(self): print("####Test media_get #############################") d1 = datetime(2010, 5, 1, 0, 0, 0) d2 = d1 + timedelta(days=7*365) sdo_data_list = media_search(DATES=[d1, d2], series='hmi.m_720s', CADENCE=['1d'], nb_res_max=2) print(sdo_data_list[0]) try: sdo_data_list[0].get_file() except Exception: raise ValueError("Failed downloading get_file")
def testSearch_sharp_720s_nrt(self): print("#### hmi.sharp_720s_nrt ################################") sdo_data_list = media_search( server=sitools2_url, dates=[d1, d2], series='hmi.sharp_720s_nrt', cadence=['12 min']) print(sdo_data_list[0:3]) self.assertEqual(len(sdo_data_list), 861)
def testMediaGetFileName(self): print("####Test media_get_filename #############################") d1 = datetime(2010, 5, 1, 0, 0, 0) d2 = d1 + timedelta(days=7 * 365) sdo_data_list = media_search(DATES=[d1, d2], series='hmi.m_720s', CADENCE=['1d'], nb_res_max=2) print(sdo_data_list[0]) try: sdo_data_list[0].get_file(TARGET_DIR='results', filename='test_hmi.m_720s.fits') except Exception: raise ValueError("Failed downloading get_file")
def testMediaGetFileName(self): print ("####Test media_get_filename #############################") d1 = datetime(2010,5,1,0,0,0) d2 = d1 + timedelta(days=7*365) sdo_data_list = media_search( DATES=[d1,d2], series='hmi.m_720s', CADENCE=['1d'], nb_res_max=2 ) print(sdo_data_list[0]) try : result = sdo_data_list[0].get_file(TARGET_DIR='results' , filename='test_hmi.m_720s.fits') except : raise ValueError("Failed downloading get_file")
def testMediaItemMetadataSearch(self): print ("####Test metadata_search #############################") d1 = datetime(2010,1,1,0,0,0) d2 = d1 + timedelta(days=365) sdo_data_list = media_search( DATES=[d1,d2], waves=['304'], CADENCE=['1d'], nb_res_max=2 ) print(sdo_data_list[0]) try : result = sdo_data_list[0].metadata_search(keywords=['datamedn', 'exptime', 'quality']) print(result) except : raise ValueError("Failed metadata_search()")
def testGetMedia(self): print("####Test media_get #############################") d1 = datetime(2012, 8, 10, 0, 0, 0) d2 = d1 + timedelta(days=1) sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '304'], CADENCE=['10 min'], nb_res_max=2) try: media_get(MEDIA_DATA_LIST=sdo_data_list, TARGET_DIR='results') except Exception: raise ValueError("Failed donwloading media data")
def testMediaItemMetadataSearch(self): print("####Test metadata_search #############################") d1 = datetime(2010, 1, 1, 0, 0, 0) d2 = d1 + timedelta(days=365) sdo_data_list = media_search(DATES=[d1, d2], waves=['304'], CADENCE=['1d'], nb_res_max=2) print(sdo_data_list[0]) try: result = sdo_data_list[0].metadata_search( keywords=['datamedn', 'exptime', 'quality']) print(result) except Exception: raise ValueError("Failed metadata_search()")
def testUser1(self): print( "####User Test media_search & metadata_search #############################" ) results_list = md.media_search(dates=[ md.datetime(2016, 1, 1, 0, 0, 0), md.datetime(2016, 1, 1, 1, 0, 0) ], waves=['193']) rnlist = [int(a.recnum) for a in results_list] print(results_list[0:3]) m = md.media_metadata_search(keywords=['date__obs', 'quality'], recnum_list=rnlist, series='aia.lev1') print(m[0:3]) self.assertEqual(len(m), 60)
import datetime as dt import sitools2.clients.sdo_client_medoc as md server = 'http://medoc-sdo.ias.u-psud.fr' ls = md.media_search( server=server, series='aia.lev1', waves=['193'], cadence=['10m'], dates=[dt.datetime(2018, 1, 1, 0, 0, 0), dt.datetime(2018, 1, 1, 1, 0, 0)]) kw = md.media_metadata_search(media_data_list=ls, keywords=['recnum', 'cdelt1', 'cdelt2'], server=server) assert(len(ls) == len(kw)) print(kw[0])
#! /usr/bin/env python """ Test Eric. B """ import sitools2.clients.sdo_client_medoc as md d1 = md.datetime(2010, 5, 1, 0, 0, 0) d2 = d1 + md.timedelta(days=7 * 365) test_eb = md.media_search(dates=[d1, d2], series='hmi.m_720s', cadence=['1d'], nb_res_max=2) test_eb[0].get_file()
def query_aia_data(dates, wl0, nb_res_max=-1, cadence='1 min', increase_date_range=True, keywords='default'): ''' Get a list of AIA data within given time limits. Parameters ========== dates : 2-tuple of datetime.datetime Interval of dates within which to search for data. wl0 : str The AIA channel to use. nb_res_max : int (default: -1) Maximum number of results to return. If set to -1, do not limit the number of results (?). This behaviour depends on sitools2.clients.sdo_client_medoc.media_search(). cadence : str (default: '1 min') Data cadence, passed to sitools2.clients.sdo_client_medoc.media_search. increase_date_range : bool (default: True) If True, increase interval of dates from (d1, d2) to (d1 - cadence, d2 + cadence) keywords : list of str, str, or None (default: 'default') The metadata keywords to return for each AIA frame. Either a list of str containing AIA metadata keywords (which can be lowercase); None to skip the keyword query; or 'default' to query the following keywords: date__obs, exptime, int_time, ctype1, cunit1, crpix1, crval1, cdelt1, ctype2, cunit2, crpix2, crval2, cdelt2, crota2, r_sun, x0_mp, y0_mp, crln_obs, crlt_obs, car_rot. Returns ======= aia_frames : list A list of md.Sdo_data objects that represent the returned AIA frames. metadata : dict A dict of lists, where each key is a metadata keyword, and each list contains metadata retrieved from the Medoc database for each search result. ''' if increase_date_range: reg = re.compile('(?:(?P<days>\d+)\s*(?:d|day|days))?\s*' '(?:(?P<hours>\d+)\s*(?:h|hour|hours))?\s*' '(?:(?P<minutes>\d+)\s*(?:m|min|minute|minutes))?\s*' '(?:(?P<seconds>\d+)\s*(?:s|second|seconds))?\s*') m = reg.match(cadence) if m: m = m.groupdict() m = {k: float(v) if v else 0 for k, v in m.items()} cadence_timedelta = datetime.timedelta(**m) else: raise ValueError('could not parse cadence') d1, d2 = dates dates = [d1 - cadence_timedelta, d2 + cadence_timedelta] # Get a list of all AIA data aia_frames = md.media_search( dates=dates, waves=[wl0], cadence=[cadence], nb_res_max=nb_res_max, ) msg = 'Reached AIA maximum results number.' assert (nb_res_max == -1) or (len(aia_frames) < nb_res_max), msg # remove frames with exposure times that are too short or too long exptime = np.array([sr.exptime for sr in aia_frames]) if np.std(exptime) > 0.1: # [s] min_exptime = np.median(exptime) - np.std(exptime) max_exptime = np.median(exptime) + np.std(exptime) exptime_mask = (min_exptime < exptime) & (exptime < max_exptime) aia_frames = list(compress(aia_frames, exptime_mask)) aia_frames = np.array(aia_frames) if len(aia_frames) == 0: msg = 'No AIA {} images found between {} and {}' msg = msg.format(wl0, dates[0], dates[1]) raise ValueError(msg) # retrieve metadata if keywords is None: metadata = {} else: if keywords is 'default': keywords = [ 'date__obs', 'exptime', 'int_time', 'ctype1', 'cunit1', 'crpix1', 'crval1', 'cdelt1', 'ctype2', 'cunit2', 'crpix2', 'crval2', 'cdelt2', 'crota2', 'r_sun', 'x0_mp', 'y0_mp', 'crln_obs', 'crlt_obs', 'car_rot', ] # query metadata for each item in aia_frames, handling possible # duplicates in this list metadata = md.media_metadata_search( keywords=keywords + ['recnum'], media_data_list=aia_frames, ) # index metadata with their recnum metadata = {meta['recnum']: meta for meta in metadata} metadata = [metadata[af.recnum] for af in aia_frames] # reshape metadata # (this drops the recnum field if it was not selected by the user) metadata = { kw: [meta_dict[kw] for meta_dict in metadata] for kw in keywords } if 'date__obs' in keywords: metadata['date__obs'] = [ sdotime_to_utc(t) for t in metadata['date__obs'] ] metadata = {k: np.array(v) for k, v in metadata.items()} return aia_frames, metadata
# This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses import datetime as dt import sitools2.clients.sdo_client_medoc as md server = 'http://medoc-sdo.ias.u-psud.fr' ls = md.media_search( server=server, series='aia.lev1', waves=['193'], cadence=['10m'], dates=[dt.datetime(2018, 1, 1, 0, 0, 0), dt.datetime(2018, 1, 1, 1, 0, 0)]) kw = md.media_metadata_search(media_data_list=ls, keywords=['recnum', 'cdelt1', 'cdelt2'], server=server) assert(len(ls) == len(kw)) print(kw[0]) server = 'http://idoc-medoc.ias.u-psud.fr' ls = md.media_search( server=server, series='hmi.m_720s', cadence=['12m'], dates=[dt.datetime(2018, 1, 1, 0, 0, 0), dt.datetime(2018, 1, 1, 1, 0, 0)]) kw = md.media_metadata_search(media_data_list=ls, keywords=['recnum', 'cdelt1', 'cdelt2'], server=server) assert(len(ls) == len(kw)) print(kw[0])
""" __author__ = "Pablo ALINGERY" from datetime import datetime from sitools2.clients.sdo_client_medoc import media_search, media_metadata_search d1 = datetime(2016, 1, 1, 0, 0, 0) d2 = datetime(2016, 1, 1, 5, 12, 0) # d2 = d1 + timedelta(minutes=5) # sdo_hmi_data_list=media_search(DATES=[d1,d2],series='hmi.sharp_cea_720s_nrt', # cadence=['1h'],nb_res_max=10) sdo_data_list = media_search( DATES=[d1, d2], WAVES=['335', '193'], CADENCE=['1m'], nb_res_max=2) # Build Recnum list [optional] recnum_list = [] for result in sdo_data_list: print(result.sunum, result.recnum, result.date_obs, result.wave) recnum_list.append(result.recnum) # Test media_metada_search # print "Exemple media_metadata_search()" # my_meta_search = media_metadata_search(KEYWORDS=['date__obs','quality', # 'cdelt1','cdelt2','crval1', 'sunum', 'recnum'], recnum_list=recnum_list, # series='hmi.sharp_cea_720s_nrt') my_meta_search = media_metadata_search( KEYWORDS=[
#! /usr/bin/env python """ Test Eric. B """ import sitools2.clients.sdo_client_medoc as md d1 = md.datetime(2010, 5, 1, 0, 0, 0) d2 = d1 + md.timedelta(days=7*365) test_eb = md.media_search(dates=[d1, d2], series='hmi.m_720s', cadence=['1d'], nb_res_max=2) test_eb[0].get_file()
#! /usr/bin/env python # SITools2 client for Python # Copyright (C) 2013 - Institut d'astrophysique spatiale # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses import datetime as dt import sitools2.clients.sdo_client_medoc as md server = 'http://idoc-medoc.ias.u-psud.fr' ls = md.media_search(server=server, series='hmi.m_720s', cadence=['12m'], dates=[dt.datetime(2018, 1, 1, 0, 0, 0), dt.datetime(2018, 1, 1, 1, 0, 0)]) kw = md.media_metadata_search(media_data_list=ls, keywords=['recnum', 'cdelt1', 'cdelt2'], server=server) assert(len(ls) == len(kw)) print(kw[0])
def get_map(obs_date, wavelength, ias_location_prefix='/', search_window=1, discard_bad_quality=False): ''' Get a map containing the closest AIA image found in Medoc for the given date and wavelength. Parameters ========== obs_date : str The date for which to search AIA data, in ISO format. wavelength : int or str The wavelength of the desired AIA channel. ias_location_prefix : str (default: '/') The local path from which to retrieve the data. eg. to use '~/sshfs' to retrieve data from ~/sshfs/SUM02/foo instead the default /SUM02/foo. search_window : float (default: 1) The window (in hours), within which to search for the data. discard_bad_quality : bool (default: False) If set to True, discard data for which quality is not 0. Raises ====== - ValueError when no data matching the requested parameters were found. ''' # AIA data: get closest AIA corresponding to EIS # dates d_eis = dateutil.parser.parse(obs_date) d1 = d_eis - dt.timedelta(hours=search_window / 2) d2 = d_eis + dt.timedelta(hours=search_window / 2) # get data from Medoc aia_res = md.media_search( dates=[d1, d2], waves=[str(wavelength)], cadence=['1m'], ) # discard images with bad quality try: if discard_bad_quality: quality = md.media_metadata_search( media_data_list=aia_res, keywords=['quality'], ) quality = np.array([q['quality'] for q in quality]) aia_res = list(np.array(aia_res)[quality == 0]) if len(aia_res) == 0: raise ValueError except ValueError: no_data_msg = 'Could not find data matching the requested parameters.' raise ValueError(no_data_msg) # get closest image in date aia_dates_obs = np.array([res.date_obs for res in aia_res]) aia_closest = np.argmin(np.abs(d_eis - aia_dates_obs)) aia_res = aia_res[aia_closest] fits_location = get_fits( aia_res, ias_location_prefix=ias_location_prefix, ) aia_map = sunpy.map.Map(fits_location) aia_map = aia_map.rotate(rmatrix=aia_map.rotation_matrix) return aia_map
""" __author__ = "Pablo ALINGERY" from datetime import datetime from sitools2.clients.sdo_client_medoc import media_search, media_metadata_search d1 = datetime(2016, 1, 1, 0, 0, 0) d2 = datetime(2016, 1, 1, 5, 12, 0) # d2 = d1 + timedelta(minutes=5) # sdo_hmi_data_list=media_search(DATES=[d1,d2],series='hmi.sharp_cea_720s_nrt', # cadence=['1h'],nb_res_max=10) sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '193'], CADENCE=['1m'], nb_res_max=2) # Build Recnum list [optional] recnum_list = [] for result in sdo_data_list: print(result.sunum, result.recnum, result.date_obs, result.wave) recnum_list.append(result.recnum) # Test media_metada_search # print "Exemple media_metadata_search()" # my_meta_search = media_metadata_search(KEYWORDS=['date__obs','quality', # 'cdelt1','cdelt2','crval1', 'sunum', 'recnum'], recnum_list=recnum_list, # series='hmi.sharp_cea_720s_nrt') my_meta_search = media_metadata_search(KEYWORDS=[
#! /usr/bin/env python """ """ from sitools2.clients.sdo_client_medoc import media_search, media_metadata_search, media_get from datetime import datetime d1 = datetime(2016, 6, 1, 5, 0, 0) d2 = datetime(2016, 6, 1, 10, 12, 0) sdo_data_list1 = media_search(DATES=[d1, d2], WAVES=['335'], CADENCE=['12s'], nb_res_max=10, server='http://medoc-sdo.ias.u-psud.fr') sdo_data_list2 = media_search(DATES=[d1, d2], WAVES=['335', '304'], CADENCE=['1m'], nb_res_max=10, server='http://idoc-medoc-test.ias.u-psud.fr') sdo_data_list3 = media_search(DATES=[d1, d2], WAVES=['94']) sdo_data_list4 = media_search(DATES=[d1, d2], WAVES=['335'], CADENCE=['12s'], nb_res_max=10) sdo_data_list5 = media_search(DATES=[d1, d2], WAVES=['171', '193'],
#! /usr/bin/env python """ Test Gabriel .P """ from sitools2.clients.sdo_client_medoc import media_search from datetime import datetime, timedelta d1 = datetime(2016, 6, 1, 5, 0, 0) d2 = d1 + timedelta(minutes=5) sdo_data_list = media_search( DATES=[d1, d2], WAVES=['335', '304'], CADENCE=['1m'], nb_res_max=10, server='http://idoc-medoc-test.ias.u-psud.fr') # And if you want to specifies files name do sthg like for item in sdo_data_list: print(item.date_obs, item.wave, item.recnum, item.sunum, item.ias_location) file_date_obs = item.date_obs.strftime('%Y-%m-%dT%H-%M-%S') file_wave = item.wave item.get_file( DECOMPRESS=False, FILENAME="toto_%s_%s.fits" % (file_date_obs, file_wave), TARGET_DIR='results' )
#! /usr/bin/env python """ Test Gabriel .P """ from sitools2.clients.sdo_client_medoc import media_search from datetime import datetime, timedelta d1 = datetime(2016, 6, 1, 5, 0, 0) d2 = d1 + timedelta(minutes=5) sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '304'], CADENCE=['1m'], nb_res_max=10, server='http://idoc-medoc-test.ias.u-psud.fr') # And if you want to specifies files name do sthg like for item in sdo_data_list: print(item.date_obs, item.wave, item.recnum, item.sunum, item.ias_location) file_date_obs = item.date_obs.strftime('%Y-%m-%dT%H-%M-%S') file_wave = item.wave item.get_file(DECOMPRESS=False, FILENAME="toto_%s_%s.fits" % (file_date_obs, file_wave), TARGET_DIR='results')
#! /usr/bin/env python """ @author: Pablo ALINGERY """ from sitools2.clients.sdo_client_medoc import media_search, media_metadata_search from datetime import datetime from sitools2.clients import constants sitools_url = constants.SITOOLS2_URL d1 = datetime(2016, 1, 1, 0, 0, 0) d2 = datetime(2016, 1, 2, 0, 0, 0) sdo_hmi_data_list = media_search(DATES=[d1, d2], SERIES='hmi.sharp_cea_720s_nrt', CADENCE=['1h'], nb_res_max=10, server=sitools_url) print(sdo_hmi_data_list[0:3]) # Metadata info meta = media_metadata_search( KEYWORDS=['date__obs', 'quality', 'cdelt1', 'cdelt2', 'crval1'], SERIES='hmi.sharp_cea_720s_nrt', MEDIA_DATA_LIST=sdo_hmi_data_list, server=sitools_url) for data in meta: print(data) # Download data for data in sdo_hmi_data_list:
""" Use sitools2.clients.sdo_client_medoc media_search, media_metadata_search """ __author__ = "Pablo ALINGERY" from sitools2.clients.sdo_client_medoc import media_search, media_metadata_search from datetime import datetime d1 = datetime(2016, 1, 1, 0, 0, 0) d2 = datetime(2016, 1, 1, 5, 12, 0) # d2 = d1 + timedelta(minutes=5) sdo_hmi_data_list = media_search(DATES=[d1, d2], series='hmi.sharp_cea_720s_nrt', cadence=['1h'], nb_res_max=10) for data in sdo_hmi_data_list: print(data) data.get_file(target_dir='results', SEGMENT=['Br']) sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '193'], CADENCE=['12s'], nb_res_max=2) recnum_list = [] for result in sdo_data_list: print(result.sunum, result.recnum, result.date_obs) recnum_list.append(result.recnum) # Test media_metada_search
Use sitools2.clients.sdo_client_medoc media_search, media_metadata_search """ __author__ = "Pablo ALINGERY" from sitools2.clients.sdo_client_medoc import media_search, media_metadata_search from datetime import datetime d1 = datetime(2016, 1, 1, 0, 0, 0) d2 = datetime(2016, 1, 1, 5, 12, 0) # d2 = d1 + timedelta(minutes=5) sdo_hmi_data_list = media_search( DATES=[d1, d2], series='hmi.sharp_cea_720s_nrt', cadence=['1h'], nb_res_max=10) for data in sdo_hmi_data_list: print(data) data.get_file(target_dir='results', SEGMENT=['Br']) sdo_data_list = media_search(DATES=[d1, d2], WAVES=['335', '193'], CADENCE=['12s'], nb_res_max=2) recnum_list = [] for result in sdo_data_list: print(result.sunum, result.recnum, result.date_obs) recnum_list.append(result.recnum) # Test media_metada_search print("Exemple media_metadata_search()")