def test_add_entry_from_hek_qr(database): hek_res = hek.HEKClient().search( net_attrs.Time('2020/08/09 07:23:56', '2020/08/09 08:23:56'), hek.attrs.EventType('FL')) assert len(database) == 0 database.add_from_hek_query_result(hek_res) assert len(database) == 90
def downloadHEKdata(fn='hekdata.pkl'): """Download flare data for all flares of magnitude greater than C1, using SunPy HEK client. Save output to FN as a pickled Pandas DataFrame. """ FIRST_DATE = datetime(2010, 5, 1) # First date for which data is available # Download 90 days of data at a time delta = timedelta(days=90) N = (datetime.now() - FIRST_DATE) // delta + 1 dates = [FIRST_DATE + x*delta for x in range(N)] params = ( hek.attrs.FL, hek.attrs.FL.GOESCls >= 'C1', hek.attrs.OBS.Instrument.like('goes') ) client = hek.HEKClient() flares = pd.DataFrame() start = datetime.now() for i, date in enumerate(dates, 1): r = client.query(hek.attrs.Time(date, date + delta), *params) flares = flares.append(pd.DataFrame.from_dict(r), ignore_index=True) printTimeInfo(start, i, N) with open(fn, 'wb') as f: pickle.dump(flares, f)
def test_add_entry_from_hek_qr(database): hek_res = hek.HEKClient().query( hek.attrs.Time('2011/08/09 07:23:56', '2011/08/09 07:24:00'), hek.attrs.EventType('FL')) assert len(database) == 0 database.add_from_hek_query_result(hek_res) assert len(database) == 1678
def test_mixed_results_get(): # To check that the following bug is fixed: # https://github.com/sunpy/sunpy/issues/3238 client = hek.HEKClient() client.search( hek.attrs.Time(parse_time('2013/02/01 00:00:00'), parse_time('2013/02/01 23:30:00')), hek.attrs.FRM.Name == 'SPoCA')
def test_add_entry_from_hek_qr(database): hek_res = hek.HEKClient().query( hek.attrs.Time('2011/08/09 07:23:56', '2011/08/09 07:24:00'), hek.attrs.EventType('FL')) assert len(database) == 0 database.add_from_hek_query_result(hek_res) # The database apparently has 1902 entries now, not 2133 assert len(database) == 1902
def get_hybrid_data(skip_one_path=False): """Gets SDO multichannel and flare size data Combines HMI magnetograms and AIA 171 images Had trouble with images produced with the scipy imsave routine Works ok with plt.imsave If it fails with a UnicodeDecodeError, just rerun the routine with skip_one_path set to true so that it skips that date the next time """ client = hek.HEKClient() cmap = cm.get_cmap('sdoaia171') path = '/sanhome/cheung/public_html/AIA/synoptic_ML/' if os.path.exists('remaining_paths.txt'): print('picking up from before') with open('remaining_paths.txt', 'r') as f: pics = f.read().split('\n') else: print('starting from beginning') pics = [f for f in os.listdir(path) if f[:4] == 'sdo_'] print('{} images found'.format(len(pics))) if skip_one_path: print(pics[0]) pics = pics[1:] for i in range(len(pics)): image = np.memmap(os.path.join(path, pics[i]), dtype=np.uint8, mode='r', shape=(128, 128, 8)) tstart = parse_time(pics[i][17:-4]) tend = tstart + timedelta(days=1) try: result = client.query(hek.attrs.Time(tstart, tend), hek.attrs.EventType('FL')) except UnicodeDecodeError: print('UnicodeDecodeError') with open('remaining_paths.txt', 'w') as f: f.write('\n'.join(pics[i:])) import sys sys.exit() cls = get_max_flare(result) img = cmap(image[:, :, 2]) # apply 171 cmap to 171 channel img2 = np.zeros((128, 128, 3)) # create array for hmi magnetogram img2[:, :, 2] = image[:, :, 7] # stuff magnetogram into blue channel img3 = img[:, :, : -1] + img2 # add 171 and magnetogram together and remove alpha save_path = '/Users/pauly/hybrid_data' plt.imsave('{}/{}/{}_orig.jpg'.format(save_path, cls, tstart), np.flipud(img3)) plt.imsave('{}/{}/{}_ud.jpg'.format(save_path, cls, tstart), img3) plt.imsave('{}/{}/{}_lr.jpg'.format(save_path, cls, tstart), np.flipud(np.fliplr(img3))) plt.imsave('{}/{}/{}_rot180.jpg'.format(save_path, cls, tstart), np.fliplr(img3)) print('\r{}%'.format(int(100 * i / len(pics))), end='')
def synoptic_glob(): day = datetime(2010, 5, 13) # series start date series_approx_end = datetime.utcnow() - timedelta( days=24) # series lags by ~24d image_cache = np.zeros( (512, 512)) # placeholder to add 1600 images to for sum norm = mov_img.colors.LogNorm(1) # log color norm for 1600 client = hek.HEKClient() # client for querying flare data while day < series_approx_end: print(day) # fetch directory location for the desired day's data r = fetch.fetch('aia.fits_web', useJSOC2=True, start=day, segments='Images') day_dir = r[0] for hour in range(24): hour_dir = os.path.join(day_dir, 'H{}/'.format(str(hour).zfill(4))) # step through data at 12 minute cadence for minute in range(0, 60, 12): fname = 'AIA{}_{}_1600.fits'.format( day.strftime('%Y%m%d'), str(hour * 100 + minute).zfill(4)) image_path = os.path.join(hour_dir, fname) try: amap = mov_img.Map(image_path) except ValueError: print(image_path) raise ValueError('File not found ^^^') data = mov_img.downscale_local_mean(amap.data, (2, 2)) image_cache += data if hour % 6 == 0: # produce image t = day + timedelta(hours=hour) ch3 = np.flipud(norm(image_cache)) ch2 = np.flipud(norm(data)) image_cache = np.zeros((512, 512)) r = fetch.fetch( 'hmi.M_720s', start=t, end_or_span=timedelta(minutes=12), segments='magnetogram', keys=['rsun_obs', 'cdelt1', 'quality', 'date__obs']) if not no_images(r): ch1 = mov_img.process_hmi(r[0][-1], float(r[0][0]), float(r[0][1]), downscale=(8, 8), single_channel=True) result = client.query( hek.attrs.Time(t, t + timedelta(days=1)), hek.attrs.EventType('FL')) cls = get_max_flare(result) out_path = '/Users/pauly/repos/aia/data/glob/{}/{}'.format( cls, r[0][-2]) img = mov_img.misc.toimage(np.array([ch1, ch2, ch3])) img.save('{}.jpg'.format(out_path)) day += timedelta(days=1)
def get_goes_event_list(timerange, goes_class_filter=None): """ Retrieve list of flares detected by GOES within a given time range. Parameters ---------- timerange: sunpy.time.TimeRange The time range to download the event list for. goes_class_filter: (optional) string A string specifying a minimum GOES class for inclusion in the list, e.g. 'M1', 'X2'. """ # use HEK module to search for GOES events client = hek.HEKClient() event_type = 'FL' tstart = timerange.start() tend = timerange.end() # query the HEK for a list of events detected by the GOES instrument # between tstart and tend (using a GOES-class filter) if goes_class_filter: result = client.query(hek.attrs.Time(tstart, tend), hek.attrs.EventType(event_type), hek.attrs.FL.GOESCls > goes_class_filter, hek.attrs.OBS.Observatory == 'GOES') else: result = client.query(hek.attrs.Time(tstart, tend), hek.attrs.EventType(event_type), hek.attrs.OBS.Observatory == 'GOES') # want to condense the results of the query into a more manageable # dictionary # keep event data, start time, peak time, end time, GOES-class, # location, active region source (as per GOES list standard) # make this into a list of dictionaries goes_event_list = [] for r in result: goes_event = { 'event_date': parse_time(r['event_starttime']).date().strftime('%Y-%m-%d'), 'start_time': parse_time(r['event_starttime']), 'peak_time': parse_time(r['event_peaktime']), 'end_time': parse_time(r['event_endtime']), 'goes_class': str(r['fl_goescls']), 'goes_location': (r['event_coord1'], r['event_coord2']), 'noaa_active_region': r['ar_noaanum'] } goes_event_list.append(goes_event) return goes_event_list
def test_mixed_results_get(): # To check that the following bug is fixed: # https://github.com/sunpy/sunpy/issues/3238 client = hek.HEKClient() result = client.search(attrs.Time('2013/02/01 00:00:00', '2013/02/01 23:30:00'), attrs.hek.FRM.Name == 'SPoCA') assert isinstance(result, hek.hek.HEKTable) assert len(result) == 89 assert result[0]["SOL_standard"] == 'SOL2013-01-31T20:13:31L219C160'
def test_add_entry_from_hek_qr(database): hek_res = hek.HEKClient().query( hek.attrs.Time('2011/08/09 07:23:56', '2011/08/09 07:24:00'), hek.attrs.EventType('FL')) assert len(database) == 0 database.add_from_hek_query_result(hek_res) # This number loves to change, so we are just going to test that it's added # *something* assert len(database) > 1
def hek_client(): startTime = '2011/08/09 07:23:56' endTime = '2011/08/09 12:40:29' eventType = 'FL' hekTime = attrs.Time(startTime, endTime) hekEvent = attrs.hek.EventType(eventType) h = hek.HEKClient() hek_query = h.search(hekTime, hekEvent) return hek_query
def test_mixed_results_get_2(): # To check that the following bug is fixed: # # https://github.com/sunpy/sunpy/issues/3898 client = hek.HEKClient() result = client.search(attrs.Time('2011/08/09 07:23:56', '2011/08/09 12:40:29'), attrs.hek.EventType("FL")) assert isinstance(result, hek.hek.HEKTable) assert len(result) == 19 assert result[0]["SOL_standard"] == 'SOL2011-08-08T01:30:04L247C075'
def test_mixed_results_get_angstrom(): # To check that the following bug is fixed: # https://github.com/sunpy/sunpy/issues/4087 client = hek.HEKClient() tstart = '2014/10/24 20:50' tend = '2014/10/25 00:14' event_type = 'FL' result = client.search(attrs.Time(tstart, tend), attrs.hek.EventType(event_type)) assert len(result) == 13 assert result[0]["SOL_standard"] == 'SOL2014-10-24T20:53:46L247C106'
def test_query_multiple_operators(): event_type = "FL" tstart = "2013/10/28" tend = "2013/10/29" client = hek.HEKClient() results = client.search(attrs.Time(tstart, tend), attrs.hek.EventType(event_type), attrs.hek.FL.GOESCls > "M1.0", attrs.hek.OBS.Observatory == "GOES") assert len(results) == 7
def test_translate_results_to_query(): """Make sure that conversion of HEK results to VSO queries is accurate""" h = hek.HEKClient() hek_query = h.query(hekTime, hekEvent) vso_query = hek2vso.translate_results_to_query(hek_query) if isinstance(hek_query, list): # Comparing length of two lists assert len(hek_query) == len(vso_query) #Comparing types of both queries assert type(hek_query) == type(vso_query)
def test_hek_client(): startTime = '2011/08/09 07:23:56' endTime = '2011/08/09 12:40:29' eventType = 'FL' hekTime = hek.attrs.Time(startTime, endTime) hekEvent = hek.attrs.EventType(eventType) h = hek.HEKClient() hek_query = h.search(hekTime, hekEvent) assert type(hek_query) == hek.hek.HEKTable
def make_hmi(start=datetime(2010, 5, 1), r=[]): """Fetch data, then filter based on quality """ if len(r) == 0: end = datetime.utcnow() delta = timedelta(days=30) while end - start > delta: r.extend( fetch.fetch( 'hmi.M_720s', start=start, end_or_span=delta, segments='magnetogram', cadence=timedelta(hours=6), keys=['rsun_obs', 'cdelt1', 'quality', 'date__obs'])) start += delta r.extend( fetch.fetch('hmi.M_720s', start=start, end_or_span=datetime.utcnow(), segments='magnetogram', cadence=timedelta(hours=6), keys=['rsun_obs', 'cdelt1', 'quality', 'date__obs'])) print('original query length:', len(r)) r = [sub for sub in r if sub[2] == '0x00000000'] print('filtered query length:', len(r)) client = hek.HEKClient() for i, sub in enumerate(r): try: path = sub[-1] hdr = getFitsHdr(path) tstart = parse_time(sub[3]) tend = tstart + timedelta(days=1) result = client.query(hek.attrs.Time(tstart, tend), hek.attrs.EventType('FL')) cls = get_max_flare(result) out_path = '/Users/pauly/repos/aia/data/hmi/{}/{}'.format( cls, tstart.strftime('%Y-%m-%dT%H:%M:%S')) img = mov_img.process_hmi(path, float(sub[0]), float(sub[1]), downscale=(8, 8)) # save image and its symmetries img.save('{}_1.jpg'.format(out_path)) ImageOps.flip(img).save('{}_2.jpg'.format(out_path)) ImageOps.mirror(img).save('{}_3.jpg'.format(out_path)) ImageOps.flip(ImageOps.mirror(img)).save( '{}_4.jpg'.format(out_path)) print('\r{}%'.format(int(100 * i / len(r))), end='') except: continue
def test_translate_results_to_query(): """Make sure that conversion of HEK results to VSO queries is accurate""" h = hek.HEKClient() hek_query = h.search(hekTime, hekEvent) vso_query = hek2vso.translate_results_to_query(hek_query) # Comparing length of two lists assert len(hek_query) == len(vso_query) # Comparing types of both queries # Not sure this test makes any sense now assert isinstance(hek_query, table.Table) assert isinstance(vso_query, list)
def test_hek_empty_search_result(): startTime = '1985-05-04 00:00:00' endTime = '1985-05-04 00:00:00' eventType = 'FL' hekTime = hek.attrs.Time(startTime, endTime) hekEvent = hek.attrs.EventType(eventType) h = hek.HEKClient() hek_query = h.search(hekTime, hekEvent) assert type(hek_query) == hek.hek.HEKTable assert len(hek_query) == 0
def get_result(self): if self._result is None: startTime = '2011/08/09 07:23:56' endTime = '2011/08/09 12:40:29' eventType = 'FL' hekTime = attrs.Time(startTime, endTime) hekEvent = attrs.hek.EventType(eventType) h = hek.HEKClient() hek_query = h.search(hekTime, hekEvent) self._result = hek_query return copy.deepcopy(self._result)
def test_hek_client(): startTime = '2011/08/09 07:23:56' endTime = '2011/08/09 12:40:29' eventType = 'FL' hekTime = hek.attrs.Time(startTime, endTime) hekEvent = hek.attrs.EventType(eventType) h = hek.HEKClient() hek_query = h.search(hekTime, hekEvent) assert hek_query[0]['event_peaktime'] == hek_query[0].get('event_peaktime') assert hek_query[0].get('') == None
def get_ar_flares(ar_number, verbose=True): """ Get flares list for active region. """ hek_client = hek.HEKClient() result_fl = hek_client.search( hek.attrs.Time(start_time_hmi, datetime.now()), hek.attrs.EventType('FL'), hek.attrs.AR.NOAANum == ar_number) result = [e for e in result_fl if e['search_instrument'] == 'GOES'] if verbose: for e in result: print(e['fl_goescls'], ' at ', e['event_peaktime']) return result
def get_observations_from_hek(self, obsdate: str, event_type: str = 'AR', observatory: str = 'SOHO'): """ Gets the observation metadata from HEK for the given obsdate. By default gets Active Region data recieved from SOHO. Parameters ---------- obsdate : str The observation time and date. event_type : str, optional The type of Event, by default 'AR' observatory : str, optional Observatory that observed the Event, by default 'SOHO' Returns ------- results = sunpy.hek.HEKTable The table of results recieved from HEK. Examples -------- >>> from pythia.seo import Sunspotter >>> sunspotter = Sunspotter() >>> obsdate = '2000-01-01 12:47:02' >>> sunspotter.get_observations_from_hek(obsdate) <HEKTable length=5> SOL_standard absnetcurrenthelicity ... unsignedvertcurrent str30 object ... object ------------------------------ --------------------- ... ------------------- SOL2000-01-01T09:35:02L054C117 None ... None SOL2000-01-01T09:35:02L058C100 None ... None SOL2000-01-01T09:35:02L333C106 None ... None SOL2000-01-01T09:35:02L033C066 None ... None SOL2000-01-01T09:35:02L012C054 None ... None """ obsdate = self.get_nearest_observation(obsdate) client = hek.HEKClient() result = client.search(hek.attrs.Time(obsdate, obsdate), hek.attrs.EventType(event_type)) obsdate = "T".join(str(obsdate).split()) result = result[result['obs_observatory'] == 'SOHO'] result = result[result['event_starttime'] <= obsdate] result = result[result['event_endtime'] > obsdate] return result
def make_aia(start=datetime(2010, 5, 1), r=[]): """Fetch data, then filter based on quality """ if len(r) == 0: end = datetime.utcnow() delta = timedelta(days=30) while end - start > delta: r.extend( fetch.fetch('aia.lev1_euv_12s', start=start, end_or_span=delta, wavelengths=171, segments='image', cadence=timedelta(hours=6))) start += delta r.extend( fetch.fetch('aia.lev1_euv_12s', start=start, end_or_span=datetime.utcnow(), wavelengths=171, segments='image', cadence=timedelta(hours=6))) print('original query length:', len(r)) r = [path for path in r if getFitsHdr(path)['quality'] == 0] print('filtered query length:', len(r)) client = hek.HEKClient() for i, path in enumerate(r): try: hdr = getFitsHdr(path) tstart = parse_time(hdr['date-obs']) tend = tstart + timedelta(days=1) result = client.query(hek.attrs.Time(tstart, tend), hek.attrs.EventType('FL')) cls = get_max_flare(result) out_path = '/Users/pauly/repos/aia/data/aia/{}/{}'.format( cls, tstart.strftime('%Y-%m-%dT%H:%M:%S')) img = mov_img.process_img(path, downscale=(8, 8)) # save image and its symmetries img.save('{}_1.jpg'.format(out_path)) ImageOps.flip(img).save('{}_2.jpg'.format(out_path)) ImageOps.mirror(img).save('{}_3.jpg'.format(out_path)) ImageOps.flip(ImageOps.mirror(img)).save( '{}_4.jpg'.format(out_path)) print('\r{}%'.format(int(100 * i / len(r))), end='') except: continue
def test_hek_query_download(monkeypatch, database, tmpdir): assert len(database) == 0 records = [ '94_1331820530-1331820530', '94_1331820542-1331820542', '94_1331820554-1331820554', '94_1331820566-1331820566', '94_1331820578-1331820578', '94_1331820590-1331820590', '94_1331820602-1331820602', '94_1331820614-1331820614', '94_1331820626-1331820626', '94_1331820638-1331820638' ] def mock_parfive_download(obj, *args, **kwargs): assert obj.queued_downloads == 10 queue = obj.http_queue if not isinstance(queue, list): queue = list(queue._queue) obj_records = [] for item in queue: url = item.keywords['url'] obj_records.append(url[-24:]) assert obj_records == records result = Results() result.append(str(tmpdir)) return result def mock_entries_from_dir(*args, **kwargs): for i in range(10): yield DatabaseEntry() monkeypatch.setattr(Downloader, "download", mock_parfive_download) monkeypatch.setattr(tables, "entries_from_dir", mock_entries_from_dir) query = hek.HEKClient().search( net_attrs.Time('2019/03/10 14:40:10', '2019/04/11 16:40:50'), hek.attrs.EventType('FL')) database.download_from_hek_query_result(query[4], path=str(tmpdir)) assert len(database) == 10
def get_hmi_data(): # get the list of the files # compile list of dates # get max flare class for each date # move each file to right location based on max flare size client = hek.HEKClient() with open('hmi_paths.txt') as f: paths = f.readlines() for i in range(len(paths)): path = paths[i][:-1] tstart = parse_time(path[46:56]) + timedelta(hours=int(path[58:60])) tend = tstart + timedelta(days=1) result = client.query(hek.attrs.Time(tstart, tend), hek.attrs.EventType('FL')) cls = get_max_flare(result) img = imread(path) img_resize = imresize(img, (256, 256, 3)) if cls == 'N': folder = 'less_than_c' elif cls == 'C': folder = 'c_class' elif cls == 'M': folder = 'm_class' else: folder = 'x_class' imsave( '/Users/pauly/Dropbox/deep/hmi_data/{}/{}_0.jpg'.format( folder, tstart), img_resize) imsave( '/Users/pauly/Dropbox/deep/hmi_data/{}/{}_1.jpg'.format( folder, tstart), np.flipud(img_resize)) imsave( '/Users/pauly/Dropbox/deep/hmi_data/{}/{}_2.jpg'.format( folder, tstart), np.fliplr(img_resize)) imsave( '/Users/pauly/Dropbox/deep/hmi_data/{}/{}_3.jpg'.format( folder, tstart), np.fliplr(np.flipud(img_resize))) print('\r{}%'.format(int(100 * i / len(paths))), end='')
def get_feature_images(self, time_start, time_end, save_dir): """ Download jp2 SDO images and create feature masks for a given time period :param time_start: String start time for HEK query :param time_end: string end time for HEK query :param save_dir: directory where images are saved :return: """ client = hek.HEKClient() result = client.search(hek.attrs.Time(time_start, time_end), hek.attrs.FRM.Name == 'SPoCA') # CH and AR result += client.search(hek.attrs.Time(time_start, time_end), hek.attrs.FRM.Name == 'EGSO_SFC') # SS times = list(set([elem["event_starttime"] for elem in result])) times.sort() ch = [elem for elem in result if elem['event_type'] == 'CH'] ar = [elem for elem in result if elem['event_type'] == 'AR'] ss = [elem for elem in result if elem['event_type'] == 'SS'] for time_in in times: image_file = self.get_all_sdo_images(time_in, save_path=save_dir) ch_mask = self.gen_feature_mask( time_in, [elem for elem in ch if elem['event_starttime'] == time_in], image_file) ar_mask = self.gen_feature_mask( time_in, [elem for elem in ar if elem['event_starttime'] == time_in], image_file) ss_mask = self.gen_feature_mask( time_in, [elem for elem in ss if elem['event_starttime'] == time_in], image_file) self.write_feature_mask(ch_mask, time_in, 'CH', save_path=save_dir) self.write_feature_mask(ar_mask, time_in, 'AR', save_path=save_dir) self.write_feature_mask(ss_mask, time_in, 'SS', save_path=save_dir)
def test_vso_attribute_parse(): """Make sure that Parsing of VSO attributes from HEK queries is accurate""" h = hek.HEKClient() hek_query = h.query(hekTime, hekEvent) vso_query = hek2vso.vso_attribute_parse(hek_query[0]) # Cheking Time # TODO # Checking Observatory assert vso_query[1].value == hek_query[0]['obs_observatory'] # Checking Instrument assert vso_query[2].value == hek_query[0]['obs_instrument'] # Checking Wavelength assert vso_query[3].min == hek_query[0]['obs_meanwavel'] * u.Unit( hek_query[0]['obs_wavelunit']) assert vso_query[3].max == hek_query[0]['obs_meanwavel'] * u.Unit( hek_query[0]['obs_wavelunit']) assert vso_query[3].unit == u.Unit('Angstrom')
def graph_sp(self): now = datetime.datetime.now() length = datetime.timedelta(hours=24 * 2) tr = TimeRange([now - length, now]) results = Fido.search(a.Time(tr), a.Instrument('XRS')) print("Nalezene soubory", results) files = Fido.fetch(results) goes = TimeSeries(files, source='XRS', concatenate=True) #goes = database.add_from_vso_query_result(results) #print(goes) client = hek.HEKClient() flares_hek = client.search(hek.attrs.Time(tr.start, tr.end), hek.attrs.FL, hek.attrs.FRM.Name == 'SWPC') #out = goes.data.to_json(orient='index', date_format='iso') print(goes.data) out = goes.data.rolling(20, min_periods=1).mean().to_csv() print(flares_hek) return out
from sunpy.net import hek client = hek.HEKClient() tstart, tend = "2011/08/09 00:00:00", "2011/08/10 00:00:00" result = client.query(hek.attrs.Time(tstart, tend), hek.attrs.EventType("FL")) # flares len(result)