def index(): """ Very simple embedding of a lightcurve chart """ # FLASK # Grab the inputs arguments from the URL # This is automated by the button args = flask.request.args _from = str(args.get('_from', str(DEFAULT_TR.start))) _to = str(args.get('_to', str(DEFAULT_TR.end))) tr = TimeRange(parse_time(_from), parse_time(_to)) if 'next' in args: tr = tr.next() if 'prev' in args: tr = tr.previous() if 'next_hour' in args: tr = TimeRange(tr.start + ONE_HOUR, tr.end + ONE_HOUR) if 'next_day' in args: tr = TimeRange(tr.start + ONE_DAY, tr.end + ONE_DAY) if 'prev_hour' in args: tr = TimeRange(tr.start - ONE_HOUR, tr.end - ONE_HOUR) if 'prev_day' in args: tr = TimeRange(tr.start - ONE_DAY, tr.end - ONE_DAY) _from = str(tr.start) _to = str(tr.end) # get the data goes = lc.GOESLightCurve.create(tr) # resample to reduce the number of points for debugging goes.data = goes.data.resample("1T").mean() # add time string for display of hover tool goes.data['time_str'] = goes.data.index.strftime('%F %H:%M:%S') source = ColumnDataSource(data=goes.data) source_static = ColumnDataSource(data=goes.data) # now create the bokeh plots # XRS-B Plot fig1 = figure(title="GOES", tools=TOOLS, plot_height=PLOT_HEIGHT, width=PLOT_WIDTH, x_axis_type='datetime', y_axis_type="log", y_range=(10**-9, 10**-2), toolbar_location="right") fig1.xaxis.formatter = formatter fig1.line('index', 'xrsb', source=source_static, color='red', line_width=2, legend="xrsa 1-8 Angstrom") fig2 = figure(title="GOES", tools=TOOLS, plot_height=PLOT_HEIGHT, width=PLOT_WIDTH, x_axis_type='datetime', y_axis_type="log", y_range=(10**-9, 10**-2)) fig2.xaxis.formatter = formatter fig2.line('index', 'xrsa', source=source_static, color='blue', line_width=2, legend="xrsa 0.5-4.0 Angstrom") # link the x-range for common panning fig2.x_range = fig1.x_range fig = Column(fig1, fig2) source_static.callback = CustomJS(code=""" var inds = cb_obj.selected['1d'].indices; var d1 = cb_obj.data; var m = 0; if (inds.length == 0) { return; } for (i = 0; i < inds.length; i++) { d1['color'][inds[i]] = "red" if (d1['y'][inds[i]] > m) { m = d1['y'][inds[i]] } } console.log(m); cb_obj.trigger('change'); """) hover = HoverTool() hover.tooltips = [("time", "@time_str"), ("xrsb", "@xrsb"), ("xrsa", "@xrsa")] fig1.add_tools(hover) hover2 = HoverTool() hover2.tooltips = [("time", "@time_str"), ("xrsb", "@xrsb"), ("xrsa", "@xrsa")] fig2.add_tools(hover2) # Configure resources to include BokehJS inline in the document. # For more details see: # http://bokeh.pydata.org/en/latest/docs/reference/resources_embedding.html#bokeh-embed js_resources = INLINE.render_js() css_resources = INLINE.render_css() # For more details see: # http://bokeh.pydata.org/en/latest/docs/user_guide/embedding.html#components script, div = components(fig, INLINE) html = flask.render_template( 'embed.html', plot_script=script, plot_div=div, js_resources=js_resources, css_resources=css_resources, _from=_from, _to=_to, ) return encode_utf8(html)
def main(date_start, date_finish, image_size_output, time_window, flag, home_dir, bases): date_time_pre_start = date_start + '-0000' date_time_start = parser.parse(date_time_pre_start) print('date_time_start:', date_time_start) date_time_pre_end = date_finish + '-2359' date_time_end = parser.parse(date_time_pre_end) print('date_time_end:', date_time_end) time_increment = 60 print('image_size_output:', image_size_output) print('flag:', flag) print('home_dir:', home_dir) url_prefix = 'https://seal.nascom.nasa.gov/' print('url_prefix:', url_prefix) look_ahead = int( np.ceil(time_window * 60 / 10.) ) #should sufficiently cover all 7 products based on their cadence. print('look_ahead:', look_ahead) diff_start_finish_total_sec = (date_time_end - date_time_start).total_seconds() print('diff_start_finish_total_sec:', diff_start_finish_total_sec) total_sec = timedelta(days=time_increment).total_seconds() print('total_sec:', total_sec) num_loops = np.ceil( diff_start_finish_total_sec / total_sec ) + 1 #num_loops would be equal to 94 + 1 for 19960101-0000' - '20110501-0000' print('num_loops:', num_loops) base_list = bases.split(',') for base in tqdm(base_list): start_process_time = process_time() #initialize clock per product type base = base.strip(' ') holes_list = [] unreadable_file_ids_product_list_global = [] print(f'{base}') base_dir = home_dir + base if not os.path.exists(base_dir): os.makedirs(base_dir) time_range = TimeRange( date_time_start, timedelta(days=time_increment) ) #time_range re-initialized here for each new base name prev_time, time_range_modified = prev_time_resumer( home_dir, base, time_range, date_time_end) for t_value in tqdm(np.arange( num_loops)): #this is the main workhorse loop of the program print('t_value:', t_value) print('prev_time:', prev_time) if time_range_modified.end > date_time_end: time_range_modified = TimeRange(time_range_modified.start, date_time_end) product_results = product_search(base, time_range_modified, date_time_start) product_results_number = product_results.file_num if product_results_number != 0: ind = index_of_sizes(base, product_results) all_size_sieved_times_pre, fetch_indices_product_orig = fetch_indices( base, ind, product_results, time_window, look_ahead, prev_time) if len(fetch_indices_product_orig) != 0: all_time_window_sieved_times_product_times_modified, holes_product_list, unreadable_file_ids_product_list_local = product_distiller( fetch_indices_product_orig, base, all_size_sieved_times_pre, ind, product_results, look_ahead, time_window, url_prefix, flag, image_size_output, home_dir) if holes_product_list: #if image had missing regions (e.g., arising from telemetry errors) holes_list.append(holes_product_list) if unreadable_file_ids_product_list_local: unreadable_file_ids_product_list_global.append( unreadable_file_ids_product_list_local) all_time_window_sieved_times_sorted = np.unique( all_time_window_sieved_times_product_times_modified) #print(f'{base} np.unique(all_size_sieved_times_pre):', np.unique(all_size_sieved_times_pre), len(np.unique(all_size_sieved_times_pre))) #print(f'{base} list(all_time_window_sieved_times_sorted):', list(all_time_window_sieved_times_sorted), len(all_time_window_sieved_times_sorted)) prev_time = [] #reset to empty list if len(all_time_window_sieved_times_sorted) != 0: prev_time.append( all_time_window_sieved_times_sorted[-1] ) #append the last good time entry from the previous loop csv_writer(base, home_dir, date_start, date_finish, flag, time_window, image_size_output, all_time_window_sieved_times_sorted) time_range_modified.next( ) #Sunpy iterator to go for the next time increment in number of days. There is also time_range_modified.previous() to go backwards in time. #print('time_range_modified next:', time_range_modified) print(f'{base} holes_list', holes_list) print(f'{base} unreadable_file_ids_product_list_global:', unreadable_file_ids_product_list_global) data_cuber(home_dir, base, date_start, date_finish, flag, time_window, image_size_output) end_process_time = process_time() time_of_process = end_process_time - start_process_time print(f'{base} time of process in seconds:', time_of_process)
def returnDataRange(centralDate, delta_min): date_rangeUp = TimeRange(centralDate, delta_min * u.min).previous() date_rangeFinal = TimeRange(date_rangeUp.start, date_rangeUp.next().end) return date_rangeFinal
delta_t = t1 - t0 print() print('-----------------------') print(t0) print(t1) print(delta_t.sec) # sunpy time ranges from sunpy.time import TimeRange time_range = TimeRange(t0, t1) print(time_range.center) tc = Time(time_range.center, scale='utc') print(tc.mjd) print(tc.datetime) print(time_range.seconds) print(time_range.next()) # example of windowing time ranges at a candence in sunpy (potentially very useful!) import astropy.units as u w = time_range.window(600 * u.second, 60 * u.second) for win in w: print(win.center) """ illustration of how I think I will use Astropy and Sunpy time objects - start with a set of dates that spans your range of interest - convert these to astropy time objects - generate a sunpy time range that spans the whole interval - divide up this interval into entries """ print('\n---------------------\n') date_start = '2012-07-01T00:00:00.000'
def main(date_start, date_finish, image_size_output, time_window, flag, home_dir, bases, fits_headers, lev1_LASCO, email, mission): warnings.simplefilter(action='ignore', category=RuntimeWarning) client = drms.Client(email=email, verbose=False) #True date_time_pre_start = date_start + '-0000' date_time_start = parser.parse(date_time_pre_start) print('date_time_start:', date_time_start) date_time_pre_end = date_finish + '-2359' date_time_end = parser.parse(date_time_pre_end) print('date_time_end:', date_time_end) time_increment = 60 print(f'time increment {time_increment} days') print('image_size_output:', image_size_output) print('flag:', flag) print('home_dir:', home_dir) url_prefix = 'https://seal.nascom.nasa.gov/' print('url_prefix:', url_prefix) look_ahead = int( np.ceil(time_window * 60 / 10.) ) #should sufficiently cover all 7 products based on their cadence. print('look_ahead:', look_ahead) diff_start_finish_total_sec = (date_time_end - date_time_start).total_seconds() print('diff_start_finish_total_sec:', diff_start_finish_total_sec) total_sec = timedelta(days=time_increment).total_seconds() print('total_sec:', total_sec) num_loops = np.ceil( diff_start_finish_total_sec / total_sec ) + 1 #num_loops would be equal to 94 + 1 for 19960101-0000' - '20110501-0000' print('num_loops:', num_loops) base_list = bases.split(',') for base in tqdm(base_list): start_process_time = process_time() #initialize clock per product type base = base.strip(' ') holes_list = [] transients_list = [] blobs_list = [] unreadable_file_ids_product_list_global = [] print(f'{base}') base_dir = home_dir + base + f'_{mission}' if not os.path.exists(base_dir): os.makedirs(base_dir) ''' if ('AIA' in base) and (int(base.split('AIA')[1]) != 4500): #AIA4500 is once an hour. So if not AIA4500, then have AIA cadences at 12s and 24s -> time_increment has to be reduced from 60d -> 1d time_increment = 1 print(f'new time increment {time_increment} days:') total_sec = timedelta(days = time_increment).total_seconds() print('new total_sec:', total_sec) num_loops = np.ceil(diff_start_finish_total_sec/total_sec) + 1 #num_loops would be equal to 94 + 1 for 19960101-0000' - '20110501-0000' print('new num_loops:', num_loops) time_range = TimeRange(date_time_start, timedelta(days = time_increment)) else: time_range = TimeRange(date_time_start, timedelta(days = time_increment)) #time_range re-initialized here for each new base name ''' time_range = TimeRange(date_time_start, timedelta(days=time_increment)) prev_time, time_range_modified = prev_time_resumer( home_dir, base, time_range, date_time_end, mission ) #time_range_modified.next() is the workshorse that advances time at the end of the time for-loop for t_value in tqdm(np.arange( num_loops)): #this is the main workhorse loop of the program print('t_value:', t_value) print('prev_time:', prev_time) if time_range_modified.end > date_time_end: time_range_modified = TimeRange(time_range_modified.start, date_time_end) product_results, client = product_search(base, time_range_modified, client, mission, time_window) if ('MDI' in base) or (mission == 'SDO'): client_export_failed = ghost_file_check(product_results) if client_export_failed == True: product_results_number = 0 elif client_export_failed == False: product_results_number = product_results.data.count( )['record'] #### old when this object was a query and not an export: product_results.count()['T_REC'] else: product_results_number = product_results.file_num client_export_failed = False if (product_results_number != 0) and (client_export_failed == False): #product_results.has_failed() ind, fits_headers = index_of_sizes( base, product_results, fits_headers, lev1_LASCO, mission ) #putting fits_headers here to insert it into __init__.py all_size_sieved_times_pre, fetch_indices_product_orig = fetch_indices( base, ind, product_results, time_window, look_ahead, prev_time, mission) if len(fetch_indices_product_orig) != 0: all_time_window_sieved_times_product_times_modified, holes_product_list, transients_product_list, blobs_product_list, unreadable_file_ids_product_list_local = product_distiller( fetch_indices_product_orig, base, all_size_sieved_times_pre, ind, product_results, look_ahead, time_window, url_prefix, flag, image_size_output, home_dir, email, fits_headers, lev1_LASCO, client, mission) if holes_product_list: #if image had missing regions (e.g., arising from telemetry errors) holes_list.append(holes_product_list) if transients_product_list: transients_list.append(transients_product_list) if blobs_product_list: blobs_list.append(blobs_product_list) if unreadable_file_ids_product_list_local: unreadable_file_ids_product_list_global.append( unreadable_file_ids_product_list_local) all_time_window_sieved_times_sorted = np.unique( all_time_window_sieved_times_product_times_modified) #print(f'{base} np.unique(all_size_sieved_times_pre):', np.unique(all_size_sieved_times_pre), len(np.unique(all_size_sieved_times_pre))) #print(f'{base} list(all_time_window_sieved_times_sorted):', list(all_time_window_sieved_times_sorted), len(all_time_window_sieved_times_sorted)) prev_time = [] #reset to empty list if len(all_time_window_sieved_times_sorted) != 0: prev_time.append( all_time_window_sieved_times_sorted[-1] ) #append the last good time entry from the previous loop csv_writer(base, home_dir, date_start, date_finish, flag, time_window, image_size_output, all_time_window_sieved_times_sorted, lev1_LASCO, mission) time_range_modified.next( ) #Sunpy iterator to go for the next time increment in number of days. There is also time_range_modified.previous() to go backwards in time. #print('time_range_modified next:', time_range_modified) print(f'{base} holes_list', holes_list) print(f'{base} transients_list', transients_list) print(f'{base} blobs_list', blobs_list) print(f'{base} unreadable_file_ids_product_list_global:', unreadable_file_ids_product_list_global) data_cuber(home_dir, base, date_start, date_finish, flag, time_window, image_size_output, lev1_LASCO, mission, fits_headers) end_process_time = process_time() time_of_process = end_process_time - start_process_time print(f'{base} time of process in seconds:', time_of_process)