def list_available_images(time_start,
time_end,
euvi_interval_cadence=2 * u.hour,
aia_search_cadence=12 * u.second,
wave_aia=193,
wave_euvi=195):
"""
This function takes in a time range and returns all available images for all available
instruments. Time ranges entered as 'astropy.time.core.Time' objects.
Optional inputs control the search cadence for EUVI 'euvi_interval_cadence', AIA
search cadence 'aia_search_cadence', EUVI wavelength 'wave_euvi', and AIA wavelength
'wave_aia'.
:return: A list of instruments, one entry per available instrument. Each list entry
is a pandas dataframe describing the available images.
"""
# query parameters
# interval_cadence = 2*u.hour
# aia_search_cadence = 12*u.second
# wave_aia = 193
# wave_euvi = 195
# generate the list of time intervals
full_range = TimeRange(time_start, time_end)
# time_ranges = full_range.window(euvi_interval_cadence, euvi_interval_cadence)
# initialize the jsoc drms helper for aia.lev1_euv_12
s12 = drms_helpers.S12(verbose=True)
# initialize the helper class for EUVI
euvi = vso_helpers.EUVI(verbose=True)
# pick a time_range to experiment with
# time_range = time_ranges[0]
time_range = full_range
# ---- Query each instrument individually -------------
# If a new instrument comes on-line, add its query routine here. Then add
# its result to f_list. Order does not matter so long as the new routine's
# output f* is in the same pandas dataframe format at fs, fa, and fb.
# query the jsoc for SDO/AIA
fs = s12.query_time_interval(time_range, wave_aia, aia_search_cadence)
# query the VSO for STA/EUVI and STB/EUVI
fa = euvi.query_time_interval(time_range, wave_euvi, craft='STEREO_A')
fb = euvi.query_time_interval(time_range, wave_euvi, craft='STEREO_B')
# combine dataframes into a simple list
f_list = [fs, fa, fb]
# # check for empty instrument results (and remove)
# f_list = [elem for elem in f_list if len(elem) > 0]
return f_list
data_dir='/Users/cdowns/work/ch_evolution/data_tests/drms_rawdata'
# this is an as is fits
infile=data_dir+'/aia_20120712T160001Z.193_drms_protocol_asis.fits'
outfile=data_dir+'/test_astropy.fits'
# this file has already been converted to a "FITS" protocol
infile=data_dir+'/2013/02/03/aia_lev1_euv_12s_20130203T000519_193.fits'
infile=data_dir+'/tmp.fits'
outfile=data_dir+'/tmp2.fits'
hdu_in = astropy.io.fits.open(infile)
hdu_in.verify('silentfix')
hdr = hdu_in[1].header
s12 = drms_helpers.S12(verbose=True)
s12.update_aia_fits_header( infile, outfile, verbose=True, force=True)
#fits_drms_helper_aia_euv_12s.update_aia_fits_header(infile, outfile, verbose=False)
"""
hdu_in = astropy.io.fits.open(infile)
hdu_in.verify('silentfix')
hdr = hdu_in[1].header
# get the corresponding header info from the JSOC as a drms frame
drms_frame=fits_drms_helper_aia_euv_12s.get_drms_info_for_image(hdr)
# update the header info
def synchronic_euv_download(synch_times,
raw_data_dir,
db_session,
download=True,
overwrite=False,
verbose=True):
"""
:param synch_times: A pandas DataFrame with columns 'target_time', 'min_time', and 'max_time'. The function will
look for images to best approximate a synchronic map at target_time, using only images that fall between min_time
and max_time.
:param raw_data_dir: Full path to where images are stored.
:param db_session:
:param download:
:param overwrite:
:param verbose:
:return: pandas dataframe with results for each download.
result -1: no image found in time range
0: image downloaded
1: download error
2: image file already exists (no download/overwrite)
3: image file already exists, but file overwritten
"""
# query parameters
interval_cadence = 2 * u.hour
aia_search_cadence = 12 * u.second
wave_aia = 193
wave_euvi = 195
n_inst = 3
# initialize the jsoc drms helper for aia.lev1_euv_12
s12 = drms_helpers.S12(verbose=True)
# initialize the helper class for EUVI
euvi = vso_helpers.EUVI(verbose=True)
# initialize a pandas dataframe to store download results
download_result = pd.DataFrame(columns=[
"instrument", "wavelength", "target_time", "min_time", "max_time",
"result", "result_desc", "url", "local_path"
])
for index, row in synch_times.iterrows():
time0 = row['target_time']
jd0 = time0.jd
print(time0)
# add new rows to download results for each instrument (defaults to 'No image in time range')
new_rows = pd.DataFrame({
'instrument': ["AIA", "EUVI-A", "EUVI-B"],
'wavelength': [wave_aia, wave_euvi, wave_euvi],
'target_time': [time0] * n_inst,
'min_time': [row['min_time']] * n_inst,
'max_time': [row['max_time']] * n_inst,
'result': [-1] * n_inst,
'result_desc': ['No image in time range'] * n_inst,
'url': [""] * n_inst,
'local_path': [""] * n_inst
})
# query various instrument repos for available images.
f_list = list_available_images(time_start=row['min_time'],
time_end=row['max_time'],
euvi_interval_cadence=interval_cadence,
aia_search_cadence=aia_search_cadence,
wave_aia=wave_aia,
wave_euvi=wave_euvi)
# check for empty instrument results or query errors (and remove)
del_list = []
for ii in range(len(f_list)):
if len(f_list[ii]) == 0:
# mark element for removal
del_list.append(ii)
# no images found in time range. do not attempt to download
elif isinstance(f_list[ii], str) and f_list[ii] == "query_error":
# mark element for removal
del_list.append(ii)
# record 'download error' in new_rows. do not attempt to download
new_rows.iloc[ii, ['result', 'result_desc']] = [
1, 'Image times query failed.'
]
# keep f_list elements that are not in del_list
f_list = [
f_list[ii] for ii in range(len(f_list)) if ii not in set(del_list)
]
if f_list.__len__() == 0:
print("No instrument images in time range.\n")
# record download results 'No image in time range'
download_result = download_result.append(new_rows)
# skip to next iteration of temporal for loop
continue
# Now loop over all the image pairs to select the "perfect" group of images.
imin = cluster_meth_1(f_list=f_list, jd0=jd0)
# download and enter into database
for ii in range(0, len(imin)):
instrument = f_list[ii]['instrument'][0]
spacecraft = f_list[ii]['spacecraft'][0]
image_num = imin[ii][0]
if download:
if instrument == "AIA":
print("Downloading AIA: ", f_list[ii].iloc[image_num].time)
subdir, fname, download_flag = s12.download_image_fixed_format(
f_list[ii].iloc[image_num],
raw_data_dir,
update=True,
overwrite=overwrite,
verbose=verbose)
results_index = 0
elif instrument == "EUVI" and spacecraft == "STEREO_A":
print("Downloading EUVI A: ",
f_list[ii].iloc[image_num].time)
subdir, fname, download_flag = euvi.download_image_fixed_format(
f_list[ii].iloc[image_num],
raw_data_dir,
compress=True,
overwrite=overwrite,
verbose=verbose)
results_index = 1
elif instrument == "EUVI" and spacecraft == "STEREO_B":
print("Downloading EUVI B: ",
f_list[ii].iloc[image_num].time)
subdir, fname, download_flag = euvi.download_image_fixed_format(
f_list[ii].iloc[image_num],
raw_data_dir,
compress=True,
overwrite=overwrite,
verbose=verbose)
results_index = 2
else:
print(
"Instrument ", instrument,
" does not yet have a download function. SKIPPING DOWNLOAD "
)
continue
# update download results
if download_flag == 1:
# download failed. do not attempt to add to DB
new_rows.iloc[results_index,
['result', 'result_desc', 'url']] = [
download_flag, 'Download failed',
f_list[ii].iloc[image_num].url
]
continue
elif download_flag == 0:
# download successful
new_rows.loc[
results_index,
['result', 'result_desc', 'url', 'local_path']] = [
download_flag, 'Image downloaded',
f_list[ii].iloc[image_num].url,
os.path.join(raw_data_dir, subdir, fname)
]
elif download_flag == 2:
new_rows.loc[
results_index,
['result', 'result_desc', 'url', 'local_path']] = [
download_flag,
'Image file already exists. No download/overwrite',
f_list[ii].iloc[image_num].url,
os.path.join(raw_data_dir, subdir, fname)
]
elif download_flag == 3:
new_rows.loc[
results_index,
['result', 'result_desc', 'url', 'local_path']] = [
download_flag,
'Image file already exists. File overwritten',
f_list[ii].iloc[image_num].url,
os.path.join(raw_data_dir, subdir, fname)
]
# print("DEBUG: download_flag=", download_flag, ". Path: ", os.path.join(raw_data_dir, subdir, fname))
# use the downloaded image to extract metadata and write a row to the database (session)
db_session = add_image2session(data_dir=raw_data_dir,
subdir=subdir,
fname=fname,
db_session=db_session)
print(
"\nDownloads complete with all images added to DB session. \nNow commit session changes to DB.\n"
)
# commit the changes to the DB, this also assigns auto-incrementing prime-keys 'data_id'
db_session.commit()
# record download outcomes to dataframe
download_result = download_result.append(new_rows)
return download_result