def query(self, date_range, aws=False):
""" Submit a query to MAST for observations in the date range
Parameters
----------
date_range : tuple
Tuple of `astropy.time.Time` objects correspond to the beginning
and end of a one month interval
aws : bool
If True, query returns references to data hosted in S3.
"""
LOG.info('Submitting query to MAST')
if aws:
Observations.enable_s3_hst_dataset()
start, stop = date_range
# there shouldn't be any data taken after the most recent file
query_params = {
'project': self.project,
'dataproduct_type': self.product_type,
'intentType': self.obstype,
'target_name': self.target_name,
'instrument_name': self.instr,
't_min': [start.mjd, stop.mjd],
't_exptime': self.t_exptime
}
with warnings.catch_warnings():
warnings.simplefilter('error')
try:
obsTable = Observations.query_criteria(**query_params)
except Exception as e:
msg = ('{}\n Date range [{}, {}]\n {}'.format(e,
start.iso,
stop.iso,
self._msg_div))
LOG.error(msg)
else:
LOG.info('Filtering observations...')
products = Observations.get_product_list(obsTable)
filter_params = {
'mrp_only': False,
'productSubGroupDescription':self.SubGroupDescription
}
filt_products = Observations.filter_products(products,
**filter_params)
key = start.datetime.date().isoformat() # 'YYYY-MM-DD'
self.products[key] = filt_products
save_files = [
'obsTables_dict_all_WFC3IR_Filiters.joblib.save',
'products_dict_list_WFC3IR_Filiters.joblib.save',
'filtered_dict_list_WFC3IR_Filiters.joblib.save',
's3_urls_dict_list_WFC3IR_Filiters.joblib.save'
]
print('[INFO]')
files_in_dir = glob('*')
all_saves_exist = np.all([sname in files_in_dir for sname in save_files])
# Enable 'S3 mode' for module which will return S3-like URLs for FITs files
# e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits
print('[INFO]')
Observations.enable_s3_hst_dataset()
print('[INFO]')
s3 = boto3.resource('s3')
# Create an authenticated S3 session. Note, download within US-East is free
# e.g. to a node on EC2.
print('[INFO]')
s3_client = boto3.client(
's3',
aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'],
aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'])
print('[INFO]')
hst_bucket = s3.Bucket('stpubdata')
