def query(self, date_range, aws=False): """ Submit a query to MAST for observations in the date range Parameters ---------- date_range : tuple Tuple of `astropy.time.Time` objects correspond to the beginning and end of a one month interval aws : bool If True, query returns references to data hosted in S3. """ LOG.info('Submitting query to MAST') if aws: Observations.enable_s3_hst_dataset() start, stop = date_range # there shouldn't be any data taken after the most recent file query_params = { 'project': self.project, 'dataproduct_type': self.product_type, 'intentType': self.obstype, 'target_name': self.target_name, 'instrument_name': self.instr, 't_min': [start.mjd, stop.mjd], 't_exptime': self.t_exptime } with warnings.catch_warnings(): warnings.simplefilter('error') try: obsTable = Observations.query_criteria(**query_params) except Exception as e: msg = ('{}\n Date range [{}, {}]\n {}'.format(e, start.iso, stop.iso, self._msg_div)) LOG.error(msg) else: LOG.info('Filtering observations...') products = Observations.get_product_list(obsTable) filter_params = { 'mrp_only': False, 'productSubGroupDescription':self.SubGroupDescription } filt_products = Observations.filter_products(products, **filter_params) key = start.datetime.date().isoformat() # 'YYYY-MM-DD' self.products[key] = filt_products
save_files = [ 'obsTables_dict_all_WFC3IR_Filiters.joblib.save', 'products_dict_list_WFC3IR_Filiters.joblib.save', 'filtered_dict_list_WFC3IR_Filiters.joblib.save', 's3_urls_dict_list_WFC3IR_Filiters.joblib.save' ] print('[INFO]') files_in_dir = glob('*') all_saves_exist = np.all([sname in files_in_dir for sname in save_files]) # Enable 'S3 mode' for module which will return S3-like URLs for FITs files # e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits print('[INFO]') Observations.enable_s3_hst_dataset() print('[INFO]') s3 = boto3.resource('s3') # Create an authenticated S3 session. Note, download within US-East is free # e.g. to a node on EC2. print('[INFO]') s3_client = boto3.client( 's3', aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY']) print('[INFO]') hst_bucket = s3.Bucket('stpubdata')