proposal_id=['12062', '12061', '12062']) # Grab the list of available data products for these observations products = Observations.get_product_list(obsTable) # Select only drizzled (DRZ) files filtered = Observations.filter_products(products, mrp_only=False, productSubGroupDescription='DRZ') # Enable 'S3 mode' for module which will return S3-like URLs for FITs files # e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits Observations.enable_s3_hst_dataset() # Grab the S3 URLs for each of the observations s3_urls = Observations.get_hst_s3_uris(filtered) print('Query returned {} entries.\n'.format(len(filtered))) print('Downloading data from S3.\n') s3 = boto3.resource('s3') # Create an authenticated S3 session. Note, download within US-East is free # e.g. to a node on EC2. s3_client = boto3.client('s3') bucket = s3.Bucket('stpubdata') # For brevity, only download the first three images for url in s3_urls[:3]:
products = Observations.get_product_list(proposal_obs) print("Products for proposal %s: %d" % (prop, len(products)), file=sys.stderr) filtered_products = Observations.filter_products( products, mrp_only=False, productSubGroupDescription='RAW', extension='fits', ) print("RAW products for proposal %s: %d" % (prop, len(filtered_products)), file=sys.stderr) if len(filtered_products) > 0: s3_uris = Observations.get_hst_s3_uris(filtered_products) for uri in s3_uris: key = uri.replace("s3://stpubdata/", "") print("Getting %s" % uri, file=sys.stderr) stpubdata.download_file(key, 'temp.fits', ExtraArgs={"RequestPayer": "requester"}) fits = astropy.io.fits.getdata('temp.fits') print("{0},{1},{2},{3}".format(prop, key.split('/')[-1], fits.shape[0], fits.shape[1])) if shape_tally.get(fits.shape) is None: shape_tally[fits.shape] = 1 else: shape_tally[fits.shape] += 1