proposal_id=['12062', '12061', '12062'])
# Grab the list of available data products for these observations
products = Observations.get_product_list(obsTable)
# Select only drizzled (DRZ) files
filtered = Observations.filter_products(products,
mrp_only=False,
productSubGroupDescription='DRZ')
# Enable 'S3 mode' for module which will return S3-like URLs for FITs files
# e.g. s3://stpubdata/hst/public/icde/icde43l0q/icde43l0q_drz.fits
Observations.enable_s3_hst_dataset()
# Grab the S3 URLs for each of the observations
s3_urls = Observations.get_hst_s3_uris(filtered)
print('Query returned {} entries.\n'.format(len(filtered)))
print('Downloading data from S3.\n')
s3 = boto3.resource('s3')
# Create an authenticated S3 session. Note, download within US-East is free
# e.g. to a node on EC2.
s3_client = boto3.client('s3')
bucket = s3.Bucket('stpubdata')
# For brevity, only download the first three images
for url in s3_urls[:3]:
products = Observations.get_product_list(proposal_obs)
print("Products for proposal %s: %d" % (prop, len(products)),
file=sys.stderr)
filtered_products = Observations.filter_products(
products,
mrp_only=False,
productSubGroupDescription='RAW',
extension='fits',
)
print("RAW products for proposal %s: %d" % (prop, len(filtered_products)),
file=sys.stderr)
if len(filtered_products) > 0:
s3_uris = Observations.get_hst_s3_uris(filtered_products)
for uri in s3_uris:
key = uri.replace("s3://stpubdata/", "")
print("Getting %s" % uri, file=sys.stderr)
stpubdata.download_file(key,
'temp.fits',
ExtraArgs={"RequestPayer": "requester"})
fits = astropy.io.fits.getdata('temp.fits')
print("{0},{1},{2},{3}".format(prop,
key.split('/')[-1], fits.shape[0],
fits.shape[1]))
if shape_tally.get(fits.shape) is None:
shape_tally[fits.shape] = 1
else:
shape_tally[fits.shape] += 1
