def create_config_file(pkg, rootname='astropy', overwrite=False):
"""
Create the default configuration file for the specified package.
If the file already exists, it is updated only if it has not been
modified. Otherwise the ``overwrite`` flag is needed to overwrite it.
Parameters
----------
pkg : str
The package to be updated.
rootname : str
Name of the root configuration directory.
overwrite : bool
Force updating the file if it already exists.
Returns
-------
updated : bool
If the profile was updated, `True`, otherwise `False`.
"""
# local import to prevent using the logger before it is configured
from astropy.logger import log
cfgfn = get_config_filename(pkg, rootname=rootname)
# generate the default config template
template_content = io.StringIO()
generate_config(pkg, template_content)
template_content.seek(0)
template_content = template_content.read()
doupdate = True
# if the file already exists, check that it has not been modified
if cfgfn is not None and path.exists(cfgfn):
with open(cfgfn, 'rt', encoding='latin-1') as fd:
content = fd.read()
doupdate = is_unedited_config_file(content, template_content)
if doupdate or overwrite:
with open(cfgfn, 'wt', encoding='latin-1') as fw:
fw.write(template_content)
log.info('The configuration file has been successfully written '
f'to {cfgfn}')
return True
elif not doupdate:
log.warning('The configuration file already exists and seems to '
'have been customized, so it has not been updated. '
'Use overwrite=True if you really want to update it.')
return False
def __init__(self, provider="AWS", profile=None, verbose=False):
"""
Initialize class to enable downloading public files from S3
instead of STScI servers.
Requires the boto3 and botocore libraries to function.
Parameters
----------
provider : str
Which cloud data provider to use. Currently only AWS S3 is supported,
so at the moment this argument is ignored.
profile : str
Profile to use to identify yourself to the cloud provider (usually in ~/.aws/config).
verbose : bool
Default False. Display extra info and warnings if true.
"""
import boto3
import botocore
self.supported_missions = [
"mast:hst/product", "mast:tess/product", "mast:kepler"
]
if profile is not None:
self.boto3 = boto3.Session(profile_name=profile)
else:
self.boto3 = boto3
self.botocore = botocore
self.pubdata_bucket = "stpubdata"
if verbose:
log.info("Using the S3 STScI public dataset")
log.warning(
"Your AWS account will be charged for access to the S3 bucket")
log.info(
"See Request Pricing in https://aws.amazon.com/s3/pricing/ for details"
)
log.info(
"If you have not configured boto3, follow the instructions here: "
"https://boto3.readthedocs.io/en/latest/guide/configuration.html"
)
def _get_password(self, service_name, username, reenter=False):
"""Get password from keyring or prompt."""
password_from_keyring = None
if reenter is False:
try:
password_from_keyring = keyring.get_password(
service_name, username)
except keyring.errors.KeyringError as exc:
log.warning("Failed to get a valid keyring for password "
"storage: {}".format(exc))
if password_from_keyring is None:
log.warning("No password was found in the keychain for the "
"provided username.")
if system_tools.in_ipynb():
log.warning("You may be using an ipython notebook:"
" the password form will appear in your terminal.")
password = getpass.getpass(
"{0}, enter your password:\n".format(username))
else:
password = password_from_keyring
return password, password_from_keyring
def _get_password(self, service_name, username, reenter=False):
"""Get password from keyring or prompt."""
password_from_keyring = None
if reenter is False:
try:
password_from_keyring = keyring.get_password(
service_name, username)
except keyring.errors.KeyringError as exc:
log.warning("Failed to get a valid keyring for password "
"storage: {}".format(exc))
if password_from_keyring is None:
log.warning("No password was found in the keychain for the "
"provided username.")
if system_tools.in_ipynb():
log.warning("You may be using an ipython notebook:"
" the password form will appear in your terminal.")
password = getpass.getpass("{0}, enter your password:\n"
.format(username))
else:
password = password_from_keyring
return password, password_from_keyring
def _download_file(self,
url,
local_filepath,
timeout=None,
auth=None,
continuation=True,
cache=False,
**kwargs):
"""
Download a file. Resembles `astropy.utils.data.download_file` but uses
the local ``_session``
"""
response = self._session.get(url,
timeout=timeout,
stream=True,
auth=auth,
**kwargs)
response.raise_for_status()
if 'content-length' in response.headers:
length = int(response.headers['content-length'])
else:
length = None
if ((os.path.exists(local_filepath)
and ('Accept-Ranges' in response.headers) and continuation)):
open_mode = 'ab'
existing_file_length = os.stat(local_filepath).st_size
if length is not None and existing_file_length >= length:
# all done!
log.info(
"Found cached file {0} with expected size {1}.".format(
local_filepath, existing_file_length))
return
elif existing_file_length == 0:
open_mode = 'wb'
else:
log.info("Continuing download of file {0}, with {1} bytes to "
"go ({2}%)".format(
local_filepath, length - existing_file_length,
(length - existing_file_length) / length * 100))
# bytes are indexed from 0:
# https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#range-request-header
end = "{0}".format(length - 1) if length is not None else ""
self._session.headers['Range'] = "bytes={0}-{1}".format(
existing_file_length, end)
response = self._session.get(url,
timeout=timeout,
stream=True,
auth=auth,
**kwargs)
elif cache and os.path.exists(local_filepath):
if length is not None:
statinfo = os.stat(local_filepath)
if statinfo.st_size != length:
log.warning("Found cached file {0} with size {1} that is "
"different from expected size {2}".format(
local_filepath, statinfo.st_size, length))
open_mode = 'wb'
else:
log.info(
"Found cached file {0} with expected size {1}.".format(
local_filepath, statinfo.st_size))
response.close()
return
else:
log.info("Found cached file {0}.".format(local_filepath))
response.close()
return
else:
open_mode = 'wb'
blocksize = astropy.utils.data.conf.download_block_size
bytes_read = 0
# Only show progress bar if logging level is INFO or lower.
if log.getEffectiveLevel() <= 20:
progress_stream = None # Astropy default
else:
progress_stream = io.StringIO()
with ProgressBarOrSpinner(
length,
('Downloading URL {0} to {1} ...'.format(url, local_filepath)),
file=progress_stream) as pb:
with open(local_filepath, open_mode) as f:
for block in response.iter_content(blocksize):
f.write(block)
bytes_read += blocksize
if length is not None:
pb.update(
bytes_read if bytes_read <= length else length)
else:
pb.update(bytes_read)
response.close()
def main():
'''
This is the main function of the system, it coordinates the Telescope_mod, the ccd_mod and the Query_mod.
First it calls load_measure to obtain some data for Telescope_mod and Query_mod, the uses Telescope_mod
with some additional parameters in input to create the PSF with the right measure for the telescope and the CCD.
At this point the function calls again the Telescope_mod to creats an empty image of the CCD, and the Query_mod
to obtain information about the position and the flux of the stars in a specific reagion of the sky,
the center of that is an imput from the operator.
The function uses the information of Query_mod to "light_up" some pixel on the CCD's image in position where shoud be the stars
and then convolve this updated CCD with the calculated PSF from Telescope_mod obtainig the CCD with the stars drawn.
If the value of realistic is True, the main calls the ccd_mod to creates the bias frame, the dark frame, the read out noise,
the backround noise and the flat frame and then combines all the frames to obtain a more realistic image on the CCD.
'''
log.info(hist())
binning = 2
#photo_filters = ['U', 'B', 'V', 'R', 'I']
photo_filters = ['V']
default_coordinates = "07 59 08.445 +15 24 42.00"
default_defocus = 0.0 #mm
default_exptime = 60
default_seeing = 3
default_method = 3 # gaussian approx
default_catalog = 1
coordinates = input(f'Coordinates. Default: {default_coordinates}. ') or default_coordinates
defocus_distance = float(input(f'Defocus distance [mm]. Default: {default_defocus}. ') or default_defocus)
exposure_time = float(input(f'Exptime [s]. Default: {default_exptime}. ') or default_exptime)
seeing = float(input(f'Seeing [arcsec]. Default: {default_seeing}. ') or default_seeing)
choose = int(input(f'Method. Kolmogorov (1), spekles (2), gaussian approximation(3). Default: {default_method}. ') or default_method)
catalogue = default_catalog # int(input(f'Catalog. Gaia(1) or Simbad(2). Default Catalog: {default_catalog}') or default_catalog )
if catalogue == 1:
catalogue = 'Gaia'
log.warning('This catalog uses its own passband, simulated results can be different from the real ones')
else:
catalogue = 'Simbad'
telescope_structure, ccd_structure, ccd_data = load_measure()
#Standard data for the noise formation
realistic = True
dark = 1.04
bias_level = 0 #760 # 2000
gain = ccd_data[0]
read_noise_electrons = ccd_data[1]
ccd_structure[2] = ccd_structure[2] * binning
ccd_structure.append(Tm.plate_scale(ccd_structure[2], telescope_structure[0])) #calculate the resolution of the CCD arcsec/pixel
if choose == 1:
CCD_sample = Tm.telescope_on_CCD(ccd_structure[3], binning, telescope_structure, defocus_distance, True, True)
else:
CCD_sample = Tm.telescope_on_CCD(ccd_structure[3], binning, telescope_structure, defocus_distance, True, False) #generates the sample of a star with the right measure
CCD = Tm.physical_CCD(ccd_structure) #generate the CCD
size = CCD.shape
if choose == 2:
CCD_seeing = seeing/(ccd_structure[3]*2)
number_of_spekle = 100 * exposure_time
seeing_image = Tm.main_spekle(number_of_spekle, CCD_seeing)
seeing_image = seeing_image/number_of_spekle
elif choose == 3:
CCD_seeing = seeing/(ccd_structure[3]*2)
seeing_image = Tm.seeing(CCD_seeing)
sky, center = Qm.query(coordinates, photo_filters, ccd_structure, exposure_time, catalogue) #call a function that gives back positions, fluxs of the stars and a data for the header
sky_counts = Qm.sky_brightness(ccd_structure[3], size[0], size[1], photo_filters, exposure_time, 3)
photons_collection_area = (np.pi/400)*(telescope_structure[1]**2-telescope_structure[2]**2)
if choose == 1:
multiplier = 2 * gain * photons_collection_area *0.05
else:
multiplier = 1 * gain * photons_collection_area *0.05 #photons_collection_area * gain * binning**2 #* 200 #I don't know where 200 cames from I'm investigating
#the flux is in ph cm^-2 so it has to be multiplied for the effective area of the aperure in cm^2
for i in range(len(sky[0])):
if sky[0][i] >=0 and sky[0][i] <=size[0]:
if sky[1][i] >=0 and sky[1][i] <=size[1]:
CCD[int(sky[0][i])][int(sky[1][i])] = sky[2][i] * multiplier
CCD = signal.fftconvolve(CCD, CCD_sample, mode='same') #convolve the position with the sample
if choose == 2 or choose == 3:
CCD = signal.fftconvolve(CCD, seeing_image, mode='same')
lines = False
dust = False
gaussian_vignetting = False
if realistic:
flat = ccd_mod.sensitivity_variations(CCD, gaussian_vignetting, dust)
if bias_level == 0:
bias_only = 0
else:
bias_only = ccd_mod.bias(CCD, bias_level, lines)
noise_only = ccd_mod.read_out_noise(CCD, read_noise_electrons, gain)
dark_only = ccd_mod.dark_current(CCD, dark, exposure_time, gain)
sky_only = ccd_mod.sky_background(CCD, sky_counts, gain)
#cosmic_rays = ccd_mod.make_cosmic_rays(CCD, np.random.randint(10,30))
CCD = bias_only + noise_only + dark_only + flat * (sky_only + CCD)
CCD = ccd_mod.saturation_controll(CCD)
save_fits(CCD, defocus_distance, center, choose) #save the fits
def download_file(self, data_product, local_path, cache=True):
"""
Takes a data product in the form of an `~astropy.table.Row` and downloads it from the cloud into
the given directory.
Parameters
----------
data_product : `~astropy.table.Row`
Product to download.
local_path : str
The local filename to which toe downloaded file will be saved.
cache : bool
Default is True. If file is found on disc it will not be downloaded again.
"""
s3 = self.boto3.resource('s3')
s3_client = self.boto3.client('s3')
bkt = s3.Bucket(self.pubdata_bucket)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
bucket_path = self.get_cloud_uri(data_product, False)
if not bucket_path:
raise Exception("Unable to locate file {}.".format(data_product['productFilename']))
# Ask the webserver (in this case S3) what the expected content length is and use that.
info_lookup = s3_client.head_object(Bucket=self.pubdata_bucket, Key=bucket_path, RequestPayer='requester')
length = info_lookup["ContentLength"]
if cache and os.path.exists(local_path):
if length is not None:
statinfo = os.stat(local_path)
if statinfo.st_size != length:
log.warning("Found cached file {0} with size {1} that is "
"different from expected size {2}"
.format(local_path,
statinfo.st_size,
length))
else:
log.info("Found cached file {0} with expected size {1}."
.format(local_path, statinfo.st_size))
return
with ProgressBarOrSpinner(length, ('Downloading URL s3://{0}/{1} to {2} ...'.format(
self.pubdata_bucket, bucket_path, local_path))) as pb:
# Bytes read tracks how much data has been received so far
# This variable will be updated in multiple threads below
global bytes_read
bytes_read = 0
progress_lock = threading.Lock()
def progress_callback(numbytes):
# Boto3 calls this from multiple threads pulling the data from S3
global bytes_read
# This callback can be called in multiple threads
# Access to updating the console needs to be locked
with progress_lock:
bytes_read += numbytes
pb.update(bytes_read)
bkt.download_file(bucket_path, local_path, ExtraArgs={"RequestPayer": "requester"},
Callback=progress_callback)
def _download_file(self, url, local_filepath, timeout=None, auth=None,
continuation=True, cache=False, method="GET", head_safe=False, **kwargs):
"""
Download a file. Resembles `astropy.utils.data.download_file` but uses
the local ``_session``
"""
if head_safe:
response = self._session.request("HEAD", url, timeout=timeout, stream=True,
auth=auth, **kwargs)
else:
response = self._session.request(method, url, timeout=timeout, stream=True,
auth=auth, **kwargs)
response.raise_for_status()
if 'content-length' in response.headers:
length = int(response.headers['content-length'])
else:
length = None
if ((os.path.exists(local_filepath) and ('Accept-Ranges' in response.headers) and continuation)):
open_mode = 'ab'
existing_file_length = os.stat(local_filepath).st_size
if length is not None and existing_file_length >= length:
# all done!
log.info("Found cached file {0} with expected size {1}."
.format(local_filepath, existing_file_length))
return
elif existing_file_length == 0:
open_mode = 'wb'
else:
log.info("Continuing download of file {0}, with {1} bytes to "
"go ({2}%)".format(local_filepath,
length - existing_file_length,
(length-existing_file_length)/length*100))
# bytes are indexed from 0:
# https://en.wikipedia.org/wiki/List_of_HTTP_header_fields#range-request-header
end = "{0}".format(length-1) if length is not None else ""
self._session.headers['Range'] = "bytes={0}-{1}".format(existing_file_length,
end)
response = self._session.request(method, url, timeout=timeout, stream=True,
auth=auth, **kwargs)
response.raise_for_status()
elif cache and os.path.exists(local_filepath):
if length is not None:
statinfo = os.stat(local_filepath)
if statinfo.st_size != length:
log.warning("Found cached file {0} with size {1} that is "
"different from expected size {2}"
.format(local_filepath,
statinfo.st_size,
length))
open_mode = 'wb'
else:
log.info("Found cached file {0} with expected size {1}."
.format(local_filepath, statinfo.st_size))
response.close()
return
else:
log.info("Found cached file {0}.".format(local_filepath))
response.close()
return
else:
open_mode = 'wb'
if head_safe:
response = self._session.request(method, url, timeout=timeout, stream=True,
auth=auth, **kwargs)
response.raise_for_status()
blocksize = astropy.utils.data.conf.download_block_size
bytes_read = 0
# Only show progress bar if logging level is INFO or lower.
if log.getEffectiveLevel() <= 20:
progress_stream = None # Astropy default
else:
progress_stream = io.StringIO()
with ProgressBarOrSpinner(
length, ('Downloading URL {0} to {1} ...'
.format(url, local_filepath)),
file=progress_stream) as pb:
with open(local_filepath, open_mode) as f:
for block in response.iter_content(blocksize):
f.write(block)
bytes_read += blocksize
if length is not None:
pb.update(bytes_read if bytes_read <= length else
length)
else:
pb.update(bytes_read)
response.close()
return response
def query(coordi, photo_filters, CCD_structure, exposure_time, catalog):
'''
This funtion uses all the functions above to query Simbad and extract the information about the stars in a
specific area of the sky
Parameters
----------
coordi : string
It contains the center of the area of interest
photo_filter : list (string elements)
It is a list where a element is a string that identifies a filter, e.g. "U","R","I"
CCD_structure : array (float elements)
This object contains the information about the measure of the CCD
[0] : the length of the CCD on the x axis in mm
[1] : the length of the CCD on the y axis in mm
[2] : the length of a single pixel in mm
exposure_time : int
It is the exposure time used to obtain the image, it is in seconds
Outputs
-------
data : list
It is a list of array that contains
[0] : array, the position of the stars along the x axis
[1] : array, the position of the stars along the y axis
[2] : array, the number of electrons generates by the stars on the CCD
center : list
It contains the center of the sky portion in "RA" and "DEC"
'''
log.info(hist())
radius = radius_sky_portion(CCD_structure)
coordi = coord.SkyCoord(coordi, frame = 'icrs', unit=(u.hourangle, u.deg))
center = [0,0]
center[0] = coordi.ra.degree
center[1] = coordi.dec.degree
data = Data_structure()
AirMass = 1 #simple Antola
if catalog == 'Simbad':
Controll_V = False
for n, i in enumerate(photo_filters):
if i == "g":
photo_filters[n] = "V"
photo_filters_temp = photo_filters
if "V" in photo_filters_temp:
photo_filters_temp.append("g")
Controll_V = True
f =['flux({0})'.format(x) for x in photo_filters_temp]
[Simbad.add_votable_fields(g) for g in f]
result_table = Simbad.query_region(coordi, radius)
Stars_number = len(result_table)
len_filter_list = len(photo_filters_temp)
if Controll_V:
log.warning('If filter V is not available, g filter will be used instead')
log.info(hist(f"Creating {Stars_number} stars:"))
for i in range(Stars_number):
log.info(f"Star n° {i+1}/{Stars_number}")
row = result_table[i]
tot = 0
magnitudo = []
for j in range(11, 11+len_filter_list): #controll
if row[j] is np.ma.masked:
row[j] = 0
magnitudo.append(row[j])
tot = magnitudo_to_electrons(magnitudo, photo_filters_temp, AirMass, exposure_time, Controll_V)
if tot != 0: #if total flux is 0 the object is not passed
c = coord.SkyCoord(row[1], row[2], frame = 'icrs', unit=(u.hourangle, u.deg))##change here
x, y = Coordinator(c, center, CCD_structure, catalog)
data[0].append(x)
data[1].append(y)
data[2].append(tot)
elif catalog == 'Gaia':
Gaia.ROW_LIMIT = -1
tables = Gaia.cone_search_async(coordi, radius)
tables = tables.get_results()
photonis = tables['phot_g_mean_flux'][:]
mag = tables['phot_g_mean_mag'][:]
pos_ra = tables['ra'][:]
pos_dec = tables['dec'][:]
number_of_stars = len(photonis)
for i in range(number_of_stars):
c = coord.SkyCoord(pos_ra[i], pos_dec[i], frame = 'icrs', unit=(u.degree, u.degree))
x, y = Coordinator(c, center, CCD_structure, catalog)
mag_atm_att = AirMass*0.2 #extinction_coefficient_mean
mag[i] = mag[i] + mag_atm_att
mag[i] = 10**(6.74-0.4*mag[i]) #here becomes photons
quantum_efficiency = 0.3
transmissivity = 0.5
multiplier = quantum_efficiency * transmissivity * exposure_time *0.5
data[0].append(x)
data[1].append(y)
data[2].append(mag[i] * multiplier)
return data, center
