def _process_photometry_from_plaintext(self, data_product):
"""
Processes the photometric data from a plaintext file into a dict, which can then be stored as a ReducedDatum
for further processing or display. File is read using astropy as specified in the below documentation. The file
is expected to be a multi-column delimited file, with headers for time, magnitude, filter, and error.
# http://docs.astropy.org/en/stable/io/ascii/read.html
:param data_product: Photometric DataProduct which will be processed into a dict
:type data_product: DataProduct
:returns: python dict containing the data from the DataProduct
:rtype: dict
"""
photometry = {}
data = ascii.read(data_product.data.name, format='fixed_width')
if len(data) < 1:
raise InvalidFileFormatException(
'Empty table or invalid file type')
utc = TimezoneInfo(tzname='UTC')
for datum in data:
time = Time(datum['MJD'], format='mjd')
value = {
'magnitude': datum['mag'],
'filter': datum['filt'],
'error': datum['dmag']
}
photometry.setdefault(time.to_datetime(timezone=utc),
[]).append(value)
return photometry
def _process_photometry_from_plaintext(self, data_product, extras):
photometry = []
data_aws = default_storage.open(data_product.data.name, 'r')
data = ascii.read(data_aws.read(),
names=['time', 'filter', 'magnitude', 'error'])
if len(data) < 1:
raise InvalidFileFormatException('Empty table or invalid file type')
for datum in data:
time = Time(float(datum['time']), format='mjd')
utc = TimezoneInfo(utc_offset=0*units.hour)
time.format = 'datetime'
value = {
'timestamp': time.to_datetime(timezone=utc),
'magnitude': datum['magnitude'],
'filter': datum['filter'],
'error': datum['error']
}
value.update(extras)
photometry.append(value)
return photometry
def _process_photometry_from_plaintext(self, data_product):
"""
Processes the photometric data from a plaintext file into a list of dicts. File is read using astropy as
specified in the below documentation. The file is expected to be a multi-column delimited file, with headers for
time, magnitude, filter, and error.
# http://docs.astropy.org/en/stable/io/ascii/read.html
:param data_product: Photometric DataProduct which will be processed into a list of dicts
:type data_product: DataProduct
:returns: python list containing the photometric data from the DataProduct
:rtype: list
"""
photometry = []
data = ascii.read(data_product.data.path)
if len(data) < 1:
raise InvalidFileFormatException('Empty table or invalid file type')
for datum in data:
time = Time(float(datum['time']), format='mjd')
utc = TimezoneInfo(utc_offset=0*units.hour)
time.format = 'datetime'
value = {
'timestamp': time.to_datetime(timezone=utc),
'magnitude': datum['magnitude'],
'filter': datum['filter'],
'error': datum['error']
}
photometry.append(value)
return photometry
def process_data(self, data_product, extras):
mimetype = mimetypes.guess_type(data_product.data.name)[0]
if mimetype in self.PLAINTEXT_MIMETYPES:
photometry = self._process_photometry_from_plaintext(data_product, extras)
return [(datum.pop('timestamp'), json.dumps(datum)) for datum in photometry]
else:
raise InvalidFileFormatException('Unsupported file type')
def process_data(self, data_product, extras):
mimetype = mimetypes.guess_type(data_product.data.name)[0]
if mimetype in self.FITS_MIMETYPES:
spectrum, obs_date = self._process_spectrum_from_fits(data_product)
elif mimetype in self.PLAINTEXT_MIMETYPES:
spectrum, obs_date = self._process_spectrum_from_plaintext(
data_product)
else:
raise InvalidFileFormatException('Unsupported file type')
serialized_spectrum = SpectrumSerializer().serialize(spectrum)
return [(obs_date, serialized_spectrum)]
def _process_spectrum_from_plaintext(self, data_product):
"""
Processes the data from a spectrum from a plaintext file into a Spectrum1D object, which can then be serialized
and stored as a ReducedDatum for further processing or display. File is read using astropy as specified in
the below documentation. The file is expected to be a multi-column delimited file, with headers for wavelength
and flux. The file also requires comments containing, at minimum, 'DATE-OBS: [value]', where value is an
Astropy Time module-readable date. It can optionally contain 'FACILITY: [value]', where the facility is a string
matching the name of a valid facility in the TOM.
# http://docs.astropy.org/en/stable/io/ascii/read.html
Parameters
----------
:param data_product: Spectroscopic DataProduct which will be processed into a Spectrum1D
:type data_product: tom_dataproducts.models.DataProduct
:returns: Spectrum1D object containing the data from the DataProduct
:rtype: specutils.Spectrum1D
:returns: Datetime of observation, if it is in the comments and the file is from a supported facility, current
datetime otherwise
:rtype: AstroPy.Time
"""
from django.core.files.storage import default_storage
data_aws = default_storage.open(data_product.data.name, 'r')
data = ascii.read(data_aws.read(), names=['wavelength', 'flux'])
if len(data) < 1:
raise InvalidFileFormatException(
'Empty table or invalid file type')
facility_name = None
date_obs = datetime.now()
comments = data.meta.get('comments', [])
for comment in comments:
if 'date-obs' in comment.lower():
date_obs = comment.split(':')[1].strip()
if 'facility' in comment.lower():
facility_name = comment.split(':')[1].strip()
facility = get_service_class(
facility_name)() if facility_name else None
wavelength_units = facility.get_wavelength_units(
) if facility else self.DEFAULT_WAVELENGTH_UNITS
flux_constant = facility.get_flux_constant(
) if facility else self.DEFAULT_FLUX_CONSTANT
spectral_axis = np.array(data['wavelength']) * wavelength_units
flux = np.array(data['flux']) * flux_constant
spectrum = Spectrum1D(flux=flux, spectral_axis=spectral_axis)
return spectrum, Time(date_obs).to_datetime()
def process_data(self, data_product):
"""
Routes a photometry processing call to a method specific to a file-format.
:param data_product: Photometric DataProduct which will be processed into the specified format for database
ingestion
:type data_product: DataProduct
:returns: python list of 2-tuples, each with a timestamp and corresponding data
:rtype: list
"""
mimetype = mimetypes.guess_type(data_product.data.path)[0]
if mimetype in self.PLAINTEXT_MIMETYPES:
photometry = self._process_photometry_from_plaintext(data_product)
return [(datum.pop('timestamp'), datum) for datum in photometry]
else:
raise InvalidFileFormatException('Unsupported file type')
def process_data(self, data_product):
"""
Routes a spectroscopy processing call to a method specific to a file-format, then serializes the returned data.
:param data_product: Spectroscopic DataProduct which will be processed into the specified format for database
ingestion
:type data_product: DataProduct
:returns: python list of 2-tuples, each with a timestamp and corresponding data
:rtype: list
"""
mimetype = mimetypes.guess_type(data_product.data.name)[0]
if mimetype in self.FITS_MIMETYPES:
spectrum, obs_date = self._process_spectrum_from_fits(data_product)
elif mimetype in self.PLAINTEXT_MIMETYPES:
spectrum, obs_date = self._process_spectrum_from_plaintext(
data_product)
else:
raise InvalidFileFormatException('Unsupported file type')
serialized_spectrum = SpectrumSerializer().serialize(spectrum)
return [(obs_date, serialized_spectrum)]