def _load_stars_object(self):
"""Load object file of list of stars"""
stars = loadFromFile(os.path.join(self.path, self.object_file_name))
if len(stars) == 0:
raise InvalidFile("There are no stars in object file")
if stars[0].__class__.__name__ != "Star":
raise InvalidFile("It is not list of stars")
return stars
def _createLcFromFits(self, fits):
time = []
mag = []
err = []
for line in fits.data:
if len(line) == 3:
t, m, e = line
elif len(line) == 2:
t, m = line
e = 0
else:
if hasattr(line, "__iter__"):
n = len(line)
else:
n = None
raise InvalidFile(
"Light curve binary extension of the fitst couldn't be parsed\nbecause of line {1}\n type: {0}, lenght: {2}"
.format(type(line), line, n))
time.append(t)
mag.append(m)
err.append(e)
meta = {
"xlabel": fits.header.get("TTYPE1", None),
"xlabel_unit": fits.header.get("TUNIT1", None),
"ylabel": fits.header.get("TTYPE2", None),
"ylabel_unit": fits.header.get("TUNIT2", None),
"color": fits.header.get("FILTER", None),
"origin": fits.header.get(self.DB_ORIGIN, None)
}
return LightCurve([time, mag, err], meta)
def loadFromFile(file_name="saved_object.pickle"):
"""
Open object from file
Parameters
----------
file_name : str
Name of the serialized file
Returns
-------
object
Loaded object
"""
try:
with open(file_name, "rb") as inputToLoad:
loaded_object = pickle.load(inputToLoad)
return loaded_object
except IOError:
raise InvalidFilesPath
except ImportError as e:
raise InvalidFile(
"Structure of project has been changed since saving this object: %s"
% str(e))
except TypeError:
return pickle.load(file_name)
def _createLcFromFits(self, fits):
time = []
mag = []
err = []
for line in fits.data:
try:
t, m, e = line
except:
raise InvalidFile(
"Light curve extension of fits couldn't be parsed\n%s" %
line)
time.append(t)
mag.append(m)
err.append(e)
meta = {
"xlabel": fits.header.get("TTYPE1", None),
"xlabel_unit": fits.header.get("TUNIT1", None),
"ylabel": fits.header.get("TTYPE2", None),
"ylabel_unit": fits.header.get("TUNIT2", None),
"color": fits.header.get("FILTER", None),
"origin": fits.header.get(self.DB_ORIGIN, None)
}
return LightCurve([time, mag, err], meta)
def _loadFromFITS(self, star_paths, files_lim=None):
if not star_paths:
return []
stars = []
for path in progressbar(star_paths, "Loading FITS files:"):
try:
fits = pyfits.open(os.path.join(self.path, path))
except:
raise InvalidFile("Invalid fits file or path: %s" % self.path)
stars.append(self._createStarFromFITS(fits))
return stars
def _loadFromFITS(self, star_paths, files_lim=None):
if not star_paths:
return []
if files_lim and isinstance(files_lim, int):
star_paths = star_paths[:files_lim]
stars = []
for path in tqdm(star_paths, desc="Loading FITS files:"):
try:
fits_file = fits.open(os.path.join(path))
except Exception as e:
raise InvalidFile("Invalid fits file or path: {}\n{}".format(
self.path, e))
stars.append(self._createStarFromFITS(fits_file))
return stars
def _loadLcFromDat(cls, file_name):
"""
Load Light curve from dat file of light curve
Parameters
-----------
file_with_path : str
Name of the light curve file with its path
Returns
--------
List of tuples of (time, mag, err)
"""
try:
dat = np.loadtxt(file_name,
usecols=(cls.TIME_COL, cls.MAG_COL, cls.ERR_COL),
skiprows=0)
except IndexError:
dat = np.loadtxt(file_name,
usecols=(cls.TIME_COL, cls.MAG_COL, cls.ERR_COL),
skiprows=2)
except IOError as Argument:
raise InvalidFilesPath("\nCannot open light curve file\n %s" %
Argument)
mag, time, err = dat.T
if not (len(mag) == len(time) == len(err)):
raise InvalidFile(
"Length of columns in light curve file is not the same")
else:
clean_dat = []
for x, y, z in zip(mag, time, err):
if (x not in cls.BAD_VALUES and y not in cls.BAD_VALUES
and z not in cls.BAD_VALUES):
clean_dat.append([
round(x, cls.ROUND_DIGITS),
round(y, cls.ROUND_DIGITS),
round(z, cls.ROUND_DIGITS)
])
return clean_dat
class FileManager(LightCurvesDb):
"""
This class is responsible for managing light curve files
Attributes
-----------
path : str
Path key of folder of light curves .
star_class : str
Name of the loaded star-like type (e.g. Cepheids)
suffix : str
Suffix of light curve files in the folder. If suffix is "fits",
files are loaded as fits files, otherwise files are considered
as .dat files of light curve such as:
#time mag err
12 13.45 0.38
files_limit : int, str
Number of files which will be loaded
db_ident : str
Name of the database to which the file name will be assigned
EXAMPLE:
For the file "my_macho_star.dat" and given db_ident as "macho"
makes Star object:
star.ident["macho"] --> my_macho_star
files_to_load : iterable of str
List of file names which should be loaded from the given folder.
If it is not specified all files will be loaded
object_file_name : str
Name of the pickle file which contains list of star objects
"""
SUFFIXES = ["dat", "txt", "fits", "FITS"]
DEFAULT_STARCLASS = "star"
FITS_RA = "RA"
FITS_DEC = "DEC"
FITS_RA_UNIT = "RA_UN"
FITS_DEC_UNIT = "DEC_UN"
FITS_NAME = "IDENT"
FITS_CLASS = "CLASS"
DB_ORIGIN = "DB_ORIGIN"
FITS_SUFFIX = ("fits", "FITS")
BAD_VALUES = ("-99", "-99.0", "99", None, "N/A", np.NaN)
TIME_COL = 0 # Order of columns in the light curve file
MAG_COL = 1
ERR_COL = 2
ROUND_DIGITS = 3
def __init__(self, obtain_params):
"""
Parameters
----------
obtain_params : dict
Query dictionary (see class Attributes doc above)
"""
if isinstance(obtain_params, list) and len(obtain_params) == 1:
obtain_params = obtain_params[0]
path = obtain_params.get("path", None)
if not path:
raise IOError("Path %s was not found" % path)
if not hasattr(path, "__iter__"):
path = [path]
self.path = path
self.star_class = obtain_params.get("star_class",
self.DEFAULT_STARCLASS)
self.suffix = obtain_params.get("suffix", None)
file_lim = obtain_params.get("files_limit")
if file_lim:
self.files_limit = int(file_lim)
else:
self.files_limit = None
self.db_ident = obtain_params.get("db_ident")
self.files_to_load = obtain_params.get("files_to_load")
self.object_file_name = obtain_params.get("object_file_name")
def getStars(self, load_lc=True):
"""
Common method for all stars provider
If there are object_file_name in query dictionary, the object file
of list of stars is loaded. In other case files from given path of
the folder is loaded into star objects.
Returns
--------
list of `Star` objects
Star objects with light curves
"""
if self.object_file_name:
return self._load_stars_object()
else:
stars = []
for path in self.path:
self.path = path
stars += self._load_stars_from_folder()
return stars
def _load_stars_from_folder(self):
"""Load all files with a certain suffix as light curves"""
# Check whether the path ends with "/" sign, if not add
if not (self.path.endswith("/")):
self.path = self.path + "/"
# Get all light curve files (all files which end with certain suffix
if not self.suffix:
starsList = []
for suffix in self.SUFFIXES:
starsList += glob.glob("%s*%s" % (self.path, suffix))
else:
starsList = glob.glob("%s*%s" % (self.path, self.suffix))
numberOfFiles = len(starsList)
if (numberOfFiles == 0):
if self.suffix:
raise InvalidFilesPath(
"There are no stars in %s with %s suffix" %
(self.path, self.suffix))
else:
raise InvalidFilesPath(
"There are no stars in %s with any of supported suffix: %s"
% (self.path, self.SUFFIXES))
if (numberOfFiles < self.files_limit):
self.files_limit = None
else:
numberOfFiles = self.files_limit
if self.suffix in self.FITS_SUFFIX:
return self._loadFromFITS(starsList, numberOfFiles)
stars = self._loadDatFiles([s for s in starsList if s.endswith("dat")],
numberOfFiles)
stars += self._loadFromFITS(
[s for s in starsList if s.endswith("fits")], numberOfFiles)
return stars
def _loadDatFiles(self, star_paths, numberOfFiles):
if not star_paths:
return []
stars = []
counter = 1
# Load every light curve and put it into star object
for singleFile in progressbar(star_paths[:numberOfFiles],
"Loading dat files:"):
if self.files_to_load and os.path.basename(
singleFile) not in self.files_to_load:
break
lc = LightCurve(self._loadLcFromDat(singleFile))
# Check if light curve is not empty
if (len(lc.mag) >= 1):
db_ident = self.parseFileName(singleFile)
if self.db_ident:
ident = {self.db_ident: {"name": db_ident}}
else:
ident = {"file": {"name": db_ident}}
star = Star(ident=ident)
star.starClass = self.star_class
star.putLightCurve(lc)
stars.append(star)
counter += 1
return stars
@classmethod
def _loadLcFromDat(cls, file_name):
"""
Load Light curve from dat file of light curve
Parameters
-----------
file_with_path : str
Name of the light curve file with its path
Returns
--------
List of tuples of (time, mag, err)
"""
try:
dat = np.loadtxt(file_name,
usecols=(cls.TIME_COL, cls.MAG_COL, cls.ERR_COL),
skiprows=0)
except IndexError:
dat = np.loadtxt(file_name,
usecols=(cls.TIME_COL, cls.MAG_COL, cls.ERR_COL),
skiprows=2)
except IOError, Argument:
raise InvalidFilesPath("\nCannot open light curve file\n %s" %
Argument)
mag, time, err = dat.T
if not (len(mag) == len(time) == len(err)):
raise InvalidFile(
"Length of columns in light curve file is not the same")
else:
clean_dat = []
for x, y, z in zip(mag, time, err):
if (x not in cls.BAD_VALUES and y not in cls.BAD_VALUES
and z not in cls.BAD_VALUES):
clean_dat.append([
round(x, cls.ROUND_DIGITS),
round(y, cls.ROUND_DIGITS),
round(z, cls.ROUND_DIGITS)
])
return clean_dat