def test():
descriptors = [AbbeValueDescr()]
# deciders = [NeuronDecider(maxEpochs=800)]
deciders = [LDADec(), QDADec(), GradBoostDec()]
s_stars = [Star(name="Searched_{}".format(i)) for i in range(100)]
c_stars = [Star(name="Contam_{}".format(i)) for i in range(100)]
x = np.linspace(0, 10, 100)
for st in s_stars:
st.putLightCurve([x, np.cos(x) - 0.5 + np.random.random_sample(100)])
for st in c_stars:
st.putLightCurve([x, np.exp(x * np.random.random_sample(100))])
filt = StarsFilter(descriptors, deciders)
filt.learn(s_stars, c_stars)
searcher = StarsSearcherRedis([filt],
db_connector="OgleII",
save_path="/tmp/test_stars")
queries = [{"field": "LMC_SC3", "starid": i} for i in range(1, 10)]
searcher.queryStars(queries)
run_workers(n_workers=1)
assert len(searcher.getPassedStars()) > 0
time.sleep(2)
assert len(searcher.getStatus()) == len(queries)
def _loadDatFiles(self, star_paths, numberOfFiles):
if not star_paths:
return []
stars = []
counter = 1
# Load every light curve and put it into star object
for singleFile in tqdm(star_paths[:numberOfFiles],
desc="Loading dat files:"):
if self.files_to_load and os.path.basename(
singleFile) not in self.files_to_load:
break
lc = LightCurve(self._loadLcFromDat(singleFile))
# Check if light curve is not empty
if (len(lc.mag) >= 1):
db_ident = self.parseFileName(singleFile)
if self.db_ident:
ident = {self.db_ident: {"name": db_ident}}
else:
ident = {"file": {"name": db_ident}}
star = Star(ident=ident)
star.starClass = self.star_class
star.putLightCurve(lc)
stars.append(star)
counter += 1
return stars
def setUp(self):
self.star1 = Star()
x = np.linspace(1, 10, 100)
self.star1.lightCurve = [x, np.sin(x)]
self.star2 = Star()
self.star2.lightCurve = [x, np.cos(x)]
self.star3 = Star()
self.star3.lightCurve = [x, np.cos(x + 0.5)]
def _createStar(self, data, keys, lc_opt, **kwargs):
"""
Create Star objects from query result
Parameters
----------
data : list
Result from query
keys : list
Name of columns of data
lc_opt : bool
Obtain light curves if True
Returns
--------
list
List of Star objects
"""
stars = []
for raw_star in data:
ident = {}
for key, value in self.IDENT_MAP.iteritems():
db_ident = {}
if isinstance(value, (list, tuple)):
for ide in value:
db_ident[ide] = raw_star[keys.index(ide)]
name = self.NAME.format(**db_ident)
else:
name = raw_star[keys.index(value)]
if not db_ident:
db_ident = None
ident[key] = {"name": name, "db_ident": db_ident}
more = {}
for key, value in self.MORE_MAP.iteritems():
more_item = raw_star[keys.index(key)]
more[value] = more_item
raw_star_dict = dict(zip(keys, raw_star))
star = Star(name=self.NAME.format(**raw_star_dict),
coo=(raw_star_dict[self.RA],
raw_star_dict[self.DEC]),
ident=ident,
more=more)
if lc_opt:
star.putLightCurve(self._getLightCurve(star=star,
file_name=raw_star_dict.get(
self.LC_FILE, None),
**kwargs))
stars.append(star)
return stars
def _createStars(self, header, rows, lc_tmp):
# [u'No', u'Field', u'StarID', u'RA', u'Decl', u'V', u'I', u'B']
# [u'No', u'Field', u'StarID', u'RA', u'Decl', u'I']
cols_map = self._parseHeader(header)
stars = []
for row in rows:
field = str(row[cols_map.get("field")])
starid = str(row[cols_map.get("starid")])
ra = float(row[cols_map.get("ra")])
dec = float(row[cols_map.get("dec")])
identifiers = {}
identifiers["Macho"] = row[cols_map.get("macho_id")]
identifiers["Asas"] = row[cols_map.get("asas_id")]
identifiers["OgleII"] = row[cols_map.get("ogle_ii_id")]
identifiers["GCVS"] = row[cols_map.get("gcvs_id")]
name = str(row[cols_map.get("name")])
ident = {"OgleIII": {"name": name,
"db_ident": {"field": field,
"starid": starid}}}
for ide, val in identifiers.iteritems():
if val != u"\xa0":
ident[ide] = {"name": str(val)}
query_ide = self._parseDbNames(ide, val)
if query_ide:
ident[ide]["db_ident"] = query_ide
more = {}
for col in self.MORE:
if cols_map.get(col) and cols_map.get(col):
val = row[cols_map.get(col)]
if val != u"\xa0":
try:
val = float(val)
except:
val = str(val)
more[col] = val
coo = (ra * 15, dec, "deg")
st = Star(ident, name, coo, more)
st.starClass = str(row[cols_map.get("type")])
if lc_tmp:
lc = self._getLc(name)
if lc and len(lc) != 0:
st.putLightCurve(np.array(lc), meta=self.LC_META)
stars.append(st)
return stars
def setUp(self):
self.descriptors = PackageReader.getClassesDict("descriptors")
self.stars = []
for i in range(self.n_stars):
star = Star()
x = np.linspace(0, 100, 100)
y = np.sin(x**i)
y[0] = np.nan
x[:20] = np.linspace(-30, -10, 20)
star.putLightCurve([x, y])
self.stars.append(star)
def _createStar(self, df, lc_opt, **kwargs):
"""
Create Star objects from query result
Parameters
----------
df :
lc_opt : bool
Obtain light curves if True
Returns
--------
list
List of Star objects
"""
stars = []
for _, _raw_star in df.iterrows():
raw_star_dict = _raw_star.to_dict()
ident = {}
for key, value in self.IDENT_MAP.items():
db_ident = {}
if isinstance(value, (list, tuple)):
for ide in value:
db_ident[ide] = raw_star_dict.get(ide)
name = self.get_name(db_ident)
else:
name = raw_star_dict.get(value)
if not db_ident:
db_ident = None
ident[key] = {"name": name, "db_ident": db_ident}
more = {}
for key, value in self.MORE_MAP.items():
more_item = raw_star_dict.get(key)
more[value] = more_item
star = Star(name=self.get_name(raw_star_dict),
coo=(raw_star_dict[self.RA],
raw_star_dict[self.DEC]),
ident=ident,
more=more)
if lc_opt:
star.putLightCurve(self._getLightCurve(star=star,
file_name=raw_star_dict.get(
self.LC_FILE, None),
**kwargs))
stars.append(star)
return stars
def parse_stars(fi):
stars = []
for st_fi in fi:
if st_fi.__dict__.get("_name", "").endswith(".dat"):
fi_io = StringIO(st_fi.read())
lc = FileManager._loadLcFromDat(fi_io)
st = Star(name=st_fi.__dict__["_name"].split(".")[0])
st.putLightCurve(lc)
stars.append(st)
else:
stars.append(FileManager._createStarFromFITS(fits.open(st_fi)))
return stars
def parseRawStar(self, raw_html, load_lc, multiple_lcs=False):
"""
Parse html retrieved from the query into Star objects
Parameters
----------
raw_html : str
Raw html retrieved from the query
load_lc : bool
load_lc : Option for downloading light curve
multiple_lcs : bool
If True search for all stars will be executed
Returns
-------
list
Star objects retrieved from the query
"""
# TODO: Multiple lcs loading. So far just the first star is retrieved
# if multiple_lcs:
# json_data = re.search('var dataSet0 = {(?P<json_data>.*)}', raw_html)
# else:
# json_data = re.search('var dataSet0 = {(?P<json_data>.*)}', raw_html)
json_data = re.search('var dataSet0 = {(?P<json_data>.*)}', raw_html)
if not json_data:
return []
json_data = json_data.group("json_data")
for to_quo in self.TO_QUO:
json_data = json_data.replace("{0}".format(to_quo),
'"{0}"'.format(to_quo))
star_id = re.search('ID=(?P<name>.*)&PLOT=plot',
raw_html).group("name")
json_data = eval("{%s}" % json_data)
star = Star(name=json_data.get("label"),
ident={"CRST": {
"name": star_id
}})
if load_lc:
lc = LightCurve(json_data["data"])
star.putLightCurve(lc)
return [star]
def setup():
descriptors = [AbbeValueDescr()]
# deciders = [NeuronDecider(maxEpochs=800)]
deciders = [LDADec(), QDADec()]
s_stars = [Star(name="Searched_{}".format(i)) for i in range(100)]
c_stars = [Star(name="Contam_{}".format(i)) for i in range(100)]
m_stars = [Star(name="Contam_{}".format(i)) for i in range(100)]
x = np.linspace(0, 10, 100)
for st in s_stars:
st.putLightCurve([x, np.cos(x) - 0.5 + np.random.random_sample(100)])
for st in c_stars:
st.putLightCurve([x, np.exp(x * np.random.random_sample(100))])
filt = StarsFilter(descriptors, deciders)
return s_stars, c_stars, m_stars, filt
def _parseStar(self, _star, lc):
"""Transform kplr Star object into package Star object"""
star = Star()
more = {}
ident = {}
data_dict = _star.__dict__
for key, value in data_dict.items():
if key in list(self.STAR_MORE_MAP.keys()):
more[self.STAR_MORE_MAP[key]] = value
elif key in list(self.IDENTIFIER.keys()):
ident[self.IDENTIFIER[key]] = {}
ident[self.IDENTIFIER[key]]["identifier"] = value
ident[self.IDENTIFIER[key]]["name"] = "kic_" + value
ra = data_dict.get(self.RA_IDENT)
dec = data_dict.get(self.DEC_IDENT)
star.name = "KIC_" + data_dict.get(self.NAME, "")
if lc:
star.lightCurve, _ = self._getLightCurve(_star, lim=1)
star.coo = (ra, dec)
star.ident = ident
star.more = more
return star
def _createStars(self, header, rows, lc_tmp):
# [u'No', u'Field', u'StarID', u'RA', u'Decl', u'V', u'I', u'B']
# [u'No', u'Field', u'StarID', u'RA', u'Decl', u'I']
cols_map = self._parseHeader(header)
stars = []
for row in rows:
field = str(row[cols_map.get("field")])
starid = int(row[cols_map.get("starid")])
ra = float(row[cols_map.get("ra")])
dec = float(row[cols_map.get("dec")])
colors = ["i_mag", "b_mag", "v_mag"]
more = {}
for col in colors:
if cols_map.get(col) and cols_map.get(col):
try:
more[col] = float(row[cols_map.get(col)])
except:
pass
name = field + "_" + str(starid)
coo = (ra * 15, dec, "deg")
ident = {
"OgleII": {
"name": name,
"db_ident": {
"field": field,
"starid": starid
}
}
}
st = Star(ident, name, coo, more)
if lc_tmp:
lc = self._getLc(field, starid, lc_tmp)
if lc and len(lc) != 0:
st.putLightCurve(np.array(lc), meta=self.LC_META)
stars.append(st)
return stars
class TestComparative(unittest.TestCase):
def setUp(self):
self.star1 = Star()
x = np.linspace(1, 10, 100)
x2 = x**2
self.star1.putLightCurve([x, x2 / x2[-1]])
self.star2 = Star()
self.star2.putLightCurve([x, np.cos(x)])
self.star3 = Star()
xx = np.linspace(1, 45, 600)
self.star3.putLightCurve([xx, np.cos(xx + 0.1) * xx])
def testCurveShape(self):
lcdes = CurvesShapeDescr([self.star2], 0.6, 10)
assert lcdes.getSpaceCoords([self.star3])[0] < lcdes.getSpaceCoords(
[self.star1])[0]
def testHistShape(self):
hist = HistShapeDescr([self.star2], 10, 5)
assert hist.getSpaceCoords([self.star3])[0] > hist.getSpaceCoords(
[self.star1])[0]
def testVarioShape(self):
vario = VariogramShapeDescr([self.star2], 10, 5)
assert vario.getSpaceCoords([self.star3]) > vario.getSpaceCoords(
[self.star1])
def parseRawStar(self, raw_html, load_lc):
json_data = re.search('var dataSet0 = {(?P<json_data>.*)}', raw_html)
if not json_data:
return []
json_data = json_data.group("json_data")
for to_quo in self.TO_QUO:
json_data = json_data.replace("{0}".format(to_quo),
'"{0}"'.format(to_quo))
star_id = re.search('ID=(?P<name>.*)&PLOT=plot',
raw_html).group("name")
json_data = eval("{%s}" % json_data)
star = Star(name=json_data.get("label"),
ident={"CRST": {
"name": star_id
}})
if load_lc:
lc = LightCurve(json_data["data"])
star.putLightCurve(lc)
return [star]
def setUp(self):
self.star1 = Star()
x = np.linspace(1, 10, 100)
x2 = x**2
self.star1.putLightCurve([x, x2 / x2[-1]])
self.star2 = Star()
self.star2.putLightCurve([x, np.cos(x)])
self.star3 = Star()
xx = np.linspace(1, 45, 600)
self.star3.putLightCurve([xx, np.cos(xx + 0.1) * xx])
def _createStarFromFITS(self, fits):
DB_NAME_END = "_name"
DB_IDENT_SEP = "_id_"
prim_hdu = fits[0].header
ra = prim_hdu.get(self.FITS_RA)
dec = prim_hdu.get(self.FITS_DEC)
ra_unit = prim_hdu.get(self.FITS_RA_UNIT)
dec_unit = prim_hdu.get(self.FITS_DEC_UNIT)
star = Star(name=prim_hdu.get(self.FITS_NAME),
coo=(ra, dec, (ra_unit, dec_unit)),
starClass=prim_hdu.get(self.FITS_CLASS))
ident = {}
more = {}
for db_name_key in list(prim_hdu.keys()):
if db_name_key.endswith(DB_NAME_END):
db_name = db_name_key[:-len(DB_NAME_END)]
ident[db_name] = {}
ident[db_name]["name"] = prim_hdu[db_name_key]
elif DB_IDENT_SEP in db_name_key:
db_name, ident_key = db_name_key.split(DB_IDENT_SEP)
if not ident[db_name].get("db_ident"):
ident[db_name]["db_ident"] = {}
ident[db_name]["db_ident"][ident_key] = prim_hdu[db_name_key]
elif db_name_key not in [
"SIMPLE", "BITPIX", "NAXIS", "EXTEND", self.FITS_RA,
self.FITS_DEC, self.FITS_RA_UNIT, self.FITS_DEC_UNIT,
self.FITS_NAME, self.FITS_CLASS
]:
more[db_name_key.lower()] = prim_hdu[db_name_key]
star.ident = ident
star.more = more
for lc_hdu in fits[1:]:
star.putLightCurve(self._createLcFromFits(lc_hdu))
fits.close()
return star
def setUp(self):
N = 20
x = np.linspace(0, 10, 100)
self.template = []
for ii in range(N):
st = Star(name="TemplateStar%i" % ii)
st.putLightCurve([x, np.cos(x) + np.random.normal(x) * 0.1])
self.template.append(st)
self.variables = []
for ii in range(N):
st = Star(name="VariableStar%i" % ii)
st.putLightCurve([x, np.sin(x) + np.random.normal(x) * 0.1])
self.variables.append(st)
self.noisy = []
for ii in range(N):
st = Star(name="NonvariableStar%i" % ii)
st.putLightCurve([x, np.random.normal(x) * 2])
self.noisy.append(st)