def testMacho(self):
#
RESULTS_NUM = 1
queries = [{"Field": 1, "Tile": 3441, "Seqn": 25}]
client = StarsProvider().getProvider(obtain_method="Macho",
obtain_params=queries)
stars = client.getStars()
self.failUnless(len(stars) == RESULTS_NUM)
self.failUnless(isinstance(stars[0], Star))
def test_OgleIII():
que1 = {"ra": 72.798405, "dec": -69.00918, "delta": 5, "nearest": True}
que3 = {"field": "LMC135.5", "starid": 19670}
client = StarsProvider().getProvider("OgleIII", [que1, que3])
stars = client.getStars()
assert len(stars) == 2
assert isinstance(stars[0], Star)
check_lc(stars)
def test_CorotBright():
RESULTS_NUM = 4
queries = [{"ra": 102.707, "dec": -0.54089, "delta": 10}, {"CoRot": 116}]
client = StarsProvider().getProvider("CorotBright", queries)
stars = client.getStars(max_bins=100)
assert len(stars) == RESULTS_NUM
assert isinstance(stars[0], Star)
check_lc(stars, 4)
def testCorotFaint(self):
queries = [{
"Corot": "102706554"
}, {
"ra": 100.94235,
"dec": -00.89651,
"delta": 10
}]
client = StarsProvider().getProvider(obtain_method="CorotFaint",
obtain_params=queries)
stars = client.getStars(max_bins=10000)
def test_OgleIII():
que1 = {"ra": 72.798405,
"dec": -69.00918, "delta": 5, "nearest": True}
que3 = {"field": "LMC135.5", "starid": 19670}
client = StarsProvider().getProvider("OgleIII", [que1, que3])
stars = client.getStars()
assert len(stars) == 2
assert len(stars[0].lightCurve.mag) == 370
assert len(stars[1].lightCurve.mag) == 370
def test_Macho():
RESULTS_NUM = 1
queries = [{"Field": 1, "Tile": 3441, "Seqn": 25}]
client = StarsProvider().getProvider("Macho", queries)
stars = client.getStars()
assert len(stars) == RESULTS_NUM
assert isinstance(stars[0], Star)
check_lc(stars)
assert len(stars) == 1
assert len(stars[0].light_curves) == 2
assert len(stars[0].light_curves[0].mag) == 1188
assert len(stars[0].light_curves[1].mag) == 676
def test_CorotFaint():
queries = [{
"Corot": "102706554"
}, {
"ra": 100.94235,
"dec": -00.89651,
"delta": 10
}]
client = StarsProvider().getProvider("CorotFaint", queries)
stars = client.getStars(max_bins=1000)
assert len(stars) > 0
assert isinstance(stars[0], Star)
check_lc(stars)
def test_FileManager():
save_stars_path = "/tmp/test_FileManage/"
path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir,
"lcc_web", "sample_data", "sample1")
stars = StarsProvider.getProvider("FileManager", {"path": path}).getStars()
assert len(stars) > 0
assert stars[0].lightCurve
saveStars(stars, path=save_stars_path)
stars_loaded = StarsProvider.getProvider("FileManager", {
"path": save_stars_path
}).getStars()
assert stars == stars_loaded
def testCorotBright(self):
RESULTS_NUM = 2
queries = [{
"ra": 102.707,
"dec": -0.54089,
"delta": 10
}, {
"CoRot": 116
}]
client = StarsProvider().getProvider(obtain_method="CorotBright",
obtain_params=queries)
stars = client.getStars(max_bins=100)
self.failUnless(len(stars) == RESULTS_NUM)
self.failIf(None in [st.lightCurve for st in stars])
def getStarsFromFolder(single_path, lcs_fold):
"""
Get stars from folder/s. If path is iterable (case that more folders were
given, light curves from that all folder will be loaded
Parameters
-----------
single_path : str
Name of the folder of lightcurves from "light_curve" directory (specified
in settings).
Returns
--------
stars : List of Star objects
Stars from the folder
"""
p, restr = _check_sample_name(single_path)
try:
st = StarsProvider().getProvider("FileManager", {
"path": os.path.join(lcs_fold, p)
}).getStars()
stars = _split_stars(st, restr)
except KeyError:
raise IOError("\n\nThere no folder with light curves named %s." % (p))
if not stars:
raise Exception(
"There are no stars in path with given restriction %s " %
single_path)
random.shuffle(stars)
return stars
def testOgle(self):
queries = [{
"starid": 2,
"field_num": 1,
"target": "lmc"
}, {
"ra": 5.545575 * 15,
"dec": -70.55272,
"delta": 30
}]
client = StarsProvider().getProvider(obtain_method="OgleII",
obtain_params=queries)
stars = client.getStars()
print len(stars)
plotStarsPicture(stars[1:])
self.failUnless(len(stars) == 2)
self.failUnless(isinstance(stars[0], Star))
def skip_test_Asas():
queries = [{
"ASAS": "000030-3937.5"
}, {
"ra": 10.08,
"dec": -39.625,
"delta": 20
}, {
"ra": 0.1251,
"dec": -39.6250,
"delta": 10
}]
client = StarsProvider().getProvider("Asas", queries)
stars = client.getStars()
assert len(stars) > 0
assert isinstance(stars[0], Star)
check_lc(stars)
def testAsas(self):
queries = [{
"ASAS": "000030-3937.5"
}, {
"ra": 10.08,
"dec": -39.625,
"delta": 20
}, {
"ra": 0.1251,
"dec": -39.6250,
"delta": 10
}]
client = StarsProvider().getProvider(obtain_method="Asas",
obtain_params=queries)
stars = client.getStars()
self.failIf(len(stars) == 0)
self.failUnless(isinstance(stars[0], Star))
print " ,".join([st.coo.to_string() for st in stars])
def test_Kepler():
RESULTS_NUM = 4
queries = [{
"ra": 297.8399,
"dec": 46.57427,
"delta": 10,
"nearest": True
}, {
"kic_num": 9787239
}, {
"kic_jkcolor": (0.3, 0.4),
"max_records": 2
}]
client = StarsProvider().getProvider("Kepler", queries)
stars = client.getStars()
assert not np.isnan(stars[1].lightCurve.getHistogram()[1]).any()
assert stars and len(stars) == RESULTS_NUM
assert stars[1].name == "KIC_9787239"
assert len(stars[1].lightCurve.time) == 1624
def getStarsFromRemoteDb(query, query_path):
"""
This method parsing the query text in order to return desired stars
from remote database.
Parameters
-----------
query : str
Query text contains db_key and query file separated by ':'
Returns
--------
List of Star objects
Example
-------
_getStarsFromRemoteDb("OgleII:query_file.txt") --> [Star objects]
query_file.txt:
#starid;field;target
1;1;lmc
10;1;smc
"""
try:
db_key, query_file = query.split(":")
except:
QueryInputError(
"Key for resolving stars source was not recognized:\n%s" % query)
queries = StatusResolver(os.path.join(query_path, query_file)).getQueries()
stars = []
for query in progressbar(queries, "Querying stars: "):
starsProvider = StarsProvider().getProvider(obtain_method=db_key,
obtain_params=query)
stars += starsProvider.getStars()
return stars
def testKepler(self):
# NOTE: Ok
RESULTS_NUM = 4
EQUAL = (1, 2)
queries = [{
"ra": 297.8399,
"dec": 46.57427,
"delta": 10,
"nearest": True
}, {
"kic_num": 9787239
}, {
"kic_jkcolor": (0.3, 0.4),
"max_records": 2
}]
client = StarsProvider().getProvider(obtain_method="Kepler",
obtain_params=queries)
stars = client.getStars()
self.failIf(np.NaN in stars[1].lightCurve.getHistogram()[1])
self.failUnless(stars and len(stars) == RESULTS_NUM)
def queryStar(self, query):
stars = StarsProvider.getProvider(self.db_connector, query).getStars()
status = {"found": [], "lc": [], "passed": [], "star_name": []}
passed_info = None
if stars:
stars_with_lc = []
for star in stars:
status["star_name"].append(star.name)
status["found"].append(True)
if star.lightCurve and len(star.lightCurve.mag):
status["lc"].append(True)
stars_with_lc.append(star)
else:
status["lc"].append(False)
# TODO: Support just one filter
# for star_filter in self.stars_filters:
if self.stars_filters:
passed_info = self.stars_filters[0].getAllPredictions(
stars_with_lc,
with_features=self.save_coords,
check_passing=True)
counter = 0
for i in range(len(stars)):
print("passed_info is none", passed_info is None)
if passed_info is None and status["lc"][i]:
status["passed"].append(True)
self.matchOccured(stars[i])
elif status["lc"][i]:
p = passed_info["passed"].values[counter]
status["passed"].append(p)
counter += 1
if p:
self.matchOccured(stars[i])
else:
status["passed"].append(False)
else:
status["found"].append(False)
status["lc"].append(False)
status["passed"].append(False)
status["star_name"].append("Noname")
self.uploadStatus(status, passed_info)
def setUp(self):
days_per_bin = 10
alphabet_size = 15
s_queries = [{"path": "quasars"}]
client = StarsProvider().getProvider(
obtain_method="FileManager", obtain_params=s_queries)
self.s_stars = client.getStars()
c_queries = [{"path": "some_stars"}]
client = StarsProvider().getProvider(
obtain_method="FileManager", obtain_params=c_queries)
self.c_stars = client.getStars()
self.N = int(np.mean([len(self.s_stars), len(self.c_stars)]))
self.lc_shape_descriptor = CurvesShapeDescr(self.s_stars[:self.N / 3],
days_per_bin, alphabet_size)
self.hist_shape_descriptor = HistShapeDescr(
self.s_stars[:self.N / 3], 10, alphabet_size)
self.qda_decider = SVCDec()
def main(project_settings, argv=None):
program_info = """ABOUT
The program downloads light curves from astronomical databases
which pass thru given filters (or all).
Database to query:
------------------
Database is specified by '-d' and name of connector class.
Note:
There is a overview of available connectors at the end (if it is
launched from command line without parameters)
Status file:
------------
Queries can be specified in the file where first
row starts with '#' and then there are keys for query a database.
Next rows consist of searched values. All columns are separated
by ';' (can be changed in settings).
Note:
Example files can be find in data/inputs/examples
Getting filter:
---------------
Filter is loaded from prepared filter object (learned). If it is desired
to load filter with certain parameters it can be also created by
tuning tool by giving one combination of parameters.
Note:
All classes which inherits BaseFilter class located
in the filters_imp package are considered as filters.
Data folder hierarchy:
-----------------------
Next to src/ (source) folder there is a data/ folder where all data files
are saved. All input/outputs are loaded/saved into a folder in data/.
This behaviour can be suppressed by entering word 'HERE:'
(e.g. 'HERE:path_of_the_file_with_its_name'). It forces to take relative
path from the directory of executing the script.
There are 5 main folders:
1. data/inputs/
Location of files of queries and files fro tuning parameters
2. data/light_curves/
Location of light curve subfolders.
3. data/star_filters/
Location where tuned filters is saved (or can be loaded by
filter_lcs script)
4. data/tuning_logs/
Location of output files from tuning - statistic for every combination
of parameters, graphs (probability distribution with train objects
and histograms).
5. data/databases/
Location of local db files (sqlite).
Running the program:
-------------------
By executing the script all inputs are verified and database is queried.
Light curves (if any) of stars passed thru filtering are saved into
'data/light_curves/' + your folder(specified via '-o') and stars are
saved into local database. So it is possible to load them with their
values or filter them by other filters.
Also result file is saved into the folder with light curves in format
'connector_name'_'filter_name'.txt.
(TODO)
It is possible to continue with unfinished query. If query file has
three more columns generated during the filtering about status of
particular queries the program will find last finished query and it will
continues form that point.
Examples
--------
*Just downloading a light curves:
For Ogle query file (named query.txt):
#starid;field_num;target
1;1;lmc
12;1;lmc
./filter_stars.py -i query.txt -o my_lc_folder -d "OgleII"
The light curves and status file will be saved into "data/light_curves/my_lc_folder" folder.
*With filtering
It is possible to insert more then one filter by adding combination
'-f' + filter_name multiple times as is shown in example below.
A command for executing searching light curves in OGLE database
with filtering:
./filter_stars.py -i query.txt -o out/ -d "OgleII" -f abbe_filter.conf -f vario_slope.pickel
"""
program_name = os.path.basename(sys.argv[0])
program_version = "v0.2"
program_build_date = "%s" % __updated__
program_version_string = '%%prog %s (%s)' % (program_version,
program_build_date)
program_longdesc = "Run script without params to get info about the program and list of available databases"
program_license = "Copyright 2016 Martin Vo"
if argv is None:
argv = sys.argv[1:]
try:
# setup option parser
parser = OptionParser(version=program_version_string,
epilog=program_longdesc,
description=program_license)
parser.add_option("-r",
"--run",
dest="run",
help="Name of this run (name of folder for results)",
type=str)
parser.add_option(
"-q",
"--query",
dest="query",
help="Name of the query file in %PROJECT_DIR/queries")
parser.add_option("-d",
"--database",
dest="db",
help="Searched database")
parser.add_option(
"-s",
"--coords",
dest="save_coords",
default="y",
help="Save params coordinates of inspected stars if 'y'.")
parser.add_option(
"-f",
"--filter",
dest="filt",
action="append",
default=[],
help=
"Name of the filter file in filters folder (%PROJECT_DIR/filters)")
# process options
opts, args = parser.parse_args(argv)
if not len(argv):
print program_info, "\n"
print json.dumps(StarsProvider().STARS_PROVIDERS.keys())
print "Run with '-h' in order to show params help\n"
return False
if opts.db not in StarsProvider().STARS_PROVIDERS:
print "Error: " + "Unresolved database %s \n" % opts.db
print json.dumps(StarsProvider().STARS_PROVIDERS.keys())
return False
# ------- Core ------
header = "#" + " " * 40 + \
"Light Curves Classifier - Filter stars" + " " * 30 + "#"
print "\n\n\t" + "#" * len(header)
print "\t#" + " " * (len(header) - 2) + "#"
print "\t" + header
print "\t#" + " " * (len(header) - 2) + "#"
UNFOUND_LIM = 2
print "Loading query..."
try:
resolver = StatusResolver(status_file_path=os.path.join(
project_settings.QUERIES, opts.query))
queries = resolver.getQueries()
except IOError:
raise IOError("Query file was not found")
except Exception as e:
print "Err:", e
raise QueryInputError("There is an issue in query file")
print "Loading filters"
star_filters = [
FiltersSerializer(filt_name,
project_settings.FILTERS).loadFilter()
for filt_name in opts.filt
]
if not star_filters:
filt_txt = ""
else:
filt_txt = [filt.__class__.__name__ for filt in star_filters]
if opts.save_coords == "y":
save_coords = True
else:
save_coords = False
prepare_run(project_settings.RESULTS, opts.run)
print _sum_txt(opts.db, len(resolver.status_queries), filt_txt)
searcher = StarsSearcher(
star_filters,
save_path=os.path.join(project_settings.RESULTS, opts.run, "lcs"),
save_lim=1,
stat_file_path=os.path.join(project_settings.RESULTS, opts.run,
"query_status.txt"),
obth_method=opts.db,
unfound_lim=UNFOUND_LIM,
save_coords=save_coords)
searcher.queryStars(queries)
except Exception, e:
print e, "\n\n"
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help")
return 2
def queryStar(self, query):
status = collections.OrderedDict(
(("found", False), ("lc", False), ("passed", False)))
try:
provider = StarsProvider().getProvider(self.obth_method, query)
if hasattr(provider, "multiproc"):
provider.multiproc = False
stars = provider.getStars()
except QueryInputError:
raise
except (KeyboardInterrupt, SystemExit):
raise
except Exception as e:
warn(str(e))
warn("Couldn't download any star for query %s" % query )
stars = []
# TODO: status attribute is rewrited and just status of the last star is noted
for one_star in stars:
status["found"] = True
contain_lc = True
try:
one_star.lightCurve.time
except AttributeError:
contain_lc = False
if contain_lc:
# TODO
if self.save_coords and self.stars_filters:
spc = self.stars_filters[
0].getSpaceCoordinates([one_star]).values
if len(spc):
self._saveCoords([one_star.name] + spc[0].tolist())
# Try to apply filters to the star
try:
passed = self.filterStar(one_star, query)
status["lc"] = True
status["passed"] = passed
except (KeyboardInterrupt, SystemExit):
raise
except IOError as err:
raise InvalidFilesPath(err)
except Exception as err:
self.failProcedure(query, err)
warn(
"Something went wrong during filtering:\n\t%s" % err)
else:
status["lc"] = False
status["passed"] = False
query["name"] = one_star.name
self.statusFile(query, status)
if not stars:
query["name"] = ""
self.statusFile(query, status)
return stars, status
def test_OgleII():
que2 = {"field": "LMC_SC3", "starid": 5}
queries = [{"field": "LMC_SC3", "starid": i} for i in range(1, 5)]
prov = StarsProvider.getProvider("OgleII", queries)
stars = prov.getStars()