def _parseQueries(self, queries):
todel_queries = []
new_queries = []
for i, query in enumerate(queries):
if "coo" in query and isinstance(query["coo"], SkyCoord):
if not "delta" in query:
query["delta"] = self.DEFAULT_DELTA
todel_queries.append(i)
coo = query["coo"]
new_queries.append(
{"ra": coo.ra.degree, "dec": coo.dec.degree,
"delta": query["delta"], "target": "all"})
if "ra" in query and "dec" in query:
if not "delta" in query:
query["delta"] = self.DEFAULT_DELTA
if "target" not in query:
query["target"] = "all"
elif "starid" in query:
if "field" in query:
query["target"] = query["field"][:3].lower()
elif "field_num" in query and "target" in query:
query["field"] = query[
"target"].upper() + "_SC" + str(query["field_num"])
else:
raise QueryInputError("Unresolved target")
if "types" in query and sum([1 for star_type in query["types"] if not star_type in self.TYPES]):
raise QueryInputError("Invalid star type in the query.\nAvailable types: %s" % self.TYPES)
return [item for i, item in enumerate(
queries) if i not in todel_queries] + new_queries
def getFeatures(self, star):
"""
Get difference in symbolic space of the investigated star from the template
Parameters
-----------
star : lcc.entities.star.Star object
Star to process
Returns
-------
list
Difference in symbolic space of the investigated star from the template
"""
if hasattr(self, "meth"):
meth = self.meth
else:
meth = "average"
coords = [
x for x in self._filtOneStar(star, search_opt="all")
if x is not None
]
logging.debug("Coords: %s" % coords)
if meth == "closest":
return np.min(coords)
elif meth == "average":
return np.mean(coords)
elif meth.startswith("best"):
n = convert_input_value(meth[4:])
if isinstance(n, float):
n = int(len(coords) * n)
if isinstance(n, str):
n = 1
if not meth:
raise QueryInputError(
"""Unresolved coordinates calculation method. String 'best' has to
be followed by integer or float number""")
return np.mean(np.sort(coords)[:n])
else:
raise QueryInputError("Unresolved coordinates calculation method")
def getSpaceCoords(self, stars):
'''
Apply all filters and get their space coordinates
Parameters
-----------
stars : Star objects
Stars to filtering
Returns
--------
list
List of coordinates
'''
try:
meth = self.method
except AttributeError:
meth = "average"
space_coordinates = []
# PB for star in progressbar(stars,"Obtaining space coordinates: "):
for star in stars:
coords = [
x for x in self._filtOneStar(star, search_opt="all") if x
]
if meth == "closest":
space_coordinates.append(np.min(coords))
elif meth == "average":
space_coordinates.append(np.mean(coords))
elif meth.startswith("best"):
n = convert_input_value(meth[4:])
if isinstance(n, float):
n = int(len(coords) * n)
if not meth:
raise QueryInputError(
"""Unresolved coordinates calculation method. String 'best' has to
be followed by integer or float number""")
space_coordinates.append(np.mean(np.argsort(coords)[:n]))
else:
raise QueryInputError(
"Unresolved coordinates calculation method")
return space_coordinates
def getFeatures(self, star):
"""
Get color indexes
Parameters
-----------
star : lcc.entities.star.Star object
Star to process
Returns
-------
list
Color indexes of the investigated star
"""
this_coords = []
for col in self.colors:
if hasattr(col, "__iter__"):
if len(col) == 2:
mag1 = star.more.get(col[0])
mag2 = star.more.get(col[1])
if mag1 and mag2:
this_coords.append(float(mag2) - float(mag1))
else:
this_coords.append(None)
else:
raise QueryInputError(
"Colors have to be list of tuples of the length of two (second - first magnitude)")
else:
this_coords.append(star.more.get(col))
return this_coords
def getSpaceCoords(self, stars):
"""
Get list of desired colors
Parameters
-----------
stars : list of Star objects
Stars with color magnitudes in their 'more' attribute
Returns
-------
List of list of floats
"""
coords = []
for star in stars:
this_coords = []
for col in self.colors:
if hasattr(col, "__iter__"):
if len(col) == 2:
mag1 = star.more.get(col[0])
mag2 = star.more.get(col[1])
if mag1 and mag2:
this_coords.append(float(mag2) - float(mag1))
else:
this_coords.append(None)
else:
raise QueryInputError(
"Colors have to be list of tuples of the length of two (second - first magnitude)"
)
else:
this_coords.append(star.more.get(col))
coords.append(this_coords)
return coords
def getQueryType(self, query):
if "RA" in query and "Dec" in query:
return "coo"
elif "ID" in query:
return "id"
else:
raise QueryInputError("Unresolved query type")
def _parse_tun_query(one_param):
this_comb = {}
for key, value in one_param.items():
x = key.split(":")
if len(x) != 2:
raise QueryInputError(
"Cannot parse tuning params header: %s. It has to be composed with 'descriptor name':'param name'"
% key)
obj_name, col = x
descr = this_comb.get(obj_name)
if not descr:
this_comb[obj_name] = {}
if isinstance(value, str):
if value == "True":
value = True
elif value == "False":
value = False
elif value == "None":
value = None
elif value.strip().startswith("`") and value.strip().endswith("`"):
try:
value = ast.literal_eval(value.strip()[1:-1])
except Exception as e:
warnings.warn(str(e))
try:
value = value.strip()[1:-1]
except:
pass
this_comb[obj_name][col] = value
return this_comb
def getStars(queries, lcs_fold, query_path=None, progb_txt="Querying stars: "):
"""
Get stars from query text. According to format of the query text different
methods are called.
1.QUERY:db_name:query_file_in_inputs_folder
--> Remote database is queried (db key is name of connector class)
2.stars_folder_key:number or stars_folder_key:float_number or stars_folder_key
--> Light curves from folder according to first key is loaded
(according to settings.STARS_PATH dictionary). All stars are
loaded if there is no number and ':', in case of integer after
':' just this number of stars are loaded and if there are float
number after ':' this percentage number of all stars are loaded.
"""
ORDINARY_QUERY_KEY = "QUERY:"
stars = []
for query in tqdm(queries, desc=progb_txt):
query = query.strip()
if query.startswith(ORDINARY_QUERY_KEY):
stars += getStarsFromRemoteDb(query[len(ORDINARY_QUERY_KEY):],
query_path)
else:
stars += getStarsFromFolder(query, lcs_fold)
if not stars:
raise QueryInputError("There no stars. Your query: %s" % queries)
return stars
def _checkDimensions(self, coords):
expected_dim = len(self.boundaries)
dim = len(coords[0])
if expected_dim != dim:
raise QueryInputError(
"Dimension of the decider boundaries (dim: %i) and given coordinates (dim: %i) dont match.\nGot: %s"
% (expected_dim, dim, coords))
def learn(self, coords):
coords = [c for c in coords if not np.NaN in c and not None in c]
if coords:
self.X = np.array(coords)
self.classifier.fit(coords)
else:
raise QueryInputError("No coordinates for learning")
def _get_where_text(self):
"""Get WHERE part for query"""
where_text = "WHERE "
for _condition in self.conditions:
condition = self._transfromCoo(_condition)
condition = [self._quoteIfNeeded(cond) for cond in condition]
if type(condition[1]) is tuple:
condition = (condition[0], condition[1][0], condition[1][1])
if len(condition) == 3:
where_text += "({0} BETWEEN {1} AND {2}) AND ".format(*
condition)
elif len(condition) == 2:
if condition[1].strip().startswith("'") or condition[1].strip().startswith('"'):
cleaned_cond = condition[1].strip()[1:-1]
else:
cleaned_cond = condition[1].strip()
if cleaned_cond[0] in self.SPECIAL_SYMB:
where_text += "({0} {1}) AND ".format(
condition[0], cleaned_cond)
else:
where_text += "({0} = {1}) AND ".format(*condition)
else:
raise QueryInputError(
"Unresolved TAP query condition: %s" % condition)
where_text = where_text[:-4]
return where_text
def _reduceDimension(self, data):
try:
if not self.pca:
self.pca = decomposition.PCA(n_components=self.red_dim)
self.pca.fit(data)
return self.pca.transform(data).tolist()
except ValueError as e:
raise QueryInputError(str(e))
def learn(self, searched, others):
"""
This method loads lists of specific values of searched objects and
others. Then the sample will be divided into train and
test samples according to user.
Parameters
-----------
searched : iterable
List of searched objects values (their "coordinates")
others : iterable
List of other objects values (their "coordinates")
Returns
-------
NoneType
None
"""
if not len(searched) or not len(others):
raise QueryInputError(
"Decider can't be learned on an empty sample")
# Input is accepted as a numpy array or as a list
if isinstance(searched, np.ndarray):
try:
searched = searched.tolist()
others = others.tolist()
except AttributeError as err:
raise AttributeError("Wrong coordinates input: %s" % err)
elif not isinstance(searched, list):
raise AttributeError("Input type ({}) not supported".format(
type(searched)))
X = np.array(searched + others)
# Note searched objects as 1 and others as 0
y = np.array([1 for i in range(len(searched))] +
[0 for i in range(len(others))])
dim = X.shape[1]
model = Sequential()
model.add(Dense(self.hiden_neurons, input_dim=dim, activation="relu"))
model.add(Dense(1, activation="sigmoid"))
# Compile model
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
# Fit the model
self.history = model.fit(X, y, epochs=150, batch_size=10)
self.model = model
logging.info("Training of NN successfully finished")
def _parseQueries(self, queries):
todel_queries = []
new_queries = []
for i, query in enumerate(queries):
if "db" not in query:
query["db"] = self.QUERY_TYPES[0]
if "coo" in query and isinstance(query["coo"],
SkyCoord) and "delta" in query:
todel_queries.append(i)
coo = query["coo"]
query["ra"] = coo.ra.degree
query["dec"] = coo.dec.degree
if "ra" in query and "dec" in query and "target" not in query:
todel_queries.append(i)
if query["db"] == "phot":
targets = self.PHOT_TARGETS
else:
targets = self.BVI_TARGETS
for target in targets:
z = query.copy()
z["target"] = target
new_queries.append(z)
elif "starid" in query:
if "field" in query:
query["target"] = query["field"][:3].lower()
elif "field_num" in query and "target" in query:
query["field"] = query["target"].upper() + "_SC" + str(
query["field_num"])
else:
raise QueryInputError("Unresolved target")
if query["db"] not in self.QUERY_TYPES:
raise QueryInputError(
"Invalid db. Available OgleII databases: %s" %
self.QUERY_TYPES)
return [
item for i, item in enumerate(queries) if i not in todel_queries
] + new_queries
def __init__(self,
stars_filters,
save_path=None,
stat_file_path=None,
db_connector=None,
save_coords=False,
multiproc=False):
"""
Parameters
----------
stars_filters : lists
Stars filters
save_path : bool, str
Path from "run" module to the folder where found
light curves will be saved. If False nothing is saved.
stat_file_path : str
Status file name
db_connector : str
Name of connector class
save_coords : bool
Save params space coordinates of inspected stars
multiproc : bool, int
If True task will be distributed into threads by using all cores. If it is number,
just that number of cores are used
"""
if not db_connector:
raise QueryInputError(
"Database for searching need to be specified.")
save_path = save_path or os.path.join(tempfile.gettempdir(), "lcc",
"stars")
stat_file_path = stat_file_path or os.path.join(
tempfile.gettempdir(), "lcc", "status_file.csv")
self.save_path = save_path
self.stat_file_path = stat_file_path
self.db_connector = db_connector
self.stars_filters = stars_filters
if multiproc:
warnings.warn("Multiprocessing not supported in current version")
multiproc = False
self.multiproc = multiproc
self.save_coords = save_coords
if os.path.exists(stat_file_path):
warnings.warn(
"Removing existing status file {}".format(stat_file_path))
os.remove(stat_file_path)
def _getStars(self, que, save_lc=True):
"""Get stars from one query"""
kic_num = que.get("kic_num", None)
ra = que.get("ra", None)
dec = que.get("dec", None)
delta = que.get("delta", None)
if kic_num:
_stars = [self.client.star(kic_num)]
self.delta = None
else:
if ra and dec and delta:
try:
delta = delta / 3600.0
self.ra, self.dec, self.delta = ra, dec, delta
except:
raise QueryInputError(
"Coordinates parameters conversion to float has failed"
)
query = {
"kic_degree_ra": "%f..%f" % (ra - delta, ra + delta),
"kic_dec": "%f..%f" % (dec - delta, dec + delta)
}
else:
query = {}
for key, value in que.items():
if hasattr(value, "__iter__"):
query[key] = "%s..%s" % (value[0], value[1])
else:
query[key] = value
try:
_stars = self.client.stars(**query)
except:
raise QueryInputError("Unresolved query.\n%s" % query)
return [self._parseStar(_star, save_lc) for _star in _stars]
def learn(self, right_coords, wrong_coords):
"""
Learn to recognize objects
Parameters
-----------
right_coords: iterable
List of coordinates (list of numbers) of searched objects
wrong_coords: iterable
List of coordinates (list of numbers) of contamination objects
Returns
--------
NoneType
None
"""
right_coords = list(right_coords)
wrong_coords = list(wrong_coords)
if not len(right_coords) or not len(wrong_coords):
raise QueryInputError(
"Decider can't be learned on an empty sample\nGot\tsearched:%s\tothers%s" % (right_coords, wrong_coords))
y = [1 for i in range(len(right_coords))]
y += [0 for i in range(len(wrong_coords))]
self.X = np.array(right_coords + wrong_coords)
self.y = np.array(y)
if not self.X.any() or not self.y.any():
raise QueryInputError(
"No stars have an attribute which are needed by filter")
try:
self.learner.fit(self.X, self.y)
except Exception as e:
raise LearningError(str(e) +
"\nCould not learn decider on the dataset:\nX = %s\n\nlabels = %s" % (self.X, self.y))
def makeDesc(descriptors, _params):
ready_descriptors = []
for i, des in enumerate(descriptors):
try:
params = _params.get(des.__name__, {})
ready_descriptors.append(des(**params))
except TypeError:
raise QueryInputError(
"Not enough parameters to construct constructor {0}\nGot: {1}".
format(des.__name__, params))
return ready_descriptors
def _get_select_text(self):
"""Get SELECT part for query"""
if isinstance(self.select, (list, tuple, set)):
select_text = "SELECT "
for sel in set(self.select):
if sel:
select_text += '"%s", ' % sel
select_text = select_text[:-2] + " "
elif isinstance(self.select, str):
select_text = "SELECT %s " % self.select
else:
raise QueryInputError(
"Select option was not resolved for TAP query\n%s" % self.select)
return select_text
def __init__(self, stars_filters, save_path=False, stat_file_path=None,
obth_method=None, save_coords=None, multiproc=True):
"""
Parameters
----------
stars_filters : lists
Stars filters
save_path : bool, str
Path from "run" module to the folder where found
light curves will be saved. If False nothing is saved.
stat_file_path : str
Status file name
obth_method : str
Name of connector class
save_coords : bool
Save params space coordinates of inspected stars
multiproc : bool, int
If True task will be distributed into threads by using all cores. If it is number,
just that number of cores are used
"""
if not obth_method:
raise QueryInputError(
"Database for searching need to be specified.")
self.save_path = save_path
self.stat_file_path = stat_file_path
self.obth_method = obth_method
self.stars_filters = stars_filters
self.not_uploaded = []
self.passed_stars = []
if save_coords:
self.que_coords = None
self.save_coords = save_coords
self.status = pd.DataFrame()
self.multiproc = multiproc
self.overview = []
self.stars = []
def coneSearch(self, coo, stars, delta_deg, nearest=False):
"""
Filter results from cone search
Parameters
----------
coo : astropy.coordinates.sky_coordinate.SkyCoord
Center of searching
stars : list of `Star` objects
Stars returned by query
delta_deg: float, astropy.units.quantity.Quantity
Radius from center of searching
nearest : bool
Nearest star to the center of searching is returned if it is True
Returns
--------
list
List of `Star` objects
"""
try:
if not isinstance(delta_deg, u.quantity.Quantity):
delta_deg = float(delta_deg) * u.deg
distances = []
passed_stars = []
for star in stars:
if star.coo:
dist = coo.separation(star.coo)
if dist < delta_deg:
passed_stars.append(star)
distances.append(dist.degree)
else:
passed_stars.append(star)
distances.append(np.inf)
except AttributeError:
raise QueryInputError("Invalid query coordinates")
if distances and (nearest or str(nearest).capitalize() == "True"):
return [passed_stars[np.argmin(distances)]]
return passed_stars
def filterStars(self, stars, pass_method="all"):
"""
Apply all deciders
Parameters
----------
stars : list, iterable
Star objects to be filtered
pass_method : str
Inspected star pass if it fulfill the selected condition.
Methods for filtering:
all - all probabilities have to be greater then the threshold
mean - mean probability has to be greater then the threshold
one - at least one has to be greater then the threshold
Returns
-------
list of `Star`s
Stars which passed thru filtering
"""
stars_coords = self.getSpaceCoordinates(stars)
threshold = np.mean([dec.threshold for dec in self.deciders])
if pass_method == "all":
probabilities = self.evaluateCoordinates(stars_coords, "lowest")
elif pass_method == "mean":
probabilities = self.evaluateCoordinates(stars_coords, "mean")
elif pass_method == "one":
probabilities = self.evaluateCoordinates(stars_coords, "highest")
else:
raise QueryInputError("Invalid filtering method")
return [
stars[i] for i, probab in enumerate(probabilities)
if probab >= threshold
]
def getQueryType(self, query):
"""
Resolve query type
Parameters
----------
query : dict
Db query
Returns
-------
str
Query type key
"""
if "RA" in query and "Dec" in query:
return "coo"
elif "ID" in query:
return "id"
else:
raise QueryInputError("Unresolved query type")
def evaluate(self, combination):
"""
Parameters
----------
combination : dict
Dictionary of dictionaries - one per a descriptor.
EXAMPLE
{'AbbeValue': {'bin':10, .. }, .. }
Returns
-------
tuple
Stars filter, statistical values
"""
descriptors = []
deciders = []
n = len(self.descriptors)
for i, des in enumerate(self.descriptors + self.deciders):
try:
static_params = self.static_params.get(des.__name__, {})
_params = combination.get(des.__name__, {})
params = _params.copy()
params.update(static_params)
if i < n:
descriptors.append(des(**params))
else:
deciders.append(des(**params))
except TypeError:
raise QueryInputError(
"Not enough parameters to construct constructor {0}\nGot: {1}"
.format(des.__name__, params))
stars_filter = StarsFilter(descriptors, deciders)
stars_filter.learn(self.searched_train, self.others_train)
stat = stars_filter.getStatistic(self.searched_test, self.others_test)
return stars_filter, stat
def learn(self, searched, others):
"""
This method loads lists of specific values of searched objects and
others. Then the sample will be divided into train and
test samples according to user.
Parameters
-----------
searched : iterable
List of searched objects values (their "coordinates")
others : iterable
List of other objects values (their "coordinates")
Returns
-------
NoneType
None
"""
if not len(searched) or not len(others):
raise QueryInputError(
"Decider can't be learned on an empty sample")
# Resolve number of input neurons
self.input_neurons = len(searched[0])
# Input is accepted as a numpy array or as a list
if type(searched) != list:
try:
X = searched.tolist() + others.tolist()
except AttributeError as err:
raise AttributeError("Wrong coordinates input: %s" % err)
elif type(searched) == list:
X = np.array(searched + others)
# Note searched objects as 1 and others as 0
self.y = np.array([1 for i in range(len(searched))] +
[0 for i in range(len(others))])
self.X = X
self.train()
def getStarsFromRemoteDb(query, query_path):
"""
This method parsing the query text in order to return desired stars
from remote database.
Parameters
-----------
query : str
Query text contains db_key and query file separated by ':'
Returns
--------
List of Star objects
Example
-------
_getStarsFromRemoteDb("OgleII:query_file.txt") --> [Star objects]
query_file.txt:
#starid;field;target
1;1;lmc
10;1;smc
"""
try:
db_key, query_file = query.split(":")
except:
QueryInputError(
"Key for resolving stars source was not recognized:\n%s" % query)
queries = StatusResolver(os.path.join(query_path, query_file)).getQueries()
stars = []
for query in progressbar(queries, "Querying stars: "):
starsProvider = StarsProvider().getProvider(obtain_method=db_key,
obtain_params=query)
stars += starsProvider.getStars()
return stars
def __init__(self, boundaries):
'''
Parameters
----------
boundaries : list, iterable
List of tuples of two values - lower and higher border value.
If one of these value is None there is no lower/upper limit.
Example
-------
[(1,10), (5,None), (None,8)]
First coordinate means "something between 1 and 10, the second
means greater then 5 and the last one means something lower
then 8
'''
if [x for x in boundaries if len(x) != 2]:
raise QueryInputError(
"List of boundaries have to be consist of tuples of two values"
)
self.boundaries = boundaries
self.threshold = 0.5
def evaluateCoordinates(self, stars_coords, meth="mean"):
"""
Get probability of membership calculated from all deciders
Parameters
----------
stars_coords : list, iterable
List of coordinates (lists)
meth : str
Method for filtering:
mean - mean probability
highest - highest probability
lowest - lowest probability
Returns
-------
list
Probabilities of membership according to selected method
"""
decisions = []
for decider in self.deciders:
decisions.append(decider.evaluate(stars_coords))
if meth == "mean":
return [round(np.mean(coo), 2) for coo in np.array(decisions).T]
elif meth == "highest":
return [round(np.max(coo), 2) for coo in np.array(decisions).T]
elif meth == "lowest":
return [round(np.min(coo), 2) for coo in np.array(decisions).T]
else:
raise QueryInputError(
"Invalid method for calculating membership probability")
def getStars(self, load_lc=True, **kwargs):
"""
Get star objects with light curves
Parameters
----------
load_lc : bool
Star is appended by light curve if True
kwargs : dict
Optional parameters which have effect just if certain database
provides this option.
For example CoRoT archive contains very large light curves,
so the dimension of light curve can be reduced by `max_bins`
keyword.
Returns
--------
list
List of stars with their light curves
"""
select = set([self.RA, self.DEC, self.LC_FILE] + list(self.MORE_MAP.keys()))
for val in self.IDENT_MAP.values():
if isinstance(val, (tuple, list, set)):
for it in val:
select.add(it)
else:
select.add(val)
select = [s for s in select if s]
select = list(select)
raw_stars = pd.DataFrame()
for _que in self.queries:
que = _que.copy()
if "ra" in que and "dec" in que:
que[self.RA] = que.pop("ra")
que[self.DEC] = que.pop("dec")
if "delta" in que:
delta = que.pop("delta")
que[self.RA], que[self.DEC] = self._areaSearch(
que[self.RA], que[self.DEC], delta)
conditions = []
for key, value in que.items():
if isinstance(value, (list, tuple)):
if len(value) == 2:
conditions.append((key, value[0], value[1]))
else:
raise QueryInputError("Invalid query range")
else:
if key != "nearest":
conditions.append((key, value))
query_inp = {"table": self.TABLE,
"select": select,
"conditions": conditions,
"URL": self.TAP_URL}
res = self.postQuery(query_inp)
if not res.empty:
raw_stars = pd.concat([raw_stars, res])
return self._createStar(raw_stars, load_lc, **kwargs)
def main(project_settings, argv=None):
program_info = """ABOUT
The program downloads light curves from astronomical databases
which pass thru given filters (or all).
Database to query:
------------------
Database is specified by '-d' and name of connector class.
Note:
There is a overview of available connectors at the end (if it is
launched from command line without parameters)
Status file:
------------
Queries can be specified in the file where first
row starts with '#' and then there are keys for query a database.
Next rows consist of searched values. All columns are separated
by ';' (can be changed in settings).
Note:
Example files can be find in data/inputs/examples
Getting filter:
---------------
Filter is loaded from prepared filter object (learned). If it is desired
to load filter with certain parameters it can be also created by
tuning tool by giving one combination of parameters.
Note:
All classes which inherits BaseFilter class located
in the filters_imp package are considered as filters.
Data folder hierarchy:
-----------------------
Next to src/ (source) folder there is a data/ folder where all data files
are saved. All input/outputs are loaded/saved into a folder in data/.
This behaviour can be suppressed by entering word 'HERE:'
(e.g. 'HERE:path_of_the_file_with_its_name'). It forces to take relative
path from the directory of executing the script.
There are 5 main folders:
1. data/inputs/
Location of files of queries and files fro tuning parameters
2. data/light_curves/
Location of light curve subfolders.
3. data/star_filters/
Location where tuned filters is saved (or can be loaded by
filter_lcs script)
4. data/tuning_logs/
Location of output files from tuning - statistic for every combination
of parameters, graphs (probability distribution with train objects
and histograms).
5. data/databases/
Location of local db files (sqlite).
Running the program:
-------------------
By executing the script all inputs are verified and database is queried.
Light curves (if any) of stars passed thru filtering are saved into
'data/light_curves/' + your folder(specified via '-o') and stars are
saved into local database. So it is possible to load them with their
values or filter them by other filters.
Also result file is saved into the folder with light curves in format
'connector_name'_'filter_name'.txt.
(TODO)
It is possible to continue with unfinished query. If query file has
three more columns generated during the filtering about status of
particular queries the program will find last finished query and it will
continues form that point.
Examples
--------
*Just downloading a light curves:
For Ogle query file (named query.txt):
#starid;field_num;target
1;1;lmc
12;1;lmc
./filter_stars.py -i query.txt -o my_lc_folder -d "OgleII"
The light curves and status file will be saved into "data/light_curves/my_lc_folder" folder.
*With filtering
It is possible to insert more then one filter by adding combination
'-f' + filter_name multiple times as is shown in example below.
A command for executing searching light curves in OGLE database
with filtering:
./filter_stars.py -i query.txt -o out/ -d "OgleII" -f abbe_filter.conf -f vario_slope.pickel
"""
program_name = os.path.basename(sys.argv[0])
program_version = "v0.2"
program_build_date = "%s" % __updated__
program_version_string = '%%prog %s (%s)' % (program_version,
program_build_date)
program_longdesc = "Run script without params to get info about the program and list of available databases"
program_license = "Copyright 2016 Martin Vo"
if argv is None:
argv = sys.argv[1:]
try:
# setup option parser
parser = OptionParser(version=program_version_string,
epilog=program_longdesc,
description=program_license)
parser.add_option("-r",
"--run",
dest="run",
help="Name of this run (name of folder for results)",
type=str)
parser.add_option(
"-q",
"--query",
dest="query",
help="Name of the query file in %PROJECT_DIR/queries")
parser.add_option("-d",
"--database",
dest="db",
help="Searched database")
parser.add_option(
"-s",
"--coords",
dest="save_coords",
default="y",
help="Save params coordinates of inspected stars if 'y'.")
parser.add_option(
"-f",
"--filter",
dest="filt",
action="append",
default=[],
help=
"Name of the filter file in filters folder (%PROJECT_DIR/filters)")
# process options
opts, args = parser.parse_args(argv)
if not len(argv):
print program_info, "\n"
print json.dumps(StarsProvider().STARS_PROVIDERS.keys())
print "Run with '-h' in order to show params help\n"
return False
if opts.db not in StarsProvider().STARS_PROVIDERS:
print "Error: " + "Unresolved database %s \n" % opts.db
print json.dumps(StarsProvider().STARS_PROVIDERS.keys())
return False
# ------- Core ------
header = "#" + " " * 40 + \
"Light Curves Classifier - Filter stars" + " " * 30 + "#"
print "\n\n\t" + "#" * len(header)
print "\t#" + " " * (len(header) - 2) + "#"
print "\t" + header
print "\t#" + " " * (len(header) - 2) + "#"
UNFOUND_LIM = 2
print "Loading query..."
try:
resolver = StatusResolver(status_file_path=os.path.join(
project_settings.QUERIES, opts.query))
queries = resolver.getQueries()
except IOError:
raise IOError("Query file was not found")
except Exception as e:
print "Err:", e
raise QueryInputError("There is an issue in query file")
print "Loading filters"
star_filters = [
FiltersSerializer(filt_name,
project_settings.FILTERS).loadFilter()
for filt_name in opts.filt
]
if not star_filters:
filt_txt = ""
else:
filt_txt = [filt.__class__.__name__ for filt in star_filters]
if opts.save_coords == "y":
save_coords = True
else:
save_coords = False
prepare_run(project_settings.RESULTS, opts.run)
print _sum_txt(opts.db, len(resolver.status_queries), filt_txt)
searcher = StarsSearcher(
star_filters,
save_path=os.path.join(project_settings.RESULTS, opts.run, "lcs"),
save_lim=1,
stat_file_path=os.path.join(project_settings.RESULTS, opts.run,
"query_status.txt"),
obth_method=opts.db,
unfound_lim=UNFOUND_LIM,
save_coords=save_coords)
searcher.queryStars(queries)
except Exception, e:
print e, "\n\n"
indent = len(program_name) * " "
sys.stderr.write(program_name + ": " + repr(e) + "\n")
sys.stderr.write(indent + " for help use --help")
return 2
