def create_estimator(self, estimator_type, path):
""" Creates and returns an estimator of indicated type that can be
found at indicated path.
Parameters
----------
estimator_type: :obj:`str`
Type of the estimator that has to be returned. Can be either
"sklearnestimator" or "kerasestimator" to create a scikit-learn
estimator from pickle or keras estimator from .hdf5 file
respectively.
path: :obj:`str`
Path to the estimator.
Returns
-------
estimator: :class:`~phenoai.conatiners.Estimator`
Class derived from :class:`~phenoai.conatiners.Estimator` of
indicated estimator type. """
if estimator_type == "sklearnestimator":
logger.debug(
"Create SklearnEstimator in EstimatorFactory instance")
return SklearnEstimator(path)
if estimator_type == "kerasestimator":
logger.debug("Create KerasEstimator in EstimatorFactory instance")
return KerasEstimator(path)
return None
def load(self):
""" Loads the estimator into the
:attr:`phenoai.estimators.SklearnEstimator.est` property from the
location stored in :attr:`phenoai.estimators.SklearnEstimator.path`.
"""
logger.debug("Loading estimator")
with open(self.path + "/estimator.pkl", 'rb') as f:
self.est = pkl.load(f)
def do_POST(self):
""" Takes care of the handling of HTTP POST requests made to PhenoAI
Performs a prediction query to PhenoAI via its
:meth:`phenoai.core.PhenoAI.run` method. Returns the resulting
:obj:`phenoai.containers.PhenoAIResults` object in the correct format.
Users dont have to interact with this method directly, it is
automatically called when needed. """
logger.info("Received POST request from {}".format(
self.client_address[0]))
logger.set_indent("+")
try:
# Get POST data
post = self.rfile.read(int(self.headers['Content-Length']))
post = post.decode('utf-8')
post = urllib.parse.parse_qs(post, keep_blank_values=1)
for k, p in post.items():
post[k] = p[0]
# Split by mode
if post["mode"] == "values":
results = self._do_post_values(post)
elif post["mode"] == "file":
results = self._do_post_file(post)
else:
raise exceptions.ServerException(
("Mode not recognized, should be either 'values' or "
"'file'. Provided was '{}'.").format(post['mode']))
if ("get_results_as_string" in post
and float(post["get_results_as_string"]) == 1.0):
logger.debug("Converting results object to string")
# Return lists of results, not PhenoAIResults object
results_txt = self.convert_result_object_to_string(results)
else:
# Convert to pickled instance
logger.debug("Encoding results object to pickle")
results_txt = io.pickle(results)
returndict = {"status": "ok", "results": results_txt}
except Exception as e:
x = traceback.format_exc()
logger.error(x)
returndict = {
"status": "error",
"type": str(type(e).__name__),
"message": str(e)
}
logger.info("Return results")
# Send response status code
self.send_response(200)
# Send headers
self.send_header('Content-type', 'text/html')
self.end_headers()
# Write content as utf-8 data
returntext = json.dumps(returndict)
self.wfile.write(bytes(returntext, "utf8"))
logger.set_indent("-")
def summary(self):
""" Prints a summary of the contents of this object """
logger.debug("Print report for AInalysisResults")
print("===========================")
print("|| PhenoAIResults report ||")
print("===========================")
print("Contains {} AInalysisResults object(s)".format(self.num()))
print("AInalysisResult IDs:")
for i in range(self.num()):
print(" - {} (length: {})".format(
self.get_ids()[i],
self.get(self.get_ids()[i]).num()))
print("Access AInalysisResults objects by id via PhenoAIResults[ id ]")
def _do_post_file(self, post):
""" Handle server queries when provided with a file
Should not be interacted with directly, but only through the do_POST
method of this class.
Parameters
----------
post: :obj:dict
Dictionary containing POST headers
Returns
-------
results: :obj:`PhenoAIResults`
Results of the prediction routine by the PhenoAI object
"""
logger.debug("Received file to be interpreted")
data_ids = ast.literal_eval(post["data_ids"])
ainalysis_ids = ast.literal_eval(post["ainalysis_ids"])
# Create files in tmp folder
filepath = "/tmp/{}.phenoai".format(utils.random_string(16))
filepath_interpreted = "/tmp/{}_interpreted.phenoai".format(
utils.random_string(16))
logger.debug("Calling run() on PhenoAI server instance")
try:
with open(filepath, "w") as tmpfile:
tmpfile.write(post['data'])
results = __serverinstance__.run(filepath,
map_data=bool(
float(post['mapping'])),
ainalysis_ids=ainalysis_ids,
data_ids=data_ids)
os.remove(filepath)
except Exception:
with open(filepath_interpreted, "w") as tmpfile:
tmpfile.write(post['data'])
results = __serverinstance__.run(filepath_interpreted,
map_data=bool(
float(post['mapping'])),
ainalysis_ids=ainalysis_ids,
data_ids=data_ids)
os.remove(filepath_interpreted)
if os.path.exists(filepath):
os.remove(filepath)
if os.path.exists(filepath_interpreted):
os.remove(filepath_interpreted)
return results
def predict(self, data):
""" Returns a prediction for the data by querying it to the loaded
estimator.
Parameters
----------
data: :obj:`numpy.ndarray`
Data to be queried to the estimator. Should have format
`(nDatapoints, nParameters)`.
Returns
-------
prediction: :obj:`numpy.ndarray`
Prediction by the loaded estimator for the provided data. Shape of
the array depends on the estimator. """
logger.debug("Querying estimator for prediction (predict)")
return self.est.predict(data)
def summary(self):
""" Prints a summary of the contents of this object """
logger.debug("Print report for AInalysisResults")
print("---------------------------")
print("| AInalysisResults report |")
print("---------------------------")
print("ID: {}".format(self.result_id))
print("Length: {}".format(self.num()))
# Information on data
contains_data = self.get_data() is not None
print("Contains data: {}".format(contains_data))
if contains_data:
print(" Data shape: {}".format(self.get_data().shape))
print(" Access via .get_data( args )")
# Information on Data ids
contains_data_ids = self.get_ids() is not None
print("Contains data IDs: {}".format(contains_data_ids))
if contains_data_ids:
print(" Data IDs length: {}".format(len(self.get_ids())))
print(" Get full list via .get_ids()")
print(" Use as argument in .get_*() methods")
# Information on data mapping
data_is_mapped = self.is_mapped()[0]
print("Data is mapped: {}".format(data_is_mapped))
if data_is_mapped:
if isinstance(self.is_mapped()[1], list):
print(" Data IDs length: {}".format(len(self.is_mapped()[1])))
elif isinstance(self.is_mapped()[1], np.ndarray):
print(" Data IDs shape: {}".format(self.is_mapped()[1].shape))
print(" Access via .is_mapped( args )")
# Information on Data ids
contains_predictions = self.get_predictions() is not None
print("Contains predictions: {}".format(contains_predictions))
if contains_predictions:
print(" Predictions shape: {}".format(
self.get_predictions().shape))
print(" Access via .get_predictions( args )")
def _do_post_values(self, post):
""" Handle server queries when provided bare values
Should not be interacted with directly, but only through the do_POST
method of this class.
Parameters
----------
post: :obj:dict
Dictionary containing POST headers
Returns
-------
results: :obj:`PhenoAIResults`
Results of the prediction routine by the PhenoAI object
"""
logger.debug("Received raw values")
# Predict lists of values
data = ast.literal_eval(post['data'])
if not "data_ids" in post:
data_ids = None
elif post["data_ids"] == "false" or post["data_ids"] == "False":
data_ids = None
else:
data_ids = ast.literal_eval(post["data_ids"])
if "ainalysis_ids" not in post:
ainalysis_ids = None
else:
ainalysis_ids = ast.literal_eval(post["ainalysis_ids"])
# Perform prediction
if ainalysis_ids == "all":
ainalysis_ids = None
logger.debug(("Calling run procedure of PhenoAI server " "instance"))
return __serverinstance__.run(np.array(data),
map_data=bool(float(post['mapping'])),
ainalysis_ids=ainalysis_ids,
data_ids=data_ids)
def predict_proba(self, data):
""" Returns a probability prediction for provided data by querying it
to the loaded estimator. Since only classifiers have this functionality
in scikit-learn, this method will return `None` if the estimator does
not have a :func:`prediction_proba` method itself.
Parameters
----------
data: :obj:`numpy.ndarray`
Data to be queried to the estimator. Should have format
`(nDatapoints, nParameters)`.
Returns
-------
prediction: :obj:`numpy.ndarray`, `None`
Prediction by the loaded estimator for the provided data. Shape of
the array depends on the estimator. If the loaded estimator does
not have a :func:`predict_proba` method itself, `None` will be
returned."""
logger.debug("Querying estimator for prediction (predict_proba)")
if hasattr(self.est, "predict_proba"):
return self.est.predict_proba(data)
logger.debug("Estimator has no predict_proba function")
return None
# 30 WARNING
# 40 ERROR
# 50 CRITICAL
# The setting below indicates that all messages with an importance of INFO or
# higher will be printed
logger.to_stream(lvl=20)
# Send output to file. The level of this output channel may differ from the lvl
# for the stream channel. Only a single filechannel can exist at a time.
logger.to_file("loggertest.out", lvl=0)
# Output a couple of messages
logger.debug("debug message")
logger.info("yo")
logger.warning("warning message")
logger.error("this is an error")
logger.critical("HELP!")
# Colour the output messages for the stream channel. The file channel is
# unaffected by this setting.
# This setting could also be put into the to_stream function via
# logger.to_stream(lvl=20, colour=True)
logger.colour_stream(True)
# Again print some messages
logger.debug("debug message")
def run(self, data, map_data=False, ainalysis_ids=None, data_ids=None):
""" Queries each added AInalysis for prediction on provided data
This run method forms the core functionality of PhenoAI objects. It
calls the run method of each of the added AInalyses with the provided
data and configuration arguments. Returns are returned in a
:obj:`phenoai.containers.PhenoAIResults` object.
The `map_data` argument is to be interpreted slightly differently here
then the map_data argument of AInalysis objects. This argument controls
if, for each of the AInalyses, the data is mapped before being
subjected to the prediction methods of the estimator in the AInalysis.
As such, this argument can either be True or False, just as the
`map_data` argument of the AInalysis run method. However, this run
method differs in the sense that is allows the entry "both", which
queries all AInalyses that allow mapping of data *twice*: once with
mapping and once without. These AInalyses thus have two
AInalysisResults instances in the returned
:obj:`phenoai.containers.PhenoAIResults` object, which can be
differentiated based on their ID: the mapped results instance has
"_mapped" appended to the ID name.
Parameters
----------
data: :obj:`numpy.ndarray`, :obj:`str`, :obj:`list(str)` Data that has
to be subjected to the estimator. Can be the raw data
(numpy.ndarray), the location of a file to be read by the built-in
file reader in the AInalysis or a list of file locations.
map_data: :obj:`bool`. Optional Determines if data has to be mapped
before prediction. Mapping uses the map_data method of this
AInalysis and follows the mapping procedure defined in the
configuration file for this AInalysis. Can also be "both", making
each mapping-allowing AInalyses to be queried twice: once with and
once without mapping. Mapped data returns an AInalysisResults
object with "_mapped" appended to the AInalysisResults ID. Default
is `False`.
ainalysis_ids: :obj:`list(str)`. Optional If set to a list of strings,
only AInalyses with an ID present in this list are queried for
prediction on provided data. If set to `None`, all AInalyses are
queried. Default is `None`.
data_ids: :obj:`list(float)`, :obj:`numpy.ndarray`. Optional List or
numpy.ndarray of IDs for the data points. By setting this value,
results can be extracted from the AInalysisResults object by using
the IDs in this list/array instead of by its location in the result
array. If `None`, this functionality will not be available. Default
is `None`."""
# Use PhenoAI object locally
# Create mapping iteration list
mapmodes = []
if map_data == "both":
mapmodes.append(True)
mapmodes.append(False)
logger.info("Running PhenoAI with mapmode 'both'")
else:
mapmodes.append(bool(map_data))
logger.info("Running PhenoAI with mapmode {}".format(
bool(map_data)))
# Create results object
results = containers.PhenoAIResults()
# Loop over ainalyses to request prediction
for ainalysis in self.ainalyses:
# Load estimator if not loaded already
loaded = True
if self.dynamic and not ainalysis.estimator.is_loaded():
logger.debug("Loading estimator of AInalysis dynamically")
ainalysis.estimator.load()
loaded = False
# Iterate over mapmodes
for mapmode in mapmodes:
# Test if this AInalysis should be queried
if (ainalysis_ids is not None
and ainalysis.ainalysis_id not in ainalysis_ids):
continue
# If multi mapmode skip mapmode True if AInalysis does not
# allow mapping
if (isinstance(ainalysis.configuration['mapping'], bool)
and ainalysis.configuration['mapping'] == 0.0
and mapmode and len(mapmodes) > 1):
continue
logger.info("Running AInalysis '{}' in map mode '{}'".format(
ainalysis.ainalysis_id, mapmode))
# Do prediction
logger.set_indent("+")
result = ainalysis.run(data,
map_data=mapmode,
data_ids=data_ids)
logger.set_indent("-")
# Alter id if multi map mode
if len(mapmodes) > 1:
if mapmode:
result.result_id += "_mapped"
# Add result to AIResultContainer container
results.add(result)
# Unload estimator
if not loaded:
logger.debug(("Clearing estimator of last AInalysis "
"from memory"))
ainalysis.estimator.clear()
logger.info("PhenoAI run finished, returning result")
# Return results object
return results