def render_save(self, varlist, outputfile=None):
"""Render and save to file"""
rendered_str = self.render(varlist)
out = Output(outputfile)
out.write(rendered_str)
out.close()
return outputfile
def walk(self):
self.co.create()
output = Output(self.kind)
output.now_dir()
print '[+] now path: ' + str(output.nowpath())
output.out_dir()
output.out_file()
print '[+] target path: ' + str(self.__path)
header = "target : " + self.__path + '\n'
print '[+] please wait ...'
output.write(header)
for root, dirs, files in os.walk(self.__path):
level = root.replace(self.__path, '').count(os.sep)
indent = ' ' * 4 * (level)
out = '{0}--=> {1}/'.format(indent, os.path.basename(root)) + '\n'
output.write(out)
subindent = ' ' * 4 * (level + 1)
for f in files:
#c_path = os.path.basename(root) + '/' + f
f_name = f
f_path = os.path.join(root, f)
if self.checking(f_path) == 2:
f_md5 = self.md5(f_path)
self.co.into(self.kind, f_name, f_path, f_md5)
out = '{0}=> {1}'.format(subindent, f) + '\n'
output.write(out)
self.co.close()
output.close()
def run(*datasources, **options):
"""Executes given Robot data sources with given options.
Data sources are paths to files and directories, similarly as when running
pybot/jybot from command line. Options are given as keywords arguments and
their names are same as long command line options without hyphens.
Examples:
run('/path/to/tests.html')
run('/path/to/tests.html', '/path/to/tests2.html', log='mylog.html')
Equivalent command line usage:
pybot /path/to/tests.html
pybot --log mylog.html /path/to/tests.html /path/to/tests2.html
"""
STOP_SIGNAL_MONITOR.start()
settings = RobotSettings(options)
LOGGER.register_console_logger(settings['MonitorWidth'],
settings['MonitorColors'])
init_global_variables(settings)
suite = TestSuite(datasources, settings)
output = Output(settings)
suite.run(output)
LOGGER.info("Tests execution ended. Statistics:\n%s"
% suite.get_stat_message())
output.close(suite)
if settings.is_rebot_needed():
output, settings = settings.get_rebot_datasource_and_settings()
ResultWriter(settings).write_robot_results(output)
LOGGER.close()
return suite
def run(*datasources, **options):
"""Executes given Robot data sources with given options.
Data sources are paths to files and directories, similarly as when running
pybot/jybot from command line. Options are given as keywords arguments and
their names are same as long command line options without hyphens.
Examples:
run('/path/to/tests.html')
run('/path/to/tests.html', '/path/to/tests2.html', log='mylog.html')
Equivalent command line usage:
pybot /path/to/tests.html
pybot --log mylog.html /path/to/tests.html /path/to/tests2.html
"""
STOP_SIGNAL_MONITOR.start()
settings = RobotSettings(options)
LOGGER.register_console_logger(settings['MonitorWidth'],
settings['MonitorColors'])
output = Output(settings)
init_global_variables(settings)
suite = TestSuite(datasources, settings)
suite.run(output)
LOGGER.info("Tests execution ended. Statistics:\n%s" %
suite.get_stat_message())
testoutput = RobotTestOutput(suite, settings)
output.close(suite)
if settings.is_rebot_needed():
datasources, settings = settings.get_rebot_datasources_and_settings()
if settings['SplitOutputs'] > 0:
testoutput = SplitIndexTestOutput(suite, datasources[0], settings)
else:
testoutput = RebotTestOutput(datasources, settings)
testoutput.serialize(settings)
LOGGER.close()
return suite
class AstroHTM(object):
"""
This class represents the hierarchical temporal memory algorithm to be
applied to astronomy data.
"""
_LOGGER = logging.getLogger(__name__)
#_SOURCE_FILE = './srcB_3to40_cl_barycorr_binned_multiD.fits'
#_SOURCE_FILE = 'nu80002092008A01_x2_bary_binned10.fits'
#_SOURCE_FILE = 'ni1103010157_0mpu7_cl_binned10.fits'
_ANOMALY_SCALE_FACTOR = 300
anomaly_count = 0
encoder_resolution_set = False
def __init__(self,
source_file,
min_var,
headers,
model_params,
output_path,
select_cols=False,
threshold=0.5):
"""
Parameters:
------------
@param source_file
The fits file name containing the data to run anomaly detection on.
Do not include '.fits' at the end of the filename, it is implied.
@param min_var (int)
The minimum variance a spectrum column will have else it is dropped
@param model_params (dictionary)
The dictionary of parameters for the HTM model to used
@param output_path (string)
The filename to whioh the output will be written (.csv extected)
@param select_cols (boolean)
True if columns should be removed for having low variance, false otherwise
Default value is False
@param threshold (float from 0 to 1)
Determines how high an anomaly score must be in order to register as an anomaly
Default value is 0.5
"""
self._SOURCE_FILE = source_file
self._MIN_VARIANCE = min_var
self._SELECT_COLS = select_cols
self._ANOMALY_THRESHOLD = threshold
self.data = Data(self._SOURCE_FILE, headers)
print len(headers), "headers given originally"
#self.model = self.createModel()
self.model_params = copy.deepcopy(model_params)
self._OUTPUT_PATH = output_path
self.data.set_input_stats()
def get_anomaly_count(self):
return self.anomaly_count
def _setRandomEncoderResolution(self, minResolution=0.001):
"""
Given model params, figure out the correct resolution for the
RandomDistributed encoders. Modifies params in place.
"""
fields = self.data.headers[1:]
for i, field in enumerate(fields):
encoder = self.model_params["modelParams"]["sensorParams"][
"encoders"][field]
if encoder[
"type"] == "RandomDistributedScalarEncoder" and "numBuckets" in encoder:
rangePadding = abs(self.data._INPUT_MAX[i] -
self.data._INPUT_MIN[i]) * 0.2
minValue = self.data._INPUT_MIN[i] - rangePadding
maxValue = self.data._INPUT_MAX[i] + rangePadding
resolution = max(minResolution, (maxValue - minValue) /
encoder.pop("numBuckets"))
encoder["resolution"] = resolution
#print "RESOLUTION:", resolution
self.model_params['modelParams']['sensorParams']['encoders'][
field] = encoder
self.encoder_resolution_set = True
for i in self.model_params['modelParams']['sensorParams'][
'encoders'].keys():
if i not in self.data.headers:
self.model_params['modelParams']['sensorParams'][
'encoders'].pop(i)
def createModel(self):
self._setRandomEncoderResolution()
return ModelFactory.create(self.model_params)
def setup_data(self):
#print("SPECTRUM BEFORE PROCESSING: ", self.data.spectrum)
if self._SELECT_COLS:
self.data.select_cols(self._MIN_VARIANCE, self.model_params)
self._SELECT_COLS = False
#self.data.replace_bad_intervals()
#print("SPECTRUM AFTER PROCESSING: ", self.data.spectrum)
def setup_output(self):
"""
Create Output object and write header line to csv file
"""
self.output = Output(self._OUTPUT_PATH)
self.output.write([
"timestamp",
str(self.data.headers[1]), "scaled_score", "anomaly_score"
])
def generate_model_input(self, index):
"""
Generate the index-th input point for the model to analyze
Parameters:
------------
@param index (int)
The row index to create the input from
"""
record = self.data.generate_record(index)
self.modelInput = dict(zip(self.data.headers, record))
for b in self.data.headers:
self.modelInput[b] = float(self.modelInput[b])
self.modelInput["float"] = self.modelInput["timestamp"]
self.modelInput["timestamp"] = datetime.datetime.fromtimestamp(
self.modelInput["timestamp"])
def run_model(self):
result = self.model.run(self.modelInput)
anomalyScore = result.inferences['anomalyScore']
scaledScore = anomalyScore * self._ANOMALY_SCALE_FACTOR
return anomalyScore, scaledScore
def output_results(self, anomalyScore, scaledScore):
"""
Output one line of results corresponding to one datapoint's analysis by the model
Parameters:
------------
@param anomalyScore (float)
The score generated by the HTM model for a given modelInput
@param scaledScore (float)
The anomalyScore multiplied by the chosen scale factor
"""
if anomalyScore > self._ANOMALY_THRESHOLD:
self.anomaly_count = self.anomaly_count + 1
self._LOGGER.info("Anomaly detected at [%s]. Anomaly score: %f.",
self.modelInput["timestamp"], anomalyScore)
self.output.write([
self.modelInput["float"], self.modelInput[self.data.headers[1]],
scaledScore,
"%.3f" % anomalyScore
])
def runAstroAnomaly(self):
"""
Process data, setup Output, and build and run anomaly detection model
"""
print "...Running with min var of: ", self._MIN_VARIANCE
self.setup_output()
self.setup_data()
self.model = self.createModel()
self.model.enableInference({'predictedField': self.data.headers[1]
}) # doesn't matter for anomaly detection
for i in tqdm.tqdm(range(0, self.data.data_size, 1),
desc='% Complete'):
self.generate_model_input(i)
anomalyScore, scaledScore = self.run_model()
self.output_results(anomalyScore, scaledScore)
print "Anomaly Scores have been written to", self._OUTPUT_PATH
print self.data.headers
self.output.close()
