def main():
# Use NullHandler for now to avoid getting the unwanted unformatted warning
# message from logger on stderr "No handlers could be found for logger".
g_log.addHandler(logging.NullHandler())
try:
outputInfo = findParameters(_readCSVFile(**vars(_parseArgs())))
sys.stdout.write(json.dumps(outputInfo))
sys.stdout.flush()
except Exception as ex: # pylint: disable=W0703
g_log.exception("ParamFinder failed")
errorMessage = {
"errorText": str(ex) or repr(ex),
"diagnosticInfo": traceback.format_exc()
}
errorMessage = "%s\n" % (json.dumps(errorMessage))
try:
sys.stderr.write(errorMessage)
sys.stderr.flush()
except Exception: # pylint: disable=W0703
g_log.exception("Failed to emit error message to stderr; msg=%s",
errorMessage)
# Use os._exit to abort the process instead of an exception to prevent
# the python runtime from dumping traceback to stderr (since we dump a json
# message to stderr, and don't want the extra text to interfere with parsing
# in the Front End)
os._exit(1) # pylint: disable=W0212
def main():
# Use NullHandler for now to avoid getting the unwanted unformatted warning
# message from logger on stderr "No handlers could be found for logger".
g_log.addHandler(logging.NullHandler())
try:
outputInfo = findParameters(_readCSVFile(**vars(_parseArgs())))
sys.stdout.write(json.dumps(outputInfo))
sys.stdout.flush()
except Exception as ex: # pylint: disable=W0703
g_log.exception("ParamFinder failed")
errorMessage = {
"errorText": str(ex) or repr(ex),
"diagnosticInfo": traceback.format_exc()
}
errorMessage = "%s\n" % (json.dumps(errorMessage))
try:
sys.stderr.write(errorMessage)
sys.stderr.flush()
except Exception: # pylint: disable=W0703
g_log.exception("Failed to emit error message to stderr; msg=%s",
errorMessage)
# Use os._exit to abort the process instead of an exception to prevent
# the python runtime from dumping traceback to stderr (since we dump a json
# message to stderr, and don't want the extra text to interfere with parsing
# in the Front End)
os._exit(1) # pylint: disable=W0212
def testFindParameters(self):
def createTestData(dataType, timeStepSeconds=300):
dayPeriod = 86400.0
weekPeriod = 604800.0
timeStep = datetime.timedelta(seconds=timeStepSeconds)
timestamp = datetime.datetime(2016, 1, 1, 0, 0, 0)
random.seed(42)
samples = []
for i in xrange(2000):
if dataType == "transaction":
timestamp += datetime.timedelta(
seconds=random.randint(1, 2*timeStepSeconds))
else:
timestamp += timeStep
if dataType == "flat":
value = 10.0
elif dataType == "daily":
value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) / dayPeriod)
elif dataType == "weekly":
value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) / weekPeriod)
elif dataType == "transaction":
value = 10.0
samples.append((timestamp, value))
return samples
outputInfo = param_finder.findParameters(createTestData("flat", 300))
self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300)
self.assertEqual(outputInfo["aggInfo"]["func"], "mean")
# Exclude timeOfDay and dayOfWeek encoder for flat line
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
outputInfo = param_finder.findParameters(createTestData("daily", 300))
self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300)
self.assertEqual(outputInfo["aggInfo"]["func"], "mean")
# Use timeOfDay but not dayOfWeek encoder for dataSet with daily period
self.assertIsNotNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
outputInfo = param_finder.findParameters(createTestData("weekly", 7200))
self.assertGreater(outputInfo["aggInfo"]["windowSize"], 7200)
self.assertEqual(outputInfo["aggInfo"]["func"], "mean")
# Use dayOfWeek but not timeOfDay encoder for dataSet with daily period
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
outputInfo = param_finder.findParameters(createTestData("transaction", 300))
self.assertGreaterEqual(outputInfo["aggInfo"]["windowSize"], 300)
self.assertEqual(outputInfo["aggInfo"]["func"], "sum")
# Use dayOfWeek but not timeOfDay encoder for dataSet with daily period
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
# Check that dayOfWeek has been disabled event if the dataset has a daily
# period.
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
def testFindParameters(self):
def createTestData(dataType, timeStepSeconds=300):
dayPeriod = 86400.0
weekPeriod = 604800.0
timeStep = datetime.timedelta(seconds=timeStepSeconds)
timestamp = datetime.datetime(2016, 1, 1, 0, 0, 0)
random.seed(42)
samples = []
for i in xrange(2000):
timestamp += timeStep
if dataType == "flat":
value = 10.0
elif dataType == "daily":
value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) /
dayPeriod)
elif dataType == "weekly":
value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) /
weekPeriod)
elif dataType == "binaryTransaction":
value = 1 if int(i / 100) % 2 else 0
samples.append((timestamp, value))
return samples
outputInfo = param_finder.findParameters(createTestData("flat", 300))
self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300)
self.assertEqual(outputInfo["aggInfo"]["func"], "sum")
# Exclude timeOfDay and dayOfWeek encoder for flat line
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
outputInfo = param_finder.findParameters(createTestData("daily", 300))
self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300)
self.assertEqual(outputInfo["aggInfo"]["func"], "mean")
# Use timeOfDay but not dayOfWeek encoder for dataSet with daily period
self.assertIsNotNone(
outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
outputInfo = param_finder.findParameters(createTestData(
"weekly", 7200))
self.assertGreater(outputInfo["aggInfo"]["windowSize"], 7200)
self.assertEqual(outputInfo["aggInfo"]["func"], "mean")
# Use dayOfWeek but not timeOfDay encoder for dataSet with daily period
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
outputInfo = param_finder.findParameters(
createTestData("binaryTransaction", 300))
self.assertGreaterEqual(outputInfo["aggInfo"]["windowSize"], 300)
self.assertEqual(outputInfo["aggInfo"]["func"], "sum")
# Use dayOfWeek but not timeOfDay encoder for dataSet with daily period
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_timeOfDay"])
# Check that dayOfWeek has been disabled event if the dataset has a daily
# period.
self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"]
["sensorParams"]["encoders"]["c0_dayOfWeek"])
