def main(): # Use NullHandler for now to avoid getting the unwanted unformatted warning # message from logger on stderr "No handlers could be found for logger". g_log.addHandler(logging.NullHandler()) try: outputInfo = findParameters(_readCSVFile(**vars(_parseArgs()))) sys.stdout.write(json.dumps(outputInfo)) sys.stdout.flush() except Exception as ex: # pylint: disable=W0703 g_log.exception("ParamFinder failed") errorMessage = { "errorText": str(ex) or repr(ex), "diagnosticInfo": traceback.format_exc() } errorMessage = "%s\n" % (json.dumps(errorMessage)) try: sys.stderr.write(errorMessage) sys.stderr.flush() except Exception: # pylint: disable=W0703 g_log.exception("Failed to emit error message to stderr; msg=%s", errorMessage) # Use os._exit to abort the process instead of an exception to prevent # the python runtime from dumping traceback to stderr (since we dump a json # message to stderr, and don't want the extra text to interfere with parsing # in the Front End) os._exit(1) # pylint: disable=W0212
def testFindParameters(self): def createTestData(dataType, timeStepSeconds=300): dayPeriod = 86400.0 weekPeriod = 604800.0 timeStep = datetime.timedelta(seconds=timeStepSeconds) timestamp = datetime.datetime(2016, 1, 1, 0, 0, 0) random.seed(42) samples = [] for i in xrange(2000): if dataType == "transaction": timestamp += datetime.timedelta( seconds=random.randint(1, 2*timeStepSeconds)) else: timestamp += timeStep if dataType == "flat": value = 10.0 elif dataType == "daily": value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) / dayPeriod) elif dataType == "weekly": value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) / weekPeriod) elif dataType == "transaction": value = 10.0 samples.append((timestamp, value)) return samples outputInfo = param_finder.findParameters(createTestData("flat", 300)) self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300) self.assertEqual(outputInfo["aggInfo"]["func"], "mean") # Exclude timeOfDay and dayOfWeek encoder for flat line self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"]) outputInfo = param_finder.findParameters(createTestData("daily", 300)) self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300) self.assertEqual(outputInfo["aggInfo"]["func"], "mean") # Use timeOfDay but not dayOfWeek encoder for dataSet with daily period self.assertIsNotNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"]) outputInfo = param_finder.findParameters(createTestData("weekly", 7200)) self.assertGreater(outputInfo["aggInfo"]["windowSize"], 7200) self.assertEqual(outputInfo["aggInfo"]["func"], "mean") # Use dayOfWeek but not timeOfDay encoder for dataSet with daily period self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"]) outputInfo = param_finder.findParameters(createTestData("transaction", 300)) self.assertGreaterEqual(outputInfo["aggInfo"]["windowSize"], 300) self.assertEqual(outputInfo["aggInfo"]["func"], "sum") # Use dayOfWeek but not timeOfDay encoder for dataSet with daily period self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) # Check that dayOfWeek has been disabled event if the dataset has a daily # period. self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"])
def testFindParameters(self): def createTestData(dataType, timeStepSeconds=300): dayPeriod = 86400.0 weekPeriod = 604800.0 timeStep = datetime.timedelta(seconds=timeStepSeconds) timestamp = datetime.datetime(2016, 1, 1, 0, 0, 0) random.seed(42) samples = [] for i in xrange(2000): timestamp += timeStep if dataType == "flat": value = 10.0 elif dataType == "daily": value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) / dayPeriod) elif dataType == "weekly": value = numpy.sin(2 * numpy.pi * (timeStep.seconds * i) / weekPeriod) elif dataType == "binaryTransaction": value = 1 if int(i / 100) % 2 else 0 samples.append((timestamp, value)) return samples outputInfo = param_finder.findParameters(createTestData("flat", 300)) self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300) self.assertEqual(outputInfo["aggInfo"]["func"], "sum") # Exclude timeOfDay and dayOfWeek encoder for flat line self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"]) outputInfo = param_finder.findParameters(createTestData("daily", 300)) self.assertGreater(outputInfo["aggInfo"]["windowSize"], 300) self.assertEqual(outputInfo["aggInfo"]["func"], "mean") # Use timeOfDay but not dayOfWeek encoder for dataSet with daily period self.assertIsNotNone( outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"]) outputInfo = param_finder.findParameters(createTestData( "weekly", 7200)) self.assertGreater(outputInfo["aggInfo"]["windowSize"], 7200) self.assertEqual(outputInfo["aggInfo"]["func"], "mean") # Use dayOfWeek but not timeOfDay encoder for dataSet with daily period self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"]) outputInfo = param_finder.findParameters( createTestData("binaryTransaction", 300)) self.assertGreaterEqual(outputInfo["aggInfo"]["windowSize"], 300) self.assertEqual(outputInfo["aggInfo"]["func"], "sum") # Use dayOfWeek but not timeOfDay encoder for dataSet with daily period self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_timeOfDay"]) # Check that dayOfWeek has been disabled event if the dataset has a daily # period. self.assertIsNone(outputInfo["modelInfo"]["modelConfig"]["modelParams"] ["sensorParams"]["encoders"]["c0_dayOfWeek"])