Beispiel #1
0
def process_with_spark(data_set, pattern_list):
    logger.debug("Processing with SPARK")
    processing_type = ProcessingType.objects.get(name="spark")
    process_instance = ProcessingInstance(processing_type=processing_type,
                                          data_set=data_set,
                                          pattern_list=pattern_list)
    process_instance.save()
    logger.debug("Loading data set from location %s", data_set.location)
    processor = SparkProcessor(process_instance, data_set, pattern_list)
    daemon_starter = DaemonStarter(processor)
    daemon_starter.daemon = True
    daemon_starter.start()
    logger.info("spark processing thread started")
Beispiel #2
0
def process_with_python(data_set, pattern_list):
    logger.debug("Processing with python")
    processing_type = ProcessingType.objects.get(name="python")
    process_instance = ProcessingInstance(processing_type=processing_type,
                                          data_set=data_set,
                                          pattern_list=pattern_list)
    process_instance.save()
    logger.debug("Process instance " + str(process_instance.id) + " saved. Loading data set from location %s", data_set.location)
    processor = PythonProcessor(process_instance, data_set, pattern_list)
    daemon_starter = DaemonStarter(processor)
    daemon_starter.daemon = True
    daemon_starter.start()
    logger.info("Python processing thread started")
    return process_instance