def process_with_spark(data_set, pattern_list):
logger.debug("Processing with SPARK")
processing_type = ProcessingType.objects.get(name="spark")
process_instance = ProcessingInstance(processing_type=processing_type,
data_set=data_set,
pattern_list=pattern_list)
process_instance.save()
logger.debug("Loading data set from location %s", data_set.location)
processor = SparkProcessor(process_instance, data_set, pattern_list)
daemon_starter = DaemonStarter(processor)
daemon_starter.daemon = True
daemon_starter.start()
logger.info("spark processing thread started")
def process_with_python(data_set, pattern_list):
logger.debug("Processing with python")
processing_type = ProcessingType.objects.get(name="python")
process_instance = ProcessingInstance(processing_type=processing_type,
data_set=data_set,
pattern_list=pattern_list)
process_instance.save()
logger.debug("Process instance " + str(process_instance.id) + " saved. Loading data set from location %s", data_set.location)
processor = PythonProcessor(process_instance, data_set, pattern_list)
daemon_starter = DaemonStarter(processor)
daemon_starter.daemon = True
daemon_starter.start()
logger.info("Python processing thread started")
return process_instance