def executeJob(self, job): time.sleep(5) job['status'] = 'RUNNING' (responseCode, job) = self.service.saveJob(job) if responseCode != 200: raise Exception("could not save job status.") filePath = conf.MOCK_DATA_PATH[job['queryType']] with open(filePath, 'r') as handle: data = json.loads(handle.read()) result_set = GeqeAPI.rawResultToResultDocument(job, data) (response, result_set) = self.service.saveResultset(result_set) if response != 200: job['status'] = 'FAILED' self.service.saveJob(job) print str(result_set) raise Exception("Could not save result set. error: " + str(response)) if 'modelSavePath' in job and job['modelSavePath'] is not None: # save the model meta data modelData = { "name": job['name'], "username": job["username"], "queryType": job["queryType"], "modelSavePath": job['modelSavePath'], "siteListId": job["siteListId"], "datasetId": job["datasetId"] } (response, modelData) = self.service.saveModelData(modelData) if response != 200: job['status'] = 'FAILED' self.service.saveJob(job) raise Exception("Could not save model metadata: " + str(response) + " \n" + str(modelData)) # save the results set into elastic search if conf.ES_HOST is not None: elaticsearchConnetor = GeqeAPI.ElasticSearchHelper( conf.ES_HOST, port=conf.ES_PORT) (response, es_result) = elaticsearchConnetor.addResultSet(result_set) if response != 201: job['status'] = 'FAILED' self.service.saveJob(job) raise Exception("Could not save result set to es. error: " + str(response) + " \n" + str(result_set)) time.sleep(5) job['status'] = 'SUCCESS' job['resultsetId'] = result_set['id'] (response, job) = self.service.saveJob(job) if response != 200: raise Exception("could not save job status.") return True
def executeJob(self, job): time.sleep(5) job["status"] = "RUNNING" (responseCode, job) = self.service.saveJob(job) if responseCode != 200: raise Exception("could not save job status.") filePath = conf.MOCK_DATA_PATH[job["queryType"]] with open(filePath, "r") as handle: data = json.loads(handle.read()) result_set = GeqeAPI.rawResultToResultDocument(job, data) (response, result_set) = self.service.saveResultset(result_set) if response != 200: job["status"] = "FAILED" self.service.saveJob(job) print str(result_set) raise Exception("Could not save result set. error: " + str(response)) if "modelSavePath" in job and job["modelSavePath"] is not None: # save the model meta data modelData = { "name": job["name"], "username": job["username"], "queryType": job["queryType"], "modelSavePath": job["modelSavePath"], "siteListId": job["siteListId"], "datasetId": job["datasetId"], } (response, modelData) = self.service.saveModelData(modelData) if response != 200: job["status"] = "FAILED" self.service.saveJob(job) raise Exception("Could not save model metadata: " + str(response) + " \n" + str(modelData)) # save the results set into elastic search if conf.ES_HOST is not None: elaticsearchConnetor = GeqeAPI.ElasticSearchHelper(conf.ES_HOST, port=conf.ES_PORT) (response, es_result) = elaticsearchConnetor.addResultSet(result_set) if response != 201: job["status"] = "FAILED" self.service.saveJob(job) raise Exception("Could not save result set to es. error: " + str(response) + " \n" + str(result_set)) time.sleep(5) job["status"] = "SUCCESS" job["resultsetId"] = result_set["id"] (response, job) = self.service.saveJob(job) if response != 200: raise Exception("could not save job status.") return True
def executeJob(dataConnector, elaticsearchConnetor, job, dataset, sitelist): #Declare Spark Context conf = SparkConf().setAppName("GeqeRunner") conf.set('spark.driver.maxResultSize', '0') sc = SparkContext(conf=conf) sqlContext = SQLContext(sc) #Create polygon list and broadcast variable based on it lPolygon = shapeReader.readInShapeDocument( sitelist) if sitelist is not None else None queryType = job['queryType'] jobname = job['username'] + '_' + job['name'] modelSavePath = job['modelSavePath'] if 'modelSavePath' in job else None datasetPath = None inputPartitions = None if job['queryType'] == 'location': datasetPath = dataset['location_path'] inputPartitions = dataset['location_partitions'] elif job['queryType'] == 'event': datasetPath = dataset['event_path'] inputPartitions = dataset['event_partitions'] else: raise ValueError("invalid query type: " + str(job['queryType'])) dictionaryPath = dataset['dictionaryPath'] nDataType = int(dataset['type']) modelData = None if 'geqeModelId' in job and job['geqeModelId'] != "": (response, modelData) = dataConnector.getModelData(job['geqeModelId']) if response != 200: raise Exception("Could not load geqeModel") #iif modelData['dictionaryPath'] != dictionaryPath: # raise ValueError("dataset dictionary and model dictionary do not match") sNum = job['limit'] if 'limit' in job else 25 result = None if modelData: modelPath = modelData['modelSavePath'] + '/' + modelData['name'] result = run_refindSimilarPlaces(sc, sqlContext, jobname, datasetPath, dictionaryPath, inputPartitions, sNum, modelPath, bByDate=queryType == 'event') elif 'location' == queryType: result = run_findSimilarPlaces(sc, sqlContext, jobname, datasetPath, dictionaryPath, nDataType, inputPartitions, sNum, lPolygon, modelSavePath) elif 'event' == queryType: result = run_findSimilarEvent(sc, sqlContext, jobname, datasetPath, dictionaryPath, nDataType, inputPartitions, sNum, lPolygon, modelSavePath) else: raise Exception("Invalid query type: " + queryType) if modelSavePath is not None: # save the model meta data modelData = { "name": jobname, "username": job["username"], "queryType": job["queryType"], "modelSavePath": modelSavePath, "siteListId": job["siteListId"], "datasetId": job["datasetId"] } (response, modelData) = dataConnector.saveModelData(modelData) if response != 200: job['status'] = 'FAILED' dataConnector.saveJob(job) raise Exception("Could not save model metadata: " + str(response) + " \n" + str(modelData)) result = GeqeAPI.rawResultToResultDocument(job, result) # save the result (response, result_set) = dataConnector.saveResultset(result) if response != 200: job['status'] = 'FAILED' dataConnector.saveJob(job) raise Exception("Could not save result set. error: " + str(response) + " \n" + str(result_set)) # save the results set into elastic search if elaticsearchConnetor is not None: (response, es_result) = elaticsearchConnetor.addResultSet(result_set) if response != 201: job['status'] = 'FAILED' dataConnector.saveJob(job) raise Exception("Could not save result set to es. error: " + str(response) + " \n" + str(result_set)) # update the job info job['status'] = 'SUCCESS' job['resultsetId'] = result_set['id'] (response, job) = dataConnector.saveJob(job) if response != 200: raise Exception("could not save job status.")
def executeJob(dataConnector,elaticsearchConnetor,job,dataset,sitelist): #Declare Spark Context conf = SparkConf().setAppName("GeqeRunner") conf.set('spark.driver.maxResultSize','0') sc = SparkContext(conf = conf) sqlContext = SQLContext(sc) #Create polygon list and broadcast variable based on it lPolygon = shapeReader.readInShapeDocument(sitelist) if sitelist is not None else None queryType = job['queryType'] jobname = job['username']+'_'+job['name'] modelSavePath = job['modelSavePath'] if 'modelSavePath' in job else None datasetPath = None inputPartitions = None if job['queryType'] == 'location': datasetPath = dataset['location_path'] inputPartitions = dataset['location_partitions'] elif job['queryType'] == 'event': datasetPath = dataset['event_path'] inputPartitions = dataset['event_partitions'] else: raise ValueError("invalid query type: "+str(job['queryType'])) dictionaryPath = dataset['dictionaryPath'] nDataType = int(dataset['type']) modelData = None if 'geqeModelId' in job and job['geqeModelId'] != "": (response,modelData) = dataConnector.getModelData(job['geqeModelId']) if response != 200: raise Exception("Could not load geqeModel") #iif modelData['dictionaryPath'] != dictionaryPath: # raise ValueError("dataset dictionary and model dictionary do not match") sNum = job['limit'] if 'limit' in job else 25 result = None if modelData: modelPath = modelData['modelSavePath']+'/'+modelData['name'] result = run_refindSimilarPlaces(sc,sqlContext,jobname,datasetPath,dictionaryPath,inputPartitions,sNum,modelPath,bByDate= queryType == 'event') elif 'location' == queryType: result = run_findSimilarPlaces(sc,sqlContext,jobname,datasetPath,dictionaryPath,nDataType,inputPartitions,sNum,lPolygon,modelSavePath) elif 'event' == queryType: result = run_findSimilarEvent(sc,sqlContext,jobname,datasetPath,dictionaryPath,nDataType,inputPartitions,sNum,lPolygon,modelSavePath) else: raise Exception("Invalid query type: "+queryType) if modelSavePath is not None: # save the model meta data modelData = { "name" : jobname, "username": job["username"], "queryType": job["queryType"], "modelSavePath": modelSavePath, "siteListId" : job["siteListId"], "datasetId" : job["datasetId"] } (response,modelData) = dataConnector.saveModelData(modelData) if response != 200: job['status'] = 'FAILED' dataConnector.saveJob(job) raise Exception("Could not save model metadata: "+str(response)+" \n"+str(modelData)) result = GeqeAPI.rawResultToResultDocument(job,result) # save the result (response,result_set) = dataConnector.saveResultset(result) if response != 200: job['status'] = 'FAILED' dataConnector.saveJob(job) raise Exception("Could not save result set. error: "+str(response)+" \n"+str(result_set)) # save the results set into elastic search if elaticsearchConnetor is not None: (response,es_result) = elaticsearchConnetor.addResultSet(result_set) if response != 201: job['status'] = 'FAILED' dataConnector.saveJob(job) raise Exception("Could not save result set to es. error: "+str(response)+" \n"+str(result_set)) # update the job info job['status'] = 'SUCCESS' job['resultsetId'] = result_set['id'] (response,job) = dataConnector.saveJob(job) if response != 200: raise Exception("could not save job status.")
job = { "name": args.name, "status": "WAITING", "queryType": "unkown", "limit": -1, "username": args.username } (response, job) = dataConnector.saveJob(job) if response != 200: print "ERROR: Could not save job. = ", response, job sys.exit(1) # save the result set with open(args.resultSetPath, 'r') as handle: data = json.loads(handle.read()) result_set = GeqeAPI.rawResultToResultDocument(job, data) (response, result_set) = dataConnector.saveResultset(result_set) if response != 200: print "ERROR: Could not save result. = ", response, result_set sys.exit(1) job['status'] = 'SUCCESS' job['queryType'] = result_set['type'] job['resultsetId'] = result_set['id'] (response, job) = dataConnector.saveJob(job) if response != 200: print "ERROR: Could not save job. = ", response, job sys.exit(1) print 'success'
"name": args.name, "status": "WAITING", "queryType": "unkown", "limit" : -1, "username": args.username } (response,job) = dataConnector.saveJob(job) if response != 200: print "ERROR: Could not save job. = ",response,job sys.exit(1) # save the result set with open(args.resultSetPath,'r') as handle: data = json.loads(handle.read()) result_set = GeqeAPI.rawResultToResultDocument(job,data) (response,result_set) = dataConnector.saveResultset(result_set) if response != 200: print "ERROR: Could not save result. = ",response,result_set sys.exit(1) job['status'] = 'SUCCESS' job['queryType'] = result_set['type'] job['resultsetId'] = result_set['id'] (response,job) = dataConnector.saveJob(job) if response != 200: print "ERROR: Could not save job. = ",response,job sys.exit(1) print 'success'