def deleteScenario(request, ncn_id): # delete form and connection to related scripts and user, # and also deletes the whole scenario scenario = models.Scenario.objects.get(ncn_id=ncn_id) scenario_id = scenario.id scripts = scenario.script_set.all() # send request/task to work-flow-manager to run delete script wfm = work_flow_manager.WorkFlowManager.Instance() user = request.user del_script = models.UserScript.objects.filter( script_name__startswith="deleteScenario-", user_id__exact=user.id)[0] current_task = work_flow_manager.WorkerTask({ "scenario_id": scenario_id, "task_type": "DELETE_SCENARIO", "scripts": ["%s/%s" % (MEDIA_ROOT, del_script.script_file)] }) wfm.put_task_to_queue(current_task) # TODO problem: wfm dele is running meanwhile django deletes scenario and # scripts ... django should wait to delete them ... # delete scenario, scripts and scenario-status from db (scenario-status is bounded to scenario) delete_scripts(scripts) scenario.delete() logger.info('Operation: delete scenario: id=%d name=%s' % (scenario.id, scenario.scenario_name), extra={'user': request.user}) port = request.META['SERVER_PORT'] return HttpResponseRedirect('http://127.0.0.1:' + port + '/scenario/overview/')
def reset_scenario_core(ncn_id): ret = {} scenario = models.Scenario.objects.get(ncn_id=ncn_id) scenario_id = scenario.id wfm = work_flow_manager.WorkFlowManager.Instance() if not wfm.lock_scenario(scenario_id): msg = "Scenario '%s' name=%s is busy." % (ncn_id, scenario.scenario_name) logger.warning("Delete Scenario refused: " + msg) ret = {'status': 1, 'message': "Error: " + msg} else: default_delete_script = os.path.join(IE_SCRIPTS_DIR, IE_DEFAULT_DEL_SCRIPT) del_scripts = [default_delete_script] scripts = scenario.script_set.all() for s in scripts: del_scripts.append("%s/%s" % (MEDIA_ROOT, s.script_file)) current_task = work_flow_manager.WorkerTask({ "scenario_id": scenario_id, "task_type": "RESET_SCENARIO", "scripts": del_scripts }) wfm.put_task_to_queue(current_task) ret = { 'status': 0, } return ret
def delete_scenario_core(scenario_id=None, ncn_id=None): ret = {} if ncn_id: ncn_id = ncn_id.encode('ascii', 'ignore') scenario = models.Scenario.objects.get(ncn_id=ncn_id) scenario_id = scenario.id else: scenario = models.Scenario.objects.get(id=int(scenario_id)) ncn_id = scenario.ncn_id # send request/task to work-flow-manager to run delete script wfm = work_flow_manager.WorkFlowManager.Instance() if not wfm.lock_scenario(scenario_id): msg = "Scenario '%s' name=%s is busy." % (ncn_id, scenario.scenario_name) logger.warning("Delete Scenario refused: " + msg) ret = {'status': 1, 'message': "Error: " + msg} else: logger.info("Submitting to Work-Queue: delete scenario ncn_id=" + ` ncn_id `) default_delete_script = os.path.join(IE_SCRIPTS_DIR, IE_DEFAULT_DEL_SCRIPT) #del_scripts = models.UserScript.objects.filter( # script_name__startswith="deleteScenario-",user_id__exact=user.id) del_scripts = [default_delete_script] scripts = scenario.script_set.all() for s in scripts: del_scripts.append("%s/%s" % (MEDIA_ROOT, s.script_file)) if len(del_scripts) == 0: logger.warning('Scenario: id=%d name=%s does not have a delete script.' \ % (scenario.id,scenario.scenario_name)) current_task = work_flow_manager.WorkerTask({ "scenario_id": scenario_id, "task_type": "DELETE_SCENARIO", "scripts": del_scripts }) wfm.put_task_to_queue(current_task) ret = { 'status': 0, } return ret
def submit_scenario(scenario, scenario_id, set_status=True): wfm = work_flow_manager.WorkFlowManager.Instance() scripts = get_scenario_script_paths(scenario) if len(scripts) == 0: msg = "Scenario '%s' name=%s does not have any scripts." \ % (scenario.ncn_id, scenario.scenario_name) logger.warning(msg) # send request/task to work-flow-manager to run ingestion current_task = work_flow_manager.WorkerTask( { "scenario_id": scenario_id, "task_type": "INGEST_SCENARIO", "scripts": scripts }, set_status) wfm.put_task_to_queue(current_task) logger.info( 'Operation: ingest scenario: id=%d name=%s' \ % (scenario.id,scenario.scenario_name))
def add_product_submit(postData, request_meta): logger.info("processing addProduct request") args = json.loads(postData) resp = {} error_str = None status = 0 ap = None from settings import IE_DEBUG if IE_DEBUG > 1: logger.debug("addProduct data: " + ` args `) try: ap = models.ProductInfo(info_date=datetime.datetime.utcnow(), info_status="processing", info_error="", new_product_id="", product_url="") ap.save() if not 'data' in args and not 'url' in args: raise AddProductError( "Missing url or data; at least one is required") # Set up to pass input data to the worker task params = { "task_type": "ADD_PRODUCT", "addProduct_id": ap.id, } if 'data' in args and 'url' in args: logger.warning( "add_product: only one of 'data' or 'url' should be used") # pathname of the product data for the product if 'data' in args: req_data = args["data"] if not req_data or None == req_data or len(req_data) < 1: logger.warning("add_product: empty 'data' in input") else: params['product'] = req_data.encode('ascii', 'ignore') # or url if any if 'url' in args: req_url = args["url"] if not req_url or req_url == None or len(req_url) < 2: logger.warning("add_product: empty 'url' in input") else: params["url"] = req_url.encode('ascii', 'ignore') # pathname of the metadata for the product if "metadata" in args: params["metadata"] = args["metadata"].encode('ascii', 'ignore') # The coverage ID of an existing product, to be replaced if "productID" in args: productID = args["productID"] if not productID or None == productID or len(productID) < 1: logger.warning("add_product: empty 'productID'") else: params["covId"] = productID.encode('ascii', 'ignore') # for time-zone aware dates use this one instead: # ap.info_date = datetime.datetime.utcnow().replace(tzinfo=utc) # process addProduct request by work_flow_manager wfm = work_flow_manager.WorkFlowManager.Instance() current_task = work_flow_manager.WorkerTask(params) wfm.put_task_to_queue(current_task) # exectues add_product_wfunc resp['opId'] = ap.id resp['status'] = 0 if ap.id > IE_MAX_ADDPRODOPS: error_str = ("Internal Error in add_product: " + "ap.id exceeded max (" + ` IE_MAX_ADDPRODOPS ` + ")") status = 103 logger.error(error_str) if ap.id > IE_PURGE_ADDPRODOPS: purge_old_ops() except AddProductError as ap_err: logger.error("AddProductError in add_product: " + ap_err.msg) status = 101 error_str = ap_err.msg except OSError as e: logger.error("OSError in add_product: " + ` e `) status = 102 error_str = ("Internal Error: " + ` e `) except Exception as e: status = 50 error_str = "Unexpected exception: " + e.__class__.__name__ logger.error("Exception in add_product: " + ` e `) from settings import IE_DEBUG if IE_DEBUG > 0: traceback.print_exc(4, sys.stdout) apID = ap.id ap = None ap = models.ProductInfo.objects.filter(id=apID)[0] resp["status"] = status if 0 != status: if None != ap: ap.info_status = "failed" if error_str: if None != ap: info_error = error_str resp["errorString"] = error_str logger.warning("addProduct request error. Status: " +`status`+\ ", error: " + error_str) else: if resp['opId']: logger.info("addProduct request status " +`status` +\ ", opId="+`resp['opId']`) else: logger.warning("addProduct: No opId nor error string, " +\ "status: "+`status`) return resp
def add_local_product_core(request, ncn_id, template, aftersave=None): # add local product to the related scenario msg = "The new product has been successfully added" if request.method == 'POST': wfm = work_flow_manager.WorkFlowManager.Instance() scenario = models.Scenario.objects.get(ncn_id=ncn_id) sc_id = scenario.id if wfm.lock_scenario(sc_id): try: form = forms.AddLocalProductForm(request.POST, request.FILES) if form.is_valid() and form.is_multipart(): full_directory_name, directory_name = create_dl_dir( ncn_id + "_") full_directory_name = full_directory_name.encode( 'ascii', 'ignore') logger.info("Product directory name:" + full_directory_name) if 'metadataFile' in request.FILES: saveFile(request.FILES['metadataFile'], full_directory_name) meta = request.FILES['metadataFile']._get_name( ).encode('ascii', 'ignore') else: meta = None saveFile(request.FILES['rasterFile'], full_directory_name) data = request.FILES['rasterFile']._get_name().encode( 'ascii', 'ignore') scripts = [] if scenario.oda_server_ingest != 0: scripts.append( os.path.join(IE_SCRIPTS_DIR, IE_DEFAULT_ADDPROD_SCRIPT)) if len(scripts) == 0: logger.warning("Scenario '%s' name='%s' does not have scripts to ingest, proceeding regardless." \ % (ncn_id,scenario.scenario_name)) # send request/task to work-flow-manager to run script current_task = work_flow_manager.WorkerTask({ "scenario_id": sc_id, "ncn_id": ncn_id, "task_type": "INGEST_LOCAL_PROD", "scripts": scripts, "cat_registration": scenario.cat_registration, "s2_preprocess": scenario.s2_preprocess, "dir_path": full_directory_name, "metadata": meta, "data": data, }) wfm.put_task_to_queue(current_task) logger.info("Local ingest scenario - submitted to queue: " + "id=%d, ncn_id='%s', name='%s'" \ % (sc_id, ncn_id, scenario.scenario_name)) else: logger.warning( 'The add product form has not been fully/correctly filled' ) except Exception, e: logger.error("Failed to add local product to Scenario '%s' name=%s" \ % (ncn_id, scenario.scenario_name) + \ 'exception = ' + `e`) finally:
def odaAddLocalProductOld(request, ncn_id): oda_init(request) # add local product to the related scenario logger.info("addLocalProduct to ncn_id " + ncn_id) msg = "The new product has been successfully added" if request.method == 'POST': wfm = work_flow_manager.WorkFlowManager.Instance() scenario = models.Scenario.objects.get(ncn_id) sc_id = scenario.id #if not wfm.lock_scenario(sc_id): # return getAddProductResult(request, sc_id, msg) if not wfm.lock_scenario(sc_id): logger.warning("Ingest Scenario refused: " + "Scenario '%s' name=%s is busy." % (scenario.ncn_id, scenario.scenario_name)) return HttpResponseRedirect('http://127.0.0.1:' + port + '/scenario/overview/') form = forms.AddLocalProductForm(request.POST, request.FILES) if form.is_valid() and form.is_multipart(): try: sc_ncn_id = scenario.ncn_id full_directory_name, directory_name = create_dl_dir(sc_ncn_id + "_") full_directory_name = full_directory_name.encode( 'ascii', 'ignore') logger.info("Product directory name:" + full_directory_name) except: wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0) logger.error("Failed to generate a new product directory name") return HttpResponseRedirect('http://127.0.0.1:' + port + '/scenario/overview/') try: saveFile(request.FILES['metadataFile'], full_directory_name) except IOError, e: wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0) logger.error("Failed to save product's metadata file") return HttpResponseRedirect('http://127.0.0.1:' + port + '/scenario/overview/') try: saveFile(request.FILES['rasterFile'], full_directory_name) except IOError, e: wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0) logger.error("Failed to save product's raster file") return HttpResponseRedirect('http://127.0.0.1:' + port + '/scenario/overview/') scripts = get_scenario_script_paths(scenario) if len(scripts) > 0: # send request/task to work-flow-manager to run script current_task = work_flow_manager.WorkerTask({ "scenario_id": sc_id, "ncn_id": sc_ncn_id, "task_type": "INGEST_LOCAL_PROD", "scripts": scripts, "cat_registration": scenario.cat_registration, "s2_preprocess": scenario.s2_preprocess, "dir_path": full_directory_name, "metadata": request.FILES['metadataFile']._get_name(), "data": request.FILES['rasterFile']._get_name(), }) wfm.put_task_to_queue(current_task) logger.info('Operation: ingest scenario: id=%d name=%s' \ % (scenario.id,scenario.scenario_name)) else: logger.error("NOT INGESTING: Scenario '%s' name=%s does not have scripts to ingest." \ % (scenario.ncn_id,scenario.scenario_name)) wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0) return HttpResponseRedirect('http://127.0.0.1:' + port + '/scenario/overview/')