def deleteScenario(request, ncn_id):
# delete form and connection to related scripts and user,
# and also deletes the whole scenario
scenario = models.Scenario.objects.get(ncn_id=ncn_id)
scenario_id = scenario.id
scripts = scenario.script_set.all()
# send request/task to work-flow-manager to run delete script
wfm = work_flow_manager.WorkFlowManager.Instance()
user = request.user
del_script = models.UserScript.objects.filter(
script_name__startswith="deleteScenario-", user_id__exact=user.id)[0]
current_task = work_flow_manager.WorkerTask({
"scenario_id":
scenario_id,
"task_type":
"DELETE_SCENARIO",
"scripts": ["%s/%s" % (MEDIA_ROOT, del_script.script_file)]
})
wfm.put_task_to_queue(current_task)
# TODO problem: wfm dele is running meanwhile django deletes scenario and
# scripts ... django should wait to delete them ...
# delete scenario, scripts and scenario-status from db (scenario-status is bounded to scenario)
delete_scripts(scripts)
scenario.delete()
logger.info('Operation: delete scenario: id=%d name=%s' %
(scenario.id, scenario.scenario_name),
extra={'user': request.user})
port = request.META['SERVER_PORT']
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
def reset_scenario_core(ncn_id):
ret = {}
scenario = models.Scenario.objects.get(ncn_id=ncn_id)
scenario_id = scenario.id
wfm = work_flow_manager.WorkFlowManager.Instance()
if not wfm.lock_scenario(scenario_id):
msg = "Scenario '%s' name=%s is busy." % (ncn_id,
scenario.scenario_name)
logger.warning("Delete Scenario refused: " + msg)
ret = {'status': 1, 'message': "Error: " + msg}
else:
default_delete_script = os.path.join(IE_SCRIPTS_DIR,
IE_DEFAULT_DEL_SCRIPT)
del_scripts = [default_delete_script]
scripts = scenario.script_set.all()
for s in scripts:
del_scripts.append("%s/%s" % (MEDIA_ROOT, s.script_file))
current_task = work_flow_manager.WorkerTask({
"scenario_id": scenario_id,
"task_type": "RESET_SCENARIO",
"scripts": del_scripts
})
wfm.put_task_to_queue(current_task)
ret = {
'status': 0,
}
return ret
def delete_scenario_core(scenario_id=None, ncn_id=None):
ret = {}
if ncn_id:
ncn_id = ncn_id.encode('ascii', 'ignore')
scenario = models.Scenario.objects.get(ncn_id=ncn_id)
scenario_id = scenario.id
else:
scenario = models.Scenario.objects.get(id=int(scenario_id))
ncn_id = scenario.ncn_id
# send request/task to work-flow-manager to run delete script
wfm = work_flow_manager.WorkFlowManager.Instance()
if not wfm.lock_scenario(scenario_id):
msg = "Scenario '%s' name=%s is busy." % (ncn_id,
scenario.scenario_name)
logger.warning("Delete Scenario refused: " + msg)
ret = {'status': 1, 'message': "Error: " + msg}
else:
logger.info("Submitting to Work-Queue: delete scenario ncn_id=" +
` ncn_id `)
default_delete_script = os.path.join(IE_SCRIPTS_DIR,
IE_DEFAULT_DEL_SCRIPT)
#del_scripts = models.UserScript.objects.filter(
# script_name__startswith="deleteScenario-",user_id__exact=user.id)
del_scripts = [default_delete_script]
scripts = scenario.script_set.all()
for s in scripts:
del_scripts.append("%s/%s" % (MEDIA_ROOT, s.script_file))
if len(del_scripts) == 0:
logger.warning('Scenario: id=%d name=%s does not have a delete script.' \
% (scenario.id,scenario.scenario_name))
current_task = work_flow_manager.WorkerTask({
"scenario_id": scenario_id,
"task_type": "DELETE_SCENARIO",
"scripts": del_scripts
})
wfm.put_task_to_queue(current_task)
ret = {
'status': 0,
}
return ret
def submit_scenario(scenario, scenario_id, set_status=True):
wfm = work_flow_manager.WorkFlowManager.Instance()
scripts = get_scenario_script_paths(scenario)
if len(scripts) == 0:
msg = "Scenario '%s' name=%s does not have any scripts." \
% (scenario.ncn_id, scenario.scenario_name)
logger.warning(msg)
# send request/task to work-flow-manager to run ingestion
current_task = work_flow_manager.WorkerTask(
{
"scenario_id": scenario_id,
"task_type": "INGEST_SCENARIO",
"scripts": scripts
}, set_status)
wfm.put_task_to_queue(current_task)
logger.info(
'Operation: ingest scenario: id=%d name=%s' \
% (scenario.id,scenario.scenario_name))
def add_product_submit(postData, request_meta):
logger.info("processing addProduct request")
args = json.loads(postData)
resp = {}
error_str = None
status = 0
ap = None
from settings import IE_DEBUG
if IE_DEBUG > 1:
logger.debug("addProduct data: " + ` args `)
try:
ap = models.ProductInfo(info_date=datetime.datetime.utcnow(),
info_status="processing",
info_error="",
new_product_id="",
product_url="")
ap.save()
if not 'data' in args and not 'url' in args:
raise AddProductError(
"Missing url or data; at least one is required")
# Set up to pass input data to the worker task
params = {
"task_type": "ADD_PRODUCT",
"addProduct_id": ap.id,
}
if 'data' in args and 'url' in args:
logger.warning(
"add_product: only one of 'data' or 'url' should be used")
# pathname of the product data for the product
if 'data' in args:
req_data = args["data"]
if not req_data or None == req_data or len(req_data) < 1:
logger.warning("add_product: empty 'data' in input")
else:
params['product'] = req_data.encode('ascii', 'ignore')
# or url if any
if 'url' in args:
req_url = args["url"]
if not req_url or req_url == None or len(req_url) < 2:
logger.warning("add_product: empty 'url' in input")
else:
params["url"] = req_url.encode('ascii', 'ignore')
# pathname of the metadata for the product
if "metadata" in args:
params["metadata"] = args["metadata"].encode('ascii', 'ignore')
# The coverage ID of an existing product, to be replaced
if "productID" in args:
productID = args["productID"]
if not productID or None == productID or len(productID) < 1:
logger.warning("add_product: empty 'productID'")
else:
params["covId"] = productID.encode('ascii', 'ignore')
# for time-zone aware dates use this one instead:
# ap.info_date = datetime.datetime.utcnow().replace(tzinfo=utc)
# process addProduct request by work_flow_manager
wfm = work_flow_manager.WorkFlowManager.Instance()
current_task = work_flow_manager.WorkerTask(params)
wfm.put_task_to_queue(current_task) # exectues add_product_wfunc
resp['opId'] = ap.id
resp['status'] = 0
if ap.id > IE_MAX_ADDPRODOPS:
error_str = ("Internal Error in add_product: " +
"ap.id exceeded max (" + ` IE_MAX_ADDPRODOPS ` + ")")
status = 103
logger.error(error_str)
if ap.id > IE_PURGE_ADDPRODOPS:
purge_old_ops()
except AddProductError as ap_err:
logger.error("AddProductError in add_product: " + ap_err.msg)
status = 101
error_str = ap_err.msg
except OSError as e:
logger.error("OSError in add_product: " + ` e `)
status = 102
error_str = ("Internal Error: " + ` e `)
except Exception as e:
status = 50
error_str = "Unexpected exception: " + e.__class__.__name__
logger.error("Exception in add_product: " + ` e `)
from settings import IE_DEBUG
if IE_DEBUG > 0:
traceback.print_exc(4, sys.stdout)
apID = ap.id
ap = None
ap = models.ProductInfo.objects.filter(id=apID)[0]
resp["status"] = status
if 0 != status:
if None != ap: ap.info_status = "failed"
if error_str:
if None != ap: info_error = error_str
resp["errorString"] = error_str
logger.warning("addProduct request error. Status: " +`status`+\
", error: " + error_str)
else:
if resp['opId']:
logger.info("addProduct request status " +`status` +\
", opId="+`resp['opId']`)
else:
logger.warning("addProduct: No opId nor error string, " +\
"status: "+`status`)
return resp
def add_local_product_core(request, ncn_id, template, aftersave=None):
# add local product to the related scenario
msg = "The new product has been successfully added"
if request.method == 'POST':
wfm = work_flow_manager.WorkFlowManager.Instance()
scenario = models.Scenario.objects.get(ncn_id=ncn_id)
sc_id = scenario.id
if wfm.lock_scenario(sc_id):
try:
form = forms.AddLocalProductForm(request.POST, request.FILES)
if form.is_valid() and form.is_multipart():
full_directory_name, directory_name = create_dl_dir(
ncn_id + "_")
full_directory_name = full_directory_name.encode(
'ascii', 'ignore')
logger.info("Product directory name:" +
full_directory_name)
if 'metadataFile' in request.FILES:
saveFile(request.FILES['metadataFile'],
full_directory_name)
meta = request.FILES['metadataFile']._get_name(
).encode('ascii', 'ignore')
else:
meta = None
saveFile(request.FILES['rasterFile'], full_directory_name)
data = request.FILES['rasterFile']._get_name().encode(
'ascii', 'ignore')
scripts = []
if scenario.oda_server_ingest != 0:
scripts.append(
os.path.join(IE_SCRIPTS_DIR,
IE_DEFAULT_ADDPROD_SCRIPT))
if len(scripts) == 0:
logger.warning("Scenario '%s' name='%s' does not have scripts to ingest, proceeding regardless." \
% (ncn_id,scenario.scenario_name))
# send request/task to work-flow-manager to run script
current_task = work_flow_manager.WorkerTask({
"scenario_id":
sc_id,
"ncn_id":
ncn_id,
"task_type":
"INGEST_LOCAL_PROD",
"scripts":
scripts,
"cat_registration":
scenario.cat_registration,
"s2_preprocess":
scenario.s2_preprocess,
"dir_path":
full_directory_name,
"metadata":
meta,
"data":
data,
})
wfm.put_task_to_queue(current_task)
logger.info("Local ingest scenario - submitted to queue: " +
"id=%d, ncn_id='%s', name='%s'" \
% (sc_id, ncn_id, scenario.scenario_name))
else:
logger.warning(
'The add product form has not been fully/correctly filled'
)
except Exception, e:
logger.error("Failed to add local product to Scenario '%s' name=%s" \
% (ncn_id, scenario.scenario_name) + \
'exception = ' + `e`)
finally:
def odaAddLocalProductOld(request, ncn_id):
oda_init(request)
# add local product to the related scenario
logger.info("addLocalProduct to ncn_id " + ncn_id)
msg = "The new product has been successfully added"
if request.method == 'POST':
wfm = work_flow_manager.WorkFlowManager.Instance()
scenario = models.Scenario.objects.get(ncn_id)
sc_id = scenario.id
#if not wfm.lock_scenario(sc_id):
# return getAddProductResult(request, sc_id, msg)
if not wfm.lock_scenario(sc_id):
logger.warning("Ingest Scenario refused: " +
"Scenario '%s' name=%s is busy." %
(scenario.ncn_id, scenario.scenario_name))
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
form = forms.AddLocalProductForm(request.POST, request.FILES)
if form.is_valid() and form.is_multipart():
try:
sc_ncn_id = scenario.ncn_id
full_directory_name, directory_name = create_dl_dir(sc_ncn_id +
"_")
full_directory_name = full_directory_name.encode(
'ascii', 'ignore')
logger.info("Product directory name:" + full_directory_name)
except:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to generate a new product directory name")
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
try:
saveFile(request.FILES['metadataFile'], full_directory_name)
except IOError, e:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to save product's metadata file")
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
try:
saveFile(request.FILES['rasterFile'], full_directory_name)
except IOError, e:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to save product's raster file")
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
scripts = get_scenario_script_paths(scenario)
if len(scripts) > 0:
# send request/task to work-flow-manager to run script
current_task = work_flow_manager.WorkerTask({
"scenario_id":
sc_id,
"ncn_id":
sc_ncn_id,
"task_type":
"INGEST_LOCAL_PROD",
"scripts":
scripts,
"cat_registration":
scenario.cat_registration,
"s2_preprocess":
scenario.s2_preprocess,
"dir_path":
full_directory_name,
"metadata":
request.FILES['metadataFile']._get_name(),
"data":
request.FILES['rasterFile']._get_name(),
})
wfm.put_task_to_queue(current_task)
logger.info('Operation: ingest scenario: id=%d name=%s' \
% (scenario.id,scenario.scenario_name))
else:
logger.error("NOT INGESTING: Scenario '%s' name=%s does not have scripts to ingest." \
% (scenario.ncn_id,scenario.scenario_name))
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')