def write_md(prodId, metadata):
try:
dl_dir, rp = create_dl_dir(prodId+"_", UQMD_SUBDIR)
except OSError as e:
return 31, "Cannot create directory for storing MD", None
leaf_root = mkFname(prodId+"_")
meta_fp, meta_fname = open_unique_file(dl_dir, leaf_root, MAX_NON_UNIQ_FILES)
meta_fp.write(metadata)
meta_fp.close()
return 0, '', meta_fname
def write_md(prodId, metadata):
try:
dl_dir, rp = create_dl_dir(prodId + "_", UQMD_SUBDIR)
except OSError as e:
return 31, "Cannot create directory for storing MD", None
leaf_root = mkFname(prodId + "_")
meta_fp, meta_fname = open_unique_file(dl_dir, leaf_root,
MAX_NON_UNIQ_FILES)
meta_fp.write(metadata)
meta_fp.close()
return 0, '', meta_fname
def add_product_wfunc(parameters):
#
# Executed by a worker thread from the the work_flow_manager
# Parameters - as set up in 'add_product' below:
# "task_type",
# "addProduct_id",
# "metadata",
# "product",
# "url"
# "covId"
#
# One of product or url are required
#
try:
addProduct = models.ProductInfo.objects.filter(
id=parameters["addProduct_id"])[0]
addProduct.info_status = "processing"
dl_dir, rel_path = create_dl_dir("ap_", ADDPRODUCT_SUBDIR)
metadata = None
product = None
if 'url' in parameters:
metadata, product = download_rem_product(
dl_dir, rel_path, parameters["url"])
elif 'product' in parameters:
replace = True if "covId" in parameters else False
metadata, product = prepare_local_product(
dl_dir,
parameters.get("metadata"),
parameters["product"],
replace)
else:
raise AddProductError("Neither product nor url in input.")
if not product or (not metadata and 'url' in parameters):
raise AddProductError(
"internal error: Could not determine base names.")
# Set up parameters for the script
# The shell script is invoked by the IE with the following params:
# <action>
# -response=<filename>
# data
# metadata
#
# where <action> is one of:
# -replace=<existing_prod_id>
# -add
# Note '-add' is used if no existing product id is known, e.g.
# when addProduct is invoked from the RCOMS system.
#
# The response filename is where the script writes the new prodID,
# assuming all goes well.
#
script = os.path.join(IE_SCRIPTS_DIR, IE_DEFAULT_ADDPROD_SCRIPT)
resp_fname = mkFname("addProdResp_")
resp_full_fname = os.path.join(dl_dir,resp_fname)
command = [script]
if "covId" in parameters:
command.append("-replace="+parameters["covId"])
else:
command.append("-add")
command.append("-dldir="+dl_dir)
command.append("-response="+resp_fname)
if metadata is not None:
command.append("-meta="+get_base_fname(metadata))
command.append("-data="+get_base_fname(product))
process_status = subprocess.call(command)
if 0 != process_status:
error_str ='AddProduct script returned status:'+`process_status`
addProduct.info_error = error_str
addProduct.info_status = "failed"
logger.error("add_product_wfunc: " + error_str)
else:
cid = parse_response_file(addProduct, resp_full_fname)
logger.info(
"add_product_wfunc: script executed successfully, "+\
"new productId="+cid)
addProduct.info_status = "success"
except AddProductError as ap_err:
logger.error("AddProductError in add_product_wfunc: " + ap_err.msg)
addProduct.info_status = "failed"
addProduct.info_error = ap_err.msg
except Exception as e:
logger.error("add_product_wfunc: Exception: " + `e`)
addProduct.info_error = "Error %s" % `e`
addProduct.info_status = "failed"
from settings import IE_DEBUG
if IE_DEBUG>0:
logger.debug(traceback.format_exc(4))
finally:
addProduct.save()
def add_product_wfunc(parameters):
#
# Executed by a worker thread from the the work_flow_manager
# Parameters - as set up in 'add_product' below:
# "task_type",
# "addProduct_id",
# "metadata",
# "product",
# "url"
# "covId"
#
# One of product or url are required
#
try:
addProduct = models.ProductInfo.objects.filter(
id=parameters["addProduct_id"])[0]
addProduct.info_status = "processing"
dl_dir, rel_path = create_dl_dir("ap_", ADDPRODUCT_SUBDIR)
metadata = None
product = None
if 'url' in parameters:
metadata, product = download_rem_product(dl_dir, rel_path,
parameters["url"])
elif 'product' in parameters:
replace = True if "covId" in parameters else False
metadata, product = prepare_local_product(
dl_dir, parameters.get("metadata"), parameters["product"],
replace)
else:
raise AddProductError("Neither product nor url in input.")
if not product or (not metadata and 'url' in parameters):
raise AddProductError(
"internal error: Could not determine base names.")
# Set up parameters for the script
# The shell script is invoked by the IE with the following params:
# <action>
# -response=<filename>
# data
# metadata
#
# where <action> is one of:
# -replace=<existing_prod_id>
# -add
# Note '-add' is used if no existing product id is known, e.g.
# when addProduct is invoked from the RCOMS system.
#
# The response filename is where the script writes the new prodID,
# assuming all goes well.
#
script = os.path.join(IE_SCRIPTS_DIR, IE_DEFAULT_ADDPROD_SCRIPT)
resp_fname = mkFname("addProdResp_")
resp_full_fname = os.path.join(dl_dir, resp_fname)
command = [script]
if "covId" in parameters:
command.append("-replace=" + parameters["covId"])
else:
command.append("-add")
command.append("-dldir=" + dl_dir)
command.append("-response=" + resp_fname)
if metadata is not None:
command.append("-meta=" + get_base_fname(metadata))
command.append("-data=" + get_base_fname(product))
process_status = subprocess.call(command)
if 0 != process_status:
error_str = 'AddProduct script returned status:' + ` process_status `
addProduct.info_error = error_str
addProduct.info_status = "failed"
logger.error("add_product_wfunc: " + error_str)
else:
cid = parse_response_file(addProduct, resp_full_fname)
logger.info(
"add_product_wfunc: script executed successfully, "+\
"new productId="+cid)
addProduct.info_status = "success"
except AddProductError as ap_err:
logger.error("AddProductError in add_product_wfunc: " + ap_err.msg)
addProduct.info_status = "failed"
addProduct.info_error = ap_err.msg
except Exception as e:
logger.error("add_product_wfunc: Exception: " + ` e `)
addProduct.info_error = "Error %s" % ` e `
addProduct.info_status = "failed"
from settings import IE_DEBUG
if IE_DEBUG > 0:
logger.debug(traceback.format_exc(4))
finally:
addProduct.save()
def add_local_product_core(request, ncn_id, template, aftersave=None):
# add local product to the related scenario
msg = "The new product has been successfully added"
if request.method == 'POST':
wfm = work_flow_manager.WorkFlowManager.Instance()
scenario = models.Scenario.objects.get(ncn_id=ncn_id)
sc_id = scenario.id
if wfm.lock_scenario(sc_id):
try:
form = forms.AddLocalProductForm(request.POST, request.FILES)
if form.is_valid() and form.is_multipart():
full_directory_name, directory_name = create_dl_dir(ncn_id+"_")
full_directory_name = full_directory_name.encode('ascii','ignore')
logger.info("Product directory name:" + full_directory_name)
if 'metadataFile' in request.FILES:
saveFile(request.FILES['metadataFile'], full_directory_name)
meta = request.FILES['metadataFile']._get_name().encode('ascii','ignore')
else:
meta = None
saveFile(request.FILES['rasterFile'], full_directory_name)
data = request.FILES['rasterFile']._get_name().encode('ascii','ignore')
scripts = []
if scenario.oda_server_ingest != 0:
scripts.append(os.path.join(IE_SCRIPTS_DIR, IE_DEFAULT_ADDPROD_SCRIPT))
if len(scripts) == 0:
logger.warning("Scenario '%s' name='%s' does not have scripts to ingest, proceeding regardless." \
% (ncn_id,scenario.scenario_name))
# send request/task to work-flow-manager to run script
current_task = work_flow_manager.WorkerTask(
{"scenario_id": sc_id,
"ncn_id" : ncn_id,
"task_type": "INGEST_LOCAL_PROD",
"scripts" : scripts,
"cat_registration" : scenario.cat_registration,
"s2_preprocess" : scenario.s2_preprocess,
"dir_path" : full_directory_name,
"metadata" : meta,
"data" : data,
})
wfm.put_task_to_queue(current_task)
logger.info("Local ingest scenario - submitted to queue: " +
"id=%d, ncn_id='%s', name='%s'" \
% (sc_id, ncn_id, scenario.scenario_name))
else:
logger.warning('The add product form has not been fully/correctly filled')
except Exception, e:
logger.error("Failed to add local product to Scenario '%s' name=%s" \
% (ncn_id, scenario.scenario_name) + \
'exception = ' + `e`)
finally:
def odaAddLocalProductOld(request, ncn_id):
oda_init(request)
# add local product to the related scenario
logger.info("addLocalProduct to ncn_id "+ncn_id)
msg = "The new product has been successfully added"
if request.method == 'POST':
wfm = work_flow_manager.WorkFlowManager.Instance()
scenario = models.Scenario.objects.get(ncn_id)
sc_id = scenario.id
#if not wfm.lock_scenario(sc_id):
# return getAddProductResult(request, sc_id, msg)
if not wfm.lock_scenario(sc_id):
logger.warning("Ingest Scenario refused: "
+ "Scenario '%s' name=%s is busy." % (scenario.ncn_id, scenario.scenario_name))
return HttpResponseRedirect('http://127.0.0.1:'+port+'/scenario/overview/')
form = forms.AddLocalProductForm(request.POST, request.FILES)
if form.is_valid() and form.is_multipart():
try:
sc_ncn_id = scenario.ncn_id
full_directory_name, directory_name = create_dl_dir(sc_ncn_id+"_")
full_directory_name = full_directory_name.encode('ascii','ignore')
logger.info("Product directory name:" + full_directory_name)
except:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to generate a new product directory name")
return HttpResponseRedirect('http://127.0.0.1:'+port+'/scenario/overview/')
try:
saveFile(request.FILES['metadataFile'], full_directory_name)
except IOError, e:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to save product's metadata file")
return HttpResponseRedirect('http://127.0.0.1:'+port+'/scenario/overview/')
try:
saveFile(request.FILES['rasterFile'], full_directory_name)
except IOError, e:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to save product's raster file")
return HttpResponseRedirect('http://127.0.0.1:'+port+'/scenario/overview/')
scripts = get_scenario_script_paths(scenario)
if len(scripts) > 0:
# send request/task to work-flow-manager to run script
current_task = work_flow_manager.WorkerTask(
{"scenario_id": sc_id,
"ncn_id" : sc_ncn_id,
"task_type": "INGEST_LOCAL_PROD",
"scripts" : scripts,
"cat_registration" : scenario.cat_registration,
"s2_preprocess" : scenario.s2_preprocess,
"dir_path" : full_directory_name,
"metadata" : request.FILES['metadataFile']._get_name(),
"data" : request.FILES['rasterFile']._get_name(),
})
wfm.put_task_to_queue(current_task)
logger.info('Operation: ingest scenario: id=%d name=%s' \
% (scenario.id,scenario.scenario_name))
else:
logger.error("NOT INGESTING: Scenario '%s' name=%s does not have scripts to ingest." \
% (scenario.ncn_id,scenario.scenario_name))
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
return HttpResponseRedirect('http://127.0.0.1:'+port+'/scenario/overview/')
def add_local_product_core(request, ncn_id, template, aftersave=None):
# add local product to the related scenario
msg = "The new product has been successfully added"
if request.method == 'POST':
wfm = work_flow_manager.WorkFlowManager.Instance()
scenario = models.Scenario.objects.get(ncn_id=ncn_id)
sc_id = scenario.id
if wfm.lock_scenario(sc_id):
try:
form = forms.AddLocalProductForm(request.POST, request.FILES)
if form.is_valid() and form.is_multipart():
full_directory_name, directory_name = create_dl_dir(
ncn_id + "_")
full_directory_name = full_directory_name.encode(
'ascii', 'ignore')
logger.info("Product directory name:" +
full_directory_name)
if 'metadataFile' in request.FILES:
saveFile(request.FILES['metadataFile'],
full_directory_name)
meta = request.FILES['metadataFile']._get_name(
).encode('ascii', 'ignore')
else:
meta = None
saveFile(request.FILES['rasterFile'], full_directory_name)
data = request.FILES['rasterFile']._get_name().encode(
'ascii', 'ignore')
scripts = []
if scenario.oda_server_ingest != 0:
scripts.append(
os.path.join(IE_SCRIPTS_DIR,
IE_DEFAULT_ADDPROD_SCRIPT))
if len(scripts) == 0:
logger.warning("Scenario '%s' name='%s' does not have scripts to ingest, proceeding regardless." \
% (ncn_id,scenario.scenario_name))
# send request/task to work-flow-manager to run script
current_task = work_flow_manager.WorkerTask({
"scenario_id":
sc_id,
"ncn_id":
ncn_id,
"task_type":
"INGEST_LOCAL_PROD",
"scripts":
scripts,
"cat_registration":
scenario.cat_registration,
"s2_preprocess":
scenario.s2_preprocess,
"dir_path":
full_directory_name,
"metadata":
meta,
"data":
data,
})
wfm.put_task_to_queue(current_task)
logger.info("Local ingest scenario - submitted to queue: " +
"id=%d, ncn_id='%s', name='%s'" \
% (sc_id, ncn_id, scenario.scenario_name))
else:
logger.warning(
'The add product form has not been fully/correctly filled'
)
except Exception, e:
logger.error("Failed to add local product to Scenario '%s' name=%s" \
% (ncn_id, scenario.scenario_name) + \
'exception = ' + `e`)
finally:
def odaAddLocalProductOld(request, ncn_id):
oda_init(request)
# add local product to the related scenario
logger.info("addLocalProduct to ncn_id " + ncn_id)
msg = "The new product has been successfully added"
if request.method == 'POST':
wfm = work_flow_manager.WorkFlowManager.Instance()
scenario = models.Scenario.objects.get(ncn_id)
sc_id = scenario.id
#if not wfm.lock_scenario(sc_id):
# return getAddProductResult(request, sc_id, msg)
if not wfm.lock_scenario(sc_id):
logger.warning("Ingest Scenario refused: " +
"Scenario '%s' name=%s is busy." %
(scenario.ncn_id, scenario.scenario_name))
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
form = forms.AddLocalProductForm(request.POST, request.FILES)
if form.is_valid() and form.is_multipart():
try:
sc_ncn_id = scenario.ncn_id
full_directory_name, directory_name = create_dl_dir(sc_ncn_id +
"_")
full_directory_name = full_directory_name.encode(
'ascii', 'ignore')
logger.info("Product directory name:" + full_directory_name)
except:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to generate a new product directory name")
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
try:
saveFile(request.FILES['metadataFile'], full_directory_name)
except IOError, e:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to save product's metadata file")
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
try:
saveFile(request.FILES['rasterFile'], full_directory_name)
except IOError, e:
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
logger.error("Failed to save product's raster file")
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')
scripts = get_scenario_script_paths(scenario)
if len(scripts) > 0:
# send request/task to work-flow-manager to run script
current_task = work_flow_manager.WorkerTask({
"scenario_id":
sc_id,
"ncn_id":
sc_ncn_id,
"task_type":
"INGEST_LOCAL_PROD",
"scripts":
scripts,
"cat_registration":
scenario.cat_registration,
"s2_preprocess":
scenario.s2_preprocess,
"dir_path":
full_directory_name,
"metadata":
request.FILES['metadataFile']._get_name(),
"data":
request.FILES['rasterFile']._get_name(),
})
wfm.put_task_to_queue(current_task)
logger.info('Operation: ingest scenario: id=%d name=%s' \
% (scenario.id,scenario.scenario_name))
else:
logger.error("NOT INGESTING: Scenario '%s' name=%s does not have scripts to ingest." \
% (scenario.ncn_id,scenario.scenario_name))
wfm.set_scenario_status(0, sc_id, 1, 'IDLE', 0)
return HttpResponseRedirect('http://127.0.0.1:' + port +
'/scenario/overview/')