def download_urls(urls_with_dirs):
dar = dar_builder.build_DAR(urls_with_dirs)
dmcontroller = DownloadManagerController.Instance()
status, dar_url, dm_dar_id = dmcontroller.submit_dar(dar)
if status != "OK":
raise DMError("DAR submit problem, status:" + status)
return dar_url, dm_dar_id
def stop_products_dl(product_list):
logger.info("Stopping products download")
dmcontroller = DownloadManagerController.Instance()
dm_url = dmcontroller._dm_url
for p in product_list:
if "productProgress" in product:
progress = product["productProgress"]
if progress["status"] == "COMPLETED":
# no point cancelling this one
continue
uuid = p['uuid']
url = dm_url + DM_PRODUCT_CANCEL_TEMPLATE % uuid
try:
dm_response = json.loads(read_from_url(url))
except urllib2.URLError as e:
logger.warning("Error from DM while cancelling download: " + ` e `)
def ingestion_logic(scid, scenario_data):
root_dl_dir = DownloadManagerController.Instance()._download_dir
custom = scenario_data['extraconditions']
eoids = scenario_data['dssids']
if scenario_data['dsrc_type'] != DSRC_EOWCS_CHOICE:
logger.warning('Data source type ' + scenario_data['dsrc_type'] +
' is not implemented')
if not os.access(root_dl_dir, os.R_OK | os.W_OK):
raise IngestionError("Cannot write/read " + root_dl_dir)
if 0 != DEBUG_MAX_DEOCS_URLS:
logger.info(" DEBUG_MAX_DEOCS_URLS = " + ` DEBUG_MAX_DEOCS_URLS `)
if 0 != DEBUG_MAX_GETCOV_URLS:
logger.info(" DEBUG_MAX_GETCOV_URLS = " + ` DEBUG_MAX_GETCOV_URLS `)
nreqs = 0
retval = (0, None, None, None, "", None)
scenario_data["sc_id"] = scid
scenario_data["custom"] = custom
ncn_id = scenario_data["ncn_id"]
dl_requests = get_download_URLs(scenario_data, eoids)
if not dl_requests or 0 == len(dl_requests):
logger.warning( ` ncn_id ` + ": no GetCoverage requests generated")
retval = (0, None, None, None, "NO_ACTION", None)
else:
if check_status_stopping(scid):
raise StopRequest("Stop Request")
nreqs = len(dl_requests)
logger.info( ` ncn_id ` + ": Submitting " + ` nreqs ` +
" URLs to the Download Manager")
dl_dir, dar_url, dar_id = \
request_download(scenario_data["ncn_id"], scid, dl_requests)
dl_errors, failed_dirs, failed_urls = wait_for_download(
scid, dar_url, dar_id, ncn_id)
if len(failed_urls) > 0:
logger.warning("Failed downloads for "+`ncn_id`+":\n" +\
'\n'.join(failed_urls))
logger.info("Products for scenario " + ncn_id + " downloaded to " +
dl_dir)
retval = (dl_errors, dl_dir, dar_url, dar_id, "OK", failed_dirs)
return retval
def create_dl_dir(leaf_name_root, extradir=None):
# structure of dirs created, shown by example:
# 1) no extradir:
# 2013
# +-10
# +-leaf_name_root05_120101_55abbbc
# ...
#
# 2) with extradir:
# 2013
# +-10
# +-extradir
# +-leaf_name_root05_120102_44deeef
# ...
root_dl_dir = DownloadManagerController.Instance().get_download_dir()
st_time = time.gmtime()
yr_name = str(st_time.tm_year)
mon_name = str(st_time.tm_mon)
leaf_dir_name = mkFname(leaf_name_root)
if extradir:
path = (yr_name, mon_name, extradir, leaf_dir_name)
else:
path = (yr_name, mon_name, leaf_dir_name)
rel_path = os.path.join(*path)
full_path = root_dl_dir
for p in path[:-1]:
full_path = os.path.join(full_path, p)
check_or_make_dir(full_path, logger)
full_path = os.path.join(full_path, leaf_dir_name)
make_new_dir(full_path, logger)
return full_path, rel_path
def get_dar_list():
dmcontroller = DownloadManagerController.Instance()
dm_url = dmcontroller._dm_url
url = dm_url + DM_DAR_STATUS_COMMAND
waiting = True
ts = time.time()
tdiff = 0
dar_status = []
err = None
while waiting:
tdiff = time.time() - ts
if tdiff > IE_DM_MAXWAIT2:
raise DMError("Unable to get DAR, timeout waiting for DM, " +
` err `)
try:
dar_status = json.loads(read_from_url(url))
waiting = False
except urllib2.HTTPError as e:
err = e
time.sleep(2)
if not "dataAccessRequests" in dar_status:
raise DMError("Bad DAR status from DM; no 'dataAccessRequests' found.")
return dar_status["dataAccessRequests"]
import work_flow_manager
import logging
import os
from utils import build_aoi_toi
from ingestion_logic import get_dssids_from_pf
from settings import \
MEDIA_ROOT, \
LOGGING_FILE, \
BROWSER_N_LOGLINES
from dm_control import DownloadManagerController
dmcontroller = DownloadManagerController.Instance()
@dajaxice_register(method='POST')
def delete_scenario_wfm(request, ncn_id):
return simplejson.dumps(views.delete_scenario_core(ncn_id=ncn_id))
@dajaxice_register(method='GET')
def test1(request):
user = request.user
scenarios = user.scenario_set.all()
results = []
for scenario in scenarios:
ss = scenario.scenariostatus
result = [