def _makecachedir(caltype):
cache = set_caches()
cachedir = join(cache["calibrations"], caltype)
if not exists(cachedir):
mkdir(cachedir)
return cachedir
def process_cal_requests(cal_requests, howmany=None):
"""
Conduct a search for calibration files for the passed list of calibration
requests. This passes the requests to the calibration_search() function,
and then examines the search results to see if a matching file, if any,
is cached. If not, then the calibration file is retrieved from the
archive.
If a calibration match is found by the calibration manager, a URL is
returned. This function will perform a cache inspection to see if the
matched calibraiton file is already present. If not, the calibration
will be downloaded and written to the cache. It is this path that is
returned in the dictionary structure. A path of 'None' indicates that no
calibration match was found.
Parameters
----------
cal_requests : list
A list of CalibrationRequest objects
howmany : int, optional
Maximum number of calibrations to return per request
(not passed to the server-side request but trims the returned list)
Returns
-------
calibration_records : dict
A set of science frames and matching calibrations.
Example
-------
The returned dictionary has the form::
{(ad): <filename_of_calibration_including_path>,
...
}
"""
calibration_records = {}
def _add_cal_record(rq, calfile):
calibration_records.update({rq.ad: calfile})
return
cache = set_caches()
for rq in cal_requests:
calname = None
calmd5 = None
calurl = None
calurl, calmd5 = calibration_search(rq, howmany=(howmany if howmany else 1))
if calurl is None:
log.error("START CALIBRATION SERVICE REPORT\n")
log.error(calmd5)
log.error("END CALIBRATION SERVICE REPORT\n")
warn = "No {} calibration file found for {}"
log.warning(warn.format(rq.caltype, rq.filename))
#_add_cal_record(rq, calname)
continue
calibs = []
for url, md5 in zip(calurl, calmd5):
log.info("Found calibration (url): {}".format(url))
components = urlparse(url)
calname = basename(components.path)
cachename, cachedir = _check_cache(calname, rq.caltype)
if cachename:
cached_md5 = generate_md5_digest(cachename)
if cached_md5 == md5:
log.stdinfo("Cached calibration {} matched.".format(cachename))
calibs.append(cachename)
else:
log.stdinfo("File {} is cached but".format(calname))
log.stdinfo("md5 checksums DO NOT MATCH")
log.stdinfo("Making request on calibration service")
log.stdinfo("Requesting URL {}".format(url))
try:
calname = get_request(url, cachename)
calibs.append(cachename)
except GetterError as err:
for message in err.messages:
log.error(message)
continue
log.status("Making request for {}".format(url))
fname = split(url)[1]
calname = join(cachedir, fname)
try:
calname = get_request(url, calname)
except GetterError as err:
for message in err.messages:
log.error(message)
else:
# hash compare
download_mdf5 = generate_md5_digest(calname)
if download_mdf5 == md5:
log.status("MD5 hash match. Download OK.")
calibs.append(calname)
else:
err = "MD5 hash of downloaded file does not match expected hash {}"
raise IOError(err.format(md5))
# If howmany=None, append the only file as a string, instead of the list
if calibs:
_add_cal_record(rq, calibs if howmany else calibs[0])
return calibration_records
def process_cal_requests(cal_requests, howmany=None):
"""
Conduct a search for calibration files for the passed list of calibration
requests. This passes the requests to the calibration_search() function,
and then examines the search results to see if a matching file, if any,
is cached. If not, then the calibration file is retrieved from the
archive.
If a calibration match is found by the calibration manager, a URL is
returned. This function will perform a cache inspection to see if the
matched calibraiton file is already present. If not, the calibration
will be downloaded and written to the cache. It is this path that is
returned in the dictionary structure. A path of 'None' indicates that no
calibration match was found.
Parameters
----------
cal_requests : <list>
A list of CalibrationRequest objects
howmany : <int>, optional
Maximum number of calibrations to return per request
(not passed to the server-side request but trims the returned list)
Returns
-------
calibration_records : <dict>
A set of science frames and matching calibrations.
Example
-------
The returned dictionary has the form::
{(ad): <filename_of_calibration_including_path>,
...
}
"""
calibration_records = {}
warn = "\tNo {} calibration file found for {}\n"
md5msg = "\tNo {} calibration found for {}\n"
def _add_cal_record(rq, calfile):
calibration_records.update({rq.ad: calfile})
return
cache = set_caches()
for rq in cal_requests:
calname = None
calmd5 = None
calurl = None
calurl, calmd5 = calibration_search(
rq, howmany=(howmany if howmany else 1))
if not calurl:
log.warning("START CALIBRATION SERVICE REPORT\n")
if not calmd5:
log.warning(md5msg.format(rq.caltype, rq.filename))
else:
log.warning("\t{}".format(calmd5))
log.warning(warn.format(rq.caltype, rq.filename))
log.warning("END CALIBRATION SERVICE REPORT\n")
continue
calibs = []
for url, md5 in zip(calurl, calmd5):
log.info("Found calibration (url): {}".format(url))
components = urlparse(url)
calname = basename(components.path)
cachename, cachedir = _check_cache(calname, rq.caltype)
if cachename:
cached_md5 = generate_md5_digest(cachename)
if cached_md5 == md5:
log.stdinfo(
"Cached calibration {} matched.".format(cachename))
calibs.append(cachename)
else:
log.stdinfo("File {} is cached but".format(calname))
log.stdinfo("md5 checksums DO NOT MATCH")
log.stdinfo("Making request on calibration service")
log.stdinfo("Requesting URL {}".format(url))
try:
calname = get_request(url, cachename)
calibs.append(cachename)
except GetterError as err:
for message in err.messages:
log.error(message)
continue
log.status("Making request for {}".format(url))
fname = split(url)[1]
calname = join(cachedir, fname)
try:
calname = get_request(url, calname)
except GetterError as err:
for message in err.messages:
log.error(message)
else:
# hash compare
download_mdf5 = generate_md5_digest(calname)
if download_mdf5 == md5:
log.status("MD5 hash match. Download OK.")
calibs.append(calname)
else:
err = "MD5 hash of downloaded file does not match expected hash {}"
raise OSError(err.format(md5))
# If howmany=None, append the only file as a string, instead of the list
if calibs:
_add_cal_record(rq, calibs if howmany else calibs[0])
return calibration_records
def _makecachedir(caltype):
cache = set_caches()
cachedir = join(cache["calibrations"], caltype)
makedirs(cachedir, exist_ok=True)
return cachedir
def _makecachedir(caltype):
cache = set_caches()
cachedir = join(cache["calibrations"], caltype)
if not exists(cachedir):
mkdir(cachedir)
return cachedir
