def executeMirroring(srvObj, iteration):
logger.info('executeMirroring for iteration %d', iteration)
try:
rx_timeout = 30 * 60
if srvObj.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(srvObj.getCfg().getVal("Mirroring[1].rx_timeout"))
pars = {
'mirror_cluster': 2,
'iteration': str(iteration),
'rx_timeout': rx_timeout,
'n_threads': getNumberOfSimultaneousFetchesPerServer(srvObj)
}
host, port = srvObj.get_self_endpoint()
ngamsHttpUtils.httpGet(host,
port,
'MIRREXEC',
pars=pars,
timeout=rx_timeout)
# TODO look at the response code
except Exception:
logger.exception("Mirroring failed")
finally:
failRemainingTransfers(srvObj, iteration)
logger.info('executeMirroring for iteration %d complete', iteration)
# remove some of the older bookkeeping entries
clean_mirroring_bookkeeping_entries(srvObj)
def execute_mirroring(ngams_server, iteration):
logger.info('executeMirroring for iteration %d', iteration)
try:
rx_timeout = 30 * 60
if ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(
ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"))
pars = {
'mirror_cluster': 2,
'iteration': str(iteration),
'rx_timeout': rx_timeout,
'n_threads': get_num_simultaneous_fetches_per_server(ngams_server)
}
host, port = ngams_server.get_self_endpoint()
# it is important to not let this operation time out. If it times out then the files being fetched will
# be eligable for re-fetching even though the spawned threads may still be executing. Chaos ensues.
ngamsHttpUtils.httpGet(host, port, 'MIRREXEC', pars=pars)
except Exception:
logger.exception("MIRREXEC command for iteration %d has failed",
iteration)
finally:
fail_remaining_transfers(ngams_server, iteration)
logger.info(
"MIRREXEC command for iteration %d has successfully completed",
iteration)
# Remove some of the older bookkeeping entries
clean_mirroring_bookkeeping_entries(ngams_server)
def test_HttpRedirection_01(self):
"""
Synopsis:
Test the HTTP redirection, contacted node redirects to sub-node.
Description:
The purpose of this test case is to verify that the HTTP redirection
works in connection with the RETRIEVE Command.
Expected Result:
The contacted node should detect that the file requested is stored
on another node (sub-node). It should resolve the address and return
an HTTP re-direction response to the client Python-API, which
internally will pick up the file.
Test Steps:
- Start simulated cluster (Proxy Mode: Off).
- Archive file onto MNU, archive file onto NCU.
- Submit RETRIEVE to retrieve file on NCU.
- Check that an HTTP redirection response was returned such
that the file was retrieved directly from the sub-node.
Remarks:
...
Test Data:
...
"""
nmuCfgPars = [["NgamsCfg.Server[1].ProxyMode", "0"],
["NgamsCfg.Log[1].LocalLogLevel", "4"]]
self.prepCluster(((8000, nmuCfgPars), 8011))
self.assertStatus(sendPclCmd(port=8000).archive("src/SmallFile.fits"))
self.assertStatus(sendPclCmd(port=8011).archive("src/SmallFile.fits"))
# The ngamsPClient handles redirects automatically,
# but we want to manually check here that things are right
fileId = "TEST.2001-05-08T15:25:00.123"
pars = (("file_id", fileId), )
resp = ngamsHttpUtils.httpGet('127.0.0.1', 8000, 'RETRIEVE', pars=pars)
self.assertEqual(303, resp.status)
resp.close()
# Follow the Location, we should get it now
host, port = resp.getheader('Location').split('/')[2].split(':')
port = int(port)
resp = ngamsHttpUtils.httpGet(host, port, 'RETRIEVE', pars=pars)
self.assertEqual(200, resp.status)
resp.close()
def _test_partial_retrieval(self, n_parts, file_size, full):
part_size, mod = divmod(file_size, n_parts)
if mod:
part_size += 1
piece_by_piece = io.BytesIO()
for n in range(n_parts):
offset = n * part_size
response = ngamsHttpUtils.httpGet(
'127.0.0.1',
8888,
'RETRIEVE',
pars=(('file_id', 'source'), ),
hdrs={'Range': 'bytes=%d-' % (offset, )})
with contextlib.closing(response):
total_read = 0
while total_read < part_size:
to_read = part_size - total_read
data = response.read(to_read)
if not data:
break
total_read += len(data)
piece_by_piece.write(data)
self.assertEqual(file_size, piece_by_piece.tell())
self.assertEqual(full.getvalue(), piece_by_piece.getvalue())
def delete_ngas_file(host, port, file_id, file_version, disk_id, timeout = 10):
pars = {'file_id': file_id,
'file_version': file_version,
'disk_id': disk_id}
resp = ngamsHttpUtils.httpGet(host, port, 'CACHEDEL', pars=pars, timeout=timeout)
with contextlib.closing(resp):
return resp.status
def _test_help(self):
self.prepExtSrv()
resp = ngamsHttpUtils.httpGet('localhost', 8888, 'HELP')
with contextlib.closing(resp):
self.assertEqual(resp.status, 308)
self.assertEqual(resp.getheader('location'),
'https://ngas.readthedocs.io')
def http_opener(host, port, file_id, file_version, srvObj):
pars = [('file_id', file_id), ('file_version', file_version)]
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
return ngamsHttpUtils.httpGet(host,
port,
NGAMS_RETRIEVE_CMD,
pars=pars,
timeout=30,
auth=authHdr)
def _checkFileAccess(srvObj,
reqPropsObj,
httpRef,
fileId,
fileVersion=-1,
diskId=""):
"""
Check if a file is accessible either on the local host or on a remotely
located host.
srvObj: Instance of the server object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
fileId: File ID (string).
fileVersion: File Version (integer).
diskId: Disk ID (string).
Returns: Returns message indicating if the file is available
(string).
"""
T = TRACE()
logger.debug("Checking for access to file with ID: %s", fileId)
# Check if the file is located on this host, or if the request should be
# forwarded (if this server should act as proxy).
location, fileHost, ipAddress, filePortNo, mountPoint, filename, fileId,\
fileVersion, mimeType =\
ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVersion,
diskId)
if (location != NGAMS_HOST_LOCAL):
# Go and get it!
host, port = srvObj.get_remote_server_endpoint(fileHost)
pars = (('file_access', fileId), ('file_version', fileVersion),
('disk_id', diskId))
resp = ngamsHttpUtils.httpGet(host, port, 'STATUS', pars, timeout=60)
with contextlib.closing(resp):
return ngamsStatus.to_status(resp, fileHost, 'STATUS').getMessage()
else:
# First check if this system allows for Retrieve Requests.
if (not srvObj.getCfg().getAllowRetrieveReq()):
msg = genLog("NGAMS_INFO_SERVICE_UNAVAIL", ["File Retrieval"])
else:
fileHost = "%s:%d" % (getHostName(), filePortNo)
msg = genLog("NGAMS_INFO_FILE_AVAIL",
[fileId + "/Version: " + str(fileVersion), fileHost])
return msg
def _test_unsupported_checksum(self, crc_variant):
# Ask for a variant that bbcp doesn't support, it should fail
self.prepExtSrv()
query_args = {'filename': '/bin/cp',
'bnum_streams': 2,
'mime_type': 'application/octet-stream',
'crc_variant': crc_variant}
response = ngamsHttpUtils.httpGet('localhost', 8888, 'BBCPARC',
pars=query_args, timeout=50)
with contextlib.closing(response):
self.assertNotEqual(200, response.status)
def send_mirrexec_command(self):
"""
Send MIRREXEC command to the input source_node
INPUT:
source_node string, Target node to send MIRREXEC
n_threads int, Number of threads per source-target connection
rx_timeout int, the socket timeout time in seconds
RETURNS: Void
"""
# Print log info
logger.info("sending MIRREXEC command to %s with (n_threads=%d)",
self.target_node, self.n_threads)
try:
host, port = self.target_node.split(":")
pars = {
'n_threads': str(self.n_threads),
'rx_timeout': str(self.rx_timeout),
'iteration': str(self.iteration)
}
start = time.time()
response = ngamsHttpUtils.httpGet(host,
int(port),
'MIRREXEC',
pars=pars,
timeout=self.rx_timeout)
with contextlib.closing(response):
failed = response.status != NGAMS_HTTP_SUCCESS or NGAMS_FAILURE in response.read(
)
elapsed_time = (time.time() - start)
# Print log info
if failed:
logger.error(
"MIRREXEC command sent to %s with (n_threads=%d) was handled with status FAILURE in %f [s]",
self.target_node, self.n_threads, elapsed_time)
else:
logger.info(
"MIRREXEC command sent to %s with (n_threads=%d) was handled with status SUCCESS in %f [s]",
self.target_node, self.n_threads, elapsed_time)
except:
logger.exception("Problems sending MIRREXEC command to %s",
self.target_node)
# Return Void
return
# EOF
def _test_correct_checksum_with_msb(self, crc_variant, msb):
_, db = self.prepExtSrv()
# Generate content until the most significant bit of the checksum is
# 1 or 0, as required
content = [b'\0'] * 1024
while True:
content[random.randint(0,
1023)] = six.b(chr(random.randint(0, 255)))
f = io.BytesIO(b''.join(content))
expected_crc = ngamsFileUtils.get_checksum(64 * 1024 * 240, f,
crc_variant)
crc_msb = (expected_crc & 0xffffffff) >> 31
if crc_msb == 0 and msb == 0:
break
elif crc_msb == 1 and msb == 1:
break
# This file needs to actually be under /tmp because our BBCP command
# rejects file pulls from some hardcoded locations (/dev/, /var, etc)
fname = ('/tmp/dummy')
with open(fname, 'wb') as f:
f.write(b''.join(content))
query_args = {
'filename': fname,
'bnum_streams': 2,
'mime_type': 'application/octet-stream',
'crc_variant': crc_variant
}
# Archive with BBCP first, then with QARCHIVE
for cmd in ('BBCPARC', 'QARCHIVE'):
response = ngamsHttpUtils.httpGet('localhost',
8888,
cmd,
pars=query_args,
timeout=50)
with contextlib.closing(response):
self.assertEqual(200, response.status)
# Both checksums are equal (i.e., when put in a set the set has one element)
checksums = db.query2(
'SELECT checksum FROM ngas_files ORDER BY file_version ASC')
checksums = set(c[0] for c in checksums)
self.assertEqual(1, len(checksums))
# And they are equal to the expected value
self.assertEqual(str(expected_crc), str(next(iter(checksums))))
self.terminateAllServer()
def _do_get(self, host, port, cmd, pars, hdrs):
start = time.time()
res = ngamsHttpUtils.httpGet(host,
port,
cmd,
pars=pars,
hdrs=hdrs,
timeout=self.timeout,
auth=self.basic_auth)
delta = time.time() - start
logger.debug("Command: %s to %s:%d handled in %.3f [s]", cmd, host,
port, delta)
return res
def test_BBCPArchive(self):
"""
Synopsis:
Test BBCP archive plugin
"""
self.prepExtSrv()
query_args = {'filename': '/bin/cp',
'bnum_streams': 2,
'mime_type': 'application/octet-stream'}
response = ngamsHttpUtils.httpGet('localhost', 8888, 'BBCPARC',
pars=query_args, timeout=50)
with contextlib.closing(response):
self.assertEqual(200, response.status)
def ngams_server_status(ngams_server):
"""
Check NGAMS server status
:param ngams_server: string, Full qualified name of ngams_server
:return: bool, True if active False if inactive
"""
try:
host, port = ngams_server.split(":")
response = ngamsHttpUtils.httpGet(host, int(port), 'STATUS')
with contextlib.closing(response):
return b'ONLINE' in response.read()
except Exception:
logger.info(
"Problem trying to reach %s, setting status to OFFLINE (0)",
ngams_server)
return False
def _sendCheckFileCmd(node, fileId, fileVersion):
"""
Send a CHECKFILE Command to the specified node. One of the following is
returned:
1. File OK: NGAMS_INFO_FILE_OK.
2. File Not OK: NGAMS_ER_FILE_NOK
3. Error: FAILURE.
node: Node to be contacted (node:port) (string).
fileId: ID of file to check (string).
fileVersion: Version of file to check (integer).
Returns: See above (NGAMS_INFO_FILE_OK | NGAMS_ER_FILE_NOK | FAILURE).
"""
T = TRACE(5)
cmdPars = [["file_id", fileId], ["file_version", fileVersion]]
data = None
try:
host, port = node.split(":")
logger.debug(
"Sending CHECKFILE Command for file: %s/%s to node: %s:%s", fileId,
str(fileVersion), host, str(port))
resp = ngamsHttpUtils.httpGet(host,
port,
NGAMS_CHECKFILE_CMD,
pars=cmdPars)
with contextlib.closing(resp):
data = resp.read()
except Exception:
logger.exception("Error contacting node %s", node)
if (data):
tmpStatObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data)
if (tmpStatObj.getMessage().find(NGAMS_INFO_FILE_OK) != -1):
return NGAMS_INFO_FILE_OK
else:
# Assume: NGAMS_ER_FILE_NOK.
return NGAMS_ER_FILE_NOK
else:
return NGAMS_FAILURE
def test_partial_retrieval(self):
self.prepExtSrv()
with open(tmp_path("source"), 'wb') as f:
f.write(os.urandom(1024))
self.archive(tmp_path("source"), mimeType='application/octet-stream')
# Retrieve the file fully first into memory
full = io.BytesIO()
response = ngamsHttpUtils.httpGet('127.0.0.1', 8888, 'RETRIEVE',
pars=(('file_id', 'source'),))
with contextlib.closing(response):
file_size = full.write(response.read())
# Now check that we can bring the same file in different number of parts
for n_parts in (1, 2, 3, 7, 11, 13, 14, 20, 100):
self._test_partial_retrieval(n_parts, file_size, full)
def test_too_many_requests(self):
saveInFile("tmp/handleHttpRequest_tmp", "handleHttpRequest_Block5secs")
self.prepExtSrv(srvModule="ngamsSrvTestDynReqCallBack",
cfgProps=(('NgamsCfg.Server[1].MaxSimReqs', '2'), ))
# Fire off two clients, each takes 5 seconds to finish
cl1, cl2 = sendPclCmd(), sendPclCmd()
threading.Thread(target=cl1.online).start()
threading.Thread(target=cl2.online).start()
# The third one should not pass through
# (assuming that 2 seconds were enough for the two clients
# to connect and be busy waiting for their reply)
time.sleep(2)
resp = ngamsHttpUtils.httpGet('127.0.0.1', 8888, 'ONLINE')
with contextlib.closing(resp):
self.assertEqual(NGAMS_HTTP_SERVICE_NA, resp.status)
def _do_get(self, host, port, cmd, pars, hdrs):
auth = None
if self.auth is not None:
auth = "Basic %s" % self.auth
start = time.time()
res = ngamsHttpUtils.httpGet(host,
port,
cmd,
pars=pars,
hdrs=hdrs,
timeout=self.timeout,
auth=auth)
delta = time.time() - start
logger.debug("Command: %s to %s:%d handled in %.3f [s]", cmd, host,
port, delta)
return res
def send_mirrexec_command(self):
"""
Send MIRREXEC command to the input source_node
"""
logger.info("Sending MIRREXEC command to %s with (n_threads=%d)",
self.target_node, self.num_threads)
try:
host, port = self.target_node.split(":")
pars = {
'n_threads': str(self.num_threads),
'rx_timeout': str(self.rx_timeout),
'iteration': str(self.iteration)
}
start_time = time.time()
# it is important to not let this operation time out. If it times out then the files being fetched will
# be eligable for re-fetching even though the spawned threads may still be executing. Chaos ensues.
response = ngamsHttpUtils.httpGet(host,
int(port),
'MIRREXEC',
pars=pars)
with contextlib.closing(response):
failed = response.status != NGAMS_HTTP_SUCCESS or NGAMS_FAILURE in utils.b2s(
response.read())
elapsed_time = time.time() - start_time
if failed:
logger.error(
"MIRREXEC command sent to %s with (n_threads=%d) was handled with status FAILURE "
"in %f [s]", self.target_node, self.num_threads,
elapsed_time)
else:
logger.info(
"MIRREXEC command sent to %s with (n_threads=%d) was handled with status SUCCESS in %f [s]",
self.target_node, self.num_threads, elapsed_time)
except Exception:
logger.exception("Problems sending MIRREXEC command to %s",
self.target_node)
return
def saveToFile(srvObj, ngamsCfgObj, reqPropsObj, trgFilename, blockSize,
startByte):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
trgFilename: Target name for file where data will be
written (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
mutexDiskAccess: Require mutual exclusion for disk access (integer).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
source_host = reqPropsObj.fileinfo['sourceHost']
file_version = reqPropsObj.fileinfo['fileVersion']
file_id = reqPropsObj.fileinfo['fileId']
logger.info("Creating path: %s", trgFilename)
checkCreatePath(os.path.dirname(trgFilename))
rx_timeout = 30 * 60
if srvObj.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(srvObj.getCfg().getVal("Mirroring[1].rx_timeout"))
host, port = source_host.split(":")
pars = {
'file_version': file_version,
'targetHost': get_full_qualified_name(srvObj),
'targetLocation': trgFilename,
'file_id': file_id
}
start = time.time()
response = ngamsHttpUtils.httpGet(host,
int(port),
'RSYNC',
pars=pars,
timeout=rx_timeout)
with contextlib.closing(response):
data = response.read()
if 'FAILURE' in data:
raise Exception(data)
deltaTime = time.time() - start
msg = "Saved data in file: %s. Bytes received: %d. Time: %.3f s. " +\
"Rate: %.2f Bytes/s"
logger.info(msg, trgFilename, int(reqPropsObj.getBytesReceived()),
deltaTime, (float(reqPropsObj.getBytesReceived()) / deltaTime))
# now check the CRC value against what we expected
sourceChecksum = reqPropsObj.checksum
start = time.time()
crc = ngamsFileUtils.get_checksum(65536, trgFilename, 'crc32')
deltaTime = time.time() - start
logger.info("crc computed in %f [s]", deltaTime)
logger.info('source checksum: %s - current checksum: %d',
str(sourceChecksum), crc)
if (crc != int(sourceChecksum)):
msg = "checksum mismatch: source=" + str(
sourceChecksum) + ", received: " + str(crc)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
return [deltaTime, crc]
# EOF
def _assert_code(self, code, bauth=None, raw_auth=None, cmd='STATUS'):
auth = raw_auth or 'Basic ' + base64.b64encode(
bauth) if bauth else None
resp = ngamsHttpUtils.httpGet('127.0.0.1', 8888, cmd, auth=auth)
with contextlib.closing(resp):
self.assertEqual(code, resp.status)
def _handleCmdRetrieve(srvObj, reqPropsObj, httpRef):
"""
Carry out the action of a RETRIEVE command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of
actions done during the request handling
(ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
# For data files, retrieval must be enabled otherwise the request is
# rejected.
if (not srvObj.getCfg().getAllowRetrieveReq()):
errMsg = genLog("NGAMS_ER_ILL_REQ", ["Retrieve"])
raise Exception(errMsg)
# Previously this command allowed to retrieve the current logging file,
# the configuration file and any internal file. We don't do this anymore
# Get query information.
if 'ng_log' in reqPropsObj or 'cfg' in reqPropsObj or 'internal' in reqPropsObj:
raise Exception(
"ng_log, cfg and internal parameters not supported anymore")
# At least file_id must be specified if not an internal file has been
# requested.
if 'file_id' not in reqPropsObj or not reqPropsObj['file_id']:
errMsg = genLog("NGAMS_ER_RETRIEVE_CMD")
raise Exception(errMsg)
fileId = reqPropsObj.getHttpPar("file_id")
logger.debug("Handling request for file with ID: %s", fileId)
fileVer = -1
if (reqPropsObj.hasHttpPar("file_version")):
fileVer = int(reqPropsObj.getHttpPar("file_version"))
diskId = ""
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
hostId = ""
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
domain = ""
if (reqPropsObj.hasHttpPar("domain")):
domain = reqPropsObj.getHttpPar("domain")
quickLocation = True
if (reqPropsObj.hasHttpPar("quick_location")):
quickLocation = int(reqPropsObj.getHttpPar("quick_location"))
# First try the quick retrieve attempt, just try to get the first
# (and best?) suitable file which is online and located on a node in the
# same domain as the contacted node.
ipAddress = None
if (quickLocation):
location, host, ipAddress, port, mountPoint, filename,\
fileVersion, mimeType =\
ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId,
hostId, domain, diskId,
fileVer)
# If not located the quick way try the normal way.
if (not ipAddress):
# Locate the file best suiting the query and send it back if possible.
location, host, ipAddress, port, mountPoint, filename, fileId,\
fileVersion, mimeType =\
ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVer,
diskId, hostId, reqPropsObj)
# If still not located, try to contact associated NGAS sites to query
# if the file is available there.
# TODO:
if (not ipAddress):
pass
if (location == NGAMS_HOST_LOCAL):
# Get the file and send back the contents from this NGAS host.
srcFilename = os.path.normpath("{0}/{1}".format(mountPoint, filename))
# Perform the possible file staging
performStaging(srvObj, reqPropsObj, httpRef, srcFilename)
# Perform the possible processing requested.
procResult = performProcessing(srvObj, reqPropsObj, srcFilename,
mimeType)
elif location in (NGAMS_HOST_CLUSTER, NGAMS_HOST_REMOTE) and \
srvObj.getCfg().getProxyMode():
logger.debug("NG/AMS Server acting as proxy - requesting file with ID: %s " +\
"from NG/AMS Server on host/port: %s/%s",
fileId, host, str(port))
# Act as proxy - get the file from the NGAS host specified and
# send back the contents. The file is temporarily stored in the
# Processing Area.
procDir = ngamsHighLevelLib.genProcDirName(srvObj.getCfg())
checkCreatePath(procDir)
pars = []
for par in reqPropsObj.getHttpParNames():
pars.append([par, reqPropsObj.getHttpPar(par)])
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
timeout = float(
reqPropsObj['timeout']) if 'timeout' in reqPropsObj else 60
conn = ngamsHttpUtils.httpGet(ipAddress,
port,
NGAMS_RETRIEVE_CMD,
pars=pars,
timeout=timeout,
auth=authHdr)
hdrs = {h[0]: h[1] for h in conn.getheaders()}
dataSize = int(hdrs["content-length"])
tmpPars = ngamsLib.parseHttpHdr(hdrs["content-disposition"])
dataFilename = tmpPars["filename"]
data = ngamsHttpUtils.sizeaware(conn, dataSize)
httpRef.send_data(data, mimeType, fname=dataFilename)
return
else:
# No proxy mode: A redirection HTTP response is generated.
httpRef.redirect(ipAddress, port)
return
# Send back reply with the result(s) queried and possibly processed.
genReplyRetrieve(srvObj, reqPropsObj, httpRef, procResult)
def _create_remote_subscriptions(srvObj, stop_evt):
"""
Creates subscriptions in remote servers so they push their data into
ours.
"""
subscriptions_in_cfg = srvObj.cfg.getSubscriptionsDic()
subscriptions = [v for _, v in subscriptions_in_cfg.items()]
subscriptions_created = 0
while True:
# Done with all of them
if not subscriptions:
break
# Iterate over copy, since we modify the original inside the loop
for subscrObj in list(subscriptions):
if (not srvObj.getThreadRunPermission()):
logger.info("Terminating Subscriber thread ...")
return
our_host, our_port = srvObj.get_self_endpoint()
subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo()
# Not subscribing to ourselves
if srvObj.is_it_us(subs_host, subs_port):
logger.warning(
"Skipping subscription to %s:%d because that's us",
subs_host, subs_port)
return
# Create the URL that needs to be set on the remote end so we
# get subscribed to it.
# TODO: include reverse proxy information when we add support
# TODO: hardcoded http will need to be changed when we add support
# for https
url = 'http://%s:%d/%s' % (our_host, our_port, subscrObj.getUrl()
or 'QARCHIVE')
logger.info("Creating subscription to %s:%d with url=%s",
subs_host, subs_port, url)
pars = [["subscr_id", url], ["priority",
subscrObj.getPriority()],
["url", url], ["start_date",
toiso8601(local=True)]]
if subscrObj.getFilterPi():
pars.append(["filter_plug_in", subscrObj.getFilterPi()])
if subscrObj.getFilterPiPars():
pars.append(["plug_in_pars", subscrObj.getFilterPiPars()])
try:
# Issue SUBSCRIBE command
resp = ngamsHttpUtils.httpGet(subs_host,
subs_port,
cmd=NGAMS_SUBSCRIBE_CMD,
pars=pars)
with contextlib.closing(resp):
stat = ngamsStatus.to_status(resp, subscrObj.getHostId(),
NGAMS_SUBSCRIBE_CMD)
if stat.getStatus() != NGAMS_SUCCESS:
msg = "Unsuccessful NGAS XML response. Status: %s, message: %s. Will try again later"
logger.warning(msg, stat.getStatus(), stat.getMessage())
continue
subscriptions_created += 1
logger.info("Successfully subscribed to %s:%d with url=%s",
subs_host, subs_port, subscrObj.getUrl())
# Remember to unsubscribe when going Offline.
srvObj.getSubscrStatusList().append(subscrObj)
subscriptions.remove(subscrObj)
except:
logger.exception(
"Error while adding subscription, will try later")
if stop_evt.wait(10):
return
if subscriptions_created:
logger.info("Successfully established %d Subscription(s)",
subscriptions_created)
else:
logger.info("No Subscriptions established")
def handleOffline(srvObj, reqPropsObj=None):
"""
Carry out the actions to put the system in Offline state (Standby).
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: NG/AMS request properties object (ngamsReqProps).
stopJanitorThr: Setting this to 0, the function will not try to
stop the Janitor Thread. The reason for this is that
the Janitor Thread may bring the server to Offline
(integer/0|1).
Returns: Void.
"""
# Stop/delete Janitor Thread + Data Check Thread + inform other
# possible threads to stop execution (if running).
srvObj.stopJanitorThread()
srvObj.stopDataCheckThread()
ngamsSubscriptionThread.stopSubscriptionThread(srvObj)
srvObj.stopUserServiceThread()
srvObj.stopMirControlThread()
srvObj.stopCacheControlThread()
srvObj.setThreadRunPermission(0)
if srvObj.getCfg().getSubscrEnable():
srvObj.remote_subscription_creation_evt.set()
logger.debug("Prepare NG/AMS for Offline State ...")
# Unsubscribe possible subscriptions. This is tried only once.
if (srvObj.getCfg().getAutoUnsubscribe()):
for subscrObj in srvObj.getSubscrStatusList():
subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo()
try:
resp = ngamsHttpUtils.httpGet(
subs_host, subs_port, NGAMS_UNSUBSCRIBE_CMD,
[["url", subscrObj.getId()]])
with contextlib.closing(resp):
stat = ngamsStatus.to_status(
resp, "%s:%d" % (subs_host, subs_port),
NGAMS_UNSUBSCRIBE_CMD)
if stat.getStatus() != NGAMS_SUCCESS:
logger.error("Error when unsubscribing from %s:%d: %s",
subs_host, subs_port, stat.getMessage())
except Exception:
msg = "Problem occurred while cancelling subscription " +\
"(host/port): %s/%d. Subscriber ID: %s"
logger.exception(msg, subscrObj.getHostId(),
subscrObj.getPortNo(), subscrObj.getId())
srvObj.resetSubscrStatusList()
# Check if there are files located in the Staging Areas of the
# Disks. In case yes, move these to the Back-Log Area to have them
# treated at a later stage.
checkStagingAreas(srvObj)
# Dump disk info on all disks, invoke the Offline Plug-In to prepare the
# disks for offline, and mark the disks as unmounted in the DB.
ngamsDiskUtils.dumpDiskInfoAllDisks(srvObj.getHostId(), srvObj.getDb(),
srvObj.getCfg())
plugIn = srvObj.getCfg().getOfflinePlugIn()
if (srvObj.getCfg().getSimulation() == 0):
logger.info("Invoking System Offline Plug-In: %s(srvObj, reqPropsObj)",
plugIn)
plugInMethod = loadPlugInEntryPoint(plugIn)
plugInRes = plugInMethod(srvObj, reqPropsObj)
else:
pass
ngamsDiskUtils.markDisksAsUnmountedInDb(srvObj.getHostId(), srvObj.getDb(),
srvObj.getCfg())
# Send out possible Retained Email Notification Messages.
flushMsg = "NOTE: Distribution of retained Email Notification Messages " +\
"forced at Offline"
ngamsNotification.checkNotifRetBuf(srvObj.getHostId(), srvObj.getCfg(), 1,
flushMsg)
logger.info("NG/AMS prepared for Offline State")
def saveToFile(srvObj,
ngamsCfgObj,
reqPropsObj,
trgFilename,
blockSize,
startByte):
"""
Save the data available on an HTTP channel into the given file.
ngamsCfgObj: NG/AMS Configuration object (ngamsConfig).
reqPropsObj: NG/AMS Request Properties object (ngamsReqProps).
trgFilename: Target name for file where data will be
written (string).
blockSize: Block size (bytes) to apply when reading the data
from the HTTP channel (integer).
mutexDiskAccess: Require mutual exclusion for disk access (integer).
diskInfoObj: Disk info object. Only needed if mutual exclusion
is required for disk access (ngamsDiskInfo).
Returns: Tuple. Element 0: Time in took to write
file (s) (tuple).
"""
disk_id = reqPropsObj.fileinfo['diskId']
source_host = reqPropsObj.fileinfo['sourceHost']
host_id = reqPropsObj.fileinfo['hostId']
file_version = reqPropsObj.fileinfo['fileVersion']
file_id = reqPropsObj.fileinfo['fileId']
host, port = source_host.split(":")
pars = {
'disk_id': disk_id,
'host_id': host_id,
'quick_location': '1',
'file_version': file_version,
'file_id': file_id
}
hdrs = {'Range': str(startByte) + '-'}
rx_timeout = 30 * 60
if srvObj.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(srvObj.getCfg().getVal("Mirroring[1].rx_timeout"))
response = ngamsHttpUtils.httpGet(host, int(port), 'RETRIEVE', pars=pars, hdrs=hdrs, timeout=rx_timeout)
# can we resume a previous download?
downloadResumeSupported = 'bytes' in response.getheader("Accept-Ranges", '')
logger.debug("Creating path: %s", trgFilename)
checkCreatePath(os.path.dirname(trgFilename))
crc_info = ngamsFileUtils.get_checksum_info('crc32')
if startByte != 0:
logger.info("resume requested")
if startByte != 0 and downloadResumeSupported:
logger.info("Resume requested and mirroring source supports resume. Appending data to previously started staging file")
crc = ngamsFileUtils.get_checksum(65536, trgFilename, 'crc32')
reqPropsObj.setBytesReceived(startByte)
fdOut = open(trgFilename, "a")
else:
if (startByte > 0):
logger.info("Resume of download requested but server does not support it. Starting from byte 0 again.")
fdOut = open(trgFilename, "w")
crc = crc_info.init
start = time.time()
# Distinguish between Archive Pull and Push Request. By Archive
# Pull we may simply read the file descriptor until it returns "".
logger.info("It is an HTTP Archive Pull Request: trying to get Content-length")
hdrs = {h[0]: h[1] for h in response.getheaders()}
if hdrs.has_key('content-length'):
remSize = int(hdrs['content-length'])
else:
logger.warning("Non Content-Lenght header found, defaulting to 1e11")
remSize = int(1e11)
# Receive the data.
buf = "-"
rdSize = blockSize
crc_m = crc_info.method
with contextlib.closing(response), contextlib.closing(fdOut):
while (remSize > 0):
if (remSize < rdSize):
rdSize = remSize
buf = response.read(rdSize)
sizeRead = len(buf)
if sizeRead == 0:
raise Exception("server is unreachable")
else:
crc = crc_m(buf, crc)
remSize -= sizeRead
reqPropsObj.setBytesReceived(reqPropsObj.getBytesReceived() +\
sizeRead)
fdOut.write(buf)
crc = crc_info.final(crc)
deltaTime = time.time() - start
msg = "Saved data in file: %s. Bytes received: %d. Time: %.3f s. " +\
"Rate: %.2f Bytes/s"
logger.info(msg, trgFilename, int(reqPropsObj.getBytesReceived()),
deltaTime, (float(reqPropsObj.getBytesReceived()) /
deltaTime))
# Raise exception if less byes were received as expected.
if (remSize != 0):
msg = "No all expected data arrived, %d bytes left to read" % (remSize,)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
# now check the CRC value against what we expected
sourceChecksum = reqPropsObj.checksum
logger.info('source checksum: %s - current checksum: %d', str(sourceChecksum), crc)
if (crc != int(sourceChecksum)):
msg = "checksum mismatch: source=" + str(sourceChecksum) + ", received: " + str(crc)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
return [deltaTime,crc]
def _create_remote_subscriptions(srvObj, stop_evt):
"""
Creates subscriptions in remote servers so they push their data into
ours.
"""
subscriptions_in_cfg = srvObj.cfg.getSubscriptionsDic()
subscriptions = [v for _, v in subscriptions_in_cfg.items()]
subscriptions_created = 0
def mark_as_active(s):
# Removes s from list of subscriptions pending creation, and
# ensures the corresponding UNSUBSCRIBE command will be called
# at server shutdown
srvObj.getSubscrStatusList().append(s)
subscriptions.remove(s)
while True:
# Done with all of them
if not subscriptions:
break
# Iterate over copy, since we modify the original inside the loop
for subscrObj in list(subscriptions):
if stop_evt.is_set():
logger.info("Terminating Subscriber thread ...")
return
our_host, our_port = srvObj.get_self_endpoint()
subs_host, subs_port = subscrObj.getHostId(), subscrObj.getPortNo()
# Not subscribing to ourselves
if srvObj.is_it_us(subs_host, subs_port):
logger.warning("Skipping subscription to %s:%d because that's us", subs_host, subs_port)
continue
# Because properly supporting the "Command" configuration mechanism
# still requires some more work, we prefer the "SubscriberUrl"
# attribute as the main source of URL information.
# We still support "Command", but with the following caveats:
# * TODO: include reverse proxy information when we add support
# * TODO: hardcoded http will need to be changed when we add support
# for https
# * TODO: fails with IpAddress == '0.0.0.0'
url = subscrObj.getUrl()
if not url:
url = 'http://%s:%d/%s' % (our_host, our_port, subscrObj.command or 'QARCHIVE')
logger.info("Creating subscription to %s:%d with url=%s", subs_host, subs_port, url)
pars = [["subscr_id", subscrObj.getId()],
["priority", subscrObj.getPriority()],
["url", url],
["start_date", toiso8601(local=True)]]
if subscrObj.getFilterPi():
pars.append(["filter_plug_in", subscrObj.getFilterPi()])
if subscrObj.getFilterPiPars():
pars.append(["plug_in_pars", subscrObj.getFilterPiPars()])
try:
# Issue SUBSCRIBE command
resp = ngamsHttpUtils.httpGet(subs_host, subs_port, cmd=NGAMS_SUBSCRIBE_CMD, pars=pars)
with contextlib.closing(resp):
stat = ngamsStatus.to_status(resp, subscrObj.getHostId(), NGAMS_SUBSCRIBE_CMD)
if stat.getStatus() != NGAMS_SUCCESS:
if stat.http_status == 409:
short_msg = "Different subscription with ID '%s' already exists, giving up"
long_msg = (
"NGAS attempted to create an automatic subscription "
"with ID=%s to obtain data from %s:%d, but the remote server "
"already has a subscription registered with the same ID, "
"but different details.\n\n"
"Instead of retrying to create this subscription over and over, "
"this server will give up now. To fix this either remove "
"the remote subscription, or change the ID of the subscription to be created "
"in the local server configuration."
)
logger.error(short_msg, subscrObj.getId())
ngamsNotification.notify(
srvObj.host_id, srvObj.cfg, NGAMS_NOTIF_ERROR,
"Automatic subscription cannot be created",
long_msg % (subscrObj.getId(), subs_host, subs_port),
force=True)
mark_as_active(subscrObj)
continue
else:
msg = "Unsuccessful NGAS XML response. Status: %s, message: %s. Will try again later"
logger.warning(msg, stat.getStatus(), stat.getMessage())
continue
subscriptions_created += 1
logger.info("Successfully subscribed to %s:%d with url=%s", subs_host, subs_port, subscrObj.getUrl())
mark_as_active(subscrObj)
except:
logger.exception("Error while adding subscription, will try later")
if stop_evt.wait(10):
return
if subscriptions_created:
logger.info("Successfully established %d Subscription(s)", subscriptions_created)
else:
logger.info("No Subscriptions established")
def save_to_file(ngams_server, request_properties, target_filename):
"""
Save the data available on an HTTP channel into the given file
:param ngams_server: NG/AMS Configuration object (ngamsConfig)
:param request_properties: NG/AMS Request Properties object (ngamsReqProps)
:param target_filename: Target name for file where data will be written (string)
:param block_size: Block size (bytes) to apply when reading the data from the HTTP channel (integer)
:param start_byte: Start byte offset
:return: Tuple. Element 0: Time in took to write file (s) (tuple)
"""
source_host = request_properties.fileinfo['sourceHost']
file_version = request_properties.fileinfo['fileVersion']
file_id = request_properties.fileinfo['fileId']
logger.info("Creating path: %s", target_filename)
checkCreatePath(os.path.dirname(target_filename))
rx_timeout = 30 * 60
if ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"):
rx_timeout = int(
ngams_server.getCfg().getVal("Mirroring[1].rx_timeout"))
host, port = source_host.split(":")
pars = {
'file_version': file_version,
'targetHost': get_fully_qualified_name(ngams_server),
'targetLocation': target_filename,
'file_id': file_id
}
fetch_start_time = time.time()
response = ngamsHttpUtils.httpGet(host,
int(port),
'RSYNC',
pars=pars,
timeout=rx_timeout)
with contextlib.closing(response):
data = response.read()
if 'FAILURE' in data:
raise Exception(data)
fetch_duration = time.time() - fetch_start_time
# Avoid divide by zeros later on, let's say it took us 1 [us] to do this
if fetch_duration == 0.0:
fetch_duration = 0.000001
msg = "Saved data in file: %s. Bytes received: %d. Time: %.3f s. Rate: %.2f Bytes/s"
logger.info(
msg, target_filename, int(request_properties.getBytesReceived()),
fetch_duration,
(float(request_properties.getBytesReceived()) / fetch_duration))
# Now check the CRC value against what we expected
checksum = request_properties.checksum
crc_variant = request_properties.checksum_plugin
crc_start_time = time.time()
crc = ngamsFileUtils.get_checksum(65536, target_filename, crc_variant)
crc_duration = time.time() - crc_start_time
logger.info("CRC computed in %f [s]", crc_duration)
logger.info('Cource checksum: %s - current checksum: %d', checksum, crc)
if crc != int(checksum):
msg = "Checksum mismatch: source={:s}, received={:d}".format(
checksum, crc)
raise ngamsFailedDownloadException.FailedDownloadException(msg)
# We half the total time for reading and writing because we do not have enough data for an accurate measurement
read_duration = fetch_duration / 2.0
write_duration = fetch_duration / 2.0
read_total_bytes = request_properties.getBytesReceived()
return archiving_results(read_total_bytes, read_duration, write_duration,
crc_duration, fetch_duration, crc_variant, crc)
def _locateArchiveFile(srvObj,
fileId,
fileVersion,
diskId,
hostId,
reqPropsObj,
files,
include_compression):
"""
See description of ngamsFileUtils.locateArchiveFile(). This function is
used simply to encapsulate the complete processing to be able to clean up.
"""
msg = "_locateArchiveFile() - Disk ID: %s - File ID: " +\
"%s - File Version: %d ..."
logger.debug(msg, str(diskId), fileId, int(fileVersion))
# Filter out files not on specified host if host ID is given.
if (hostId):
files = filter(lambda x: x[1] == hostId, files)
# If no file was found we raise an exception.
if not files:
tmpFileRef = fileId
if (fileVersion > 0): tmpFileRef += "/Version: " + str(fileVersion)
if (diskId): tmpFileRef += "/Disk ID: " + diskId
if (hostId): tmpFileRef += "/Host ID: " + hostId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [tmpFileRef])
raise Exception(errMsg)
# We now sort the file information sub-lists in the file list.
# The priori is as follows:
#
# 1. Local host.
# 2. Same cluster.
# 3. Same domain (e.g. hq.eso.org).
# 4. Other files (remote files).
localHostFileList = []
clusterFileList = []
domainFileList = []
remoteFileList = []
all_hosts = set([x[1] for x in files])
hostDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
all_hosts)
# Loop over the candidate files and sort them.
fileCount = idx = 0
for fileInfo in files:
fileHost = fileInfo[1]
if (hostDic[fileHost].getHostType() == NGAMS_HOST_LOCAL):
localHostFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_CLUSTER):
clusterFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_DOMAIN):
domainFileList.append(fileInfo)
else:
# NGAMS_HOST_REMOTE:
remoteFileList.append(fileInfo)
idx += 1
fileCount += 1
# The highest priority of the file is determined by the File Version,
# the latest version is preferred even though this may be stored
# on another NGAS host.
# A dictionary is built up, which contains the candidate files. The
# format is such there each version found is one key. For each key
# (= version) there is a list with the corresponding file information
# order according to the location.
candFileDic = {}
fileLists = [[NGAMS_HOST_LOCAL, localHostFileList],
[NGAMS_HOST_CLUSTER, clusterFileList],
[NGAMS_HOST_DOMAIN, domainFileList],
[NGAMS_HOST_REMOTE, remoteFileList]]
for fileListInfo in fileLists:
location = fileListInfo[0]
fileList = fileListInfo[1]
for fileInfo in fileList:
fileVer = fileInfo[0].getFileVersion()
# Create a list in connection with each File Version key.
if fileVer not in candFileDic:
candFileDic[fileVer] = []
candFileDic[fileVer].append([location, fileInfo[0], fileInfo[1]])
fileVerList = list(candFileDic)
fileVerList.sort(reverse=True)
if logger.level <= logging.DEBUG:
msg = ""
count = 1
for fileVer in fileVerList:
for fi in candFileDic[fileVer]:
msg += "(" + str(count) + ": Location:" + fi[0] +\
", Host:" + fi[2] + ", Version:" +\
str(fi[1].getFileVersion()) + ") "
count += 1
logger.debug("File list to check: " + msg)
# If no files were found we raise an exception.
if (len(candFileDic) == 0):
if (fileVersion != -1):
fileRef = fileId + "/V" + str(fileVersion)
else:
fileRef = fileId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileRef])
raise Exception(errMsg)
# We generate a list with the Disk IDs (which we need later).
# Generate a dictionary with Disk Info Objects.
diskIdDic = {}
for fileVer in fileVerList:
for fileInfo in candFileDic[fileVer]:
diskIdDic[fileInfo[1].getDiskId()] = fileInfo[1].getDiskId()
sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskIdList(list(diskIdDic))
diskInfoDic = {}
for diskInfo in sqlDiskInfo:
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(diskInfo)
diskInfoDic[diskInfoObj.getDiskId()] = diskInfoObj
logger.debug("Disk Info Objects Dictionary: %s", str(diskInfoDic))
# Check if the files are accessible - when the first accessible file
# in the fileList is found, the information is returned as the file wanted.
# To check the file accessibility, it is also checked if the NG/AMS
# 'responsible' for the file, allows for Retrieve Requests (only done
# in connection with a Retrieve Request).
logger.debug("Checking which of the candidate files should be selected ...")
foundFile = 0
for fileVer in fileVerList:
if (foundFile): break
for fileInfo in candFileDic[fileVer]:
location = fileInfo[0]
fileInfoObj = fileInfo[1]
host = fileInfo[2]
diskInfoObj = diskInfoDic[fileInfoObj.getDiskId()]
port = hostDic[host].getSrvPort()
logger.debug("Checking candidate file with ID: %s on host/port: %s/%s. " + \
"Location: %s",
fileInfoObj.getFileId(), host, str(port), location)
# If the file is stored locally we check if it is accessible,
# otherwise we send a STATUS/file_access request to the
# host in question.
if (location == NGAMS_HOST_LOCAL):
# Check first if the local system supports retrieve requests.
# (if relevant).
if (reqPropsObj):
if (reqPropsObj.getCmd() == NGAMS_RETRIEVE_CMD):
if (not srvObj.getCfg().getAllowRetrieveReq()):
continue
# Check if the file is accessible.
filename = os.path.normpath(diskInfoObj.getMountPoint()+"/" +\
fileInfoObj.getFilename())
logger.debug("Checking if local file with name: %s is available", filename)
if (not os.path.exists(filename)):
logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
else:
logger.debug("Checking if file with ID/Version: %s/%s " +\
"is available on host/port: %s/%s",
fileInfoObj.getFileId(), str(fileInfoObj.getFileVersion()),
host, str(port))
# If a server hosting a file is suspended, it is woken up
# to be able to check if the file is really accessible.
if (hostDic[host].getSrvSuspended() == 1):
logger.debug("Server hosting requested file (%s/%s) is suspended " + \
"- waking up server ...",
host, str(port))
try:
ngamsSrvUtils.wakeUpHost(srvObj, host)
logger.debug("Suspended server hosting requested file (%s/%s) " +\
"has been woken up",
host, str(port))
except Exception:
logger.exception("Error waking up server hosting selected " +\
"file")
continue
# The file is hosted on a host, which is not suspended or
# which was successfully woken up.
pars = [["file_access", fileInfoObj.getFileId()]]
if (fileInfoObj.getFileVersion() != -1):
pars.append(["file_version", fileInfoObj.getFileVersion()])
ipAddress = hostDic[host].getIpAddress()
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
resp = ngamsHttpUtils.httpGet(ipAddress, port, NGAMS_STATUS_CMD,
pars=pars, auth=authHdr)
with contextlib.closing(resp):
data = resp.read()
statusObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Result of File Access Query: %s",
re.sub("\n", "", str(statusObj.genXml().toprettyxml(' ', '\n'))))
if ((statusObj.getMessage().\
find("NGAMS_INFO_FILE_AVAIL") == -1)):
logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
# If no file was found we raise an exception.
if (not foundFile):
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileId])
raise Exception(errMsg)
# The file was found, get the info necessary for the acquiring the file.
ipAddress = hostDic[host].getIpAddress()
srcFileInfo = [location, host, ipAddress, port,
diskInfoObj.getMountPoint(),
fileInfoObj.getFilename(), fileInfoObj.getFileId(),
fileInfoObj.getFileVersion(), fileInfoObj.getFormat()]
if include_compression:
srcFileInfo.append(fileInfoObj.getCompression())
msg = "Located suitable file for request - File ID: %s. " +\
"Info for file found - Location: %s - Host ID/IP: %s/%s - " +\
"Port Number: %s - File Version: %d - Filename: %s - " +\
"Mime-type: %s"
logger.debug(msg, fileId, location, host, ipAddress, port,
fileInfoObj.getFileVersion(), fileInfoObj.getFilename(),
fileInfoObj.getFormat())
return srcFileInfo
def lookup_partner_site_file_status(ngas_server, file_id, file_version,
request_properties):
"""
Lookup the file indicated by the File ID using a NGAS partner site (if one
is specified in the configuration). Returns a tuple of status objects.
Parameters:
ngas_server: Reference to NG/AMS server class object (ngamsServer).
file_id: File ID of file to locate (string).
file_version: Version of the file (integer).
request_properties: Request Property object to keep track of actions done
during the request handling (ngamsReqProps|None).
Returns:
host: Partner site host name (or IP address)
port: Partner site port number
status_info: Status response object (ngamsStatus)
disk_info: Status response disk information object (ngamsDiskInfo)
file_info: Status response file information object (ngamsFileInfo)
"""
file_reference = file_id
if file_version > 0:
file_reference += "/Version: " + str(file_version)
# If the request came from a partner site. We will not continue to
# propagate the request to avoid a death loop scenario. We will raise an
# exception.
if request_properties.hasHttpPar("partner_site_redirect"):
error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference])
raise Exception(error_message)
# Check partner sites is enabled are available from the configuration
if not ngas_server.is_partner_sites_proxy_mode()\
or not ngas_server.get_partner_sites_address_list():
error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference])
raise Exception(error_message)
# Lets query the partner sites for the availability of the requested file
authentication_header = ngamsSrvUtils.genIntAuthHdr(ngas_server)
parameter_list = [["file_id", file_id]]
if file_version != -1:
parameter_list.append(["file_version", file_version])
parameter_list.append(["partner_site_redirect", 1])
host, port, status_info, disk_info, file_info = None, None, None, None, None
for partner_site in ngas_server.get_partner_sites_address_list():
partner_address, partner_domain, partner_port = parse_host_id(
partner_site)
try:
logger.info("Looking up file ID %s on partner site %s", file_id,
partner_site)
response = ngamsHttpUtils.httpGet(partner_address,
partner_port,
NGAMS_STATUS_CMD,
parameter_list,
auth=authentication_header)
with contextlib.closing(response):
response_info = response.read()
except:
# We ignore this error, and try the next partner site, if any
continue
status_info = ngamsStatus.ngamsStatus().unpackXmlDoc(response_info, 1)
logger.info("Result of File Access Query: {}".format(
re.sub("\n", "", str(status_info.genXml().toprettyxml(' ',
'\n')))))
if status_info.getStatus() == "FAILURE":
logger.info(
genLog("NGAMS_INFO_FILE_NOT_AVAIL",
[file_id, partner_address]))
else:
logger.info(
genLog("NGAMS_INFO_FILE_AVAIL", [file_id, partner_address]))
disk_info = status_info.getDiskStatusList()[0]
file_info = disk_info.getFileObjList()[0]
# This is a bit of a hack because the host ID may not contain
# the fully qualified address. We append the domain name from
# the partner site address. This should work because they are
# part of the same cluster.
host, domain, port = parse_host_id(disk_info.getHostId())
if domain is None and partner_domain is not None:
host = host + "." + partner_domain
break
if status_info is None or status_info.getStatus() == "FAILURE":
# Failed to find file on a partner site
error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference])
raise Exception(error_message)
return host, port, status_info, disk_info, file_info