def cretrieve(self, containerName, containerId=None, targetDir='.'):
"""
Sends a CRETRIEVE command to NG/AMS to retrieve the full contents of a
container and dumps them into the file system.
"""
if (not containerId and not containerName):
msg = "Must specify parameter -containerId or -containerName for " +\
"a CRETRIEVE Command"
raise Exception(msg)
if not targetDir:
targetDir = '.'
pars = []
if containerId:
pars.append(("container_id", containerId))
if containerName:
pars.append(("container_name", containerName))
resp, host, port = self._get('CRETRIEVE', pars=pars)
host_id = "%s:%d" % (host, port)
with contextlib.closing(resp):
if resp.status != NGAMS_HTTP_SUCCESS:
return ngamsStatus.ngamsStatus().unpackXmlDoc(resp.read(), 1)
size = int(resp.getheader('Content-Length'))
handler = ngamsMIMEMultipart.FilesystemWriterHandler(
1024, basePath=targetDir)
parser = ngamsMIMEMultipart.MIMEMultipartParser(
handler, resp, size, 65536)
parser.parse()
return ngamsStatus.dummy_success_stat(host_id)
def _httpPost(srvObj, url, filename, sessionId):
"""
A wrapper for _httpPostUrl
return success 0 or failure 1
filename full path for the file to be sent
sessionId session uuid
"""
#info(3, "xxxxxx ---- filename %s" % filename)
stat = ngamsStatus.ngamsStatus()
baseName = os.path.basename(filename)
contDisp = "attachment; filename=\"" + baseName + "\""
contDisp += "; no_versioning=1"
logger.debug("Async Delivery Thread [%s] Delivering file: %s - to: %s",
str(thread.get_ident()), baseName, url)
ex = ""
try:
reply, msg, hdrs, data = \
_httpPostUrl(url, fileMimeType,
contDisp, filename, "FILE",
blockSize=\
srvObj.getCfg().getBlockSize(), session_uuid = sessionId)
if (reply == None and msg == None and hdrs == None
and data == None): # transfer cancelled/suspended
return 1
if (data.strip() != ""):
stat.clear().unpackXmlDoc(data)
else:
stat.clear().setStatus(NGAMS_SUCCESS)
except Exception, e:
ex = str(e)
def _checkFile(srvObj,
fileInfoObj,
stagFile,
httpHeaders,
checkChecksum):
"""
Make a consistency check of the Staging File.
srvObj: Reference to instance of Server Object (ngamsServer).
fileInfoObj: File info object with info about the file
(ngamsFileInfo).
stagFile: Staging filename (string).
httpHeaders: HTTP headers (mimetools.Message)
checkChecksum: Carry out checksum check (0|1/integer).
Returns: Void.
"""
# First ensure to flush file caches.
ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagFile])
# Check file size.
fileSize = getFileSize(stagFile)
if (fileSize != fileInfoObj.getFileSize()):
# If the mime-type is 'text/xml' we check if the returned
# document is an NG/AMS XML Status reporting a problem.
tmpStat = None
if (httpHeaders.type == NGAMS_XML_MT):
try:
tmpStat = ngamsStatus.ngamsStatus().load(stagFile,getStatus=1)
if (tmpStat.getStatus() != NGAMS_FAILURE):
del tmpStat
tmpStat = None
except:
# Was apparently not an NG/AMS Status Document.
pass
if (tmpStat):
# An error response was received from the source node.
raise Exception(tmpStat.getMessage())
else:
# The file seems not to have been transferred completely.
errMsg = "Size of cloned file wrong (expected: %d/actual: %d)"
raise Exception(errMsg % (fileInfoObj.getFileSize(), fileSize))
logger.debug("Size of cloned Staging File OK: %s", stagFile)
# The file size was correct.
if (checkChecksum):
if ngamsFileUtils.check_checksum(srvObj, fileInfoObj, stagFile):
logger.debug("Checksum of cloned Staging File OK: %s", stagFile)
else:
logger.debug("No Checksum or Checksum Plug-In specified for file")
def _httpPost(srvObj, url, filename, sessionId=None):
"""
A wrapper for _httpPostUrl
return success 0 or failure 1
filename full path for the file to be sent
sessionId session uuid
"""
#info(3, "xxxxxx ---- filename %s" % filename)
stat = ngamsStatus.ngamsStatus()
baseName = os.path.basename(filename)
contDisp = "attachment; filename=\"" + baseName + "\""
contDisp += "; no_versioning=1"
thread_id = threading.current_thread().ident
logger.debug("Async Delivery Thread [%s] Delivering file: %s - to: %s ...",
str(thread_id), baseName, url)
ex = ""
try:
#TODO - the fileMimeType should be queried from the database
reply, msg, hdrs, data = _httpPostUrl(url, fileMimeType,
contDisp, filename, "FILE",
blockSize=\
srvObj.getCfg().getBlockSize(), session_uuid = sessionId)
#reply, msg, hdrs, data = ngamsLib.httpPostUrl(url, fileMimeType, contDisp, filename, "FILE", blockSize=srvObj.getCfg().getBlockSize())
if (reply == None and msg == None and hdrs == None
and data == None): # transfer cancelled/suspended
return 1
if (data.strip() != ""):
stat.clear().unpackXmlDoc(data)
else:
stat.clear().setStatus(NGAMS_SUCCESS)
except Exception as e:
ex = str(e)
if ((ex != "") or (reply != NGAMS_HTTP_SUCCESS)
or (stat.getStatus() == NGAMS_FAILURE)):
errMsg = "Error occurred while async delivering file: " + baseName +\
" - to url: " + url + " by Data Delivery Thread [" + str(thread_id) + "]"
if (ex != ""): errMsg += " Exception: " + ex + "."
if (stat.getMessage() != ""):
errMsg += " Message: " + stat.getMessage()
logger.warning(errMsg + '\n' + traceback.format_exc())
return 1
else:
logger.debug(
"File: %s delivered to url: %s by Async Delivery Thread [%s]",
baseName, url, str(thread_id))
return 0
def _sendCheckFileCmd(node, fileId, fileVersion):
"""
Send a CHECKFILE Command to the specified node. One of the following is
returned:
1. File OK: NGAMS_INFO_FILE_OK.
2. File Not OK: NGAMS_ER_FILE_NOK
3. Error: FAILURE.
node: Node to be contacted (node:port) (string).
fileId: ID of file to check (string).
fileVersion: Version of file to check (integer).
Returns: See above (NGAMS_INFO_FILE_OK | NGAMS_ER_FILE_NOK | FAILURE).
"""
T = TRACE(5)
cmdPars = [["file_id", fileId], ["file_version", fileVersion]]
data = None
try:
host, port = node.split(":")
logger.debug(
"Sending CHECKFILE Command for file: %s/%s to node: %s:%s", fileId,
str(fileVersion), host, str(port))
resp = ngamsHttpUtils.httpGet(host,
port,
NGAMS_CHECKFILE_CMD,
pars=cmdPars)
with contextlib.closing(resp):
data = resp.read()
except Exception:
logger.exception("Error contacting node %s", node)
if (data):
tmpStatObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data)
if (tmpStatObj.getMessage().find(NGAMS_INFO_FILE_OK) != -1):
return NGAMS_INFO_FILE_OK
else:
# Assume: NGAMS_ER_FILE_NOK.
return NGAMS_ER_FILE_NOK
else:
return NGAMS_FAILURE
def test_status_retrieve_sequence(self):
bad_file_name = "dummy.fits"
# We create two NGAS clusters each containing a single NGAS node
# We configure the first NGAS cluster to use the second NGAS cluster
# as a partner site
partner_host_id = "{0}:9011".format("localhost")
config_list_1 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas1"),
("NgamsCfg.Server[1].IpAddress", "0.0.0.0"),
("NgamsCfg.PartnerSites[1].ProxyMode", "1"),
("NgamsCfg.PartnerSites[1].PartnerSite[1].Address",
partner_host_id)]
self._prepare_partner_site_cluster((9001, config_list_1))
config_list_2 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas2"),
("NgamsCfg.Server[1].IpAddress", "0.0.0.0")]
self._prepare_partner_site_cluster((9011, config_list_2))
# We check the status of a file ID found on the partner site cluster
command_status = functools.partial(ngamsHttpUtils.httpGet,
"localhost",
9001,
"STATUS",
timeout=60)
argument_list = {"file_id": bad_file_name}
with contextlib.closing(
command_status(pars=argument_list)) as response:
self.assertEqual(response.status, 400)
# Parse the response XML status information
response_data = response.read()
status_info = ngamsStatus.ngamsStatus().unpackXmlDoc(
response_data, 1)
self.assertEqual(status_info.getStatus(), "FAILURE")
self.assertEqual(
status_info.getMessage(),
"NGAMS_ER_UNAVAIL_FILE:4019:ERROR: File with ID: dummy.fits appears not to be available."
)
# Retrieve a file found on the partner site cluster
retrieve_file_path = tmp_path(bad_file_name)
self.retrieve(9001,
bad_file_name,
targetFile=retrieve_file_path,
expectedStatus='FAILURE')
def _execCClient(unpackXmlStat = 1,
pars = []):
"""
Execute the NG/AMS C-Client on the shell.
cmdLineParsDic: Dictionary with command line parameters (dictionary).
pars: Extra parameters for invoking NGAMS C-Client (list).
Returns: List with status objects or stdout output from c-client:
[<stat obj>, ...] or <stdout c-client> (list|string).
"""
cmd = ['ngamsCClient']
for opt, val in pars:
cmd.append(opt)
cmd.append(str(val))
if '-servers' not in (x for x, _ in pars):
cmd.append('-host')
cmd.append('127.0.0.1')
cmd.append('-status')
env = os.environ.copy()
env['NGAMS_VERBOSE_LEVEL'] = '0'
_, out, _ = execCmd(cmd, shell=False)
out = utils.b2s(out)
if (unpackXmlStat):
statObjList = []
xmlStatList = out.split("Command repetition counter:")
for xmlStat in xmlStatList:
# Clean up the output.
xmlStat = xmlStat.strip()
if (xmlStat == ""): continue
idx = 0
while (xmlStat[idx] != "<"): idx += 1
xmlStat = xmlStat[idx:]
# Unpack it.
statObj = ngamsStatus.ngamsStatus().unpackXmlDoc(xmlStat)
statObjList.append(statObj)
out = statObjList
return out
def _post(self, cmd, mime_type, data, pars=[]):
if self.timeout:
pars.append(["time_out", str(self.timeout)])
if self.reload_mod:
pars.append(["reload", "1"])
# For now we try the serves in random order.
servers = self.servers
random.shuffle(servers)
for i, host_port in enumerate(servers):
host, port = host_port
try:
_, _, _, data = self._do_post(host, port, cmd, mime_type, data,
pars)
return ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
except socket.error:
if i == len(servers) - 1:
raise
def processHttpReply(http, basename, url):
"""
After file is sent, collect the ngams status from the remote server
parse error message, and log if necessary
http: the HTTP Client
basename Name of the file sent to the remote ngas server (used in the content disposition)
url: the url of the remote ngas server that receives the file
"""
logger.debug("Waiting for reply ...")
# ngamsLib._setSocketTimeout(None, http)
reply, msg, hdrs = http.getreply()
if (hdrs == None):
errMsg = "Illegal/no response to HTTP request encountered!"
raise Exception(errMsg)
if (hdrs.has_key("content-length")):
dataSize = int(hdrs["content-length"])
else:
dataSize = 0
data = http.getfile().read(dataSize)
stat = ngamsStatus.ngamsStatus()
if (data.strip() != ""):
stat.clear().unpackXmlDoc(data)
else:
# TODO: For the moment assume success in case no
# exception was thrown.
stat.clear().setStatus(NGAMS_SUCCESS)
if ((reply != NGAMS_HTTP_SUCCESS) or
(stat.getStatus() == NGAMS_FAILURE)):
errMsg = 'Error occurred while proxy quick archive file %s to %s' % (basename, url)
if (stat.getMessage() != ""):
errMsg += " Message: " + stat.getMessage()
raise Exception(errMsg)
def _httpPost(srvObj, url, filename, sessionId):
"""
A wrapper for _httpPostUrl
return success 0 or failure 1
filename full path for the file to be sent
sessionId session uuid
"""
#info(3, "xxxxxx ---- filename %s" % filename)
stat = ngamsStatus.ngamsStatus()
baseName = os.path.basename(filename)
contDisp = "attachment; filename=\"" + baseName + "\""
contDisp += "; no_versioning=1"
thread_id = threading.current_thread().ident
logger.debug("Async Delivery Thread [%s] Delivering file: %s - to: %s",
str(thread_id), baseName, url)
ex = ""
try:
reply, msg, hdrs, data = \
_httpPostUrl(url, fileMimeType,
contDisp, filename, "FILE",
blockSize=\
srvObj.getCfg().getBlockSize(), session_uuid = sessionId)
if (reply == None and msg == None and hdrs == None
and data == None): # transfer cancelled/suspended
return 1
if (data.strip() != ""):
stat.clear().unpackXmlDoc(data)
else:
stat.clear().setStatus(NGAMS_SUCCESS)
except Exception as e:
ex = str(e)
if ((ex != "") or (reply != NGAMS_HTTP_SUCCESS)
or (stat.getStatus() == NGAMS_FAILURE)):
errMsg = "Error occurred while async delivering file: " + baseName +\
" - to url: " + url + " by Data Delivery Thread [" + str(thread_id) + "]"
if (ex != ""): errMsg += " Exception: " + ex + "."
if (stat.getMessage() != ""):
errMsg += " Message: " + stat.getMessage()
logger.warning(errMsg)
jobManHost = srvObj.getCfg().getNGASJobMANHost()
if (jobManHost):
try:
if (not ex):
ex = ''
rereply = urlrequest.urlopen(
'http://%s/failtodeliverfile?file_id=%s&to_url=%s&err_msg=%s'
% (jobManHost, baseName, urlparse.quote(url),
urlparse.quote(ex)),
timeout=15).read()
logger.debug(
'Reply from sending file %s failtodeliver event to server %s - %s',
baseName, jobManHost, rereply)
except Exception as err:
logger.error(
'Fail to send fail-to-deliver event to server %s, Exception: %s',
jobManHost, str(err))
return 1
else:
logger.debug(
"File: %s - delivered to url: %s by Async Delivery Thread [%s]",
baseName, url, str(thread_id))
return 0
def test_NormalOp_1(self):
"""
Synopsis:
Test normal operation of the NG/AMS Archive Client.
Description:
It is tested that a file can be archived via a link in the Archive
Queue.
Expected Result:
After the archiving the link is moved to the Archive Files
Area. It is also checked that the NG/AMS XML Status Document is
created in the Archived Files Area. After the expiration time for
keeping archived files has expired, the archived file and the XML
status document should be deleted.
Test Steps:
- Start NG/AMS Server.
- Start instance of the NG/AMS Archive Client.
- Create a link from a legal test (FITS) file into the Archive Queue.
- Test that the file is archived within 20s and moved to the Archived
Files Area.
- Test that the XML Status Document from NG/AMS is stored in the
Archived Files Area.
- Check that after the given expiration time for the Archived Files
Area, that the archived file + the XML Status Document are removed.
- Stop the Archive Client.
Remarks:
...
"""
self.prepExtSrv()
# Make sure the the queue subdir exist before the launch the client;
# otherwise the client and this test might find themselves in a race
# condition and the test might fail
d = os.path.abspath(os.path.join(arcCliDir, 'queue'))
if not os.path.exists(d):
os.makedirs(d)
self.startArchiveClient()
# Archive a file as copy and link.
# Make sure at least the quee dir is already created
srcFile = self.resource("src/SmallFile.fits")
self.cp(srcFile, os.path.join(arcCliDir, 'queue'))
os.symlink(srcFile, os.path.join(arcCliDir, 'queue', 'Test.fits'))
# Check that files are being archived (within 20s) + NG/AMS Status
# Documents created.
file1Pat = arcCliDir + "/archived/*___SmallFile.fits"
file1StatPat = file1Pat + "___STATUS.xml"
file2Pat = arcCliDir + "/archived/*___Test.fits"
file2StatPat = file2Pat + "___STATUS.xml"
startTime = time.time()
filesFound = 0
while ((time.time() - startTime) < 20):
globFile1Pat = glob.glob(file1Pat)
globFile1StatPat = glob.glob(file1StatPat)
globFile2Pat = glob.glob(file2Pat)
globFile2StatPat = glob.glob(file2StatPat)
if ((len(globFile1Pat) == 1) and (len(globFile1StatPat) == 1)
and (len(globFile2Pat) == 1)
and (len(globFile2StatPat) == 1)):
filesFound = 1
break
if (not filesFound):
if (not len(globFile1Pat)):
errMsg = "Did not find status file: " + file1Pat
elif (not len(globFile1StatPat)):
errMsg = "Did not find status XML document: " + file1StatPat
elif (not len(globFile2Pat)):
errMsg = "Did not find status file: " + file2Pat
else:
# (not len(globFile2StatPat)):
errMsg = "Did not find status XML document: " + file2StatPat
self.fail(errMsg)
# Check the contents of one of the status documents.
statObj = ngamsStatus.ngamsStatus().load(globFile1StatPat[0])
refStatFile = "ref/ngamsArchiveClientTest_test_NormalOp_1_1_ref"
filters = ("BytesStored:", "NumberOfFiles:", "FileName:",
"FileVersion:")
self.assert_status_ref_file(refStatFile, statObj, filters=filters)
# Check that the status documents are removed within 10s.
filesRemoved = 0
startTime = time.time()
while ((time.time() - startTime) < 20):
globFile1Pat = glob.glob(file1Pat)
globFile1StatPat = glob.glob(file1StatPat)
globFile2Pat = glob.glob(file2Pat)
globFile2StatPat = glob.glob(file2StatPat)
if ((len(globFile1Pat) == 0) and (len(globFile1StatPat) == 0)
and (len(globFile2Pat) == 0)
and (len(globFile2StatPat) == 0)):
filesRemoved = 1
break
if (not filesRemoved):
if (len(globFile1Pat)):
errMsg = "Did not remove status file: " + globFile1Pat[0]
elif (len(globFile1StatPat)):
errMsg = "Did not remove status XML document: " +\
globFile1StatPat[0]
elif (len(globFile2Pat)):
errMsg = "Did not remove status file: " + file2Pat[0]
else:
# (len(globFile2StatPat)):
errMsg = "Did not remove status XML document: " + file2StatPat[
0]
self.fail(errMsg)
def handleCmd(srvObj, reqPropsObj, httpRef):
"""
Compress the file based on file_path (file_id, disk_id, and file_version)
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler
object (ngamsHttpRequestHandler).
Returns: Void.
"""
# target_host does not contain port
required_params = [
'file_path', 'file_id', 'file_version', 'disk_id', 'target_host',
'crc_db', 'debug'
]
filename = reqPropsObj.getHttpPar('file_path')
if (not os.path.exists(filename)):
errMsg = 'File does not exist: %s' % filename
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=501)
return
parDic = reqPropsObj.getHttpParsDic()
for rp in required_params:
if (not parDic.has_key(rp)):
errMsg = 'Parameter missing: %s' % rp
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=502)
return
logger.debug('Moving file %s', filename)
fileId = parDic['file_id']
fileVersion = int(parDic['file_version'])
diskId = parDic['disk_id']
tgtHost = parDic['target_host']
crcDb = parDic['crc_db']
debug = int(parDic['debug'])
if (not isMWAVisFile(fileId)):
errMsg = 'Not MWA visibilty file: %' % (fileId)
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=503)
return
"""
if (fileOnHost(srvObj, fileId, tgtHost)):
errMsg = "File %s already on host %s" % (fileId, tgtHost)
warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT) # make it correct
return
"""
if (debug):
logger.debug('Only for file movment debugging, return now')
httpRef.send_data('DEBUG ONLY', NGAMS_TEXT_MT)
return
fileCRC = getCRCFromFile(filename)
if (fileCRC != crcDb):
errMsg = 'File %s on source host %s already corrupted, moving request rejected' % (
filename, srvObj.getHostId().replace(':', '-'))
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=504)
return
sendUrl = 'http://%s:7777/QARCHIVE' % tgtHost
logger.debug("Moving to %s", sendUrl)
fileMimeType = 'application/octet-stream'
baseName = os.path.basename(filename)
contDisp = "attachment; filename=\"" + baseName + "\""
deliver_success = False
last_deliv_err = ''
stat = ngamsStatus.ngamsStatus()
hdrs = {NGAMS_HTTP_HDR_CHECKSUM: fileCRC}
for i in range(3): # total trials - 3 times
with open(filename, "rb") as f:
try:
reply, msg, hdrs, data = ngamsHttpUtils.httpPostUrl(
sendUrl, f, fileMimeType, contDisp=contDisp, hdrs=hdrs)
if (data.strip() != ""):
stat.clear().unpackXmlDoc(data)
else:
stat.clear().setStatus(NGAMS_SUCCESS)
if (reply != NGAMS_HTTP_SUCCESS
or stat.getStatus() == NGAMS_FAILURE):
logger.warning("Attempt %d failed: %s", i,
stat.getMessage())
last_deliv_err = stat.getMessage()
continue
else:
deliver_success = True
break
except Exception as hexp:
logger.warning("Attempt %d failed: %s", i, str(hexp))
last_deliv_err = str(hexp).replace('\n', '--')
continue
if (not deliver_success):
errMsg = 'File %s failed to be moved to %s: %s' % (fileId, tgtHost,
last_deliv_err)
logger.warning(errMsg)
httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=505)
return
try:
ngamsDiscardCmd._discardFile(srvObj,
diskId,
fileId,
fileVersion,
execute=1)
except Exception as e1:
logger.warning('Fail to remove file %s: %s', filename, str(e1))
httpRef.send_data('Remove error: %s' % str(e1).replace('\n', '--'),
NGAMS_TEXT_MT)
return
httpRef.send_data(MOVE_SUCCESS, NGAMS_TEXT_MT)
def lookup_partner_site_file_status(ngas_server, file_id, file_version,
request_properties):
"""
Lookup the file indicated by the File ID using a NGAS partner site (if one
is specified in the configuration). Returns a tuple of status objects.
Parameters:
ngas_server: Reference to NG/AMS server class object (ngamsServer).
file_id: File ID of file to locate (string).
file_version: Version of the file (integer).
request_properties: Request Property object to keep track of actions done
during the request handling (ngamsReqProps|None).
Returns:
host: Partner site host name (or IP address)
port: Partner site port number
status_info: Status response object (ngamsStatus)
disk_info: Status response disk information object (ngamsDiskInfo)
file_info: Status response file information object (ngamsFileInfo)
"""
file_reference = file_id
if file_version > 0:
file_reference += "/Version: " + str(file_version)
# If the request came from a partner site. We will not continue to
# propagate the request to avoid a death loop scenario. We will raise an
# exception.
if request_properties.hasHttpPar("partner_site_redirect"):
error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference])
raise Exception(error_message)
# Check partner sites is enabled are available from the configuration
if not ngas_server.is_partner_sites_proxy_mode()\
or not ngas_server.get_partner_sites_address_list():
error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference])
raise Exception(error_message)
# Lets query the partner sites for the availability of the requested file
authentication_header = ngamsSrvUtils.genIntAuthHdr(ngas_server)
parameter_list = [["file_id", file_id]]
if file_version != -1:
parameter_list.append(["file_version", file_version])
parameter_list.append(["partner_site_redirect", 1])
host, port, status_info, disk_info, file_info = None, None, None, None, None
for partner_site in ngas_server.get_partner_sites_address_list():
partner_address, partner_domain, partner_port = parse_host_id(
partner_site)
try:
logger.info("Looking up file ID %s on partner site %s", file_id,
partner_site)
response = ngamsHttpUtils.httpGet(partner_address,
partner_port,
NGAMS_STATUS_CMD,
parameter_list,
auth=authentication_header)
with contextlib.closing(response):
response_info = response.read()
except:
# We ignore this error, and try the next partner site, if any
continue
status_info = ngamsStatus.ngamsStatus().unpackXmlDoc(response_info, 1)
logger.info("Result of File Access Query: {}".format(
re.sub("\n", "", str(status_info.genXml().toprettyxml(' ',
'\n')))))
if status_info.getStatus() == "FAILURE":
logger.info(
genLog("NGAMS_INFO_FILE_NOT_AVAIL",
[file_id, partner_address]))
else:
logger.info(
genLog("NGAMS_INFO_FILE_AVAIL", [file_id, partner_address]))
disk_info = status_info.getDiskStatusList()[0]
file_info = disk_info.getFileObjList()[0]
# This is a bit of a hack because the host ID may not contain
# the fully qualified address. We append the domain name from
# the partner site address. This should work because they are
# part of the same cluster.
host, domain, port = parse_host_id(disk_info.getHostId())
if domain is None and partner_domain is not None:
host = host + "." + partner_domain
break
if status_info is None or status_info.getStatus() == "FAILURE":
# Failed to find file on a partner site
error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference])
raise Exception(error_message)
return host, port, status_info, disk_info, file_info
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
T = TRACE()
status = ngamsStatus.ngamsStatus()
status.\
setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = reqPropsObj.getHttpPar("file_version")
if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = "-"
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception, e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception, errMsg
return
def _locateArchiveFile(srvObj,
fileId,
fileVersion,
diskId,
hostId,
reqPropsObj,
files,
include_compression):
"""
See description of ngamsFileUtils.locateArchiveFile(). This function is
used simply to encapsulate the complete processing to be able to clean up.
"""
msg = "_locateArchiveFile() - Disk ID: %s - File ID: " +\
"%s - File Version: %d ..."
logger.debug(msg, str(diskId), fileId, int(fileVersion))
# Filter out files not on specified host if host ID is given.
if (hostId):
files = filter(lambda x: x[1] == hostId, files)
# If no file was found we raise an exception.
if not files:
tmpFileRef = fileId
if (fileVersion > 0): tmpFileRef += "/Version: " + str(fileVersion)
if (diskId): tmpFileRef += "/Disk ID: " + diskId
if (hostId): tmpFileRef += "/Host ID: " + hostId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [tmpFileRef])
raise Exception(errMsg)
# We now sort the file information sub-lists in the file list.
# The priori is as follows:
#
# 1. Local host.
# 2. Same cluster.
# 3. Same domain (e.g. hq.eso.org).
# 4. Other files (remote files).
localHostFileList = []
clusterFileList = []
domainFileList = []
remoteFileList = []
all_hosts = set([x[1] for x in files])
hostDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(),
srvObj.getDb(),
srvObj.getCfg(),
all_hosts)
# Loop over the candidate files and sort them.
fileCount = idx = 0
for fileInfo in files:
fileHost = fileInfo[1]
if (hostDic[fileHost].getHostType() == NGAMS_HOST_LOCAL):
localHostFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_CLUSTER):
clusterFileList.append(fileInfo)
elif (hostDic[fileHost].getHostType() == NGAMS_HOST_DOMAIN):
domainFileList.append(fileInfo)
else:
# NGAMS_HOST_REMOTE:
remoteFileList.append(fileInfo)
idx += 1
fileCount += 1
# The highest priority of the file is determined by the File Version,
# the latest version is preferred even though this may be stored
# on another NGAS host.
# A dictionary is built up, which contains the candidate files. The
# format is such there each version found is one key. For each key
# (= version) there is a list with the corresponding file information
# order according to the location.
candFileDic = {}
fileLists = [[NGAMS_HOST_LOCAL, localHostFileList],
[NGAMS_HOST_CLUSTER, clusterFileList],
[NGAMS_HOST_DOMAIN, domainFileList],
[NGAMS_HOST_REMOTE, remoteFileList]]
for fileListInfo in fileLists:
location = fileListInfo[0]
fileList = fileListInfo[1]
for fileInfo in fileList:
fileVer = fileInfo[0].getFileVersion()
# Create a list in connection with each File Version key.
if fileVer not in candFileDic:
candFileDic[fileVer] = []
candFileDic[fileVer].append([location, fileInfo[0], fileInfo[1]])
fileVerList = list(candFileDic)
fileVerList.sort(reverse=True)
if logger.level <= logging.DEBUG:
msg = ""
count = 1
for fileVer in fileVerList:
for fi in candFileDic[fileVer]:
msg += "(" + str(count) + ": Location:" + fi[0] +\
", Host:" + fi[2] + ", Version:" +\
str(fi[1].getFileVersion()) + ") "
count += 1
logger.debug("File list to check: " + msg)
# If no files were found we raise an exception.
if (len(candFileDic) == 0):
if (fileVersion != -1):
fileRef = fileId + "/V" + str(fileVersion)
else:
fileRef = fileId
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileRef])
raise Exception(errMsg)
# We generate a list with the Disk IDs (which we need later).
# Generate a dictionary with Disk Info Objects.
diskIdDic = {}
for fileVer in fileVerList:
for fileInfo in candFileDic[fileVer]:
diskIdDic[fileInfo[1].getDiskId()] = fileInfo[1].getDiskId()
sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskIdList(list(diskIdDic))
diskInfoDic = {}
for diskInfo in sqlDiskInfo:
diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(diskInfo)
diskInfoDic[diskInfoObj.getDiskId()] = diskInfoObj
logger.debug("Disk Info Objects Dictionary: %s", str(diskInfoDic))
# Check if the files are accessible - when the first accessible file
# in the fileList is found, the information is returned as the file wanted.
# To check the file accessibility, it is also checked if the NG/AMS
# 'responsible' for the file, allows for Retrieve Requests (only done
# in connection with a Retrieve Request).
logger.debug("Checking which of the candidate files should be selected ...")
foundFile = 0
for fileVer in fileVerList:
if (foundFile): break
for fileInfo in candFileDic[fileVer]:
location = fileInfo[0]
fileInfoObj = fileInfo[1]
host = fileInfo[2]
diskInfoObj = diskInfoDic[fileInfoObj.getDiskId()]
port = hostDic[host].getSrvPort()
logger.debug("Checking candidate file with ID: %s on host/port: %s/%s. " + \
"Location: %s",
fileInfoObj.getFileId(), host, str(port), location)
# If the file is stored locally we check if it is accessible,
# otherwise we send a STATUS/file_access request to the
# host in question.
if (location == NGAMS_HOST_LOCAL):
# Check first if the local system supports retrieve requests.
# (if relevant).
if (reqPropsObj):
if (reqPropsObj.getCmd() == NGAMS_RETRIEVE_CMD):
if (not srvObj.getCfg().getAllowRetrieveReq()):
continue
# Check if the file is accessible.
filename = os.path.normpath(diskInfoObj.getMountPoint()+"/" +\
fileInfoObj.getFilename())
logger.debug("Checking if local file with name: %s is available", filename)
if (not os.path.exists(filename)):
logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
else:
logger.debug("Checking if file with ID/Version: %s/%s " +\
"is available on host/port: %s/%s",
fileInfoObj.getFileId(), str(fileInfoObj.getFileVersion()),
host, str(port))
# If a server hosting a file is suspended, it is woken up
# to be able to check if the file is really accessible.
if (hostDic[host].getSrvSuspended() == 1):
logger.debug("Server hosting requested file (%s/%s) is suspended " + \
"- waking up server ...",
host, str(port))
try:
ngamsSrvUtils.wakeUpHost(srvObj, host)
logger.debug("Suspended server hosting requested file (%s/%s) " +\
"has been woken up",
host, str(port))
except Exception:
logger.exception("Error waking up server hosting selected " +\
"file")
continue
# The file is hosted on a host, which is not suspended or
# which was successfully woken up.
pars = [["file_access", fileInfoObj.getFileId()]]
if (fileInfoObj.getFileVersion() != -1):
pars.append(["file_version", fileInfoObj.getFileVersion()])
ipAddress = hostDic[host].getIpAddress()
authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj)
resp = ngamsHttpUtils.httpGet(ipAddress, port, NGAMS_STATUS_CMD,
pars=pars, auth=authHdr)
with contextlib.closing(resp):
data = resp.read()
statusObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Result of File Access Query: %s",
re.sub("\n", "", str(statusObj.genXml().toprettyxml(' ', '\n'))))
if ((statusObj.getMessage().\
find("NGAMS_INFO_FILE_AVAIL") == -1)):
logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host]))
else:
logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host]))
foundFile = 1
break
# If no file was found we raise an exception.
if (not foundFile):
errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileId])
raise Exception(errMsg)
# The file was found, get the info necessary for the acquiring the file.
ipAddress = hostDic[host].getIpAddress()
srcFileInfo = [location, host, ipAddress, port,
diskInfoObj.getMountPoint(),
fileInfoObj.getFilename(), fileInfoObj.getFileId(),
fileInfoObj.getFileVersion(), fileInfoObj.getFormat()]
if include_compression:
srcFileInfo.append(fileInfoObj.getCompression())
msg = "Located suitable file for request - File ID: %s. " +\
"Info for file found - Location: %s - Host ID/IP: %s/%s - " +\
"Port Number: %s - File Version: %d - Filename: %s - " +\
"Mime-type: %s"
logger.debug(msg, fileId, location, host, ipAddress, port,
fileInfoObj.getFileVersion(), fileInfoObj.getFilename(),
fileInfoObj.getFormat())
return srcFileInfo
def test_archive_status_retrieve_sequence(self):
cfg = self.env_aware_cfg()
if 'sqlite' not in cfg.getDbInterface():
self.skipTest("This test works only against the sqlite db")
host_name = getHostName()
sample_file_name = "SmallFile.fits"
sample_file_path = self.resource(os.path.join("src", sample_file_name))
sample_file_size = os.path.getsize(sample_file_path)
sample_mime_type = "application/octet-stream"
# We create two NGAS clusters each containing a single NGAS node
# We configure the first NGAS cluster to use the second NGAS cluster
# as a partner site
partner_host_id = "{0}:9011".format("localhost")
config_list_1 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas1"),
("NgamsCfg.Server[1].IpAddress", "0.0.0.0"),
("NgamsCfg.PartnerSites[1].ProxyMode", "1"),
("NgamsCfg.PartnerSites[1].PartnerSite[1].Address",
partner_host_id)]
self._prepare_partner_site_cluster((9001, config_list_1))
config_list_2 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas2"),
("NgamsCfg.Server[1].IpAddress", "0.0.0.0")]
self._prepare_partner_site_cluster((9011, config_list_2))
# We archive a test sample file on the partner site cluster
self.archive(9011, sample_file_path, mimeType=sample_mime_type)
# We check the status of a file ID found on the partner site cluster
command_status = functools.partial(ngamsHttpUtils.httpGet,
"localhost",
9001,
"STATUS",
timeout=60)
argument_list = {"file_id": sample_file_name}
with contextlib.closing(
command_status(pars=argument_list)) as response:
self.assertEqual(response.status, 200)
# Parse the response XML status information
response_data = response.read()
status_info = ngamsStatus.ngamsStatus().unpackXmlDoc(
response_data, 1)
disk_info = status_info.getDiskStatusList()[0]
file_info = disk_info.getFileObjList()[0]
self.assertEqual(disk_info.getHostId(),
"{0}:9011".format(host_name))
self.assertEqual(file_info.getFileId(), sample_file_name)
self.assertEqual(file_info.getFileVersion(), 1)
self.assertEqual(file_info.getFormat(), sample_mime_type)
# Retrieve a file found on the partner site cluster
retrieve_file_path = tmp_path(sample_file_name)
self.retrieve(9001, sample_file_name, targetFile=retrieve_file_path)
retrieve_file_size = os.path.getsize(retrieve_file_path)
self.assertEqual(sample_file_size, retrieve_file_size)
# Retrieve a file with a 100 bytes offset
headers = {"range": "bytes=100-"}
self.retrieve(9001,
sample_file_name,
targetFile=retrieve_file_path,
hdrs=headers)
retrieve_file_size = os.path.getsize(retrieve_file_path)
self.assertEqual(sample_file_size - 100, retrieve_file_size)
def handleCmd(srvObj,
reqPropsObj,
httpRef):
"""
Handle STATUS command.
srvObj: Reference to NG/AMS server class object (ngamsServer).
reqPropsObj: Request Property object to keep track of actions done
during the request handling (ngamsReqProps).
httpRef: Reference to the HTTP request handler object
(ngamsHttpRequestHandler).
Returns: Void.
"""
status = ngamsStatus.ngamsStatus()
status.setDate(toiso8601()).\
setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\
setStatus(NGAMS_SUCCESS).\
setMessage("Successfully handled command STATUS").\
setState(srvObj.getState()).setSubState(srvObj.getSubState())
reqPropsObjRef = reqPropsObj
# Get the information requested.
diskId = ""
fileId = ""
fileVersion = ""
configurationFile = ""
fileAccess = ""
hostId = ""
requestId = ""
dbTime = ""
dbTimeReset = ""
fileList = ""
fileListId = ""
maxElements = 100000
if (reqPropsObj.hasHttpPar("disk_id")):
diskId = reqPropsObj.getHttpPar("disk_id")
if (reqPropsObj.hasHttpPar("file_id")):
fileId = reqPropsObj.getHttpPar("file_id")
if (reqPropsObj.hasHttpPar("file_version")):
fileVersion = int(reqPropsObj.getHttpPar("file_version"))
if (reqPropsObj.hasHttpPar("configuration_file")):
configurationFile = reqPropsObj.getHttpPar("configuration_file")
if (reqPropsObj.hasHttpPar("file_access")):
fileAccess = reqPropsObj.getHttpPar("file_access")
if (reqPropsObj.hasHttpPar("host_id")):
hostId = reqPropsObj.getHttpPar("host_id")
if (reqPropsObj.hasHttpPar("max_elements")):
maxElements = int(reqPropsObj.getHttpPar("max_elements"))
if (reqPropsObj.hasHttpPar("request_id")):
requestId = reqPropsObj.getHttpPar("request_id")
if (reqPropsObj.hasHttpPar("db_time")):
dbTime = True
if (reqPropsObj.hasHttpPar("db_time_reset")):
dbTimeReset = True
if (reqPropsObj.hasHttpPar("flush_log")):
# in the past this called flushLog()
# do we really want to keep that functionality? I doubt it
pass
if (reqPropsObj.hasHttpPar("file_list")):
fileList = int(reqPropsObj.getHttpPar("file_list"))
if (reqPropsObj.hasHttpPar("file_list_id")):
fileListId = reqPropsObj.getHttpPar("file_list_id")
if (reqPropsObj.hasHttpPar("dump_object_info")):
# Dump status of all objects allocated into file specified.
targFile = reqPropsObj.getHttpPar("dump_object_info")
rmFile(targFile)
fo = open(targFile, "w")
fo.write(_genObjectStatus())
fo.close()
# Handle request for file info.
genCfgStatus = 0
genDiskStatus = 0
genFileStatus = 0
genStatesStatus = 1
genRequestStatus = 0
msg = ""
help = 0
if (reqPropsObj.hasHttpPar("help")):
global _help
msg = _help
help = 1
elif hostId and hostId != srvObj.getHostId():
host, port = srvObj.get_remote_server_endpoint(hostId)
cfgObj = srvObj.getCfg()
if not cfgObj.getProxyMode():
httpRef.redirect(host, port)
return
else:
try:
httpRef.proxy_request(hostId, host, port)
except Exception as e:
ex = re.sub("<|>", "", str(e))
errMsg = genLog("NGAMS_ER_COM",
[host, port,ex])
raise Exception(errMsg)
return
elif (fileList):
if (not fileListId):
# It's a new STATUS?file_list request.
fileListId = _handleFileList(srvObj, reqPropsObj, httpRef)
# Send back data from the request.
_handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId,
maxElements)
elif (diskId):
diskObj = ngamsDiskInfo.ngamsDiskInfo()
diskObj.read(srvObj.getDb(), diskId)
status.addDiskStatus(diskObj)
genDiskStatus = 1
elif (fileId):
if (not fileVersion): fileVersion = -1
try:
fileObj = ngamsFileInfo.ngamsFileInfo()
fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion)
diskObj = ngamsDiskInfo.ngamsDiskInfo()
try:
diskObj.read(srvObj.getDb(), fileObj.getDiskId())
except:
errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId)
raise Exception(errMsg)
diskObj.addFileObj(fileObj)
status.addDiskStatus(diskObj)
except:
# The file was not found in the database. Check if it is available
# on a remote partner site.
host, port, status_info, disk_info, file_info = \
ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId,
fileVersion,
reqPropsObj)
# Update status reply using the disk status and file status
# information retrieved from the partner site
status.addDiskStatus(disk_info)
genDiskStatus = 1
genFileStatus = 1
elif (requestId):
logger.debug("Checking status of request with ID: %s", requestId)
reqPropsObjRef = srvObj.getRequest(requestId)
if (not reqPropsObjRef):
errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId])
raise Exception(errMsg)
genRequestStatus = 1
elif (configurationFile):
msg = "configuration_file=" + srvObj.cfg_fname
genCfgStatus = 1
# Hidden feature to return complete Cfg!!
try:
genCfgStatus = int(configurationFile)
except ValueError:
genCfgStatus = 1
status.setNgamsCfgObj(srvObj.getCfg())
elif (fileAccess):
if (not fileVersion): fileVersion = -1
fileId = fileAccess
msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId,
fileVersion, diskId)
elif (dbTime):
logger.debug("Querying total DB time")
msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime()
elif (dbTimeReset):
msg = "Resetting DB timer"
logger.debug(msg)
srvObj.getDb().resetDbTime()
else:
msg = "Successfully handled command STATUS"
if (reqPropsObjRef == reqPropsObj):
reqPropsObj.setCompletionTime()
srvObj.updateRequestDb(reqPropsObj)
if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus):
status.setReqStatFromReqPropsObj(reqPropsObjRef)
# Generate XML reply.
logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus]))
xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus,
genStatesStatus)
xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat)
httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT)
elif not httpRef.reply_sent:
httpRef.send_status(msg)
if (msg and (not help)):
logger.info(msg)
else:
logger.info("Successfully handled command STATUS")
def downloadFiles(host, port, srcHost, path, outputDir, list=0):
"""
Download the files matching the given pattern via the host specified,
from the host(s) indicated.
The files downloaded will be stored in a directory structure of the
form:
<Output Dir>/<Source Host>/<File Path>
I.e., the filepath of the file is preserved, and the source host name is
also kept.
host: Name of host to be contacted (string).
port: Port number of NG/AMS Server (integer).
srcHost: List of hosts to be contacted (<Host 1>,<Host 2>,...).
Can also be a single host, wildcards is allowed (string).
path: File pattern of file(s) to be retrieved. If this is a
directory, the files in the directory will be retrieved
(string).
outputDir: Output directory. If not given, the current working point
will be chosen (string).
list: If set to 1 and the path given is a directory, rather than
retrieving the file in the remote directory, the names of
the remote files are listed on stdout (integer/0|1).
Returns: Void.
"""
if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir)
reply, msg, hdrs, data = ngamsLib.httpGet(host,
port,
NGAMS_RETRIEVE_CMD,
pars=[["internal", path],
["host_id", srcHost]],
timeOut=10)
if (isNgasXmlStatusDoc(data)):
stat = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1)
if (stat.getStatus() == NGAMS_FAILURE):
msg = "Problem handling request: %s" % stat.getMessage()
raise Exception, msg
else:
stat = None
if (data.find("<?xml version=") == -1):
if (not list):
storeFile(hdrs, data, outputDir)
else:
print path
return
# The reply is a file list. Retrieve the file in question.
remDir = stat.getFileListList()[0].getComment().split(": ")[-1]
remDir = os.path.normpath(remDir)
outputDir = os.path.normpath("%s/%s" % (outputDir, remDir.split("/")[-1]))
if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir)
client = ngamsPClient.ngamsPClient()
for fileInfo in stat.getFileListList()[0].getFileInfoObjList():
# If we should only list the contents and if the entry is a file,
# we only display its information.
if (list):
print "%s %-8s %-8s %-10s %-16s %s" %\
(fileInfo.getPermissions(), fileInfo.getOwner(),
fileInfo.getGroup(), str(fileInfo.getFileSize()),
fileInfo.getModDate(), fileInfo.getFilename())
# If the entry is a file and we should only list, we go to the
# next item.
if (fileInfo.getPermissions()[0] != "d"): continue
# Handle/download the item.
fn = fileInfo.getFilename()
basename = os.path.basename(fn)
trgFile = os.path.normpath("%s/%s" % (outputDir, basename))
if (not list): info(1, "Requesting: %s:%s ..." % (srcHost, fn))
reply2, msg2, hdrs2, data2 = \
ngamsLib.httpGet(host, port, NGAMS_RETRIEVE_CMD,
pars=[["internal", fn],
["host_id", srcHost]], timeOut=10)
# If the remote object specified is a file, retrieve it, otherwise if a
# directory, call this function recursively to retrieve the contents of
# the sub-folder(s).
if (isNgasXmlStatusDoc(data2)):
stat2 = ngamsStatus.ngamsStatus().unpackXmlDoc(data2, 1)
if (stat2.getStatus() == NGAMS_FAILURE):
msg = "Problem handling request: %s" % stat2.getMessage()
raise Exception, msg
# It seems to be another File List, we retrieve the contents
# of this recursively.
if (not list): info(1, "Remote object found: %s" % fn)
downloadFiles(host, port, srcHost, fn, outputDir, list)
else:
# Just retrieve the file.
storeFile(hdrs2, data2, outputDir)
return
