def cretrieve(self, containerName, containerId=None, targetDir='.'): """ Sends a CRETRIEVE command to NG/AMS to retrieve the full contents of a container and dumps them into the file system. """ if (not containerId and not containerName): msg = "Must specify parameter -containerId or -containerName for " +\ "a CRETRIEVE Command" raise Exception(msg) if not targetDir: targetDir = '.' pars = [] if containerId: pars.append(("container_id", containerId)) if containerName: pars.append(("container_name", containerName)) resp, host, port = self._get('CRETRIEVE', pars=pars) host_id = "%s:%d" % (host, port) with contextlib.closing(resp): if resp.status != NGAMS_HTTP_SUCCESS: return ngamsStatus.ngamsStatus().unpackXmlDoc(resp.read(), 1) size = int(resp.getheader('Content-Length')) handler = ngamsMIMEMultipart.FilesystemWriterHandler( 1024, basePath=targetDir) parser = ngamsMIMEMultipart.MIMEMultipartParser( handler, resp, size, 65536) parser.parse() return ngamsStatus.dummy_success_stat(host_id)
def _httpPost(srvObj, url, filename, sessionId): """ A wrapper for _httpPostUrl return success 0 or failure 1 filename full path for the file to be sent sessionId session uuid """ #info(3, "xxxxxx ---- filename %s" % filename) stat = ngamsStatus.ngamsStatus() baseName = os.path.basename(filename) contDisp = "attachment; filename=\"" + baseName + "\"" contDisp += "; no_versioning=1" logger.debug("Async Delivery Thread [%s] Delivering file: %s - to: %s", str(thread.get_ident()), baseName, url) ex = "" try: reply, msg, hdrs, data = \ _httpPostUrl(url, fileMimeType, contDisp, filename, "FILE", blockSize=\ srvObj.getCfg().getBlockSize(), session_uuid = sessionId) if (reply == None and msg == None and hdrs == None and data == None): # transfer cancelled/suspended return 1 if (data.strip() != ""): stat.clear().unpackXmlDoc(data) else: stat.clear().setStatus(NGAMS_SUCCESS) except Exception, e: ex = str(e)
def _checkFile(srvObj, fileInfoObj, stagFile, httpHeaders, checkChecksum): """ Make a consistency check of the Staging File. srvObj: Reference to instance of Server Object (ngamsServer). fileInfoObj: File info object with info about the file (ngamsFileInfo). stagFile: Staging filename (string). httpHeaders: HTTP headers (mimetools.Message) checkChecksum: Carry out checksum check (0|1/integer). Returns: Void. """ # First ensure to flush file caches. ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagFile]) # Check file size. fileSize = getFileSize(stagFile) if (fileSize != fileInfoObj.getFileSize()): # If the mime-type is 'text/xml' we check if the returned # document is an NG/AMS XML Status reporting a problem. tmpStat = None if (httpHeaders.type == NGAMS_XML_MT): try: tmpStat = ngamsStatus.ngamsStatus().load(stagFile,getStatus=1) if (tmpStat.getStatus() != NGAMS_FAILURE): del tmpStat tmpStat = None except: # Was apparently not an NG/AMS Status Document. pass if (tmpStat): # An error response was received from the source node. raise Exception(tmpStat.getMessage()) else: # The file seems not to have been transferred completely. errMsg = "Size of cloned file wrong (expected: %d/actual: %d)" raise Exception(errMsg % (fileInfoObj.getFileSize(), fileSize)) logger.debug("Size of cloned Staging File OK: %s", stagFile) # The file size was correct. if (checkChecksum): if ngamsFileUtils.check_checksum(srvObj, fileInfoObj, stagFile): logger.debug("Checksum of cloned Staging File OK: %s", stagFile) else: logger.debug("No Checksum or Checksum Plug-In specified for file")
def _httpPost(srvObj, url, filename, sessionId=None): """ A wrapper for _httpPostUrl return success 0 or failure 1 filename full path for the file to be sent sessionId session uuid """ #info(3, "xxxxxx ---- filename %s" % filename) stat = ngamsStatus.ngamsStatus() baseName = os.path.basename(filename) contDisp = "attachment; filename=\"" + baseName + "\"" contDisp += "; no_versioning=1" thread_id = threading.current_thread().ident logger.debug("Async Delivery Thread [%s] Delivering file: %s - to: %s ...", str(thread_id), baseName, url) ex = "" try: #TODO - the fileMimeType should be queried from the database reply, msg, hdrs, data = _httpPostUrl(url, fileMimeType, contDisp, filename, "FILE", blockSize=\ srvObj.getCfg().getBlockSize(), session_uuid = sessionId) #reply, msg, hdrs, data = ngamsLib.httpPostUrl(url, fileMimeType, contDisp, filename, "FILE", blockSize=srvObj.getCfg().getBlockSize()) if (reply == None and msg == None and hdrs == None and data == None): # transfer cancelled/suspended return 1 if (data.strip() != ""): stat.clear().unpackXmlDoc(data) else: stat.clear().setStatus(NGAMS_SUCCESS) except Exception as e: ex = str(e) if ((ex != "") or (reply != NGAMS_HTTP_SUCCESS) or (stat.getStatus() == NGAMS_FAILURE)): errMsg = "Error occurred while async delivering file: " + baseName +\ " - to url: " + url + " by Data Delivery Thread [" + str(thread_id) + "]" if (ex != ""): errMsg += " Exception: " + ex + "." if (stat.getMessage() != ""): errMsg += " Message: " + stat.getMessage() logger.warning(errMsg + '\n' + traceback.format_exc()) return 1 else: logger.debug( "File: %s delivered to url: %s by Async Delivery Thread [%s]", baseName, url, str(thread_id)) return 0
def _sendCheckFileCmd(node, fileId, fileVersion): """ Send a CHECKFILE Command to the specified node. One of the following is returned: 1. File OK: NGAMS_INFO_FILE_OK. 2. File Not OK: NGAMS_ER_FILE_NOK 3. Error: FAILURE. node: Node to be contacted (node:port) (string). fileId: ID of file to check (string). fileVersion: Version of file to check (integer). Returns: See above (NGAMS_INFO_FILE_OK | NGAMS_ER_FILE_NOK | FAILURE). """ T = TRACE(5) cmdPars = [["file_id", fileId], ["file_version", fileVersion]] data = None try: host, port = node.split(":") logger.debug( "Sending CHECKFILE Command for file: %s/%s to node: %s:%s", fileId, str(fileVersion), host, str(port)) resp = ngamsHttpUtils.httpGet(host, port, NGAMS_CHECKFILE_CMD, pars=cmdPars) with contextlib.closing(resp): data = resp.read() except Exception: logger.exception("Error contacting node %s", node) if (data): tmpStatObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data) if (tmpStatObj.getMessage().find(NGAMS_INFO_FILE_OK) != -1): return NGAMS_INFO_FILE_OK else: # Assume: NGAMS_ER_FILE_NOK. return NGAMS_ER_FILE_NOK else: return NGAMS_FAILURE
def test_status_retrieve_sequence(self): bad_file_name = "dummy.fits" # We create two NGAS clusters each containing a single NGAS node # We configure the first NGAS cluster to use the second NGAS cluster # as a partner site partner_host_id = "{0}:9011".format("localhost") config_list_1 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas1"), ("NgamsCfg.Server[1].IpAddress", "0.0.0.0"), ("NgamsCfg.PartnerSites[1].ProxyMode", "1"), ("NgamsCfg.PartnerSites[1].PartnerSite[1].Address", partner_host_id)] self._prepare_partner_site_cluster((9001, config_list_1)) config_list_2 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas2"), ("NgamsCfg.Server[1].IpAddress", "0.0.0.0")] self._prepare_partner_site_cluster((9011, config_list_2)) # We check the status of a file ID found on the partner site cluster command_status = functools.partial(ngamsHttpUtils.httpGet, "localhost", 9001, "STATUS", timeout=60) argument_list = {"file_id": bad_file_name} with contextlib.closing( command_status(pars=argument_list)) as response: self.assertEqual(response.status, 400) # Parse the response XML status information response_data = response.read() status_info = ngamsStatus.ngamsStatus().unpackXmlDoc( response_data, 1) self.assertEqual(status_info.getStatus(), "FAILURE") self.assertEqual( status_info.getMessage(), "NGAMS_ER_UNAVAIL_FILE:4019:ERROR: File with ID: dummy.fits appears not to be available." ) # Retrieve a file found on the partner site cluster retrieve_file_path = tmp_path(bad_file_name) self.retrieve(9001, bad_file_name, targetFile=retrieve_file_path, expectedStatus='FAILURE')
def _execCClient(unpackXmlStat = 1, pars = []): """ Execute the NG/AMS C-Client on the shell. cmdLineParsDic: Dictionary with command line parameters (dictionary). pars: Extra parameters for invoking NGAMS C-Client (list). Returns: List with status objects or stdout output from c-client: [<stat obj>, ...] or <stdout c-client> (list|string). """ cmd = ['ngamsCClient'] for opt, val in pars: cmd.append(opt) cmd.append(str(val)) if '-servers' not in (x for x, _ in pars): cmd.append('-host') cmd.append('127.0.0.1') cmd.append('-status') env = os.environ.copy() env['NGAMS_VERBOSE_LEVEL'] = '0' _, out, _ = execCmd(cmd, shell=False) out = utils.b2s(out) if (unpackXmlStat): statObjList = [] xmlStatList = out.split("Command repetition counter:") for xmlStat in xmlStatList: # Clean up the output. xmlStat = xmlStat.strip() if (xmlStat == ""): continue idx = 0 while (xmlStat[idx] != "<"): idx += 1 xmlStat = xmlStat[idx:] # Unpack it. statObj = ngamsStatus.ngamsStatus().unpackXmlDoc(xmlStat) statObjList.append(statObj) out = statObjList return out
def _post(self, cmd, mime_type, data, pars=[]): if self.timeout: pars.append(["time_out", str(self.timeout)]) if self.reload_mod: pars.append(["reload", "1"]) # For now we try the serves in random order. servers = self.servers random.shuffle(servers) for i, host_port in enumerate(servers): host, port = host_port try: _, _, _, data = self._do_post(host, port, cmd, mime_type, data, pars) return ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1) except socket.error: if i == len(servers) - 1: raise
def processHttpReply(http, basename, url): """ After file is sent, collect the ngams status from the remote server parse error message, and log if necessary http: the HTTP Client basename Name of the file sent to the remote ngas server (used in the content disposition) url: the url of the remote ngas server that receives the file """ logger.debug("Waiting for reply ...") # ngamsLib._setSocketTimeout(None, http) reply, msg, hdrs = http.getreply() if (hdrs == None): errMsg = "Illegal/no response to HTTP request encountered!" raise Exception(errMsg) if (hdrs.has_key("content-length")): dataSize = int(hdrs["content-length"]) else: dataSize = 0 data = http.getfile().read(dataSize) stat = ngamsStatus.ngamsStatus() if (data.strip() != ""): stat.clear().unpackXmlDoc(data) else: # TODO: For the moment assume success in case no # exception was thrown. stat.clear().setStatus(NGAMS_SUCCESS) if ((reply != NGAMS_HTTP_SUCCESS) or (stat.getStatus() == NGAMS_FAILURE)): errMsg = 'Error occurred while proxy quick archive file %s to %s' % (basename, url) if (stat.getMessage() != ""): errMsg += " Message: " + stat.getMessage() raise Exception(errMsg)
def _httpPost(srvObj, url, filename, sessionId): """ A wrapper for _httpPostUrl return success 0 or failure 1 filename full path for the file to be sent sessionId session uuid """ #info(3, "xxxxxx ---- filename %s" % filename) stat = ngamsStatus.ngamsStatus() baseName = os.path.basename(filename) contDisp = "attachment; filename=\"" + baseName + "\"" contDisp += "; no_versioning=1" thread_id = threading.current_thread().ident logger.debug("Async Delivery Thread [%s] Delivering file: %s - to: %s", str(thread_id), baseName, url) ex = "" try: reply, msg, hdrs, data = \ _httpPostUrl(url, fileMimeType, contDisp, filename, "FILE", blockSize=\ srvObj.getCfg().getBlockSize(), session_uuid = sessionId) if (reply == None and msg == None and hdrs == None and data == None): # transfer cancelled/suspended return 1 if (data.strip() != ""): stat.clear().unpackXmlDoc(data) else: stat.clear().setStatus(NGAMS_SUCCESS) except Exception as e: ex = str(e) if ((ex != "") or (reply != NGAMS_HTTP_SUCCESS) or (stat.getStatus() == NGAMS_FAILURE)): errMsg = "Error occurred while async delivering file: " + baseName +\ " - to url: " + url + " by Data Delivery Thread [" + str(thread_id) + "]" if (ex != ""): errMsg += " Exception: " + ex + "." if (stat.getMessage() != ""): errMsg += " Message: " + stat.getMessage() logger.warning(errMsg) jobManHost = srvObj.getCfg().getNGASJobMANHost() if (jobManHost): try: if (not ex): ex = '' rereply = urlrequest.urlopen( 'http://%s/failtodeliverfile?file_id=%s&to_url=%s&err_msg=%s' % (jobManHost, baseName, urlparse.quote(url), urlparse.quote(ex)), timeout=15).read() logger.debug( 'Reply from sending file %s failtodeliver event to server %s - %s', baseName, jobManHost, rereply) except Exception as err: logger.error( 'Fail to send fail-to-deliver event to server %s, Exception: %s', jobManHost, str(err)) return 1 else: logger.debug( "File: %s - delivered to url: %s by Async Delivery Thread [%s]", baseName, url, str(thread_id)) return 0
def test_NormalOp_1(self): """ Synopsis: Test normal operation of the NG/AMS Archive Client. Description: It is tested that a file can be archived via a link in the Archive Queue. Expected Result: After the archiving the link is moved to the Archive Files Area. It is also checked that the NG/AMS XML Status Document is created in the Archived Files Area. After the expiration time for keeping archived files has expired, the archived file and the XML status document should be deleted. Test Steps: - Start NG/AMS Server. - Start instance of the NG/AMS Archive Client. - Create a link from a legal test (FITS) file into the Archive Queue. - Test that the file is archived within 20s and moved to the Archived Files Area. - Test that the XML Status Document from NG/AMS is stored in the Archived Files Area. - Check that after the given expiration time for the Archived Files Area, that the archived file + the XML Status Document are removed. - Stop the Archive Client. Remarks: ... """ self.prepExtSrv() # Make sure the the queue subdir exist before the launch the client; # otherwise the client and this test might find themselves in a race # condition and the test might fail d = os.path.abspath(os.path.join(arcCliDir, 'queue')) if not os.path.exists(d): os.makedirs(d) self.startArchiveClient() # Archive a file as copy and link. # Make sure at least the quee dir is already created srcFile = self.resource("src/SmallFile.fits") self.cp(srcFile, os.path.join(arcCliDir, 'queue')) os.symlink(srcFile, os.path.join(arcCliDir, 'queue', 'Test.fits')) # Check that files are being archived (within 20s) + NG/AMS Status # Documents created. file1Pat = arcCliDir + "/archived/*___SmallFile.fits" file1StatPat = file1Pat + "___STATUS.xml" file2Pat = arcCliDir + "/archived/*___Test.fits" file2StatPat = file2Pat + "___STATUS.xml" startTime = time.time() filesFound = 0 while ((time.time() - startTime) < 20): globFile1Pat = glob.glob(file1Pat) globFile1StatPat = glob.glob(file1StatPat) globFile2Pat = glob.glob(file2Pat) globFile2StatPat = glob.glob(file2StatPat) if ((len(globFile1Pat) == 1) and (len(globFile1StatPat) == 1) and (len(globFile2Pat) == 1) and (len(globFile2StatPat) == 1)): filesFound = 1 break if (not filesFound): if (not len(globFile1Pat)): errMsg = "Did not find status file: " + file1Pat elif (not len(globFile1StatPat)): errMsg = "Did not find status XML document: " + file1StatPat elif (not len(globFile2Pat)): errMsg = "Did not find status file: " + file2Pat else: # (not len(globFile2StatPat)): errMsg = "Did not find status XML document: " + file2StatPat self.fail(errMsg) # Check the contents of one of the status documents. statObj = ngamsStatus.ngamsStatus().load(globFile1StatPat[0]) refStatFile = "ref/ngamsArchiveClientTest_test_NormalOp_1_1_ref" filters = ("BytesStored:", "NumberOfFiles:", "FileName:", "FileVersion:") self.assert_status_ref_file(refStatFile, statObj, filters=filters) # Check that the status documents are removed within 10s. filesRemoved = 0 startTime = time.time() while ((time.time() - startTime) < 20): globFile1Pat = glob.glob(file1Pat) globFile1StatPat = glob.glob(file1StatPat) globFile2Pat = glob.glob(file2Pat) globFile2StatPat = glob.glob(file2StatPat) if ((len(globFile1Pat) == 0) and (len(globFile1StatPat) == 0) and (len(globFile2Pat) == 0) and (len(globFile2StatPat) == 0)): filesRemoved = 1 break if (not filesRemoved): if (len(globFile1Pat)): errMsg = "Did not remove status file: " + globFile1Pat[0] elif (len(globFile1StatPat)): errMsg = "Did not remove status XML document: " +\ globFile1StatPat[0] elif (len(globFile2Pat)): errMsg = "Did not remove status file: " + file2Pat[0] else: # (len(globFile2StatPat)): errMsg = "Did not remove status XML document: " + file2StatPat[ 0] self.fail(errMsg)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Compress the file based on file_path (file_id, disk_id, and file_version) srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ # target_host does not contain port required_params = [ 'file_path', 'file_id', 'file_version', 'disk_id', 'target_host', 'crc_db', 'debug' ] filename = reqPropsObj.getHttpPar('file_path') if (not os.path.exists(filename)): errMsg = 'File does not exist: %s' % filename logger.warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=501) return parDic = reqPropsObj.getHttpParsDic() for rp in required_params: if (not parDic.has_key(rp)): errMsg = 'Parameter missing: %s' % rp httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=502) return logger.debug('Moving file %s', filename) fileId = parDic['file_id'] fileVersion = int(parDic['file_version']) diskId = parDic['disk_id'] tgtHost = parDic['target_host'] crcDb = parDic['crc_db'] debug = int(parDic['debug']) if (not isMWAVisFile(fileId)): errMsg = 'Not MWA visibilty file: %' % (fileId) logger.warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=503) return """ if (fileOnHost(srvObj, fileId, tgtHost)): errMsg = "File %s already on host %s" % (fileId, tgtHost) warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT) # make it correct return """ if (debug): logger.debug('Only for file movment debugging, return now') httpRef.send_data('DEBUG ONLY', NGAMS_TEXT_MT) return fileCRC = getCRCFromFile(filename) if (fileCRC != crcDb): errMsg = 'File %s on source host %s already corrupted, moving request rejected' % ( filename, srvObj.getHostId().replace(':', '-')) logger.warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=504) return sendUrl = 'http://%s:7777/QARCHIVE' % tgtHost logger.debug("Moving to %s", sendUrl) fileMimeType = 'application/octet-stream' baseName = os.path.basename(filename) contDisp = "attachment; filename=\"" + baseName + "\"" deliver_success = False last_deliv_err = '' stat = ngamsStatus.ngamsStatus() hdrs = {NGAMS_HTTP_HDR_CHECKSUM: fileCRC} for i in range(3): # total trials - 3 times with open(filename, "rb") as f: try: reply, msg, hdrs, data = ngamsHttpUtils.httpPostUrl( sendUrl, f, fileMimeType, contDisp=contDisp, hdrs=hdrs) if (data.strip() != ""): stat.clear().unpackXmlDoc(data) else: stat.clear().setStatus(NGAMS_SUCCESS) if (reply != NGAMS_HTTP_SUCCESS or stat.getStatus() == NGAMS_FAILURE): logger.warning("Attempt %d failed: %s", i, stat.getMessage()) last_deliv_err = stat.getMessage() continue else: deliver_success = True break except Exception as hexp: logger.warning("Attempt %d failed: %s", i, str(hexp)) last_deliv_err = str(hexp).replace('\n', '--') continue if (not deliver_success): errMsg = 'File %s failed to be moved to %s: %s' % (fileId, tgtHost, last_deliv_err) logger.warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=505) return try: ngamsDiscardCmd._discardFile(srvObj, diskId, fileId, fileVersion, execute=1) except Exception as e1: logger.warning('Fail to remove file %s: %s', filename, str(e1)) httpRef.send_data('Remove error: %s' % str(e1).replace('\n', '--'), NGAMS_TEXT_MT) return httpRef.send_data(MOVE_SUCCESS, NGAMS_TEXT_MT)
def lookup_partner_site_file_status(ngas_server, file_id, file_version, request_properties): """ Lookup the file indicated by the File ID using a NGAS partner site (if one is specified in the configuration). Returns a tuple of status objects. Parameters: ngas_server: Reference to NG/AMS server class object (ngamsServer). file_id: File ID of file to locate (string). file_version: Version of the file (integer). request_properties: Request Property object to keep track of actions done during the request handling (ngamsReqProps|None). Returns: host: Partner site host name (or IP address) port: Partner site port number status_info: Status response object (ngamsStatus) disk_info: Status response disk information object (ngamsDiskInfo) file_info: Status response file information object (ngamsFileInfo) """ file_reference = file_id if file_version > 0: file_reference += "/Version: " + str(file_version) # If the request came from a partner site. We will not continue to # propagate the request to avoid a death loop scenario. We will raise an # exception. if request_properties.hasHttpPar("partner_site_redirect"): error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference]) raise Exception(error_message) # Check partner sites is enabled are available from the configuration if not ngas_server.is_partner_sites_proxy_mode()\ or not ngas_server.get_partner_sites_address_list(): error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference]) raise Exception(error_message) # Lets query the partner sites for the availability of the requested file authentication_header = ngamsSrvUtils.genIntAuthHdr(ngas_server) parameter_list = [["file_id", file_id]] if file_version != -1: parameter_list.append(["file_version", file_version]) parameter_list.append(["partner_site_redirect", 1]) host, port, status_info, disk_info, file_info = None, None, None, None, None for partner_site in ngas_server.get_partner_sites_address_list(): partner_address, partner_domain, partner_port = parse_host_id( partner_site) try: logger.info("Looking up file ID %s on partner site %s", file_id, partner_site) response = ngamsHttpUtils.httpGet(partner_address, partner_port, NGAMS_STATUS_CMD, parameter_list, auth=authentication_header) with contextlib.closing(response): response_info = response.read() except: # We ignore this error, and try the next partner site, if any continue status_info = ngamsStatus.ngamsStatus().unpackXmlDoc(response_info, 1) logger.info("Result of File Access Query: {}".format( re.sub("\n", "", str(status_info.genXml().toprettyxml(' ', '\n'))))) if status_info.getStatus() == "FAILURE": logger.info( genLog("NGAMS_INFO_FILE_NOT_AVAIL", [file_id, partner_address])) else: logger.info( genLog("NGAMS_INFO_FILE_AVAIL", [file_id, partner_address])) disk_info = status_info.getDiskStatusList()[0] file_info = disk_info.getFileObjList()[0] # This is a bit of a hack because the host ID may not contain # the fully qualified address. We append the domain name from # the partner site address. This should work because they are # part of the same cluster. host, domain, port = parse_host_id(disk_info.getHostId()) if domain is None and partner_domain is not None: host = host + "." + partner_domain break if status_info is None or status_info.getStatus() == "FAILURE": # Failed to find file on a partner site error_message = genLog("NGAMS_ER_UNAVAIL_FILE", [file_reference]) raise Exception(error_message) return host, port, status_info, disk_info, file_info
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle STATUS command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ T = TRACE() status = ngamsStatus.ngamsStatus() status.\ setDate(toiso8601()).\ setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\ setStatus(NGAMS_SUCCESS).\ setMessage("Successfully handled command STATUS").\ setState(srvObj.getState()).setSubState(srvObj.getSubState()) reqPropsObjRef = reqPropsObj # Get the information requested. diskId = "" fileId = "" fileVersion = "" configurationFile = "" fileAccess = "" hostId = "" requestId = "" dbTime = "" dbTimeReset = "" fileList = "" fileListId = "" maxElements = 100000 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): fileVersion = reqPropsObj.getHttpPar("file_version") if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = "-" if (reqPropsObj.hasHttpPar("file_access")): fileAccess = reqPropsObj.getHttpPar("file_access") if (reqPropsObj.hasHttpPar("host_id")): hostId = reqPropsObj.getHttpPar("host_id") if (reqPropsObj.hasHttpPar("max_elements")): maxElements = int(reqPropsObj.getHttpPar("max_elements")) if (reqPropsObj.hasHttpPar("request_id")): requestId = reqPropsObj.getHttpPar("request_id") if (reqPropsObj.hasHttpPar("db_time")): dbTime = True if (reqPropsObj.hasHttpPar("db_time_reset")): dbTimeReset = True if (reqPropsObj.hasHttpPar("flush_log")): # in the past this called flushLog() # do we really want to keep that functionality? I doubt it pass if (reqPropsObj.hasHttpPar("file_list")): fileList = int(reqPropsObj.getHttpPar("file_list")) if (reqPropsObj.hasHttpPar("file_list_id")): fileListId = reqPropsObj.getHttpPar("file_list_id") if (reqPropsObj.hasHttpPar("dump_object_info")): # Dump status of all objects allocated into file specified. targFile = reqPropsObj.getHttpPar("dump_object_info") rmFile(targFile) fo = open(targFile, "w") fo.write(_genObjectStatus()) fo.close() # Handle request for file info. genCfgStatus = 0 genDiskStatus = 0 genFileStatus = 0 genStatesStatus = 1 genRequestStatus = 0 msg = "" help = 0 if (reqPropsObj.hasHttpPar("help")): global _help msg = _help help = 1 elif hostId and hostId != srvObj.getHostId(): host, port = srvObj.get_remote_server_endpoint(hostId) cfgObj = srvObj.getCfg() if not cfgObj.getProxyMode(): httpRef.redirect(host, port) return else: try: httpRef.proxy_request(hostId, host, port) except Exception, e: ex = re.sub("<|>", "", str(e)) errMsg = genLog("NGAMS_ER_COM", [host, port,ex]) raise Exception, errMsg return
def _locateArchiveFile(srvObj, fileId, fileVersion, diskId, hostId, reqPropsObj, files, include_compression): """ See description of ngamsFileUtils.locateArchiveFile(). This function is used simply to encapsulate the complete processing to be able to clean up. """ msg = "_locateArchiveFile() - Disk ID: %s - File ID: " +\ "%s - File Version: %d ..." logger.debug(msg, str(diskId), fileId, int(fileVersion)) # Filter out files not on specified host if host ID is given. if (hostId): files = filter(lambda x: x[1] == hostId, files) # If no file was found we raise an exception. if not files: tmpFileRef = fileId if (fileVersion > 0): tmpFileRef += "/Version: " + str(fileVersion) if (diskId): tmpFileRef += "/Disk ID: " + diskId if (hostId): tmpFileRef += "/Host ID: " + hostId errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [tmpFileRef]) raise Exception(errMsg) # We now sort the file information sub-lists in the file list. # The priori is as follows: # # 1. Local host. # 2. Same cluster. # 3. Same domain (e.g. hq.eso.org). # 4. Other files (remote files). localHostFileList = [] clusterFileList = [] domainFileList = [] remoteFileList = [] all_hosts = set([x[1] for x in files]) hostDic = ngamsHighLevelLib.resolveHostAddress(srvObj.getHostId(), srvObj.getDb(), srvObj.getCfg(), all_hosts) # Loop over the candidate files and sort them. fileCount = idx = 0 for fileInfo in files: fileHost = fileInfo[1] if (hostDic[fileHost].getHostType() == NGAMS_HOST_LOCAL): localHostFileList.append(fileInfo) elif (hostDic[fileHost].getHostType() == NGAMS_HOST_CLUSTER): clusterFileList.append(fileInfo) elif (hostDic[fileHost].getHostType() == NGAMS_HOST_DOMAIN): domainFileList.append(fileInfo) else: # NGAMS_HOST_REMOTE: remoteFileList.append(fileInfo) idx += 1 fileCount += 1 # The highest priority of the file is determined by the File Version, # the latest version is preferred even though this may be stored # on another NGAS host. # A dictionary is built up, which contains the candidate files. The # format is such there each version found is one key. For each key # (= version) there is a list with the corresponding file information # order according to the location. candFileDic = {} fileLists = [[NGAMS_HOST_LOCAL, localHostFileList], [NGAMS_HOST_CLUSTER, clusterFileList], [NGAMS_HOST_DOMAIN, domainFileList], [NGAMS_HOST_REMOTE, remoteFileList]] for fileListInfo in fileLists: location = fileListInfo[0] fileList = fileListInfo[1] for fileInfo in fileList: fileVer = fileInfo[0].getFileVersion() # Create a list in connection with each File Version key. if fileVer not in candFileDic: candFileDic[fileVer] = [] candFileDic[fileVer].append([location, fileInfo[0], fileInfo[1]]) fileVerList = list(candFileDic) fileVerList.sort(reverse=True) if logger.level <= logging.DEBUG: msg = "" count = 1 for fileVer in fileVerList: for fi in candFileDic[fileVer]: msg += "(" + str(count) + ": Location:" + fi[0] +\ ", Host:" + fi[2] + ", Version:" +\ str(fi[1].getFileVersion()) + ") " count += 1 logger.debug("File list to check: " + msg) # If no files were found we raise an exception. if (len(candFileDic) == 0): if (fileVersion != -1): fileRef = fileId + "/V" + str(fileVersion) else: fileRef = fileId errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileRef]) raise Exception(errMsg) # We generate a list with the Disk IDs (which we need later). # Generate a dictionary with Disk Info Objects. diskIdDic = {} for fileVer in fileVerList: for fileInfo in candFileDic[fileVer]: diskIdDic[fileInfo[1].getDiskId()] = fileInfo[1].getDiskId() sqlDiskInfo = srvObj.getDb().getDiskInfoFromDiskIdList(list(diskIdDic)) diskInfoDic = {} for diskInfo in sqlDiskInfo: diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().unpackSqlResult(diskInfo) diskInfoDic[diskInfoObj.getDiskId()] = diskInfoObj logger.debug("Disk Info Objects Dictionary: %s", str(diskInfoDic)) # Check if the files are accessible - when the first accessible file # in the fileList is found, the information is returned as the file wanted. # To check the file accessibility, it is also checked if the NG/AMS # 'responsible' for the file, allows for Retrieve Requests (only done # in connection with a Retrieve Request). logger.debug("Checking which of the candidate files should be selected ...") foundFile = 0 for fileVer in fileVerList: if (foundFile): break for fileInfo in candFileDic[fileVer]: location = fileInfo[0] fileInfoObj = fileInfo[1] host = fileInfo[2] diskInfoObj = diskInfoDic[fileInfoObj.getDiskId()] port = hostDic[host].getSrvPort() logger.debug("Checking candidate file with ID: %s on host/port: %s/%s. " + \ "Location: %s", fileInfoObj.getFileId(), host, str(port), location) # If the file is stored locally we check if it is accessible, # otherwise we send a STATUS/file_access request to the # host in question. if (location == NGAMS_HOST_LOCAL): # Check first if the local system supports retrieve requests. # (if relevant). if (reqPropsObj): if (reqPropsObj.getCmd() == NGAMS_RETRIEVE_CMD): if (not srvObj.getCfg().getAllowRetrieveReq()): continue # Check if the file is accessible. filename = os.path.normpath(diskInfoObj.getMountPoint()+"/" +\ fileInfoObj.getFilename()) logger.debug("Checking if local file with name: %s is available", filename) if (not os.path.exists(filename)): logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host])) else: logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host])) foundFile = 1 break else: logger.debug("Checking if file with ID/Version: %s/%s " +\ "is available on host/port: %s/%s", fileInfoObj.getFileId(), str(fileInfoObj.getFileVersion()), host, str(port)) # If a server hosting a file is suspended, it is woken up # to be able to check if the file is really accessible. if (hostDic[host].getSrvSuspended() == 1): logger.debug("Server hosting requested file (%s/%s) is suspended " + \ "- waking up server ...", host, str(port)) try: ngamsSrvUtils.wakeUpHost(srvObj, host) logger.debug("Suspended server hosting requested file (%s/%s) " +\ "has been woken up", host, str(port)) except Exception: logger.exception("Error waking up server hosting selected " +\ "file") continue # The file is hosted on a host, which is not suspended or # which was successfully woken up. pars = [["file_access", fileInfoObj.getFileId()]] if (fileInfoObj.getFileVersion() != -1): pars.append(["file_version", fileInfoObj.getFileVersion()]) ipAddress = hostDic[host].getIpAddress() authHdr = ngamsSrvUtils.genIntAuthHdr(srvObj) resp = ngamsHttpUtils.httpGet(ipAddress, port, NGAMS_STATUS_CMD, pars=pars, auth=authHdr) with contextlib.closing(resp): data = resp.read() statusObj = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1) if logger.isEnabledFor(logging.DEBUG): logger.debug("Result of File Access Query: %s", re.sub("\n", "", str(statusObj.genXml().toprettyxml(' ', '\n')))) if ((statusObj.getMessage().\ find("NGAMS_INFO_FILE_AVAIL") == -1)): logger.debug(genLog("NGAMS_INFO_FILE_NOT_AVAIL", [fileId, host])) else: logger.debug(genLog("NGAMS_INFO_FILE_AVAIL", [fileId, host])) foundFile = 1 break # If no file was found we raise an exception. if (not foundFile): errMsg = genLog("NGAMS_ER_UNAVAIL_FILE", [fileId]) raise Exception(errMsg) # The file was found, get the info necessary for the acquiring the file. ipAddress = hostDic[host].getIpAddress() srcFileInfo = [location, host, ipAddress, port, diskInfoObj.getMountPoint(), fileInfoObj.getFilename(), fileInfoObj.getFileId(), fileInfoObj.getFileVersion(), fileInfoObj.getFormat()] if include_compression: srcFileInfo.append(fileInfoObj.getCompression()) msg = "Located suitable file for request - File ID: %s. " +\ "Info for file found - Location: %s - Host ID/IP: %s/%s - " +\ "Port Number: %s - File Version: %d - Filename: %s - " +\ "Mime-type: %s" logger.debug(msg, fileId, location, host, ipAddress, port, fileInfoObj.getFileVersion(), fileInfoObj.getFilename(), fileInfoObj.getFormat()) return srcFileInfo
def test_archive_status_retrieve_sequence(self): cfg = self.env_aware_cfg() if 'sqlite' not in cfg.getDbInterface(): self.skipTest("This test works only against the sqlite db") host_name = getHostName() sample_file_name = "SmallFile.fits" sample_file_path = self.resource(os.path.join("src", sample_file_name)) sample_file_size = os.path.getsize(sample_file_path) sample_mime_type = "application/octet-stream" # We create two NGAS clusters each containing a single NGAS node # We configure the first NGAS cluster to use the second NGAS cluster # as a partner site partner_host_id = "{0}:9011".format("localhost") config_list_1 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas1"), ("NgamsCfg.Server[1].IpAddress", "0.0.0.0"), ("NgamsCfg.PartnerSites[1].ProxyMode", "1"), ("NgamsCfg.PartnerSites[1].PartnerSite[1].Address", partner_host_id)] self._prepare_partner_site_cluster((9001, config_list_1)) config_list_2 = [("NgamsCfg.Server[1].RootDirectory", "/tmp/ngas2"), ("NgamsCfg.Server[1].IpAddress", "0.0.0.0")] self._prepare_partner_site_cluster((9011, config_list_2)) # We archive a test sample file on the partner site cluster self.archive(9011, sample_file_path, mimeType=sample_mime_type) # We check the status of a file ID found on the partner site cluster command_status = functools.partial(ngamsHttpUtils.httpGet, "localhost", 9001, "STATUS", timeout=60) argument_list = {"file_id": sample_file_name} with contextlib.closing( command_status(pars=argument_list)) as response: self.assertEqual(response.status, 200) # Parse the response XML status information response_data = response.read() status_info = ngamsStatus.ngamsStatus().unpackXmlDoc( response_data, 1) disk_info = status_info.getDiskStatusList()[0] file_info = disk_info.getFileObjList()[0] self.assertEqual(disk_info.getHostId(), "{0}:9011".format(host_name)) self.assertEqual(file_info.getFileId(), sample_file_name) self.assertEqual(file_info.getFileVersion(), 1) self.assertEqual(file_info.getFormat(), sample_mime_type) # Retrieve a file found on the partner site cluster retrieve_file_path = tmp_path(sample_file_name) self.retrieve(9001, sample_file_name, targetFile=retrieve_file_path) retrieve_file_size = os.path.getsize(retrieve_file_path) self.assertEqual(sample_file_size, retrieve_file_size) # Retrieve a file with a 100 bytes offset headers = {"range": "bytes=100-"} self.retrieve(9001, sample_file_name, targetFile=retrieve_file_path, hdrs=headers) retrieve_file_size = os.path.getsize(retrieve_file_path) self.assertEqual(sample_file_size - 100, retrieve_file_size)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Handle STATUS command. srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ status = ngamsStatus.ngamsStatus() status.setDate(toiso8601()).\ setVersion(getNgamsVersion()).setHostId(srvObj.getHostId()).\ setStatus(NGAMS_SUCCESS).\ setMessage("Successfully handled command STATUS").\ setState(srvObj.getState()).setSubState(srvObj.getSubState()) reqPropsObjRef = reqPropsObj # Get the information requested. diskId = "" fileId = "" fileVersion = "" configurationFile = "" fileAccess = "" hostId = "" requestId = "" dbTime = "" dbTimeReset = "" fileList = "" fileListId = "" maxElements = 100000 if (reqPropsObj.hasHttpPar("disk_id")): diskId = reqPropsObj.getHttpPar("disk_id") if (reqPropsObj.hasHttpPar("file_id")): fileId = reqPropsObj.getHttpPar("file_id") if (reqPropsObj.hasHttpPar("file_version")): fileVersion = int(reqPropsObj.getHttpPar("file_version")) if (reqPropsObj.hasHttpPar("configuration_file")): configurationFile = reqPropsObj.getHttpPar("configuration_file") if (reqPropsObj.hasHttpPar("file_access")): fileAccess = reqPropsObj.getHttpPar("file_access") if (reqPropsObj.hasHttpPar("host_id")): hostId = reqPropsObj.getHttpPar("host_id") if (reqPropsObj.hasHttpPar("max_elements")): maxElements = int(reqPropsObj.getHttpPar("max_elements")) if (reqPropsObj.hasHttpPar("request_id")): requestId = reqPropsObj.getHttpPar("request_id") if (reqPropsObj.hasHttpPar("db_time")): dbTime = True if (reqPropsObj.hasHttpPar("db_time_reset")): dbTimeReset = True if (reqPropsObj.hasHttpPar("flush_log")): # in the past this called flushLog() # do we really want to keep that functionality? I doubt it pass if (reqPropsObj.hasHttpPar("file_list")): fileList = int(reqPropsObj.getHttpPar("file_list")) if (reqPropsObj.hasHttpPar("file_list_id")): fileListId = reqPropsObj.getHttpPar("file_list_id") if (reqPropsObj.hasHttpPar("dump_object_info")): # Dump status of all objects allocated into file specified. targFile = reqPropsObj.getHttpPar("dump_object_info") rmFile(targFile) fo = open(targFile, "w") fo.write(_genObjectStatus()) fo.close() # Handle request for file info. genCfgStatus = 0 genDiskStatus = 0 genFileStatus = 0 genStatesStatus = 1 genRequestStatus = 0 msg = "" help = 0 if (reqPropsObj.hasHttpPar("help")): global _help msg = _help help = 1 elif hostId and hostId != srvObj.getHostId(): host, port = srvObj.get_remote_server_endpoint(hostId) cfgObj = srvObj.getCfg() if not cfgObj.getProxyMode(): httpRef.redirect(host, port) return else: try: httpRef.proxy_request(hostId, host, port) except Exception as e: ex = re.sub("<|>", "", str(e)) errMsg = genLog("NGAMS_ER_COM", [host, port,ex]) raise Exception(errMsg) return elif (fileList): if (not fileListId): # It's a new STATUS?file_list request. fileListId = _handleFileList(srvObj, reqPropsObj, httpRef) # Send back data from the request. _handleFileListReply(srvObj, reqPropsObj, httpRef, fileListId, maxElements) elif (diskId): diskObj = ngamsDiskInfo.ngamsDiskInfo() diskObj.read(srvObj.getDb(), diskId) status.addDiskStatus(diskObj) genDiskStatus = 1 elif (fileId): if (not fileVersion): fileVersion = -1 try: fileObj = ngamsFileInfo.ngamsFileInfo() fileObj.read(srvObj.getHostId(), srvObj.getDb(), fileId, fileVersion) diskObj = ngamsDiskInfo.ngamsDiskInfo() try: diskObj.read(srvObj.getDb(), fileObj.getDiskId()) except: errMsg = "Illegal Disk ID found: {0} for file with ID: {1}".format(fileObj.getDiskId(), fileId) raise Exception(errMsg) diskObj.addFileObj(fileObj) status.addDiskStatus(diskObj) except: # The file was not found in the database. Check if it is available # on a remote partner site. host, port, status_info, disk_info, file_info = \ ngamsFileUtils.lookup_partner_site_file_status(srvObj, fileId, fileVersion, reqPropsObj) # Update status reply using the disk status and file status # information retrieved from the partner site status.addDiskStatus(disk_info) genDiskStatus = 1 genFileStatus = 1 elif (requestId): logger.debug("Checking status of request with ID: %s", requestId) reqPropsObjRef = srvObj.getRequest(requestId) if (not reqPropsObjRef): errMsg = genLog("NGAMS_ER_ILL_REQ_ID", [requestId]) raise Exception(errMsg) genRequestStatus = 1 elif (configurationFile): msg = "configuration_file=" + srvObj.cfg_fname genCfgStatus = 1 # Hidden feature to return complete Cfg!! try: genCfgStatus = int(configurationFile) except ValueError: genCfgStatus = 1 status.setNgamsCfgObj(srvObj.getCfg()) elif (fileAccess): if (not fileVersion): fileVersion = -1 fileId = fileAccess msg = _checkFileAccess(srvObj, reqPropsObj, httpRef, fileId, fileVersion, diskId) elif (dbTime): logger.debug("Querying total DB time") msg = "Total DB time: %.6fs" % srvObj.getDb().getDbTime() elif (dbTimeReset): msg = "Resetting DB timer" logger.debug(msg) srvObj.getDb().resetDbTime() else: msg = "Successfully handled command STATUS" if (reqPropsObjRef == reqPropsObj): reqPropsObj.setCompletionTime() srvObj.updateRequestDb(reqPropsObj) if (genCfgStatus or genDiskStatus or genFileStatus or genRequestStatus): status.setReqStatFromReqPropsObj(reqPropsObjRef) # Generate XML reply. logger.debug("Status requests: " + str([genCfgStatus, genDiskStatus, genFileStatus, genStatesStatus])) xmlStat = status.genXmlDoc(genCfgStatus, genDiskStatus, genFileStatus, genStatesStatus) xmlStat = ngamsHighLevelLib.addStatusDocTypeXmlDoc(srvObj, xmlStat) httpRef.send_data(six.b(xmlStat), NGAMS_XML_MT) elif not httpRef.reply_sent: httpRef.send_status(msg) if (msg and (not help)): logger.info(msg) else: logger.info("Successfully handled command STATUS")
def downloadFiles(host, port, srcHost, path, outputDir, list=0): """ Download the files matching the given pattern via the host specified, from the host(s) indicated. The files downloaded will be stored in a directory structure of the form: <Output Dir>/<Source Host>/<File Path> I.e., the filepath of the file is preserved, and the source host name is also kept. host: Name of host to be contacted (string). port: Port number of NG/AMS Server (integer). srcHost: List of hosts to be contacted (<Host 1>,<Host 2>,...). Can also be a single host, wildcards is allowed (string). path: File pattern of file(s) to be retrieved. If this is a directory, the files in the directory will be retrieved (string). outputDir: Output directory. If not given, the current working point will be chosen (string). list: If set to 1 and the path given is a directory, rather than retrieving the file in the remote directory, the names of the remote files are listed on stdout (integer/0|1). Returns: Void. """ if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir) reply, msg, hdrs, data = ngamsLib.httpGet(host, port, NGAMS_RETRIEVE_CMD, pars=[["internal", path], ["host_id", srcHost]], timeOut=10) if (isNgasXmlStatusDoc(data)): stat = ngamsStatus.ngamsStatus().unpackXmlDoc(data, 1) if (stat.getStatus() == NGAMS_FAILURE): msg = "Problem handling request: %s" % stat.getMessage() raise Exception, msg else: stat = None if (data.find("<?xml version=") == -1): if (not list): storeFile(hdrs, data, outputDir) else: print path return # The reply is a file list. Retrieve the file in question. remDir = stat.getFileListList()[0].getComment().split(": ")[-1] remDir = os.path.normpath(remDir) outputDir = os.path.normpath("%s/%s" % (outputDir, remDir.split("/")[-1])) if (not list): commands.getstatusoutput("mkdir -p %s" % outputDir) client = ngamsPClient.ngamsPClient() for fileInfo in stat.getFileListList()[0].getFileInfoObjList(): # If we should only list the contents and if the entry is a file, # we only display its information. if (list): print "%s %-8s %-8s %-10s %-16s %s" %\ (fileInfo.getPermissions(), fileInfo.getOwner(), fileInfo.getGroup(), str(fileInfo.getFileSize()), fileInfo.getModDate(), fileInfo.getFilename()) # If the entry is a file and we should only list, we go to the # next item. if (fileInfo.getPermissions()[0] != "d"): continue # Handle/download the item. fn = fileInfo.getFilename() basename = os.path.basename(fn) trgFile = os.path.normpath("%s/%s" % (outputDir, basename)) if (not list): info(1, "Requesting: %s:%s ..." % (srcHost, fn)) reply2, msg2, hdrs2, data2 = \ ngamsLib.httpGet(host, port, NGAMS_RETRIEVE_CMD, pars=[["internal", fn], ["host_id", srcHost]], timeOut=10) # If the remote object specified is a file, retrieve it, otherwise if a # directory, call this function recursively to retrieve the contents of # the sub-folder(s). if (isNgasXmlStatusDoc(data2)): stat2 = ngamsStatus.ngamsStatus().unpackXmlDoc(data2, 1) if (stat2.getStatus() == NGAMS_FAILURE): msg = "Problem handling request: %s" % stat2.getMessage() raise Exception, msg # It seems to be another File List, we retrieve the contents # of this recursively. if (not list): info(1, "Remote object found: %s" % fn) downloadFiles(host, port, srcHost, fn, outputDir, list) else: # Just retrieve the file. storeFile(hdrs2, data2, outputDir) return