def bbcp_transfer(request, out_fname, crc_name, skip_crc): # source_file must have the IP of the client, so bbcp can contact it and # launch a copy ot itself. # OTOH, target_file must have the IP through which we received the original # HTTP request to ensure the same network path is used by bbcp bparam = get_params(request) source_file = get_source_file(request) target_file = request.getHttpHdr('host') + ':' + out_fname # Make target file writable if existing. if (os.path.exists(out_fname)): os.chmod(out_fname, 420) checkCreatePath(os.path.dirname(out_fname)) # perform the bbcp transfer, we will always return the checksum start = time.time() checksum = bbcpFile(source_file, target_file, bparam, crc_name, skip_crc) size = getFileSize(out_fname) totaltime = time.time() - start # Feedback the file size into the request object now that we know it, # just in case we need it later request.setSize(size) # Artificially split the total time between read and write (+ 0 crc), # we don't actually know how much was spent on what half = totaltime / 2 return ngamsArchiveUtils.archiving_results(size, half, half, 0, totaltime, crc_name, checksum)
def finfo_from_database(fileInfo, srvObj, reqPropsObj): fileId = fileInfo.getFileId() fileVer = fileInfo.getFileVersion() # Locate the file best suiting the query and send it back if possible. location, _, ipAddress, port, mountPoint, filename, fileId,\ fileVersion, mimeType =\ ngamsFileUtils.locateArchiveFile(srvObj, fileId, fileVersion=fileVer, reqPropsObj=reqPropsObj) basename = os.path.basename(filename) if location == NGAMS_HOST_LOCAL: absname = os.path.normpath(os.path.join(mountPoint, filename)) size = getFileSize(absname) opener = fopener(absname) elif location == NGAMS_HOST_CLUSTER or location == NGAMS_HOST_REMOTE: size = srvObj.getDb().getFileSize(fileId, fileVersion) opener = http_opener(ipAddress, port, fileId, fileVersion, srvObj) else: raise Exception("Unknown location type: %s" % (location, )) return ngamsMIMEMultipart.file_info(mimeType, basename, size, opener)
def ngamsGLEAM_Fix_Phase2_JobPI(srvObj, plugInPars, filename, fileId, fileVersion, diskId): """ srvObj: Reference to NG/AMS Server Object (ngamsServer). plugInPars: Parameters to take into account for the plug-in execution (string).(e.g. scale_factor=4,threshold=1E-5) fileId: File ID for file to test (string). filename: Filename of (complete) (string). fileVersion: Version of file to test (integer). Returns: the return code of the compression plugin (integer). """ date_obs = pyfits.getval(filename, 'DATE-OBS').split('T')[0] if (not dict_dec.has_key(date_obs)): return (1, 'no date for %s' % filename) # copy it to some tmp directory fndir = os.path.dirname(filename) bname = os.path.basename(filename) execCmd('cp %s %s/' % (filename, work_dir)) # add header keyword fn = '%s/%s' % (work_dir, bname) os.chmod(fn, 644) # owner writable hdulist = pyfits.open(fn, mode='update') prihdr = hdulist[0].header prihdr['DEC_PNT'] = dict_dec[date_obs] hdulist.close() # double check if the key is really added or the file is not corrupted if 'DEC_PNT' not in pyfits.getheader(filename): # header is not added # remove the temp file os.remove(fn) # raise error raise Exception('%s still does not have key DEC_PNT' % (fn)) # calculate the checksum, new filesize csum = getFileCRC(fn) fsize = getFileSize(fn) # move the original file to a different name os.chmod(filename, 644) execCmd('mv %s %s/%s_origin' % (filename, fndir, bname)) # move the new file back under the original name execCmd('mv %s %s/' % (fn, fndir)) os.chmod(filename, 444) # make it readonly # if all fine, remove the original file (under the different name) # otherwise, remove the new file, and move the original file back to the original name os.remove('%s/%s_origin' % (fndir, bname))
def _checkFile(srvObj, fileInfoObj, stagFile, httpHeaders, checkChecksum): """ Make a consistency check of the Staging File. srvObj: Reference to instance of Server Object (ngamsServer). fileInfoObj: File info object with info about the file (ngamsFileInfo). stagFile: Staging filename (string). httpHeaders: HTTP headers (mimetools.Message) checkChecksum: Carry out checksum check (0|1/integer). Returns: Void. """ # First ensure to flush file caches. ngamsFileUtils.syncCachesCheckFiles(srvObj, [stagFile]) # Check file size. fileSize = getFileSize(stagFile) if (fileSize != fileInfoObj.getFileSize()): # If the mime-type is 'text/xml' we check if the returned # document is an NG/AMS XML Status reporting a problem. tmpStat = None if (httpHeaders.type == NGAMS_XML_MT): try: tmpStat = ngamsStatus.ngamsStatus().load(stagFile,getStatus=1) if (tmpStat.getStatus() != NGAMS_FAILURE): del tmpStat tmpStat = None except: # Was apparently not an NG/AMS Status Document. pass if (tmpStat): # An error response was received from the source node. raise Exception(tmpStat.getMessage()) else: # The file seems not to have been transferred completely. errMsg = "Size of cloned file wrong (expected: %d/actual: %d)" raise Exception(errMsg % (fileInfoObj.getFileSize(), fileSize)) logger.debug("Size of cloned Staging File OK: %s", stagFile) # The file size was correct. if (checkChecksum): if ngamsFileUtils.check_checksum(srvObj, fileInfoObj, stagFile): logger.debug("Checksum of cloned Staging File OK: %s", stagFile) else: logger.debug("No Checksum or Checksum Plug-In specified for file")
def _assertEqualsDir(self, dir1, dir2): # Entries in dir are the same files1 = os.listdir(dir1) files2 = os.listdir(dir2) files1.sort() files2.sort() self.assertListEqual(files1, files2) # Add dir names so we can treat them as proper paths addDir = lambda d, f: d + os.sep + f files1 = list(map(addDir, [dir1 for _ in range(len(files1))], files1)) files2 = list(map(addDir, [dir2 for _ in range(len(files2))], files2)) # Files actually have the same size, we recurse on dirs for idx, f in [(idx, f) for idx, f in enumerate(files1)]: if os.path.isfile(f): self.assertEqual(getFileSize(f), getFileSize(files2[idx])) elif os.path.isdir(f): self._assertEqualsDir(f, files2[idx])
def __retrieveThread(optDic, taskCtrl, dummy): """ """ info(4, "Entering retrieveThread(%s) ..." % getThreadName()) client = ngamsPClient.ngamsPClient().\ parseSrvList(optDic["servers"][NGAS_OPT_VAL]) while (1): nextFileId = taskCtrl.getNextFileId() if (not nextFileId): thread.exit() nextFileId = nextFileId[:-1] stat = client.retrieve2File(nextFileId, targetFile=nextFileId) fileSize = getFileSize(nextFileId) taskCtrl.incBytesRecv(fileSize) info(1, "Next File ID: %s" % nextFileId)
def performStaging(srvObj, reqPropsObj, httpRef, filename): """ if the staging plugin is set, then perform staging using the registered staging plugin if the file is offline (i.e. on Tape) srvObj: Reference to NG/AMS server class object (ngamsServer). filename: File to be processed (string). """ if srvObj.getCfg().getFileStagingEnable() != 1: return fspi = srvObj.getCfg().getFileStagingPlugIn() if not fspi: return logger.info("Invoking FSPI.isFileOffline: %s to check file: %s", fspi, filename) isFileOffline = loadPlugInEntryPoint(fspi, 'isFileOffline') if isFileOffline(filename) == 0: return logger.info("Invoking FSPI.stageFiles: %s to check file: %s", fspi, filename) stageFiles = loadPlugInEntryPoint(fspi, 'stageFiles') try: st = time.time() stageFiles(filenames=[filename], requestObj=reqPropsObj, serverObj=srvObj) howlong = time.time() - st fileSize = getFileSize(filename) logger.debug('Staging rate = %.0f Bytes/s (%.0f seconds) for file %s', fileSize / howlong, howlong, filename) except socket.timeout: errMsg = 'Staging timed out: %s' % filename logger.warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=504) raise
def bbcp_transfer(request, out_fname, crc_name, skip_crc): bparam = get_params(request) # perform the bbcp transfer, we will always return the checksum start = time.time() checksum = bbcpFile(request.getFileUri(), out_fname, bparam, crc_name, skip_crc) size = getFileSize(out_fname) totaltime = time.time() - start # Feedback the file size into the request object now that we know it, # just in case we need it later request.setSize(size) # Artificially split the total time between read and write (+ 0 crc), # we don't actually know how much was spent on what half = totaltime / 2 return ngamsArchiveUtils.archiving_results(size, half, half, 0, totaltime, crc_name, checksum)
def createFitsFiles(testParDic): """ Create the test 'archived' files according to the specified parameters. This function handles FITS files. Note: - The files is archived as it is. I.e., if it is not compressed, it remains like that. - Even if rearchiving the same file with new File IDs, no parameters of the header are updated. - It is assumed that the file has a valid FITS checksum. - It is assumed that the file has a valid ARCFILE. - Only new versions are created. I.e., the File ID is incremented at each iteration. testParDic: Dictionary with parameters for running the test (dictionary). Returns: Void. """ from ngamsPlugIns import ngamsFitsPlugIn filesPerDay = 10000 # Max. number of files to generate per day. baseFile = testParDic["BASEFILE"] ext = baseFile.split(".")[-1] if ((ext == "Z") or (ext == "gz")): try: tmpFile1 = "/tmp/ngasCreateVolume_%f" % time.time() tmpFile2 = "%s.%s" % (tmpFile1, ext) os.system("cp %s %s" % (baseFile, tmpFile2)) os.system("gunzip %s" % tmpFile2) arcFile, dpId, dateDir = ngamsFitsPlugIn.getDpIdInfo(tmpFile1) uncomprSize = getFileSize(tmpFile1) os.system("rm -f %s*" % tmpFile1) except Exception, e: os.system("rm -f %s*" % tmpFile1) raise e
def finfo_from_database(fileInfo, srvObj, reqPropsObj): fileId = fileInfo.getFileId() fileVer = fileInfo.getFileVersion() # Locate the file best suiting the query and send it back if possible. location, _, ipAddress, port, mountPoint, filename, fileVersion, mimeType = \ ngamsFileUtils.quickFileLocate(srvObj, reqPropsObj, fileId, fileVersion=fileVer) basename = os.path.basename(filename) if location == NGAMS_HOST_LOCAL: absname = os.path.normpath(os.path.join(mountPoint, filename)) size = getFileSize(absname) opener = fopener(absname) elif location == NGAMS_HOST_CLUSTER or location == NGAMS_HOST_REMOTE: # TODO: int in python2 guarantees at least 32 bits, so we may overflow here size = int(srvObj.getDb().getFileSize(fileId, fileVersion)) opener = http_opener(ipAddress, port, fileId, fileVersion, srvObj) else: raise Exception("Unknown location type: %s" % (location,)) return ngamsMIMEMultipart.file_info(mimeType, basename, size, opener)
1] logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) # remove the temp compressed file cmd2 = 'rm %s' % newfn execCmd(cmd2) # move the uncompressed file back cmd2 = 'mv %s/%s_uncompressed %s' % (fndir, bname, filename) ret = execCmd(cmd2) if (ret[0] != 0): logger.error('Fail to recover from a failed compression: %s', cmd2) return # change the CRC in the database new_fs = getFileSize(filename) query = "UPDATE ngas_files SET checksum = {0}, file_size = {1}, compression = 'RICE', format = 'application/fits' WHERE file_id = {2} AND disk_id = {3} AND file_version = {4}" logger.debug("Updating CRC SQL: %s", str(query)) if (debug): pass else: try: srvObj.getDb().query2(query, args=(crc, new_fs, fileId, diskId, fileVersion)) except Exception, ex: errMsg = 'Fail to update crc for file %s/%d/%s: %s' % ( fileId, fileVersion, diskId, str(ex)) logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) # remove the compressed file that have been copied in
def handleCmd(srvObj, reqPropsObj, httpRef): """ Compress the file based on file_path (file_id, disk_id, and file_version) srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ required_params = ['file_path', 'file_id', 'file_version', 'disk_id'] parDic = reqPropsObj.getHttpParsDic() for rp in required_params: if (not parDic.has_key(rp)): errMsg = 'Parameter missing: %s' % rp httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) return filename = reqPropsObj.getHttpPar('file_path') logger.debug('Compressing file %s', filename) if (not os.path.exists(filename)): errMsg = 'File Not found: %s' % filename logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=404) return fileId = parDic['file_id'] fileVersion = int(parDic['file_version']) diskId = parDic['disk_id'] if (not isMWAVisFile(fileId)): errMsg = 'Not MWA visibilty file: %' % (fileId) logger.warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) return if (hasCompressed(filename)): errMsg = 'OK' logger.debug('File compressed %s already, return now', filename) httpRef.send_data(errMsg, NGAMS_TEXT_MT) return if (parDic.has_key('scale_factor')): sf = int(parDic['scale_factor']) else: sf = 4 if (parDic.has_key('threshold')): th = float(parDic['threshold']) # currently not used if (parDic.has_key('bins')): bins = int(parDic['bins']) else: bins = 0 if (parDic.has_key('remove_uc')): remove_uc = int(parDic['remove_uc']) # currently not used if (parDic.has_key('timeout')): timeout = int(parDic['timeout']) if (timeout <= 0): timeout = 600 if (parDic.has_key('debug')): debug = int(parDic['debug']) else: debug = 0 if (bins): binstr = '-h %d' % bins else: binstr = '' bname = os.path.basename(filename) newfn = '%s/%s' % (work_dir, bname) fndir = os.path.dirname(filename) old_fs = getFileSize(filename) # do compression cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn) logger.debug('Compression: %s', cmd) if (debug): pass else: ret = execCmd(cmd) if (ret[0] != 0): errMsg = 'Failed to compress %s' % ret[1] logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) return # calculate the CRC code logger.debug('Calculating CRC for file %s', newfn) if (debug): crc = 1234567 else: try: crc = getFileCRC(newfn) except Exception as exp: errMsg = 'Failed to calculate the CRC for file %s: %s' % (newfn, str(exp)) logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) # remove the temp compressed file cmd1 = 'rm %s' % newfn execCmd(cmd1) return # rename the uncompressed file (under the same directory) cmd1 = 'mv %s %s/%s_uncompressed' % (filename, fndir, bname) logger.debug('Renaming the uncompressed file: %s', cmd1) if (debug): pass else: ret = execCmd(cmd1) if (ret[0] != 0): errMsg = 'Failed to move/rename uncompressed file %s: %s' % (filename, ret[1]) logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) # remove the temp compressed file cmd1 = 'rm %s' % newfn execCmd(cmd1) return # move the compressed file to replace the uncompressed file cmd2 = 'mv %s %s' % (newfn, filename) logger.debug('Moving the compressed file to NGAS volume: %s', cmd2) if (debug): pass else: ret = execCmd(cmd2) if (ret[0] != 0): errMsg = 'Failed to move the compressed file to NGAS volume: %s' % ret[1] logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) # remove the temp compressed file cmd2 = 'rm %s' % newfn execCmd(cmd2) # move the uncompressed file back cmd2 = 'mv %s/%s_uncompressed %s' % (fndir, bname, filename) ret = execCmd(cmd2) if (ret[0] != 0): logger.error('Fail to recover from a failed compression: %s', cmd2) return # change the CRC in the database new_fs = getFileSize(filename) query = "UPDATE ngas_files SET checksum = {0}, file_size = {1}, compression = 'RICE', format = 'application/fits' WHERE file_id = {2} AND disk_id = {3} AND file_version = {4}" logger.debug("Updating CRC SQL: %s", str(query)) if (debug): pass else: try: srvObj.getDb().query2(query, args=(crc, new_fs, fileId, diskId, fileVersion)) except Exception as ex: errMsg = 'Fail to update crc for file %s/%d/%s: %s' % (fileId, fileVersion, diskId, str(ex)) logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) # remove the compressed file that have been copied in cmd2 = 'rm %s' % filename execCmd(cmd2) # recover the uncompressed back cmd2 = 'mv %s/%s_uncompressed %s' % (fndir, bname, filename) ret = execCmd(cmd2) if (ret[0] != 0): logger.error('Fail to recover from a failed compression: %s', cmd2) return # remove the uncompressed file errMsg = 'Compression completed %s.' % (filename) if (not debug): #new_fs = getFileSize(filename) cmd3 = 'rm %s/%s_uncompressed' % (fndir, bname) ret = execCmd(cmd3) if (ret[0] != 0): errMsg += '. But fail to remove the uncompressed file %s/%s: %s' % (fndir, bname, ret[1]) logger.warning(errMsg) logger.debug(errMsg + '\. Result: %d - %d = %d, compress ratio: %.2f' % (old_fs, new_fs, (old_fs - new_fs), new_fs * 1.0 / old_fs)) else: logger.debug(errMsg) httpRef.send_data(errMsg + '\n', NGAMS_TEXT_MT)
def checkCorrectFileSizeDb(diskId, correct=0, notifEmail=None): """ Check the size of the files registered on the given disk. If a file has a size different from the one stored in the DB, the size is corrected in the DB diskId: ID of disk for which to check the files (string). Returns: Void """ server, db, user, password = ngasUtilsLib.getDbPars() db = ngamsDb.ngamsDb(server, db, user, password, createSnapshot=0) tmpDiskInfo = ngamsDiskInfo.ngamsDiskInfo().read(getHostName(), db, diskId) if (tmpDiskInfo.getHostId() != getHostName()): raise Exception, "The disk: %s is not inserted in this system!!" %\ diskId notifMsg = "\nFILE SIZE CHECK/CORRECTION REPORT:\n" +\ "(only files with an incorrect size in the DB are reported)\n"+\ "\nDisk ID: " + diskId + "\n" +\ 128 * "=" sys.stdout.write(notifMsg) dbCursor = db.getFileSummary1(getHostName(), [diskId], fileStatus=[]) while (1): res = dbCursor.fetch(128) if (not res): break for fileInfo in res: # IMPL: This works only on calibration frames for the moment. # (these are uncompressed -> file size == uncompr file size) filename = fileInfo[2] if (filename.find("/M.") == -1): continue mountPoint = fileInfo[1] dbChecksum = fileInfo[3] fileId = fileInfo[5] fileVer = fileInfo[6] dbFileSize = int(fileInfo[7]) diskId = fileInfo[9] complFilename = os.path.normpath(mountPoint + "/" + filename) diskFileSize = getFileSize(complFilename) msg = "\nCheck size of file: %s/%s/%d: " % (diskId, fileId, fileVer) if (diskFileSize == dbFileSize): msg += "Size OK" else: msg += "SIZE WRONG!" # Check checksum. from ngamsPlugIns import ngamsGenCrc32 checksum = ngamsGenCrc32.ngamsGenCrc32(None, complFilename, 0) if (checksum != dbChecksum): msg += " ILLEGAL CHECKSUM!" else: msg += " CHECKSUM OK!" if (correct): tmpFileInfo = ngamsFileInfo.\ ngamsFileInfo().read(db, fileId, fileVer, diskId) tmpFileInfo.setFileSize(diskFileSize) tmpFileInfo.setUncompressedFileSize(diskFileSize) tmpFileInfo.setFileStatus(NGAMS_FILE_STATUS_OK) if (not tmpFileInfo.getCreationDate()): tmpFileInfo.setCreationDate(tmpFileInfo.\ getIngestionDate()) tmpFileInfo.write(getHostName(), db, 0, 1) msg += " FILE SIZE CORRECTED!" notifMsg += msg sys.stdout.write(msg) sys.stdout.flush() notifMsg += "\n%s\n" % (128 * "=") print "\n" + 128 * "=" if (notifEmail): ngasUtilsLib.sendEmail("FILE SIZE CHECK/CORRECTION REPORT", notifEmail, notifMsg, "text/plain", "FILE-SIZE-CHECK-REP-%s" % diskId)
def _httpPostUrl(url, mimeType, contDisp = "", dataRef = "", dataSource = "BUFFER", dataTargFile = "", blockSize = 65536, suspTime = 0.0, timeOut = None, authHdrVal = "", dataSize = -1, session_uuid = ""): """ Post the the data referenced on the given URL. This function is adapted from ngamsLib.httpPostUrl, which does not support block-level suspension and cancelling for file transfer The data send back from the remote server + the HTTP header information is return in a list with the following contents: [<HTTP status code>, <HTTP status msg>, <HTTP headers (list)>, <data>] url: URL to where data is posted (string). mimeType: Mime-type of message (string). contDisp: Content-Disposition of the data (string). dataRef: Data to post or name of file containing data to send (string). dataSource: Source where to pick up the data (string/BUFFER|FILE|FD). dataTargFile: If a filename is specified with this parameter, the data received is stored into a file of that name (string). blockSize: Block size (in bytes) used when sending the data (integer). suspTime: Time in seconds to suspend between each block (double). timeOut: Timeout in seconds to wait for replies from the server (double). authHdrVal: Authorization HTTP header value as it should be sent in the query (string). dataSize: Size of data to send if read from a socket (integer). Returns: List with information from reply from contacted NG/AMS Server (reply, msg, hdrs, data) (list). """ T = TRACE() # Separate the URL from the command. idx = (url[7:].find("/") + 7) tmpUrl = url[7:idx] cmd = url[(idx + 1):] http = httplib.HTTP(tmpUrl) logger.debug("HTTP Header: %s: %s", NGAMS_HTTP_POST, cmd) http.putrequest(NGAMS_HTTP_POST, cmd) logger.debug("HTTP Header: Content-Type: %s", mimeType) http.putheader("Content-Type", mimeType) if (contDisp != ""): logger.debug("HTTP Header: Content-Disposition: %s", contDisp) http.putheader("Content-Disposition", contDisp) if (authHdrVal): if (authHdrVal[-1] == "\n"): authHdrVal = authHdrVal[:-1] logger.debug("HTTP Header: Authorization: %s", authHdrVal) http.putheader("Authorization", authHdrVal) if (dataSource == "FILE"): dataSize = getFileSize(dataRef) elif (dataSource == "BUFFER"): dataSize = len(dataRef) if (dataSize != -1): logger.debug("HTTP Header: Content-Length: %s", str(dataSize)) http.putheader("Content-Length", str(dataSize)) logger.debug("HTTP Header: Host: %s", getHostName()) http.putheader("Host", getHostName()) http.endheaders() logger.debug("HTTP header sent") http._conn.sock.settimeout(timeOut) # Send the data. logger.debug("Sending data ...") if (dataSource == "FILE"): fdIn = open(dataRef) block = "-" blockAccu = 0 http._conn.sock.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 87380) # to fit Fornax while (block != ""): """ if (threadRunDic.has_key(session_uuid) and threadRunDic[session_uuid] == 0): info(3, "Received cancel/suspend request, discard remaining blocks") break """ block = fdIn.read(blockSize) blockAccu += len(block) http._conn.sock.sendall(block) if (suspTime > 0.0): time.sleep(suspTime) fdIn.close() elif (dataSource == "FD"): fdIn = dataRef dataRead = 0 while (dataRead < dataSize): if ((dataSize - dataRead) < blockSize): rdSize = (dataSize - dataRead) else: rdSize = blockSize block = fdIn.read(rdSize) http._conn.sock.sendall(block) dataRead += len(block) if (suspTime > 0.0): time.sleep(suspTime) else: # dataSource == "BUFFER" http.send(dataRef) logger.debug("Data sent") if (threadRunDic.has_key(session_uuid) and threadRunDic[session_uuid] == 0): logger.debug("Received cancel/suspend request, close HTTP connection and return None values") if (http != None): http.close() del http return [None, None, None, None] # Receive + unpack reply. logger.debug("Waiting for reply ...") reply, msg, hdrs = http.getreply() if (hdrs == None): errMsg = "Illegal/no response to HTTP request encountered!" raise Exception(errMsg) if (hdrs.has_key("content-length")): dataSize = int(hdrs["content-length"]) else: dataSize = 0 if (dataTargFile == ""): data = http.getfile().read(dataSize) else: fd = None try: data = dataTargFile fd = open(dataTargFile, "w") fd.write(http.getfile().read(dataSize)) fd.close() except Exception as e: if (fd != None): fd.close() raise e # Dump HTTP headers if Verbose Level >= 4. logger.debug("HTTP Header: HTTP/1.0 " + str(reply) + " " + msg) if logger.isEnabledFor(logging.DEBUG): for hdr in hdrs.keys(): logger.debug("HTTP Header: %s: %s", hdr, hdrs[hdr]) if (http != None): http.close() del http return [reply, msg, hdrs, data]
def checkFile(srvObj, sum1FileInfo, checkReport, skipCheckSum = 0, executor=None): """ Function to carry out a consistency check on a file. If `stop_evt` and `allowed_evt` are given, then `get_checksum_interruptible` is used internally by this method; otherwise `get_checksum` is used. If `executor` is given, then it is used to carry out the execution of the checksum calculation; otherwise `get_checksum` is used. """ executor = executor or get_checksum foundProblem = 0 fileInfo = sum1FileInfo diskId = fileInfo[ngamsDbCore.SUM1_DISK_ID] slotId = fileInfo[ngamsDbCore.SUM1_SLOT_ID] filename = os.path.\ normpath(fileInfo[ngamsDbCore.SUM1_MT_PT] + "/" +\ fileInfo[ngamsDbCore.SUM1_FILENAME]) if (fileInfo[ngamsDbCore.SUM1_CHECKSUM] == None): checksumDb = "" else: checksumDb = fileInfo[ngamsDbCore.SUM1_CHECKSUM].strip() crc_variant = fileInfo[ngamsDbCore.SUM1_CHECKSUM_PI] fileId = fileInfo[ngamsDbCore.SUM1_FILE_ID] fileVersion = fileInfo[ngamsDbCore.SUM1_VERSION] dbFileSize = fileInfo[ngamsDbCore.SUM1_FILE_SIZE] logger.debug("Checking file with ID: %s and filename: %s on NGAS host: %s", fileId, filename, srvObj.getHostId()) ## Set the file status "checking bit" temporary to 1. #tmpFileStatus = fileStatus[0] + "1" + fileStatus[2:] #srvObj.getDb().setFileStatus(fileId, fileVersion, diskId, tmpFileStatus) # Create name of temporary file in "<NGAS Rt Pt>/cache" indicating which # file is being checked: # <NGAS Rt Pt>/cache/<Thr ID>___<Disk ID>___<File ID>___<File Version>.\ # check fileChecked = os.path.normpath("%s|%s___%s___%s___%s.check" % ( NGAMS_CACHE_DIR, NGAMS_DATA_CHECK_THR, diskId, fileId, str(fileVersion))).replace("/", "_") fileChecked = srvObj.getCfg().getRootDirectory() + '/' + fileChecked.replace("|","/") fileCheckedFo = None try: # Create file indicating which data file is being checked. rmFile(fileChecked) fileCheckedFo = open(fileChecked, "w") fileCheckedFo.close() # Check if file exists. fileExists = os.path.exists(filename) if (not fileExists): foundProblem = 1 logger.error('File %s does not exist on disk', filename) checkReport.append(["ERROR: File in DB missing on disk", fileId, fileVersion, slotId, diskId, filename]) srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, False) # Check if file has the correct size. if ((not foundProblem) and fileExists): fileSize = getFileSize(filename) if (fileSize != dbFileSize): foundProblem = 1 logger.error('File %s has wrong size. Expected: %d/Actual: %d', filename, dbFileSize, fileSize) format = "ERROR: File has wrong size. Expected: %d/Actual: %d." checkReport.append([format % (dbFileSize , fileSize), fileId, fileVersion, slotId, diskId, filename]) srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, False) # Check checksum if this test should not be skipped. if ((not foundProblem) and (not skipCheckSum)): if crc_variant is None: crc_variant = srvObj.getCfg().getCRCVariant() checksum_info = get_checksum_info(crc_variant) if crc_variant is not None: try: blockSize = srvObj.getCfg().getBlockSize() if blockSize == -1: blockSize = 4096 checksum_typ = get_checksum_name(crc_variant) # Calculate the checksum, possibly under the executor start = time.time() checksumFile = executor(blockSize, filename, crc_variant) duration = time.time() - start fsize_mb = getFileSize(filename) / 1024. / 1024. logger.info("Checked %s in %.4f [s] using %s. Check ran at %.3f [MB/s]. Checksum file/db: %s / %s", filename, duration, str(crc_variant), fsize_mb / duration, str(checksumFile), checksumDb) except Exception: # We assume an IO error: # "[Errno 2] No such file or directory" # This problem has already been registered further # up so we ignore it here. rmFile(fileChecked) logger.exception("Error while checking file %s", filename) return else: checksumFile = "" if not checksumDb and checksumFile: checkReport.append(["NOTICE: Missing checksum - generated", fileId, fileVersion, slotId, diskId, filename]) srvObj.getDb().setFileChecksum(srvObj.getHostId(), fileId, fileVersion, diskId, str(checksumFile), checksum_typ) elif not checksumDb and not checksumFile: checkReport.append(["NOTICE: Missing checksum - " +\ "cannot generate - update DB", fileId, fileVersion, slotId, diskId, filename]) elif not checksum_info.equals(checksumDb, checksumFile): logger.error("File %s has inconsistent checksum! file/db: %s / %s", filename, str(checksumFile), checksumDb) checkReport.append(["ERROR: Inconsistent checksum found", fileId, fileVersion, slotId, diskId, filename]) srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, False) # If file is OK but is marked as 'bad', reset the flag. if not foundProblem: srvObj.db.set_valid_checksum(fileId, fileVersion, diskId, True) ## Set the checking flag back to 0 in the DB. #srvObj.getDb().setFileStatus(fileId, fileVersion, diskId, fileStatus) # Reset file indicating which data file is being checked. rmFile(fileChecked) except Exception: if (fileCheckedFo): fileCheckedFo.close() # Reset file indicating which data file is being checked. rmFile(fileChecked) raise
def _filesSize(self): return sum([getFileSize(f) for f in self.myfiles])
ext = baseFile.split(".")[-1] if ((ext == "Z") or (ext == "gz")): try: tmpFile1 = "/tmp/ngasCreateVolume_%f" % time.time() tmpFile2 = "%s.%s" % (tmpFile1, ext) os.system("cp %s %s" % (baseFile, tmpFile2)) os.system("gunzip %s" % tmpFile2) arcFile, dpId, dateDir = ngamsFitsPlugIn.getDpIdInfo(tmpFile1) uncomprSize = getFileSize(tmpFile1) os.system("rm -f %s*" % tmpFile1) except Exception, e: os.system("rm -f %s*" % tmpFile1) raise e else: arcFile, dpId, dateDir = ngamsFitsPlugIn.getDpIdInfo(baseFile) uncomprSize = getFileSize(baseFile) insId = arcFile.split(".")[0] isoTime = arcFile[(arcFile.find(".") + 1):] baseTimeMjd = PccUtTime.TimeStamp(isoTime).getMjd() # Generate 'static' file info for this run. ngasDiskInfo = os.path.normpath("%s/NgasDiskInfo" %\ testParDic["MOUNTPOINT"]) ngasDiskInfoDoc = open(ngasDiskInfo).read() diskInfoObj = ngamsDiskInfo.ngamsDiskInfo().\ unpackXmlDoc(ngasDiskInfoDoc, ignoreVarDiskPars=1) diskId = diskInfoObj.getDiskId() fileVer = 1 if (ext == "Z"): format = "application/x-cfits" compression = "compress -f"
def test_AppendRemove(self): # Server and client self.prepExtSrv() # Create a container, shouldn't be a problem containerName = "testing" self.ccreate(containerName) self._checkContainerClosed(containerName, False) #------------------------------------------------------------------ # We start testing with a single file append/removal, # which is a simple use case #------------------------------------------------------------------ # Archive a single file, append it to the container # and check the file is there and it contributes to the container size myfile = self.myfiles[0] myfileId = os.path.basename(myfile) myfileSize = getFileSize(myfile) self.qarchive(myfile, "application/octet-stream") self.cappend(myfileId, containerName=containerName) self._checkFilesAndContainerSize(containerName, 1, myfileSize) self._checkContainerClosed(containerName, False) # Remove the file now and check that the file is not there anymore, # decreasing the container size self.cremove(myfileId, containerName=containerName) self._checkFilesAndContainerSize(containerName, 0, 0) self._checkContainerClosed(containerName, False) #------------------------------------------------------------------ # We continue testing with a list of files for append/removal, # which is a bit more complex #------------------------------------------------------------------ # Store the rest of the files # Append all files to the container (including the first one, we removed it) # and check that files are there, and that they contribute to the container size fileIds = ':'.join([os.path.basename(f) for f in self.myfiles]) allFilesSize = self._filesSize() for myfile in self.myfiles[1:]: self.qarchive(myfile, "application/octet-stream") self.cappend(None, fileIdList=fileIds, containerName=containerName) self._checkFilesAndContainerSize(containerName, len(self.myfiles), allFilesSize) self._checkContainerClosed(containerName, False) # Remove the files now and check that the files are not part of the container anymore, # decreasing the container size self.cremove(None, fileIdList=fileIds, containerName=containerName) self._checkFilesAndContainerSize(containerName, 0, 0) self._checkContainerClosed(containerName, False) #------------------------------------------------------------------ # Now we re-archive the same files, effectively creating a second # version of them. We then check which version is considered as # part of the container. # # This test comes in two flavors: # - Add the files to the container, re-archive them # - Re-archive the files, then add them to the container #------------------------------------------------------------------ # Add the files to the container and then re-archive them # The new version of the files should be 2, and the new container size # should correspond to the new files' sizes self._createFiles() allFilesSize = self._filesSize() self.cappend(None, fileIdList=fileIds, containerName=containerName) for myfile in self.myfiles: self.qarchive(myfile, "application/octet-stream") self._checkFilesAndContainerSize(containerName, len(self.myfiles), allFilesSize, fileVersion=2) self._checkContainerClosed(containerName, False) # Remove all files and check that the container is empty self.cremove(None, fileIdList=fileIds, containerName=containerName) self._checkFilesAndContainerSize(containerName, 0, 0) self._checkContainerClosed(containerName, False) # Re-archive the files, add them to the container and check again # The new version of the files should be 3, and the new container size # should correspond to the new files' sizes self._createFiles() allFilesSize = self._filesSize() for myfile in self.myfiles: self.qarchive(myfile, "application/octet-stream") self.cappend(None, fileIdList=fileIds, containerName=containerName) self._checkFilesAndContainerSize(containerName, len(self.myfiles), allFilesSize, fileVersion=3) self._checkContainerClosed(containerName, False) # Remove all files and check that the container is empty self.cremove(None, fileIdList=fileIds, containerName=containerName) self._checkFilesAndContainerSize(containerName, 0, 0) self._checkContainerClosed(containerName, False) #------------------------------------------------------------------ # The final test in this section is about "closing" a container # When a container is created via CCREATE it remains "opened", # until a CAPPEND command specifies that it wants to "close" the # container. A container is closed when its ingestion date is set. #------------------------------------------------------------------ # First of all, check that the container is still currently opened self._checkContainerClosed(containerName, False) # Append a file, check that the container is still opened myfileId = os.path.basename(self.myfiles[0]) self.cappend(myfileId, containerName=containerName) self._checkContainerClosed(containerName, False) # Append a second file and mark the container as closed myfileId = os.path.basename(self.myfiles[1]) self.cappend(myfileId, containerName=containerName, closeContainer=True) self._checkContainerClosed(containerName, True)
def handleCmd(srvObj, reqPropsObj, httpRef): """ Compress the file based on file_path (file_id, disk_id, and file_version) srvObj: Reference to NG/AMS server class object (ngamsServer). reqPropsObj: Request Property object to keep track of actions done during the request handling (ngamsReqProps). httpRef: Reference to the HTTP request handler object (ngamsHttpRequestHandler). Returns: Void. """ required_params = ['file_path', 'file_id', 'file_version', 'disk_id'] parDic = reqPropsObj.getHttpParsDic() for rp in required_params: if (not parDic.has_key(rp)): errMsg = 'Parameter missing: %s' % rp httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) return filename = reqPropsObj.getHttpPar('file_path') logger.debug('Compressing file %s', filename) if (not os.path.exists(filename)): errMsg = 'File Not found: %s' % filename logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=404) return fileId = parDic['file_id'] fileVersion = int(parDic['file_version']) diskId = parDic['disk_id'] if (not isMWAVisFile(fileId)): errMsg = 'Not MWA visibilty file: %' % (fileId) logger.warning(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) return if (hasCompressed(filename)): errMsg = 'OK' logger.debug('File compressed %s already, return now', filename) httpRef.send_data(errMsg, NGAMS_TEXT_MT) return if (parDic.has_key('scale_factor')): sf = int(parDic['scale_factor']) else: sf = 4 if (parDic.has_key('threshold')): th = float(parDic['threshold']) # currently not used if (parDic.has_key('bins')): bins = int(parDic['bins']) else: bins = 0 if (parDic.has_key('remove_uc')): remove_uc = int(parDic['remove_uc']) # currently not used if (parDic.has_key('timeout')): timeout = int(parDic['timeout']) if (timeout <= 0): timeout = 600 if (parDic.has_key('debug')): debug = int(parDic['debug']) else: debug = 0 if (bins): binstr = '-h %d' % bins else: binstr = '' bname = os.path.basename(filename) newfn = '%s/%s' % (work_dir, bname) fndir = os.path.dirname(filename) old_fs = getFileSize(filename) # do compression cmd = "%s -d %d %s %s %s" % (uvcompress, sf, binstr, filename, newfn) logger.debug('Compression: %s', cmd) if (debug): pass else: ret = execCmd(cmd) if (ret[0] != 0): errMsg = 'Failed to compress %s' % ret[1] logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) return # calculate the CRC code logger.debug('Calculating CRC for file %s', newfn) if (debug): crc = 1234567 else: try: crc = getFileCRC(newfn) except Exception, exp: errMsg = 'Failed to calculate the CRC for file %s: %s' % (newfn, str(exp)) logger.error(errMsg) httpRef.send_data(errMsg, NGAMS_TEXT_MT, code=500) # remove the temp compressed file cmd1 = 'rm %s' % newfn execCmd(cmd1) return