def need_split(self,data) : # check if the bulletin needs to be segmented # AM bulletin has the following form : 128 bytes for a struct # followed by the bulletin. unBulletinAm = bulletinAm.bulletinAm(data,self.logger,lineSeparator='\r\r\n') limit = self.maxLength - 128 header = unBulletinAm.getHeader() lheader = len(header) + 1 tosplit = False if len(data) > limit : tosplit = True return tosplit
def need_split(self, data): # check if the bulletin needs to be segmented # AM bulletin has the following form : 128 bytes for a struct # followed by the bulletin. unBulletinAm = bulletinAm.bulletinAm(data, self.logger, lineSeparator='\r\r\n') limit = self.maxLength - 128 header = unBulletinAm.getHeader() lheader = len(header) + 1 tosplit = False if len(data) > limit: tosplit = True return tosplit
def write_data(self, data): unBulletinAm = bulletinAm.bulletinAm(data, self.logger, lineSeparator='\r\r\n') # applying the dx_script to the data if self.client.dx_execfile != None: try: newBulletinAm = self.client.run_dx_script( unBulletinAm, self.logger) if newBulletinAm == None: self.logger.warning("DX script ignored the data") return False, 0 elif newBulletinAm == unBulletinAm: self.logger.warning("DX script kept the data as is") else: self.logger.info("DX script modified data") unBulletinAm = newBulletinAm except: self.logger.error("DX script bombed") return False, 0 # determine the destination thread for that bulletin dest_thread = 255 if self.client.am_dest_thread != None: if data[:2] in self.client.am_dest_thread: dest_thread = self.client.am_dest_thread[data[:2]] self.logger.debug("%s bulletin assigned thread %d" % (data[:2], dest_thread)) elif '*' in self.client.am_dest_thread: dest_thread = self.client.am_dest_thread['*'] self.logger.debug("%s bulletin assigned thread %d" % (data[:2], dest_thread)) # The check to see of the connection must be re-initialized is in the sendBullein routine. succes, nbBytesSent = self.unSocketManagerAm.sendBulletin( unBulletinAm, dest_thread) #if the bulletin was sent successfully, erase the file. if succes: self.tallyBytes(nbBytesSent) return (succes, nbBytesSent)
def write_data(self,data): unBulletinAm = bulletinAm.bulletinAm(data,self.logger,lineSeparator='\r\r\n') # applying the dx_script to the data if self.client.dx_execfile != None : try : newBulletinAm = self.client.run_dx_script(unBulletinAm,self.logger) if newBulletinAm == None : self.logger.warning("DX script ignored the data") return False, 0 elif newBulletinAm == unBulletinAm : self.logger.warning("DX script kept the data as is") else : self.logger.info("DX script modified data") unBulletinAm = newBulletinAm except : self.logger.error("DX script bombed") return False, 0 # determine the destination thread for that bulletin dest_thread = 255 if self.client.am_dest_thread != None : if data[:2] in self.client.am_dest_thread : dest_thread = self.client.am_dest_thread[data[:2]] self.logger.debug("%s bulletin assigned thread %d" % (data[:2],dest_thread)) elif '*' in self.client.am_dest_thread : dest_thread = self.client.am_dest_thread['*'] self.logger.debug("%s bulletin assigned thread %d" % (data[:2],dest_thread)) # The check to see of the connection must be re-initialized is in the sendBullein routine. succes, nbBytesSent = self.unSocketManagerAm.sendBulletin(unBulletinAm,dest_thread) #if the bulletin was sent successfully, erase the file. if succes: self.tallyBytes(nbBytesSent) return (succes, nbBytesSent)
def _bulletinManager__generateBulletin(self,rawBulletin): __doc__ = bulletinManager.bulletinManager._bulletinManager__generateBulletin.__doc__ + \ """Override here to pass the correct arguments to bulletinAm """ return bulletinAm.bulletinAm(rawBulletin,self.logger,self.lineSeparator,self.mapEntetes,self.SMHeaderFormat)
def write_segmented_data(self,data,path): unBulletinAm = bulletinAm.bulletinAm(data,self.logger,lineSeparator='\r\r\n') limit = self.maxLength - 128 header = unBulletinAm.getHeader() lheader = len(header) + 1 # SHOULD SEGMENT BUT : At the moment BUFR are not segmented but discarded if data[lheader:lheader+4] == "BUFR" : self.logger.error("Unable to segment and send %s ! Reason : type %s, Size: %s" % (path, "BUFR", len(data) )) self.unlink_file(path) return ( False, 0 ) # SHOULD SEGMENT BUT : At the moment GRIB are not segmented but discarded if data[lheader:lheader+4] == "GRIB" : self.logger.error("Unable to segment and send %s ! Reason : type %s, Size: %s" % (path, "GRIB", len(data) )) self.unlink_file(path) return ( False, 0 ) # SHOULD SEGMENT BUT : the bulletin already have a BBB group -> not segmented but discarded # FIXME should validate that the 4th token is realy a BBB (AAa-z CCa-z RRa-z Pa-za-z AMD COR RTM) tokn = header.split() if len(tokn) == 4 : self.logger.error("Unable to send %s Segmented ! Reason : BBB = %s, Size: %s" % (path, tokn[3], len(data) )) self.unlink_file(path) return ( False, 0 ) # Perform segmentation # segmentation the block size is computed like this : # maxLength - 128 (Am struct size) - (len(header) + '\n\ + ' ' + BBB ) limit = self.maxLength - 128 - (lheader + 4) blocks = TextSplitter(data[lheader:], limit ).breakMarker() self.logger.info("(%i Bytes) Bulletin %s delivered segmented in %d parts" % (len(data),os.path.basename(path),len(blocks))) self.logger.debug("Bulletin is \n%s" % data) i = 0 totSent = 0 priority = path.split('/')[-3] alpha=['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z'] for part in blocks : l1 = alpha[i/24] l2 = alpha[i%24] i = i + 1 if i == len(blocks) : l1 = 'Z' rawSegment = header + " P" + l1 + l2 + '\n' + part if self.client.nodups and priority != '0' and self.in_cache( rawSegment, False, None ) : continue succes, nbBytesSent = self.write_data(rawSegment) if succes : self.tallyBytes(nbBytesSent) self.logger.info("(%i Bytes) Bulletin Segment number %d sent (%s)" % (nbBytesSent,i,header + " P" + l1 + l2)) self.logger.debug("Bulletin is \n%s" % rawSegment) else : return (False, totSent) return (True, totSent)
def write_segmented_data(self, data, path): unBulletinAm = bulletinAm.bulletinAm(data, self.logger, lineSeparator='\r\r\n') limit = self.maxLength - 128 header = unBulletinAm.getHeader() lheader = len(header) + 1 # SHOULD SEGMENT BUT : At the moment BUFR are not segmented but discarded if data[lheader:lheader + 4] == "BUFR": self.logger.error( "Unable to segment and send %s ! Reason : type %s, Size: %s" % (path, "BUFR", len(data))) self.unlink_file(path) return (False, 0) # SHOULD SEGMENT BUT : At the moment GRIB are not segmented but discarded if data[lheader:lheader + 4] == "GRIB": self.logger.error( "Unable to segment and send %s ! Reason : type %s, Size: %s" % (path, "GRIB", len(data))) self.unlink_file(path) return (False, 0) # SHOULD SEGMENT BUT : the bulletin already have a BBB group -> not segmented but discarded # FIXME should validate that the 4th token is realy a BBB (AAa-z CCa-z RRa-z Pa-za-z AMD COR RTM) tokn = header.split() if len(tokn) == 4: self.logger.error( "Unable to send %s Segmented ! Reason : BBB = %s, Size: %s" % (path, tokn[3], len(data))) self.unlink_file(path) return (False, 0) # Perform segmentation # segmentation the block size is computed like this : # maxLength - 128 (Am struct size) - (len(header) + '\n\ + ' ' + BBB ) limit = self.maxLength - 128 - (lheader + 4) blocks = TextSplitter(data[lheader:], limit).breakMarker() self.logger.info( "(%i Bytes) Bulletin %s delivered segmented in %d parts" % (len(data), os.path.basename(path), len(blocks))) self.logger.debug("Bulletin is \n%s" % data) i = 0 totSent = 0 priority = path.split('/')[-3] alpha = [ 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z' ] for part in blocks: l1 = alpha[i / 24] l2 = alpha[i % 24] i = i + 1 if i == len(blocks): l1 = 'Z' rawSegment = header + " P" + l1 + l2 + '\n' + part if self.client.nodups and priority != '0' and self.in_cache( rawSegment, False, None): continue succes, nbBytesSent = self.write_data(rawSegment) if succes: self.tallyBytes(nbBytesSent) self.logger.info( "(%i Bytes) Bulletin Segment number %d sent (%s)" % (nbBytesSent, i, header + " P" + l1 + l2)) self.logger.debug("Bulletin is \n%s" % rawSegment) else: return (False, totSent) return (True, totSent)