Ejemplo n.º 1
0
    def need_split(self,data) :

        # check if the bulletin needs to be segmented
        # AM bulletin has the following form : 128 bytes for a struct 
        # followed by the bulletin.

        unBulletinAm = bulletinAm.bulletinAm(data,self.logger,lineSeparator='\r\r\n')
        limit   = self.maxLength - 128
        header  = unBulletinAm.getHeader()
        lheader = len(header) + 1

        tosplit = False
        if  len(data) > limit :
            tosplit = True

        return tosplit
Ejemplo n.º 2
0
    def need_split(self, data):

        # check if the bulletin needs to be segmented
        # AM bulletin has the following form : 128 bytes for a struct
        # followed by the bulletin.

        unBulletinAm = bulletinAm.bulletinAm(data,
                                             self.logger,
                                             lineSeparator='\r\r\n')
        limit = self.maxLength - 128
        header = unBulletinAm.getHeader()
        lheader = len(header) + 1

        tosplit = False
        if len(data) > limit:
            tosplit = True

        return tosplit
Ejemplo n.º 3
0
    def write_data(self, data):
        unBulletinAm = bulletinAm.bulletinAm(data,
                                             self.logger,
                                             lineSeparator='\r\r\n')

        # applying the dx_script to the data

        if self.client.dx_execfile != None:
            try:
                newBulletinAm = self.client.run_dx_script(
                    unBulletinAm, self.logger)
                if newBulletinAm == None:
                    self.logger.warning("DX script ignored the data")
                    return False, 0
                elif newBulletinAm == unBulletinAm:
                    self.logger.warning("DX script kept the data as is")
                else:
                    self.logger.info("DX script modified data")
                    unBulletinAm = newBulletinAm
            except:
                self.logger.error("DX script bombed")
                return False, 0

        # determine the destination thread for that bulletin
        dest_thread = 255
        if self.client.am_dest_thread != None:
            if data[:2] in self.client.am_dest_thread:
                dest_thread = self.client.am_dest_thread[data[:2]]
                self.logger.debug("%s bulletin assigned thread %d" %
                                  (data[:2], dest_thread))
            elif '*' in self.client.am_dest_thread:
                dest_thread = self.client.am_dest_thread['*']
                self.logger.debug("%s bulletin assigned thread %d" %
                                  (data[:2], dest_thread))

        # The check to see of the connection must be re-initialized is in the sendBullein routine.
        succes, nbBytesSent = self.unSocketManagerAm.sendBulletin(
            unBulletinAm, dest_thread)

        #if the bulletin was sent successfully, erase the file.
        if succes:
            self.tallyBytes(nbBytesSent)

        return (succes, nbBytesSent)
Ejemplo n.º 4
0
    def write_data(self,data):
        unBulletinAm = bulletinAm.bulletinAm(data,self.logger,lineSeparator='\r\r\n')

        # applying the dx_script to the data

        if self.client.dx_execfile != None :
            try : 
                     newBulletinAm = self.client.run_dx_script(unBulletinAm,self.logger)
                     if newBulletinAm == None :
                        self.logger.warning("DX script ignored the data")
                        return False, 0
                     elif newBulletinAm == unBulletinAm :
                        self.logger.warning("DX script kept the data as is")
                     else :
                        self.logger.info("DX script modified data")
                        unBulletinAm = newBulletinAm
            except : 
                     self.logger.error("DX script bombed")
                     return False, 0

        # determine the destination thread for that bulletin
        dest_thread = 255
        if self.client.am_dest_thread != None :
           if data[:2] in self.client.am_dest_thread :
              dest_thread = self.client.am_dest_thread[data[:2]]
              self.logger.debug("%s bulletin assigned thread %d" % (data[:2],dest_thread))
           elif '*'    in self.client.am_dest_thread :
              dest_thread = self.client.am_dest_thread['*']
              self.logger.debug("%s bulletin assigned thread %d" % (data[:2],dest_thread))

        # The check to see of the connection must be re-initialized is in the sendBullein routine.
        succes, nbBytesSent = self.unSocketManagerAm.sendBulletin(unBulletinAm,dest_thread)

        #if the bulletin was sent successfully, erase the file.
        if succes:
           self.tallyBytes(nbBytesSent)

        return (succes, nbBytesSent)
Ejemplo n.º 5
0
    def _bulletinManager__generateBulletin(self,rawBulletin):
        __doc__ = bulletinManager.bulletinManager._bulletinManager__generateBulletin.__doc__ + \
        """Override here to pass the correct arguments to bulletinAm 

        """
        return bulletinAm.bulletinAm(rawBulletin,self.logger,self.lineSeparator,self.mapEntetes,self.SMHeaderFormat)
Ejemplo n.º 6
0
    def write_segmented_data(self,data,path):

        unBulletinAm = bulletinAm.bulletinAm(data,self.logger,lineSeparator='\r\r\n')
        limit   = self.maxLength - 128
        header  = unBulletinAm.getHeader()
        lheader = len(header) + 1

        # SHOULD SEGMENT BUT : At the moment BUFR are not segmented but discarded
        if data[lheader:lheader+4] == "BUFR" :
           self.logger.error("Unable to segment and send %s ! Reason : type %s, Size: %s" % (path, "BUFR", len(data) ))
           self.unlink_file(path)
           return ( False, 0 )

        # SHOULD SEGMENT BUT : At the moment GRIB are not segmented but discarded
        if data[lheader:lheader+4] == "GRIB" :
           self.logger.error("Unable to segment and send %s ! Reason : type %s, Size: %s" % (path, "GRIB", len(data) ))
           self.unlink_file(path)
           return ( False, 0 )

        # SHOULD SEGMENT BUT : the bulletin already have a BBB group -> not segmented but discarded
        # FIXME should validate that the 4th token is realy a BBB (AAa-z CCa-z RRa-z Pa-za-z AMD COR RTM)
        tokn = header.split()
        if len(tokn) == 4 :
           self.logger.error("Unable to send %s Segmented ! Reason : BBB = %s, Size: %s" % (path, tokn[3], len(data) ))
           self.unlink_file(path)
           return ( False, 0 )

        # Perform segmentation
        # segmentation the block size is computed like this :
        # maxLength - 128 (Am struct size) - (len(header) + '\n\ + ' ' + BBB  )

        limit   = self.maxLength - 128 - (lheader + 4)
        blocks  = TextSplitter(data[lheader:], limit ).breakMarker()
        self.logger.info("(%i Bytes) Bulletin %s  delivered segmented in %d parts" % (len(data),os.path.basename(path),len(blocks)))
        self.logger.debug("Bulletin is \n%s" % data)
                    
        i       =  0
        totSent =  0


        priority = path.split('/')[-3]

        alpha=['A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z']

        for part in blocks :
            l1 = alpha[i/24]
            l2 = alpha[i%24]
            i  = i + 1
            if i == len(blocks) : l1 = 'Z'
            rawSegment = header + " P" + l1 + l2 + '\n' + part

            if self.client.nodups and priority != '0' and self.in_cache( rawSegment, False, None ) :
               continue

            succes, nbBytesSent = self.write_data(rawSegment)
            if succes :
               self.tallyBytes(nbBytesSent)
               self.logger.info("(%i Bytes) Bulletin Segment number %d sent (%s)" % (nbBytesSent,i,header + " P" + l1 + l2))
               self.logger.debug("Bulletin is \n%s" % rawSegment)
            else :
               return (False, totSent)

        return (True, totSent)
Ejemplo n.º 7
0
    def write_segmented_data(self, data, path):

        unBulletinAm = bulletinAm.bulletinAm(data,
                                             self.logger,
                                             lineSeparator='\r\r\n')
        limit = self.maxLength - 128
        header = unBulletinAm.getHeader()
        lheader = len(header) + 1

        # SHOULD SEGMENT BUT : At the moment BUFR are not segmented but discarded
        if data[lheader:lheader + 4] == "BUFR":
            self.logger.error(
                "Unable to segment and send %s ! Reason : type %s, Size: %s" %
                (path, "BUFR", len(data)))
            self.unlink_file(path)
            return (False, 0)

        # SHOULD SEGMENT BUT : At the moment GRIB are not segmented but discarded
        if data[lheader:lheader + 4] == "GRIB":
            self.logger.error(
                "Unable to segment and send %s ! Reason : type %s, Size: %s" %
                (path, "GRIB", len(data)))
            self.unlink_file(path)
            return (False, 0)

        # SHOULD SEGMENT BUT : the bulletin already have a BBB group -> not segmented but discarded
        # FIXME should validate that the 4th token is realy a BBB (AAa-z CCa-z RRa-z Pa-za-z AMD COR RTM)
        tokn = header.split()
        if len(tokn) == 4:
            self.logger.error(
                "Unable to send %s Segmented ! Reason : BBB = %s, Size: %s" %
                (path, tokn[3], len(data)))
            self.unlink_file(path)
            return (False, 0)

        # Perform segmentation
        # segmentation the block size is computed like this :
        # maxLength - 128 (Am struct size) - (len(header) + '\n\ + ' ' + BBB  )

        limit = self.maxLength - 128 - (lheader + 4)
        blocks = TextSplitter(data[lheader:], limit).breakMarker()
        self.logger.info(
            "(%i Bytes) Bulletin %s  delivered segmented in %d parts" %
            (len(data), os.path.basename(path), len(blocks)))
        self.logger.debug("Bulletin is \n%s" % data)

        i = 0
        totSent = 0

        priority = path.split('/')[-3]

        alpha = [
            'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
            'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z'
        ]

        for part in blocks:
            l1 = alpha[i / 24]
            l2 = alpha[i % 24]
            i = i + 1
            if i == len(blocks): l1 = 'Z'
            rawSegment = header + " P" + l1 + l2 + '\n' + part

            if self.client.nodups and priority != '0' and self.in_cache(
                    rawSegment, False, None):
                continue

            succes, nbBytesSent = self.write_data(rawSegment)
            if succes:
                self.tallyBytes(nbBytesSent)
                self.logger.info(
                    "(%i Bytes) Bulletin Segment number %d sent (%s)" %
                    (nbBytesSent, i, header + " P" + l1 + l2))
                self.logger.debug("Bulletin is \n%s" % rawSegment)
            else:
                return (False, totSent)

        return (True, totSent)