Exemple #1
0
    def run(self):
        self.server = nntp.NNTP(**self.credentials)
        try:
            while True:
                if self.stop:
                    self.server.quit()
                    return

                # Try to grab a segment from queue
                f, segment, bytes = self.segments.get(False)

                # Check for the article on the server
                try:
                    self.server.stat(segment)
                    # print "Found: %s" % segment
                except nntplib.NNTPTemporaryError, e:
                    # Error code 430 is "No such article"
                    error = nntp.get_error_code(e)
                    if error == "430":
                        # Found missing segment
                        self.missing.put((f, segment, bytes))
                        self.missing.task_done()
                        # print "Missing: %s" % segment
                    else:
                        # Some other error, put the segment back in the
                        # queue to be checked again
                        print "Error: %s" % e
                        self.segments.put(segment)
                except Exception, e:
                    print "Unknown error: %s" % e
                    return
Exemple #2
0
 def run(self):
     self.server = nntp.NNTP(**self.credentials)
     try:
         while True:
             if self.stop:
                 self.server.quit()
                 return
             
             # Try to grab a segment from queue
             f, segment, bytes = self.segments.get(False)
             
             # Check for the article on the server
             try:
                 self.server.stat(segment)
                 #print "Found: %s" % segment
             except nntplib.NNTPTemporaryError, e:
                 # Error code 430 is "No such article"
                 error = nntp.get_error_code(e)
                 if error == '430':
                     # Found missing segment                    
                     self.missing.put((f, segment, bytes))
                     self.missing.task_done()
                     #print "Missing: %s" % segment
                 else:
                     # Some other error, put the segment back in the
                     # queue to be checked again
                     print "Error: %s" % e
                     self.segments.put(segment)
             except Exception, e:
                 print "Unknown error: %s" % e
                 return
Exemple #3
0
    def check_segment(self, connection, segment, segments, missing):
        """
        Check to see if this server has the requested segment.

        Arguments:
            connection  - NNTP connection
            segment     - Tuple of format: (file name, message id, bytes)
            segments    - Global segments queue
            missing     - Global missing segments queue

        If the NNTP connection has be closed (due to idle) will raise an
        ``ConnectionClosedException`` exception.
        """
        if self.has_missing_segment(segment):
            # We already know we don't have this segment, so send it to the
            # next server
            self.log(
                log.info,
                "Segment '%s' already marked as missing, passing off to next server"
                % segment[1])
            self.try_next_server(segment, missing)
            return

        f, msgid, bytes = segment

        # Check for the article on the server
        try:
            # Found the segment
            connection.stat(msgid)

            if self.conf.get("backup"):
                self.log(log.info,
                         "Found segment '%s' on backup server" % segment[1])

        except nntplib.NNTPTemporaryError as e:
            # Error code 430 is "No such article"
            error = nntp.get_error_code(e)
            if error == '430':
                # Found missing segment.  Mark it as missing from this server
                # and pass it to the highest priority server, where it'll then
                # trickle its way down through all the servers
                self.add_missing_segment(segment)
                self._servers[0].segments.put(segment)
                return

            elif error == "400":
                # Code 400 means the connection is closed, usually do to a timeout
                raise ConnectionClosedException(
                    "Connection closed - idle timeout?")

            else:
                # Some other error, put the segment back in the
                # queue to be checked again
                log.error("Error: %s" % e)
                segments.put(segment)
                return

        except Exception as e:
            log.error("Unknown error: %s" % e)
            return
Exemple #4
0
    def check_segment(self, connection, segment, segments, missing):
        """
        Check to see if this server has the requested segment.

        Arguments:
            connection  - NNTP connection
            segment     - Tuple of format: (file name, message id, bytes)
            segments    - Global segments queue
            missing     - Global missing segments queue

        If the NNTP connection has be closed (due to idle) will raise an
        ``ConnectionClosedException`` exception.
        """
        if self.has_missing_segment(segment):
            # We already know we don't have this segment, so send it to the
            # next server
            self.log(log.info, "Segment '%s' already marked as missing, passing off to next server" % segment[1])
            self.try_next_server(segment, missing)
            return

        f, msgid, bytes = segment

        # Check for the article on the server
        try:
            # Found the segment
            connection.stat(msgid)

            if self.conf.get("backup"):
                self.log(log.info, "Found segment '%s' on backup server" % segment[1])

        except nntplib.NNTPTemporaryError, e:
            # Error code 430 is "No such article"
            error = nntp.get_error_code(e)
            if error == '430':
                # Found missing segment.  Mark it as missing from this server
                # and pass it to the highest priority server, where it'll then
                # trickle its way down through all the servers
                self.add_missing_segment(segment)
                self._servers[0].segments.put(segment)
                return

            elif error == "400":
                # Code 400 means the connection is closed, usually do to a timeout
                raise ConnectionClosedException("Connection closed - idle timeout?")

            else:
                # Some other error, put the segment back in the
                # queue to be checked again
                log.error("Error: %s" % e)
                segments.put(segment)
                return
	def stat_all(self, msg):
		lastgrp = ''

		#~ for info in infodata['detail']:
		#~ if(info[2] == 0 or info[2] == 1 or info[2] == 4):


		for m in msg['detail']:			
			if(m[2] == 0 or m[2] == 1 or m[2] == 4):			
				#~ simple semaphor
				if(m[5] != -3):
					continue
				with lock:
					global completion_counter
					completion_counter += 1				
				m[5] = -1
				repeat = MAXREPEAT
				while (repeat):
					try:
						if(lastgrp != m[4][0]):
							lastgrp = m[4][0]
							self.tn.group(m[4][0])
							#~ print 'change grp'
							
						self.tn.stat("<"+m[3]+">")
						m[5] = 1
						repeat = 0
					except Exception as e:	
						error = nntp.get_error_code(e)
						if error == '430':
							#~ print "Missing article " + m[3] + " : %s" % e
							print "*",
							m[5] = 0
							repeat = 0
						else:
							#~ print "Error ("+ str(repeat) +" more attempts, trying again: " + m[3] + " %s" % e
							print "e",
							repeat -= 1
    def stat_all(self, msg):
        lastgrp = ''

        #~ for info in infodata['detail']:
        #~ if(info[2] == 0 or info[2] == 1 or info[2] == 4):

        for m in msg['detail']:
            if (m[2] == 0 or m[2] == 1 or m[2] == 4):
                #~ simple semaphor
                if (m[5] != -3):
                    continue
                with lock:
                    global completion_counter
                    completion_counter += 1
                m[5] = -1
                repeat = MAXREPEAT
                while (repeat):
                    try:
                        if (lastgrp != m[4][0]):
                            lastgrp = m[4][0]
                            self.tn.group(m[4][0])
                            #~ print 'change grp'

                        self.tn.stat("<" + m[3] + ">")
                        m[5] = 1
                        repeat = 0
                    except Exception as e:
                        error = nntp.get_error_code(e)
                        if error == '430':
                            #~ print "Missing article " + m[3] + " : %s" % e
                            print "*",
                            m[5] = 0
                            repeat = 0
                        else:
                            #~ print "Error ("+ str(repeat) +" more attempts, trying again: " + m[3] + " %s" % e
                            print "e",
                            repeat -= 1