Esempio n. 1
0
    def do_broker_callback_tasks(self, queue):
        while True:
            try:
                msg = yield queue.get()
                log.msg("DEBUG: do_broker_callback_tasks: got msg %s" % msg)
                unpacked_msg = self.service.unpack(msg.content.body)
                log.msg("DEBUG: do_broker_callback_tasks: unpacked_msg is %s" % str(unpacked_msg))
                corr_id = msg.content.properties['correlation id']
                if corr_id not in self.broker_callback_map:
                    log.err("do_broker_callback_tasks: invalid corr_id %s" % corr_id)
                    continue

                d, f_name, exchange = self.broker_callback_map[corr_id]
                if self.is_broker_error(unpacked_msg):
                    if unpacked_msg["err_id"] == error.ERROR_INTERNAL_SERVICE_ERROR:
                        d.errback(BrokerError(unpacked_msg["err_id"], unpacked_msg["err_msg"],
                                              f_name=f_name, exchange=exchange))
                    else:
                        d.errback(error.EInternalError(unpacked_msg["err_id"]))
                else:
                    d.callback(unpacked_msg)

                del self.broker_callback_map[corr_id]
            except Closed, e_inst:
                log.msg("do_broker_callback_tasks: queue is closed: %s" % str(e_inst))
                break
            except Exception:
                log.error(str(msg))
                formatted_traceback = traceback.format_exc()
                log.error(formatted_traceback)
    def requestAvatarId(self, c):
        def cbRequest(response):
                log.msg('cbRequest Got Response code: %s'%response.code)
                for s in response.headers.getRawHeaders("Set-Cookie", ()):
                    (key, value) = s.split("=")
                    if key == "session_cookie":
                        self.env["BSC_SESSION_COOKIE"] = value

                if response.code == 200:
                    return defer.succeed(c.username)
                else:
                    return defer.fail(error.UnauthorizedLogin())
        def cbError(reason):
            print "+++ error during http request +++"
            reason.printTraceback()
            return defer.fail(error.UnauthorizedLogin(reason.getErrorMessage()))

        headers = {'Content-type': [ 'application/json' ]}
        req = {'user': c.username, 'password' : c.password }
        serialized_payload = json.dumps(req)
        log.msg("Requesting Payload %s" % serialized_payload)
        try:
            d = agent.request('POST', self.url, Headers(headers), StringProducer(serialized_payload))
            d.addCallbacks(cbRequest)
            d.addErrback(cbError)
        except Exception as e:
            log.error("Auth failed",e)
            return defer.fail(error.UnauthorizedLogin())
        log.msg("Returning deferred")
        return d
Esempio n. 3
0
def refresh_mirror( working_dir, first_block, last_block ):
   """
   Refresh the mirror:
   * process all new invalidations
   * grab and mirror any new profiles from the DHT
   
   This gets called by Twisted every time there ought to be a new block.
   """

   from twisted.python import log
   from kademlia.network import Server 
   
   # make soure our bitcoind cached block index is up-to-speed 
   nameop_sequence = sync_blockchain( working_dir, first_block, last_block )
   if nameop_sequence is None:
      
      log.error("sync_blockchain(%s-%s) failed" % (first_block, last_block))
      return None
   
   # synchronize name registrations...
   
   
   server = Server()
   server.listen( dht.plugin.DHT_SERVER_PORT )
   server.bootstrap( dht.plugin.DEFAULT_DHT_SERVERS ).addCallback( connect_done, server )
   
   pass
Esempio n. 4
0
  def onMessage(self, msg):
    log.debug(u"Received %s message from %s: %s" % (msg['type'], msg['from'], msg.body))

    if msg.x and msg.x.defaultUri == 'jabber:x:delay':
      log.debug(u"Ignoring delayed message")
      return

    if msg.body is None:
      log.debug(u'Ignoring empty message')
      self.chat(msg['from'], "Use $start to create menu")
      return

    try:
      if msg['type'] == 'error':
        log.debug("Error message received from %s: %s" % (msg['from'], msg.body))
        return
      elif msg['type'] == 'chat':
        if self.message_handler:
          self.message_handler.answer(
            transport = self.chat,
            recipient = msg['from'],
            message = msg.body)
        else:
          self.chat(msg['from'], "Message received sir!")
    except Exception, e:
      log.error("Some error occurred: %s" % e)
Esempio n. 5
0
 def answer(self, transport, recipient, message):
   user = self.auth.authenticate(recipient)
   if user:
     session = USSDSession.objects.recent(user)
     if session:
       log.debug("User: %s USSD Session: %s" % (user, session))
       msg = str(message)
       if msg == "$start":
         response = handle_start(session)
       elif msg == "$end":
         response = handle_end(session)
       elif msg == "$error":
         import pdb
         pdb.set_trace()
       else:
         if session.current_menu and not session.current_menu.is_finished():
           response = handle_session(session, msg)
         else:
           response = "Use $start to create menu"
       if recipient and response:
         transport(recipient, response)
       else:
         log.error("Either recipient (%s) or response (%s) is not valid!" % (recipient, response))
       return
     else:
       transport(recipient, "Error: No USSD Session!")
   else:
     transport(recipient, "Error: No User!")
Esempio n. 6
0
    def joinNetwork(self, knownNodeAddresses=None):
        """ Causes the Node to join the Kademlia network; normally, this
        should be called before any other DHT operations.

        @param knownNodeAddresses: A sequence of tuples containing IP address
                                   information for existing nodes on the
                                   Kademlia network, in the format:
                                   C{(<ip address>, (udp port>)}
        @type knownNodeAddresses: tuple
        """
        # Prepare the underlying Kademlia protocol
        if self.port is not None:
            try:
                self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol)
            except error.CannotListenError as e:
                import traceback
                log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
                raise ValueError("%s lbrynet may already be running." % str(e))
        #IGNORE:E1101
        # Create temporary contact information for the list of addresses of known nodes
        if knownNodeAddresses != None:
            bootstrapContacts = []
            for address, port in knownNodeAddresses:
                contact = Contact(self._generateID(), address, port, self._protocol)
                bootstrapContacts.append(contact)
        else:
            bootstrapContacts = None
        # Initiate the Kademlia joining sequence - perform a search for this node's own ID
        self._joinDeferred = self._iterativeFind(self.id, bootstrapContacts)
#        #TODO: Refresh all k-buckets further away than this node's closest neighbour
        # Start refreshing k-buckets periodically, if necessary
        self.next_refresh_call = twisted.internet.reactor.callLater(
            constants.checkRefreshInterval, self._refreshNode) #IGNORE:E1101
        self.hash_watcher.tick()
        return self._joinDeferred
Esempio n. 7
0
 def build_TLV_response(self, requestdata):
     responsedata = OrderedDict()
     for typ, value in requestdata.items():
         if typ == 'NAME':
             # send full host name - no truncation
             value = self.service.hostname.encode("UTF-8")
         elif typ == 'IPAD':
             # send ipaddress as a string only if it is set
             value = self.transport.getHost().host
             # :todo: IPv6
             if value == '0.0.0.0':
                 # do not send back an ip address
                 typ = None
         elif typ == 'JSON':
             # send port as a string
             json_port = 9000 # todo: web.service.port
             value = str(json_port)
         elif typ == 'VERS':
             # send server version
              value = squeal.__version__
         elif typ == 'UUID':
             # send server uuid
             value = self.service.uuid
         elif typ == 'JVID':
             # not handle, just log the information
             typ = None
             log.msg("Jive: %x:%x:%x:%x:%x:%x:" % struct.unpack('>6B', value),
                     logLevel=logging.INFO)
         else:
             log.error('Unexpected information request: %r', typ)
             typ = None
         if typ:
             responsedata[typ] = value
     return responsedata
Esempio n. 8
0
def twisted_log(eventDict):
    log = logging.getLogger('twisted')
    if 'failure' in eventDict:
        log.error(eventDict.get('why') or 'Unhandled exception' + '\n' + str(eventDict['failure'].getTraceback()))
    elif 'warning' in eventDict:
        log.warning(eventDict['warning'])
    else:
        log.debug(' '.join([str(m) for m in eventDict['message']]))
Esempio n. 9
0
 def removeListener(self, eventvalue, callback):
     if not hasattr(callback, "__call__"):
          callback = getattr(callback, eventvalue+"Callback"
                  ) if hasattr(callback, eventvalue+"Callback"
                  ) else getattr(callback, "callback")
     try:
         self.event_registry[eventvalue].remove(callback)
     except Exception as ex:
          log.error(ex)
Esempio n. 10
0
 def log_error(err, n):
     if err.check(protocol.TimeoutError):
         log.debug(
             "Timeout while storing blob_hash %s at %s",
             binascii.hexlify(blob_hash), n)
     else:
         log.error(
             "Unexpected error while storing blob_hash %s at %s: %s",
             binascii.hexlify(blob_hash), n, err.getErrorMessage())
Esempio n. 11
0
    def checkIfValidEmail(self, email):
        """
        Checks if the passed email is valid based on the regex string
        """

        valid_email = re.compile(
            r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")

        if not valid_email.match(email):
            log.error("Invalid email: %s" % email)
            raise ValueError(email)
Esempio n. 12
0
 def get_msg(self, my_queue):
     """
     The function takes message from the queue
     """
     self.channel.basic_qos(prefetch_count=COUNT)
     try:
         for method_frame, properties, body in self.channel.consume(my_queue):
             self.channel.basic_ack(method_frame.delivery_tag)
             return body
     except pika.exceptions, err_msg:
         log.error(err_msg)
         return False
Esempio n. 13
0
def failure(failure, log, msg, *args):
    """Log a failure message from a deferred.

    Args:
        failure: twisted.python.failure.Failure
        log: a python logger instance
        msg: the message to log. Can use normal logging string interpolation.
             the last argument will be set to the error message from the failure.
        args: values to substitute into `msg`
    """
    args += (failure.getErrorMessage(),)
    exc_info = (failure.type, failure.value, failure.getTracebackObject())
    log.error(msg, *args, exc_info=exc_info)
Esempio n. 14
0
def _parseStunResponse(dgram, address, expectedTID=None, oldtids=[]):
    mt, pktlen, tid = struct.unpack('!hh16s', dgram[:20])
    if expectedTID is not None and expectedTID != tid:
        # a response from an earlier request
        if tid in oldtids:
            # discard
            if STUNVERBOSE:
                log.msg("ignoring belated STUN response to %r from %s"%(
                            hexify(tid), repr(address)), system='stun')
            return
        log.msg("got unexpected STUN response %r != %r from %s"%
                        (hexify(expectedTID), hexify(tid),
                        repr(address),), system='stun')
        return
    resdict = {}
    if mt == 0x0101:
        #服务器的response message
        log.msg("got STUN response to %r from %s"%(hexify(expectedTID),
                                                   repr(address)),
                                                        system='stun')
        # response
        remainder = dgram[20:]
        while remainder:
            avtype, avlen = struct.unpack('!hh', remainder[:4])
            val = remainder[4:4+avlen]
            avtype = StunTypes.get(avtype, '(Unknown type %04x)'%avtype)
            remainder = remainder[4+avlen:]
            if avtype in ('MAPPED-ADDRESS',
                          'CHANGED-ADDRESS',
                          'SOURCE-ADDRESS'):
                dummy,family,port,addr = struct.unpack('!ccH4s', val)
                addr = socket.inet_ntoa(addr)
                if STUNVERBOSE:
                    print avtype, addr, port
                if avtype == 'MAPPED-ADDRESS':
                    resdict['externalAddress'] = (addr, port)
                elif avtype == 'CHANGED-ADDRESS':
                    resdict['_altStunAddress'] = (addr, address[1])
                elif address[0] != addr:
                    # Some son of a bitch is rewriting packets on the way
                    # back. AAARGH.
                    log.msg('WARNING: packets are being rewritten %r != %r'%
                            (address, (addr,port)), system='stun')
                    return
            else:
                log.msg("STUN: unhandled AV %s, val %r"%(avtype,
                                                         repr(val)),
                                                         system='stun')
    elif mt == 0x0111:
        log.error("STUN got an error response")
    return resdict
Esempio n. 15
0
def _parseStunResponse(dgram, address, expectedTID=None, oldtids=[]):
    mt, pktlen, tid = struct.unpack('!hh16s', dgram[:20])
    if expectedTID is not None and expectedTID != tid:
        # a response from an earlier request
        if tid in oldtids:
            # discard
            if STUNVERBOSE:
                log.msg("ignoring belated STUN response to %r from %s"%(
                            hexify(tid), repr(address)), system='stun')
            return
        log.msg("got unexpected STUN response %r != %r from %s"%
                        (hexify(expectedTID), hexify(tid),
                        repr(address),), system='stun')
        return
    resdict = {}
    if mt == 0x0101:
        log.msg("got STUN response to %r from %s"%(hexify(expectedTID),
                                                   repr(address)),
                                                        system='stun')
        # response
        remainder = dgram[20:]
        while remainder:
            avtype, avlen = struct.unpack('!hh', remainder[:4])
            val = remainder[4:4+avlen]
            avtype = StunTypes.get(avtype, '(Unknown type %04x)'%avtype)
            remainder = remainder[4+avlen:]
            if avtype in ('MAPPED-ADDRESS',
                          'CHANGED-ADDRESS',
                          'SOURCE-ADDRESS'):
                dummy,family,port,addr = struct.unpack('!ccH4s', val)
                addr = socket.inet_ntoa(addr)
                if STUNVERBOSE:
                    print((avtype, addr, port))
                if avtype == 'MAPPED-ADDRESS':
                    resdict['externalAddress'] = (addr, port)
                elif avtype == 'CHANGED-ADDRESS':
                    resdict['_altStunAddress'] = (addr, address[1])
                elif address[0] != addr:
                    # Some son of a bitch is rewriting packets on the way
                    # back. AAARGH.
                    log.msg('WARNING: packets are being rewritten %r != %r'%
                            (address, (addr,port)), system='stun')
                    return
            else:
                log.msg("STUN: unhandled AV %s, val %r"%(avtype,
                                                         repr(val)),
                                                         system='stun')
    elif mt == 0x0111:
        log.error("STUN got an error response")
    return resdict
Esempio n. 16
0
    def checkIfValidPassword(self, password):
        """
        Checks if the passed password is valid based on the regex string
        """

        min_len = 6
        max_len = 64

        valid_password = re.compile(
            r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))

        if not valid_password.match(password):
            log.error("Invalid password: %s" % password)
            raise ValueError(password)
Esempio n. 17
0
 def resumeProducing(self):
     if not self.request:
         log.error("Resuming a producer with no request object")
         return
     data = yield self.fileObject.read(
         min(self.bufferSize, self.size - self.bytesWritten))
     if data:
         self.request.write(data)
         self.bytesWritten += len(data)
     else:
         log.err("Finished reading from grid file: %s" % self.fileObject)
         self.request.unregisterProducer()
         self.request.finish()
         self.stopProducing()
Esempio n. 18
0
    def checkIfValidNickname(self, nickname):
        """
        Checks if the passed nickname is valid based on the regex string
        """

        min_len = 3
        max_len = 64

        valid_nickname = re.compile(
            r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))

        if not valid_nickname.match(nickname):
            log.error("Invalid nick: %s" % nickname)
            raise ValueError(nickname)
Esempio n. 19
0
    def lineReceived(self, line):
        """Dispatches commands from the child.  So far there's only one.  And
        you have to use it.  Or everything dies.
        """
        args = line.split(' ')
        command = args.pop(0)

        if command == 'set-port':
            port = int(args[0])
            self.protocol.set_port(port)
        elif command == 'request-done':
            self.protocol.track_request_done()
        else:
            log.error("Child is babbling nonsense: {0!r}".format(line))
Esempio n. 20
0
    def checkIfValidNickname(self, nickname):
        """
        Checks if the passed nickname is valid based on the regex string
        """

        min_len = 3
        max_len = 64

        valid_nickname = re.compile(
            r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))

        if not valid_nickname.match(nickname):
            log.error("Invalid nick: %s" % nickname)
            raise ValueError(nickname)
Esempio n. 21
0
    def checkIfValidPassword(self, password):
        """
        Checks if the passed password is valid based on the regex string
        """

        min_len = 6
        max_len = 64

        valid_password = re.compile(
            r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))

        if not valid_password.match(password):
            log.error("Invalid password: %s" % password)
            raise ValueError(password)
Esempio n. 22
0
    def scanip(self, entry):
        """
        Scan IP againt Greynoise API
        """
        def message(query):
            log.msg(
                eventid='cowrie.greynoise.result',
                format=
                'greynoise: Scan for %(IP)s with %(tag)s have %(conf)s confidence'
                ' along with the following %(meta)s metadata',
                IP=entry['src_ip'],
                tag=query['name'],
                conf=query['confidence'],
                meta=query['metadata'])

        gnUrl = '{0}query/ip'.format(GNAPI_URL).encode('utf8')
        headers = ({'User-Agent': [COWRIE_USER_AGENT]})
        fields = {'key': self.apiKey, 'ip': entry['src_ip']}

        try:
            response = yield treq.post(url=gnUrl,
                                       data=fields,
                                       headers=headers,
                                       timeout=10)
        except (defer.CancelledError, error.ConnectingCancelledError,
                error.DNSLookupError):
            log.msg("GreyNoise requests timeout")
            return

        if response.code != 200:
            rsp = yield response.text()
            log.error("greynoise: got error {}".format(rsp))
            return

        j = yield response.json()
        if self.debug:
            log.msg("greynoise: debug: " + repr(j))

        if j['status'] == "ok":
            if "all" not in self.tags:
                for query in j['records']:
                    if query['name'] in self.tags:
                        message(query)
            else:
                for query in j['records']:
                    message(query)
        else:
            log.msg("greynoise: no results for for IP {0}".format(
                entry['src_ip']))
Esempio n. 23
0
 def joinNetwork(self, knownNodeAddresses=None):
     """ Causes the Node to join the Kademlia network; normally, this
     should be called before any other DHT operations.
     
     @param knownNodeAddresses: A sequence of tuples containing IP address
                                information for existing nodes on the
                                Kademlia network, in the format:
                                C{(<ip address>, (udp port>)}
     @type knownNodeAddresses: tuple
     """
     # Prepare the underlying Kademlia protocol
     if self.port is not None:
         try:
             self._listeningPort = twisted.internet.reactor.listenUDP(
                 self.port, self._protocol)
         except error.CannotListenError as e:
             import traceback
             log.error("Couldn't bind to port %d. %s", self.port,
                       traceback.format_exc())
             raise ValueError("%s lbrynet may already be running." % str(e))
     #IGNORE:E1101
     # Create temporary contact information for the list of addresses of known nodes
     if knownNodeAddresses != None:
         bootstrapContacts = []
         for address, port in knownNodeAddresses:
             contact = Contact(self._generateID(), address, port,
                               self._protocol)
             bootstrapContacts.append(contact)
     else:
         bootstrapContacts = None
     # Initiate the Kademlia joining sequence - perform a search for this node's own ID
     self._joinDeferred = self._iterativeFind(self.id, bootstrapContacts)
     #        #TODO: Refresh all k-buckets further away than this node's closest neighbour
     #        def getBucketAfterNeighbour(*args):
     #            for i in range(160):
     #                if len(self._buckets[i]) > 0:
     #                    return i+1
     #            return 160
     #        df.addCallback(getBucketAfterNeighbour)
     #        df.addCallback(self._refreshKBuckets)
     #protocol.reactor.callLater(10, self.printContacts)
     #self._joinDeferred.addCallback(self._persistState)
     #self._joinDeferred.addCallback(self.printContacts)
     # Start refreshing k-buckets periodically, if necessary
     self.next_refresh_call = twisted.internet.reactor.callLater(
         constants.checkRefreshInterval, self._refreshNode)  #IGNORE:E1101
     self.hash_watcher.tick()
     return self._joinDeferred
Esempio n. 24
0
def run_server(foreground=False):
    """ run the blockstored server
    """

    global bitcoind
    prompt_user_for_chaincom_details()
    bitcoind = init_bitcoind()

    from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE
    from .lib.config import BLOCKSTORED_TAC_FILE
    from .lib.config import START_BLOCK

    working_dir = get_working_dir()

    current_dir = os.path.abspath(os.path.dirname(__file__))

    tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE)
    log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE)
    pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE)

    start_block, current_block = get_index_range()

    if foreground:
        command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
    else:
        command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file,
                                                              log_file,
                                                              tac_file)

    try:
        # refresh_index(335563, 335566, initial_index=True)
        if start_block != current_block:
            refresh_index(start_block, current_block, initial_index=True)
        blockstored = subprocess.Popen(
            command, shell=True, preexec_fn=os.setsid)
        log.info('Blockstored successfully started')

    except IndexError, ie:
        # indicates that we don't have the latest block 
        log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \
                   "Please verify that your bitcoin provider has " + \
                   "processed up to block %s.\n" % (START_BLOCK) + \
                   "    Example:  bitcoin-cli getblockcount" )
        try:
            os.killpg(blockstored.pid, signal.SIGTERM)
        except:
            pass
        exit(1)
Esempio n. 25
0
def run_server(foreground=False):
    """ run the blockstored server
    """

    global bitcoind
    prompt_user_for_chaincom_details()
    bitcoind = init_bitcoind()

    from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE
    from .lib.config import BLOCKSTORED_TAC_FILE
    from .lib.config import START_BLOCK

    working_dir = get_working_dir()

    current_dir = os.path.abspath(os.path.dirname(__file__))

    tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE)
    log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE)
    pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE)

    start_block, current_block = get_index_range()

    if foreground:
        command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
    else:
        command = 'twistd --pidfile=%s --logfile=%s -y %s' % (
            pid_file, log_file, tac_file)

    try:
        # refresh_index(335563, 335566, initial_index=True)
        if start_block != current_block:
            refresh_index(start_block, current_block, initial_index=True)
        blockstored = subprocess.Popen(command,
                                       shell=True,
                                       preexec_fn=os.setsid)
        log.info('Blockstored successfully started')

    except IndexError, ie:
        # indicates that we don't have the latest block
        log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \
                   "Please verify that your bitcoin provider has " + \
                   "processed up to block %s.\n" % (START_BLOCK) + \
                   "    Example:  bitcoin-cli getblockcount" )
        try:
            os.killpg(blockstored.pid, signal.SIGTERM)
        except:
            pass
        exit(1)
Esempio n. 26
0
  def dictionary(self, args, irc):
    '''(dict [word or phrase]) -- Lookup up a word using WordNet.
    '''
    if not args:
        return u'Missing word or phrase to look up'

    try:
        cmnd = 'sdcv -u WordNet -n --utf8-input --utf8-output'.split() + args
        o = [line.strip() for line in sp.check_output(cmnd).split('\n') if line]

        if len(o) == 1:
            return o[0]
        else:
            return u' '.join(o[3:])
    except:
        log.error('[Error]: dict {}'.format(sys.exc_info()[0]))
Esempio n. 27
0
	def handle_line(self, line, repo_lock=defer.DeferredLock()):
		try:
			line = line.decode('utf-8', 'ignore').strip()
			match = re.search(r'(^|\s+)!pq\s+(?P<link>\S+)(\s+::\S+|$)', line)
			if not match:
				log.noise('Non-patchbot line, ignoring: {}'.format(line.encode('utf-8', 'ignore')))
				defer.returnValue(None)
			link = match.group('link').encode('ascii')
			if not re.search('https?://', link, re.IGNORECASE):
				log.warn('Incorrect non-http link, skipping: {}'.format(link))
				defer.returnValue(None)
		except UnicodeError as err:
			log.warn('Failed to recode line ({!r}): {}'.format(line, err))
			defer.returnValue(None)

		## Grab the patch
		dst_base = '{}.patch'.format(sha1(link).hexdigest())
		dst_path = self.dst_path.child(dst_base)
		if dst_path.exists():
			log.debug( 'Patch already exists'
				' (file: {}, link: {}), skipping'.format(dst_path, link) )
			defer.returnValue(None)
		# Not via tmpfile to prevent multiple downloads of the same paste
		try: yield downloadPage(link, dst_path.open('wb'), timeout=120)
		except:
			if dst_path.exists(): dst_path.remove()
			raise

		## Commit into repo and push
		yield repo_lock.acquire()
		try:
			for cmd, check in [
					(['add', dst_base], True),
					(['commit', '-m', 'New patch: {}'.format(link)], False),
					(['push'], True) ]:
				out, err, code = yield getProcessOutputAndValue(
					'/usr/bin/git', cmd, path=self.dst_path.path )
				if check and code:
					log.error('\n'.join([
						'Failed to commit/push new patch into repo',
						'Command: {}'.format(cmd), 'Exit code:  {}'.format(code),
						'Stdout:\n  {}'.format('\n  '.join(out.splitlines())),
						'Stderr:\n  {}'.format('\n  '.join(err.splitlines())) ]))
					break
			else: log.debug('Successfully pushed paste: {}'.format(link))
		finally: repo_lock.release()
Esempio n. 28
0
 def setStunnedAddress(self, results):
     ''' Handle results of the rtp/rtcp STUN. We have to check that
         the results have the same IP and usable port numbers
     '''
     log.msg("got NAT mapping back! %r"%(results), system='rtp')
     rtpres, rtcpres = results
     if rtpres[0] != defer.SUCCESS or rtcpres[0] != defer.SUCCESS:
         # barf out.
         log.msg("uh oh, stun failed %r"%(results), system='rtp')
     else:
         # a=RTCP might help for wacked out RTCP/RTP pairings
         # format is something like "a=RTCP:AUDIO 16387"
         # See RFC 3605
         code1, rtp = rtpres
         code2, rtcp = rtcpres
         if rtp[0] != rtcp[0]:
             print "stun gave different IPs for rtp and rtcp", results
         # We _should_ try and see if we have working rtp and rtcp, but
         # this seems almost impossible with most firewalls. So just try
         # to get a working rtp port (an even port number is required).
         elif False and ((rtp[1] % 2) != 0):
             log.error("stun: unusable RTP/RTCP ports %r, retry #%d"%
                                         (results, self._stunAttempts),
                                         system='rtp')
             # XXX close connection, try again, tell user
             if self._stunAttempts > 8:
                 # XXX
                 print "Giving up. Made %d attempts to get a working port" \
                     % (self._stunAttempts)
             self._stunAttempts += 1
             defer.maybeDeferred(
                         self.rtpListener.stopListening).addCallback( \
                 lambda x:self.rtcpListener.stopListening()).addCallback( \
                 lambda x:self._socketCreationAttempt())
         else:
             # phew. working NAT
             log.msg("stun: sane NAT for RTP/RTCP; rtp addr: %s" \
                         % (rtp,), system='rtp')
             #Register address and port
             self._extIP, self._extRTPPort = rtp
             self._stunAttempts = 0
             d = self._socketCompleteDef
             del self._socketCompleteDef
             d.callback(self.cookie)
Esempio n. 29
0
 def setStunnedAddress(self, results):
     ''' Handle results of the rtp/rtcp STUN. We have to check that
         the results have the same IP and usable port numbers
     '''
     log.msg("got NAT mapping back! %r" % (results), system='rtp')
     rtpres, rtcpres = results
     if rtpres[0] != defer.SUCCESS or rtcpres[0] != defer.SUCCESS:
         # barf out.
         log.msg("uh oh, stun failed %r" % (results), system='rtp')
     else:
         # a=RTCP might help for wacked out RTCP/RTP pairings
         # format is something like "a=RTCP:AUDIO 16387"
         # See RFC 3605
         code1, rtp = rtpres
         code2, rtcp = rtcpres
         if rtp[0] != rtcp[0]:
             print "stun gave different IPs for rtp and rtcp", results
         # We _should_ try and see if we have working rtp and rtcp, but
         # this seems almost impossible with most firewalls. So just try
         # to get a working rtp port (an even port number is required).
         elif False and ((rtp[1] % 2) != 0):
             log.error("stun: unusable RTP/RTCP ports %r, retry #%d" %
                       (results, self._stunAttempts),
                       system='rtp')
             # XXX close connection, try again, tell user
             if self._stunAttempts > 8:
                 # XXX
                 print "Giving up. Made %d attempts to get a working port" \
                     % (self._stunAttempts)
             self._stunAttempts += 1
             defer.maybeDeferred(
                         self.rtpListener.stopListening).addCallback( \
                 lambda x:self.rtcpListener.stopListening()).addCallback( \
                 lambda x:self._socketCreationAttempt())
         else:
             # phew. working NAT
             log.msg("stun: sane NAT for RTP/RTCP; rtp addr: %s" \
                         % (rtp,), system='rtp')
             #Register address and port
             self._extIP, self._extRTPPort = rtp
             self._stunAttempts = 0
             d = self._socketCompleteDef
             del self._socketCompleteDef
             d.callback(self.cookie)
Esempio n. 30
0
 def consume(self, data):
     """
     We've monkey patched the consumer's `consume` method to point to this
     one intead. All incoming data are messages that need to be sent
     to TruTeq over USSD. Use `ssmi_client.send_ussd` to do that.
     """
     # default to ussd
     if data.get('type', 'ussd') == 'ussd':
         self.ssmi_client.send_ussd(
             str(data['msisdn']),    # str everything because the SSMIClient
             str(data['message']),   # isn't happy with Unicode
             str(data['ussd_type']))
     elif data['type'] == 'sms':
         self.ssmi_client.send_sms(
             str(data['msisdn']),
             str(data['message'])
         )
     else:
         log.error('Unknown TruTeq message type in %s' % data)
Esempio n. 31
0
 def __init__(self, url, filepath, callback=None):
     """
     @param url = resource to fetch
     @param filepath = string path of file to read
     @param callback = function to call with the filepath as arguement 
         after sucess response from url
     """
     if not os.path.exists(filepath):
         e = FileRouterException('{0} not found'.format(filepath))
         log.err(e)
         raise e
     with open(filepath) as ofile:
         try:
             self.data = json.loads(ofile.read())
         except Exception as e:
             log.error(e)
             raise e
     self.url = url
     self.file_callback = callback
Esempio n. 32
0
    def run(self):
        if not os.path.exists("dist/%s.app/" % config.appname):
            log.error("You have to run py2app first")
            return
        # getting Python.framework path
        cmd = "python-config --includes | awk -F'-I' '{print $2}' | sed 's/\/include.*$//'"
        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
        pythonFrameworkPath = process.stdout.read().strip()
        version = pythonFrameworkPath.split("/")[-1]

        # folder of the app bundle
        frameFolder = ("dist/%s.app/Contents/Frameworks/Python.framework/Versions/%s"
                       % (config.appname, version))

        # copy it if it does't exists
        if not os.path.exists(frameFolder):
            os.makedirs(frameFolder)
            os.system("cp %s/Python %s" % (pythonFrameworkPath, frameFolder))
            os.system("chmod +x dist/%s.app/Contents/Resources/contrib/tor" % config.appname)
Esempio n. 33
0
    def run(self):
        if not os.path.exists("dist/%s.app/" % config.appname):
            log.error("You have to run py2app first")
            return
        # getting Python.framework path
        cmd = "python-config --includes | awk -F'-I' '{print $2}' | sed 's/\/include.*$//'"
        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
        pythonFrameworkPath = process.stdout.read().strip()
        version = pythonFrameworkPath.split("/")[-1]

        # folder of the app bundle
        frameFolder = (
            "dist/%s.app/Contents/Frameworks/Python.framework/Versions/%s" %
            (config.appname, version))

        # copy it if it does't exists
        if not os.path.exists(frameFolder):
            os.makedirs(frameFolder)
            os.system("cp %s/Python %s" % (pythonFrameworkPath, frameFolder))
            os.system("chmod +x dist/%s.app/Contents/Resources/contrib/tor" %
                      config.appname)
Esempio n. 34
0
        def getTags(_file):

            try:

                tags = File(_file, easy=True)

                return {
                    'title': tags['TITLE'][0].encode('utf8'),
                    'artist': tags['ARTIST'][0].encode('utf8'),
                    'album': tags['ALBUM'][0].encode('utf8'),
                    'genre': tags['GENRE'][0].encode('utf8'),
                    'art': hasEmbeddedArt(_file)
                }

            except Exception as err:

                Logger.error('Error on file {} - {}'.format(
                    str(_file),
                    err)
                )

                return False
Esempio n. 35
0
def initalizeDBus():
    global flag
    if flag:
        return

    global dbus, bus, manager
    try:
        import dbus
        try:
            bus = dbus.SessionBus()
            manager = dbus.Interface(bus.get_object("org.bluez", "/"),
                "org.bluez.Manager")
        except:
            bus = dbus.SystemBus()
            manager = dbus.Interface(bus.get_object("org.bluez", "/"),
                "org.bluez.Manager")
        log.msg("DBus available")
        flag = True
    except Exception, err:
        log.msg("DBus unavailable")
        log.error(err)
        if isAndroid(): flag = True
Esempio n. 36
0
File: bluez.py Progetto: ygl-rg/AIRi
def initalizeDBus():
    global flag
    if flag:
        return

    global dbus, bus, manager
    try:
        import dbus
        try:
            bus = dbus.SessionBus()
            manager = dbus.Interface(bus.get_object("org.bluez", "/"),
                                     "org.bluez.Manager")
        except:
            bus = dbus.SystemBus()
            manager = dbus.Interface(bus.get_object("org.bluez", "/"),
                                     "org.bluez.Manager")
        log.msg("DBus available")
        flag = True
    except Exception, err:
        log.msg("DBus unavailable")
        log.error(err)
        if isAndroid(): flag = True
Esempio n. 37
0
 def dataReceived(self, data):
     if self.transport.protocol.state == txws.FRAMES:
         if not self._session:
             session_id = self.transport.protocol.location.rstrip('/').split('/')[-1]
             if not base.simple_id(session_id):
                 log.err("Invalid Session ID: %r"%( session_id, ))
                 self.write_error("session invalid")
                 self.transport.loseConnection()
                 return
             log.msg("New Connection on Session: %r"%( session_id, ))
             session = self.factory.server.session(session_id, create=False)
             if not session:
                 log.err("Unkown/no-auth session: %r"%( session_id, ))
                 self.write_error("session unknown")
                 self.transport.loseConnection()
                 return
             self._session = session
             self._session.protocols.append(self)
             self.write_welcome()
         else:
             session = self._session
         try:
             channel_id, data = data.split(',', 1)
         except ValueError:
             log.error('Mis-formatted request (no ,)')
             self.write_error('Missing comma in request')
             return 
         if not channel_id:
             """Protocol-level messages, currently nothing"""
             pass 
         elif not base.simple_id(channel_id):
             log.err("Invalid Channel ID: %r"%( channel_id, ))
             self.write_error("invalid channel")
             self.transport.loseConnection()
             return
         else:
             session.on_incoming(channel_id, data)
         self.ready = True
Esempio n. 38
0
    def __init__(self, target, listener = None, connectionLost = None):
        if SCOConnected(target):
            log.error("Can't connect twice to the same device")
            return

        self.sock = bluetooth.BluetoothSocket(bluetooth.SCO)
        try:
            self.sock.connect((target,))
        except Exception, err:
            log.error("Failed while trying to connect SCO")
            log.error(err)
            return
Esempio n. 39
0
File: bluez.py Progetto: ygl-rg/AIRi
    def __init__(self, target, listener=None, connectionLost=None):
        if SCOConnected(target):
            log.error("Can't connect twice to the same device")
            return

        self.sock = bluetooth.BluetoothSocket(bluetooth.SCO)
        try:
            self.sock.connect((target, ))
        except Exception, err:
            log.error("Failed while trying to connect SCO")
            log.error(err)
            return
Esempio n. 40
0
    def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None:
        # logfile path relative to config dir if not abs path
        log_filename = logfile.get()
        if log_filename.strip():  # catches empty filename
            if not os.path.isabs(log_filename):
                log_filename = os.path.join(config.config_dir, log_filename)
            ensure_dir_exists(log_filename)
            if logging_rotate_daily.get():
                logging_file = DailyLogFile(log_filename, '.')
            else:
                logging_file = open(log_filename, 'a')
            globalLogPublisher.addObserver(textFileLogObserver(logging_file))
            globalLogPublisher.addObserver(textFileLogObserver(sys.stderr))
            log.info('piqueserver started on %s' % time.strftime('%c'))

        self.config = config_dict
        if random_rotation:
            self.map_rotator_type = random_choice_cycle
        else:
            self.map_rotator_type = itertools.cycle  # pylint: disable=redefined-variable-type
        self.default_time_limit = default_time_limit.get()
        self.default_cap_limit = cap_limit.get()
        self.advance_on_win = int(advance_on_win.get())
        self.win_count = itertools.count(1)
        self.bans = NetworkDict()

        # attempt to load a saved bans list
        try:
            with open(os.path.join(config.config_dir, bans_file.get()),
                      'r') as f:
                self.bans.read_list(json.load(f))
            log.debug("loaded {count} bans", count=len(self.bans))
        except FileNotFoundError:
            log.debug("skip loading bans: file unavailable",
                      count=len(self.bans))
        except IOError as e:
            log.error('Could not read bans.txt: {}'.format(e))
        except ValueError as e:
            log.error('Could not parse bans.txt: {}'.format(e))

        self.hard_bans = set()  # possible DDoS'ers are added here
        self.player_memory = deque(maxlen=100)
        if len(self.name) > MAX_SERVER_NAME_SIZE:
            log.warn('(server name too long; it will be truncated to "%s")' %
                     (self.name[:MAX_SERVER_NAME_SIZE]))
        self.respawn_time = respawn_time_option.get()
        self.respawn_waves = respawn_waves.get()
        if game_mode.get() == 'ctf':
            self.game_mode = CTF_MODE
        elif game_mode.get() == 'tc':
            self.game_mode = TC_MODE
        elif self.game_mode is None:
            raise NotImplementedError('invalid game mode: %s' % game_mode)
        self.game_mode_name = game_mode.get().split('.')[-1]
        self.team1_name = team1_name.get()
        self.team2_name = team2_name.get()
        self.team1_color = tuple(team1_color.get())
        self.team2_color = tuple(team2_color.get())
        self.friendly_fire = friendly_fire.get()
        self.friendly_fire_on_grief = friendly_fire_on_grief.get()
        self.friendly_fire_time = grief_friendly_fire_time.get()
        self.spade_teamkills_on_grief = spade_teamkills_on_grief.get()
        self.fall_damage = fall_damage.get()
        self.teamswitch_interval = teamswitch_interval.get()
        self.teamswitch_allowed = teamswitch_allowed.get()
        self.max_players = max_players.get()
        self.melee_damage = melee_damage.get()
        self.max_connections_per_ip = max_connections_per_ip.get()
        self.passwords = passwords.get()
        self.server_prefix = server_prefix.get()
        self.time_announcements = time_announcements.get()
        self.balanced_teams = balanced_teams.get()
        self.login_retries = login_retries.get()

        # voting configuration
        self.default_ban_time = default_ban_duration.get()

        self.speedhack_detect = speedhack_detect.get()
        if user_blocks_only.get():
            self.user_blocks = set()
        self.set_god_build = set_god_build.get()
        self.debug_log = debug_log_enabled.get()
        if self.debug_log:
            # TODO: make this configurable
            pyspades.debug.open_debug_log(
                os.path.join(config.config_dir, 'debug.log'))
        if ssh_enabled.get():
            from piqueserver.ssh import RemoteConsole
            self.remote_console = RemoteConsole(self)
        irc = irc_options.get()
        if irc.get('enabled', False):
            from piqueserver.irc import IRCRelay
            self.irc_relay = IRCRelay(self, irc)
        if status_server_enabled.get():
            from piqueserver.statusserver import StatusServerFactory
            self.status_server = StatusServerFactory(self)
        if ban_publish.get():
            from piqueserver.banpublish import PublishServer
            self.ban_publish = PublishServer(self, ban_publish_port.get())
        if bans_urls.get():
            from piqueserver import bansubscribe
            self.ban_manager = bansubscribe.BanManager(self)
        self.start_time = reactor.seconds()
        self.end_calls = []
        # TODO: why is this here?
        create_console(self)

        for user_type, func_names in rights.get().items():
            for func_name in func_names:
                commands.add_rights(user_type, func_name)

        port = self.port = port_option.get()
        ServerProtocol.__init__(self, port, interface)
        self.host.intercept = self.receive_callback
        try:
            self.set_map_rotation(self.config['rotation'])
        except MapNotFound as e:
            log.critical('Invalid map in map rotation (%s), exiting.' % e.map)
            raise SystemExit

        self.update_format()
        self.tip_frequency = tip_frequency.get()
        if self.tips is not None and self.tip_frequency > 0:
            reactor.callLater(self.tip_frequency * 60, self.send_tip)

        self.master = register_master_option.get()
        self.set_master()

        self.http_agent = web_client.Agent(reactor)

        ip_getter = ip_getter_option.get()
        if ip_getter:
            self.get_external_ip(ip_getter)
Esempio n. 41
0
def run() -> None:
    """
    runs the server
    """

    # apply scripts

    protocol_class = FeatureProtocol
    connection_class = FeatureConnection

    script_objects = []
    script_names = scripts_option.get()
    script_dir = os.path.join(config.config_dir, 'scripts/')

    for script in script_names[:]:
        try:
            # this finds and loads scripts directly from the script dir
            # no need for messing with sys.path
            f, filename, desc = imp.find_module(script, [script_dir])
            module = imp.load_module('piqueserver_script_namespace_' + script,
                                     f, filename, desc)
            script_objects.append(module)
        except ImportError as e:
            # warning: this also catches import errors from inside the script
            # module it tried to load
            try:
                module = importlib.import_module(script)
                script_objects.append(module)
            except ImportError as e:
                log.error("(script '{}' not found: {!r})".format(script, e))
                script_names.remove(script)

    for script in script_objects:
        protocol_class, connection_class = script.apply_script(
            protocol_class, connection_class, config.get_dict())

    # apply the game_mode script
    if game_mode.get() not in ('ctf', 'tc'):
        # must be a script with this game mode
        module = None
        try:
            game_mode_dir = os.path.join(config.config_dir, 'game_modes/')
            f, filename, desc = imp.find_module(game_mode.get(),
                                                [game_mode_dir])
            module = imp.load_module(
                'piqueserver_gamemode_namespace_' + game_mode.get(), f,
                filename, desc)
        except ImportError as e:
            try:
                module = importlib.import_module(game_mode.get())
            except ImportError as e:
                log.error("(game_mode '%s' not found: %r)" %
                          (game_mode.get(), e))

        if module:
            protocol_class, connection_class = module.apply_script(
                protocol_class, connection_class, config.get_dict())

    protocol_class.connection_class = connection_class

    interface = network_interface.get().encode('utf-8')

    # instantiate the protocol class once. It will set timers and hooks to keep
    # itself running once we start the reactor
    protocol_class(interface, config.get_dict())

    log.debug('Checking for unregistered config items...')
    unused = config.check_unused()
    if unused:
        log.warn('The following config items are not used:')
        pprint(unused)

    log.info('Started server...')

    profile = logging_profile_option.get()
    if profile:
        import cProfile
        cProfile.runctx('reactor.run()', None, globals())
    else:
        reactor.run()
Esempio n. 42
0
import web
import sys, os
from twisted.python import log
from twisted.internet import defer, reactor
                                                                                

def main(config_file):
    log.startLogging(sys.stdout)                                                
    application = web.Application(config_file)

    port = os.environ.get("PORT", 8888)                               
    reactor.listenTCP(int(port), application)
    reactor.run()                                                               
                                                                                
if __name__ == "__main__":                                                      
    if len(sys.argv) > 1:
        main(sys.argv[1])
    else:
        log.error("no config file given")
        sys.exit(-1)
Esempio n. 43
0
 def connectionLost(self, reason):
     if reason.type != ConnectionLost:
         log.debug('Disconnect from %s' % self.transport.getPeer())
     else:
         log.error('Disconnect from %s: %s' %
                   (self.transport.getPeer(), reason))
Esempio n. 44
0
 def openFailed(self, reason):
     log.error("Open failed due to %r", reason)
     if self.start_defer:
         self.start_defer.errback(self)
Esempio n. 45
0
    def run(self):
        def safe_unicode(obj, *args):
            try:
                return unicode(obj, *args)
            except UnicodeDecodeError:
                # obj is byte string
                ascii_text = str(obj).encode('string_escape')
                return unicode(ascii_text)

        def get_appname(rts):
            rts = rts.lower()
            for name in ('paypage', 'ccentry', 'mis'):
                if name in rts:
                    return name
            return 'UnknownEntryPoint'

        log.debug("Execute model checks and execution of loop")
        # process the accumulated data into the model
        bl_trace_id = []
        traces, last_trace_end_str = {}, None
        trace_id = self.guid
        start_trace = False
        local = []
        latest_datetime = None
        try:
            for line in self.accumulator:
                if not start_trace:
                    if 'TRACE START' == line[:11]:
                        start_trace = True
                        latest_datetime = extract_datetime(line)
                        local = []
                else:
                    if 'TRACE END' in line[:9]:
                        appname = get_appname(local[0])
                        trace_id = time.strftime(
                            "%Y-%m-%d_%H-%M-%S",
                            latest_datetime) + '_' + appname + '_' + str(
                                hex(hash(repr(local))))
                        if trace_id not in bl_trace_id:
                            bl_trace_id.append(trace_id)
                            last_trace_end_str = line
                            start_trace = False
                            traces[trace_id] = local

                            if trace_id not in self.cache:
                                self.cache[trace_id] = local

                        local = []
                        start_trace = False
                    else:
                        local.append(line)

            # keep only what hasn't been processed yet
            if last_trace_end_str in self.accumulator:
                self.accumulator = self.accumulator[self.accumulator.
                                                    index(last_trace_end_str):]

            log.debug("Process all %d traces available" % len(traces))
            for trace_id in traces:
                self.processTrace(traces[trace_id], trace_id)
                try:
                    o = open('./cache/' + trace_id + '.html', 'w')

                    o.write(
                        pygment_content("<?php\n" + '\n'.join([
                            safe_unicode(rts) for rts in self.cache[trace_id]
                            if not has_blacklisted_function(rts)
                        ]) + "\n?>"))
                    o.close()
                except IOError, error:
                    log.error("Exception writing %s (error:%s)" %
                              ('./cache/' + trace_id + '.html', error))

            # generate the JSON file for frontend to consume
            self.update_JSON()
Esempio n. 46
0
 def failure(self, format, failure, *args, **kwargs):
     log.error(failure, format.format(args, **kwargs))
Esempio n. 47
0
 def log_error(err):
     log.error(err.getErrorMessage())
Esempio n. 48
0
class Modeler(object):
    def __init__(self):
        self.times = []
        self.guid = 1
        self.cuid = 1
        self.g = digraph()
        self.context = {}
        self.rtids = {}
        self.node_location = {}
        self.accumulator = []
        self.trace_ids = []
        self.cache = {}

    def feedAccumulator(self, data):
        self.accumulator.append(data)

    @staticmethod
    def __node_uid(trace_id, node_str, cur_index=''):
        return hash(str(trace_id) + node_str + str(cur_index))

    @staticmethod
    def add_node(g,
                 nid,
                 location=None,
                 linenumber=None,
                 context_id=None,
                 trace_id=None,
                 method=None,
                 data=None,
                 _type=None,
                 _id=None):
        if not g.has_node(nid):
            g.add_node(nid)
            g.node_attr[nid] = {
                'return': '',
                'context_id': context_id,
                'trace_id': trace_id,
                'location': location,
                'linenumber': linenumber,
                'method': method,
                'data': data,
                'type': _type,
                'rtid': False,
                'sequence': _id
            }

    @staticmethod
    def add_edge(g, e):
        if not g.has_edge(e):
            g.add_edge(e)
        else:
            g.set_edge_weight(e, g.edge_weight(e) + 1)

    def update_rtid(self, lst):
        for e in lst:
            if e not in self.rtids:
                self.rtids[e] = re.compile(e, re.I)

    @staticmethod
    def regexp_interesect(data, dct):
        if not isinstance(data, str):
            data = str(repr(data))
        for reg in dct:
            if dct[reg].search(data):
                return True
        return False

    def update_model_taints(self):
        # get latest list of rtids
        self.update_rtid(db_list_rtid())

        # make sure we don't iterate if no data to check with
        if 1 > len(self.rtids):
            return

        # loop over the entire graph and update the 'rtid' node attr
        for n in self.g:
            if Modeler.regexp_interesect(
                    self.g.node_attr[n]['method'],
                    self.rtids) or Modeler.regexp_interesect(
                        self.g.node_attr[n]['data'],
                        self.rtids) or Modeler.regexp_interesect(
                            self.g.node_attr[n]['return'], self.rtids):
                # update taint
                self.g.node_attr[n]['rtid'] = True

    def processTrace(self, trace, trace_id):
        callstack = Stack()
        prev_node, cur_node = None, None
        pre_l = None
        cur_i = 0

        if trace_id not in self.trace_ids:
            self.trace_ids.append(trace_id)

        for l in trace:
            self.guid += 1
            cur_i += 1
            suc, trace, token = get_cleaned_trace_node(l)
            if not suc:
                continue
            else:
                trace_info = extract_information(token, trace)
                if trace_info and isinstance(trace_info, dict):
                    _type = trace_info['type']

                    if _type == 'main':
                        main_nuid = Modeler.__node_uid(trace_id, l)
                        Modeler.add_node(self.g, main_nuid,
                                         trace_info['filename'],
                                         trace_info['line'], None, trace_id,
                                         'main', self.guid)
                        callstack.push(main_nuid)
                    elif _type == 'return':
                        # only keep the latest  value
                        if not callstack.empty():
                            prev_node = callstack.pop()
                            self.g.node_attr[prev_node]['return'] = trace_info[
                                'value']

                    elif _type == 'call':
                        cur_node = Modeler.__node_uid(trace_id, l)
                        Modeler.add_node(self.g, cur_node,
                                         trace_info['filename'],
                                         trace_info['line'], None, trace_id,
                                         trace_info['function'],
                                         trace_info['parameters'], 'call',
                                         self.guid)
                        if not callstack.empty():
                            prev_node = callstack.top()
                            edge = (prev_node, cur_node)
                            Modeler.add_edge(self.g, edge)
                            # set label to the edge as parameters
                            self.g.set_edge_label(
                                edge, ','.join(trace_info['parameters']))
                        callstack.push(cur_node)

                    elif _type == 'assignment':
                        cur_node = Modeler.__node_uid(trace_id, l)
                        Modeler.add_node(self.g, cur_node,
                                         trace_info['filename'],
                                         trace_info['line'], None, trace_id,
                                         trace_info['variable'],
                                         trace_info['value'], 'assignment',
                                         self.guid)

                        if not callstack.empty():
                            prev_node = callstack.top()
                            edge = (prev_node, cur_node)
                            Modeler.add_edge(self.g, edge)

                            # set label to the edge as parameters
                            self.g.set_edge_label(edge, trace_info['value'])
                        # callstack.push(cur_node)

        self.update_model_taints()
        # self.write_dot_slice(trace_id)

    def update_JSON(self):
        log.debug("update_JSON")
        # only output simple paths and possible ways to investigate
        jsonified_data = {
            'injected': self.rtids.keys(),
            'hash': '',
            'trace': {},
            'knowledge': {
                'function': {},
                'file': {}
            }
        }

        def safe_unicode(obj, *args):
            try:
                return unicode(obj, *args)
            except UnicodeDecodeError:
                # obj is byte string
                ascii_text = str(obj).encode('string_escape')
                return unicode(ascii_text)

        def lmt_size(rts):
            length = len(rts)
            if 256 >= length:
                return rts
            return rts[:256]

        # always output the top 50 traces
        sorted_traces = self.trace_ids
        sorted_traces.sort(reverse=True)

        for trace_id in sorted_traces[:min(100, len(sorted_traces))]:
            if trace_id in jsonified_data['trace']:
                continue

            jsonified_data['trace'][trace_id] = {
                'callchain': [],
                'function': {},
                'file': {},
            }

            tlst_nodes = {}
            for n in self.g:
                if trace_id == self.g.node_attr[n]['trace_id']:
                    tlst_nodes[self.g.node_attr[n]['sequence']] = n
            slst_nodes = [tlst_nodes[k] for k in sorted(tlst_nodes.keys())]

            if 0 == len(slst_nodes):
                del jsonified_data['trace'][trace_id]
            else:
                for n in slst_nodes:
                    if self.g.node_attr[n]['type'] not in ('call', 'main'):
                        continue

                    if self.g.node_attr[n]['rtid']:
                        nd = self.g.node_attr[n]
                        method = safe_unicode(nd['method'])
                        data = safe_unicode(nd['data'])
                        ret = safe_unicode(nd['return'])

                        if has_blacklisted_function(nd['location']):
                            continue

                        jsonified_data['trace'][trace_id]['callchain'].append([
                            self.g.node_attr[n]['type'], nd['location'],
                            nd['linenumber'], method,
                            lmt_size(data),
                            lmt_size(ret)
                        ])
                        """
						if method not in jsonified_data['trace'][trace_id]['function']:
							jsonified_data['trace'][trace_id]['function'][method] = []
						jsonified_data['trace'][trace_id]['function'][method].append(data)

						if method not in jsonified_data['knowledge']['function']:
							jsonified_data['knowledge']['function'][method] = []
						if data not in jsonified_data['knowledge']['function'][method]:
							jsonified_data['knowledge']['function'][method] = data

						if nd['location'] not in jsonified_data['trace'][trace_id]['file']:
							jsonified_data['trace'][trace_id]['file'][nd['location']] = []
						jsonified_data['trace'][trace_id]['file'][nd['location']].append(nd['linenumber'])
						"""

                if 0 == len(jsonified_data['trace'][trace_id]['callchain']):
                    del jsonified_data['trace'][trace_id]

        jsonified_data['hash'] = hash(
            str(
                repr(jsonified_data['trace']) +
                repr(jsonified_data['injected'])))

        o = open('assmnt-data.json', 'w')
        o.write(
            json.dumps(jsonified_data,
                       ensure_ascii=False,
                       separators=(',', ':')))
        o.close()

    def write_dot_slice(self, trace_id):
        new_g = digraph()
        new_g.add_graph(self.g)

        # create new graph with only the CWE instances
        for n in new_g:
            if self.g.node_attr[n]['trace_id'] != trace_id:
                new_g.del_node(n)
                continue

        fname = './graphs/%s-trace' % str(trace_id)
        self.write_dot(new_g, fname)
        print "Generated: %s" % fname

    def write_dot(self, g, fname):
        if 1 > len(g):
            return

        def pen_width(num, local_max):
            max_penwidth = 7
            val = int(
                math.ceil(float(num) / float(local_max) * float(max_penwidth)))
            return max(1, min(val, max_penwidth))

        def node_call_labels(dct):
            """
				<f0> Function Name| $value1 | ... | return values"
			"""
            lbl_str = "%s:%s |" % (dct['location'], dct['linenumber'])
            lbl_str += "<f0> " + xml_entities(dct['method']) + "()"
            for e in dct['data']:
                lbl_str += ' | ' + xml_entities(e)
            lbl_str += '| <f1> T_RET: ' + xml_entities(dct['return'])
            return lbl_str

        # generate DOT file with custom information (source, sink, and custom colors)
        dstr = """digraph runtime {
			graph [rankdir = "LR"];
			node [fontsize = 12];
		"""
        for n in g:
            node_type = 'shape=ellipse'
            if self.g.node_attr[n]['type'] == 'call':
                node_type = 'shape=record'
                node_type += ", label=\"%s\"" % node_call_labels(
                    self.g.node_attr[n])
            elif self.g.node_attr[n]['type'] == 'assignment':
                node_type = 'shape=record, style="filled,bold" penwidth=1'
                node_type += ",label=\"%s=%s\"" % (
                    xml_entities(self.g.node_attr[n]['method']),
                    xml_entities(self.g.node_attr[n]['data']))
            else:
                if self.g.node_attr[n]['method'] == 'return':
                    node_type += ", label=\"T_RETURN:%s\"" % self.g.node_attr[
                        n]['data'].replace('"', '\\"')
                else:
                    node_type += ", label=\"%s\"" % self.g.node_attr[n][
                        'method']

            if self.g.node_attr[n]['rtid']:
                node_type += ', fillcolor="#AA0114"'

            #if 'source' in g.node_attr[n]['nodetype'] or 'sink' in g.node_attr[n]['nodetype']:
            #	node_type = 'shape=box, style=filled, fillcolor="%s"' % color_node(g.node_attr[n]['nodetype'])
            # get the size of the pen based on the number of GUID
            #node_type += ', penwidth=%d' % int(pen_width(len(g.node_attr[n]['guid']), max_guid))
            dstr += '  "%s" [%s];\n' % (n, node_type)

        for e in g.edges():
            n1, n2 = e
            node_type = ""
            #l_e_prop = self.g.edge_label(e).split(',')
            if self.g.node_attr[n1]['rtid'] and self.g.node_attr[n2]['rtid']:
                node_type += 'color="#AA0114"'
            else:
                node_type += 'color="black"'
            node_type += ', penwidth=%d' % pen_width(g.edge_weight(e), 10)

            #node_type += ', label="%s"' %  ','.join(l_e_prop).replace('"', '\\"')
            addon_n1 = ':f0' if self.g.node_attr[n1]['type'] in (
                'call', 'return') else ''
            addon_n2 = ':f0' if self.g.node_attr[n2]['type'] in (
                'call', 'return') else ''
            dstr += ' "%s"%s -> "%s"%s [%s];\n' % (n1, addon_n1, n2, addon_n2,
                                                   node_type)
        dstr += '}'

        dot_fname = fname + '.dot'
        o = open(dot_fname, 'w')
        o.write(dstr)
        o.close()

        #cmd_line = "dot %s -Tsvg -O -Kdot -x" % (dot_fname)
        # os.system(cmd_line)
        syscall("dot", [dot_fname, "-Tsvg", "-O", "-Kdot", "-x"])

    def export(self):
        return
        if 0 < len(self.g):
            self.write_dot(self.g, './graphs/global_file')

    def run(self):
        def safe_unicode(obj, *args):
            try:
                return unicode(obj, *args)
            except UnicodeDecodeError:
                # obj is byte string
                ascii_text = str(obj).encode('string_escape')
                return unicode(ascii_text)

        def get_appname(rts):
            rts = rts.lower()
            for name in ('paypage', 'ccentry', 'mis'):
                if name in rts:
                    return name
            return 'UnknownEntryPoint'

        log.debug("Execute model checks and execution of loop")
        # process the accumulated data into the model
        bl_trace_id = []
        traces, last_trace_end_str = {}, None
        trace_id = self.guid
        start_trace = False
        local = []
        latest_datetime = None
        try:
            for line in self.accumulator:
                if not start_trace:
                    if 'TRACE START' == line[:11]:
                        start_trace = True
                        latest_datetime = extract_datetime(line)
                        local = []
                else:
                    if 'TRACE END' in line[:9]:
                        appname = get_appname(local[0])
                        trace_id = time.strftime(
                            "%Y-%m-%d_%H-%M-%S",
                            latest_datetime) + '_' + appname + '_' + str(
                                hex(hash(repr(local))))
                        if trace_id not in bl_trace_id:
                            bl_trace_id.append(trace_id)
                            last_trace_end_str = line
                            start_trace = False
                            traces[trace_id] = local

                            if trace_id not in self.cache:
                                self.cache[trace_id] = local

                        local = []
                        start_trace = False
                    else:
                        local.append(line)

            # keep only what hasn't been processed yet
            if last_trace_end_str in self.accumulator:
                self.accumulator = self.accumulator[self.accumulator.
                                                    index(last_trace_end_str):]

            log.debug("Process all %d traces available" % len(traces))
            for trace_id in traces:
                self.processTrace(traces[trace_id], trace_id)
                try:
                    o = open('./cache/' + trace_id + '.html', 'w')

                    o.write(
                        pygment_content("<?php\n" + '\n'.join([
                            safe_unicode(rts) for rts in self.cache[trace_id]
                            if not has_blacklisted_function(rts)
                        ]) + "\n?>"))
                    o.close()
                except IOError, error:
                    log.error("Exception writing %s (error:%s)" %
                              ('./cache/' + trace_id + '.html', error))

            # generate the JSON file for frontend to consume
            self.update_JSON()
        except Exception, error:
            log.error("run- %s" % error)
            sys.exit()
Esempio n. 49
0
File: bluez.py Progetto: ygl-rg/AIRi
 def connectionLost(self, reason):
     if callable(self._connectionLost):
         try:
             self._connectionLost(self)
         except Exception, err:
             log.error(err)
Esempio n. 50
0
    def _process_changes(self, newRev, branch):
        """
        Read changes since last change.

        - Read list of commit hashes.
        - Extract details from each commit.
        - Add changes to database.
        """

        # initial run, don't parse all history
        if not self.lastRev:
            return
        rebuild = False
        if newRev in itervalues(self.lastRev):
            if self.buildPushesWithNoCommits and \
               branch not in iterkeys(self.lastRev):
                # we know the newRev but not for this branch
                log.msg('gitpoller: rebuilding %s for new branch "%s"' %
                        (newRev, branch))
                rebuild = True

        # get the change list
        revListArgs = ([r'--format=%H', r'%s' % newRev] +
                       [r'^%s' % rev.encode('ascii', 'ignore')
                        for rev in itervalues(self.lastRev)] +
                       [r'--'])
        self.changeCount = 0
        results = yield self._dovccmd('log', revListArgs, path=self.workdir)

        # process oldest change first
        revList = results.split()
        revList.reverse()

        if rebuild and len(revList) == 0:
            revList = [newRev]

        self.changeCount = len(revList)
        self.lastRev[branch] = newRev

        if self.changeCount:
            log.msg('gitpoller: processing %d changes: %s from "%s" branch "%s"'
                    % (self.changeCount, revList, self.repourl, branch))

        for rev in revList:
            dl = defer.DeferredList([
                self._get_commit_timestamp(rev),
                self._get_commit_author(rev),
                self._get_commit_files(rev),
                self._get_commit_comments(rev),
            ], consumeErrors=True)

            results = yield dl

            # check for failures
            failures = [r[1] for r in results if not r[0]]
            if failures:
                for failure in failures:
                    log.error(
                        failure, "while processing changes for {} {}".format(newRev, branch))
                # just fail on the first error; they're probably all related!
                failures[0].raiseException()

            timestamp, author, files, comments = [r[1] for r in results]

            yield self.master.data.updates.addChange(
                author=author, revision=ascii2unicode(rev), files=files,
                comments=comments, when_timestamp=timestamp,
                branch=ascii2unicode(self._removeHeads(branch)),
                project=self.project, repository=ascii2unicode(self.repourl),
                category=self.category, src=u'git')
Esempio n. 51
0
def validate_settings(settings):
    if not settings:
        log.error("No settings at all?")
        return False
    if not settings['nickname'] or not isinstance(settings['nickname'], str):
        log.error("Settings is missing nickname field")
        return False
    if not settings['realname'] or not isinstance(settings['realname'], str):
        log.error("Settings is missing realname field")
        return False
    if not settings['username'] or not isinstance(settings['username'], str):
        log.error("Settings is missing username field")
        return False
    if not settings['servers']:
        log.error("Settings is missing servers?")
        return False
    for server in settings['servers']:
        if not server:
            log.error("Got empty server in server settings")
            return False
        if not server['name'] or not isinstance(server['name'], str):
            log.error("Missing name for server")
            return False
        if not server['host'] or not isinstance(server['host'], str):
            log.error("Got misconfigured host setting")
            return False
        if not server['port'] or not isinstance(server['port'], int):
            log.error("Got misconfigured ip setting")
            return False
    if not settings['plugins']:
        log.error("Got no plugin, bot will be useless?")
        return False
    for plugin in settings['plugins']:
        if not plugin:
            log.error("Settings found empty plugin")
            return False
        if not plugin['name'] or not isinstance(plugin['name'], str):
            log.error("Settings got plugin with no name")
            return False
    return True
Esempio n. 52
0
 def openFailed(self, reason):
     log.error("Open failed due to %r", reason)
     if self.start_defer:
         self.start_defer.errback(self)
Esempio n. 53
0
 def eb_produce(f):
     log.error("Produce failed",
               exc_info=(f.type, f.value, f.getTracebackObject()))
Esempio n. 54
0
def replaceTwistedLoggers():
    """
    Visit all Python modules that have been loaded and:

     - replace L{twisted.python.log} with a L{LegacyLogger}

     - replace L{twisted.python.log.msg} with a L{LegacyLogger}'s C{msg}

     - replace L{twisted.python.log.err} with a L{LegacyLogger}'s C{err}
    """
    log = Logger()

    for moduleName, module in sys.modules.iteritems():
        # Oddly, this happens
        if module is None:
            continue

        # Don't patch Twisted's logging module
        if module in (twisted.python, twisted.python.log):
            continue

        # Don't patch this module
        if moduleName is __name__:
            continue

        try:
            for name, obj in module.__dict__.iteritems():
                try:
                    newLogger = Logger(namespace=module.__name__)
                except AttributeError:
                    # Can't look up __name__.  A hack in the "six" module causes
                    # this.  Skip the module.
                    # See https://trac.calendarserver.org/ticket/832
                    continue

                legacyLogger = LegacyLogger(logger=newLogger)

                if obj is twisted.python.log:
                    log.info("Replacing Twisted log module object {0} in {1}".
                             format(name, module.__name__))
                    setattr(module, name, legacyLogger)

                elif obj is twisted.python.log.msg:
                    log.info(
                        "Replacing Twisted log.msg object {0} in {1}".format(
                            name, module.__name__))
                    setattr(module, name, legacyLogger.msg)

                elif obj is twisted.python.log.err:
                    log.info(
                        "Replacing Twisted log.err object {0} in {1}".format(
                            name, module.__name__))
                    setattr(module, name, legacyLogger.err)
        except RuntimeError as e:
            # Python could use more specific exceptions, eh.
            # What we mean to catch is:
            # RuntimeError: dictionary changed size during iteration
            log.error(
                "Unable to replace twisted loggers for module {module}: "
                "{error}",
                module=module,
                error=e)
Esempio n. 55
0
 def _errback(self, reason):
     log.error("Processing request error: %s" % reason)
     self._send_action('DUNNO')
Esempio n. 56
0
 def connection_failed(self, reason):
     log.error("Connection failed: %s" % (reason, ))
     self.defer = None
     self.secs_to_wait = 10 if self.secs_to_wait >= 10 else self.secs_to_wait * 2
     self.connection_lost()