def do_broker_callback_tasks(self, queue): while True: try: msg = yield queue.get() log.msg("DEBUG: do_broker_callback_tasks: got msg %s" % msg) unpacked_msg = self.service.unpack(msg.content.body) log.msg("DEBUG: do_broker_callback_tasks: unpacked_msg is %s" % str(unpacked_msg)) corr_id = msg.content.properties['correlation id'] if corr_id not in self.broker_callback_map: log.err("do_broker_callback_tasks: invalid corr_id %s" % corr_id) continue d, f_name, exchange = self.broker_callback_map[corr_id] if self.is_broker_error(unpacked_msg): if unpacked_msg["err_id"] == error.ERROR_INTERNAL_SERVICE_ERROR: d.errback(BrokerError(unpacked_msg["err_id"], unpacked_msg["err_msg"], f_name=f_name, exchange=exchange)) else: d.errback(error.EInternalError(unpacked_msg["err_id"])) else: d.callback(unpacked_msg) del self.broker_callback_map[corr_id] except Closed, e_inst: log.msg("do_broker_callback_tasks: queue is closed: %s" % str(e_inst)) break except Exception: log.error(str(msg)) formatted_traceback = traceback.format_exc() log.error(formatted_traceback)
def requestAvatarId(self, c): def cbRequest(response): log.msg('cbRequest Got Response code: %s'%response.code) for s in response.headers.getRawHeaders("Set-Cookie", ()): (key, value) = s.split("=") if key == "session_cookie": self.env["BSC_SESSION_COOKIE"] = value if response.code == 200: return defer.succeed(c.username) else: return defer.fail(error.UnauthorizedLogin()) def cbError(reason): print "+++ error during http request +++" reason.printTraceback() return defer.fail(error.UnauthorizedLogin(reason.getErrorMessage())) headers = {'Content-type': [ 'application/json' ]} req = {'user': c.username, 'password' : c.password } serialized_payload = json.dumps(req) log.msg("Requesting Payload %s" % serialized_payload) try: d = agent.request('POST', self.url, Headers(headers), StringProducer(serialized_payload)) d.addCallbacks(cbRequest) d.addErrback(cbError) except Exception as e: log.error("Auth failed",e) return defer.fail(error.UnauthorizedLogin()) log.msg("Returning deferred") return d
def refresh_mirror( working_dir, first_block, last_block ): """ Refresh the mirror: * process all new invalidations * grab and mirror any new profiles from the DHT This gets called by Twisted every time there ought to be a new block. """ from twisted.python import log from kademlia.network import Server # make soure our bitcoind cached block index is up-to-speed nameop_sequence = sync_blockchain( working_dir, first_block, last_block ) if nameop_sequence is None: log.error("sync_blockchain(%s-%s) failed" % (first_block, last_block)) return None # synchronize name registrations... server = Server() server.listen( dht.plugin.DHT_SERVER_PORT ) server.bootstrap( dht.plugin.DEFAULT_DHT_SERVERS ).addCallback( connect_done, server ) pass
def onMessage(self, msg): log.debug(u"Received %s message from %s: %s" % (msg['type'], msg['from'], msg.body)) if msg.x and msg.x.defaultUri == 'jabber:x:delay': log.debug(u"Ignoring delayed message") return if msg.body is None: log.debug(u'Ignoring empty message') self.chat(msg['from'], "Use $start to create menu") return try: if msg['type'] == 'error': log.debug("Error message received from %s: %s" % (msg['from'], msg.body)) return elif msg['type'] == 'chat': if self.message_handler: self.message_handler.answer( transport = self.chat, recipient = msg['from'], message = msg.body) else: self.chat(msg['from'], "Message received sir!") except Exception, e: log.error("Some error occurred: %s" % e)
def answer(self, transport, recipient, message): user = self.auth.authenticate(recipient) if user: session = USSDSession.objects.recent(user) if session: log.debug("User: %s USSD Session: %s" % (user, session)) msg = str(message) if msg == "$start": response = handle_start(session) elif msg == "$end": response = handle_end(session) elif msg == "$error": import pdb pdb.set_trace() else: if session.current_menu and not session.current_menu.is_finished(): response = handle_session(session, msg) else: response = "Use $start to create menu" if recipient and response: transport(recipient, response) else: log.error("Either recipient (%s) or response (%s) is not valid!" % (recipient, response)) return else: transport(recipient, "Error: No USSD Session!") else: transport(recipient, "Error: No User!")
def joinNetwork(self, knownNodeAddresses=None): """ Causes the Node to join the Kademlia network; normally, this should be called before any other DHT operations. @param knownNodeAddresses: A sequence of tuples containing IP address information for existing nodes on the Kademlia network, in the format: C{(<ip address>, (udp port>)} @type knownNodeAddresses: tuple """ # Prepare the underlying Kademlia protocol if self.port is not None: try: self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol) except error.CannotListenError as e: import traceback log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) raise ValueError("%s lbrynet may already be running." % str(e)) #IGNORE:E1101 # Create temporary contact information for the list of addresses of known nodes if knownNodeAddresses != None: bootstrapContacts = [] for address, port in knownNodeAddresses: contact = Contact(self._generateID(), address, port, self._protocol) bootstrapContacts.append(contact) else: bootstrapContacts = None # Initiate the Kademlia joining sequence - perform a search for this node's own ID self._joinDeferred = self._iterativeFind(self.id, bootstrapContacts) # #TODO: Refresh all k-buckets further away than this node's closest neighbour # Start refreshing k-buckets periodically, if necessary self.next_refresh_call = twisted.internet.reactor.callLater( constants.checkRefreshInterval, self._refreshNode) #IGNORE:E1101 self.hash_watcher.tick() return self._joinDeferred
def build_TLV_response(self, requestdata): responsedata = OrderedDict() for typ, value in requestdata.items(): if typ == 'NAME': # send full host name - no truncation value = self.service.hostname.encode("UTF-8") elif typ == 'IPAD': # send ipaddress as a string only if it is set value = self.transport.getHost().host # :todo: IPv6 if value == '0.0.0.0': # do not send back an ip address typ = None elif typ == 'JSON': # send port as a string json_port = 9000 # todo: web.service.port value = str(json_port) elif typ == 'VERS': # send server version value = squeal.__version__ elif typ == 'UUID': # send server uuid value = self.service.uuid elif typ == 'JVID': # not handle, just log the information typ = None log.msg("Jive: %x:%x:%x:%x:%x:%x:" % struct.unpack('>6B', value), logLevel=logging.INFO) else: log.error('Unexpected information request: %r', typ) typ = None if typ: responsedata[typ] = value return responsedata
def twisted_log(eventDict): log = logging.getLogger('twisted') if 'failure' in eventDict: log.error(eventDict.get('why') or 'Unhandled exception' + '\n' + str(eventDict['failure'].getTraceback())) elif 'warning' in eventDict: log.warning(eventDict['warning']) else: log.debug(' '.join([str(m) for m in eventDict['message']]))
def removeListener(self, eventvalue, callback): if not hasattr(callback, "__call__"): callback = getattr(callback, eventvalue+"Callback" ) if hasattr(callback, eventvalue+"Callback" ) else getattr(callback, "callback") try: self.event_registry[eventvalue].remove(callback) except Exception as ex: log.error(ex)
def log_error(err, n): if err.check(protocol.TimeoutError): log.debug( "Timeout while storing blob_hash %s at %s", binascii.hexlify(blob_hash), n) else: log.error( "Unexpected error while storing blob_hash %s at %s: %s", binascii.hexlify(blob_hash), n, err.getErrorMessage())
def checkIfValidEmail(self, email): """ Checks if the passed email is valid based on the regex string """ valid_email = re.compile( r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)") if not valid_email.match(email): log.error("Invalid email: %s" % email) raise ValueError(email)
def get_msg(self, my_queue): """ The function takes message from the queue """ self.channel.basic_qos(prefetch_count=COUNT) try: for method_frame, properties, body in self.channel.consume(my_queue): self.channel.basic_ack(method_frame.delivery_tag) return body except pika.exceptions, err_msg: log.error(err_msg) return False
def failure(failure, log, msg, *args): """Log a failure message from a deferred. Args: failure: twisted.python.failure.Failure log: a python logger instance msg: the message to log. Can use normal logging string interpolation. the last argument will be set to the error message from the failure. args: values to substitute into `msg` """ args += (failure.getErrorMessage(),) exc_info = (failure.type, failure.value, failure.getTracebackObject()) log.error(msg, *args, exc_info=exc_info)
def _parseStunResponse(dgram, address, expectedTID=None, oldtids=[]): mt, pktlen, tid = struct.unpack('!hh16s', dgram[:20]) if expectedTID is not None and expectedTID != tid: # a response from an earlier request if tid in oldtids: # discard if STUNVERBOSE: log.msg("ignoring belated STUN response to %r from %s"%( hexify(tid), repr(address)), system='stun') return log.msg("got unexpected STUN response %r != %r from %s"% (hexify(expectedTID), hexify(tid), repr(address),), system='stun') return resdict = {} if mt == 0x0101: #服务器的response message log.msg("got STUN response to %r from %s"%(hexify(expectedTID), repr(address)), system='stun') # response remainder = dgram[20:] while remainder: avtype, avlen = struct.unpack('!hh', remainder[:4]) val = remainder[4:4+avlen] avtype = StunTypes.get(avtype, '(Unknown type %04x)'%avtype) remainder = remainder[4+avlen:] if avtype in ('MAPPED-ADDRESS', 'CHANGED-ADDRESS', 'SOURCE-ADDRESS'): dummy,family,port,addr = struct.unpack('!ccH4s', val) addr = socket.inet_ntoa(addr) if STUNVERBOSE: print avtype, addr, port if avtype == 'MAPPED-ADDRESS': resdict['externalAddress'] = (addr, port) elif avtype == 'CHANGED-ADDRESS': resdict['_altStunAddress'] = (addr, address[1]) elif address[0] != addr: # Some son of a bitch is rewriting packets on the way # back. AAARGH. log.msg('WARNING: packets are being rewritten %r != %r'% (address, (addr,port)), system='stun') return else: log.msg("STUN: unhandled AV %s, val %r"%(avtype, repr(val)), system='stun') elif mt == 0x0111: log.error("STUN got an error response") return resdict
def _parseStunResponse(dgram, address, expectedTID=None, oldtids=[]): mt, pktlen, tid = struct.unpack('!hh16s', dgram[:20]) if expectedTID is not None and expectedTID != tid: # a response from an earlier request if tid in oldtids: # discard if STUNVERBOSE: log.msg("ignoring belated STUN response to %r from %s"%( hexify(tid), repr(address)), system='stun') return log.msg("got unexpected STUN response %r != %r from %s"% (hexify(expectedTID), hexify(tid), repr(address),), system='stun') return resdict = {} if mt == 0x0101: log.msg("got STUN response to %r from %s"%(hexify(expectedTID), repr(address)), system='stun') # response remainder = dgram[20:] while remainder: avtype, avlen = struct.unpack('!hh', remainder[:4]) val = remainder[4:4+avlen] avtype = StunTypes.get(avtype, '(Unknown type %04x)'%avtype) remainder = remainder[4+avlen:] if avtype in ('MAPPED-ADDRESS', 'CHANGED-ADDRESS', 'SOURCE-ADDRESS'): dummy,family,port,addr = struct.unpack('!ccH4s', val) addr = socket.inet_ntoa(addr) if STUNVERBOSE: print((avtype, addr, port)) if avtype == 'MAPPED-ADDRESS': resdict['externalAddress'] = (addr, port) elif avtype == 'CHANGED-ADDRESS': resdict['_altStunAddress'] = (addr, address[1]) elif address[0] != addr: # Some son of a bitch is rewriting packets on the way # back. AAARGH. log.msg('WARNING: packets are being rewritten %r != %r'% (address, (addr,port)), system='stun') return else: log.msg("STUN: unhandled AV %s, val %r"%(avtype, repr(val)), system='stun') elif mt == 0x0111: log.error("STUN got an error response") return resdict
def checkIfValidPassword(self, password): """ Checks if the passed password is valid based on the regex string """ min_len = 6 max_len = 64 valid_password = re.compile( r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len)) if not valid_password.match(password): log.error("Invalid password: %s" % password) raise ValueError(password)
def resumeProducing(self): if not self.request: log.error("Resuming a producer with no request object") return data = yield self.fileObject.read( min(self.bufferSize, self.size - self.bytesWritten)) if data: self.request.write(data) self.bytesWritten += len(data) else: log.err("Finished reading from grid file: %s" % self.fileObject) self.request.unregisterProducer() self.request.finish() self.stopProducing()
def checkIfValidNickname(self, nickname): """ Checks if the passed nickname is valid based on the regex string """ min_len = 3 max_len = 64 valid_nickname = re.compile( r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len)) if not valid_nickname.match(nickname): log.error("Invalid nick: %s" % nickname) raise ValueError(nickname)
def lineReceived(self, line): """Dispatches commands from the child. So far there's only one. And you have to use it. Or everything dies. """ args = line.split(' ') command = args.pop(0) if command == 'set-port': port = int(args[0]) self.protocol.set_port(port) elif command == 'request-done': self.protocol.track_request_done() else: log.error("Child is babbling nonsense: {0!r}".format(line))
def scanip(self, entry): """ Scan IP againt Greynoise API """ def message(query): log.msg( eventid='cowrie.greynoise.result', format= 'greynoise: Scan for %(IP)s with %(tag)s have %(conf)s confidence' ' along with the following %(meta)s metadata', IP=entry['src_ip'], tag=query['name'], conf=query['confidence'], meta=query['metadata']) gnUrl = '{0}query/ip'.format(GNAPI_URL).encode('utf8') headers = ({'User-Agent': [COWRIE_USER_AGENT]}) fields = {'key': self.apiKey, 'ip': entry['src_ip']} try: response = yield treq.post(url=gnUrl, data=fields, headers=headers, timeout=10) except (defer.CancelledError, error.ConnectingCancelledError, error.DNSLookupError): log.msg("GreyNoise requests timeout") return if response.code != 200: rsp = yield response.text() log.error("greynoise: got error {}".format(rsp)) return j = yield response.json() if self.debug: log.msg("greynoise: debug: " + repr(j)) if j['status'] == "ok": if "all" not in self.tags: for query in j['records']: if query['name'] in self.tags: message(query) else: for query in j['records']: message(query) else: log.msg("greynoise: no results for for IP {0}".format( entry['src_ip']))
def joinNetwork(self, knownNodeAddresses=None): """ Causes the Node to join the Kademlia network; normally, this should be called before any other DHT operations. @param knownNodeAddresses: A sequence of tuples containing IP address information for existing nodes on the Kademlia network, in the format: C{(<ip address>, (udp port>)} @type knownNodeAddresses: tuple """ # Prepare the underlying Kademlia protocol if self.port is not None: try: self._listeningPort = twisted.internet.reactor.listenUDP( self.port, self._protocol) except error.CannotListenError as e: import traceback log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc()) raise ValueError("%s lbrynet may already be running." % str(e)) #IGNORE:E1101 # Create temporary contact information for the list of addresses of known nodes if knownNodeAddresses != None: bootstrapContacts = [] for address, port in knownNodeAddresses: contact = Contact(self._generateID(), address, port, self._protocol) bootstrapContacts.append(contact) else: bootstrapContacts = None # Initiate the Kademlia joining sequence - perform a search for this node's own ID self._joinDeferred = self._iterativeFind(self.id, bootstrapContacts) # #TODO: Refresh all k-buckets further away than this node's closest neighbour # def getBucketAfterNeighbour(*args): # for i in range(160): # if len(self._buckets[i]) > 0: # return i+1 # return 160 # df.addCallback(getBucketAfterNeighbour) # df.addCallback(self._refreshKBuckets) #protocol.reactor.callLater(10, self.printContacts) #self._joinDeferred.addCallback(self._persistState) #self._joinDeferred.addCallback(self.printContacts) # Start refreshing k-buckets periodically, if necessary self.next_refresh_call = twisted.internet.reactor.callLater( constants.checkRefreshInterval, self._refreshNode) #IGNORE:E1101 self.hash_watcher.tick() return self._joinDeferred
def run_server(foreground=False): """ run the blockstored server """ global bitcoind prompt_user_for_chaincom_details() bitcoind = init_bitcoind() from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE from .lib.config import BLOCKSTORED_TAC_FILE from .lib.config import START_BLOCK working_dir = get_working_dir() current_dir = os.path.abspath(os.path.dirname(__file__)) tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE) log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE) pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE) start_block, current_block = get_index_range() if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file, log_file, tac_file) try: # refresh_index(335563, 335566, initial_index=True) if start_block != current_block: refresh_index(start_block, current_block, initial_index=True) blockstored = subprocess.Popen( command, shell=True, preexec_fn=os.setsid) log.info('Blockstored successfully started') except IndexError, ie: # indicates that we don't have the latest block log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \ "Please verify that your bitcoin provider has " + \ "processed up to block %s.\n" % (START_BLOCK) + \ " Example: bitcoin-cli getblockcount" ) try: os.killpg(blockstored.pid, signal.SIGTERM) except: pass exit(1)
def run_server(foreground=False): """ run the blockstored server """ global bitcoind prompt_user_for_chaincom_details() bitcoind = init_bitcoind() from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE from .lib.config import BLOCKSTORED_TAC_FILE from .lib.config import START_BLOCK working_dir = get_working_dir() current_dir = os.path.abspath(os.path.dirname(__file__)) tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE) log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE) pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE) start_block, current_block = get_index_range() if foreground: command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file) else: command = 'twistd --pidfile=%s --logfile=%s -y %s' % ( pid_file, log_file, tac_file) try: # refresh_index(335563, 335566, initial_index=True) if start_block != current_block: refresh_index(start_block, current_block, initial_index=True) blockstored = subprocess.Popen(command, shell=True, preexec_fn=os.setsid) log.info('Blockstored successfully started') except IndexError, ie: # indicates that we don't have the latest block log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \ "Please verify that your bitcoin provider has " + \ "processed up to block %s.\n" % (START_BLOCK) + \ " Example: bitcoin-cli getblockcount" ) try: os.killpg(blockstored.pid, signal.SIGTERM) except: pass exit(1)
def dictionary(self, args, irc): '''(dict [word or phrase]) -- Lookup up a word using WordNet. ''' if not args: return u'Missing word or phrase to look up' try: cmnd = 'sdcv -u WordNet -n --utf8-input --utf8-output'.split() + args o = [line.strip() for line in sp.check_output(cmnd).split('\n') if line] if len(o) == 1: return o[0] else: return u' '.join(o[3:]) except: log.error('[Error]: dict {}'.format(sys.exc_info()[0]))
def handle_line(self, line, repo_lock=defer.DeferredLock()): try: line = line.decode('utf-8', 'ignore').strip() match = re.search(r'(^|\s+)!pq\s+(?P<link>\S+)(\s+::\S+|$)', line) if not match: log.noise('Non-patchbot line, ignoring: {}'.format(line.encode('utf-8', 'ignore'))) defer.returnValue(None) link = match.group('link').encode('ascii') if not re.search('https?://', link, re.IGNORECASE): log.warn('Incorrect non-http link, skipping: {}'.format(link)) defer.returnValue(None) except UnicodeError as err: log.warn('Failed to recode line ({!r}): {}'.format(line, err)) defer.returnValue(None) ## Grab the patch dst_base = '{}.patch'.format(sha1(link).hexdigest()) dst_path = self.dst_path.child(dst_base) if dst_path.exists(): log.debug( 'Patch already exists' ' (file: {}, link: {}), skipping'.format(dst_path, link) ) defer.returnValue(None) # Not via tmpfile to prevent multiple downloads of the same paste try: yield downloadPage(link, dst_path.open('wb'), timeout=120) except: if dst_path.exists(): dst_path.remove() raise ## Commit into repo and push yield repo_lock.acquire() try: for cmd, check in [ (['add', dst_base], True), (['commit', '-m', 'New patch: {}'.format(link)], False), (['push'], True) ]: out, err, code = yield getProcessOutputAndValue( '/usr/bin/git', cmd, path=self.dst_path.path ) if check and code: log.error('\n'.join([ 'Failed to commit/push new patch into repo', 'Command: {}'.format(cmd), 'Exit code: {}'.format(code), 'Stdout:\n {}'.format('\n '.join(out.splitlines())), 'Stderr:\n {}'.format('\n '.join(err.splitlines())) ])) break else: log.debug('Successfully pushed paste: {}'.format(link)) finally: repo_lock.release()
def setStunnedAddress(self, results): ''' Handle results of the rtp/rtcp STUN. We have to check that the results have the same IP and usable port numbers ''' log.msg("got NAT mapping back! %r"%(results), system='rtp') rtpres, rtcpres = results if rtpres[0] != defer.SUCCESS or rtcpres[0] != defer.SUCCESS: # barf out. log.msg("uh oh, stun failed %r"%(results), system='rtp') else: # a=RTCP might help for wacked out RTCP/RTP pairings # format is something like "a=RTCP:AUDIO 16387" # See RFC 3605 code1, rtp = rtpres code2, rtcp = rtcpres if rtp[0] != rtcp[0]: print "stun gave different IPs for rtp and rtcp", results # We _should_ try and see if we have working rtp and rtcp, but # this seems almost impossible with most firewalls. So just try # to get a working rtp port (an even port number is required). elif False and ((rtp[1] % 2) != 0): log.error("stun: unusable RTP/RTCP ports %r, retry #%d"% (results, self._stunAttempts), system='rtp') # XXX close connection, try again, tell user if self._stunAttempts > 8: # XXX print "Giving up. Made %d attempts to get a working port" \ % (self._stunAttempts) self._stunAttempts += 1 defer.maybeDeferred( self.rtpListener.stopListening).addCallback( \ lambda x:self.rtcpListener.stopListening()).addCallback( \ lambda x:self._socketCreationAttempt()) else: # phew. working NAT log.msg("stun: sane NAT for RTP/RTCP; rtp addr: %s" \ % (rtp,), system='rtp') #Register address and port self._extIP, self._extRTPPort = rtp self._stunAttempts = 0 d = self._socketCompleteDef del self._socketCompleteDef d.callback(self.cookie)
def setStunnedAddress(self, results): ''' Handle results of the rtp/rtcp STUN. We have to check that the results have the same IP and usable port numbers ''' log.msg("got NAT mapping back! %r" % (results), system='rtp') rtpres, rtcpres = results if rtpres[0] != defer.SUCCESS or rtcpres[0] != defer.SUCCESS: # barf out. log.msg("uh oh, stun failed %r" % (results), system='rtp') else: # a=RTCP might help for wacked out RTCP/RTP pairings # format is something like "a=RTCP:AUDIO 16387" # See RFC 3605 code1, rtp = rtpres code2, rtcp = rtcpres if rtp[0] != rtcp[0]: print "stun gave different IPs for rtp and rtcp", results # We _should_ try and see if we have working rtp and rtcp, but # this seems almost impossible with most firewalls. So just try # to get a working rtp port (an even port number is required). elif False and ((rtp[1] % 2) != 0): log.error("stun: unusable RTP/RTCP ports %r, retry #%d" % (results, self._stunAttempts), system='rtp') # XXX close connection, try again, tell user if self._stunAttempts > 8: # XXX print "Giving up. Made %d attempts to get a working port" \ % (self._stunAttempts) self._stunAttempts += 1 defer.maybeDeferred( self.rtpListener.stopListening).addCallback( \ lambda x:self.rtcpListener.stopListening()).addCallback( \ lambda x:self._socketCreationAttempt()) else: # phew. working NAT log.msg("stun: sane NAT for RTP/RTCP; rtp addr: %s" \ % (rtp,), system='rtp') #Register address and port self._extIP, self._extRTPPort = rtp self._stunAttempts = 0 d = self._socketCompleteDef del self._socketCompleteDef d.callback(self.cookie)
def consume(self, data): """ We've monkey patched the consumer's `consume` method to point to this one intead. All incoming data are messages that need to be sent to TruTeq over USSD. Use `ssmi_client.send_ussd` to do that. """ # default to ussd if data.get('type', 'ussd') == 'ussd': self.ssmi_client.send_ussd( str(data['msisdn']), # str everything because the SSMIClient str(data['message']), # isn't happy with Unicode str(data['ussd_type'])) elif data['type'] == 'sms': self.ssmi_client.send_sms( str(data['msisdn']), str(data['message']) ) else: log.error('Unknown TruTeq message type in %s' % data)
def __init__(self, url, filepath, callback=None): """ @param url = resource to fetch @param filepath = string path of file to read @param callback = function to call with the filepath as arguement after sucess response from url """ if not os.path.exists(filepath): e = FileRouterException('{0} not found'.format(filepath)) log.err(e) raise e with open(filepath) as ofile: try: self.data = json.loads(ofile.read()) except Exception as e: log.error(e) raise e self.url = url self.file_callback = callback
def run(self): if not os.path.exists("dist/%s.app/" % config.appname): log.error("You have to run py2app first") return # getting Python.framework path cmd = "python-config --includes | awk -F'-I' '{print $2}' | sed 's/\/include.*$//'" process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) pythonFrameworkPath = process.stdout.read().strip() version = pythonFrameworkPath.split("/")[-1] # folder of the app bundle frameFolder = ("dist/%s.app/Contents/Frameworks/Python.framework/Versions/%s" % (config.appname, version)) # copy it if it does't exists if not os.path.exists(frameFolder): os.makedirs(frameFolder) os.system("cp %s/Python %s" % (pythonFrameworkPath, frameFolder)) os.system("chmod +x dist/%s.app/Contents/Resources/contrib/tor" % config.appname)
def run(self): if not os.path.exists("dist/%s.app/" % config.appname): log.error("You have to run py2app first") return # getting Python.framework path cmd = "python-config --includes | awk -F'-I' '{print $2}' | sed 's/\/include.*$//'" process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) pythonFrameworkPath = process.stdout.read().strip() version = pythonFrameworkPath.split("/")[-1] # folder of the app bundle frameFolder = ( "dist/%s.app/Contents/Frameworks/Python.framework/Versions/%s" % (config.appname, version)) # copy it if it does't exists if not os.path.exists(frameFolder): os.makedirs(frameFolder) os.system("cp %s/Python %s" % (pythonFrameworkPath, frameFolder)) os.system("chmod +x dist/%s.app/Contents/Resources/contrib/tor" % config.appname)
def getTags(_file): try: tags = File(_file, easy=True) return { 'title': tags['TITLE'][0].encode('utf8'), 'artist': tags['ARTIST'][0].encode('utf8'), 'album': tags['ALBUM'][0].encode('utf8'), 'genre': tags['GENRE'][0].encode('utf8'), 'art': hasEmbeddedArt(_file) } except Exception as err: Logger.error('Error on file {} - {}'.format( str(_file), err) ) return False
def initalizeDBus(): global flag if flag: return global dbus, bus, manager try: import dbus try: bus = dbus.SessionBus() manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.bluez.Manager") except: bus = dbus.SystemBus() manager = dbus.Interface(bus.get_object("org.bluez", "/"), "org.bluez.Manager") log.msg("DBus available") flag = True except Exception, err: log.msg("DBus unavailable") log.error(err) if isAndroid(): flag = True
def dataReceived(self, data): if self.transport.protocol.state == txws.FRAMES: if not self._session: session_id = self.transport.protocol.location.rstrip('/').split('/')[-1] if not base.simple_id(session_id): log.err("Invalid Session ID: %r"%( session_id, )) self.write_error("session invalid") self.transport.loseConnection() return log.msg("New Connection on Session: %r"%( session_id, )) session = self.factory.server.session(session_id, create=False) if not session: log.err("Unkown/no-auth session: %r"%( session_id, )) self.write_error("session unknown") self.transport.loseConnection() return self._session = session self._session.protocols.append(self) self.write_welcome() else: session = self._session try: channel_id, data = data.split(',', 1) except ValueError: log.error('Mis-formatted request (no ,)') self.write_error('Missing comma in request') return if not channel_id: """Protocol-level messages, currently nothing""" pass elif not base.simple_id(channel_id): log.err("Invalid Channel ID: %r"%( channel_id, )) self.write_error("invalid channel") self.transport.loseConnection() return else: session.on_incoming(channel_id, data) self.ready = True
def __init__(self, target, listener = None, connectionLost = None): if SCOConnected(target): log.error("Can't connect twice to the same device") return self.sock = bluetooth.BluetoothSocket(bluetooth.SCO) try: self.sock.connect((target,)) except Exception, err: log.error("Failed while trying to connect SCO") log.error(err) return
def __init__(self, target, listener=None, connectionLost=None): if SCOConnected(target): log.error("Can't connect twice to the same device") return self.sock = bluetooth.BluetoothSocket(bluetooth.SCO) try: self.sock.connect((target, )) except Exception, err: log.error("Failed while trying to connect SCO") log.error(err) return
def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None: # logfile path relative to config dir if not abs path log_filename = logfile.get() if log_filename.strip(): # catches empty filename if not os.path.isabs(log_filename): log_filename = os.path.join(config.config_dir, log_filename) ensure_dir_exists(log_filename) if logging_rotate_daily.get(): logging_file = DailyLogFile(log_filename, '.') else: logging_file = open(log_filename, 'a') globalLogPublisher.addObserver(textFileLogObserver(logging_file)) globalLogPublisher.addObserver(textFileLogObserver(sys.stderr)) log.info('piqueserver started on %s' % time.strftime('%c')) self.config = config_dict if random_rotation: self.map_rotator_type = random_choice_cycle else: self.map_rotator_type = itertools.cycle # pylint: disable=redefined-variable-type self.default_time_limit = default_time_limit.get() self.default_cap_limit = cap_limit.get() self.advance_on_win = int(advance_on_win.get()) self.win_count = itertools.count(1) self.bans = NetworkDict() # attempt to load a saved bans list try: with open(os.path.join(config.config_dir, bans_file.get()), 'r') as f: self.bans.read_list(json.load(f)) log.debug("loaded {count} bans", count=len(self.bans)) except FileNotFoundError: log.debug("skip loading bans: file unavailable", count=len(self.bans)) except IOError as e: log.error('Could not read bans.txt: {}'.format(e)) except ValueError as e: log.error('Could not parse bans.txt: {}'.format(e)) self.hard_bans = set() # possible DDoS'ers are added here self.player_memory = deque(maxlen=100) if len(self.name) > MAX_SERVER_NAME_SIZE: log.warn('(server name too long; it will be truncated to "%s")' % (self.name[:MAX_SERVER_NAME_SIZE])) self.respawn_time = respawn_time_option.get() self.respawn_waves = respawn_waves.get() if game_mode.get() == 'ctf': self.game_mode = CTF_MODE elif game_mode.get() == 'tc': self.game_mode = TC_MODE elif self.game_mode is None: raise NotImplementedError('invalid game mode: %s' % game_mode) self.game_mode_name = game_mode.get().split('.')[-1] self.team1_name = team1_name.get() self.team2_name = team2_name.get() self.team1_color = tuple(team1_color.get()) self.team2_color = tuple(team2_color.get()) self.friendly_fire = friendly_fire.get() self.friendly_fire_on_grief = friendly_fire_on_grief.get() self.friendly_fire_time = grief_friendly_fire_time.get() self.spade_teamkills_on_grief = spade_teamkills_on_grief.get() self.fall_damage = fall_damage.get() self.teamswitch_interval = teamswitch_interval.get() self.teamswitch_allowed = teamswitch_allowed.get() self.max_players = max_players.get() self.melee_damage = melee_damage.get() self.max_connections_per_ip = max_connections_per_ip.get() self.passwords = passwords.get() self.server_prefix = server_prefix.get() self.time_announcements = time_announcements.get() self.balanced_teams = balanced_teams.get() self.login_retries = login_retries.get() # voting configuration self.default_ban_time = default_ban_duration.get() self.speedhack_detect = speedhack_detect.get() if user_blocks_only.get(): self.user_blocks = set() self.set_god_build = set_god_build.get() self.debug_log = debug_log_enabled.get() if self.debug_log: # TODO: make this configurable pyspades.debug.open_debug_log( os.path.join(config.config_dir, 'debug.log')) if ssh_enabled.get(): from piqueserver.ssh import RemoteConsole self.remote_console = RemoteConsole(self) irc = irc_options.get() if irc.get('enabled', False): from piqueserver.irc import IRCRelay self.irc_relay = IRCRelay(self, irc) if status_server_enabled.get(): from piqueserver.statusserver import StatusServerFactory self.status_server = StatusServerFactory(self) if ban_publish.get(): from piqueserver.banpublish import PublishServer self.ban_publish = PublishServer(self, ban_publish_port.get()) if bans_urls.get(): from piqueserver import bansubscribe self.ban_manager = bansubscribe.BanManager(self) self.start_time = reactor.seconds() self.end_calls = [] # TODO: why is this here? create_console(self) for user_type, func_names in rights.get().items(): for func_name in func_names: commands.add_rights(user_type, func_name) port = self.port = port_option.get() ServerProtocol.__init__(self, port, interface) self.host.intercept = self.receive_callback try: self.set_map_rotation(self.config['rotation']) except MapNotFound as e: log.critical('Invalid map in map rotation (%s), exiting.' % e.map) raise SystemExit self.update_format() self.tip_frequency = tip_frequency.get() if self.tips is not None and self.tip_frequency > 0: reactor.callLater(self.tip_frequency * 60, self.send_tip) self.master = register_master_option.get() self.set_master() self.http_agent = web_client.Agent(reactor) ip_getter = ip_getter_option.get() if ip_getter: self.get_external_ip(ip_getter)
def run() -> None: """ runs the server """ # apply scripts protocol_class = FeatureProtocol connection_class = FeatureConnection script_objects = [] script_names = scripts_option.get() script_dir = os.path.join(config.config_dir, 'scripts/') for script in script_names[:]: try: # this finds and loads scripts directly from the script dir # no need for messing with sys.path f, filename, desc = imp.find_module(script, [script_dir]) module = imp.load_module('piqueserver_script_namespace_' + script, f, filename, desc) script_objects.append(module) except ImportError as e: # warning: this also catches import errors from inside the script # module it tried to load try: module = importlib.import_module(script) script_objects.append(module) except ImportError as e: log.error("(script '{}' not found: {!r})".format(script, e)) script_names.remove(script) for script in script_objects: protocol_class, connection_class = script.apply_script( protocol_class, connection_class, config.get_dict()) # apply the game_mode script if game_mode.get() not in ('ctf', 'tc'): # must be a script with this game mode module = None try: game_mode_dir = os.path.join(config.config_dir, 'game_modes/') f, filename, desc = imp.find_module(game_mode.get(), [game_mode_dir]) module = imp.load_module( 'piqueserver_gamemode_namespace_' + game_mode.get(), f, filename, desc) except ImportError as e: try: module = importlib.import_module(game_mode.get()) except ImportError as e: log.error("(game_mode '%s' not found: %r)" % (game_mode.get(), e)) if module: protocol_class, connection_class = module.apply_script( protocol_class, connection_class, config.get_dict()) protocol_class.connection_class = connection_class interface = network_interface.get().encode('utf-8') # instantiate the protocol class once. It will set timers and hooks to keep # itself running once we start the reactor protocol_class(interface, config.get_dict()) log.debug('Checking for unregistered config items...') unused = config.check_unused() if unused: log.warn('The following config items are not used:') pprint(unused) log.info('Started server...') profile = logging_profile_option.get() if profile: import cProfile cProfile.runctx('reactor.run()', None, globals()) else: reactor.run()
import web import sys, os from twisted.python import log from twisted.internet import defer, reactor def main(config_file): log.startLogging(sys.stdout) application = web.Application(config_file) port = os.environ.get("PORT", 8888) reactor.listenTCP(int(port), application) reactor.run() if __name__ == "__main__": if len(sys.argv) > 1: main(sys.argv[1]) else: log.error("no config file given") sys.exit(-1)
def connectionLost(self, reason): if reason.type != ConnectionLost: log.debug('Disconnect from %s' % self.transport.getPeer()) else: log.error('Disconnect from %s: %s' % (self.transport.getPeer(), reason))
def openFailed(self, reason): log.error("Open failed due to %r", reason) if self.start_defer: self.start_defer.errback(self)
def run(self): def safe_unicode(obj, *args): try: return unicode(obj, *args) except UnicodeDecodeError: # obj is byte string ascii_text = str(obj).encode('string_escape') return unicode(ascii_text) def get_appname(rts): rts = rts.lower() for name in ('paypage', 'ccentry', 'mis'): if name in rts: return name return 'UnknownEntryPoint' log.debug("Execute model checks and execution of loop") # process the accumulated data into the model bl_trace_id = [] traces, last_trace_end_str = {}, None trace_id = self.guid start_trace = False local = [] latest_datetime = None try: for line in self.accumulator: if not start_trace: if 'TRACE START' == line[:11]: start_trace = True latest_datetime = extract_datetime(line) local = [] else: if 'TRACE END' in line[:9]: appname = get_appname(local[0]) trace_id = time.strftime( "%Y-%m-%d_%H-%M-%S", latest_datetime) + '_' + appname + '_' + str( hex(hash(repr(local)))) if trace_id not in bl_trace_id: bl_trace_id.append(trace_id) last_trace_end_str = line start_trace = False traces[trace_id] = local if trace_id not in self.cache: self.cache[trace_id] = local local = [] start_trace = False else: local.append(line) # keep only what hasn't been processed yet if last_trace_end_str in self.accumulator: self.accumulator = self.accumulator[self.accumulator. index(last_trace_end_str):] log.debug("Process all %d traces available" % len(traces)) for trace_id in traces: self.processTrace(traces[trace_id], trace_id) try: o = open('./cache/' + trace_id + '.html', 'w') o.write( pygment_content("<?php\n" + '\n'.join([ safe_unicode(rts) for rts in self.cache[trace_id] if not has_blacklisted_function(rts) ]) + "\n?>")) o.close() except IOError, error: log.error("Exception writing %s (error:%s)" % ('./cache/' + trace_id + '.html', error)) # generate the JSON file for frontend to consume self.update_JSON()
def failure(self, format, failure, *args, **kwargs): log.error(failure, format.format(args, **kwargs))
def log_error(err): log.error(err.getErrorMessage())
class Modeler(object): def __init__(self): self.times = [] self.guid = 1 self.cuid = 1 self.g = digraph() self.context = {} self.rtids = {} self.node_location = {} self.accumulator = [] self.trace_ids = [] self.cache = {} def feedAccumulator(self, data): self.accumulator.append(data) @staticmethod def __node_uid(trace_id, node_str, cur_index=''): return hash(str(trace_id) + node_str + str(cur_index)) @staticmethod def add_node(g, nid, location=None, linenumber=None, context_id=None, trace_id=None, method=None, data=None, _type=None, _id=None): if not g.has_node(nid): g.add_node(nid) g.node_attr[nid] = { 'return': '', 'context_id': context_id, 'trace_id': trace_id, 'location': location, 'linenumber': linenumber, 'method': method, 'data': data, 'type': _type, 'rtid': False, 'sequence': _id } @staticmethod def add_edge(g, e): if not g.has_edge(e): g.add_edge(e) else: g.set_edge_weight(e, g.edge_weight(e) + 1) def update_rtid(self, lst): for e in lst: if e not in self.rtids: self.rtids[e] = re.compile(e, re.I) @staticmethod def regexp_interesect(data, dct): if not isinstance(data, str): data = str(repr(data)) for reg in dct: if dct[reg].search(data): return True return False def update_model_taints(self): # get latest list of rtids self.update_rtid(db_list_rtid()) # make sure we don't iterate if no data to check with if 1 > len(self.rtids): return # loop over the entire graph and update the 'rtid' node attr for n in self.g: if Modeler.regexp_interesect( self.g.node_attr[n]['method'], self.rtids) or Modeler.regexp_interesect( self.g.node_attr[n]['data'], self.rtids) or Modeler.regexp_interesect( self.g.node_attr[n]['return'], self.rtids): # update taint self.g.node_attr[n]['rtid'] = True def processTrace(self, trace, trace_id): callstack = Stack() prev_node, cur_node = None, None pre_l = None cur_i = 0 if trace_id not in self.trace_ids: self.trace_ids.append(trace_id) for l in trace: self.guid += 1 cur_i += 1 suc, trace, token = get_cleaned_trace_node(l) if not suc: continue else: trace_info = extract_information(token, trace) if trace_info and isinstance(trace_info, dict): _type = trace_info['type'] if _type == 'main': main_nuid = Modeler.__node_uid(trace_id, l) Modeler.add_node(self.g, main_nuid, trace_info['filename'], trace_info['line'], None, trace_id, 'main', self.guid) callstack.push(main_nuid) elif _type == 'return': # only keep the latest value if not callstack.empty(): prev_node = callstack.pop() self.g.node_attr[prev_node]['return'] = trace_info[ 'value'] elif _type == 'call': cur_node = Modeler.__node_uid(trace_id, l) Modeler.add_node(self.g, cur_node, trace_info['filename'], trace_info['line'], None, trace_id, trace_info['function'], trace_info['parameters'], 'call', self.guid) if not callstack.empty(): prev_node = callstack.top() edge = (prev_node, cur_node) Modeler.add_edge(self.g, edge) # set label to the edge as parameters self.g.set_edge_label( edge, ','.join(trace_info['parameters'])) callstack.push(cur_node) elif _type == 'assignment': cur_node = Modeler.__node_uid(trace_id, l) Modeler.add_node(self.g, cur_node, trace_info['filename'], trace_info['line'], None, trace_id, trace_info['variable'], trace_info['value'], 'assignment', self.guid) if not callstack.empty(): prev_node = callstack.top() edge = (prev_node, cur_node) Modeler.add_edge(self.g, edge) # set label to the edge as parameters self.g.set_edge_label(edge, trace_info['value']) # callstack.push(cur_node) self.update_model_taints() # self.write_dot_slice(trace_id) def update_JSON(self): log.debug("update_JSON") # only output simple paths and possible ways to investigate jsonified_data = { 'injected': self.rtids.keys(), 'hash': '', 'trace': {}, 'knowledge': { 'function': {}, 'file': {} } } def safe_unicode(obj, *args): try: return unicode(obj, *args) except UnicodeDecodeError: # obj is byte string ascii_text = str(obj).encode('string_escape') return unicode(ascii_text) def lmt_size(rts): length = len(rts) if 256 >= length: return rts return rts[:256] # always output the top 50 traces sorted_traces = self.trace_ids sorted_traces.sort(reverse=True) for trace_id in sorted_traces[:min(100, len(sorted_traces))]: if trace_id in jsonified_data['trace']: continue jsonified_data['trace'][trace_id] = { 'callchain': [], 'function': {}, 'file': {}, } tlst_nodes = {} for n in self.g: if trace_id == self.g.node_attr[n]['trace_id']: tlst_nodes[self.g.node_attr[n]['sequence']] = n slst_nodes = [tlst_nodes[k] for k in sorted(tlst_nodes.keys())] if 0 == len(slst_nodes): del jsonified_data['trace'][trace_id] else: for n in slst_nodes: if self.g.node_attr[n]['type'] not in ('call', 'main'): continue if self.g.node_attr[n]['rtid']: nd = self.g.node_attr[n] method = safe_unicode(nd['method']) data = safe_unicode(nd['data']) ret = safe_unicode(nd['return']) if has_blacklisted_function(nd['location']): continue jsonified_data['trace'][trace_id]['callchain'].append([ self.g.node_attr[n]['type'], nd['location'], nd['linenumber'], method, lmt_size(data), lmt_size(ret) ]) """ if method not in jsonified_data['trace'][trace_id]['function']: jsonified_data['trace'][trace_id]['function'][method] = [] jsonified_data['trace'][trace_id]['function'][method].append(data) if method not in jsonified_data['knowledge']['function']: jsonified_data['knowledge']['function'][method] = [] if data not in jsonified_data['knowledge']['function'][method]: jsonified_data['knowledge']['function'][method] = data if nd['location'] not in jsonified_data['trace'][trace_id]['file']: jsonified_data['trace'][trace_id]['file'][nd['location']] = [] jsonified_data['trace'][trace_id]['file'][nd['location']].append(nd['linenumber']) """ if 0 == len(jsonified_data['trace'][trace_id]['callchain']): del jsonified_data['trace'][trace_id] jsonified_data['hash'] = hash( str( repr(jsonified_data['trace']) + repr(jsonified_data['injected']))) o = open('assmnt-data.json', 'w') o.write( json.dumps(jsonified_data, ensure_ascii=False, separators=(',', ':'))) o.close() def write_dot_slice(self, trace_id): new_g = digraph() new_g.add_graph(self.g) # create new graph with only the CWE instances for n in new_g: if self.g.node_attr[n]['trace_id'] != trace_id: new_g.del_node(n) continue fname = './graphs/%s-trace' % str(trace_id) self.write_dot(new_g, fname) print "Generated: %s" % fname def write_dot(self, g, fname): if 1 > len(g): return def pen_width(num, local_max): max_penwidth = 7 val = int( math.ceil(float(num) / float(local_max) * float(max_penwidth))) return max(1, min(val, max_penwidth)) def node_call_labels(dct): """ <f0> Function Name| $value1 | ... | return values" """ lbl_str = "%s:%s |" % (dct['location'], dct['linenumber']) lbl_str += "<f0> " + xml_entities(dct['method']) + "()" for e in dct['data']: lbl_str += ' | ' + xml_entities(e) lbl_str += '| <f1> T_RET: ' + xml_entities(dct['return']) return lbl_str # generate DOT file with custom information (source, sink, and custom colors) dstr = """digraph runtime { graph [rankdir = "LR"]; node [fontsize = 12]; """ for n in g: node_type = 'shape=ellipse' if self.g.node_attr[n]['type'] == 'call': node_type = 'shape=record' node_type += ", label=\"%s\"" % node_call_labels( self.g.node_attr[n]) elif self.g.node_attr[n]['type'] == 'assignment': node_type = 'shape=record, style="filled,bold" penwidth=1' node_type += ",label=\"%s=%s\"" % ( xml_entities(self.g.node_attr[n]['method']), xml_entities(self.g.node_attr[n]['data'])) else: if self.g.node_attr[n]['method'] == 'return': node_type += ", label=\"T_RETURN:%s\"" % self.g.node_attr[ n]['data'].replace('"', '\\"') else: node_type += ", label=\"%s\"" % self.g.node_attr[n][ 'method'] if self.g.node_attr[n]['rtid']: node_type += ', fillcolor="#AA0114"' #if 'source' in g.node_attr[n]['nodetype'] or 'sink' in g.node_attr[n]['nodetype']: # node_type = 'shape=box, style=filled, fillcolor="%s"' % color_node(g.node_attr[n]['nodetype']) # get the size of the pen based on the number of GUID #node_type += ', penwidth=%d' % int(pen_width(len(g.node_attr[n]['guid']), max_guid)) dstr += ' "%s" [%s];\n' % (n, node_type) for e in g.edges(): n1, n2 = e node_type = "" #l_e_prop = self.g.edge_label(e).split(',') if self.g.node_attr[n1]['rtid'] and self.g.node_attr[n2]['rtid']: node_type += 'color="#AA0114"' else: node_type += 'color="black"' node_type += ', penwidth=%d' % pen_width(g.edge_weight(e), 10) #node_type += ', label="%s"' % ','.join(l_e_prop).replace('"', '\\"') addon_n1 = ':f0' if self.g.node_attr[n1]['type'] in ( 'call', 'return') else '' addon_n2 = ':f0' if self.g.node_attr[n2]['type'] in ( 'call', 'return') else '' dstr += ' "%s"%s -> "%s"%s [%s];\n' % (n1, addon_n1, n2, addon_n2, node_type) dstr += '}' dot_fname = fname + '.dot' o = open(dot_fname, 'w') o.write(dstr) o.close() #cmd_line = "dot %s -Tsvg -O -Kdot -x" % (dot_fname) # os.system(cmd_line) syscall("dot", [dot_fname, "-Tsvg", "-O", "-Kdot", "-x"]) def export(self): return if 0 < len(self.g): self.write_dot(self.g, './graphs/global_file') def run(self): def safe_unicode(obj, *args): try: return unicode(obj, *args) except UnicodeDecodeError: # obj is byte string ascii_text = str(obj).encode('string_escape') return unicode(ascii_text) def get_appname(rts): rts = rts.lower() for name in ('paypage', 'ccentry', 'mis'): if name in rts: return name return 'UnknownEntryPoint' log.debug("Execute model checks and execution of loop") # process the accumulated data into the model bl_trace_id = [] traces, last_trace_end_str = {}, None trace_id = self.guid start_trace = False local = [] latest_datetime = None try: for line in self.accumulator: if not start_trace: if 'TRACE START' == line[:11]: start_trace = True latest_datetime = extract_datetime(line) local = [] else: if 'TRACE END' in line[:9]: appname = get_appname(local[0]) trace_id = time.strftime( "%Y-%m-%d_%H-%M-%S", latest_datetime) + '_' + appname + '_' + str( hex(hash(repr(local)))) if trace_id not in bl_trace_id: bl_trace_id.append(trace_id) last_trace_end_str = line start_trace = False traces[trace_id] = local if trace_id not in self.cache: self.cache[trace_id] = local local = [] start_trace = False else: local.append(line) # keep only what hasn't been processed yet if last_trace_end_str in self.accumulator: self.accumulator = self.accumulator[self.accumulator. index(last_trace_end_str):] log.debug("Process all %d traces available" % len(traces)) for trace_id in traces: self.processTrace(traces[trace_id], trace_id) try: o = open('./cache/' + trace_id + '.html', 'w') o.write( pygment_content("<?php\n" + '\n'.join([ safe_unicode(rts) for rts in self.cache[trace_id] if not has_blacklisted_function(rts) ]) + "\n?>")) o.close() except IOError, error: log.error("Exception writing %s (error:%s)" % ('./cache/' + trace_id + '.html', error)) # generate the JSON file for frontend to consume self.update_JSON() except Exception, error: log.error("run- %s" % error) sys.exit()
def connectionLost(self, reason): if callable(self._connectionLost): try: self._connectionLost(self) except Exception, err: log.error(err)
def _process_changes(self, newRev, branch): """ Read changes since last change. - Read list of commit hashes. - Extract details from each commit. - Add changes to database. """ # initial run, don't parse all history if not self.lastRev: return rebuild = False if newRev in itervalues(self.lastRev): if self.buildPushesWithNoCommits and \ branch not in iterkeys(self.lastRev): # we know the newRev but not for this branch log.msg('gitpoller: rebuilding %s for new branch "%s"' % (newRev, branch)) rebuild = True # get the change list revListArgs = ([r'--format=%H', r'%s' % newRev] + [r'^%s' % rev.encode('ascii', 'ignore') for rev in itervalues(self.lastRev)] + [r'--']) self.changeCount = 0 results = yield self._dovccmd('log', revListArgs, path=self.workdir) # process oldest change first revList = results.split() revList.reverse() if rebuild and len(revList) == 0: revList = [newRev] self.changeCount = len(revList) self.lastRev[branch] = newRev if self.changeCount: log.msg('gitpoller: processing %d changes: %s from "%s" branch "%s"' % (self.changeCount, revList, self.repourl, branch)) for rev in revList: dl = defer.DeferredList([ self._get_commit_timestamp(rev), self._get_commit_author(rev), self._get_commit_files(rev), self._get_commit_comments(rev), ], consumeErrors=True) results = yield dl # check for failures failures = [r[1] for r in results if not r[0]] if failures: for failure in failures: log.error( failure, "while processing changes for {} {}".format(newRev, branch)) # just fail on the first error; they're probably all related! failures[0].raiseException() timestamp, author, files, comments = [r[1] for r in results] yield self.master.data.updates.addChange( author=author, revision=ascii2unicode(rev), files=files, comments=comments, when_timestamp=timestamp, branch=ascii2unicode(self._removeHeads(branch)), project=self.project, repository=ascii2unicode(self.repourl), category=self.category, src=u'git')
def validate_settings(settings): if not settings: log.error("No settings at all?") return False if not settings['nickname'] or not isinstance(settings['nickname'], str): log.error("Settings is missing nickname field") return False if not settings['realname'] or not isinstance(settings['realname'], str): log.error("Settings is missing realname field") return False if not settings['username'] or not isinstance(settings['username'], str): log.error("Settings is missing username field") return False if not settings['servers']: log.error("Settings is missing servers?") return False for server in settings['servers']: if not server: log.error("Got empty server in server settings") return False if not server['name'] or not isinstance(server['name'], str): log.error("Missing name for server") return False if not server['host'] or not isinstance(server['host'], str): log.error("Got misconfigured host setting") return False if not server['port'] or not isinstance(server['port'], int): log.error("Got misconfigured ip setting") return False if not settings['plugins']: log.error("Got no plugin, bot will be useless?") return False for plugin in settings['plugins']: if not plugin: log.error("Settings found empty plugin") return False if not plugin['name'] or not isinstance(plugin['name'], str): log.error("Settings got plugin with no name") return False return True
def eb_produce(f): log.error("Produce failed", exc_info=(f.type, f.value, f.getTracebackObject()))
def replaceTwistedLoggers(): """ Visit all Python modules that have been loaded and: - replace L{twisted.python.log} with a L{LegacyLogger} - replace L{twisted.python.log.msg} with a L{LegacyLogger}'s C{msg} - replace L{twisted.python.log.err} with a L{LegacyLogger}'s C{err} """ log = Logger() for moduleName, module in sys.modules.iteritems(): # Oddly, this happens if module is None: continue # Don't patch Twisted's logging module if module in (twisted.python, twisted.python.log): continue # Don't patch this module if moduleName is __name__: continue try: for name, obj in module.__dict__.iteritems(): try: newLogger = Logger(namespace=module.__name__) except AttributeError: # Can't look up __name__. A hack in the "six" module causes # this. Skip the module. # See https://trac.calendarserver.org/ticket/832 continue legacyLogger = LegacyLogger(logger=newLogger) if obj is twisted.python.log: log.info("Replacing Twisted log module object {0} in {1}". format(name, module.__name__)) setattr(module, name, legacyLogger) elif obj is twisted.python.log.msg: log.info( "Replacing Twisted log.msg object {0} in {1}".format( name, module.__name__)) setattr(module, name, legacyLogger.msg) elif obj is twisted.python.log.err: log.info( "Replacing Twisted log.err object {0} in {1}".format( name, module.__name__)) setattr(module, name, legacyLogger.err) except RuntimeError as e: # Python could use more specific exceptions, eh. # What we mean to catch is: # RuntimeError: dictionary changed size during iteration log.error( "Unable to replace twisted loggers for module {module}: " "{error}", module=module, error=e)
def _errback(self, reason): log.error("Processing request error: %s" % reason) self._send_action('DUNNO')
def connection_failed(self, reason): log.error("Connection failed: %s" % (reason, )) self.defer = None self.secs_to_wait = 10 if self.secs_to_wait >= 10 else self.secs_to_wait * 2 self.connection_lost()