def do_broker_callback_tasks(self, queue):
while True:
try:
msg = yield queue.get()
log.msg("DEBUG: do_broker_callback_tasks: got msg %s" % msg)
unpacked_msg = self.service.unpack(msg.content.body)
log.msg("DEBUG: do_broker_callback_tasks: unpacked_msg is %s" % str(unpacked_msg))
corr_id = msg.content.properties['correlation id']
if corr_id not in self.broker_callback_map:
log.err("do_broker_callback_tasks: invalid corr_id %s" % corr_id)
continue
d, f_name, exchange = self.broker_callback_map[corr_id]
if self.is_broker_error(unpacked_msg):
if unpacked_msg["err_id"] == error.ERROR_INTERNAL_SERVICE_ERROR:
d.errback(BrokerError(unpacked_msg["err_id"], unpacked_msg["err_msg"],
f_name=f_name, exchange=exchange))
else:
d.errback(error.EInternalError(unpacked_msg["err_id"]))
else:
d.callback(unpacked_msg)
del self.broker_callback_map[corr_id]
except Closed, e_inst:
log.msg("do_broker_callback_tasks: queue is closed: %s" % str(e_inst))
break
except Exception:
log.error(str(msg))
formatted_traceback = traceback.format_exc()
log.error(formatted_traceback)
def requestAvatarId(self, c):
def cbRequest(response):
log.msg('cbRequest Got Response code: %s'%response.code)
for s in response.headers.getRawHeaders("Set-Cookie", ()):
(key, value) = s.split("=")
if key == "session_cookie":
self.env["BSC_SESSION_COOKIE"] = value
if response.code == 200:
return defer.succeed(c.username)
else:
return defer.fail(error.UnauthorizedLogin())
def cbError(reason):
print "+++ error during http request +++"
reason.printTraceback()
return defer.fail(error.UnauthorizedLogin(reason.getErrorMessage()))
headers = {'Content-type': [ 'application/json' ]}
req = {'user': c.username, 'password' : c.password }
serialized_payload = json.dumps(req)
log.msg("Requesting Payload %s" % serialized_payload)
try:
d = agent.request('POST', self.url, Headers(headers), StringProducer(serialized_payload))
d.addCallbacks(cbRequest)
d.addErrback(cbError)
except Exception as e:
log.error("Auth failed",e)
return defer.fail(error.UnauthorizedLogin())
log.msg("Returning deferred")
return d
def refresh_mirror( working_dir, first_block, last_block ):
"""
Refresh the mirror:
* process all new invalidations
* grab and mirror any new profiles from the DHT
This gets called by Twisted every time there ought to be a new block.
"""
from twisted.python import log
from kademlia.network import Server
# make soure our bitcoind cached block index is up-to-speed
nameop_sequence = sync_blockchain( working_dir, first_block, last_block )
if nameop_sequence is None:
log.error("sync_blockchain(%s-%s) failed" % (first_block, last_block))
return None
# synchronize name registrations...
server = Server()
server.listen( dht.plugin.DHT_SERVER_PORT )
server.bootstrap( dht.plugin.DEFAULT_DHT_SERVERS ).addCallback( connect_done, server )
pass
def onMessage(self, msg):
log.debug(u"Received %s message from %s: %s" % (msg['type'], msg['from'], msg.body))
if msg.x and msg.x.defaultUri == 'jabber:x:delay':
log.debug(u"Ignoring delayed message")
return
if msg.body is None:
log.debug(u'Ignoring empty message')
self.chat(msg['from'], "Use $start to create menu")
return
try:
if msg['type'] == 'error':
log.debug("Error message received from %s: %s" % (msg['from'], msg.body))
return
elif msg['type'] == 'chat':
if self.message_handler:
self.message_handler.answer(
transport = self.chat,
recipient = msg['from'],
message = msg.body)
else:
self.chat(msg['from'], "Message received sir!")
except Exception, e:
log.error("Some error occurred: %s" % e)
def answer(self, transport, recipient, message):
user = self.auth.authenticate(recipient)
if user:
session = USSDSession.objects.recent(user)
if session:
log.debug("User: %s USSD Session: %s" % (user, session))
msg = str(message)
if msg == "$start":
response = handle_start(session)
elif msg == "$end":
response = handle_end(session)
elif msg == "$error":
import pdb
pdb.set_trace()
else:
if session.current_menu and not session.current_menu.is_finished():
response = handle_session(session, msg)
else:
response = "Use $start to create menu"
if recipient and response:
transport(recipient, response)
else:
log.error("Either recipient (%s) or response (%s) is not valid!" % (recipient, response))
return
else:
transport(recipient, "Error: No USSD Session!")
else:
transport(recipient, "Error: No User!")
def joinNetwork(self, knownNodeAddresses=None):
""" Causes the Node to join the Kademlia network; normally, this
should be called before any other DHT operations.
@param knownNodeAddresses: A sequence of tuples containing IP address
information for existing nodes on the
Kademlia network, in the format:
C{(<ip address>, (udp port>)}
@type knownNodeAddresses: tuple
"""
# Prepare the underlying Kademlia protocol
if self.port is not None:
try:
self._listeningPort = twisted.internet.reactor.listenUDP(self.port, self._protocol)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. %s", self.port, traceback.format_exc())
raise ValueError("%s lbrynet may already be running." % str(e))
#IGNORE:E1101
# Create temporary contact information for the list of addresses of known nodes
if knownNodeAddresses != None:
bootstrapContacts = []
for address, port in knownNodeAddresses:
contact = Contact(self._generateID(), address, port, self._protocol)
bootstrapContacts.append(contact)
else:
bootstrapContacts = None
# Initiate the Kademlia joining sequence - perform a search for this node's own ID
self._joinDeferred = self._iterativeFind(self.id, bootstrapContacts)
# #TODO: Refresh all k-buckets further away than this node's closest neighbour
# Start refreshing k-buckets periodically, if necessary
self.next_refresh_call = twisted.internet.reactor.callLater(
constants.checkRefreshInterval, self._refreshNode) #IGNORE:E1101
self.hash_watcher.tick()
return self._joinDeferred
def build_TLV_response(self, requestdata):
responsedata = OrderedDict()
for typ, value in requestdata.items():
if typ == 'NAME':
# send full host name - no truncation
value = self.service.hostname.encode("UTF-8")
elif typ == 'IPAD':
# send ipaddress as a string only if it is set
value = self.transport.getHost().host
# :todo: IPv6
if value == '0.0.0.0':
# do not send back an ip address
typ = None
elif typ == 'JSON':
# send port as a string
json_port = 9000 # todo: web.service.port
value = str(json_port)
elif typ == 'VERS':
# send server version
value = squeal.__version__
elif typ == 'UUID':
# send server uuid
value = self.service.uuid
elif typ == 'JVID':
# not handle, just log the information
typ = None
log.msg("Jive: %x:%x:%x:%x:%x:%x:" % struct.unpack('>6B', value),
logLevel=logging.INFO)
else:
log.error('Unexpected information request: %r', typ)
typ = None
if typ:
responsedata[typ] = value
return responsedata
def twisted_log(eventDict):
log = logging.getLogger('twisted')
if 'failure' in eventDict:
log.error(eventDict.get('why') or 'Unhandled exception' + '\n' + str(eventDict['failure'].getTraceback()))
elif 'warning' in eventDict:
log.warning(eventDict['warning'])
else:
log.debug(' '.join([str(m) for m in eventDict['message']]))
def removeListener(self, eventvalue, callback):
if not hasattr(callback, "__call__"):
callback = getattr(callback, eventvalue+"Callback"
) if hasattr(callback, eventvalue+"Callback"
) else getattr(callback, "callback")
try:
self.event_registry[eventvalue].remove(callback)
except Exception as ex:
log.error(ex)
def log_error(err, n):
if err.check(protocol.TimeoutError):
log.debug(
"Timeout while storing blob_hash %s at %s",
binascii.hexlify(blob_hash), n)
else:
log.error(
"Unexpected error while storing blob_hash %s at %s: %s",
binascii.hexlify(blob_hash), n, err.getErrorMessage())
def checkIfValidEmail(self, email):
"""
Checks if the passed email is valid based on the regex string
"""
valid_email = re.compile(
r"(^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$)")
if not valid_email.match(email):
log.error("Invalid email: %s" % email)
raise ValueError(email)
def get_msg(self, my_queue):
"""
The function takes message from the queue
"""
self.channel.basic_qos(prefetch_count=COUNT)
try:
for method_frame, properties, body in self.channel.consume(my_queue):
self.channel.basic_ack(method_frame.delivery_tag)
return body
except pika.exceptions, err_msg:
log.error(err_msg)
return False
def failure(failure, log, msg, *args):
"""Log a failure message from a deferred.
Args:
failure: twisted.python.failure.Failure
log: a python logger instance
msg: the message to log. Can use normal logging string interpolation.
the last argument will be set to the error message from the failure.
args: values to substitute into `msg`
"""
args += (failure.getErrorMessage(),)
exc_info = (failure.type, failure.value, failure.getTracebackObject())
log.error(msg, *args, exc_info=exc_info)
def _parseStunResponse(dgram, address, expectedTID=None, oldtids=[]):
mt, pktlen, tid = struct.unpack('!hh16s', dgram[:20])
if expectedTID is not None and expectedTID != tid:
# a response from an earlier request
if tid in oldtids:
# discard
if STUNVERBOSE:
log.msg("ignoring belated STUN response to %r from %s"%(
hexify(tid), repr(address)), system='stun')
return
log.msg("got unexpected STUN response %r != %r from %s"%
(hexify(expectedTID), hexify(tid),
repr(address),), system='stun')
return
resdict = {}
if mt == 0x0101:
#服务器的response message
log.msg("got STUN response to %r from %s"%(hexify(expectedTID),
repr(address)),
system='stun')
# response
remainder = dgram[20:]
while remainder:
avtype, avlen = struct.unpack('!hh', remainder[:4])
val = remainder[4:4+avlen]
avtype = StunTypes.get(avtype, '(Unknown type %04x)'%avtype)
remainder = remainder[4+avlen:]
if avtype in ('MAPPED-ADDRESS',
'CHANGED-ADDRESS',
'SOURCE-ADDRESS'):
dummy,family,port,addr = struct.unpack('!ccH4s', val)
addr = socket.inet_ntoa(addr)
if STUNVERBOSE:
print avtype, addr, port
if avtype == 'MAPPED-ADDRESS':
resdict['externalAddress'] = (addr, port)
elif avtype == 'CHANGED-ADDRESS':
resdict['_altStunAddress'] = (addr, address[1])
elif address[0] != addr:
# Some son of a bitch is rewriting packets on the way
# back. AAARGH.
log.msg('WARNING: packets are being rewritten %r != %r'%
(address, (addr,port)), system='stun')
return
else:
log.msg("STUN: unhandled AV %s, val %r"%(avtype,
repr(val)),
system='stun')
elif mt == 0x0111:
log.error("STUN got an error response")
return resdict
def _parseStunResponse(dgram, address, expectedTID=None, oldtids=[]):
mt, pktlen, tid = struct.unpack('!hh16s', dgram[:20])
if expectedTID is not None and expectedTID != tid:
# a response from an earlier request
if tid in oldtids:
# discard
if STUNVERBOSE:
log.msg("ignoring belated STUN response to %r from %s"%(
hexify(tid), repr(address)), system='stun')
return
log.msg("got unexpected STUN response %r != %r from %s"%
(hexify(expectedTID), hexify(tid),
repr(address),), system='stun')
return
resdict = {}
if mt == 0x0101:
log.msg("got STUN response to %r from %s"%(hexify(expectedTID),
repr(address)),
system='stun')
# response
remainder = dgram[20:]
while remainder:
avtype, avlen = struct.unpack('!hh', remainder[:4])
val = remainder[4:4+avlen]
avtype = StunTypes.get(avtype, '(Unknown type %04x)'%avtype)
remainder = remainder[4+avlen:]
if avtype in ('MAPPED-ADDRESS',
'CHANGED-ADDRESS',
'SOURCE-ADDRESS'):
dummy,family,port,addr = struct.unpack('!ccH4s', val)
addr = socket.inet_ntoa(addr)
if STUNVERBOSE:
print((avtype, addr, port))
if avtype == 'MAPPED-ADDRESS':
resdict['externalAddress'] = (addr, port)
elif avtype == 'CHANGED-ADDRESS':
resdict['_altStunAddress'] = (addr, address[1])
elif address[0] != addr:
# Some son of a bitch is rewriting packets on the way
# back. AAARGH.
log.msg('WARNING: packets are being rewritten %r != %r'%
(address, (addr,port)), system='stun')
return
else:
log.msg("STUN: unhandled AV %s, val %r"%(avtype,
repr(val)),
system='stun')
elif mt == 0x0111:
log.error("STUN got an error response")
return resdict
def checkIfValidPassword(self, password):
"""
Checks if the passed password is valid based on the regex string
"""
min_len = 6
max_len = 64
valid_password = re.compile(
r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))
if not valid_password.match(password):
log.error("Invalid password: %s" % password)
raise ValueError(password)
def resumeProducing(self):
if not self.request:
log.error("Resuming a producer with no request object")
return
data = yield self.fileObject.read(
min(self.bufferSize, self.size - self.bytesWritten))
if data:
self.request.write(data)
self.bytesWritten += len(data)
else:
log.err("Finished reading from grid file: %s" % self.fileObject)
self.request.unregisterProducer()
self.request.finish()
self.stopProducing()
def checkIfValidNickname(self, nickname):
"""
Checks if the passed nickname is valid based on the regex string
"""
min_len = 3
max_len = 64
valid_nickname = re.compile(
r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))
if not valid_nickname.match(nickname):
log.error("Invalid nick: %s" % nickname)
raise ValueError(nickname)
def lineReceived(self, line):
"""Dispatches commands from the child. So far there's only one. And
you have to use it. Or everything dies.
"""
args = line.split(' ')
command = args.pop(0)
if command == 'set-port':
port = int(args[0])
self.protocol.set_port(port)
elif command == 'request-done':
self.protocol.track_request_done()
else:
log.error("Child is babbling nonsense: {0!r}".format(line))
def checkIfValidNickname(self, nickname):
"""
Checks if the passed nickname is valid based on the regex string
"""
min_len = 3
max_len = 64
valid_nickname = re.compile(
r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))
if not valid_nickname.match(nickname):
log.error("Invalid nick: %s" % nickname)
raise ValueError(nickname)
def checkIfValidPassword(self, password):
"""
Checks if the passed password is valid based on the regex string
"""
min_len = 6
max_len = 64
valid_password = re.compile(
r"^(?i)[a-z0-9_-]{%s,%s}$" % (min_len, max_len))
if not valid_password.match(password):
log.error("Invalid password: %s" % password)
raise ValueError(password)
def scanip(self, entry):
"""
Scan IP againt Greynoise API
"""
def message(query):
log.msg(
eventid='cowrie.greynoise.result',
format=
'greynoise: Scan for %(IP)s with %(tag)s have %(conf)s confidence'
' along with the following %(meta)s metadata',
IP=entry['src_ip'],
tag=query['name'],
conf=query['confidence'],
meta=query['metadata'])
gnUrl = '{0}query/ip'.format(GNAPI_URL).encode('utf8')
headers = ({'User-Agent': [COWRIE_USER_AGENT]})
fields = {'key': self.apiKey, 'ip': entry['src_ip']}
try:
response = yield treq.post(url=gnUrl,
data=fields,
headers=headers,
timeout=10)
except (defer.CancelledError, error.ConnectingCancelledError,
error.DNSLookupError):
log.msg("GreyNoise requests timeout")
return
if response.code != 200:
rsp = yield response.text()
log.error("greynoise: got error {}".format(rsp))
return
j = yield response.json()
if self.debug:
log.msg("greynoise: debug: " + repr(j))
if j['status'] == "ok":
if "all" not in self.tags:
for query in j['records']:
if query['name'] in self.tags:
message(query)
else:
for query in j['records']:
message(query)
else:
log.msg("greynoise: no results for for IP {0}".format(
entry['src_ip']))
def joinNetwork(self, knownNodeAddresses=None):
""" Causes the Node to join the Kademlia network; normally, this
should be called before any other DHT operations.
@param knownNodeAddresses: A sequence of tuples containing IP address
information for existing nodes on the
Kademlia network, in the format:
C{(<ip address>, (udp port>)}
@type knownNodeAddresses: tuple
"""
# Prepare the underlying Kademlia protocol
if self.port is not None:
try:
self._listeningPort = twisted.internet.reactor.listenUDP(
self.port, self._protocol)
except error.CannotListenError as e:
import traceback
log.error("Couldn't bind to port %d. %s", self.port,
traceback.format_exc())
raise ValueError("%s lbrynet may already be running." % str(e))
#IGNORE:E1101
# Create temporary contact information for the list of addresses of known nodes
if knownNodeAddresses != None:
bootstrapContacts = []
for address, port in knownNodeAddresses:
contact = Contact(self._generateID(), address, port,
self._protocol)
bootstrapContacts.append(contact)
else:
bootstrapContacts = None
# Initiate the Kademlia joining sequence - perform a search for this node's own ID
self._joinDeferred = self._iterativeFind(self.id, bootstrapContacts)
# #TODO: Refresh all k-buckets further away than this node's closest neighbour
# def getBucketAfterNeighbour(*args):
# for i in range(160):
# if len(self._buckets[i]) > 0:
# return i+1
# return 160
# df.addCallback(getBucketAfterNeighbour)
# df.addCallback(self._refreshKBuckets)
#protocol.reactor.callLater(10, self.printContacts)
#self._joinDeferred.addCallback(self._persistState)
#self._joinDeferred.addCallback(self.printContacts)
# Start refreshing k-buckets periodically, if necessary
self.next_refresh_call = twisted.internet.reactor.callLater(
constants.checkRefreshInterval, self._refreshNode) #IGNORE:E1101
self.hash_watcher.tick()
return self._joinDeferred
def run_server(foreground=False):
""" run the blockstored server
"""
global bitcoind
prompt_user_for_chaincom_details()
bitcoind = init_bitcoind()
from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE
from .lib.config import BLOCKSTORED_TAC_FILE
from .lib.config import START_BLOCK
working_dir = get_working_dir()
current_dir = os.path.abspath(os.path.dirname(__file__))
tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE)
log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE)
pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE)
start_block, current_block = get_index_range()
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (pid_file,
log_file,
tac_file)
try:
# refresh_index(335563, 335566, initial_index=True)
if start_block != current_block:
refresh_index(start_block, current_block, initial_index=True)
blockstored = subprocess.Popen(
command, shell=True, preexec_fn=os.setsid)
log.info('Blockstored successfully started')
except IndexError, ie:
# indicates that we don't have the latest block
log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \
"Please verify that your bitcoin provider has " + \
"processed up to block %s.\n" % (START_BLOCK) + \
" Example: bitcoin-cli getblockcount" )
try:
os.killpg(blockstored.pid, signal.SIGTERM)
except:
pass
exit(1)
def run_server(foreground=False):
""" run the blockstored server
"""
global bitcoind
prompt_user_for_chaincom_details()
bitcoind = init_bitcoind()
from .lib.config import BLOCKSTORED_PID_FILE, BLOCKSTORED_LOG_FILE
from .lib.config import BLOCKSTORED_TAC_FILE
from .lib.config import START_BLOCK
working_dir = get_working_dir()
current_dir = os.path.abspath(os.path.dirname(__file__))
tac_file = os.path.join(current_dir, BLOCKSTORED_TAC_FILE)
log_file = os.path.join(working_dir, BLOCKSTORED_LOG_FILE)
pid_file = os.path.join(working_dir, BLOCKSTORED_PID_FILE)
start_block, current_block = get_index_range()
if foreground:
command = 'twistd --pidfile=%s -noy %s' % (pid_file, tac_file)
else:
command = 'twistd --pidfile=%s --logfile=%s -y %s' % (
pid_file, log_file, tac_file)
try:
# refresh_index(335563, 335566, initial_index=True)
if start_block != current_block:
refresh_index(start_block, current_block, initial_index=True)
blockstored = subprocess.Popen(command,
shell=True,
preexec_fn=os.setsid)
log.info('Blockstored successfully started')
except IndexError, ie:
# indicates that we don't have the latest block
log.error("\n\nFailed to find the first blockstore record (got block %s).\n" % current_block + \
"Please verify that your bitcoin provider has " + \
"processed up to block %s.\n" % (START_BLOCK) + \
" Example: bitcoin-cli getblockcount" )
try:
os.killpg(blockstored.pid, signal.SIGTERM)
except:
pass
exit(1)
def dictionary(self, args, irc):
'''(dict [word or phrase]) -- Lookup up a word using WordNet.
'''
if not args:
return u'Missing word or phrase to look up'
try:
cmnd = 'sdcv -u WordNet -n --utf8-input --utf8-output'.split() + args
o = [line.strip() for line in sp.check_output(cmnd).split('\n') if line]
if len(o) == 1:
return o[0]
else:
return u' '.join(o[3:])
except:
log.error('[Error]: dict {}'.format(sys.exc_info()[0]))
def handle_line(self, line, repo_lock=defer.DeferredLock()):
try:
line = line.decode('utf-8', 'ignore').strip()
match = re.search(r'(^|\s+)!pq\s+(?P<link>\S+)(\s+::\S+|$)', line)
if not match:
log.noise('Non-patchbot line, ignoring: {}'.format(line.encode('utf-8', 'ignore')))
defer.returnValue(None)
link = match.group('link').encode('ascii')
if not re.search('https?://', link, re.IGNORECASE):
log.warn('Incorrect non-http link, skipping: {}'.format(link))
defer.returnValue(None)
except UnicodeError as err:
log.warn('Failed to recode line ({!r}): {}'.format(line, err))
defer.returnValue(None)
## Grab the patch
dst_base = '{}.patch'.format(sha1(link).hexdigest())
dst_path = self.dst_path.child(dst_base)
if dst_path.exists():
log.debug( 'Patch already exists'
' (file: {}, link: {}), skipping'.format(dst_path, link) )
defer.returnValue(None)
# Not via tmpfile to prevent multiple downloads of the same paste
try: yield downloadPage(link, dst_path.open('wb'), timeout=120)
except:
if dst_path.exists(): dst_path.remove()
raise
## Commit into repo and push
yield repo_lock.acquire()
try:
for cmd, check in [
(['add', dst_base], True),
(['commit', '-m', 'New patch: {}'.format(link)], False),
(['push'], True) ]:
out, err, code = yield getProcessOutputAndValue(
'/usr/bin/git', cmd, path=self.dst_path.path )
if check and code:
log.error('\n'.join([
'Failed to commit/push new patch into repo',
'Command: {}'.format(cmd), 'Exit code: {}'.format(code),
'Stdout:\n {}'.format('\n '.join(out.splitlines())),
'Stderr:\n {}'.format('\n '.join(err.splitlines())) ]))
break
else: log.debug('Successfully pushed paste: {}'.format(link))
finally: repo_lock.release()
def setStunnedAddress(self, results):
''' Handle results of the rtp/rtcp STUN. We have to check that
the results have the same IP and usable port numbers
'''
log.msg("got NAT mapping back! %r"%(results), system='rtp')
rtpres, rtcpres = results
if rtpres[0] != defer.SUCCESS or rtcpres[0] != defer.SUCCESS:
# barf out.
log.msg("uh oh, stun failed %r"%(results), system='rtp')
else:
# a=RTCP might help for wacked out RTCP/RTP pairings
# format is something like "a=RTCP:AUDIO 16387"
# See RFC 3605
code1, rtp = rtpres
code2, rtcp = rtcpres
if rtp[0] != rtcp[0]:
print "stun gave different IPs for rtp and rtcp", results
# We _should_ try and see if we have working rtp and rtcp, but
# this seems almost impossible with most firewalls. So just try
# to get a working rtp port (an even port number is required).
elif False and ((rtp[1] % 2) != 0):
log.error("stun: unusable RTP/RTCP ports %r, retry #%d"%
(results, self._stunAttempts),
system='rtp')
# XXX close connection, try again, tell user
if self._stunAttempts > 8:
# XXX
print "Giving up. Made %d attempts to get a working port" \
% (self._stunAttempts)
self._stunAttempts += 1
defer.maybeDeferred(
self.rtpListener.stopListening).addCallback( \
lambda x:self.rtcpListener.stopListening()).addCallback( \
lambda x:self._socketCreationAttempt())
else:
# phew. working NAT
log.msg("stun: sane NAT for RTP/RTCP; rtp addr: %s" \
% (rtp,), system='rtp')
#Register address and port
self._extIP, self._extRTPPort = rtp
self._stunAttempts = 0
d = self._socketCompleteDef
del self._socketCompleteDef
d.callback(self.cookie)
def setStunnedAddress(self, results):
''' Handle results of the rtp/rtcp STUN. We have to check that
the results have the same IP and usable port numbers
'''
log.msg("got NAT mapping back! %r" % (results), system='rtp')
rtpres, rtcpres = results
if rtpres[0] != defer.SUCCESS or rtcpres[0] != defer.SUCCESS:
# barf out.
log.msg("uh oh, stun failed %r" % (results), system='rtp')
else:
# a=RTCP might help for wacked out RTCP/RTP pairings
# format is something like "a=RTCP:AUDIO 16387"
# See RFC 3605
code1, rtp = rtpres
code2, rtcp = rtcpres
if rtp[0] != rtcp[0]:
print "stun gave different IPs for rtp and rtcp", results
# We _should_ try and see if we have working rtp and rtcp, but
# this seems almost impossible with most firewalls. So just try
# to get a working rtp port (an even port number is required).
elif False and ((rtp[1] % 2) != 0):
log.error("stun: unusable RTP/RTCP ports %r, retry #%d" %
(results, self._stunAttempts),
system='rtp')
# XXX close connection, try again, tell user
if self._stunAttempts > 8:
# XXX
print "Giving up. Made %d attempts to get a working port" \
% (self._stunAttempts)
self._stunAttempts += 1
defer.maybeDeferred(
self.rtpListener.stopListening).addCallback( \
lambda x:self.rtcpListener.stopListening()).addCallback( \
lambda x:self._socketCreationAttempt())
else:
# phew. working NAT
log.msg("stun: sane NAT for RTP/RTCP; rtp addr: %s" \
% (rtp,), system='rtp')
#Register address and port
self._extIP, self._extRTPPort = rtp
self._stunAttempts = 0
d = self._socketCompleteDef
del self._socketCompleteDef
d.callback(self.cookie)
def consume(self, data):
"""
We've monkey patched the consumer's `consume` method to point to this
one intead. All incoming data are messages that need to be sent
to TruTeq over USSD. Use `ssmi_client.send_ussd` to do that.
"""
# default to ussd
if data.get('type', 'ussd') == 'ussd':
self.ssmi_client.send_ussd(
str(data['msisdn']), # str everything because the SSMIClient
str(data['message']), # isn't happy with Unicode
str(data['ussd_type']))
elif data['type'] == 'sms':
self.ssmi_client.send_sms(
str(data['msisdn']),
str(data['message'])
)
else:
log.error('Unknown TruTeq message type in %s' % data)
def __init__(self, url, filepath, callback=None):
"""
@param url = resource to fetch
@param filepath = string path of file to read
@param callback = function to call with the filepath as arguement
after sucess response from url
"""
if not os.path.exists(filepath):
e = FileRouterException('{0} not found'.format(filepath))
log.err(e)
raise e
with open(filepath) as ofile:
try:
self.data = json.loads(ofile.read())
except Exception as e:
log.error(e)
raise e
self.url = url
self.file_callback = callback
def run(self):
if not os.path.exists("dist/%s.app/" % config.appname):
log.error("You have to run py2app first")
return
# getting Python.framework path
cmd = "python-config --includes | awk -F'-I' '{print $2}' | sed 's/\/include.*$//'"
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
pythonFrameworkPath = process.stdout.read().strip()
version = pythonFrameworkPath.split("/")[-1]
# folder of the app bundle
frameFolder = ("dist/%s.app/Contents/Frameworks/Python.framework/Versions/%s"
% (config.appname, version))
# copy it if it does't exists
if not os.path.exists(frameFolder):
os.makedirs(frameFolder)
os.system("cp %s/Python %s" % (pythonFrameworkPath, frameFolder))
os.system("chmod +x dist/%s.app/Contents/Resources/contrib/tor" % config.appname)
def run(self):
if not os.path.exists("dist/%s.app/" % config.appname):
log.error("You have to run py2app first")
return
# getting Python.framework path
cmd = "python-config --includes | awk -F'-I' '{print $2}' | sed 's/\/include.*$//'"
process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE)
pythonFrameworkPath = process.stdout.read().strip()
version = pythonFrameworkPath.split("/")[-1]
# folder of the app bundle
frameFolder = (
"dist/%s.app/Contents/Frameworks/Python.framework/Versions/%s" %
(config.appname, version))
# copy it if it does't exists
if not os.path.exists(frameFolder):
os.makedirs(frameFolder)
os.system("cp %s/Python %s" % (pythonFrameworkPath, frameFolder))
os.system("chmod +x dist/%s.app/Contents/Resources/contrib/tor" %
config.appname)
def getTags(_file):
try:
tags = File(_file, easy=True)
return {
'title': tags['TITLE'][0].encode('utf8'),
'artist': tags['ARTIST'][0].encode('utf8'),
'album': tags['ALBUM'][0].encode('utf8'),
'genre': tags['GENRE'][0].encode('utf8'),
'art': hasEmbeddedArt(_file)
}
except Exception as err:
Logger.error('Error on file {} - {}'.format(
str(_file),
err)
)
return False
def initalizeDBus():
global flag
if flag:
return
global dbus, bus, manager
try:
import dbus
try:
bus = dbus.SessionBus()
manager = dbus.Interface(bus.get_object("org.bluez", "/"),
"org.bluez.Manager")
except:
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez", "/"),
"org.bluez.Manager")
log.msg("DBus available")
flag = True
except Exception, err:
log.msg("DBus unavailable")
log.error(err)
if isAndroid(): flag = True
def initalizeDBus():
global flag
if flag:
return
global dbus, bus, manager
try:
import dbus
try:
bus = dbus.SessionBus()
manager = dbus.Interface(bus.get_object("org.bluez", "/"),
"org.bluez.Manager")
except:
bus = dbus.SystemBus()
manager = dbus.Interface(bus.get_object("org.bluez", "/"),
"org.bluez.Manager")
log.msg("DBus available")
flag = True
except Exception, err:
log.msg("DBus unavailable")
log.error(err)
if isAndroid(): flag = True
def dataReceived(self, data):
if self.transport.protocol.state == txws.FRAMES:
if not self._session:
session_id = self.transport.protocol.location.rstrip('/').split('/')[-1]
if not base.simple_id(session_id):
log.err("Invalid Session ID: %r"%( session_id, ))
self.write_error("session invalid")
self.transport.loseConnection()
return
log.msg("New Connection on Session: %r"%( session_id, ))
session = self.factory.server.session(session_id, create=False)
if not session:
log.err("Unkown/no-auth session: %r"%( session_id, ))
self.write_error("session unknown")
self.transport.loseConnection()
return
self._session = session
self._session.protocols.append(self)
self.write_welcome()
else:
session = self._session
try:
channel_id, data = data.split(',', 1)
except ValueError:
log.error('Mis-formatted request (no ,)')
self.write_error('Missing comma in request')
return
if not channel_id:
"""Protocol-level messages, currently nothing"""
pass
elif not base.simple_id(channel_id):
log.err("Invalid Channel ID: %r"%( channel_id, ))
self.write_error("invalid channel")
self.transport.loseConnection()
return
else:
session.on_incoming(channel_id, data)
self.ready = True
def __init__(self, target, listener = None, connectionLost = None):
if SCOConnected(target):
log.error("Can't connect twice to the same device")
return
self.sock = bluetooth.BluetoothSocket(bluetooth.SCO)
try:
self.sock.connect((target,))
except Exception, err:
log.error("Failed while trying to connect SCO")
log.error(err)
return
def __init__(self, target, listener=None, connectionLost=None):
if SCOConnected(target):
log.error("Can't connect twice to the same device")
return
self.sock = bluetooth.BluetoothSocket(bluetooth.SCO)
try:
self.sock.connect((target, ))
except Exception, err:
log.error("Failed while trying to connect SCO")
log.error(err)
return
def __init__(self, interface: bytes, config_dict: Dict[str, Any]) -> None:
# logfile path relative to config dir if not abs path
log_filename = logfile.get()
if log_filename.strip(): # catches empty filename
if not os.path.isabs(log_filename):
log_filename = os.path.join(config.config_dir, log_filename)
ensure_dir_exists(log_filename)
if logging_rotate_daily.get():
logging_file = DailyLogFile(log_filename, '.')
else:
logging_file = open(log_filename, 'a')
globalLogPublisher.addObserver(textFileLogObserver(logging_file))
globalLogPublisher.addObserver(textFileLogObserver(sys.stderr))
log.info('piqueserver started on %s' % time.strftime('%c'))
self.config = config_dict
if random_rotation:
self.map_rotator_type = random_choice_cycle
else:
self.map_rotator_type = itertools.cycle # pylint: disable=redefined-variable-type
self.default_time_limit = default_time_limit.get()
self.default_cap_limit = cap_limit.get()
self.advance_on_win = int(advance_on_win.get())
self.win_count = itertools.count(1)
self.bans = NetworkDict()
# attempt to load a saved bans list
try:
with open(os.path.join(config.config_dir, bans_file.get()),
'r') as f:
self.bans.read_list(json.load(f))
log.debug("loaded {count} bans", count=len(self.bans))
except FileNotFoundError:
log.debug("skip loading bans: file unavailable",
count=len(self.bans))
except IOError as e:
log.error('Could not read bans.txt: {}'.format(e))
except ValueError as e:
log.error('Could not parse bans.txt: {}'.format(e))
self.hard_bans = set() # possible DDoS'ers are added here
self.player_memory = deque(maxlen=100)
if len(self.name) > MAX_SERVER_NAME_SIZE:
log.warn('(server name too long; it will be truncated to "%s")' %
(self.name[:MAX_SERVER_NAME_SIZE]))
self.respawn_time = respawn_time_option.get()
self.respawn_waves = respawn_waves.get()
if game_mode.get() == 'ctf':
self.game_mode = CTF_MODE
elif game_mode.get() == 'tc':
self.game_mode = TC_MODE
elif self.game_mode is None:
raise NotImplementedError('invalid game mode: %s' % game_mode)
self.game_mode_name = game_mode.get().split('.')[-1]
self.team1_name = team1_name.get()
self.team2_name = team2_name.get()
self.team1_color = tuple(team1_color.get())
self.team2_color = tuple(team2_color.get())
self.friendly_fire = friendly_fire.get()
self.friendly_fire_on_grief = friendly_fire_on_grief.get()
self.friendly_fire_time = grief_friendly_fire_time.get()
self.spade_teamkills_on_grief = spade_teamkills_on_grief.get()
self.fall_damage = fall_damage.get()
self.teamswitch_interval = teamswitch_interval.get()
self.teamswitch_allowed = teamswitch_allowed.get()
self.max_players = max_players.get()
self.melee_damage = melee_damage.get()
self.max_connections_per_ip = max_connections_per_ip.get()
self.passwords = passwords.get()
self.server_prefix = server_prefix.get()
self.time_announcements = time_announcements.get()
self.balanced_teams = balanced_teams.get()
self.login_retries = login_retries.get()
# voting configuration
self.default_ban_time = default_ban_duration.get()
self.speedhack_detect = speedhack_detect.get()
if user_blocks_only.get():
self.user_blocks = set()
self.set_god_build = set_god_build.get()
self.debug_log = debug_log_enabled.get()
if self.debug_log:
# TODO: make this configurable
pyspades.debug.open_debug_log(
os.path.join(config.config_dir, 'debug.log'))
if ssh_enabled.get():
from piqueserver.ssh import RemoteConsole
self.remote_console = RemoteConsole(self)
irc = irc_options.get()
if irc.get('enabled', False):
from piqueserver.irc import IRCRelay
self.irc_relay = IRCRelay(self, irc)
if status_server_enabled.get():
from piqueserver.statusserver import StatusServerFactory
self.status_server = StatusServerFactory(self)
if ban_publish.get():
from piqueserver.banpublish import PublishServer
self.ban_publish = PublishServer(self, ban_publish_port.get())
if bans_urls.get():
from piqueserver import bansubscribe
self.ban_manager = bansubscribe.BanManager(self)
self.start_time = reactor.seconds()
self.end_calls = []
# TODO: why is this here?
create_console(self)
for user_type, func_names in rights.get().items():
for func_name in func_names:
commands.add_rights(user_type, func_name)
port = self.port = port_option.get()
ServerProtocol.__init__(self, port, interface)
self.host.intercept = self.receive_callback
try:
self.set_map_rotation(self.config['rotation'])
except MapNotFound as e:
log.critical('Invalid map in map rotation (%s), exiting.' % e.map)
raise SystemExit
self.update_format()
self.tip_frequency = tip_frequency.get()
if self.tips is not None and self.tip_frequency > 0:
reactor.callLater(self.tip_frequency * 60, self.send_tip)
self.master = register_master_option.get()
self.set_master()
self.http_agent = web_client.Agent(reactor)
ip_getter = ip_getter_option.get()
if ip_getter:
self.get_external_ip(ip_getter)
def run() -> None:
"""
runs the server
"""
# apply scripts
protocol_class = FeatureProtocol
connection_class = FeatureConnection
script_objects = []
script_names = scripts_option.get()
script_dir = os.path.join(config.config_dir, 'scripts/')
for script in script_names[:]:
try:
# this finds and loads scripts directly from the script dir
# no need for messing with sys.path
f, filename, desc = imp.find_module(script, [script_dir])
module = imp.load_module('piqueserver_script_namespace_' + script,
f, filename, desc)
script_objects.append(module)
except ImportError as e:
# warning: this also catches import errors from inside the script
# module it tried to load
try:
module = importlib.import_module(script)
script_objects.append(module)
except ImportError as e:
log.error("(script '{}' not found: {!r})".format(script, e))
script_names.remove(script)
for script in script_objects:
protocol_class, connection_class = script.apply_script(
protocol_class, connection_class, config.get_dict())
# apply the game_mode script
if game_mode.get() not in ('ctf', 'tc'):
# must be a script with this game mode
module = None
try:
game_mode_dir = os.path.join(config.config_dir, 'game_modes/')
f, filename, desc = imp.find_module(game_mode.get(),
[game_mode_dir])
module = imp.load_module(
'piqueserver_gamemode_namespace_' + game_mode.get(), f,
filename, desc)
except ImportError as e:
try:
module = importlib.import_module(game_mode.get())
except ImportError as e:
log.error("(game_mode '%s' not found: %r)" %
(game_mode.get(), e))
if module:
protocol_class, connection_class = module.apply_script(
protocol_class, connection_class, config.get_dict())
protocol_class.connection_class = connection_class
interface = network_interface.get().encode('utf-8')
# instantiate the protocol class once. It will set timers and hooks to keep
# itself running once we start the reactor
protocol_class(interface, config.get_dict())
log.debug('Checking for unregistered config items...')
unused = config.check_unused()
if unused:
log.warn('The following config items are not used:')
pprint(unused)
log.info('Started server...')
profile = logging_profile_option.get()
if profile:
import cProfile
cProfile.runctx('reactor.run()', None, globals())
else:
reactor.run()
import web
import sys, os
from twisted.python import log
from twisted.internet import defer, reactor
def main(config_file):
log.startLogging(sys.stdout)
application = web.Application(config_file)
port = os.environ.get("PORT", 8888)
reactor.listenTCP(int(port), application)
reactor.run()
if __name__ == "__main__":
if len(sys.argv) > 1:
main(sys.argv[1])
else:
log.error("no config file given")
sys.exit(-1)
def connectionLost(self, reason):
if reason.type != ConnectionLost:
log.debug('Disconnect from %s' % self.transport.getPeer())
else:
log.error('Disconnect from %s: %s' %
(self.transport.getPeer(), reason))
def openFailed(self, reason):
log.error("Open failed due to %r", reason)
if self.start_defer:
self.start_defer.errback(self)
def run(self):
def safe_unicode(obj, *args):
try:
return unicode(obj, *args)
except UnicodeDecodeError:
# obj is byte string
ascii_text = str(obj).encode('string_escape')
return unicode(ascii_text)
def get_appname(rts):
rts = rts.lower()
for name in ('paypage', 'ccentry', 'mis'):
if name in rts:
return name
return 'UnknownEntryPoint'
log.debug("Execute model checks and execution of loop")
# process the accumulated data into the model
bl_trace_id = []
traces, last_trace_end_str = {}, None
trace_id = self.guid
start_trace = False
local = []
latest_datetime = None
try:
for line in self.accumulator:
if not start_trace:
if 'TRACE START' == line[:11]:
start_trace = True
latest_datetime = extract_datetime(line)
local = []
else:
if 'TRACE END' in line[:9]:
appname = get_appname(local[0])
trace_id = time.strftime(
"%Y-%m-%d_%H-%M-%S",
latest_datetime) + '_' + appname + '_' + str(
hex(hash(repr(local))))
if trace_id not in bl_trace_id:
bl_trace_id.append(trace_id)
last_trace_end_str = line
start_trace = False
traces[trace_id] = local
if trace_id not in self.cache:
self.cache[trace_id] = local
local = []
start_trace = False
else:
local.append(line)
# keep only what hasn't been processed yet
if last_trace_end_str in self.accumulator:
self.accumulator = self.accumulator[self.accumulator.
index(last_trace_end_str):]
log.debug("Process all %d traces available" % len(traces))
for trace_id in traces:
self.processTrace(traces[trace_id], trace_id)
try:
o = open('./cache/' + trace_id + '.html', 'w')
o.write(
pygment_content("<?php\n" + '\n'.join([
safe_unicode(rts) for rts in self.cache[trace_id]
if not has_blacklisted_function(rts)
]) + "\n?>"))
o.close()
except IOError, error:
log.error("Exception writing %s (error:%s)" %
('./cache/' + trace_id + '.html', error))
# generate the JSON file for frontend to consume
self.update_JSON()
def failure(self, format, failure, *args, **kwargs):
log.error(failure, format.format(args, **kwargs))
def log_error(err):
log.error(err.getErrorMessage())
class Modeler(object):
def __init__(self):
self.times = []
self.guid = 1
self.cuid = 1
self.g = digraph()
self.context = {}
self.rtids = {}
self.node_location = {}
self.accumulator = []
self.trace_ids = []
self.cache = {}
def feedAccumulator(self, data):
self.accumulator.append(data)
@staticmethod
def __node_uid(trace_id, node_str, cur_index=''):
return hash(str(trace_id) + node_str + str(cur_index))
@staticmethod
def add_node(g,
nid,
location=None,
linenumber=None,
context_id=None,
trace_id=None,
method=None,
data=None,
_type=None,
_id=None):
if not g.has_node(nid):
g.add_node(nid)
g.node_attr[nid] = {
'return': '',
'context_id': context_id,
'trace_id': trace_id,
'location': location,
'linenumber': linenumber,
'method': method,
'data': data,
'type': _type,
'rtid': False,
'sequence': _id
}
@staticmethod
def add_edge(g, e):
if not g.has_edge(e):
g.add_edge(e)
else:
g.set_edge_weight(e, g.edge_weight(e) + 1)
def update_rtid(self, lst):
for e in lst:
if e not in self.rtids:
self.rtids[e] = re.compile(e, re.I)
@staticmethod
def regexp_interesect(data, dct):
if not isinstance(data, str):
data = str(repr(data))
for reg in dct:
if dct[reg].search(data):
return True
return False
def update_model_taints(self):
# get latest list of rtids
self.update_rtid(db_list_rtid())
# make sure we don't iterate if no data to check with
if 1 > len(self.rtids):
return
# loop over the entire graph and update the 'rtid' node attr
for n in self.g:
if Modeler.regexp_interesect(
self.g.node_attr[n]['method'],
self.rtids) or Modeler.regexp_interesect(
self.g.node_attr[n]['data'],
self.rtids) or Modeler.regexp_interesect(
self.g.node_attr[n]['return'], self.rtids):
# update taint
self.g.node_attr[n]['rtid'] = True
def processTrace(self, trace, trace_id):
callstack = Stack()
prev_node, cur_node = None, None
pre_l = None
cur_i = 0
if trace_id not in self.trace_ids:
self.trace_ids.append(trace_id)
for l in trace:
self.guid += 1
cur_i += 1
suc, trace, token = get_cleaned_trace_node(l)
if not suc:
continue
else:
trace_info = extract_information(token, trace)
if trace_info and isinstance(trace_info, dict):
_type = trace_info['type']
if _type == 'main':
main_nuid = Modeler.__node_uid(trace_id, l)
Modeler.add_node(self.g, main_nuid,
trace_info['filename'],
trace_info['line'], None, trace_id,
'main', self.guid)
callstack.push(main_nuid)
elif _type == 'return':
# only keep the latest value
if not callstack.empty():
prev_node = callstack.pop()
self.g.node_attr[prev_node]['return'] = trace_info[
'value']
elif _type == 'call':
cur_node = Modeler.__node_uid(trace_id, l)
Modeler.add_node(self.g, cur_node,
trace_info['filename'],
trace_info['line'], None, trace_id,
trace_info['function'],
trace_info['parameters'], 'call',
self.guid)
if not callstack.empty():
prev_node = callstack.top()
edge = (prev_node, cur_node)
Modeler.add_edge(self.g, edge)
# set label to the edge as parameters
self.g.set_edge_label(
edge, ','.join(trace_info['parameters']))
callstack.push(cur_node)
elif _type == 'assignment':
cur_node = Modeler.__node_uid(trace_id, l)
Modeler.add_node(self.g, cur_node,
trace_info['filename'],
trace_info['line'], None, trace_id,
trace_info['variable'],
trace_info['value'], 'assignment',
self.guid)
if not callstack.empty():
prev_node = callstack.top()
edge = (prev_node, cur_node)
Modeler.add_edge(self.g, edge)
# set label to the edge as parameters
self.g.set_edge_label(edge, trace_info['value'])
# callstack.push(cur_node)
self.update_model_taints()
# self.write_dot_slice(trace_id)
def update_JSON(self):
log.debug("update_JSON")
# only output simple paths and possible ways to investigate
jsonified_data = {
'injected': self.rtids.keys(),
'hash': '',
'trace': {},
'knowledge': {
'function': {},
'file': {}
}
}
def safe_unicode(obj, *args):
try:
return unicode(obj, *args)
except UnicodeDecodeError:
# obj is byte string
ascii_text = str(obj).encode('string_escape')
return unicode(ascii_text)
def lmt_size(rts):
length = len(rts)
if 256 >= length:
return rts
return rts[:256]
# always output the top 50 traces
sorted_traces = self.trace_ids
sorted_traces.sort(reverse=True)
for trace_id in sorted_traces[:min(100, len(sorted_traces))]:
if trace_id in jsonified_data['trace']:
continue
jsonified_data['trace'][trace_id] = {
'callchain': [],
'function': {},
'file': {},
}
tlst_nodes = {}
for n in self.g:
if trace_id == self.g.node_attr[n]['trace_id']:
tlst_nodes[self.g.node_attr[n]['sequence']] = n
slst_nodes = [tlst_nodes[k] for k in sorted(tlst_nodes.keys())]
if 0 == len(slst_nodes):
del jsonified_data['trace'][trace_id]
else:
for n in slst_nodes:
if self.g.node_attr[n]['type'] not in ('call', 'main'):
continue
if self.g.node_attr[n]['rtid']:
nd = self.g.node_attr[n]
method = safe_unicode(nd['method'])
data = safe_unicode(nd['data'])
ret = safe_unicode(nd['return'])
if has_blacklisted_function(nd['location']):
continue
jsonified_data['trace'][trace_id]['callchain'].append([
self.g.node_attr[n]['type'], nd['location'],
nd['linenumber'], method,
lmt_size(data),
lmt_size(ret)
])
"""
if method not in jsonified_data['trace'][trace_id]['function']:
jsonified_data['trace'][trace_id]['function'][method] = []
jsonified_data['trace'][trace_id]['function'][method].append(data)
if method not in jsonified_data['knowledge']['function']:
jsonified_data['knowledge']['function'][method] = []
if data not in jsonified_data['knowledge']['function'][method]:
jsonified_data['knowledge']['function'][method] = data
if nd['location'] not in jsonified_data['trace'][trace_id]['file']:
jsonified_data['trace'][trace_id]['file'][nd['location']] = []
jsonified_data['trace'][trace_id]['file'][nd['location']].append(nd['linenumber'])
"""
if 0 == len(jsonified_data['trace'][trace_id]['callchain']):
del jsonified_data['trace'][trace_id]
jsonified_data['hash'] = hash(
str(
repr(jsonified_data['trace']) +
repr(jsonified_data['injected'])))
o = open('assmnt-data.json', 'w')
o.write(
json.dumps(jsonified_data,
ensure_ascii=False,
separators=(',', ':')))
o.close()
def write_dot_slice(self, trace_id):
new_g = digraph()
new_g.add_graph(self.g)
# create new graph with only the CWE instances
for n in new_g:
if self.g.node_attr[n]['trace_id'] != trace_id:
new_g.del_node(n)
continue
fname = './graphs/%s-trace' % str(trace_id)
self.write_dot(new_g, fname)
print "Generated: %s" % fname
def write_dot(self, g, fname):
if 1 > len(g):
return
def pen_width(num, local_max):
max_penwidth = 7
val = int(
math.ceil(float(num) / float(local_max) * float(max_penwidth)))
return max(1, min(val, max_penwidth))
def node_call_labels(dct):
"""
<f0> Function Name| $value1 | ... | return values"
"""
lbl_str = "%s:%s |" % (dct['location'], dct['linenumber'])
lbl_str += "<f0> " + xml_entities(dct['method']) + "()"
for e in dct['data']:
lbl_str += ' | ' + xml_entities(e)
lbl_str += '| <f1> T_RET: ' + xml_entities(dct['return'])
return lbl_str
# generate DOT file with custom information (source, sink, and custom colors)
dstr = """digraph runtime {
graph [rankdir = "LR"];
node [fontsize = 12];
"""
for n in g:
node_type = 'shape=ellipse'
if self.g.node_attr[n]['type'] == 'call':
node_type = 'shape=record'
node_type += ", label=\"%s\"" % node_call_labels(
self.g.node_attr[n])
elif self.g.node_attr[n]['type'] == 'assignment':
node_type = 'shape=record, style="filled,bold" penwidth=1'
node_type += ",label=\"%s=%s\"" % (
xml_entities(self.g.node_attr[n]['method']),
xml_entities(self.g.node_attr[n]['data']))
else:
if self.g.node_attr[n]['method'] == 'return':
node_type += ", label=\"T_RETURN:%s\"" % self.g.node_attr[
n]['data'].replace('"', '\\"')
else:
node_type += ", label=\"%s\"" % self.g.node_attr[n][
'method']
if self.g.node_attr[n]['rtid']:
node_type += ', fillcolor="#AA0114"'
#if 'source' in g.node_attr[n]['nodetype'] or 'sink' in g.node_attr[n]['nodetype']:
# node_type = 'shape=box, style=filled, fillcolor="%s"' % color_node(g.node_attr[n]['nodetype'])
# get the size of the pen based on the number of GUID
#node_type += ', penwidth=%d' % int(pen_width(len(g.node_attr[n]['guid']), max_guid))
dstr += ' "%s" [%s];\n' % (n, node_type)
for e in g.edges():
n1, n2 = e
node_type = ""
#l_e_prop = self.g.edge_label(e).split(',')
if self.g.node_attr[n1]['rtid'] and self.g.node_attr[n2]['rtid']:
node_type += 'color="#AA0114"'
else:
node_type += 'color="black"'
node_type += ', penwidth=%d' % pen_width(g.edge_weight(e), 10)
#node_type += ', label="%s"' % ','.join(l_e_prop).replace('"', '\\"')
addon_n1 = ':f0' if self.g.node_attr[n1]['type'] in (
'call', 'return') else ''
addon_n2 = ':f0' if self.g.node_attr[n2]['type'] in (
'call', 'return') else ''
dstr += ' "%s"%s -> "%s"%s [%s];\n' % (n1, addon_n1, n2, addon_n2,
node_type)
dstr += '}'
dot_fname = fname + '.dot'
o = open(dot_fname, 'w')
o.write(dstr)
o.close()
#cmd_line = "dot %s -Tsvg -O -Kdot -x" % (dot_fname)
# os.system(cmd_line)
syscall("dot", [dot_fname, "-Tsvg", "-O", "-Kdot", "-x"])
def export(self):
return
if 0 < len(self.g):
self.write_dot(self.g, './graphs/global_file')
def run(self):
def safe_unicode(obj, *args):
try:
return unicode(obj, *args)
except UnicodeDecodeError:
# obj is byte string
ascii_text = str(obj).encode('string_escape')
return unicode(ascii_text)
def get_appname(rts):
rts = rts.lower()
for name in ('paypage', 'ccentry', 'mis'):
if name in rts:
return name
return 'UnknownEntryPoint'
log.debug("Execute model checks and execution of loop")
# process the accumulated data into the model
bl_trace_id = []
traces, last_trace_end_str = {}, None
trace_id = self.guid
start_trace = False
local = []
latest_datetime = None
try:
for line in self.accumulator:
if not start_trace:
if 'TRACE START' == line[:11]:
start_trace = True
latest_datetime = extract_datetime(line)
local = []
else:
if 'TRACE END' in line[:9]:
appname = get_appname(local[0])
trace_id = time.strftime(
"%Y-%m-%d_%H-%M-%S",
latest_datetime) + '_' + appname + '_' + str(
hex(hash(repr(local))))
if trace_id not in bl_trace_id:
bl_trace_id.append(trace_id)
last_trace_end_str = line
start_trace = False
traces[trace_id] = local
if trace_id not in self.cache:
self.cache[trace_id] = local
local = []
start_trace = False
else:
local.append(line)
# keep only what hasn't been processed yet
if last_trace_end_str in self.accumulator:
self.accumulator = self.accumulator[self.accumulator.
index(last_trace_end_str):]
log.debug("Process all %d traces available" % len(traces))
for trace_id in traces:
self.processTrace(traces[trace_id], trace_id)
try:
o = open('./cache/' + trace_id + '.html', 'w')
o.write(
pygment_content("<?php\n" + '\n'.join([
safe_unicode(rts) for rts in self.cache[trace_id]
if not has_blacklisted_function(rts)
]) + "\n?>"))
o.close()
except IOError, error:
log.error("Exception writing %s (error:%s)" %
('./cache/' + trace_id + '.html', error))
# generate the JSON file for frontend to consume
self.update_JSON()
except Exception, error:
log.error("run- %s" % error)
sys.exit()
def connectionLost(self, reason):
if callable(self._connectionLost):
try:
self._connectionLost(self)
except Exception, err:
log.error(err)
def _process_changes(self, newRev, branch):
"""
Read changes since last change.
- Read list of commit hashes.
- Extract details from each commit.
- Add changes to database.
"""
# initial run, don't parse all history
if not self.lastRev:
return
rebuild = False
if newRev in itervalues(self.lastRev):
if self.buildPushesWithNoCommits and \
branch not in iterkeys(self.lastRev):
# we know the newRev but not for this branch
log.msg('gitpoller: rebuilding %s for new branch "%s"' %
(newRev, branch))
rebuild = True
# get the change list
revListArgs = ([r'--format=%H', r'%s' % newRev] +
[r'^%s' % rev.encode('ascii', 'ignore')
for rev in itervalues(self.lastRev)] +
[r'--'])
self.changeCount = 0
results = yield self._dovccmd('log', revListArgs, path=self.workdir)
# process oldest change first
revList = results.split()
revList.reverse()
if rebuild and len(revList) == 0:
revList = [newRev]
self.changeCount = len(revList)
self.lastRev[branch] = newRev
if self.changeCount:
log.msg('gitpoller: processing %d changes: %s from "%s" branch "%s"'
% (self.changeCount, revList, self.repourl, branch))
for rev in revList:
dl = defer.DeferredList([
self._get_commit_timestamp(rev),
self._get_commit_author(rev),
self._get_commit_files(rev),
self._get_commit_comments(rev),
], consumeErrors=True)
results = yield dl
# check for failures
failures = [r[1] for r in results if not r[0]]
if failures:
for failure in failures:
log.error(
failure, "while processing changes for {} {}".format(newRev, branch))
# just fail on the first error; they're probably all related!
failures[0].raiseException()
timestamp, author, files, comments = [r[1] for r in results]
yield self.master.data.updates.addChange(
author=author, revision=ascii2unicode(rev), files=files,
comments=comments, when_timestamp=timestamp,
branch=ascii2unicode(self._removeHeads(branch)),
project=self.project, repository=ascii2unicode(self.repourl),
category=self.category, src=u'git')
def validate_settings(settings):
if not settings:
log.error("No settings at all?")
return False
if not settings['nickname'] or not isinstance(settings['nickname'], str):
log.error("Settings is missing nickname field")
return False
if not settings['realname'] or not isinstance(settings['realname'], str):
log.error("Settings is missing realname field")
return False
if not settings['username'] or not isinstance(settings['username'], str):
log.error("Settings is missing username field")
return False
if not settings['servers']:
log.error("Settings is missing servers?")
return False
for server in settings['servers']:
if not server:
log.error("Got empty server in server settings")
return False
if not server['name'] or not isinstance(server['name'], str):
log.error("Missing name for server")
return False
if not server['host'] or not isinstance(server['host'], str):
log.error("Got misconfigured host setting")
return False
if not server['port'] or not isinstance(server['port'], int):
log.error("Got misconfigured ip setting")
return False
if not settings['plugins']:
log.error("Got no plugin, bot will be useless?")
return False
for plugin in settings['plugins']:
if not plugin:
log.error("Settings found empty plugin")
return False
if not plugin['name'] or not isinstance(plugin['name'], str):
log.error("Settings got plugin with no name")
return False
return True
def openFailed(self, reason):
log.error("Open failed due to %r", reason)
if self.start_defer:
self.start_defer.errback(self)
def eb_produce(f):
log.error("Produce failed",
exc_info=(f.type, f.value, f.getTracebackObject()))
def replaceTwistedLoggers():
"""
Visit all Python modules that have been loaded and:
- replace L{twisted.python.log} with a L{LegacyLogger}
- replace L{twisted.python.log.msg} with a L{LegacyLogger}'s C{msg}
- replace L{twisted.python.log.err} with a L{LegacyLogger}'s C{err}
"""
log = Logger()
for moduleName, module in sys.modules.iteritems():
# Oddly, this happens
if module is None:
continue
# Don't patch Twisted's logging module
if module in (twisted.python, twisted.python.log):
continue
# Don't patch this module
if moduleName is __name__:
continue
try:
for name, obj in module.__dict__.iteritems():
try:
newLogger = Logger(namespace=module.__name__)
except AttributeError:
# Can't look up __name__. A hack in the "six" module causes
# this. Skip the module.
# See https://trac.calendarserver.org/ticket/832
continue
legacyLogger = LegacyLogger(logger=newLogger)
if obj is twisted.python.log:
log.info("Replacing Twisted log module object {0} in {1}".
format(name, module.__name__))
setattr(module, name, legacyLogger)
elif obj is twisted.python.log.msg:
log.info(
"Replacing Twisted log.msg object {0} in {1}".format(
name, module.__name__))
setattr(module, name, legacyLogger.msg)
elif obj is twisted.python.log.err:
log.info(
"Replacing Twisted log.err object {0} in {1}".format(
name, module.__name__))
setattr(module, name, legacyLogger.err)
except RuntimeError as e:
# Python could use more specific exceptions, eh.
# What we mean to catch is:
# RuntimeError: dictionary changed size during iteration
log.error(
"Unable to replace twisted loggers for module {module}: "
"{error}",
module=module,
error=e)
def _errback(self, reason):
log.error("Processing request error: %s" % reason)
self._send_action('DUNNO')
def connection_failed(self, reason):
log.error("Connection failed: %s" % (reason, ))
self.defer = None
self.secs_to_wait = 10 if self.secs_to_wait >= 10 else self.secs_to_wait * 2
self.connection_lost()
