def main():
global outstream
if len(sys.argv) < 5:
print >> sys.stderr, "Usage: python %s <host> <port> <ident> <secret> <channel,channel2,channel3,...> [logfile]"%(sys.argv[0])
sys.exit(1)
HOST, PORT, IDENT, SECRET, CHANNELS = [arg.encode("utf-8") for arg in sys.argv[1:6]]
CHANNELS = CHANNELS.split(",")
PORT = int(PORT)
if len(sys.argv) > 6:
outstream = open(sys.argv[6], "a")
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >>sys.stderr, 'connected to', hpc.brokername
def on_message(identifier, channel, payload):
try:
payload = str(payload).strip()
record = json.loads(payload)
if 'source' in record:
record['src'] = record['source']
del record['source']
print >>outstream, unicode(datetime.datetime.utcnow()) + u' '+ u' '.join([ u'='.join([unicode(k),unicode(v)]) for k,v in record.items()])
outstream.flush()
except Exception, e:
print >> sys.stderr, "Error", e
def main():
try: outfd = open(OUTFILE, 'a')
except:
print >>sys.stderr, 'could not open output file for message log.'
return 1
if not os.path.exists(OUTDIR): os.mkdir(OUTDIR)
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >>sys.stderr, 'connected to', hpc.brokername
def on_message(identifier, channel, payload):
# now store the file itself
md5sum = hashlib.md5(payload).hexdigest()
fpath = os.path.join(OUTDIR, md5sum)
try:
open(fpath, 'wb').write(payload)
except:
print >>outfd, '{0} ERROR could not write to {1}'.format(datetime.datetime.now().ctime(), fpath)
outfd.flush()
def on_error(payload):
print >>sys.stderr, ' -> errormessage from server: {0}'.format(payload)
hpc.stop()
hpc.subscribe(CHANNELS)
hpc.run(on_message, on_error)
hpc.close()
return 0
def __init__(self):
try:
self.hpc = hpfeeds.new(config.hpfriends_host, config.hpfriends_port,
config.hpfriends_ident, config.hpfriends_secret)
self.hpc.connect()
except Exception as e:
raise
def hpfeeds_connect(host, port, ident, secret):
logger.info('Connecting to %s@%s:%s ...', ident, host, port)
try:
connection = hpfeeds.new(host, port, ident, secret)
except hpfeeds.FeedException, e:
logger.error('feed exception: %s'%e)
sys.exit(1)
def main():
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >>sys.stderr, 'connected to', hpc.brokername
def on_message(ident, channel, payload):
try:
dec = json.loads(str(payload))
del dec['daddr']
dec['identifier'] = ident
enc = json.dumps(dec)
except:
traceback.print_exc()
print >>sys.stderr, 'forward error for message from {0}.'.format(ident)
return
hpc.publish(RELAYCHAN, enc)
def on_error(payload):
print >>sys.stderr, ' -> errormessage from server: {0}'.format(payload)
hpc.stop()
hpc.subscribe(CHANNELS)
hpc.run(on_message, on_error)
hpc.close()
return 0
def run(self, botnet_queue):
self.botnet_queue = botnet_queue
try:
hpc = hpfeeds.new(self.host, self.port, self.ident, self.secret)
except hpfeeds.FeedException, e:
self.log("Feed exception: %s" % e)
return 1
def start(self):
if not self._initial_connection_happend:
self.hp_connection = hpfeeds.new(self.host, self.port, self.ident, self.secret, True)
self._initial_connection_happend = True
logger.info('HpFeeds logger connected to %s:%s.', self.host, self.port)
# after we established that we can connect enter the subscribe and enter the polling loop
super().start()
def run(self):
def on_message(identifier, channel, payload):
m = hashlib.md5()
m.update(payload)
outmsg = 'PUBLISH channel = %s, identifier = %s, MAEC = %s' % (channel, identifier, m.hexdigest())
self.log.info(outmsg)
if self.opts['logdir'] is None:
return
fpath = os.path.join(self.opts['logdir'], m.hexdigest())
with open(fpath, 'wb') as fd:
fd.write(payload)
def on_error(payload):
self.log.critical("Error message from server: %s" % (payload, ))
self.hpc.stop()
while True:
try:
self.hpc = hpfeeds.new(self.opts['host'], int(self.opts['port']), self.opts['ident'], self.opts['secret'])
self.log.info("Connected to %s" % (self.hpc.brokername, ))
self.hpc.subscribe(self.opts['channel'])
except hpfeeds.FeedException:
break
try:
self.hpc.run(on_message, on_error)
except:
self.hpc.close()
time.sleep(20)
def start_listening(self):
gevent.spawn_later(15, self._activity_checker)
while self.enabled:
try:
self.hpc = hpfeeds.new(self.host, self.port, self.ident, self.secret)
def on_error(payload):
logger.error('Error message from broker: {0}'.format(payload))
self.hpc.stop()
def on_message(ident, chan, payload):
self.last_received = datetime.now()
db_conn = self.db_connect()
db_cursor = db_conn.cursor()
handler = BaseHandler(ident,db_cursor,self.config)
handler.select_module(chan)
handler.handle_payload(payload)
db_conn.close()
self.hpc.subscribe(self.feeds)
self.hpc.run(on_message, on_error)
except Exception as ex:
print ex
self.hpc.stop()
logger.exception('Exception caught: {0}'.format(ex))
#throttle
gevent.sleep(5)
def main():
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >>sys.stderr, 'connected to', hpc.brokername
def on_message(identifier, channel, payload):
try: decoded = json.loads(str(payload))
except: decoded = {'raw': payload}
print 'incoming message from {0} on channel {1}, length {2}'.format(identifier, channel, len(payload))
def on_error(payload):
print >>sys.stderr, ' -> errormessage from server: {0}'.format(payload)
hpc.stop()
#hpc.subscribe(CHANNELS)
hpc.s.settimeout(0.01)
while True:
rrdy, wrdy, xrdy = select.select([hpc.s, sys.stdin], [], [])
if hpc.s in rrdy:
try: hpc.run(on_message, on_error)
except socket.timeout: pass
if sys.stdin in rrdy:
try: l = sys.stdin.readline()
except: traceback.print_exc()
else:
if l.strip(): hpc.publish(CHANNELS, l)
print 'quit.'
hpc.close()
return 0
def main():
try: outfd = open(OUTFILE, 'a')
except:
print >>sys.stderr, 'could not open output file for message log.'
return 1
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >>sys.stderr, 'connected to', hpc.brokername
def on_message(identifier, channel, payload):
try: decoded = json.loads(str(payload))
except: decoded = {'raw': payload}
csv = ', '.join(['{0}={1}'.format(i,j) for i,j in decoded.items()])
outmsg = '{0} PUBLISH chan={1}, identifier={2}, {3}'.format(
datetime.datetime.now().ctime(), channel, identifier, csv
)
print >>outfd, outmsg
outfd.flush()
def on_error(payload):
print >>sys.stderr, ' -> errormessage from server: {0}'.format(payload)
hpc.stop()
hpc.subscribe(CHANNELS)
hpc.run(on_message, on_error)
hpc.close()
return 0
def start_listening(self):
gevent.spawn_later(15, self._activity_checker)
while self.enabled:
try:
self.hpc = hpfeeds.new(self.host, self.port, self.ident, self.secret)
def on_error(payload):
print 'Error message from broker: {0}'.format(payload)
self.hpc.stop()
def on_message(ident, chan, payload):
self.last_received = datetime.now()
data = json.loads(str(payload))
site_id = data['id']
url = data['url'].encode('unicode-escape')
self.handler = UrlHandler(url)
self.handler.process()
#self.handle_url(url)
#print "Time: %s --- Site: %s - URL: %s" % (self.last_received, site_id, url)
self.hpc.subscribe(self.feeds)
self.hpc.run(on_message, on_error)
except Exception as ex:
print ex
self.hpc.stop()
gevent.sleep(5)
def __init__(self, host, port, ident, secret, channels):
self.channels = channels
try:
with gevent.Timeout(2):
self.hpc = hpfeeds.new(host, port, ident, secret, reconnect=False)
except:
raise Exception("Connection to HPFriends timed out")
def hpfeedsend(esm):
try:
hpc = hpfeeds.new(ECFG["host"],int(ECFG["port"]),ECFG["ident"],ECFG["secret"])
logme("hpfeedsend","Connect to (%s)" % format(hpc.brokername) ,("P3","VERBOSE"),ECFG)
except hpfeeds.FeedException, e:
logme("hpfeedsend","HPFeeds Error (%s)" % format(e) ,("LOG","VERBOSE"),ECFG)
return False
def main():
gi = GeoIP.open("/opt/GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def main():
import socket
gi = geoip2.database.Reader("/opt/GeoLite2-City.mmdb")
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def onincident(details):
logger.warning('Detection! PID %d, TID %d' % (details['PID'], details['TID']))
activate_readonly()
wire_report = dict(details)
wire_report['Ident'] = HPFEEDS_IDENT
incident_hpc = hpfeeds.new(HPFEEDS_HOST, HPFEEDS_PORT, HPFEEDS_IDENT, HPFEEDS_SECRET)
incident_hpc.publish(HPFEEDS_REPORT_CHANNEL, json.dumps(wire_report))
incident_hpc.publish(HPFEEDS_STATUS_CHANNEL, create_status_update('detection'))
incident_hpc.close()
def main():
conn = psycopg2.connect("host=localhost dbname=hpfeeds user=username password=pw")
cur = conn.cursor()
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def __init__(self,host,port,ident, secret,channels):
logging.Handler.__init__(self)
self.host=str(host)
self.port=int(port)
self.ident=str(ident)
self.secret=str(secret)
self.channels=map(str,channels)
hpc=hpfeeds.new(self.host, self.port, self.ident, self.secret)
hpc.subscribe(channels)
self.hpc=hpc
def main():
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
glastopf_logger = logging.getLogger('python-logstash-logger')
glastopf_logger.setLevel(logging.INFO)
glastopf_logger.addHandler(logstash.LogstashHandler('127.0.0.1', 8070, version=1))
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def main():
logger.info('Initializing Ghost...')
g = ghost.Ghost(GHOST_DEVICE_ID)
g.settimeout(GHOST_DURATION)
logger.info('Ready')
while True:
logger.info('Mounting the virtual device')
hpc = hpfeeds.new(HPFEEDS_HOST, HPFEEDS_PORT, HPFEEDS_IDENT, HPFEEDS_SECRET)
hpc.publish(HPFEEDS_STATUS_CHANNEL, create_status_update('mount'))
hpc.close()
g.run(onincident)
logger.info('Virtual device removed')
hpc = hpfeeds.new(HPFEEDS_HOST, HPFEEDS_PORT, HPFEEDS_IDENT, HPFEEDS_SECRET)
hpc.publish(HPFEEDS_STATUS_CHANNEL, create_status_update('remove'))
hpc.close()
sleep(GHOST_INTERVAL)
def start(self):
import socket
self.hpc = None
self.gi = {}
self.gi[socket.AF_INET] = GeoIP.open("GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)
self.gi[socket.AF_INET6] = GeoIP.open("GeoLiteCityv6.dat",GeoIP.GEOIP_STANDARD)
try:
self.hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
logger.error('[hpfeed] feed exception: %s'% e)
def run(self):
"""
Fetches feed from hpfriends and create a Feed object then put it to socketQueue
"""
global IS_RUNNING
try:
self.hpc = hpfeeds.new(self.host, self.port, self.ident, self.secret)
except hpfeeds.FeedException, e:
self.log('Error: {0}'.format(e))
return 1
def main():
import socket
gi = {}
gi[socket.AF_INET] = GeoIP.open("/opt/GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)
gi[socket.AF_INET6] = GeoIP.open("/opt/GeoLiteCityv6.dat",GeoIP.GEOIP_STANDARD)
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def main():
import socket
gi = {}
gi[socket.AF_INET] = GeoIP.open("/opt/GeoLiteCity.dat",
GeoIP.GEOIP_STANDARD)
gi[socket.AF_INET6] = GeoIP.open("/opt/GeoLiteCityv6.dat",
GeoIP.GEOIP_STANDARD)
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >> sys.stderr, 'feed exception:', e
return 1
def __init__(self, config):
self.config = config
self.log_get = self.config.getboolean('jsonlog',
'log_get',
fallback=False)
self.channel = "big-hp.events"
self.server = config.get('hpfeeds', 'server')
self.port = config.getint('hpfeeds', 'port')
self.ident = config.get('hpfeeds', 'ident')
self.secret = config.get('hpfeeds', 'secret')
self.tags = config.get('hpfeeds', 'tags')
self.client = hpfeeds.new(self.server, self.port, self.ident,
self.secret)
def start(self, cfg):
self.sessions= {}
self.HOST = cfg.get('database_hpfeed', "host")
self.PORT = int(cfg.get('database_hpfeed', "port"))
self.CHANNELS= []
self.CHANNELS.append(cfg.get("database_hpfeed", "channel"))
self.gd = pygeoip.GeoIP(cfg.get('database_hpfeed', 'geolitecity'))
self.IDENT = cfg.get("database_hpfeed", 'ident')
self.SECRET = cfg.get("database_hpfeed", 'secret')
self.hpc = hpfeeds.new(self.HOST, self.PORT, self.IDENT, self.SECRET)
log.msg('connected to %s'% self.hpc.brokername)
logger.info("Connected to %s" % self.hpc.brokername)
def main():
logger.info('Initializing Ghost...')
g = ghost.Ghost(GHOST_DEVICE_ID)
g.settimeout(GHOST_DURATION)
logger.info('Ready')
while True:
logger.info('Mounting the virtual device')
hpc = hpfeeds.new(HPFEEDS_HOST, HPFEEDS_PORT, HPFEEDS_IDENT,
HPFEEDS_SECRET)
hpc.publish(HPFEEDS_STATUS_CHANNEL, create_status_update('mount'))
hpc.close()
g.run(onincident)
logger.info('Virtual device removed')
hpc = hpfeeds.new(HPFEEDS_HOST, HPFEEDS_PORT, HPFEEDS_IDENT,
HPFEEDS_SECRET)
hpc.publish(HPFEEDS_STATUS_CHANNEL, create_status_update('remove'))
hpc.close()
sleep(GHOST_INTERVAL)
def get_hpfeeds_client(config):
hpc = None
if config["hpfeeds.enabled"].lower() == "true":
LOGGER.info(
"hpfeeds enabled, creating connection to {}:{}".format(config["hpfeeds.host"], config["hpfeeds.port"])
)
hpc = hpfeeds.new(
config["hpfeeds.host"], int(config["hpfeeds.port"]), config["hpfeeds.identity"], config["hpfeeds.secret"]
)
hpc.s.settimeout(0.01)
else:
LOGGER.info("hpfeeds is disabled")
return hpc
def start(self, cfg):
self.sessions = {}
self.HOST = cfg.get('database_hpfeed', "host")
self.PORT = int(cfg.get('database_hpfeed', "port"))
self.CHANNELS = []
self.CHANNELS.append(cfg.get("database_hpfeed", "channel"))
self.gd = pygeoip.GeoIP(cfg.get('database_hpfeed', 'geolitecity'))
self.IDENT = cfg.get("database_hpfeed", 'ident')
self.SECRET = cfg.get("database_hpfeed", 'secret')
self.hpc = hpfeeds.new(self.HOST, self.PORT, self.IDENT, self.SECRET)
log.msg('connected to %s' % self.hpc.brokername)
logger.info("Connected to %s" % self.hpc.brokername)
def main():
try:
outfd = open(OUTFILE, 'a')
except:
print >> sys.stderr, 'could not open output file for message log.'
return 1
if not os.path.exists(OUTDIR): os.mkdir(OUTDIR)
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >> sys.stderr, 'connected to', hpc.brokername
def on_message(identifier, channel, payload):
try:
decoded = json.loads(str(payload))
except:
decoded = {'raw': payload}
if not 'sha1' in decoded or not 'data' in decoded:
print >> sys.stderr, "Message received does not contain hash or data :( - ignoring it..."
else:
print >> sys.stderr, "Got a message with sha1 and data"
csv = ', '.join([
'{0}={1}'.format(i, decoded[i])
for i in ['sha1', 'type', 'md5']
])
outmsg = '{0} PUBLISH chan={1}, identifier={2}, {3}'.format(
datetime.datetime.now().ctime(), chan, ident, csv)
print >> outfd, outmsg
# now store the file itself
filedata = decoded['data'].decode('base64')
fpath = os.path.join(OUTDIR, decoded['sha1'])
try:
open(fpath, 'wb').write(filedata)
except:
print >> outfd, '{0} ERROR could not write to {1}'.format(
datetime.datetime.now().ctime(), fpath)
outfd.flush()
def on_error(payload):
print >> sys.stderr, ' -> errormessage from server: {0}'.format(
payload)
hpc.stop()
hpc.subscribe(CHANNELS)
hpc.run(on_message, on_error)
hpc.close()
return 0
def sendfeed():
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/hpfeeds/")
import hpfeeds
host = server.shivaconf.get('hpfeeds', 'host')
port = server.shivaconf.getint('hpfeeds', 'port')
ident = server.shivaconf.get('hpfeeds', 'ident')
secret = server.shivaconf.get('hpfeeds', 'secret')
channel = {"parsed": "shiva.parsed", "ip_url": "shiva.ip.url"}
try:
hpc = hpfeeds.new(host, port, ident, secret)
except Exception, e:
logging.critical("Cannot connect. %s" % e)
def sendfeed():
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/hpfeeds/")
import hpfeeds
host = server.shivaconf.get('hpfeeds', 'host')
port = server.shivaconf.getint('hpfeeds', 'port')
ident = server.shivaconf.get('hpfeeds', 'ident')
secret = server.shivaconf.get('hpfeeds', 'secret')
channel = {"parsed": "shiva.parsed", "ip_url": "shiva.ip.url"}
try:
hpc = hpfeeds.new(host, port, ident, secret)
except Exception, e:
logging.critical("Cannot connect. %s" % e)
def run(self):
def on_message(identifier, channel, payload):
try:
decoded = json.loads(str(payload))
except:
decoded = {'raw': payload}
if not 'md5' in decoded or not 'data' in decoded:
self.log.info(
"Received message does not contain hash or data - Ignoring it"
)
return
csv = ', '.join([
'{0} = {1}'.format(i, decoded[i])
for i in ['md5', 'sha1', 'type']
])
outmsg = 'PUBLISH channel = %s, identifier = %s, %s' % (
channel, identifier, csv)
self.log.info(outmsg)
if self.opts['logdir'] is None:
return
filedata = decoded['data'].decode('base64')
fpath = os.path.join(self.opts['logdir'], decoded['md5'])
with open(fpath, 'wb') as fd:
fd.write(filedata)
def on_error(payload):
self.log.critical("Error message from server: %s", payload)
self.hpc.stop()
while True:
try:
self.hpc = hpfeeds.new(self.opts['host'],
int(self.opts['port']),
self.opts['ident'], self.opts['secret'])
self.log.info("Connected to %s", self.hpc.brokername)
self.hpc.subscribe([
self.opts['channel'],
])
except hpfeeds.FeedException:
break
try:
self.hpc.run(on_message, on_error)
except:
self.hpc.close()
time.sleep(20)
def main(opts, action, pubdata=None):
outfd = None
if opts.output:
try: outfd = open(opts.output, 'a')
except:
log('could not open output file for message log.')
return 1
else:
outfd = sys.stdout
try: hpc = hpfeeds.new(opts.host, opts.port, opts.ident, opts.secret)
except hpfeeds.FeedException, e:
log('Error: {0}'.format(e))
return 1
def get_hpfeeds_client(config):
hpc = None
if config['hpfeeds.enabled'].lower() == 'true':
LOGGER.info('hpfeeds enabled, creating connection to {}:{}'.format(config['hpfeeds.host'], config['hpfeeds.port']))
hpc = hpfeeds.new(
config['hpfeeds.host'],
int(config['hpfeeds.port']),
config['hpfeeds.identity'],
config['hpfeeds.secret']
)
hpc.s.settimeout(0.01)
else:
LOGGER.info( 'hpfeeds is disabled')
return hpc
def main():
import socket
gi = {}
gi[socket.AF_INET] = GeoIP.open("/opt/GeoLiteCity.dat",GeoIP.GEOIP_STANDARD)
gi[socket.AF_INET6] = GeoIP.open("/opt/GeoLiteCityv6.dat",GeoIP.GEOIP_STANDARD)
with open(sys.argv[2], 'r') as f:
dl = dict(line.strip().split(None, 1) for line in f)
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >>sys.stderr, 'feed exception:', e
return 1
def main():
if len(sys.argv) < 2:
return 1
config = parse_config(sys.argv[1])
host = config['hpf_host']
port = config['hpf_port']
channels = [c.encode('utf-8') for c in config['hpf_feeds']]
ident = config['hpf_ident'].encode('utf-8')
secret = config['hpf_secret'].encode('utf-8')
include_hp_tags = config['include_hp_tags']
ignore_cidr_l = parse_ignore_cidr_option(config['ignore_cidr'])
cif_token = config['cif_token']
cif_host = config['cif_host']
cif_provider = config['cif_provider']
cif_tlp = config['cif_tlp']
cif_confidence = config['cif_confidence']
cif_tags = config['cif_tags']
cif_group = config['cif_group']
cif_verify_ssl = config['cif_verify_ssl']
cache = RedisCache()
processor = processors.HpfeedsMessageProcessor(ignore_cidr_list=ignore_cidr_l)
logging.debug('Initializing HPFeeds connection with {0}, {1}, {2}, {3}'.format(host,port,ident,secret))
try:
hpc = hpfeeds.new(host, port, ident, secret)
except hpfeeds.FeedException as e:
logging.error('Experienced FeedException: {0}'.format(repr(e)))
return 1
def on_message(identifier, channel, payload):
for msg in processor.process(identifier, channel, payload, ignore_errors=True):
handle_message(msg, cif_host, cif_token, cif_provider, cif_tlp, cif_confidence, cif_tags, cif_group,
cif_verify_ssl, cache, include_hp_tags)
def on_error(payload):
sys.stderr.write("Handling error.")
hpc.stop()
hpc.subscribe(channels)
try:
hpc.run(on_message, on_error)
except:
pass
finally:
hpc.close()
return 0
def connect_hpfeeds():
# set hpfeeds related data
hpfeeds_server = args.hpfserver
hpfeeds_port = args.hpfport
hpfeeds_ident = args.hpfident
hpfeeds_secret = args.hpfsecret
hpfeeds_prefix = args.hpfchannelprefix
if hpfeeds_server and hpfeeds_port and hpfeeds_ident and hpfeeds_secret and hpfeeds_prefix:
try:
hpc = hpfeeds.new(hpfeeds_server, int(hpfeeds_port), hpfeeds_ident,
hpfeeds_secret)
return hpc, hpfeeds_prefix
except (hpfeeds.FeedException, socket.error, hpfeeds.Disconnect), e:
print "hpfeeds connection not successful"
logger.warn('Exception while connecting: {0}'.format(e))
def main():
import socket
gi = {}
gi[socket.AF_INET] = GeoIP.open("/opt/GeoLiteCity.dat",
GeoIP.GEOIP_STANDARD)
gi[socket.AF_INET6] = GeoIP.open("/opt/GeoLiteCityv6.dat",
GeoIP.GEOIP_STANDARD)
with open(sys.argv[2], 'r') as f:
dl = dict(line.strip().split(None, 1) for line in f)
try:
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
except hpfeeds.FeedException, e:
print >> sys.stderr, 'feed exception:', e
return 1
def main():
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >> sys.stderr, 'connected to', hpc.brokername
def on_message(identifier, channel, payload):
print 'msg', identifier, channel, payload
def on_error(payload):
print >> sys.stderr, ' -> errormessage from server: {0}'.format(
payload)
hpc.stop()
hpc.subscribe(CHANNELS)
hpc.run(on_message, on_error)
hpc.close()
return 0
def _main(opts, action, pubdata=None):
tls = opts.tls
if opts.certfile:
tls = True
try:
hpc = hpfeeds.new(
opts.host,
opts.port,
opts.ident,
opts.secret,
certfile=opts.certfile,
tls=tls,
)
except hpfeeds.FeedException as e:
log('Error: {0}'.format(e))
return 1
def on_error(payload):
log('Error message from broker: {0}'.format(payload))
hpc.stop()
log('connected to {0}'.format(hpc.brokername))
if action == 'subscribe':
hpc.subscribe(opts.channels)
try:
hpc.run(on_message, on_error)
except hpfeeds.FeedException as e:
log('Error: {0}'.format(e))
return 1
elif action == 'publish':
hpc.publish(opts.channels, pubdata)
emsg = hpc.wait()
if emsg:
print('got error from server:', emsg)
elif action == 'sendfile':
pubfile = open(pubdata, 'rb').read()
hpc.publish(opts.channels, pubfile)
log('closing connection.')
hpc.close()
return 0
def _main(opts, action, pubdata=None):
try:
hpc = hpfeeds.new(
opts.host,
opts.port,
opts.ident,
opts.secret,
certfile=opts.certfile,
)
except hpfeeds.FeedException as e:
log('Error: {0}'.format(e))
return 1
log('connected to {0}'.format(hpc.brokername))
if action == 'subscribe':
def on_message(ident, chan, payload):
if [i for i in payload[:20] if i not in string.printable.encode('utf-8')]:
payload = payload[:20].encode('hex') + '...'
log('publish to {0} by {1}: {2}'.format(chan, ident, payload))
def on_error(payload):
log('Error message from broker: {0}'.format(payload))
hpc.stop()
hpc.subscribe(opts.channels)
try:
hpc.run(on_message, on_error)
except hpfeeds.FeedException as e:
log('Error: {0}'.format(e))
return 1
elif action == 'publish':
hpc.publish(opts.channels, pubdata)
emsg = hpc.wait()
if emsg:
print('got error from server:', emsg)
elif action == 'sendfile':
pubfile = open(pubdata, 'rb').read()
hpc.publish(opts.channels, pubfile)
log('closing connection.')
hpc.close()
return 0
def __init__(self, hpfserver, hpfport, hpfident, hpfsecret, hpfchannel, serverid):
self.hpfserver = hpfserver
self.hpfport = hpfport
self.hpfident = hpfident
self.hpfsecret = hpfsecret
self.hpfchannel = hpfchannel
self.serverid = serverid
self.hpc = None
if (self.hpfserver and self.hpfport and self.hpfident and self.hpfport and self.hpfchannel and self.serverid):
import logging
logging.basicConfig()
import hpfeeds
try:
self.hpc = hpfeeds.new(self.hpfserver, self.hpfport, self.hpfident, self.hpfsecret)
self.status = "Logging to hpfeeds using server: {0}, channel {1}.".format(self.hpfserver, self.hpfchannel)
except (hpfeeds.FeedException, socket.error, hpfeeds.Disconnect):
self.status = "hpfeeds connection not successful"
def run(self):
def on_message(identifier, channel, payload):
try:
decoded = json.loads(str(payload))
except:
decoded = {'raw': payload}
if not 'md5' in decoded or not 'data' in decoded:
log.info(
"Received message does not contain hash or data - Ignoring it"
)
return
csv = ', '.join([
'{0} = {1}'.format(i, decoded[i])
for i in ['md5', 'sha1', 'type']
])
outmsg = 'PUBLISH channel = %s, identifier = %s, %s' % (
channel, identifier, csv)
log.info(outmsg)
filedata = decoded['data'].decode('base64')
fpath = os.path.join(OUTDIR, decoded['md5'])
with open(fpath, 'wb') as fd:
fd.write(filedata)
def on_error(payload):
log.critical("Error message from server: %s" % (payload, ))
self.hpc.stop()
while True:
try:
self.hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
log.info("Connected to %s" % (self.hpc.brokername, ))
self.hpc.subscribe(CHANNELS)
except hpfeeds.FeedException:
break
try:
self.hpc.run(on_message, on_error)
except:
self.hpc.close()
time.sleep(20)
def get_hpc():
"""
@summary: initialise hpfeeds connection
"""
if (os.environ.get('HPFEEDS_SERVER') and os.environ.get('HPFEEDS_SECRET')
and os.environ.get('HPFEEDS_IDENT')
and os.environ.get('HPFEEDS_PORT')
and os.environ.get('HPFEEDS_CHANNEL')):
try:
hpc = hpfeeds.new(os.environ.get('HPFEEDS_SERVER'),
int(os.environ.get('HPFEEDS_PORT')),
os.environ.get('HPFEEDS_IDENT'),
os.environ.get('HPFEEDS_SECRET'))
log.info("hpfeeds connection successful")
return hpc
except (hpfeeds.FeedException, socket.error, hpfeeds.Disconnect), e:
log.info("hpfeeds connection not successful")
log.info(
"Exception while connecting to hpfeeds server: {0}".format(e))
def __init__(self, hpfserver, hpfport, hpfident, hpfsecret, hpfchannel,
serverid, verbose):
self.hpfserver = hpfserver
self.hpfport = hpfport
self.hpfident = hpfident
self.hpfsecret = hpfsecret
self.hpfchannel = hpfchannel
self.serverid = serverid
self.hpc = None
self.verbose = verbose
if (self.hpfserver and self.hpfport and self.hpfident and self.hpfport
and self.hpfchannel and self.serverid):
import hpfeeds
try:
self.hpc = hpfeeds.new(self.hpfserver, self.hpfport,
self.hpfident, self.hpfsecret)
logger.debug(
"Logging to hpfeeds using server: {0}, channel {1}.".
format(self.hpfserver, self.hpfchannel))
except (hpfeeds.FeedException, socket.error, hpfeeds.Disconnect):
logger.critical("hpfeeds connection not successful")
def main():
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >> sys.stderr, 'connected to', hpc.brokername
def on_message(identifier, channel, payload):
try:
decoded = json.loads(str(payload))
except:
decoded = {'raw': payload}
print 'incoming message from {0} on channel {1}, length {2}'.format(
identifier, channel, len(payload))
def on_error(payload):
print >> sys.stderr, ' -> errormessage from server: {0}'.format(
payload)
hpc.stop()
#hpc.subscribe(CHANNELS)
hpc.s.settimeout(0.01)
while True:
rrdy, wrdy, xrdy = select.select([hpc.s, sys.stdin], [], [])
if hpc.s in rrdy:
try:
hpc.run(on_message, on_error)
except socket.timeout:
pass
if sys.stdin in rrdy:
try:
l = sys.stdin.readline()
except:
traceback.print_exc()
else:
if l.strip(): hpc.publish(CHANNELS, l)
print 'quit.'
hpc.close()
return 0
def hpfeeds_connect(host, port, ident, secret):
try:
connection = hpfeeds.new(host, port, ident, secret)
except hpfeeds.FeedException, e:
logger.error('feed exception: %s' % e)
sys.exit(1)
logging.basicConfig()
import hpfeeds
from src.core import *
from src.smtp import *
# port ranges to spawn pulled from config
ports = check_config("PORTS=")
# check to see what IP we need to bind to
bind_interface = check_config("BIND_INTERFACE=")
send_email = check_config("ALERT_USER_EMAIL=")
hpfeeds_host = check_config("HPFEEDS_HOST=")
if hpfeeds_host:
hpc = hpfeeds.new(hpfeeds_host, int(check_config("HPFEEDS_PORT=")),
check_config("HPFEEDS_IDENT="),
check_config("HPFEEDS_SECRET="))
write_log("Connected to {0}".format(hpc.brokername))
hpf_channels_string = check_config("HPFEEDS_CHANNELS=")
hpf_channels = hpf_channels_string[1:-1].split(',')
# main socket server listener for responses
class SocketListener((SocketServer.BaseRequestHandler)):
def handle(self):
#print self.server.server_name, self.server.server_port
pass
def setup(self):
def main(): hpc = hpfeeds.new(HPFEEDS_HOST, HPFEEDS_PORT, HPFEEDS_IDENT, HPFEEDS_SECRET) hpc.subscribe([HPFEEDS_REPORT_CHANNEL, HPFEEDS_STATUS_CHANNEL]) hpc.run(on_message, on_error)
def handle(self):
hpclient = hpfeeds.new(HPF_HOST, HPF_PORT, HPF_IDENT, HPF_SECRET)
# self.request is the TCP socket connected to the client
source_ip = self.client_address[0]
source_port = self.client_address[1]
dest_ip = socket.gethostbyname(socket.gethostname())
dest_port = self.server.server_address[1]
printer = Printer(logger, source_ip)
logger.info("handle - open_conn - " + source_ip + ":" + str(source_port))
printer.events_list.append("handle - open_conn - " + source_ip + ":" + str(source_port))
emptyRequest = False
while emptyRequest == False:
# Wait a maximum of conn_timeout seconds for another request
# If conn_timeout elapses without request, close the connection
ready = select.select([self.request], [], [], conn_timeout)
if not ready[0]:
break
try:
self.data = self.request.recv(1024).strip()
except Exception as e:
logger.error("handle - receive - Error receiving data - possible port scan")
printer.events_list.append("handle - receive - Error receiving data - possible port scan")
emptyRequest = True
break
request = self.data.decode('UTF-8')
request = request.replace('\x1b%-12345X', '')
if request[0:2] == '%!':
printer.receiving_postscript = True
printer.postscript_data = request
logger.info('handle - postscript - Received first postscript request of file')
printer.events_list.append('handle - postscript - Received first postscript request of file')
continue
elif printer.receiving_postscript:
printer.postscript_data += request
if '%%EOF' in request:
printer.save_postscript()
printer.receiving_postscript = False
continue
commands = self.parse_commands(request)
logger.debug('handle - request - ' + str(request.encode('UTF-8')))
printer.events_list.append('handle - request - ' + str(request.encode('UTF-8')))
if len(commands) == 0: # If we're sent an empty request, close the connection
emptyRequest = True
break
try:
response = ''
for command in commands:
command = command.lstrip()
if command.startswith("@PJL "):
command = command[5:]
if printer.printing_raw_job:
printer.save_raw_print_job()
if command.startswith("ECHO"):
response += printer.command_echo(command)
elif command.startswith("USTATUSOFF"):
response += printer.command_ustatusoff(command)
elif command.startswith("INFO ID"):
response += printer.command_info_id(command)
elif command.startswith("INFO STATUS"):
response += printer.command_info_status(command)
elif command.startswith("FSDIRLIST"):
response += printer.command_fsdirlist(command)
elif command.startswith("FSQUERY"):
response += printer.command_fsquery(command)
elif command.startswith("FSMKDIR"):
response += printer.command_fsmkdir(command)
elif command.startswith("FSUPLOAD"):
response += printer.command_fsupload(command)
elif command.startswith("FSDOWNLOAD"):
response += printer.command_fsdownload(command)
elif command.startswith("RDYMSG"):
response += printer.command_rdymsg(command)
else:
logger.error("handle - cmd_unknown - " + str(command))
printer.events_list.append("handle - cmd_unknown - " + str(command))
else:
response += printer.append_raw_print_job(command)
logger.info("handle - response - " + str(response.encode('UTF-8')))
printer.events_list.append("handle - response - " + str(response.encode('UTF-8')))
self.request.sendall(response.encode('UTF-8'))
except Exception as e:
tb = sys.exc_info()[2]
traceback.print_tb(tb)
logger.error("handle - error_caught - " + str(e))
printer.events_list.append("handle - error_caught - " + str(e))
if printer.printing_raw_job:
printer.save_raw_print_job()
logger.info("handle - close_conn - " + source_ip + ":" + str(source_port))
printer.events_list.append("handle - close_conn - " + source_ip + ":" + str(source_port))
now = datetime.now()
events_json = {"source_ip": source_ip, "source_port" : source_port, "dest_ip": dest_ip, "dest_port": dest_port,"timestamp": datetime.timestamp(now), "events": printer.events_list}
hpclient.publish(HPF_CHAN, json.dumps(events_json))
def main():
hpc = hpfeeds.new(HOST, PORT, IDENT, SECRET)
print >> sys.stderr, 'connected to', hpc.brokername
insertCon = pymongo.Connection(host="localhost", port=27017)
db = None
collection = None
def on_message(identifier, channel, payload):
if channel == 'dionaea.connections':
try:
msg = ast.literal_eval(str(payload))
except:
print 'exception processing dionaea.connections event', repr(
payload)
else:
msg["time"] = datetime.datetime.utcfromtimestamp(msg['time'])
msg['rport'] = int(msg['rport'])
msg['lport'] = int(msg['lport'])
print 'inserting...', msg
db = insertCon['dionaea']
collection = db['connection']
collection.insert(msg)
elif channel == 'geoloc.events':
try:
payload_python = str(payload)
msg = ast.literal_eval(payload_python.replace("null", "None"))
except:
print 'exception processing geoloc.events', repr(payload)
else:
msg['time'] = datetime.datetime.strptime(
msg['time'], "%Y-%m-%d %H:%M:%S")
print 'inserting...', msg
db = insertCon['geoloc']
collection = db['events']
collection.insert(msg)
elif channel == 'dionaea.dcerpcrequests':
try:
payload_python = str(payload)
msg = ast.literal_eval(payload_python.replace("null", "None"))
except:
print 'exception processing dionaea.dcerpcrequests', repr(
payload)
else:
dt = datetime.datetime.now()
msg['time'] = dt.strftime('%Y-%m-%d %H:%M:%S')
print 'inserting...', msg
db = insertCon['dionaea']
collection = db['dcerpcrequests']
collection.insert(msg)
elif channel == 'dionaea.shellcodeprofiles':
try:
payload_python = str(payload)
msg = ast.literal_eval(payload_python.replace("null", "None"))
except:
print 'exception processing dionaea.shellcodeprofiles', repr(
payload)
else:
dt = datetime.datetime.now()
msg['time'] = dt.strftime('%Y-%m-%d %H:%M:%S')
print 'inserting...', msg
db = insertCon['dionaea']
collection = db['shellcodeprofiles']
collection.insert(msg)
elif channel == 'mwbinary.dionaea.sensorunique':
try:
payload_python = str(payload)
except:
print 'exception processing mwbinary.dionaea.sensorunique', repr(
payload)
else:
hash = md5.new()
hash.update(payload_python)
msg = hash.hexdigest()
print 'inserting mwbinary...', msg
db = insertCon['dionaea']
gfsDate = GridFS(db)
gfsDate.put(payload_python, filename=msg)
elif channel == 'dionaea.capture':
try:
payload_python = str(payload)
msg = ast.literal_eval(payload_python.replace("null", "None"))
except:
print 'exception processing dionaea.capture', repr(payload)
else:
dt = datetime.datetime.now()
msg['time'] = dt.strftime('%Y-%m-%d %H:%M:%S')
print 'inserting...', msg
db = insertCon['dionaea']
collection = db['capture']
collection.insert(msg)
def on_error(payload):
print >> sys.stderr, ' -> errormessage from server: {0}'.format(
payload)
hpc.stop()
hpc.subscribe(CHANNELS)
hpc.run(on_message, on_error)
hpc.close()
return 0
rediscl = None
# determine if hpfeeds is used
LogToHpfeeds = False
if config.has_section('log_hpfeed'):
import hpfeeds
import pygeoip
LogToHpfeeds = True
hpf_host = config.get('log_hpfeed', 'host')
hpf_port = int(config.get('log_hpfeed', 'port'))
hpf_CHANNELS = []
hpf_CHANNELS.append(config.get('log_hpfeed', 'channel'))
hpf_IDENT = config.get('log_hpfeed', 'ident')
hpf_SECRET = config.get('log_hpfeed', 'secret')
hpc = hpfeeds.new(hpf_host, hpf_port, hpf_IDENT, hpf_SECRET)
gd = pygeoip.GeoIP('GeoLiteCity.dat')
logger.info("Connected to %s" % hpc.brokername)
# determine if log to redis db
LogToRedis = False
if config.has_section('log_redis'):
import redis
LogToRedis = True
redis_db = int(config.get('log_redis', 'redis_db'))
redis_host = config.get('log_redis', 'redis_host')
redis_port = int(config.get('log_redis', 'redis_port'))
rediscl = redis.Redis(host=redis_host, port=redis_port, db=redis_db)
def __init__(self, sensor_uuid, alert_file, host, port, ident, secret):
hpc = hpfeeds.new(host=host, port=port, ident=ident, secret=secret)
self.alerter = AlertEventHandler(sensor_uuid, alert_file, hpc)
def _start_connection(self, host, port, ident, secret):
# if no initial connection to hpfeeds this will hang forever, reconnect=True only comes into play
# when lost connection after the initial connect happend.
self.hpc = hpfeeds.new(host, port, ident, secret)
self._initial_connection_happend = True
#!/usr/bin/env python2
import hpfeeds
import json
import csv
import datetime
import os
import sys
hpc = hpfeeds.new('hpfriends.honeycloud.net', 20000, 'username', 'password')
print "Connected to {0}".format(hpc.brokername)
hpf_channel = 'turris'
dict = {}
if os.path.isfile('honey.json'):
with open('honey.json', 'r') as json_file_r:
dict = json.load(json_file_r)
json_file_r.close()
new_dict = dict.copy()
date_ago = datetime.datetime.utcnow() - datetime.timedelta(hours=10)
for key in dict:
date_string = key.split(',')[1].split('+')[0]
date = datetime.datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')
if date_ago > date:
del new_dict[key]
if not os.path.isfile('honey.csv'):
sys.stderr.write('file honey.csv not found\n')
# Import config from file
conffile = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'../wordpot.conf')
LOGGER.info('Loading conf file: %s', conffile)
try:
app.config.from_pyfile(conffile)
except:
LOGGER.error('Can\'t load conf file')
check_options()
if app.config['HPFEEDS_ENABLED']:
import hpfeeds
print('Connecting to hpfeeds broker {}:{}'.format(
app.config['HPFEEDS_HOST'], app.config['HPFEEDS_PORT']))
app.config['hpfeeds_client'] = hpfeeds.new(app.config['HPFEEDS_HOST'],
app.config['HPFEEDS_PORT'],
app.config['HPFEEDS_IDENT'],
app.config['HPFEEDS_SECRET'])
app.config['hpfeeds_client'].s.settimeout(0.01)
else:
LOGGER.warn('hpfeeds is disabled')
# ------------------------
# Add Custom Server Header
#-------------------------
@app.after_request
def add_server_header(response):
if app.config['SERVER']:
response.headers['Server'] = app.config['SERVER']
d = {'s_id': spam_id, 'attachment': attachment, 'name': name}
data = json.dumps(d)
with lock:
hpc.publish(channels['attachments'], data)
print "[+] Attachment Published"
shutil.move(path['attach'] + f, path['hpfeedattach'])
else:
print "nothing to send on hpfeeds channel shiva.attachments"
def main():
try:
raw_thread = threading.Thread(target=send_raw, args=[]).run()
attach_thread = threading.Thread(target=send_attach, args=[]).run()
except Exception, e:
print "[-] shivasendfiles main: Error. %s" % e
sys.exit(1)
try:
while raw_thread.isAlive() or attach_thread.isAlive():
pass
except Exception, e:
pass
if __name__ == '__main__':
hpc = hpfeeds.new(host, port, ident, secret)
main()
