def _scan(self, context):
logger.debug("Setting up shop...")
shop_path = "%s/shop" % self.base_dir
if not os.path.exists(self.base_dir):
self._error("ChopShop path does not exist")
elif not os.path.exists(shop_path):
self._error("ChopShop shop path does not exist")
else:
sys.path.append(shop_path)
import ChopLib as CL
# I wanted to do this check in validate, but if it fails and
# then you fix the path to point to the appropriate chopshop
# it requires a webserver restart to take effect. So just do
# the check at each scan.
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
self._error("Need ChopShop 4.0 or newer")
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = self.base_dir
# XXX: Convert from unicode to str...
choplib.modules = str(self.modules)
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
chopui.bind(choplib)
chopui.start()
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
def _scan(self, context):
logger.debug("Setting up shop...")
shop_path = "%s/shop" % self.base_dir
if not os.path.exists(self.base_dir):
self._error("ChopShop path does not exist")
elif not os.path.exists(shop_path):
self._error("ChopShop shop path does not exist")
else:
sys.path.append(shop_path)
import ChopLib as CL
# I wanted to do this check in validate, but if it fails and
# then you fix the path to point to the appropriate chopshop
# it requires a webserver restart to take effect. So just do
# the check at each scan.
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
self._error("Need ChopShop 4.0 or newer")
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = self.base_dir
# XXX: Convert from unicode to str...
choplib.modules = str(self.modules)
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
chopui.bind(choplib)
chopui.start()
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
def run(self, obj, config):
logger.debug("Setting up shop...")
base_dir = config['basedir']
shop_path = "%s/shop" % base_dir
if not os.path.exists(base_dir):
self._error("ChopShop path does not exist")
return
elif not os.path.exists(shop_path):
self._error("ChopShop shop path does not exist")
return
sys.path.append(shop_path)
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = base_dir
choplib.modules = "metacap -b"
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
try:
chopui.bind(choplib)
chopui.start()
while chopui.jsonclass == None:
time.sleep(.1)
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
except Exception as e:
self._error(str(e))
finally:
chopui.join()
choplib.finish()
choplib.join()
def run(self, obj, config):
logger.debug("Setting up shop...")
base_dir = config['basedir']
shop_path = "%s/shop" % base_dir
if not os.path.exists(base_dir):
self._error("ChopShop path does not exist")
return
elif not os.path.exists(shop_path):
self._error("ChopShop shop path does not exist")
return
sys.path.append(shop_path)
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = base_dir
choplib.modules = "metacap -b"
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
try:
chopui.bind(choplib)
chopui.start()
while chopui.jsonclass == None:
time.sleep(.1)
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
except Exception as e:
self._error(str(e))
finally:
chopui.join()
choplib.finish()
choplib.join()
def _scan(self, context):
logger.debug("Setting up shop...")
shop_path = "%s/shop" % self.base_dir
if not os.path.exists(self.base_dir):
raise ServiceConfigError("ChopShop path does not exist")
elif not os.path.exists(shop_path):
raise ServiceConfigError("ChopShop shop path does not exist")
else:
sys.path.append(shop_path)
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = self.base_dir
# XXX: Convert from unicode to str...
choplib.modules = str(self.modules)
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
chopui.bind(choplib)
chopui.start()
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
def run_module_info(self, modules):
clib = ChopLib()
clib.text = True
clib.modules = modules
clib.modinfo = True
clib.start()
stopped = False
message_queue = clib.get_message_queue()
while not stopped and clib.is_alive():
try:
message = message_queue.get(True, .1)
except Queue.Empty, e:
continue
#clean up messages
if message['type'] == 'ctrl':
#self.send_message(message['data']['msg'] )
if message['data']['msg'] == 'finished':
stopped = True
elif message['type'] == 'text':
self.send_message(message['data']['data'])
def run_module_info(self, modules):
clib = ChopLib()
clib.text = True
clib.modules = modules
clib.modinfo = True
clib.start()
stopped = False
message_queue = clib.get_message_queue()
while not stopped and clib.is_alive():
try:
message = message_queue.get(True, .1)
except Queue.Empty, e:
continue
#clean up messages
if message['type'] == 'ctrl':
#self.send_message(message['data']['msg'] )
if message['data']['msg'] == 'finished':
stopped = True
elif message['type'] == 'text':
self.send_message(message['data']['data'])
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
user = get_user_info(analyst)
if not sc:
return {'success': False, 'message': 'Could not find ChopShop service.'}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {'success': False, 'message': "ChopShop shop path does not exist."}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {'success': False, 'message': "Can not process SMTP and raw simultaneously."}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {'success': False, 'message': "Could not get PCAP from GridFS: %s" % pcap_md5}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {'success': False,
'message': 'Lost race condition in chopui. Try again.'}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if user.has_access_to(SampleACL.WRITE) and handle_file(name, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', md5_digest=md5_digest, related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob, source, None, analyst, 'ChopShop FileCarver',
related_id=pcap.id, related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (reverse('crits-emails-views-email_detail', args=[ret['object'].id]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
if user.has_access_to(SampleACL.WRITE):
md5_digest = handle_file(id_, blob, source, related_md5=pcap_md5, user=user, source_method='ChopShop Filecarver', related_type='PCAP')
else:
md5_digest = None
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (id_, reverse('crits-samples-views-detail', args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def chopshop_carver(pcap_md5, options, analyst):
# Make sure we can find ChopShop
sc = get_config('ChopShop')
if not sc:
return {
'success': False,
'message': 'Could not find ChopShop service.'
}
shop_path = "%s/shop" % str(sc['basedir'])
if not os.path.exists(shop_path):
return {
'success': False,
'message': "ChopShop shop path does not exist."
}
sys.path.append(shop_path)
import ChopLib as CL
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
return {'success': False, 'message': 'Need ChopShop 4.0 or newer'}
# Until we have an smtp_extractor in ChopShop we have to resort to
# to (ab)using payloads to dump the entire TCP stream and letting
# handle_eml() process everything. We also use the payloads module
# for handling raw carves. If a user wants to do SMTP and raw
# simultaneously it won't work because we can't distinguish one
# payloads module from another.
if options.get('raw', False) and options.get('smtp', False):
return {
'success': False,
'message': "Can not process SMTP and raw simultaneously."
}
# Make sure we have a PCAP to work with
pcap = PCAP.objects(md5=pcap_md5).first()
if not pcap:
return {'success': False, 'message': "No PCAP found."}
pcap_data = pcap.filedata.read()
if not pcap_data:
return {
'success': False,
'message': "Could not get PCAP from GridFS: %s" % pcap_md5
}
source = pcap['source'][0]['name'] # XXX: This kind of sucks...
# Create module string to pass to ChopShop
modules = []
if options.get('http_resp', False) or options.get('http_req', False):
modules.append("http | http_extractor")
if options.get('smtp', False) or options.get('raw', False):
# ChopShop really needs an smtp_extractor, but there's no good
# capability to do that yet. Maybe one day I'll build one. :)
# For now, just use payloads and let handle_eml() sort it out.
#
# Raw carving works exactly the same way, just post-processed
# differently.
modules.append("payloads -b")
if not modules:
return {'success': False, 'message': "No modules specified."}
mod_string = ';'.join(mod for mod in modules)
from ChopLib import ChopLib
from ChopUi import ChopUi
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = str(sc['basedir'])
choplib.modules = mod_string
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# Write the pcap data to a temporary file.
temp_pcap = tempfile.NamedTemporaryFile(delete=False)
temp_pcap.write(pcap_data)
temp_pcap.close()
choplib.filename = temp_pcap.name
chopui.bind(choplib)
chopui.start()
if chopui.jsonclass == None:
os.unlink(temp_pcap.name)
chopui.join()
choplib.finish()
choplib.join()
return {
'success': False,
'message': 'Lost race condition in chopui. Try again.'
}
# ChopUI must be started before the jsonhandler class is insantiated.
# Tell the class what we are looking for now that it exists.
chopui.jsonclass.parse_options(options)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
os.unlink(temp_pcap.name)
message = ''
# Grab any carved HTTP bodies.
for (md5_digest, (name, blob)) in chopui.jsonclass.http_files.items():
if handle_file(name,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
md5_digest=md5_digest,
related_type='PCAP'):
# Specifically not using name here as I don't want to deal
# with sanitizing it
message += "Saved HTTP body: <a href=\"%s\">%s</a><br />" % (
reverse('crits.samples.views.detail', args=[md5_digest
]), md5_digest)
else:
message += "Failed to save file %s." % md5_digest
# Grab any carved SMTP returns.
for blob in chopui.jsonclass.smtp_returns.values():
ret = handle_eml(blob,
source,
None,
analyst,
'ChopShop FileCarver',
related_id=pcap.id,
related_type='PCAP',
relationship_type=RelationshipTypes.RELATED_TO)
if not ret['status']:
message += ret['reason']
continue
message += "Saved email: <a href=\"%s\">%s</a><br />%i attachment(s)<br />" % (
reverse('crits.emails.views.email_detail', args=[
ret['object'].id
]), ret['object'].id, len(ret['attachments'].keys()))
for md5_digest in ret['attachments'].keys():
message += "<a href=\"%s\">%s</a><br />" % (reverse(
'crits.samples.views.detail', args=[md5_digest]), md5_digest)
# Handle raw returns.
for id_, blob in chopui.jsonclass.raw_returns.items():
md5_digest = handle_file(id_,
blob,
source,
related_md5=pcap_md5,
user=analyst,
method='ChopShop Filecarver',
related_type='PCAP')
if md5_digest:
message += "Saved raw %s: <a href=\"%s\">%s</a><br />" % (
id_, reverse('crits.samples.views.detail',
args=[md5_digest]), md5_digest)
else:
message += "Failed to save raw %s." % md5_digest
# It's possible to have no files here if nothing matched.
# Still return True as there were no problems.
if not message:
message = 'No files found.'
return {'success': True, 'message': message}
def run(self, obj, config):
# When running under mod_wsgi we have to make sure sys.stdout is not
# going to the real stdout. This is because multiprocessing (used by
# choplib internally) does sys.stdout.flush(), which mod_wsgi doesn't
# like. Work around by pointing sys.stdout somewhere that mod_wsgi
# doesn't care about.
sys.stdout = sys.stderr
sys.stdin = open(os.devnull)
logger.debug("Initializing ChopShop service.")
basedir = config['basedir']
modules = ""
if 'HTTP' in config['modules']:
modules += ";http | http_extractor -m"
if 'DNS' in config['modules']:
modules += ";dns | dns_extractor"
logger.debug("Setting up shop...")
shop_path = "%s/shop" % basedir
if not os.path.exists(basedir):
self._error("ChopShop path does not exist")
return
elif not os.path.exists(shop_path):
self._error("ChopShop shop path does not exist")
return
sys.path.append(shop_path)
import ChopLib as CL
# I wanted to do this check in validate, but if it fails and
# then you fix the path to point to the appropriate chopshop
# it requires a webserver restart to take effect. So just do
# the check at each scan.
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
self._error("Need ChopShop 4.0 or newer")
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = basedir
# XXX: Convert from unicode to str...
choplib.modules = str(modules)
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
chopui.bind(choplib)
chopui.start()
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()
def run(self, obj, config):
# When running under mod_wsgi we have to make sure sys.stdout is not
# going to the real stdout. This is because multiprocessing (used by
# choplib internally) does sys.stdout.flush(), which mod_wsgi doesn't
# like. Work around by pointing sys.stdout somewhere that mod_wsgi
# doesn't care about.
sys.stdout = sys.stderr
sys.stdin = open(os.devnull)
logger.debug("Initializing ChopShop service.")
basedir = config['basedir']
modules = ""
if 'HTTP' in config['modules']:
modules += ";http | http_extractor -m"
if 'DNS' in config['modules']:
modules += ";dns | dns_extractor"
logger.debug("Setting up shop...")
shop_path = "%s/shop" % basedir
if not os.path.exists(basedir):
self._error("ChopShop path does not exist")
return
elif not os.path.exists(shop_path):
self._error("ChopShop shop path does not exist")
return
sys.path.append(shop_path)
import ChopLib as CL
# I wanted to do this check in validate, but if it fails and
# then you fix the path to point to the appropriate chopshop
# it requires a webserver restart to take effect. So just do
# the check at each scan.
if StrictVersion(str(CL.VERSION)) < StrictVersion('4.0'):
self._error("Need ChopShop 4.0 or newer")
from ChopLib import ChopLib
from ChopUi import ChopUi
logger.debug("Scanning...")
choplib = ChopLib()
chopui = ChopUi()
choplib.base_dir = basedir
# XXX: Convert from unicode to str...
choplib.modules = str(modules)
chopui.jsonout = jsonhandler
choplib.jsonout = True
# ChopShop (because of pynids) needs to read a file off disk.
# The services framework forces you to use 'with' here. It's not
# possible to just get a path to a file on disk.
with self._write_to_file() as pcap_file:
choplib.filename = pcap_file
chopui.bind(choplib)
chopui.start()
chopui.jsonclass.set_service(self)
choplib.start()
while chopui.is_alive():
time.sleep(.1)
chopui.join()
choplib.finish()
choplib.join()