def run(self):
url = 'http://urlvoid.com/scan/%s/' % self.artifact['name']
try:
status, response = get(url, headers=self.headers)
if status:
data = BeautifulSoup(response.text)
if data.findAll('div',
attrs={'class': 'bs-callout bs-callout-info'}):
pass
elif data.findAll('div',
attrs={
'class': 'bs-callout bs-callout-warning'
}):
self.artifact['data']['urlvoid'] = {}
for each in data.findAll('img', alt='Alert'):
site = each.parent.parent.td.text.lstrip()
url = each.parent.a['href']
self.artifact['data']['urlvoid'][site] = url
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def ip(self):
url = 'https://api.threatminer.org/v2/host.php?q=%s&rt=2' % self.artifact[
'name']
# check for passive DNS results
try:
status, response = get(url)
if status:
data = response.json()
if data['status_code'] == '200' and 'status_message' == 'Results found.':
self.artifact['data']['threatminer'] = {
'passivedns': data['results']
}
# check for potential children artifacts
if isinstance(self.artifact['data']['threatminer'], dict):
for entry in self.artifact['data']['threatminer'][
'passivedns']:
if detect_type(entry['domain']) == 'host':
self.artifact['children'].append({
'name':
entry['domain'],
'type':
'host',
'subtype':
'fqdn',
'source':
'threatminer'
})
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def on_change(self, callback, field=False):
match = re.match('^(.*?)\((.*)\)$', callback)
if not match:
raise Exception, 'ERROR: Wrong on_change trigger: %s' % callback
func_name = match.group(1)
arg_names = [n.strip() for n in match.group(2).split(',') if n.strip()]
idx = False
if 'context' in arg_names:
idx = arg_names.index('context')
args = [self.expr_eval(arg) for arg in arg_names]
if idx and field:
args[idx].update(field.context_get(self))
ids = self.id and [self.id] or []
response = getattr(self.rpc, func_name)(ids, *args)
if response:
self.set(response.get('value', {}), modified=True)
if 'domain' in response:
for fieldname, value in response['domain'].items():
if fieldname not in self.mgroup.mfields:
continue
self.mgroup.mfields[fieldname].attrs['domain'] = value
if 'context' in response:
value = response.get('context', {})
self.mgroup.context = value
warning = response.get('warning', {})
if warning:
parent = self.mgroup.screen and self.mgroup.screen.current_view and self.mgroup.screen.current_view.window or False
common.warning(warning['message'], warning['title'], parent=parent)
self.signal('record-changed')
def run(self):
url = 'http://api.ipstack.com/%s?access_key=%s&hostname=1' % (
self.artifact['name'], self.api_key)
try:
status, response = get(url, headers=self.headers)
if status:
results = response.json()
self.artifact['data']['geoip'] = results
if 'hostname' in results.keys():
if results['hostname'] != self.artifact[
'name'] and results['hostname'] != '':
self.artifact.children.append({
'name':
results['hostname'],
'type':
'host',
'subtype':
'fqdn',
'source':
'ipstack'
})
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def run(self):
url = 'https://api.github.com/users/%s' % self.artifact['name']
headers = {
'User-Agent': 'OSINT Omnibus (https://github.com/InQuest/Omnibus)'
}
try:
status, response = get(url, headers=headers)
if status:
self.artifact.data['github'] = response.json()
if 'email' in self.artifact.data['github'].keys():
if self.artifact.data['github']['email']:
self.artifact['children'].append({
'name':
self.artifact.data['github']['email'],
'type':
'email',
'subtype':
None,
'source':
'github'
})
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def report(text, fname):
common.progress('saving report')
try:
with open(fname, 'w') as outfile:
outfile.write(text.encode('utf8'))
except (IOError, OSError, UnicodeEncodeError):
common.warning('Report output failed.')
def machine(self, session, artifact):
""" Run all modules against an artifact of a given type """
is_key, value = lookup_key(session, artifact)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % artifact)
return
elif is_key and value is not None:
artifact = value
else:
pass
artifact_type = detect_type(artifact)
artifact = self.db.find(artifact_type, {'name': artifact}, one=True)
for key in self.modules.keys():
if artifact['type'] == key:
modules = self.modules[artifact['type']]
elif artifact['subtype'] == key:
modules = self.modules[artifact['subtype']]
results = []
for m in modules:
result = self.run(m, artifact)
if m in result['data'].keys():
if result['data'][m] is not None:
if self.db.exists(artifact['type'],
{'name': artifact['name']}):
for child in result['children']:
child_artifact = create_artifact(
child['name'],
parent=artifact['name'],
_type=child['type'],
source=child['source'],
subtype=child['subtype'])
if not self.db.exists(child['type'],
{'name': child['name']}):
self.db.insert_one(child['type'],
child_artifact)
self.db.update_one(artifact['type'],
{'name': artifact['name']}, result)
if len(result['children']) > 0:
info('Created child artifacts: %d' %
len(result['children']))
results.append({'[%s]' % m: result['data'][m]})
else:
warning('No results found (%s)' % m)
else:
warning('Failed to get module results (%s)' % m)
success('Machine completed')
def fqdn(self):
url = 'http://pgp.mit.edu/pks/lookup?op=index&search=%s' % self.artifact[
'name']
try:
status, response = get(url, headers=self.headers)
if status:
if 'No results found' in response.text:
pass
else:
data = BeautifulSoup(response.text)
items = data.fetch('a')
for item in items:
matches = re.findall(re_email, item)
for m in matches:
if isinstance(self.artifact['data']['pgp'], list):
self.artifact['data']['pgp'].append(m)
else:
self.artifact['data']['pgp'] = []
self.artifact['data']['pgp'].append(m)
self.artifact['children'].append({
'name': m,
'type': 'email',
'source': 'PGP',
'subtype': None
})
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def check_upload_submit_consistency(self):
issue_id = self.issue
self.issue_dict = common.load_issue_info(self.blade_root_dir)
if not self.issue_dict.has_key(issue_id):
warning('issue %s not found in issue_info_dict' % issue_id)
return
# TODO(naicaisun): submit.py support --files like upload.py
relative_path = os.path.relpath(common.get_cwd(), self.blade_root_dir)
file_list = common.get_all_diff_files()
file_list = set(
[os.path.join(relative_path, filename) for filename in file_list])
# TODO(naicaisun): delete backward compatible after running serveral weeks
difference_files = []
if isinstance(self.issue_dict[issue_id], set):
difference_files = file_list.symmetric_difference(
self.issue_dict[issue_id])
else:
issue_info = self.issue_dict[issue_id]
if issue_info["upload_path"] != relative_path:
error_exit('the submit path: %s is not the same as upload path'
' : %s' %
(relative_path, issue_info["upload_path"]))
difference_files = file_list.symmetric_difference(
issue_info["filelist"])
if difference_files:
warning('the following file[s] you submit are not consistent with '
'the ones you uploaded\n%s' % "\n".join(difference_files))
answer = raw_input('Continue?(y/N) ').strip()
if answer != 'y':
error_exit('Exit')
else:
if isinstance(self.issue_dict[issue_id], set):
self.issue_dict[issue_id] = file_list
else:
self.issue_dict[issue_id]["filelist"] = file_list
def rpc_exec_auth(self, obj, method, *args):
if self._open:
try:
sock = self._gw(self._url, self.db, self.uid, self._passwd, obj)
return sock.exec_auth(method, *args)
except socket.error, e:
common.message(_('Unable to reach to OpenERP server !\nYou should check your connection to the network and the OpenERP server.'), _('Connection Error'), type=gtk.MESSAGE_ERROR)
raise rpc_exception(69, 'Connection refused!')
except Exception, e:
if isinstance(e, xmlrpclib.Fault) \
or isinstance(e, tiny_socket.Myexception):
a = rpc_exception(e.faultCode, e.faultString)
if a.type in ('warning','UserError'):
if a.message in ('ConcurrencyException') and len(args) > 4:
if common.concurrency(args[0], args[2][0], args[4]):
if CONCURRENCY_CHECK_FIELD in args[4]:
del args[4][CONCURRENCY_CHECK_FIELD]
return self.rpc_exec_auth(obj, method, *args)
else:
common.warning(a.data, a.message)
else:
common.error(_('Application Error'), e.faultCode, e.faultString)
else:
common.error(_('Application Error'), _('View details'), str(e))
#TODO Must propagate the exception?
raise
def rpc_exec_auth(self, obj, method, *args):
if self._open:
try:
sock = self._gw(self._url, self.db, self.uid, self._passwd, obj)
return sock.exec_auth(method, *args)
except socket.error, e:
common.message(_('Unable to reach to OpenERP server !\nYou should check your connection to the network and the OpenERP server.'), _('Connection Error'), type=gtk.MESSAGE_ERROR)
raise rpc_exception(69, 'Connection refused!')
except Exception, e:
if isinstance(e, (xmlrpclib.Fault, tiny_socket.Myexception)):
a = rpc_exception(e.faultCode, e.faultString)
if isinstance(e.faultCode, int):
a.type = 'UserError'
a.message = 'Error %s' % e.faultCode
a.data = e.faultString
if a.type in ('warning','UserError'):
if a.message in ('ConcurrencyException') and len(args) > 4:
if common.concurrency(args[0], args[2][0], args[4]):
if CONCURRENCY_CHECK_FIELD in args[4]:
del args[4][CONCURRENCY_CHECK_FIELD]
return self.rpc_exec_auth(obj, method, *args)
else:
common.warning(a.data, a.message)
else:
common.error(_('Application Error'), e.faultCode, e.faultString)
else:
common.error(_('Application Error'), _('View details'), str(e))
#TODO Must propagate the exception?
raise
def __init__(self, name, type, source=None, subtype=None, case_id=None):
self.name = name
self.created = timestamp()
self.type = type
self.subtype = subtype
self.source = source
self.parent = None
self.children = []
self.case_id = case_id
self.tags = []
self.notes = []
self.data = {}
if self.type == 'host':
if is_ipv4(self.name):
self.subtype = 'ipv4'
elif is_ipv6(self.name):
self.subtype = 'ipv6'
elif is_fqdn(self.name):
self.subtype = 'fqdn'
else:
warning('host type cannot be determined. must be one of: ipv4, ipv6, fqdn')
self.subtype = 'unknown'
elif self.type == 'hash':
result = is_hash(self.name)
if result is None:
warning('hash is not a valid md5, sha1, sha256, or sha512')
self.subtype = 'unknown'
else:
self.subtype = result
def import_csv(csv_data, f, model, fields, context=None):
fname = csv_data['fname']
content = file(fname, 'rb').read()
input = cStringIO.StringIO(content)
input.seek(0)
data = list(
csv.reader(input,
quotechar=csv_data['del'] or '"',
delimiter=csv_data['sep']))[int(csv_data['skip']):]
datas = []
for line in data:
if not line:
continue
datas.append(
map(lambda x: x.decode(csv_data['combo']).encode('utf-8'), line))
if not datas:
common.warning(_('The file is empty !'), _('Importation !'))
return False
try:
res = rpc.session.rpc_exec_auth('/object', 'execute', model,
'import_data', f, datas, 'init', '',
False, context)
except Exception, e:
common.warning(str(e), _('XML-RPC error !'))
return False
def email(self):
url = 'http://pgp.mit.edu/pks/lookup?op=index&search=%s' % self.artifact[
'name']
try:
status, response = get(url, headers=self.headers)
if status:
if 'No results found' in response.text:
pass
else:
data = BeautifulSoup(response.text)
hrefs = data.fetch('a')
for href in hrefs:
content = href.contents
if self.artifact['name'] in content[0]:
try:
name = content[0].split('<')[0]
if isinstance(self.artifact['data']['pgp'],
list):
self.artifact['data']['pgp'].append(name)
else:
self.artifact['data']['pgp'] = []
self.artifact['data']['pgp'].append(name)
except IndexError:
warning(
'Unable to parse returned PGP web data')
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def run(self):
if self.artifact['type'] == 'email':
self.email()
elif self.artifact['type'] == 'fqdn':
self.fqdn()
else:
warning('PGP module only accepts artifact types email or fqdn')
def on_change(self, callback):
match = re.match("^\s?(.*?)\((.*?)\)\s?$", callback)
if not match:
raise Exception, "ERROR: Wrong on_change trigger: %s" % callback
func_name = match.group(1)
arg_names = [n.strip() for n in match.group(2).split(",") if n.strip()]
args = [self.expr_eval(arg) for arg in arg_names]
ids = self.id and [self.id] or []
response = getattr(self.rpc, func_name)(ids, *args)
if response:
self.set(response.get("value", {}), modified=True)
if "domain" in response:
for fieldname, value in response["domain"].items():
if fieldname not in self.mgroup.mfields:
continue
self.mgroup.mfields[fieldname].attrs["domain"] = value
if "context" in response:
value = response.get("context", {})
self.mgroup.context = value
warning = response.get("warning", {})
if warning:
parent = self.mgroup.screen and self.mgroup.screen.current_view.window or False
common.warning(warning["message"], warning["title"], parent=parent)
self.signal("record-changed")
def ip(self):
whois_data = IPWhois(self.artifact['name'])
try:
data = whois_data.lookup_whois()
if data is not None:
self.artifact['data']['whois'] = {}
# collect ASN information
self.artifact['data']['whois']['asn'] = data['asn']
self.artifact['data']['whois']['asn']['cidr'] = data[
'asn_cidr']
self.artifact['data']['whois']['asn']['description'] = data[
'asn_description']
self.artifact['data']['whois']['asn']['country'] = data[
'asn_country_code']
if 'nets' in data.keys() and len(data['nets']) > 0:
net_data = data['nets'][0]
self.artifact['data']['whois']['address'] = net_data[
'address']
self.artifact['data']['whois']['state'] = net_data['state']
self.artifact['data']['whois']['emails'] = net_data[
'emails']
except Exception as err:
warning('Caught unhandled exception: %s' % str(err))
def context_reload(self):
"""Reload the context for the current user
"""
self.context = {'client': 'web'}
self.timezone = 'utc'
# self.uid
context = self.execute('object', 'execute', 'ir.values', 'get', 'meta', False, [('res.users', self.uid or False)], False, {}, True, True, False)
for c in context:
if c[2]:
self.context[c[1]] = c[2]
if c[1] == 'lang':
pass
# ids = self.execute('object', 'execute', 'res.lang', 'search', [('code', '=', c[2])])
# if ids:
# l = self.execute('object', 'execute', 'res.lang', 'read', ids, ['direction'])
# if l and 'direction' in l[0]:
# common.DIRECTION = l[0]['direction']
# import gtk
# if common.DIRECTION == 'rtl':
# gtk.widget_set_default_direction(gtk.TEXT_DIR_RTL)
# else:
# gtk.widget_set_default_direction(gtk.TEXT_DIR_LTR)
elif c[1] == 'tz':
self.timezone = self.execute('common', 'timezone_get')
try:
import pytz
except:
common.warning(_('You select a timezone but Tiny ERP could not find pytz library !\nThe timezone functionality will be disable.'))
# set locale in session
self.locale = self.context.get('lang')
def run(self):
if self.artifact['subtype'] == 'ipv4':
self.ip()
elif self.artifact['subtype'] == 'fqdn':
self.fqdn()
else:
warning('Threatcrowd module only supports artifacts of type ipv4 or fqdn')
def main(artifact):
if artifact['subtype'] == 'ipv4':
plugin = Plugin(artifact)
plugin.run()
else:
warning('IPVoid only accepts artifacts of subtype IPv4')
return plugin.artifact
def sig_autodetect(self, widget=None):
fname= self.filechooser.get_filename()
if not fname:
common.message('You must select an import file first !')
return True
csvsep= self.ui.get_object('import_csv_sep').get_text()
csvdel= self.ui.get_object('import_csv_del').get_text()
csvcode= self.ui.get_object('import_csv_combo').get_active_text() or 'UTF-8'
self.ui.get_object('import_csv_skip').set_value(1)
try:
data = csv.reader(file(fname), quotechar=csvdel or '"', delimiter=csvsep)
except:
common.warning(_('Error opening .CSV file'), _('Input Error.'), parent=self.win)
return True
self.sig_unsel_all()
word=''
try:
for line in data:
for word in line:
word = word.encode(csvcode.lower())
if (word in self.fields):
num = self.model2.append()
self.model2.set(num, 0, self.fields[word], 1, word)
elif word in self.fields_invert:
self.invert = True
num = self.model2.append()
self.model2.set(num, 0, word, 1, word)
else:
raise Exception(_("You cannot import this field %s, because we cannot auto-detect it"))
break
except:
common.warning(_('Error processing your first line of the file.\nField %s is unknown !') % (word,), _('Import Error.'), parent=self.win)
return True
def _send_cpplint_result_to_comments(self, issue_id):
# get trunk url of current dir
cmd = ["svn", "info"]
svn_info, returncode = common.run_shell(cmd)
if returncode:
error_exit("failed to run '%s'" % " ".join(cmd))
current_trunk_dir = svn_info.split('\n')[2].split()[1]
comment = []
for msg in self.cpplint_err_msgs:
err_info = msg.split(":")
if len(err_info) < 3:
continue
temp_dict = {}
temp_dict["file"] = os.path.join(current_trunk_dir, err_info[0])
temp_dict["line"] = err_info[1]
temp_dict["type"] = "3"
temp_dict["comment"] = "CPPLINT:" + err_info[2]
comment.append(temp_dict)
summary = ('cpplint check result: %d new errors found' %
len(self.cpplint_err_msgs))
json_content = {"comments": comment, "summary": summary}
(fd, comment_file) = tempfile.mkstemp(suffix=".json")
with open(comment_file, 'w') as fd:
json.dump(json_content, fd)
# send comment
cmd = ["%s/app/qzap/common/tool/tcr.py" % self.current_source_dir]
cmd.extend(["-i", issue_id, "--comment", comment_file])
output, ret = common.run_shell(cmd)
if ret:
common.warning("Failed to send comments! ret=%d" % ret)
os.remove(comment_file)
def create_artifact(artifact_name,
_type=None,
source=None,
subtype=None,
parent=None):
if _type is None:
artifact_type = detect_type(artifact_name)
else:
artifact_type = _type
if artifact_type not in artifact_types:
warning(
'Artifact must be one of: email, ipv4, fqdn, user, hash, bitcoin address'
)
return None
created = Artifact(name=artifact_name,
type=artifact_type,
subtype=subtype,
source=source)
if parent is not None:
created.parent = parent
return created
def parse_cmdline(argv):
"""
Returns the parsed argument list and return code.
`argv` is a list of arguments, or `None` for ``sys.argv[1:]``.
"""
if argv is None:
argv = sys.argv[1:]
# initialize the parser object:
parser = argparse.ArgumentParser()
parser.add_argument(
"-s",
"--structural",
help="Calculate structural properties (lattice parameters and angles)",
action='store_true')
parser.add_argument(
"-e",
"--electronic",
help="Calculate electronic properties (band gap, VBM, CBM, etc.)",
action='store_true')
parser.add_argument(
"-a",
"--all",
help="Calculate both structural and electronic properties",
action='store_true')
args = None
try:
args = parser.parse_args(argv)
except ArgumentParserError as e:
warning("Argument Parser Error:", e)
parser.print_help()
return args, INPUT_ERROR
return args, GOOD_RET
def on_change(self, callback, field=False):
match = re.match('^(.*?)\((.*)\)$', callback)
if not match:
raise Exception, 'ERROR: Wrong on_change trigger: %s' % callback
func_name = match.group(1)
arg_names = [n.strip() for n in match.group(2).split(',') if n.strip()]
idx = False
if 'context' in arg_names:
idx = arg_names.index('context')
args = [self.expr_eval(arg) for arg in arg_names]
if idx and field:
args[idx].update(field.context_get(self))
ids = self.id and [self.id] or []
response = getattr(self.rpc, func_name)(ids, *args)
if response:
self.set(response.get('value', {}), modified=True)
if 'domain' in response:
for fieldname, value in response['domain'].items():
if fieldname not in self.mgroup.mfields:
continue
self.mgroup.mfields[fieldname].attrs['domain'] = value
if 'context' in response:
value = response.get('context', {})
self.mgroup.context = value
warning = response.get('warning', {})
if warning:
parent = self.mgroup.screen and self.mgroup.screen.current_view and self.mgroup.screen.current_view.window or False
common.warning(warning['message'],
warning['title'],
parent=parent)
self.signal('record-changed')
def sig_save_as(self, widget):
if not self._value:
common.warning('There is no image!',_('Warning'))
else:
filename = common.file_selection(_('Save As...'), parent=self._window,
action=gtk.FILE_CHOOSER_ACTION_SAVE)
if filename:
file(filename, 'wb').write(decodestring(self._value))
def sig_save(self, widget=None, sig_new=True, auto_continue=True):
id = self.screen.save_current()
if id:
self.message_state(_('Document Saved.'), color="darkgreen")
else:
common.warning(_('Invalid form, correct red fields !'),_('Error !'))
self.message_state(_('Invalid form, correct red fields !'), color="red")
return bool(id)
def main(artifact):
if artifact['subtype'] == 'ipv4':
plugin = Plugin(artifact)
plugin.run()
else:
warning('IPVoid only accepts artifacts of subtype IPv4')
return plugin.artifact
def run(self):
if self.artifact['subtype'] == 'ipv4':
self.ip()
elif self.artifact['subtype'] == 'fqdn':
self.fqdn()
else:
warning(
'Threatcrowd module only supports artifacts of type ipv4 or fqdn'
)
def run(self):
url = 'https://censys.io/api/v1/view/ipv4/%s' % self.artifact['name']
try:
status, response = get(url, auth=(self.api_key['token'], self.api_key['secret']), headers=self.headers)
if status:
self.artifact['data']['censys'] = response.json()
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def run(self):
url = 'https://blockchain.info/rawaddr/%s' % self.artifact['name']
try:
status, response = get(url, headers=self.headers)
if status:
self.artifact['data']['blockchain'] = response.json()
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def close(self):
try:
del self.routeCursor
arcpy.SelectLayerByAttribute_management(self.places, 'CLEAR_SELECTION')
except:
pass
remapErrors = self.placeMapper.getErrorCount()
if remapErrors:
common.warning('Route not found for %i records, setting shape to NULL' % remapErrors)
TableLinker.close(self)
def __init__(self, artifact):
self.artifact = artifact
if not self.artifact['name'].startwith(
'https://') or not self.artifact['name'].startwith('http://'):
warning(
'Artifact is not a valid URL: pre-pendig "http://" to artifact for this module to run\n \
This will be reset after the module executes.')
self.artifact['name'] = 'http://' + self.artifact['name']
self.artifact['data']['wappalyzer'] = None
def run(self):
if self.artifact['type'] == 'host':
if self.artifact['subtype'] == 'ipv4':
self.ip()
elif self.artifact['subtype'] == 'fqdn':
self.host()
elif self.artifact['type'] == 'hash':
self.hash()
else:
warning('OTX module accepts artifact types host or hash')
def run(self):
if self.artifact['type'] == 'host':
if self.artifact['subtype'] == 'ipv4':
self.ip()
elif self.artifact['subtype'] == 'fqdn':
self.host()
elif self.artifact['type'] == 'hash':
self.hash()
else:
warning('OTX module accepts artifact types host or hash')
def pastes(self):
url = 'https://haveibeenpwned.com/api/v2/pasteaccount/%s' % self.artifact[
'name']
try:
status, response = get(url, headers=self.headers)
if status:
self.artifact['data']['hibp']['pastes'] = response.json()
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def ip(self):
url = 'https://api.shodan.io/shodan/host/%s?key=%s' % (
self.artifact['name'], self.api_key)
try:
status, response = get(url, headers=self.headers)
if status:
self.artifact['data']['shodan'] = response.json()
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def fqdn(self):
url = 'https://api.shodan.io/shodan/host/search?key=%s&query=hostname:%s&facets={facets}' % (
self.api_key, self.artifact['name'])
try:
status, response = get(url, headers=self.headers)
if status:
self.artifact['data']['shodan'] = response.json()
except Exception as err:
warning('Caught exception in module (%s)' % str(err))
def go(self):
while True:
button = self.win.run()
if button == gtk.RESPONSE_OK:
if not len(self.model2):
common.warning(
_("You have not selected any fields to import"))
continue
fields = []
fields2 = []
iter = self.model2.get_iter_root()
while iter:
fields.append(self.model2.get_value(iter, 1))
fields2.append(self.model2.get_value(iter, 0))
iter = self.model2.iter_next(iter)
csv = {
'fname':
self.glade.get_widget('import_csv_file').get_filename(),
'sep':
self.glade.get_widget('import_csv_sep').get_text(),
'del':
self.glade.get_widget('import_csv_del').get_text(),
'skip':
self.glade.get_widget('import_csv_skip').get_value(),
'combo':
self.glade.get_widget(
'import_csv_combo').get_active_text() or 'UTF-8'
}
self.parent.present()
self.win.destroy()
if csv['fname']:
if self.invert:
inverted = []
for f in fields:
for key, value in self.fields_invert.items():
if key.encode('utf8') == f:
inverted.append(value)
return import_csv(csv,
inverted,
self.model,
self.fields_invert,
context=self.context)
else:
return import_csv(csv,
fields,
self.model,
self.fields,
context=self.context)
return False
else:
self.parent.present()
self.win.destroy()
return False
def _menu_sig_default_get(self):
try:
if self._view.modelfield.get_state_attrs(self._view.model).get('readonly', False):
return False
model = self._view.modelfield.parent.resource
res = rpc.session.rpc_exec_auth_try('/object', 'execute', model, 'default_get', [self.attrs['name']])
self._view.modelfield.set(self._view.model, res.get(self.attrs['name'], False), modified=True)
self.display(self._view.model, self._view.modelfield)
except:
common.warning('You can not set to the default value here !', 'Operation not permited')
return False
def machine(self, session, artifact):
""" Run all modules against an artifact of a given type """
is_key, value = lookup_key(session, artifact)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % artifact)
return
elif is_key and value is not None:
artifact = value
else:
pass
artifact_type = detect_type(artifact)
artifact = self.db.find(artifact_type, {'name': artifact}, one=True)
for key in self.modules.keys():
if artifact['type'] == key:
modules = self.modules[artifact['type']]
elif artifact['subtype'] == key:
modules = self.modules[artifact['subtype']]
results = []
for m in modules:
result = self.run(m, artifact)
if m in result['data'].keys():
if result['data'][m] is not None:
if self.db.exists(artifact['type'], {'name': artifact['name']}):
for child in result['children']:
child_artifact = create_artifact(child['name'], parent=artifact['name'],
_type=child['type'], source=child['source'], subtype=child['subtype'])
if not self.db.exists(child['type'], {'name': child['name']}):
self.db.insert_one(child['type'], child_artifact)
self.db.update_one(artifact['type'], {'name': artifact['name']}, result)
if len(result['children']) > 0:
info('Created child artifacts: %d' % len(result['children']))
results.append({'[%s]' % m: result['data'][m]})
else:
warning('No results found (%s)' % m)
else:
warning('Failed to get module results (%s)' % m)
success('Machine completed')
def sig_save_as(self, widget=None):
fields = []
while(self.screen.view_to_load):
self.screen.load_view_to_load()
screen_fields = copy.deepcopy(self.screen.models.fields)
ids = self.screen.ids_get()
if not ids:
msg = _('No records to export! \n Select at least one record ')
if not ids and self.context.get('group_by_no_leaf', False):
msg = _('You cannot export these record(s) ! Either use copy and paste')
common.warning(msg,_('Warning !'), parent=self.window)
return
win = win_export.win_export(self.model, ids, screen_fields, fields, parent=self.window, context=self.context)
res = win.go()
def button_clicked(self, widget):
model = self.form.screen.current_model
self.form.set_value()
button_type = self.attrs.get('special', '')
if button_type=='cancel':
self.form.screen.window.destroy()
if 'name' in self.attrs.keys():
type_button = self.attrs.get('type','object')
if type_button == 'action':
obj = service.LocalService('action.main')
action_id = int(self.attrs['name'])
context_action = self.form.screen.context.copy()
if 'context' in self.attrs:
context_action.update(self.form.screen.current_model.expr_eval(self.attrs['context'], check_load=False))
obj.execute(action_id, {'model':self.form.screen.name, 'id': False, 'ids': [], 'report_type': 'pdf'}, context=context_action)
elif type_button == 'object':
result = rpc.session.rpc_exec_auth(
'/object', 'execute',
self.form.screen.name,
self.attrs['name'],[], model.context_get())
datas = {}
obj = service.LocalService('action.main')
obj._exec_action(result,datas,context=self.form.screen.context)
else:
raise Exception, 'Unallowed button type'
elif model.validate():
id = self.form.screen.save_current()
model.get_button_action(self.form.screen, id, self.attrs)
self.warn('misc-message', '')
else:
fields = common.get_invalid_field(self.form.screen.current_model, self.form.screen)
msg = ''
for req, inv in fields:
if not inv:
msg += req + ' (<b>invisible</b>) '
else:
msg += req
msg += '\n'
common.warning(_('Correct following red fields !\n\n%s') % ( msg ),_('Input Error !'), parent=self.form.screen.current_view.window)
self.warn('misc-message', _('Invalid form, correct red fields !'), "red")
self.form.screen.display()
self.form.screen.current_view.set_cursor()
def create_artifact(artifact_name, _type=None, source=None, subtype=None, parent=None):
if _type is None:
artifact_type = detect_type(artifact_name)
else:
artifact_type = _type
if artifact_type not in artifact_types:
warning('Artifact must be one of: email, ipv4, fqdn, user, hash, bitcoin address')
return None
created = Artifact(name=artifact_name, type=artifact_type, subtype=subtype, source=source)
if parent is not None:
created.parent = parent
return created
def sig_save(self, widget=None, sig_new=True, auto_continue=True):
res = self.screen.save_current()
warning = False
if isinstance(res,dict):
id = res.get('id',False)
warning = res.get('warning',False)
else:
id = res
if id:
self.message_state(_('Document Saved.'), color="darkgreen")
elif len(self.screen.models.models) and res != None:
common.warning(_('Invalid form, correct red fields !'),_('Error !'), parent=self.screen.current_view.window)
self.message_state(_('Invalid form, correct red fields !'), color="red")
if warning:
common.warning(warning,_('Warning !'), parent=self.screen.current_view.window)
return bool(id)
def context_reload(self):
"""Reload the context for the current user
"""
self.storage['_context'] = {'client': 'web'}
# self.uid
context = self.execute('object', 'execute', 'res.users', 'context_get')
self._context.update(context or {})
self.storage['remote_timezone'] = 'utc'
self.storage['client_timezone'] = self.context.get("tz", False)
if self.storage.get('client_timezone'):
self.storage['remote_timezone'] = self.execute('common', 'timezone_get')
try:
import pytz
except:
raise common.warning(_('You select a timezone but OpenERP could not find pytz library!\nThe timezone functionality will be disable.'))
# set locale in session
self.storage['locale'] = self.context.get('lang', 'en_US')
lang_ids = self.execute(
'object', 'execute', 'res.lang',
'search', [('code', '=', self.storage['locale'])])
if lang_ids:
self.storage['lang'] = self.execute(
'object', 'execute', 'res.lang', 'read', lang_ids[0], [])
def submit(self, session, module, artifact, no_argument=False):
""" Run a single module against an artifact """
if no_argument:
module_result = self.run(module, None)
return module_result
is_key, value = lookup_key(session, artifact)
if is_key and value is None:
error('Unable to find artifact key in session (%s)' % artifact)
return
elif is_key and value is not None:
artifact = value
else:
pass
artifact_type = detect_type(artifact)
artifact = self.db.find(artifact_type, {'name': artifact}, one=True)
if artifact is None:
warning('Unable to find artifact in database (%s)' % artifact['name'])
return None
if module in self.modules[artifact['type']] or module in self.modules[artifact['subtype']]:
pass
else:
warning('Artifact is not supported by module (%s)' % (artifact['name']))
return None
result = self.run(module, artifact)
if module in result['data'].keys():
if result['data'][module] is not None:
if self.db.exists(artifact['type'], {'name': artifact['name']}):
for child in result['children']:
child_artifact = create_artifact(child['name'], parent=artifact['name'],
_type=child['type'], source=child['source'], subtype=child['subtype'])
if not self.db.exists(child['type'], {'name': child['name']}):
self.db.insert_one(child['type'], child_artifact)
self.db.update_one(artifact['type'], {'name': artifact['name']}, result)
if len(result['children']) > 0:
info('Created child artifacts: %d' % len(result['children']))
return result['data'][module]
else:
warning('No results found (%s)' % module)
return None
else:
warning('Failed to get module results (%s)' % module)
def run(self):
nm = NmapProcess(targets=str(self.artifact['name']), options='-sT -sV -Pn -T5 -p21,22,23,25,80,6667,1337')
nm.run()
if nm.is_successful():
report = NmapParser.parse_fromstring(nm.stdout)
for host in report.hosts:
if host.is_up():
results = {
'ports': host.get_open_ports(),
'services': []
}
for service in host.services:
if service.state == 'open':
serv = {
'banner': service.banner,
'protocol': service.protocol,
'service': service.service,
'port': service.port}
results['services'].append(serv)
if self.artifact['subtype'] == 'ipv4':
results['hostnames'] = host.hostnames
for h in host.hostnames:
self.artifact['children'].append({
'name': h,
'type': 'host',
'subtype': 'fqdn',
'source': 'Nmap'
})
elif self.artifact['subtype'] == 'fqdn':
results['ipv4'] = host.address
self.artifact['children'].append({
'name': host.address,
'type': 'host',
'subtype': 'ipv4',
'source': 'Nmap'
})
self.artifact['data']['nmap'] = results
else:
warning('Nmap scanner failed - no results')
def run(self):
url = 'https://person.clearbit.com/v1/people/email/%s' % self.artifact['name']
headers = {
'Authorization': 'Bearer %s' % self.api_key,
'User-Agent': 'OSINT Omnibus (https://github.com/InQuest/Omnibus)'
}
try:
status, response = get(url, headers=headers)
if status:
if 'error' in response.content and 'queued' in response.content:
warning('results are queued by Clearbit. please re-run module after 5-10 minutes.')
else:
self.artifact['data']['fullcontact'] = response.json()
except:
pass
def import_csv(csv_data, f, model, fields, context=None, parent=None):
fname = csv_data['fname']
content = file(fname,'rb').read()
input=cStringIO.StringIO(content)
input.seek(0)
data = list(csv.reader(input, quotechar=csv_data['del'] or '"', delimiter=csv_data['sep']))[int(csv_data['skip']):]
datas = []
for line in data:
if not line:
continue
datas.append(map(lambda x:x.decode(csv_data['combo']).encode('utf-8'), line))
if not datas:
common.warning(_('The file is empty !'), _('Importation !'), parent=parent)
return False
try:
res = rpc.session.rpc_exec_auth('/object', 'execute', model, 'import_data', f, datas, 'init', '', False, context)
except Exception, e:
common.warning(str(e), _('XML-RPC error !'), parent=parent)
return False
def on_change(self, callback):
match = re.match('^(.*?)\((.*)\)$', callback)
if not match:
raise Exception, 'ERROR: Wrong on_change trigger: %s' % callback
func_name = match.group(1)
arg_names = [n.strip() for n in match.group(2).split(',') if n.strip()]
args = [self.expr_eval(arg) for arg in arg_names]
ids = self.id and [self.id] or []
response = getattr(self.rpc, func_name)(ids, *args)
if response:
self.set(response.get('value', {}), modified=True)
if 'domain' in response:
for fieldname, value in response['domain'].items():
if fieldname not in self.mgroup.mfields:
continue
self.mgroup.mfields[fieldname].attrs['domain'] = value
warning=response.get('warning',{})
if warning:
common.warning(warning['message'], warning['title'])
self.signal('record-changed')
def execute(self, obj, method, *args):
if not self.is_logged():
raise common.warning(_('Not logged...'), _('Authorization Error!'))
try:
result = self.gateway.execute(obj, method, *args)
return self.__convert(result)
except socket.error, (e1, e2):
raise common.message(_('Connection refused!'))
def remapData(self, source, output, mappers=[], processor=None):
count = common.count(source)
if count == 0:
common.warning('remapping source is empty, empty output generated')
return
prog = common.progressor('remapping records', count)
inCur = arcpy.SearchCursor(source)
outCur = arcpy.InsertCursor(output)
# common.debug('remapping')
for inRow in inCur:
outRow = outCur.newRow()
for mapper in mappers:
outRow = mapper.remap(inRow, outRow, processor)
# common.debug(mapper, outRow)
if outRow is None:
break
if outRow is not None:
outCur.insertRow(outRow)
prog.move()
prog.end()
del inCur, inRow, outCur, outRow
def __plot_br_ex_to_br_int (self):
namespaces = self.info['namespaces']
for nms in namespaces.keys():
if not re.search('^qrouter-', nms):
continue
if not namespaces[nms].has_key('interfaces'):
warning('namespace %s does not have any interface' % nms)
continue
qg_intf = None
for intf in namespaces[nms]['interfaces'].keys():
if re.search('^qg-', intf):
qg_intf = intf
break;
for intf in namespaces[nms]['interfaces'].keys():
if re.search('^qr-', intf):
src_tag = 'br_ex:' + qg_intf
dst_tag = 'network_br_int:' + intf
self.__html_edge(src_tag, dst_tag, self.__get_color('edge'))
pass
def go(self):
while True:
button = self.win.run()
if button == gtk.RESPONSE_OK:
if not len(self.model2):
common.warning(_("You have not selected any fields to import"), parent=self.win)
continue
fields = []
fields2 = []
iter = self.model2.get_iter_root()
while iter:
fields.append(self.model2.get_value(iter, 1))
fields2.append(self.model2.get_value(iter, 0))
iter = self.model2.iter_next(iter)
csv = {
'fname': self.glade.get_widget('import_csv_file').get_filename(),
'sep': self.glade.get_widget('import_csv_sep').get_text(),
'del': self.glade.get_widget('import_csv_del').get_text(),
'skip': self.glade.get_widget('import_csv_skip').get_value(),
'combo': self.glade.get_widget('import_csv_combo').get_active_text() or 'UTF-8'
}
self.parent.present()
self.win.destroy()
if csv['fname']:
if self.invert:
inverted = []
for f in fields:
for key, value in self.fields_invert.items():
if key.encode('utf8') == f:
inverted.append(value)
return import_csv(csv, inverted, self.model, self.fields_invert, context=self.context, parent=self.win)
else:
return import_csv(csv, fields, self.model, self.fields, context=self.context, parent=self.win)
return False
else:
self.parent.present()
self.win.destroy()
return False
def _on_cell_renderer_edited( self, cell, path_string, new_text ):
model = self.treeview.get_model()
iter = model.get_iter_from_string(path_string)
(path,column) = self.treeview.get_cursor()
column_id = self.treeview.get_columns().index(column)
old_text = model.get_value( iter, column_id )
if column_id == 0:
old_text = old_text.lower()
new_text = new_text.lower()
if old_text <> new_text:
if column_id == 0:
if new_text in [ ext for ext, app, app_print in model ]:
common.warning(_('This extension is already defined'), _('Extension Manager'), parent=self.win)
return
else:
model.set(iter, column_id, new_text)
else:
model.set(iter, column_id, new_text)
