def load_client(context):
"""Get an instance of a loaded client."""
username = context.getTransformSetting('username')
api_key = context.getTransformSetting('aKey')
test_status = context.getTransformSetting('test_local')
if test_status and test_status == 'True':
server = context.getTransformSetting('server')
version = context.getTransformSetting('version')
return AttributeRequest(username, api_key, server, version)
else:
return AttributeRequest(username, api_key, headers=gen_debug(request))
def get_attribute(self, **kwargs):
client = AttributeRequest(self.username, self.apikey)
keys = ['query', 'type']
params = self._cleanup_params(keys, **kwargs)
if params.get('type') == 'tracker':
return client.get_host_attribute_trackers(**params)
else:
return client.get_host_attribute_components(**params)
def get_attribute(self, **kwargs):
client = AttributeRequest(self.username, self.apikey)
keys = ['query', 'type']
params = self._cleanup_params(keys, **kwargs)
if params.get('type') == 'tracker':
return client.get_host_attribute_trackers(**params)
else:
return client.get_host_attribute_components(**params)
def call_attribute(args):
"""Abstract call to attribute-based queries."""
client = AttributeRequest.from_config()
pruned = prune_args(query=args.query, type=args.type)
if args.type == 'tracker':
data = client.get_host_attribute_trackers(**pruned)
else:
data = client.get_host_attribute_components(**pruned)
return data
def __init__(self):
try:
self.clients = {
'ssl': SslRequest.from_config(),
'dns': DnsRequest.from_config(),
'enrichment': EnrichmentRequest.from_config(),
'whois': WhoisRequest.from_config(),
'attribute': AttributeRequest.from_config(),
}
except Exception:
self.clients = None
def call_attribute(args):
"""Abstract call to attribute-based queries."""
client = AttributeRequest.from_config()
pruned = prune_args(
query=args.query,
type=args.type
)
if args.type == 'tracker':
data = client.get_host_attribute_trackers(**pruned)
else:
data = client.get_host_attribute_components(**pruned)
return data
from passivetotal.libs.attributes import AttributeRequest
from passivetotal.libs.enrichment import EnrichmentRequest
def show_tagged(direction, enriched):
for host, data in enriched.get("results", {}).iteritems():
if len(data['tags']) == 0:
continue
print data['queryValue'], ','.join(data['tags'])
query = sys.argv[1]
direction = sys.argv[2]
result_key = {'parents': 'parent', 'children': 'child'}
if len(sys.argv) != 3:
print "Usage: python host_pair_sentinel.py <query> <parents|children>"
sys.exit(1)
if direction not in ['children', 'parents']:
print "[!] Direction must be 'children' or 'parents' to work"
sys.exit(1)
client = AttributeRequest.from_config()
matches = client.get_host_attribute_pairs(query=query, direction=direction)
hostnames = [x[result_key[direction]] for x in matches.get("results", list())]
client = EnrichmentRequest.from_config()
enriched = client.get_bulk_enrichment(query=hostnames)
show_tagged(direction, enriched)
class AttributeTestCase(unittest.TestCase):
"""Test case for attribute methods."""
formats = ['json', 'xml', 'csv', 'table']
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = AttributeRequest('--No-User--', '--No-Key--')
def teardown_class(self):
self.patcher.stop()
def test_trackers(self):
"""Test getting tracker codes."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_trackers(**payload)
assert ('results' in response)
def test_process_trackers(self):
"""Test processing tracker data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_trackers(**payload)
wrapped = AttributeResponse(response)
record = wrapped.get_records().pop(0)
assert (record.hostname) == 'passivetotal.org'
assert (record.lastSeen) == '2016-01-26 13:47:45'
assert (record.attributeType) == 'GoogleAnalyticsAccountNumber'
assert (record.firstSeen) == '2015-10-09 17:05:38'
assert (record.attributeValue) == 'UA-61048133'
def test_components(self):
"""Test getting component data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_components(**payload)
assert ('results' in response)
def test_process_components(self):
"""Test processing component data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_components(**payload)
wrapped = AttributeResponse(response)
record = wrapped.get_records().pop(0)
assert (record.hostname) == 'passivetotal.org'
assert (record.lastSeen) == '2016-01-07 21:52:30'
assert (record.category) == 'JavaScript Library'
assert (record.firstSeen) == '2015-12-26 11:17:43'
assert (record.label) == 'jQuery'
def test_trackers_search(self):
"""Test searching trakcer data."""
payload = {'query': 'UA-49901229', 'type': 'GoogleAnalyticsAccountNumber'}
response = self.client.search_trackers(**payload)
assert ('results' in response)
def test_process_trackers_search(self):
"""Test processing component data."""
payload = {'query': 'UA-49901229', 'type': 'GoogleAnalyticsAccountNumber'}
response = self.client.search_trackers(**payload)
wrapped = AttributeResponse(response)
record = wrapped.get_records().pop(0)
assert not (record.everBlacklisted)
assert (record.alexaRank) == 38
assert (record.hostname) == 'demo.paypal.com'
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = AttributeRequest('--No-User--', '--No-Key--')
1) Take in a domain or IP 2) Identify all tracking codes associated with the query 3) Search for other sites not matching the original query using any codes 4) Construct a table output with data for easy consumption """ __author__ = 'Brandon Dixon ([email protected])' __version__ = '1.0.0' __description__ = "Surface related entities based on tracking codes" __keywords__ = ['trackers', 'phishing', 'crimeware', 'analysis'] import sys from tabulate import tabulate from passivetotal.libs.attributes import AttributeRequest query = sys.argv[1] client = AttributeRequest.from_config() # client.set_debug(True) processed_values = list() def surface_values(item): """Identify items that could be interesting.""" if item.get('attributeValue') in processed_values: return {} children = client.search_trackers( query=item.get('attributeValue'), type=item.get('attributeType') ) interesting = dict()
class AttributeTestCase(unittest.TestCase):
"""Test case for attribute methods."""
formats = ['json', 'xml', 'csv', 'table']
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = AttributeRequest('--No-User--', '--No-Key--')
def teardown_class(self):
self.patcher.stop()
def test_trackers(self):
"""Test getting tracker codes."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_trackers(**payload)
assert ('results' in response)
def test_process_trackers(self):
"""Test processing tracker data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_trackers(**payload)
wrapped = Response(response)
record = wrapped.results.pop(0)
record = Response(record)
assert (record.hostname) == 'passivetotal.org'
assert (record.lastSeen) == '2016-01-26 13:47:45'
assert (record.attributeType) == 'GoogleAnalyticsAccountNumber'
assert (record.firstSeen) == '2015-10-09 17:05:38'
assert (record.attributeValue) == 'UA-61048133'
def test_components(self):
"""Test getting component data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_components(**payload)
assert ('results' in response)
def test_process_components(self):
"""Test processing component data."""
payload = {'query': 'passivetotal.org'}
response = self.client.get_host_attribute_components(**payload)
wrapped = Response(response)
record = wrapped.results.pop(0)
record = Response(record)
assert (record.hostname) == 'passivetotal.org'
assert (record.lastSeen) == '2016-01-07 21:52:30'
assert (record.category) == 'JavaScript Library'
assert (record.firstSeen) == '2015-12-26 11:17:43'
assert (record.label) == 'jQuery'
def test_trackers_search(self):
"""Test searching trakcer data."""
payload = {'query': 'UA-49901229', 'type': 'GoogleAnalyticsAccountNumber'}
response = self.client.search_trackers(**payload)
assert ('results' in response)
def test_process_trackers_search(self):
"""Test processing component data."""
payload = {'query': 'UA-49901229', 'type': 'GoogleAnalyticsAccountNumber'}
response = self.client.search_trackers(**payload)
wrapped = Response(response)
record = wrapped.results.pop(0)
record = Response(record)
assert not (record.everBlacklisted)
assert (record.alexaRank) == 38
assert (record.hostname) == 'demo.paypal.com'
def setup_class(self):
self.patcher = patch('passivetotal.api.Client._get', fake_request)
self.patcher.start()
self.client = AttributeRequest('--No-User--', '--No-Key--')
try:
logger.info("Starting command processing")
input_events, dummyresults, settings = splunk.Intersplunk.getOrganizedResults(
)
keywords, options = splunk.Intersplunk.getKeywordsAndOptions()
query_value = options.get("query", "")
logger.info("Query target: %s" % query_value)
logger.debug("Raw options: %s" % str(options))
configuration = get_config("passivetotal", "api-setup")
username = configuration.get('username', None)
api_key = configuration.get('apikey', None)
output_events = []
tmp = AttributeRequest(
username, api_key,
headers=build_headers()).get_host_attribute_trackers(query=query_value)
if 'error' in tmp:
raise Exception(
"Whoa there, looks like you reached your quota for today! Please come back tomorrow to resume your investigation or contact support for details on enterprise plans."
)
for result in tmp.get("results", []):
output_events.append(result)
splunk.Intersplunk.outputResults(output_events)
except Exception, e:
stack = traceback.format_exc()
splunk.Intersplunk.generateErrorResults(str(e))
logger.error(str(e) + ". Traceback: " + str(stack))
logger = setup_logging()
try:
logger.info("Starting command processing")
input_events, dummyresults, settings = splunk.Intersplunk.getOrganizedResults()
keywords, options = splunk.Intersplunk.getKeywordsAndOptions()
query_value = options.get("query", "")
logger.info("Query target: %s" % query_value)
logger.debug("Raw options: %s" % str(options))
configuration = get_config("passivetotal", "api-setup")
username = configuration.get('username', None)
api_key = configuration.get('apikey', None)
output_events = []
tmp = AttributeRequest(username, api_key, headers=build_headers()).get_host_attribute_trackers(
query=query_value)
if 'error' in tmp:
raise Exception("Whoa there, looks like you reached your quota for today! Please come back tomorrow to resume your investigation or contact support for details on enterprise plans.")
for result in tmp.get("results", []):
output_events.append(result)
splunk.Intersplunk.outputResults(output_events)
except Exception, e:
stack = traceback.format_exc()
splunk.Intersplunk.generateErrorResults(str(e))
logger.error(str(e) + ". Traceback: " + str(stack))
except:
ip = "domain did not resolve"
# Get geo-coordinates of the IP address
coordinates = None
if not row['lat'] and not row['lng']:
try:
coordinates = DbIpCity.get(ip, api_key='free')
except:
pass
# Gather web trackers (Google Analytics Tracking ID, FacebookId, etc.)
username = ""
api_key = ""
trackers = AttributeRequest(
username=username,
api_key=api_key).get_host_attribute_trackers(
query=row['domain'])
# Create dictionary that will be written at the end of processing the entire file.
# We do this because the RiskIQ Tracker API call can return a varying amount of fields
# so we don't know what headers to write in the CSV file until the processing is complete.
csv_row = {}
csv_row["domain"] = row['domain']
csv_row["ip"] = ip
csv_row["locationVerified"] = row['locationVerified']
if coordinates:
csv_row["latitude"] = str(coordinates.latitude)
csv_row["longitude"] = str(coordinates.longitude)
else:
csv_row["latitude"] = row['lat']
