def test_double_env_var(self, monkeypatch):
expected_token = 'app_id|app_secret'
monkeypatch.setenv(te.TX_APP_ID, 'app_id')
monkeypatch.setenv(te.TX_APP_SECRET, 'app_secret')
access_token.access_token()
self.verify_token(expected_token)
def __init__(self):
'''register our criteria for being passed a message
as a list of lower case strings to match with an rest endpoint
(i.e. blockip matches /blockip)
set the priority if you have a preference for order of plugins
0 goes first, 100 is assumed/default if not sent
Plugins will register in Meteor with attributes:
name: (as below)
description: (as below)
priority: (as below)
file: "plugins.filename" where filename.py is the plugin code.
Plugin gets sent main rest options as:
self.restoptions
self.restoptions['configfile'] will be the .conf file
used by the restapi's index.py file.
'''
self.registration = ['blockip']
self.priority = 10
self.name = "ThreatExchange"
self.description = "Facebook ThreatExchange"
# set my own conf file
# relative path to the rest index.py file
self.configfile = './plugins/facebookThreatExchange.conf'
self.options = None
if os.path.exists(self.configfile):
sys.stdout.write('found conf file {0}\n'.format(self.configfile))
self.initConfiguration()
# set up the threat exchange secret
access_token(self.options.appid, self.options.appsecret)
def test_double_env_var(self, monkeypatch):
expected_token = "app_id|app_secret"
monkeypatch.setenv(te.TX_APP_ID, "app_id")
monkeypatch.setenv(te.TX_APP_SECRET, "app_secret")
access_token.access_token()
self.verify_token(expected_token)
def test_explicit_token_file(self):
expected_token = 'app_id|app_secret'
file_contents = 'app_id|app_secret'
with nested(
patch('pytx.access_token._find_token_file', return_value=None),
patch('pytx.access_token._read_token_file', return_value=file_contents)
):
access_token.access_token(token_file='/foobar/mocked/away')
self.verify_token(expected_token)
def test_explicit_token_file(self):
expected_token = 'app_id|app_secret'
file_contents = 'app_id|app_secret'
with nested(
patch('pytx.access_token._find_token_file', return_value=None),
patch('pytx.access_token._read_token_file',
return_value=file_contents)):
access_token.access_token(token_file='/foobar/mocked/away')
self.verify_token(expected_token)
def setup_access():
sc = get_config('ThreatExchange')
config = CRITsConfig.objects().first()
access_token.access_token(app_id=sc['app_id'], app_secret=sc['app_secret'])
headers = None
if len(sc['headers']) > 0:
hlist = sc['headers'].split(',')
headers = {}
for h in hlist:
tmp = h.split(':')
if len(tmp) == 2:
headers[tmp[0].strip()] = tmp[1].strip()
proxies = {'http': config.http_proxy, 'https': config.http_proxy}
connection(headers=headers, proxies=proxies, verify=sc['verify'])
return
def setup_access():
sc = get_config('ThreatExchange')
config = CRITsConfig.objects().first()
access_token.access_token(app_id=sc['app_id'],
app_secret=sc['app_secret'])
headers = None
if len(sc['headers']) > 0:
hlist = sc['headers'].split(',')
headers = {}
for h in hlist:
tmp = h.split(':')
if len(tmp) == 2:
headers[tmp[0].strip()] = tmp[1].strip()
proxies = {'http': config.http_proxy,
'https': config.http_proxy}
connection(headers=headers,
proxies=proxies,
verify=sc['verify'])
return
def validate_config(self):
super(ThreatExchangeConnector, self).validate_config()
self.check_required_options(["tx_app_id", "tx_secret_key", "carbonblack_server_token"])
self.bridge_options["listener_port"] = self.get_config_integer("listener_port", 6120)
self.bridge_options["feed_host"] = self.get_config_string("feed_host", "127.0.0.1")
self.bridge_options["listener_address"] = self.get_config_string("listener_address", "0.0.0.0")
self.bridge_auth["app_id"] = self.get_config_string("tx_app_id")
self.bridge_auth["secret_key"] = self.get_config_string("tx_secret_key")
access_token(self.bridge_auth["app_id"], self.bridge_auth["secret_key"])
ioc_types = self.get_config_string("tx_ioc_types", None)
if not ioc_types or len(ioc_types.strip()) == 0:
ioc_types = processing_engines.ALL_INDICATOR_TYPES
ioc_types = ioc_types.split(',')
self.bridge_options["ioc_types"] = []
for ioc_type in ioc_types:
if ioc_type not in processing_engines.INDICATOR_PROCESSORS:
self.logger.warning("%s is not a valid IOC type, ignoring" % ioc_type)
else:
self.bridge_options["ioc_types"].append(ioc_type)
self.bridge_options["historical_days"] = self.get_config_integer("tx_historical_days", 7)
# retrieve once a day by default
self.bridge_options["feed_retrieval_minutes"] = self.get_config_integer("feed_retrieval_minutes", 120)
self.bridge_options["minimum_severity"] = self.get_config_string("tx_minimum_severity", "WARNING")
self.bridge_options["minimum_confidence"] = int(self.get_config_string("tx_minimum_confidence", 50))
status_filter = self.get_config_string("tx_status_filter", None)
if type(status_filter) == str:
self.bridge_options["status_filter"] = status_filter.split(',')
else:
self.bridge_options["status_filter"] = None
return True
def main():
logger.debug('Connecting to Elasticsearch')
client = ElasticsearchClient(options.esservers)
logger.debug('Connecting to threat exchange')
access_token(options.appid, options.appsecret)
state = State(options.state_file_name)
current_timestamp = toUTC(datetime.now()).isoformat()
# We're setting a default for the past 2 days of data
# if there isnt a state file
since_date_obj = toUTC(datetime.now()) - timedelta(days=2)
since_date = since_date_obj.isoformat()
if 'lastrun' in state.data.keys():
since_date = state.data['lastrun']
# A master dict of all the different types of
# data we want to pull from threat exchange
params = {
'malware_hash': {
'threat_class': Malware,
'query_params': {},
},
'ip_address': {
'threat_class': ThreatDescriptor,
'query_params': {
'type_': 'IP_ADDRESS',
}
},
'domain': {
'threat_class': ThreatDescriptor,
'query_params': {
'type_': 'DOMAIN',
}
},
'uri': {
'threat_class': ThreatDescriptor,
'query_params': {
'type_': 'URI',
}
},
'debug_string': {
'threat_class': ThreatDescriptor,
'query_params': {
'type_': 'DEBUG_STRING',
}
},
'banner': {
'threat_class': ThreatDescriptor,
'query_params': {
'type_': 'BANNER',
}
},
'email_address': {
'threat_class': ThreatDescriptor,
'query_params': {
'type_': 'EMAIL_ADDRESS',
}
},
'file_created': {
'threat_class': ThreatDescriptor,
'query_params': {
'type_': 'FILE_CREATED',
}
},
}
docs = {}
for param_key, param in params.iteritems():
param['query_params']['since'] = str(since_date)
param['query_params']['until'] = str(current_timestamp)
param['query_params']['dict_generator'] = True
docs = pull_threat_exchange_data(param_key, param)
logger.debug('Saving {0} {1} to ES'.format(len(docs), param_key))
for doc in docs:
client.save_object(index='threat-exchange',
doc_type=param_key,
body=doc)
state.data['lastrun'] = current_timestamp
state.save()
def test_single_env_var(self, monkeypatch):
expected_token = 'app_id|app_secret'
monkeypatch.setenv(te.TX_ACCESS_TOKEN, expected_token)
access_token.access_token()
self.verify_token(expected_token)
def test_app_id_and_secret(self, mock_patch):
expected_token = 'app_id|app_secret'
access_token.access_token(app_id='app_id', app_secret='app_secret')
self.verify_token(expected_token)
def test_only_app_secret(self, mock_patch):
with pytest.raises(pytxAccessTokenError):
access_token.access_token(app_secret="app_secret")
def test_no_token(self, mock_patch):
with pytest.raises(pytxAccessTokenError):
access_token.access_token()
def test_app_id_and_secret(self):
expected_token = 'app_id|app_secret'
with patch('pytx.access_token._find_token_file', return_value=None):
access_token.access_token(app_id='app_id', app_secret='app_secret')
self.verify_token(expected_token)
def test_only_app_secret(self):
with nested(
pytest.raises(pytxAccessTokenError),
patch('pytx.access_token._find_token_file', return_value=None)
):
access_token.access_token(app_secret='app_secret')
def test_no_token(self):
with nested(
pytest.raises(pytxAccessTokenError),
patch('pytx.access_token._find_token_file', return_value=None)
):
access_token.access_token()
def test_no_token(self, mock_patch):
with pytest.raises(pytxAccessTokenError):
access_token.access_token()
def test_only_app_secret(self, mock_patch):
with pytest.raises(pytxAccessTokenError):
access_token.access_token(app_secret='app_secret')
def test_app_id_and_secret(self, mock_patch):
expected_token = "app_id|app_secret"
access_token.access_token(app_id="app_id", app_secret="app_secret")
self.verify_token(expected_token)
def test_explicit_token_file(self, mock_patch, mock_patch_two):
expected_token = 'app_id|app_secret'
access_token.access_token(token_file='/foobar/mocked/away')
self.verify_token(expected_token)
def test_only_app_secret(self):
with nested(
pytest.raises(pytxAccessTokenError),
patch('pytx.access_token._find_token_file',
return_value=None)):
access_token.access_token(app_secret='app_secret')
def test_explicit_token_file(self, mock_patch, mock_patch_two):
expected_token = "app_id|app_secret"
access_token.access_token(token_file="/foobar/mocked/away")
self.verify_token(expected_token)
def test_app_id_and_secret(self):
expected_token = 'app_id|app_secret'
with patch('pytx.access_token._find_token_file', return_value=None):
access_token.access_token(app_id='app_id', app_secret='app_secret')
self.verify_token(expected_token)
def test_single_env_var(self, monkeypatch):
expected_token = "app_id|app_secret"
monkeypatch.setenv(te.TX_ACCESS_TOKEN, expected_token)
access_token.access_token()
self.verify_token(expected_token)
def test_no_token(self):
with nested(
pytest.raises(pytxAccessTokenError),
patch('pytx.access_token._find_token_file',
return_value=None)):
access_token.access_token()