def run_client(parser, run_func):
args = parser.parse_args()
level = logging.DEBUG if args.verbose else logging.INFO
configure_color_logging(level=level)
poll_headers = (prepare_headers(args.poll_headers)
if args.poll_headers else None)
inbox_headers = (prepare_headers(args.inbox_headers)
if args.inbox_headers else None)
poll_client = create_client(version=args.poll_taxii_version,
headers=poll_headers)
inbox_client = create_client(version=args.inbox_taxii_version,
headers=inbox_headers)
poll_client.set_auth(
username=args.poll_username,
password=args.poll_password,
jwt_auth_url=args.poll_jwt_auth_url
)
inbox_client.set_auth(
username=args.inbox_username,
password=args.inbox_password,
jwt_auth_url=args.inbox_jwt_auth_url
)
try:
run_func(poll_client, inbox_client, args)
except Exception as e:
log.error(e, exc_info=args.verbose)
def run_client(parser, run_func):
args = parser.parse_args()
level = logging.DEBUG if args.verbose else logging.INFO
configure_color_logging(level=level)
poll_headers = prepare_headers(
args.poll_headers) if args.poll_headers else None
inbox_headers = prepare_headers(
args.inbox_headers) if args.inbox_headers else None
poll_client = create_client(version=args.poll_taxii_version,
headers=poll_headers)
inbox_client = create_client(version=args.inbox_taxii_version,
headers=inbox_headers)
if args.poll_verify == "yes":
poll_verify_ssl = True
elif args.poll_verify == "no":
poll_verify_ssl = False
elif args.poll_verify:
poll_verify_ssl = args.poll_verify
else:
poll_verify_ssl = True
poll_client.set_auth(
cert_file=args.poll_cert,
key_file=args.poll_key,
username=args.poll_username,
password=args.poll_password,
key_password=args.poll_key_password,
jwt_auth_url=args.poll_jwt_auth_url,
verify_ssl=poll_verify_ssl,
)
if args.inbox_verify == "yes":
inbox_verify_ssl = True
elif args.inbox_verify == "no":
inbox_verify_ssl = False
elif args.inbox_verify:
inbox_verify_ssl = args.inbox_verify
else:
inbox_verify_ssl = True
inbox_client.set_auth(
cert_file=args.inbox_cert,
key_file=args.inbox_key,
username=args.inbox_username,
password=args.inbox_password,
key_password=args.inbox_key_password,
jwt_auth_url=args.inbox_jwt_auth_url,
verify_ssl=inbox_verify_ssl,
)
try:
run_func(poll_client, inbox_client, args)
except Exception as e:
log.error(e, exc_info=args.verbose)
def process(self):
end = datetime.datetime.now(datetime.timezone.utc)
start = end - datetime.timedelta(seconds=self.time_delta)
self.logger.debug('Fetching data back to %s.',
start.isoformat(timespec='seconds'))
self.logger.debug('Authenticating.')
client = cabby.create_client(self.endpoint,
discovery_path="/taxiiservice/discovery",
use_https=True)
client.set_auth(username=self.user, password=self.passwd)
self.logger.debug(
'Authentication succeeded. Polling data from ESET TAXII endpoint.')
for item in client.poll(self.collection,
begin_date=start,
end_date=end):
if not item.content:
continue # skip empty items
report = self.new_report()
report.add("feed.url",
"https://%s/taxiiservice/discovery" % self.endpoint)
report.add('extra.eset_feed', self.collection)
report.add('raw', item.content)
self.send_message(report)
def test_url_parse():
url = 'https://eiq-test.com:1337/path/to/discovery'
client = cabby.create_client(discovery_url=url)
assert client.host == 'eiq-test.com'
assert client.port == 1337
assert client.use_https is True
assert client.discovery_path == '/path/to/discovery'
def get_all_collections(self, is_raise_error=False):
"""Gets a list of all collections listed in the discovery service instance.
Args:
is_raise_error(bool): Whether to raise an error when one occurs.
Returns:
list. A list of all collection names in discovery service.
"""
if self.discovery_service:
taxii_client = cabby.create_client(discovery_path=self.discovery_service)
if self.username:
taxii_client.set_auth(username=str(self.username), password=self.password, verify_ssl=self.verify_cert)
elif self.api_key:
taxii_client.set_auth(username=str(self.api_key), verify_ssl=self.verify_cert)
else:
taxii_client.set_auth(verify_ssl=self.verify_cert)
try:
all_collections = taxii_client.get_collections()
except Exception as e:
if is_raise_error:
raise ConnectionError()
return_error(f'{INTEGRATION_NAME} - An error occurred when trying to fetch available collections.\n{e}')
return [collection.name for collection in all_collections]
return []
def create_taxii_client(self):
"""Connects to a TAXII server using cabby and configuration entries."""
conf = self.config
if not conf.start_date:
logging.error(
f"A start_date is required for site {conf.site}. Exiting.")
return
if not conf.ssl_verify:
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
try:
client = create_client(conf.site,
use_https=conf.use_https,
discovery_path=conf.discovery_path)
client.set_auth(username=conf.username,
password=conf.password,
verify_ssl=conf.ssl_verify,
ca_cert=conf.ca_cert,
cert_file=conf.cert_file,
key_file=conf.key_file)
proxy_dict = dict()
if conf.http_proxy_url:
proxy_dict['http'] = conf.http_proxy_url
if conf.https_proxy_url:
proxy_dict['https'] = conf.https_proxy_url
if proxy_dict:
client.set_proxies(proxy_dict)
self.client = client
except handled_exceptions as e:
logging.error(f"Error creating client: {e}")
def create_taxii_client(self):
conf = self.config
try:
client = create_client(conf.site,
use_https=conf.use_https,
discovery_path=conf.discovery_path)
client.set_auth(username=conf.username,
password=conf.password,
verify_ssl=conf.ssl_verify,
ca_cert=conf.ca_cert,
cert_file=conf.cert_file,
key_file=conf.key_file)
proxy_dict = dict()
if conf.http_proxy_url:
proxy_dict['http'] = conf.http_proxy_url
if conf.https_proxy_url:
proxy_dict['https'] = conf.https_proxy_url
if proxy_dict:
client.set_proxies(proxy_dict)
self.client = client
except handled_exceptions as e:
log.error(f"Error creating client: {e}")
def __init__(self,
api_key: str,
collection: str,
insecure: bool = False,
proxy: bool = False,
all_collections: bool = False,
tags: list = [],
tlp_color: Optional[str] = None):
taxii_client = cabby.create_client(
discovery_path="https://otx.alienvault.com/taxii/discovery")
taxii_client.set_auth(username=str(api_key),
password="******",
verify_ssl=not insecure)
if proxy:
taxii_client.set_proxies(handle_proxy())
self.taxii_client = taxii_client
self.tags = tags
self.tlp_color = tlp_color
self.all_collections = all_collections
if all_collections:
self.collections = self.get_all_collections()
else:
if collection is None or collection == '':
return_error(
f"No collection set. Here is a list of all accessible collections: "
f"{self.get_all_collections()}")
self.collections = collection.split(',')
def run_taxii():
client = create_client('otx.alienvault.com',
use_https=True,
discovery_path='/taxii/discovery')
collection_services = []
for service in client.discover_services():
print('Service type={s.type}, available={s.available}, '
'address={s.address}'.format(s=service))
if service.type == 'COLLECTION_MANAGEMENT' and service.available:
collection_services.append(service.address)
if not collection_services:
print('No available collection services found')
return
collections = client.get_collections(uri=collection_services[0])
print('Found %s collections' % (len(collections)))
if not collections:
return
collection = collections[0]
print('collection name {c.name}, type {c.type}, '
'available {c.available}, volume {c.volume}'.format(c=collection))
print_collection_content_bindings(collection)
print_collection_subscription_methods(collection)
def _taxii_inbox(content, config):
client = create_client(config['host'], use_https=config[
'ssl'], discovery_path=config['discovery_path'])
content = content
binding = config['binding']
client.set_auth(username=config['username'], password=config['password'])
client.push(content, binding, uri=config['inbox_path'])
def _taxii_inbox(content, config):
client = create_client(config['host'],
use_https=config['ssl'],
discovery_path=config['discovery_path'])
content = content
binding = config['binding']
client.set_auth(username=config['username'], password=config['password'])
client.push(content, binding, uri=config['inbox_path'])
def test_set_auth_ssl(httpsserver):
host, port = httpsserver.server_address
client = cabby.create_client(host=host,
port=port,
use_https=True,
discovery_path=fixtures11.DISCOVERY_PATH)
auth_params = dict(
username="******",
password="******",
ca_cert="tests/ssl_test_files/root_ca.pem",
cert_file="tests/ssl_test_files/client.pem",
key_file="tests/ssl_test_files/client.key",
key_password="******",
jwt_auth_url="/auth",
verify_ssl=True,
)
# Test client certificate with key_password.
# This doesn't use the requests library, instead
# it uses urllib2 or 3 through the 'request_with_key_password' function:
client.set_auth(**auth_params)
services = client.discover_services()
assert len(services) == 4
# Test we get the correct error when forgetting a key_password:
missing_key_password = dict(auth_params, key_password=None)
client.set_auth(**missing_key_password)
error_msg = "Key file is encrypted but key password was not provided"
with pytest.raises(ValueError, match=error_msg):
client.discover_services()
# Test that server certificate authentication can fail:
invalid_server_ca_cert = dict(auth_params, ca_cert=None)
client.set_auth(**invalid_server_ca_cert)
error_msg = "certificate verify failed"
with pytest.raises(urllib.error.URLError, match=error_msg):
client.discover_services()
# Test that verify_ssl=False ignores the invalid certificate:
invalid_server_ca_cert_no_verify = dict(auth_params,
ca_cert=None,
verify_ssl=False)
client.set_auth(**invalid_server_ca_cert_no_verify)
services = client.discover_services()
assert len(services) == 4
# Test client certificate with key_password.
# This uses the requests library as usual:
client_key_without_passphrase = dict(
auth_params,
key_file="tests/ssl_test_files/client_no_pass.key",
cert_file="tests/ssl_test_files/client_no_pass.pem",
)
client.set_auth(**client_key_without_passphrase)
services = client.discover_services()
assert len(services) == 4
def __init__(self, url, use_https, discovery_path):
self.feed_list = []
self.url = url
self.use_https = use_https
self.discovery_path = discovery_path
self.client = create_client(self.url, self.use_https,
self.discovery_path)
""" (host=None, port=None, discovery_path=None, use_https=False, version='1.1', headers=None)
def set_TAXI_client(self):
# Create a Client
client = create_client(
self.hostURL, use_https=self.useHTTPS, discovery_path=self.discoveryPath
)
client.set_auth(
username=self.usernameKey, password=self.userPassword, verify_ssl=False
)
return client
def test_set_auth_ssl(httpsserver):
host, port = httpsserver.server_address
client = cabby.create_client(
host=host,
port=port,
use_https=True,
discovery_path=fixtures11.DISCOVERY_PATH)
auth_params = dict(
username='******',
password='******',
ca_cert='tests/ssl_test_files/root_ca.pem',
cert_file='tests/ssl_test_files/client.pem',
key_file='tests/ssl_test_files/client.key',
key_password='******',
jwt_auth_url='/auth',
verify_ssl=True)
# Test client certificate with key_password.
# This doesn't use the requests library, instead
# it uses urllib2 or 3 through the 'request_with_key_password' function:
client.set_auth(**auth_params)
services = client.discover_services()
assert len(services) == 4
# Test we get the correct error when forgetting a key_password:
missing_key_password = dict(auth_params, key_password=None)
client.set_auth(**missing_key_password)
error_msg = "Key file is encrypted but key password was not provided"
with pytest.raises(ValueError, match=error_msg):
client.discover_services()
# Test that server certificate authentication can fail:
invalid_server_ca_cert = dict(auth_params, ca_cert=None)
client.set_auth(**invalid_server_ca_cert)
error_msg = "certificate verify failed"
with pytest.raises(urllib.error.URLError, match=error_msg):
client.discover_services()
# Test that verify_ssl=False ignores the invalid certificate:
invalid_server_ca_cert_no_verify = dict(
auth_params, ca_cert=None, verify_ssl=False)
client.set_auth(**invalid_server_ca_cert_no_verify)
services = client.discover_services()
assert len(services) == 4
# Test client certificate with key_password.
# This uses the requests library as usual:
client_key_without_passphrase = dict(
auth_params,
key_file='tests/ssl_test_files/client_no_pass.key',
cert_file='tests/ssl_test_files/client_no_pass.pem')
client.set_auth(**client_key_without_passphrase)
services = client.discover_services()
assert len(services) == 4
def poll_repository(repository: dict) -> list:
logging.debug("Connecting to %s", repository['name'])
client = cabby.create_client(**repository['client'])
collections = (c for c in client.get_collections()
if c.name not in repository.get('exclusions', ()))
for collection in collections:
yield from poll_collection(client, collection.name)
logging.info("Repository %s exhausted", repository['name'])
def __init__(self,
url=None,
discovery_path=None,
https=True,
username=None,
password=None,
cert=None,
key=None):
self.client = create_client(url,
use_https=https,
discovery_path=discovery_path)
self.client.set_auth(username=username,
password=password,
cert_file=cert,
key_file=key)
def perform(self):
"""
:param self:
:param enumerate_collections_only:
:return:
"""
for site in self.sites:
client = create_client(site.get('site'),
use_https=site.get('use_https'),
discovery_path=site.get('discovery_path'))
if not site.get('collection_management_path', ''):
collections = client.get_collections()
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('collection_management_path')
logger.info('Collection Management Path: {}'.format(uri))
collections = client.get_collections(uri=uri)
for collection in collections:
logger.info('Collection Name: {}, Collection Type: {}'.format(collection.name, collection.type))
if len(collections) == 0:
logger.info('Unable to find any collections. Exiting...')
sys.exit(0)
desired_collections = site.get('collections').lower().split(',')
want_all = False
if '*' in desired_collections:
want_all = True
for collection in collections:
if collection.type != 'DATA_FEED':
continue
if want_all or collection.name.lower() in desired_collections:
self._import_collection(client, site, collection)
def _build_iterator(self, now):
# create cabby client
LOG.info(
'{} - Creating a cabby client with host={}, discovery={}, port={}, https={} and version={}'
.format(self.name, self.host, self.discovery_service, self.port,
self.use_https, self.version))
client = create_client(host=self.host,
discovery_path=self.discovery_service,
port=self.port,
use_https=self.use_https,
version=self.version)
# basic authentication
client.set_auth(username=self.username,
password=self.password,
cert_file=self.cert_file,
key_file=self.key_file)
if self.poll_service is not None:
discovered_poll_service = self.poll_service
else:
discovered_poll_service = self._discover_poll_service(client)
LOG.debug('{} - poll service: {!r}'.format(self.name,
discovered_poll_service))
last_run = self.last_taxii_run
if last_run is None:
last_run = now - (self.initial_interval * 1000)
begin = datetime.utcfromtimestamp(last_run / 1000)
begin = begin.replace(microsecond=0, tzinfo=pytz.UTC)
end = datetime.utcfromtimestamp(now / 1000)
end = end.replace(tzinfo=pytz.UTC)
if self.lower_timestamp_precision:
end = end.replace(second=0, microsecond=0)
begin = begin.replace(second=0, microsecond=0)
return self._incremental_poll_collection(client,
discovered_poll_service,
begin=begin,
end=end)
def create_client_10(**kwargs):
client = create_client(HOST, version="1.0", **kwargs)
return client
def poll_taxii():
global f_hashes, f_manifest, f_events
results_dict = {}
client = create_client(
CYBERSAIYAN_FEED_URL,
use_https=TAXII_USE_TLS,
discovery_path=TAXII_DISCOVERY_PATH
)
blocks = client.poll(collection_name=CYBERSAIYAN_COLLECTION_NAME)
for block in blocks:
content = block.content
if content:
if type(content) == str:
continue
elif type(content) == bytes:
content = content.decode('utf-8')
pkg = STIXPackage.from_xml(StringIO(content))
title = pkg.stix_header.title
information_source = pkg.stix_header.information_source.identity.name
cs_event = (title, information_source)
cs_event_hash = hash(cs_event)
db_cursor.execute("SELECT uuid FROM hashes WHERE hash = '%s'" % cs_event_hash)
element = db_cursor.fetchone()
if element:
e_uuid = element[0]
else:
e_uuid = str(uuid.uuid4())
db_cursor.execute("INSERT INTO hashes VALUES (?,?)", (cs_event_hash,e_uuid,))
if cs_event_hash not in results_dict:
results_dict[cs_event_hash] = MISPEvent()
m_ev = results_dict[cs_event_hash]
m_ev.info = str(pkg.stix_header.description)
m_ev.analysis = 0
m_ev.uuid = e_uuid
#m_ev.org = "CyberSaiyan"
csorg = MISPOrganisation()
csorg.name = "CyberSaiyan"
csorg.uuid = "8aaa81ed-72ef-4fb1-8e96-fa1bc200faeb"
m_ev.orgc = csorg
marking = pkg.stix_header.handling.marking
tlp = 0
found_tlp = False
for m in marking:
for struct in m.marking_structures:
if struct._XSI_TYPE == "tlpMarking:TLPMarkingStructureType":
found_tlp = True
tlp = max(TLP[struct.color.lower()], tlp)
if tlp == 0 and not found_tlp:
tlp = TLP["amber"]
m_ev.add_tag("tlp:"+TLP[tlp])
m_ev.add_tag("CyberSaiyan")
indicators = pkg.indicators
last_ts = utc.localize(datetime.datetime(1970,1,1))
for indicator in indicators:
cur_ts = indicator.timestamp
if cur_ts > last_ts:
last_ts = cur_ts
obj = indicator.observable.object_
obj_d = obj.properties.to_dict()
attr_type = obj_d["xsi:type"]
if attr_type == "AddressObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "ip-dst"
attr.value = obj_d["address_value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "DomainNameObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "domain"
attr.value = obj_d["value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "URIObjectType":
attr = MISPAttribute()
attr.category = "Network activity"
attr.type = "url"
attr.value = obj_d["value"]
attr.disable_correlation = False
attr.to_ids = True
elif attr_type == "FileObjectType":
hash_type = obj_d["hashes"][0]["type"]["value"].lower()
hash_value = obj_d["hashes"][0]["simple_hash_value"]
attr = MISPAttribute()
attr.category = "Payload delivery"
assert hash_type in ('md5', "sha1", "sha224",
"sha256", "sha384", "sha512", "ssdeep")
attr.type = hash_type
attr.value = hash_value
attr.disable_correlation = False
attr.to_ids = True
m_ev.date = last_ts.strftime("%Y-%m-%d")
m_ev.attributes.append(attr)
db_conn.commit()
c_hashes, c_manifest, c_events = list(), dict(), dict()
for event in results_dict.values():
e_feed = event.to_feed(with_meta=True).get("Event")
c_hashes += [[h, event.uuid] for h in e_feed.pop("_hashes")]
c_manifest.update(e_feed.pop('_manifest'))
c_events[event.uuid] = e_feed
f_hashes, f_manifest, f_events = c_hashes, c_manifest, c_events
log.debug("Opening config file %s", configFile)
with open(configFile, "r") as f:
config = yaml.load(f.read())
log.debug("Config read %s", config)
# Read in the local server configuration
localConfig = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
log.debug("Reading local server config")
with open(localConfig, "r") as f:
localConfig = yaml.load(f.read())
# Attempt to make contact with the local server
log.info("Connecting to local server...")
localClient = create_client(host=localConfig["host"],
port=localConfig["port"],
discovery_path=localConfig["discovery_path"],
use_https=localConfig["use_https"],
version=localConfig["taxii_version"],
headers=localConfig["headers"])
localClient.username = localConfig["auth"]["username"]
localClient.password = localConfig["auth"]["password"]
localInbox = "{}://{}:{}{}".format(
"https" if localConfig["use_https"] else "http", localConfig["host"],
localConfig["port"], localConfig["inbox_path"])
# Check that we're all good and authenticated
try:
list(localClient.discover_services())
except Exception as ex:
log.fatal("Could not connect to local server")
log.fatal(ex)
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from cabby import create_client
try:
# create a connection
client = create_client(host='{{ opentaxii_host }}',
port='{{ opentaxii_port }}',
discovery_path='/services/discovery')
# iterate through each defined collection
collections = client.get_collections(
uri='{{ opentaxii_domain }}/services/collection')
for collection in collections:
# how many records in each collection?
count = client.get_content_count(
collection_name=collection.name,
uri='{{ opentaxii_domain }}/services/poll')
print "%-50s %-10d" % (collection.name, count.count)
except:
print "Services not defined"
def perform(self):
"""
:param self:
:param enumerate_collections_only:
:return:
"""
for site in self.sites:
client = create_client(site.get('site'),
use_https=site.get('use_https'),
discovery_path=site.get('discovery_path'))
#
# Set verify_ssl and ca_cert inside the client
#
client.set_auth(verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'))
#
# Proxy Settings
#
proxy_dict = dict()
if self.http_proxy_url:
logger.info("Found HTTP Proxy: {}".format(self.http_proxy_url))
proxy_dict['http'] = self.http_proxy_url
if self.https_proxy_url:
logger.info("Found HTTPS Proxy: {}".format(
self.https_proxy_url))
proxy_dict['https'] = self.https_proxy_url
if proxy_dict:
client.set_proxies(proxy_dict)
if site.get('username') or site.get('cert_file'):
#
# If a username is supplied use basic authentication
#
logger.info("Found Username in config, using basic auth...")
client.set_auth(username=site.get('username'),
password=site.get('password'),
verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'),
cert_file=site.get('cert_file'),
key_file=site.get('key_file'))
if not site.get('collection_management_path', ''):
collections = client.get_collections()
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('collection_management_path')
logger.info('Collection Management Path: {}'.format(uri))
collections = client.get_collections(uri=uri)
for collection in collections:
logger.info('Collection Name: {}, Collection Type: {}'.format(
collection.name, collection.type))
if len(collections) == 0:
logger.info('Unable to find any collections. Exiting...')
sys.exit(0)
desired_collections = [
x.strip() for x in site.get('collections').lower().split(',')
]
want_all = False
if '*' in desired_collections:
want_all = True
for collection in collections:
if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET':
continue
if collection.type == 'DATA_SET':
data_set = True
else:
data_set = False
if want_all or collection.name.lower() in desired_collections:
self._import_collection(client, site, collection, data_set)
log.info("Subscribing to tcp://{}:{}".format(
config["zmq"]["host"],
config["zmq"]["port"]
))
# Connect to the socket
socket.connect("tcp://{}:{}".format(
config["zmq"]["host"],
config["zmq"]["port"]
))
# Set the option to subscribe
socket.setsockopt_string(zmq.SUBSCRIBE, '')
# Connct to TAXII as well
cli = create_client(discovery_path="{}://{}/services/discovery".format(config.get("protocol", "http"), config["domain"]))
cli.set_auth(username = config["taxii"]["auth"]["username"],
password = config["taxii"]["auth"]["password"]
)
if not config.get("verify_ssl", True):
cli.verify_ssl = False
while True:
# Wait for something to come in on the ZMQ socket
message = socket.recv().decode("utf-8")
log.info("Recieved a message!")
topic = message.split(' ', 1)[0]
if topic != 'misp_json':
log.info("Ignoring " + topic + "...")
continue
def make_client(version, **kwargs):
client = create_client(get_fix(version).HOST,
version=("1.1" if version == 11 else "1.0"),
**kwargs)
return client
def __init__(self):
self.cabby = cabby.create_client( config.TAXII_SERVER, config.TAXII_PORT, discovery_path='/services/discovery-internal', version='1.1')
# self.cabby.set_proxy(self.cabby.NO_PROXY)
self.cabby.set_auth(username=config.TAXII_USER, password=config.TAXII_PASS, jwt_auth_url='/management/auth')
pass
def create_client_11(**kwargs):
client = create_client(HOST, version="1.1", **kwargs)
return client
###############################################################################
from cabby import create_client
from stix.core import STIXPackage
import smtplib
from email.header import Header
from email.mime.text import MIMEText
from io import StringIO
from datetime import datetime, timedelta
import pytz
from pprint import pprint
# conexión al server taxii
client = create_client(TAXII_SERVER_HOST,
use_https=TAXII_SERVER_SSL,
discovery_path=TAXII_DISCOVERY_PATH)
# fecha para poll (bloques registrados con fecha >= X)
fechahora_desde = datetime.now(pytz.utc) - timedelta(hours=4)
#print("fecha/hora desde: " + str(fechahora_desde))
# resumen de indicadores para mail...
mail_indicadores = 'INDICADORES DESDE ' + str(
fechahora_desde) + ' [' + TAXII_SERVER_HOST + ']\n\n'
# poll de bloques...
content_blocks = client.poll(collection_name=TAXII_COLLECTION_NAME,
begin_date=fechahora_desde)
for block in content_blocks:
#print(type(block.content))
from StixExport import StixExport
from OTXv2 import OTXv2
from cabby import create_client
import ConfigParser
import datetime
import sys
binding = 'urn:stix.mitre.org:xml:1.1.1'
config = ConfigParser.ConfigParser()
config.read('config.cfg')
otx = OTXv2(config.get('otx', 'key'))
client = create_client(config.get('taxii', 'server_ip'),
discovery_path=config.get('taxii', 'discovery_path'))
client.set_auth(username=config.get('taxii', 'username'),
password=config.get('taxii', 'password'))
def saveTimestamp(timestamp=None):
mtimestamp = timestamp
if not timestamp:
mtimestamp = datetime.datetime.now().isoformat()
fname = "timestamp"
f = open(fname, "w")
f.write(mtimestamp)
f.close()
def readTimestamp():
def create_haila_client():
print("[+] Create client")
client = create_client('hailataxii.com',
use_https=False,
discovery_path='/taxii-discovery-service')
return client
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from cabby import create_client
try:
# create a connection
client = create_client('{{ opentaxii_domain }}', discovery_path='/services/discovery')
# iterate through each defined collection
collections = client.get_collections(uri='/services/collection')
for collection in collections:
# how many records in each collection?
count = client.get_content_count(collection_name=collection.name)
print "%-50s %-10d" % (collection.name, count.count)
except:
print "Services not defined"
from StixExport import StixExport
from OTXv2 import OTXv2
from cabby import create_client
import ConfigParser
import datetime
import sys
binding = 'urn:stix.mitre.org:xml:1.1.1'
config = ConfigParser.ConfigParser()
config.read('config.cfg')
otx = OTXv2(config.get('otx', 'key'))
client = create_client(config.get('taxii', 'server_ip'), discovery_path=config.get('taxii', 'discovery_path'))
client.set_auth(username=config.get('taxii', 'username'), password=config.get('taxii', 'password'))
def saveTimestamp(timestamp=None):
mtimestamp = timestamp
if not timestamp:
mtimestamp = datetime.datetime.now().isoformat()
fname = "timestamp"
f = open(fname, "w")
f.write(mtimestamp)
f.close()
def readTimestamp():
fname = "timestamp"
f = open(fname, "r")
def main():
args = process_arguments()
client = create_client('limo.anomali.com',\
use_https=True,\
discovery_path='/api/v1/taxii/taxii-discovery-service/')
client.set_auth(username=args.username, password=args.password)
collections = client.get_collections(
uri=''
'https://limo.anomali.com/api/v1/taxii/collection_management/')
for c in collections:
print "[*]-Fetching collection name: {} ".format(c.name)
content_blocks = client.poll(collection_name=c.name,\
begin_date=begin_date(args.days),\
end_date=end_date())
for block in content_blocks:
with open(c.name + '.xml', 'wb') as file_handle:
file_handle.write(block.content)
es = Elasticsearch([{'host': args.es, 'port': args.port}])
try:
if es.ping():
print "\n[*]-Connection with ES was successful."
except ValueError:
print "\n[*]-Connection with ES was failed."
sys.exit(-1)
try:
if es.indices.exists(args.index):
re = es.indices.delete(index=args.index, ignore=[400, 404])
print "[*]-Deleting {} index... Status: {}".format(
args.index, re.get('acknowledged'))
except:
print "[*]-Something was wrong with the deletion of the index {}.".format(
args.index)
sys.exit(-1)
document = {}
print "[*]-Creating {} index...".format(args.index)
res = es.indices.create(index=args.index, body=document)
if res:
print "[*]-Index {} created successully... Status: {}".format(
args.index, res.get('acknowledged'))
else:
print "[*]-Something went wrong with the creation of the index {}.".format(
args.index)
sys.exit(-1)
xml_files = glob.glob("*.xml")
for xml in xml_files:
pkg = STIXPackage.from_xml(xml)
pkg_dict = pkg.to_dict()
for v in pkg_dict.get('indicators'):
description = v.get('description').split(';')
Produced_Time = v.get('producer').get('time').get('produced_time')
description = v.get('description').split(';')
State = description[3].split(':')
Org = description[4].split(':')
Source = description[5].split(':')
Indtitle = v.get('title').strip(' ').split(':')
IndicatorType = ','.join(
ind.get('value') for ind in v.get('indicator_types'))
Severity = ''.join(v.get('observable').get('keywords'))
IPAddress = v.get('observable').get('object').get(
'properties').get('address_value')
Value = v.get('observable').get('object').get('properties').get(
'value')
Confidence = v.get('confidence').get('value').get('value')
document = {
'Value': Value,
'IPAddress': IPAddress,
'Category': Indtitle[0],
'Indicator_Type': IndicatorType,
'Severity': Severity,
'Confidence': Confidence,
'State': State[1],
'Organisation': Org[1],
'Source': Source[1],
'Produced_Time': Produced_Time
}
es.index(index=args.index,
doc_type='anomali_threat_feed_free',
body=document)
es.indices.refresh(index=args.index)
total_res = es.search(index=args.index)
print "[*]-{} documents have been saved.".format(
total_res['hits']['total'])
def make_client(version, **kwargs):
client = create_client(
get_fix(version).HOST,
version=("1.1" if version == 11 else "1.0"),
**kwargs)
return client
log.debug("Opening config file %s", config_file)
with open(config_file, "r") as f:
config = yaml.load(f.read())
log.debug("Config read %s", config)
# Read in the local server configuration
local_config = "{}/local-server.yml".format(os.path.expanduser(args.configdir))
log.debug("Reading local server config")
with open(local_config, "r") as f:
local_config = yaml.load(f.read())
# Attempt to make contact with the local server
log.info("Connecting to local server...")
local_client = create_client(host=local_config["host"],
port=local_config["port"],
discovery_path=local_config["discovery_path"],
use_https=local_config["use_https"],
version=local_config["taxii_version"],
headers=local_config["headers"])
local_client.username = local_config["auth"]["username"]
local_client.password = local_config["auth"]["password"]
local_inbox = "{}://{}:{}{}".format(
"https" if local_config["use_https"] else "http",
local_config["host"], local_config["port"],
local_config["inbox_path"])
# Check that we're all good and authenticated
try:
list(local_client.discover_services())
def taxii(content, host, https, discovery, binding, username, password, inbox):
client = create_client(host, use_https=https, discovery_path=discovery)
content = content
binding = binding
client.set_auth(username=username, password=password)
client.push(content, binding, uri=inbox)
def taxiiPush(self, **kwargs):
"""Pushes taxii data to a taxii server
``jwt_auth_url`` is required for JWT based authentication. If
it is not specified but ``username`` and ``password`` are provided,
client will configure Basic authentication.
SSL authentication can be combined with JWT and Basic
authentication.
:param str ca_cert: a path to CA SSL certificate file
:param str cert_file: a path to SSL certificate file
:param str key_file: a path to SSL key file
:param str username: username, used in basic auth or JWT auth
:param str password: password, used in basic auth or JWT auth
:param str key_password: same argument as in
``ssl.SSLContext.load_cert_chain`` - may be a function to call
to get the password for decrypting the private key or
string/bytes/bytearray. It will only be called if the private
key is encrypted and a password is necessary.
:param str jwt_auth_url: URL used to obtain JWT token
:param bool/str verify_ssl: set to False to skip checking host's SSL
certificate. Set to True to check certificate against public CAs or
set to filepath to check against custom CA bundle.
:param str content: content to push
:param content_binding: content binding for a content
:type content_binding: string or
:py:class:`cabby.entities.ContentBinding`
:param list collection_names:
destination collection names
:param datetime timestamp: timestamp label of the content block
(current UTC time by default)
:param str uri: URI path to a specific Inbox Service
:raises ValueError:
if URI provided is invalid or schema is not supported
:raises `cabby.exceptions.HTTPError`:
if HTTP error happened
:raises `cabby.exceptions.UnsuccessfulStatusError`:
if Status Message received and status_type is not `SUCCESS`
:raises `cabby.exceptions.ServiceNotFoundError`:
if no service found
:raises `cabby.exceptions.AmbiguousServicesError`:
more than one service with type specified
:raises `cabby.exceptions.NoURIProvidedError`:
no URI provided and client can't discover services
:return: STIX object from python-stix
:rtype: ``stix.core.stix_package.STIXPackage``
"""
client = create_client()
content = kwargs.get("content")
if 'username' in kwargs:
self.logger.debug("message=\"Using basic auth\"")
client.set_auth(username=kwargs.get("username"), password=kwargs.get("password"), jwt_auth_url=kwargs.get("jwt_auth_url"), verify_ssl=kwargs.get("verify_ssl"))
elif 'cert_file' in kwargs:
self.logger.debug("message=\"Using cert auth\"")
client.set_auth(ca_cert=kwargs.get("ca_cert"), cert_file=kwargs.get("cert_file"), key_file=kwargs.get("key_file"), key_password=kwargs.get("key_password"), verify_ssl=kwargs.get("verify_ssl"))
content_io = StringIO(content)
stix = STIXPackage().from_xml(content_io)
file_name = stix.id_
self.logger.info("message=\"Pushing STIX " + file_name + "\"")
client.push(
content=content,
content_binding=kwargs.get("content_binding"),
collection_names=kwargs.get("collection_names"),
timestamp=kwargs.get("timestamp"),
uri=kwargs.get("uri")
)
return stix
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
from cabby import create_client
try:
# create a connection
client = create_client(host='{{ opentaxii_host }}', port='{{ opentaxii_port }}', discovery_path='/services/discovery')
# iterate through each defined collection
collections = client.get_collections(uri='{{ opentaxii_domain }}/services/collection')
for collection in collections:
# how many records in each collection?
count = client.get_content_count(collection_name=collection.name, uri='{{ opentaxii_domain }}/services/poll')
print "%-50s %-10d" % (collection.name, count.count)
except:
print "Services not defined"
def perform(self) -> None:
"""
Perform the taxii hailing service.
"""
for site in self.sites:
client: Union[Client10, Client11] = create_client(site.get('site'),
use_https=site.get('use_https'),
discovery_path=site.get('discovery_path'))
#
# Set verify_ssl and ca_cert inside the client
#
client.set_auth(verify_ssl=site.get('ssl_verify'), ca_cert=site.get('ca_cert'))
#
# Proxy Settings
#
proxy_dict = dict()
if self.http_proxy_url:
_logger.info(f"Found HTTP Proxy: {self.http_proxy_url}")
proxy_dict['http'] = self.http_proxy_url
if self.https_proxy_url:
_logger.info(f"Found HTTPS Proxy: {self.https_proxy_url}")
proxy_dict['https'] = self.https_proxy_url
if proxy_dict:
client.set_proxies(proxy_dict)
# If a username is supplied use basic authentication
if site.get('username') or site.get('cert_file'):
_logger.info("Found Username in config, using basic auth...")
client.set_auth(username=site.get('username'),
password=site.get('password'),
verify_ssl=site.get('ssl_verify'),
ca_cert=site.get('ca_cert'),
cert_file=site.get('cert_file'),
key_file=site.get('key_file'))
if not site.get('collection_management_path', ''):
collections = client.get_collections()
else:
uri = ''
if site.get('use_https'):
uri += 'https://'
else:
uri += 'http://'
uri += site.get('site')
uri += site.get('collection_management_path')
_logger.info('Collection Management Path: {}'.format(uri))
collections: List[CabbyCollection] = client.get_collections(uri=uri)
if len(collections) == 0:
_logger.info('Unable to find any collections. Exiting...')
sys.exit(0)
_logger.info("=" * 80)
for collection in collections:
_logger.info(f'Collection Name: {collection.name}, Collection Type: {collection.type}')
_logger.info("=" * 80 + "\n")
desired_collections = [x.strip() for x in site.get('collections').lower().split(',')]
want_all = False
if '*' in desired_collections:
want_all = True
for collection in collections:
if collection.type != 'DATA_FEED' and collection.type != 'DATA_SET':
continue
if collection.type == 'DATA_SET':
data_set = True
else:
data_set = False
if want_all or collection.name.lower() in desired_collections:
self._import_collection(client, site, collection, data_set)
