def __init__(self, dxl_config_file, dxl_topic,
object_report, object_send_message, object_logger):
self.config = DxlClientConfig.create_dxl_config_from_file(dxl_config_file)
self.dxl_topic = dxl_topic
self.send_message = object_send_message
self.report = object_report
self.logger = object_logger
root_dir = os.path.dirname(os.path.abspath(__file__))
sys.path.append(root_dir + "/../..")
sys.path.append(root_dir + "/..")
from dxlciscopxgridclient.client import CiscoPxGridClient
from dxlciscopxgridclient.callbacks import AncApplyEndpointPolicyCallback
# Import common logging and configuration
from common import *
# Configure local logger
logging.getLogger().setLevel(logging.ERROR)
logger = logging.getLogger(__name__)
# Create DXL configuration from file
config = DxlClientConfig.create_dxl_config_from_file(CONFIG_FILE)
# Create the client
with DxlClient(config) as dxl_client:
# Connect to the fabric
dxl_client.connect()
logger.info("Connected to DXL fabric.")
# Create client wrapper
client = CiscoPxGridClient(dxl_client)
class MyAncApplyEndpointPolicyCallback(AncApplyEndpointPolicyCallback):
def on_apply_endpoint_policy(self, apply_dict):
print("on_apply_endpoint_policy\n" +
def init(self):
if DxlClient is None:
raise ValueError('Could not import dxlclient or dxlmarclient. '
'Please install them.')
self.config = DxlClientConfig.create_dxl_config_from_file(self.parameters.dxl_config_file)
#!/usr/bin/python
import sys, getopt,socket,json,os
from dxlclient.client import DxlClient
from dxlclient.client_config import DxlClientConfig
config = DxlClientConfig.create_dxl_config_from_file(os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dxl.conf'))
from dxlclient.message import Event
IP = socket.gethostname()
DXL_MESSAGE = {}
def main(argv):
TOPIC_DESTINATION = '/events/syslog'
TYPE_PAYLOAD = 'syslog'
PAYLOAD = sys.argv[1]
DXL_MESSAGE['SRC_HOST'] = IP
DXL_MESSAGE['TYPE_PAYLOAD'] = TYPE_PAYLOAD
DXL_MESSAGE['PAYLOAD'] = PAYLOAD
with DxlClient(config) as client:
client.connect()
event = Event(TOPIC_DESTINATION)
event.payload = str(json.dumps(DXL_MESSAGE)).encode()
client.send_event(event)
if __name__ == "__main__":
main(sys.argv[1:])
def __init__(self):
self.config = DxlClientConfig.create_dxl_config_from_file(dxl_config)
self.tie_rep = tie_rep
def __init__(self):
self.config = DxlClientConfig.create_dxl_config_from_file(
os.getenv("DXL_CONNECTOR_CLIENT_CONFIG_PATH"))
def client_config(self):
return DxlClientConfig.create_dxl_config_from_file(
self.app.bootstrap_app.client_config_path)
def main(argv):
# parse the args
arg_parser = create_arg_parser()
args = arg_parser.parse_args()
# set logging level
set_logging_level(logger, args.loglevel)
# configure local logger for requests (Urllib3) and set its level
set_logging_level(logging.getLogger("urllib3"), args.loglevel)
# read cfg file
try:
config = ConfigObj(args.configfile, raise_errors=True, file_error=True)
except:
# TODO - enhance error handling here
logger.error("Could not parse config file!")
exit(1)
#
# get token
#
# TODO - handle HTTPS nicely
urllib3.disable_warnings()
token = get_staxx_token(config['STAXX']['Address'],
config['STAXX']['Port'], config['STAXX']['User'],
config['STAXX']['Password'])
if not token:
logger.error("Exiting...")
exit(1)
#
# DXL initialization
#
# TODO - enhance error handling here
if not args.dryrun:
# DxlClientConfig from DXL configuration file
logger.info("Loading DXL config from: %s", config['DXL']['Config'])
dxl_config = DxlClientConfig.create_dxl_config_from_file(
config['DXL']['Config'])
#
# build the topics
#
obs_types = config['Observable Types']
logger.debug("Observable types: %s", obs_types)
obs_topics = config['Observable Topics']
#
# timed loop (synchronous "obs export / msg publish")
#
while True:
#
# export observables into JSON object
#
req_error, json_obs = get_staxx_observables(config['STAXX']['Address'],
config['STAXX']['Port'],
token, args.filter_query)
if req_error:
logger.error("Failure exporting observables.")
if not args.singleshot:
logger.info("Sleeping until next polling cycle...\n")
time.sleep(args.time)
logger.info("New polling cycle.")
continue
if json_obs:
logger.info(
"{0} observable(s) exported from Anomali STAXX.".format(
len(json_obs)))
if args.pprint:
logger.info("Printing observables to STDOUT...")
print json.dumps(json_obs, indent=2, sort_keys=False)
if not args.dryrun:
#
# Connect to DXL and publish the observables as events
#
try:
with DxlClient(dxl_config) as dxl_client:
# Connect to DXL Broker
logger.info("Connecting to DXL broker...")
dxl_client.connect()
# TODO - handle possible connection errors
logger.info(
"Filtering observables and publishing events...")
count = 0
for ob in json_obs:
key = is_observable_type_listed(
obs_types, ob['itype'])
if key:
count += 1
logger.debug(
"Publishing message for observable (itype: %s, topic: %s).",
ob['itype'], obs_topics[key])
dxl_event = Event(obs_topics[key])
payload_str = json.dumps(ob)
logger.debug("Msg payload: %s", payload_str)
dxl_event.payload = str(payload_str).encode()
dxl_client.send_event(dxl_event)
else:
logger.info(
"Observable not published (itype: %s not listed).",
ob['itype'])
logger.info("%s event(s) published to DXL fabric.",
count)
except Exception as e:
logger.error(
"Could not initialize OpenDXL client ({0}).".format(
e.message))
exit(1)
else:
logger.info("No observable exported from Anomali STAXX.")
# wait for next cycle (if not single shot mode)
if args.singleshot:
logger.info("Exiting (single shot mode).")
exit(0)
else:
logger.info("Sleeping until next polling cycle...\n")
time.sleep(float(args.time))
logger.info("New polling cycle.")
print "TEST: FILE, FOLDER STRUCTURE"
#print "FOLDER EXISTS = {}".format(aws_machine.check_folder_exists('/var/McAfee/dxlbroker/keystore/'))
#print "FILE EXISTS = {}".format(aws_machine.check_file_exists('/var/McAfee/dxlbroker/keystore/broker.crt'))
#print "FOLDER EXISTS = {}".format(aws_machine.check_folder_exists('/var/McAfee/dxlbroker/keystore2/'))
#print "FILE EXISTS = {}".format(aws_machine.check_file_exists('/var/McAfee/dxlbroker/keystore/broker2.crt'))
# STEP-4: connect python client (need SG with rule for port 8883: "Custom TCP Rule | TCP | 8883 | 0.0.0.0/0")
print "TEST: PYTHON CLIENT"
brokerCaBundle = "C:\\test\\dxlbroker_install_files\\keystore\\ca-broker.crt"
certFile = "C:\\test\\dxlbroker_install_files\\keystore\\broker.crt"
privateKey = "C:\\test\\dxlbroker_install_files\\keystore\\broker.key"
brokerString = "ssl://{}".format(aws_machine.ip)
action = "publish_event"
topic = "/mcafee/client/controlevent"
config = DxlClientConfig(broker_ca_bundle=brokerCaBundle,
cert_file=certFile,
private_key=privateKey,
brokers=[Broker.parse(brokerString)])
with DxlClient(config) as dxl_client:
# Connect to the fabric
dxl_client.connect()
if dxl_client.connected:
print "Connected ... \n"
else:
print "Not Connected ... \n"
sleepTime = 1
rb = os.urandom(100)
event = Event(str(topic))
event.payload = rb
print "payload={}".format(rb)
def __init__(self):
self.config = DxlClientConfig.create_dxl_config_from_file(dxl_config)
self.misp = ExpandedPyMISP(misp_url, misp_key, misp_verify)
self.tags = self.misp.tags()
self.attributes = []
self.found = False
def main():
# Create DXL configuration from file
config = DxlClientConfig.create_dxl_config_from_file('dxlclient.config')
logger.info("Starting the ATD/PAN FW OpenDXL integrator")
# Create the client
# Condition/lock used to protect changes to counter
event_count_condition = Condition()
event_count = {}
event_count[0] = 0
conn = sqlite3.connect(settings['sqlite_file'])
curs = conn.cursor()
curs.execute("""
CREATE TABLE IF NOT EXISTS domains (domain text,
seen_count int)
""")
curs.execute("""
CREATE TABLE IF NOT EXISTS ips (ip text,
seen_count int)
""")
conn.close()
# Create the client
with DxlClient(config) as client:
logger.info("Connecting to OpenDXL")
# Connect to the fabric
client.connect()
logger.info("Connected to OpenDXL")
#
# Register callback and subscribe
#
# Create and add event listener. This EventCallBack is executed whenever an event is placed on the current
# topic
class MyEventCallback(EventCallback):
def on_event(self, event):
with event_count_condition:
conn = sqlite3.connect(settings['sqlite_file'])
curs = conn.cursor()
logger.info("Received event")
# The ATD json can sometimes have non-standard json characters at the end.
# Let's clean it up.
# End the json at the last curly brace: }
payload_end = event.payload.rfind('}')+1
# Offset the payload and escape any backslashes to be interpreted as literals not escape/control
# characters
clean_payload = event.payload[:payload_end].replace('\\', '\\\\')
# Print and assign the payload for the received event
atd_result = json.loads(clean_payload)
logger.debug(atd_result)
# Process URLs discovered in ATD analysis
try:
for remote in atd_result["Summary"]["Urls"]:
if int(remote["Severity"]) >= settings['min_sev']:
# This domain is relevant
current_domain = remote["Url"].split('/')[0]
logger.info("Severity is {}".format(int(remote["Severity"])))
logger.info("URL {} is a candidate for Blacklisting".format(current_domain))
# First, check to see if the current domain exists
curs.execute("SELECT domain, seen_count FROM domains WHERE domain =?",
[current_domain])
rows = curs.fetchall()
if rows:
# The row exists
# Add 1 to the seen count
curs.execute("UPDATE domains set seen_count = seen_count+1 WHERE domain =?",
[current_domain])
conn.commit()
else:
# The row is new
# Add an entry for the domain
curs.execute("INSERT INTO domains VALUES(?, 1)", [current_domain])
conn.commit()
except KeyError as ke:
logger.info("Summary/URLs does not exist.")
logger.info(ke)
# Process IPs discovered in ATD analysis
try:
for remote in atd_result["Summary"]["Ips"]:
if int(remote["Severity"]) >= settings['min_sev']:
# This domain is relevant
current_ip = remote["Ipv4"]
logger.info("Severity is {}".format(int(remote["Severity"])))
logger.info("IP {} is a candidate for Blacklisting".format(current_ip))
# First, check to see if the current IP exists
curs.execute("SELECT ip, seen_count FROM ips WHERE ip =?",
[current_ip])
rows = curs.fetchall()
if rows:
# The row exists
# Add 1 to the seen count
curs.execute("UPDATE ips set seen_count = seen_count+1 WHERE ip =?",
[current_ip])
conn.commit()
else:
# The row is new
# Add an entry for the ip
curs.execute("INSERT INTO ips VALUES(?, 1)", [current_ip])
conn.commit()
except KeyError as ke:
logger.info("Summary/Ips does not exist.")
logger.info(ke)
# Increment the count
event_count[0] += 1
# Notify that the count was increment
event_count_condition.notify_all()
conn.close()
# Register the callback with the client
client.add_event_callback(settings['event_topic'], MyEventCallback())
# Wait until all events have been received
logger.info("Waiting for events to be received...")
if __name__ == "__main__":
# This ensures we don't run the web service twice
app.run()
def __init__(self):
dxl_config = 'config/dxlclient.config'
self.config = DxlClientConfig.create_dxl_config_from_file(dxl_config)
def start(self,
config_file="./dxlclient.config",
service="/dsa/dxl/test",
*args):
if self.started:
raise DxlJythonException(2000, "Already started")
if len(args) == 2:
# Topic and callback were specified in separate parameters
if not args[1]:
raise DxlJythonException(2100, "DXL callback is required")
topic = args[0] or "/dsa/dxl/test/event2"
callbacks_by_topic = {topic: args[1]}
topic_info = "topic '%s'" % topic
elif len(args) == 1:
# Topics/callbacks were specified in a map
callbacks_by_topic = args[0]
topic_info = "topics '%s'" % ",".join(
list(callbacks_by_topic.keys()))
else:
raise DxlJythonException(2100, "DXL callback is required")
try:
logger.info("Starting service '%s' on %s", service, topic_info)
logger.info("Reading configuration file from '%s'", config_file)
config = DxlClientConfig.create_dxl_config_from_file(config_file)
# Initialize DXL client using our configuration
with DxlClient(config) as client:
# Connect to DXL Broker
client.connect()
class MyRequestCallback(RequestCallback):
def __init__(self, dxl_callback):
self.dxlCallback = dxl_callback
def on_request(self, request):
dxl_message = JavaDxlMessage()
dxl_message.setTopic(request.destination_topic)
dxl_message.setMessageVersion(request.version)
dxl_message.setMessageId(request.message_id)
dxl_message.setClientId(request.source_client_id)
dxl_message.setBrokerId(request.source_broker_id)
dxl_message.setMessageType(request.message_type)
dxl_message.setBrokerIdList(request.broker_ids)
dxl_message.setClientIdList(request.client_ids)
dxl_message.setReplyTopic(request.reply_to_topic)
dxl_message.setServiceId(request.service_id)
dxl_message.setPayload(request.payload.decode())
dxl_message.setReplyTopic(request.reply_to_topic)
dxl_message.setServiceId(request.service_id)
response = Response(request)
resp = self.dxlCallback.callbackEvent(dxl_message)
response.payload = resp.encode()
client.send_response(response)
# Create DXL Service Registration object
service_registration_info = ServiceRegistrationInfo(
client, str(service))
# Add topics for the service to respond to
service_registration_info.add_topics({
str(k): MyRequestCallback(v)
for k, v in callbacks_by_topic.iteritems()
})
# Register the service with the DXL fabric (with a wait up to 10 seconds for registration to complete)
client.register_service_sync(service_registration_info, 10)
self.started = True
while self.loop:
time.sleep(1)
logger.info("Shutting down service '%s' on %s", service,
topic_info)
return "Shutting down service provider on %s" % topic_info
except Exception as e:
logger.error("Exception $s", e.message)
raise DxlJythonException(
1010, "Unable to communicate with a DXL broker")
logger = logging.getLogger(__name__)
# The topic to publish to
EVENT_TOPIC = "/isecg/sample/basicevent"
# The total number of events to send
TOTAL_EVENTS = 1000
# Condition/lock used to protect changes to counter
event_count_condition = Condition()
# The events received (use an array so we can modify in callback)
event_count = [0]
# Create DXL configuration from file
config = DxlClientConfig.create_dxl_config_from_file(
"/vagrant/dxlclient.config")
# Create the client
with DxlClient(config) as client:
# Connect to the fabric
client.connect()
#
# Register callback and subscribe
#
# Create and add event listener
class MyEventCallback(EventCallback):
def on_event(self, event):
with event_count_condition:
import time
from flask import Flask, render_template, request, jsonify
from dxlclient.client_config import DxlClientConfig
from dxlclient.client import DxlClient
from dxltieclient import TieClient
requests.packages.urllib3.disable_warnings()
app = Flask(__name__)
FILTERS = ['md5', 'sha1', 'sha256', 'hostname']
DXL_CERTS = '/path/to/dxlclient.config'
DXL_CONFIG = DxlClientConfig.create_dxl_config_from_file(DXL_CERTS)
EPO_IP = '1.1.1.1'
EPO_PORT = '8443'
EPO_USER = '******'
EPO_PW = 'password'
class MAR():
def __init__(self):
self.url = 'https://{0}:{1}'.format(EPO_IP, EPO_PORT)
self.headers = {'Content-Type': 'application/json'}
self.verify = False
self.auth = (EPO_USER, EPO_PW)
