def query_service_registry(self, client, query):
request = Request(self.DXL_SERVICE_REGISTRY_QUERY_TOPIC)
if not query:
query = {}
request.payload = json.dumps(query)
response = client.sync_request(request, timeout=self.RESPONSE_WAIT)
return json.loads(
response.payload.decode("utf8").rstrip("\0"))["services"]
def set_certificate_reputation(self, trust_level, sha1, public_key_sha1=None, comment=""):
"""
Sets the "Enterprise" reputation (`trust level`) of a specified certificate (as identified by hashes).
.. note::
**Client Authorization**
The OpenDXL Python client invoking this method must have permission to send messages to the
``/mcafee/service/tie/cert/reputation/set`` topic which is part of the
``TIE Server Set Enterprise Reputation`` authorization group.
The following page provides an example of authorizing a Python client to send messages to an
`authorization group`. While the example is based on McAfee Active Response (MAR), the
instructions are the same with the exception of swapping the ``TIE Server Set Enterprise Reputation``
`authorization group` in place of ``Active Response Server API``:
`<https://opendxl.github.io/opendxl-client-python/pydoc/marsendauth.html>`_
**Example Usage**
.. code-block:: python
# Set the enterprise reputation (trust level) for the certificate to Known Trusted
tie_client.set_certificate_reputation(
TrustLevel.KNOWN_TRUSTED,
"1C26E2037C8E205B452CAB3565D696512207D66D",
public_key_sha1="B4C3B2D596D1461C1BB417B92DCD74817ABB829D",
comment="Reputation set via OpenDXL")
:param trust_level: The new `trust level` for the file. The list of standard `trust levels` can be found in the
:class:`dxltieclient.constants.TrustLevel` constants class.
:param sha1: The SHA-1 of the certificate
:param public_key_sha1: The SHA-1 of the certificate's public key (optional)
:param comment: A comment to associate with the certificate (optional)
"""
# Create the request message
req = Request(TIE_SET_CERT_REPUTATION_TOPIC)
# Create a dictionary for the payload
payload_dict = {
"trustLevel": trust_level,
"providerId": CertProvider.ENTERPRISE,
"comment": comment,
"hashes": [
{"type": "sha1", "value": base64.b64encode(sha1.decode('hex'))}
]}
# Add public key SHA-1 (if specified)
if public_key_sha1:
payload_dict["publicKeySha1"] = base64.b64encode(public_key_sha1.decode('hex'))
# Set the payload
req.payload = json.dumps(payload_dict).encode(encoding="UTF-8")
# Send the request
self.__dxl_sync_request(req)
def test_execute_registry_query(self):
with self.create_client() as client:
client.connect()
topic = "/mcafee/service/dxl/brokerregistry/query"
req = Request(topic)
req.payload = "{}"
response = client.sync_request(req)
self.assertNotIsInstance(response, ErrorResponse)
print("## sourceBrokerGuid: " + str(response.source_broker_id))
print("## sourceClientGuid: " + str(response.source_client_id))
print(str(response.payload))
def test_execute_message_payload(self):
# Create a server that handles a request, unpacks the payload, and
# asserts that the information in the payload was delivered successfully.
with self.create_client(max_retries=0) as server:
test_service = TestService(server, 1)
server.connect()
topic = UuidGenerator.generate_id_as_string()
reg_info = ServiceRegistrationInfo(
server, "message_payload_runner_service")
# callback definition
def on_request(request):
unpacker = Unpacker(
file_like=StringIO.StringIO(request.payload))
with self.request_complete_condition:
try:
self.assertEquals(unpacker.next(), self.TEST_STRING)
self.assertEquals(unpacker.next(), self.TEST_BYTE)
self.assertEquals(unpacker.next(), self.TEST_INT)
self.received_request = True
except Exception, e:
print e.message
self.request_complete_condition.notify_all()
request_callback = RequestCallback()
request_callback.on_request = on_request
reg_info.add_topic(topic, request_callback)
# Register the service
server.register_service_sync(reg_info, self.DEFAULT_TIMEOUT)
with self.create_client() as client:
client.connect()
packer = Packer()
# Send a request to the server with information contained
# in the payload
request = Request(destination_topic=topic)
request.payload = packer.pack(self.TEST_STRING)
request.payload += packer.pack(self.TEST_BYTE)
request.payload += packer.pack(self.TEST_INT)
client.async_request(request, request_callback)
with self.request_complete_condition:
if not self.received_request:
# Wait until the request has been processed
self.request_complete_condition.wait(self.MAX_WAIT)
if not self.received_request:
self.fail("Request not received.")
def test_execute_message_payload(self):
# Create a server that handles a request, unpacks the payload, and
# asserts that the information in the payload was delivered successfully.
with self.create_client(max_retries=0) as service_client:
service_client.connect()
topic = UuidGenerator.generate_id_as_string()
reg_info = ServiceRegistrationInfo(
service_client, "message_payload_runner_service")
# callback definition
def on_request(request):
with self.request_complete_condition:
try:
self.request_received = request
except Exception as ex: # pylint: disable=broad-except
print(ex)
self.request_complete_condition.notify_all()
request_callback = RequestCallback()
request_callback.on_request = on_request
reg_info.add_topic(topic, request_callback)
# Register the service
service_client.register_service_sync(reg_info,
self.DEFAULT_TIMEOUT)
with self.create_client() as request_client:
request_client.connect()
packer = msgpack.Packer()
# Send a request to the server with information contained
# in the payload
request = Request(destination_topic=topic)
request.payload = packer.pack(self.TEST_STRING)
request.payload += packer.pack(self.TEST_BYTE)
request.payload += packer.pack(self.TEST_INT)
request_client.async_request(request, request_callback)
start = time.time()
# Wait until the request has been processed
with self.request_complete_condition:
while (time.time() - start < self.MAX_WAIT) and \
not self.request_received:
self.request_complete_condition.wait(self.MAX_WAIT)
self.assertIsNotNone(self.request_received)
unpacker = msgpack.Unpacker(file_like=BytesIO(request.payload))
self.assertEqual(
next(unpacker).decode('utf8'), self.TEST_STRING)
self.assertEqual(next(unpacker), self.TEST_BYTE)
self.assertEqual(next(unpacker), self.TEST_INT)
def _invoke_mar_search_api(self, payload_dict):
"""
Executes a query against the MAR search API
:param payload_dict: The payload
:return: A dictionary containing the results of the query
"""
# Create the request message
req = Request(MAR_SEARCH_TOPIC)
# Set the payload
req.payload = json.dumps(payload_dict).encode(encoding="UTF-8")
# Display the request that is going to be sent
logger.debug(
"Request:\n%s",
json.dumps(payload_dict,
sort_keys=True,
indent=4,
separators=(',', ': ')))
# Send the request and wait for a response (synchronous)
res = self._dxl_sync_request(req)
# Return a dictionary corresponding to the response payload
resp_dict = MessageUtils.json_payload_to_dict(res)
# Display the response
logger.debug(
"Response:\n%s",
json.dumps(resp_dict,
sort_keys=True,
indent=4,
separators=(',', ': ')))
if "code" in resp_dict:
code = resp_dict['code']
if code < 200 or code >= 300:
if "body" in resp_dict and "applicationErrorList" in \
resp_dict["body"]:
error = resp_dict["body"]["applicationErrorList"][0]
raise Exception(error["message"] + ": " +
str(error["code"]))
elif "body" in resp_dict:
raise Exception(resp_dict["body"] + ": " + str(code))
else:
raise Exception("Error: Received failure response code: " +
str(code))
else:
raise Exception("Error: unable to find response code")
return resp_dict
def test_register_service_and_send_request(self):
with self.create_client() as client:
self.add_client_callbacks(client)
client.register_service_async(self.info)
time.sleep(self.POST_OP_DELAY)
client.connect()
time.sleep(self.POST_OP_DELAY)
request = Request("/mcafee/service/JTI/file/reputation/" + self.info.service_id)
request.payload = bytes("Test")
response = client.sync_request(request, self.POST_OP_DELAY)
logging.info("Response payload: {0}".format(str(response.payload)))
self.assertEquals("Ok", str(response.payload))
def test_register_service_and_send_request(self):
with self.create_client() as client:
self.add_client_callbacks(client)
client.register_service_async(self.info)
client.connect()
self.assertTrue(self.wait_info_registered())
request = Request("/mcafee/service/JTI/file/reputation/" +
self.info.service_id)
request.payload = "Test"
response = client.sync_request(request, self.POST_OP_DELAY)
logging.info("Response payload: %s",
response.payload.decode("utf8"))
self.assertEqual("Ok", response.payload.decode("utf8"))
def test_register_service_weak_reference_after_connect_and_send_request(
self):
with self.create_client() as client:
self.add_client_callbacks(client)
ref = weakref.ref(self.info)
client.register_service_async(self.info)
client.connect()
info_guid = self.info.service_id
# Theoretically, sending the destroy method when creating the weakref
# would made the magic of calling that method when info get unref
self.info._destroy()
# Deleted the service registration
self.info = 1
time.sleep(self.POST_OP_DELAY * 2)
# Enforce garbage collection
gc.collect()
# Weak reference should now be null
self.assertEquals(None, ref())
# Sending an request should result in a WaitTimeoutException since the destroy() method
# of ServiceRegistrationInfo will unregister the service
request = Request("/mcafee/service/JTI/file/reputation/" +
info_guid)
request.payload = bytes("Test")
try:
response = client.sync_request(request, 2)
# Depending upon the timing, the broker can respond with 404 or the request might timeout
# self.assertIsInstance(response, ErrorResponse, "response is instance of ErrorResponse")
self.assertTrue(isinstance(response, ErrorResponse),
response.__class__)
except WaitTimeoutException as ex:
assert (ex.message.__contains__(request.message_id))
self.assertEquals(1, self.register_callback.get())
logging.debug("Waiting for unregister event...")
ttw = 30
while self.unregister_callback.get() < 1 and ttw > 0:
time.sleep(self.POST_OP_DELAY)
ttw -= 1
def test_register_service_weak_reference_after_connect_and_send_request(
self):
with self.create_client() as client:
self.add_client_callbacks(client)
client.register_service_async(self.info)
client.connect()
service_id = self.info.service_id
self.assertTrue(self.wait_info_registered())
# Deleted the service registration
self.info = 1
# Enforce garbage collection
gc.collect()
# Service should be implicitly unregistered from the broker
# as the weak reference to the service is cleaned up.
self.assertTrue(self.wait_info_not_registered())
# Sending an request should result in a WaitTimeoutException since
# the destroy() method of ServiceRegistrationInfo will unregister
# the service
request = Request(
"/mcafee/service/JTI/file/reputation/" + service_id)
request.payload = "Test"
try:
response = client.sync_request(request, 2)
# Depending upon the timing, the broker can respond with 404 or the request might timeout
# self.assertIsInstance(response, ErrorResponse, "response is instance of ErrorResponse")
self.assertTrue(isinstance(response, ErrorResponse),
response.__class__)
except WaitTimeoutException as ex:
self.assertIn(request.message_id, str(ex))
self.assertEqual(1, self.info_registrations)
def test_request_and_response(self):
reply_to_channel = "/mcafee/client/" + UuidGenerator.generate_id_as_string()
service_guid = UuidGenerator.generate_id_as_string()
source_client_guid = UuidGenerator.generate_id_as_string()
source_broker_guid = UuidGenerator.generate_id_as_string()
request = Request(destination_topic="")
request.reply_to_channel = reply_to_channel
request.service_id = service_guid
request._source_client_id = source_client_guid
request._source_broker_id = source_broker_guid
request.broker_ids = ["{66000000-0000-0000-0000-000000000001}",
"{66000000-0000-0000-0000-000000000002}",
"{66000000-0000-0000-0000-000000000003}"]
request.client_ids = ["{25000000-0000-0000-0000-000000000001}",
"{25000000-0000-0000-0000-000000000002}",
"{25000000-0000-0000-0000-000000000003}"]
request.payload = "REQUEST".encode()
PP.pprint(vars(request))
message = request._to_bytes()
PP.pprint(message)
self.assertEqual(source_client_guid, request.source_client_id)
result = Message._from_bytes(message)
PP.pprint(vars(result))
response = Response(request=request)
response.payload = "RESPONSE".encode()
PP.pprint(vars(response))
message = response._to_bytes()
PP.pprint(message)
result = Message._from_bytes(message)
PP.pprint(vars(result))
self.assertEqual(Message.MESSAGE_TYPE_RESPONSE, result.message_type)
def test_request(self):
reply_to_channel = "/mcafee/client/" + UuidGenerator.generate_id_as_string()
service_guid = UuidGenerator.generate_id_as_string()
source_client_guid = UuidGenerator.generate_id_as_string()
source_broker_guid = UuidGenerator.generate_id_as_string()
request = Request(destination_topic="")
request.reply_to_topic = reply_to_channel
request.service_id = service_guid
request._source_client_id = source_client_guid
request._source_broker_id = source_broker_guid
request.broker_ids = ["{66000000-0000-0000-0000-000000000001}",
"{66000000-0000-0000-0000-000000000002}",
"{66000000-0000-0000-0000-000000000003}"]
request.client_ids = ["{25000000-0000-0000-0000-000000000001}",
"{25000000-0000-0000-0000-000000000002}",
"{25000000-0000-0000-0000-000000000003}"]
request.payload = str.encode("REQUEST")
pp.pprint(vars(request))
message = request._to_bytes()
pp.pprint(message)
result = Message._from_bytes(message)
pp.pprint(vars(result))
assert result.reply_to_topic == reply_to_channel
assert result.service_id == service_guid
assert result.source_client_id == source_client_guid
assert result.source_broker_id == source_broker_guid
assert result.broker_ids == ["{66000000-0000-0000-0000-000000000001}",
"{66000000-0000-0000-0000-000000000002}",
"{66000000-0000-0000-0000-000000000003}"]
assert result.client_ids == ["{25000000-0000-0000-0000-000000000001}",
"{25000000-0000-0000-0000-000000000002}",
"{25000000-0000-0000-0000-000000000003}"]
assert result.payload == str.encode("REQUEST")
assert result.message_type == Message.MESSAGE_TYPE_REQUEST
def test_eporequestcallback(self):
mock_dxl_client = MockDxlClient()
with MockServerRunner() as server_list:
server_info = server_list[0]
test_topic = "/test/topic"
epo = dxleposervice._epo._Epo(
server_info[SERVER_INFO_SERVER_NAME_KEY],
LOCALHOST_IP,
server_info[SERVER_INFO_SERVER_PORT_KEY],
TEST_USER,
TEST_PASSWORD,
False
)
epo_topics = {test_topic: epo}
test_request = Request(test_topic)
# Set the payload
test_request.payload = json.dumps(
{
"command": "core.help",
"output": "json",
"params": {}
}
).encode(encoding="UTF-8")
epo_request_callback = \
dxleposervice.app._EpoRequestCallback(mock_dxl_client, epo_topics)
epo_request_callback.on_request(test_request)
self.assertIn(
HELP_CMD_RESPONSE_PAYLOAD,
mock_dxl_client.latest_sent_message._payload
)
def set_file_reputation(self, trust_level, hashes, filename="", comment=""):
"""
Sets the "Enterprise" reputation (`trust level`) of a specified file (as identified by hashes).
.. note::
**Client Authorization**
The OpenDXL Python client invoking this method must have permission to send messages to the
``/mcafee/service/tie/file/reputation/set`` topic which is part of the
``TIE Server Set Enterprise Reputation`` authorization group.
The following page provides an example of authorizing a Python client to send messages to an
`authorization group`. While the example is based on McAfee Active Response (MAR), the
instructions are the same with the exception of swapping the ``TIE Server Set Enterprise Reputation``
`authorization group` in place of ``Active Response Server API``:
`<https://opendxl.github.io/opendxl-client-python/pydoc/marsendauth.html>`_
**Example Usage**
.. code-block:: python
# Set the Enterprise reputation (trust level) for notepad.exe to Known Trusted
tie_client.set_file_reputation(
TrustLevel.KNOWN_TRUSTED, {
HashType.MD5: "f2c7bb8acc97f92e987a2d4087d021b1",
HashType.SHA1: "7eb0139d2175739b3ccb0d1110067820be6abd29",
HashType.SHA256: "142e1d688ef0568370c37187fd9f2351d7ddeda574f8bfa9b0fa4ef42db85aa2"
},
filename="notepad.exe",
comment="Reputation set via OpenDXL")
:param trust_level: The new `trust level` for the file. The list of standard `trust levels` can be found in the
:class:`dxltieclient.constants.TrustLevel` constants class.
:param hashes: A ``dict`` (dictionary) of hashes that identify the file to update the reputation for.
The ``key`` in the dictionary is the `hash type` and the ``value`` is the `hex` representation of the
hash value. See the :class:`dxltieclient.constants.HashType` class for the list of `hash type`
constants.
:param filename: A file name to associate with the file (optional)
:param comment: A comment to associate with the file (optional)
"""
# Create the request message
req = Request(TIE_SET_FILE_REPUTATION_TOPIC)
# Create a dictionary for the payload
payload_dict = {
"trustLevel": trust_level,
"providerId": FileProvider.ENTERPRISE,
"filename": filename,
"comment": comment,
"hashes": []}
for key, value in hashes.items():
payload_dict["hashes"].append({"type": key, "value": base64.b64encode(value.decode('hex'))})
# Set the payload
req.payload = json.dumps(payload_dict).encode(encoding="UTF-8")
# Send the request
self.__dxl_sync_request(req)
def test_async_flood(self):
channel = UuidGenerator.generate_id_as_string()
with self.create_client() as client:
self.m_info = ServiceRegistrationInfo(client, channel)
client.connect()
client.subscribe(channel)
def my_request_callback(request):
try:
time.sleep(0.05)
resp = Response(request)
resp.payload = request.payload
client.send_response(resp)
except Exception as ex: # pylint: disable=broad-except
print(ex)
req_callback = RequestCallback()
req_callback.on_request = my_request_callback
self.m_info.add_topic(channel, req_callback)
client.register_service_sync(self.m_info, 10)
with self.create_client() as client2:
client2.connect()
def my_response_callback(response):
if response.message_type == Message.MESSAGE_TYPE_ERROR:
print("Received error response: " +
response._error_response)
with self.resp_condition:
self.error_count += 1
self.resp_condition.notify_all()
else:
with self.resp_condition:
if self.response_count % 10 == 0:
print("Received request " +
str(self.response_count))
self.response_count += 1
self.resp_condition.notify_all()
callback = ResponseCallback()
callback.on_response = my_response_callback
client2.add_response_callback("", callback)
for i in range(0, self.REQUEST_COUNT):
if i % 100 == 0:
print("Sent: " + str(i))
request = Request(channel)
request.payload = str(i)
client2.async_request(request)
if self.error_count > 0:
break
# Wait for all responses, an error to occur, or we timeout
start_time = time.time()
with self.resp_condition:
while (self.response_count != self.REQUEST_COUNT) and (
self.error_count == 0) and (
time.time() - start_time < self.WAIT_TIME):
self.resp_condition.wait(5)
if self.error_count != 0:
raise Exception("Received an error response!")
self.assertEqual(self.REQUEST_COUNT, self.response_count,
"Did not receive all messages!")
def _mcafee_publish_to_dxl_function(self, event, *args, **kwargs):
"""Function: A function which takes 3 inputs:
mcafee_topic_name: String of the topic name. ie: /mcafee/service/epo/remote/epo1.
mcafee_dxl_payload: The text of the payload to publish to the topic.
mcafee_return_request: Specify whether or not to wait for and return the response.
The function will publish the provided payload to the provided topic.
Indicate whether acknowledgment response should be returned."""
try:
yield StatusMessage("Starting...")
# Get the function parameters:
mcafee_topic_name = kwargs.get("mcafee_topic_name") # text
if not mcafee_topic_name:
yield FunctionError("mcafee_topic_name is required")
mcafee_dxl_payload = kwargs.get("mcafee_dxl_payload") # text
if not mcafee_dxl_payload:
yield FunctionError("mcafee_dxl_payload is required")
mcafee_publish_method = self.get_select_param(
kwargs.get("mcafee_publish_method")
) # select, values: "Event", "Service"
if not mcafee_publish_method:
yield FunctionError("mcafee_publish_method is required")
mcafee_wait_for_response = self.get_select_param(
kwargs.get(
"mcafee_wait_for_response")) # select, values: "Yes", "No"
log.info("mcafee_topic_name: %s", mcafee_topic_name)
log.info("mcafee_dxl_payload: %s", mcafee_dxl_payload)
log.info("mcafee_publish_method: %s", mcafee_publish_method)
log.info("mcafee_wait_for_response: %s", mcafee_wait_for_response)
response = None
# Publish Event
if mcafee_publish_method == "Event":
event = Event(mcafee_topic_name)
event.payload = mcafee_dxl_payload
yield StatusMessage("Publishing Event...")
self.client.send_event(event)
# Invoke Service
else:
req = Request(mcafee_topic_name)
req.payload = mcafee_dxl_payload
yield StatusMessage("Invoking Service...")
if mcafee_wait_for_response == "No":
self.client.async_request(req)
else:
response = Response(
self.client.sync_request(req, timeout=300))
yield StatusMessage("Done...")
r = {
"mcafee_topic_name": mcafee_topic_name,
"mcafee_dxl_payload": mcafee_dxl_payload,
"mcafee_publish_method": mcafee_publish_method,
"mcafee_wait_for_response": mcafee_wait_for_response
}
# Return response from publishing to topic
if response is not None:
r["response"] = vars(response)
yield FunctionResult(r)
else:
yield FunctionResult(r)
except Exception as e:
yield FunctionError(e)
def get_subs_count(topic):
req = Request("/mcafee/service/dxl/broker/subs")
req.payload = "{\"topic\":\"" + topic + "\"}"
resp = client.sync_request(req, 5)
return json.loads(
resp.payload.decode("utf8").rstrip("\0"))["count"]
def get_certificate_first_references(self, sha1, public_key_sha1=None, query_limit=500):
"""
Retrieves the set of systems which have referenced the specified certificate (as
identified by hashes).
**Example Usage**
.. code-block:: python
# Get the list of systems that have referenced the certificate
system_list = \\
tie_client.get_certificate_first_references(
"6EAE26DB8C13182A7947982991B4321732CC3DE2",
public_key_sha1="3B87A2D6F39770160364B79A152FCC73BAE27ADF")
**Systems**
The systems that have referenced the certificate are returned as a Python ``list``.
An example ``list`` is shown below:
.. code-block:: python
[
{
"agentGuid": "{3a6f574a-3e6f-436d-acd4-bcde336b054d}",
"date": 1475873692
},
{
"agentGuid": "{68125cd6-a5d8-11e6-348e-000c29663178}",
"date": 1478626172
}
]
Each entry in the ``list`` is a ``dict`` (dictionary) containing details about a system that has
referenced the certificate. See the :class:`dxltieclient.constants.FirstRefProp` constants class for details
about the information that is available for each system in the ``dict`` (dictionary).
:param sha1: The SHA-1 of the certificate
:param public_key_sha1: The SHA-1 of the certificate's public key (optional)
:param query_limit: The maximum number of results to return
:return: A ``list`` containing a ``dict`` (dictionary) for each system that has referenced the certificate.
See the :class:`dxltieclient.constants.FirstRefProp` constants class for details about the information that
is available for each system in the ``dict`` (dictionary).
"""
# Create the request message
req = Request(TIE_GET_CERT_FIRST_REFS)
# Create a dictionary for the payload
payload_dict = {
"queryLimit": query_limit,
"hashes": [
{"type": "sha1", "value": base64.b64encode(sha1.decode('hex'))}
]}
# Add public key SHA-1 (if specified)
if public_key_sha1:
payload_dict["publicKeySha1"] = base64.b64encode(public_key_sha1.decode('hex'))
# Set the payload
req.payload = json.dumps(payload_dict).encode(encoding="UTF-8")
# Send the request
response = self.__dxl_sync_request(req)
resp_dict = json.loads(response.payload.decode(encoding="UTF-8"))
# Return the agents list
if "agents" in resp_dict:
return resp_dict["agents"]
else:
return []
def get_certificate_reputation(self, sha1, public_key_sha1=None):
"""
Retrieves the reputations for the specified certificate (as identified by the SHA-1 of the certificate
and optionally the SHA-1 of the certificate's public key)
While the SHA-1 of the certificate is required, passing the optional SHA-1 of the certificate's public key
can result in additional reputations being located across the set of `certificate reputation providers`.
**Example Usage**
.. code-block:: python
# Determine reputations for certificate (identified by hashes)
reputations_dict = \\
tie_client.get_certificate_reputation(
"6EAE26DB8C13182A7947982991B4321732CC3DE2",
public_key_sha1="3B87A2D6F39770160364B79A152FCC73BAE27ADF")
**Reputations**
The `Reputations` for the certificate are returned as a Python ``dict`` (dictionary).
The `key` for each entry in the ``dict`` (dictionary) corresponds to a particular `provider` of the
associated `reputation`. The list of `certificate reputation providers` can be found in the
:class:`dxltieclient.constants.CertProvider` constants class.
An example reputations ``dict`` is shown below:
.. code-block:: python
{
"2": {
"attributes": {
"2108821": "92",
"2109077": "1454912619",
"2117524": "0",
"2120596": "0"
},
"createDate": 1476318514,
"providerId": 2,
"trustLevel": 99
},
"4": {
"attributes": {
"2109333": "4",
"2109589": "1476318514",
"2139285": "73183493944770750"
},
"createDate": 1476318514,
"providerId": 4,
"trustLevel": 0
}
}
The ``"2"`` `key` corresponds to a reputation from the "Global Threat Intelligence (GTI)" reputation
provider while the ``"4"`` `key` corresponds to a reputation from the "Enterprise" reputation provider.
Each reputation contains a standard set of properties (trust level, creation date, etc.). These
properties are listed in the :class:`dxltieclient.constants.ReputationProp` constants class.
The following example shows how to access the `trust level` property of the "Enterprise" reputation:
.. code-block:: python
trust_level = reputations_dict[CertProvider.ENTERPRISE][ReputationProp.TRUST_LEVEL]
Each reputation can also contain a provider-specific set of attributes as a Python ``dict`` (dictionary).
These attributes can be found in the :class:`dxltieclient.constants` module:
:class:`dxltieclient.constants.CertEnterpriseAttrib`
Attributes associated with the `Enterprise` reputation provider for certificates
:class:`dxltieclient.constants.CertGtiAttrib`
Attributes associated with the `Global Threat Intelligence (GTI)` reputation provider for certificates
The following example shows how to access the `prevalence` attribute from the "Enterprise" reputation:
.. code-block:: python
ent_rep = reputations_dict[CertProvider.ENTERPRISE]
ent_rep_attribs = ent_rep[ReputationProp.ATTRIBUTES]
prevalence = int(ent_rep_attribs[CertEnterpriseAttrib.PREVALENCE])
:param sha1: The SHA-1 of the certificate
:param public_key_sha1: The SHA-1 of the certificate's public key (optional)
:return: A ``dict`` (dictionary) where each `value` is a reputation from a particular `reputation provider`
which is identified by the `key`. The list of `certificate reputation providers` can be found in the
:class:`dxltieclient.constants.CertProvider` constants class.
"""
# Create the request message
req = Request(TIE_GET_CERT_REPUTATION_TOPIC)
# Create a dictionary for the payload
payload_dict = {
"hashes": [
{"type": "sha1", "value": base64.b64encode(sha1.decode('hex'))}
]}
# Add public key SHA-1 (if specified)
if public_key_sha1:
payload_dict["publicKeySha1"] = base64.b64encode(public_key_sha1.decode('hex'))
# Set the payload
req.payload = json.dumps(payload_dict).encode(encoding="UTF-8")
# Send the request
response = self.__dxl_sync_request(req)
resp_dict = json.loads(response.payload.decode(encoding="UTF-8"))
# Transform reputations to be simpler to use
if "reputations" in resp_dict:
return TieClient._transform_reputations(resp_dict["reputations"])
else:
return {}
def _create_request(self, topic):
req = Request("/mcafee/service/dxl/broker/subs")
req.payload = "{\"topic\":\"" + topic + "\"}"
return req
def get_file_first_references(self, hashes, query_limit=500):
"""
Retrieves the set of systems which have referenced (typically executed) the specified file (as
identified by hashes).
**Example Usage**
.. code-block:: python
# Get the list of systems that have referenced the file
system_list = \\
tie_client.get_file_first_references({
HashType.MD5: "f2c7bb8acc97f92e987a2d4087d021b1",
HashType.SHA1: "7eb0139d2175739b3ccb0d1110067820be6abd29",
HashType.SHA256: "142e1d688ef0568370c37187fd9f2351d7ddeda574f8bfa9b0fa4ef42db85aa2"
})
**Systems**
The systems that have referenced the file are returned as a Python ``list``.
An example ``list`` is shown below:
.. code-block:: python
[
{
"agentGuid": "{3a6f574a-3e6f-436d-acd4-bcde336b054d}",
"date": 1475873692
},
{
"agentGuid": "{68125cd6-a5d8-11e6-348e-000c29663178}",
"date": 1478626172
}
]
Each entry in the ``list`` is a ``dict`` (dictionary) containing details about a system that has
referenced the file. See the :class:`dxltieclient.constants.FirstRefProp` constants class for details
about the information that is available for each system in the ``dict`` (dictionary).
:param hashes: A ``dict`` (dictionary) of hashes that identify the file to lookup.
The ``key`` in the dictionary is the `hash type` and the ``value`` is the `hex` representation of the
hash value. See the :class:`dxltieclient.constants.HashType` class for the list of `hash type`
constants.
:param query_limit: The maximum number of results to return
:return: A ``list`` containing a ``dict`` (dictionary) for each system that has referenced the file. See the
:class:`dxltieclient.constants.FirstRefProp` constants class for details about the information that
is available for each system in the ``dict`` (dictionary).
"""
# Create the request message
req = Request(TIE_GET_FILE_FIRST_REFS)
# Create a dictionary for the payload
payload_dict = {
"queryLimit": query_limit,
"hashes": []
}
for key, value in hashes.items():
payload_dict["hashes"].append({"type": key, "value": base64.b64encode(value.decode('hex'))})
# Set the payload
req.payload = json.dumps(payload_dict).encode(encoding="UTF-8")
# Send the request
response = self.__dxl_sync_request(req)
resp_dict = json.loads(response.payload.decode(encoding="UTF-8"))
# Return the agents list
if "agents" in resp_dict:
return resp_dict["agents"]
else:
return []
def test_register_service_call_from_request_callback(self):
# While in the request callback for a service invocation, attempt to
# register a second service. Confirm that a call to the second service
# is successful. This test ensures that concurrent add/remove service
# calls and processing of incoming messages do not produce deadlocks.
with self.create_client(self.DEFAULT_RETRIES, 2) as client:
expected_second_service_response_payload = \
"Second service request okay too"
second_service_callback = RequestCallback()
def on_second_service_request(request):
response = Response(request)
response.payload = expected_second_service_response_payload
try:
client.send_response(response)
except DxlException as ex:
print("Failed to send response" + str(ex))
second_service_callback.on_request = on_second_service_request
second_service_info = ServiceRegistrationInfo(
client, "/mcafee/service/JTI2")
second_service_info.add_topic(
"/mcafee/service/JTI2/file/reputation/" +
second_service_info.service_id, second_service_callback)
def register_second_service():
client.register_service_sync(second_service_info,
self.REG_DELAY)
register_second_service_thread = threading.Thread(
target=register_second_service)
register_second_service_thread.daemon = True
# Perform the second service registration from a separate thread
# in order to ensure that locks taken by the callback and
# service managers do not produce deadlocks between the
# thread from which the service registration request is made and
# any threads on which response messages are received from the
# broker.
def on_first_service_request():
register_second_service_thread.start()
register_second_service_thread.join()
self.add_client_callbacks(client, on_first_service_request)
client.connect()
client.register_service_sync(self.info, self.REG_DELAY)
first_service_request = Request(
"/mcafee/service/JTI/file/reputation/" + self.info.service_id)
first_service_request.payload = "Test"
first_service_response = client.sync_request(
first_service_request, self.POST_OP_DELAY)
first_service_response_payload = first_service_response. \
payload.decode("utf8")
logging.info("First service response payload: %s",
first_service_response_payload)
self.assertEqual("Ok", first_service_response_payload)
second_service_request = Request(
"/mcafee/service/JTI2/file/reputation/" +
second_service_info.service_id)
second_service_request.payload = "Test"
second_service_response = client.sync_request(
second_service_request, self.POST_OP_DELAY)
actual_second_service_response_payload = second_service_response. \
payload.decode("utf8")
logging.info("Second service response payload: %s",
actual_second_service_response_payload)
self.assertEqual(expected_second_service_response_payload,
actual_second_service_response_payload)
def get_file_reputation(self, hashes):
"""
Retrieves the reputations for the specified file (as identified by hashes)
At least one `hash value` of a particular `hash type` (MD5, SHA-1, etc.) must be specified.
Passing additional hashes increases the likelihood of other reputations being located across the
set of `file reputation providers`.
**Example Usage**
.. code-block:: python
# Determine reputations for file (identified by hashes)
reputations_dict = \\
tie_client.get_file_reputation({
HashType.MD5: "f2c7bb8acc97f92e987a2d4087d021b1",
HashType.SHA1: "7eb0139d2175739b3ccb0d1110067820be6abd29",
HashType.SHA256: "142e1d688ef0568370c37187fd9f2351d7ddeda574f8bfa9b0fa4ef42db85aa2"
})
**Reputations**
The `Reputations` for the file are returned as a Python ``dict`` (dictionary).
The `key` for each entry in the ``dict`` (dictionary) corresponds to a particular `provider` of the
associated `reputation`. The list of `file reputation providers` can be found in the
:class:`dxltieclient.constants.FileProvider` constants class.
An example reputations ``dict`` is shown below:
.. code-block:: python
{
"1": {
"attributes": {
"2120340": "2139160704"
},
"createDate": 1480455704,
"providerId": 1,
"trustLevel": 99
},
"3": {
"attributes": {
"2101652": "52",
"2102165": "1476902802",
"2111893": "56",
"2114965": "1",
"2139285": "73183493944770750"
},
"createDate": 1476902802,
"providerId": 3,
"trustLevel": 99
}
}
The ``"1"`` `key` corresponds to a reputation from the "Global Threat Intelligence (GTI)" reputation
provider while the ``"3"`` `key` corresponds to a reputation from the "Enterprise" reputation provider.
Each reputation contains a standard set of properties (trust level, creation date, etc.). These
properties are listed in the :class:`dxltieclient.constants.ReputationProp` constants class.
The following example shows how to access the `trust level` property of the "Enterprise" reputation:
.. code-block:: python
trust_level = reputations_dict[FileProvider.ENTERPRISE][ReputationProp.TRUST_LEVEL]
Each reputation can also contain a provider-specific set of attributes as a Python ``dict`` (dictionary).
These attributes can be found in the :class:`dxltieclient.constants` module:
:class:`dxltieclient.constants.FileEnterpriseAttrib`
Attributes associated with the `Enterprise` reputation provider for files
:class:`dxltieclient.constants.FileGtiAttrib`
Attributes associated with the `Global Threat Intelligence (GTI)` reputation provider for files
:class:`dxltieclient.constants.AtdAttrib`
Attributes associated with the `Advanced Threat Defense (ATD)` reputation provider
The following example shows how to access the `prevalence` attribute from the "Enterprise" reputation:
.. code-block:: python
ent_rep = reputations_dict[FileProvider.ENTERPRISE]
ent_rep_attribs = ent_rep[ReputationProp.ATTRIBUTES]
prevalence = int(ent_rep_attribs[FileEnterpriseAttrib.PREVALENCE])
:param hashes: A ``dict`` (dictionary) of hashes that identify the file to retrieve the reputations for.
The ``key`` in the dictionary is the `hash type` and the ``value`` is the `hex` representation of the
hash value. See the :class:`dxltieclient.constants.HashType` class for the list of `hash type`
constants.
:return: A ``dict`` (dictionary) where each `value` is a reputation from a particular `reputation provider`
which is identified by the `key`. The list of `file reputation providers` can be found in the
:class:`dxltieclient.constants.FileProvider` constants class.
"""
# Create the request message
req = Request(TIE_GET_FILE_REPUTATION_TOPIC)
# Create a dictionary for the payload
payload_dict = {"hashes": []}
for key, value in hashes.items():
payload_dict["hashes"].append({"type": key, "value": base64.b64encode(value.decode('hex'))})
# Set the payload
req.payload = json.dumps(payload_dict).encode(encoding="UTF-8")
# Send the request
response = self.__dxl_sync_request(req)
resp_dict = json.loads(response.payload.decode(encoding="UTF-8"))
# Transform reputations to be simpler to use
if "reputations" in resp_dict:
return TieClient._transform_reputations(resp_dict["reputations"])
else:
return {}
