def _post_network_request(event_tag: EventTag, url: str, data: str):
details_str = 'TA SENDING POST ' + url.replace(_BASE_URL, '') + ' with '
if event_tag.event_type == EventType.STATUS:
if url.startswith(URL_TH_ACTION):
details_str += '/action/' + url[url.rfind('/') + 1:]
send_data = data
elif event_tag == EventTags.THSubmitReady:
details_str += json.dumps(json.loads(data),
indent=4,
separators=(',', ': '))
send_data = data
else:
send_data = data
details_str += 'STATUS: ' + json.loads(data)['STATUS']
elif event_tag.event_type == EventType.ERROR:
details_str += 'ERROR: ' + json.loads(data)['ERROR']
send_data = json.dumps(data)
else:
raise Exception(
'Unknown combination of routable data submitted for network sending! Details: '
'url=' + url + ', eventTag=' + str(event_tag))
# TODO: objectify after comparison with prior build
ig.get_event_router().submit(EventTags.NetworkSentPost, details_str)
response = requests.post(url=url, headers=_JSON_HEADERS, data=send_data)
ig.get_event_router().submit(
EventTags.NetworkReceivedPostResponse,
'TA RECEIVED ACK: ' + str(response.status_code))
def set_adaptation_result(self, result: AdaptationResult):
with self._lock:
if result.adaptationStatusValue == 'SUCCESSFUL':
self._test_adapter_state.adaptation.adaptationStatus = Status.SUCCESS
elif result.adaptationStatusValue == 'UNSUCCESSFUL':
self._test_adapter_state.adaptation.adaptationStatus = Status.FAILURE
elif result.adaptationStatusValue == 'ERROR':
raise Exception('Error executing the adaptation on the das! details: ' + result.to_json_str_pretty())
else:
raise Exception('Unexpected adaptationStatusValue returned!' + result.to_json_str_pretty())
self._test_adapter_state.adaptation.details = result
self._das_not_finished = False
if self._test_adapter_state.adaptation.adaptationStatus == Status.SUCCESS:
ig.get_event_router().submit(event_tag=EventTags.THStatusAdaptationCompleted,
data=self._test_adapter_state)
elif self._test_adapter_state.adaptation.adaptationStatus == Status.FAILURE:
ig.get_event_router().submit(event_tag=EventTags.THStatusMissionAborted,
data=self._test_adapter_state)
self._attempt_finish_submission()
def start(self):
with self._lock:
ig.get_event_router().subscribe_listener(EventType.ANALYSIS,
self.receive_event)
self._current_result: TestResult = TestResult(
validatorIdentifier=self.identifier(),
currentState=Status.RUNNING,
errorMessages=list(),
detailMessages=list())
def _receive_event(self, event_tag: EventTag, data: AnalyticsEvent):
if isinstance(data, list):
data_list = data
else:
data_list = [data]
for event in data_list:
if self._current_result.currentState == Status.RUNNING:
if event_tag == EventTags.AnalyticsEventServerNetworkTrafficMeasuredBytes:
bandwidth_calculated_event = None
if len(self._offline_clients) == 0:
self._timestamp_list.append(event.eventTime)
self._byte_list.append(int(event.data))
logging.debug("Bandwidth Timestamp: " + str(event.eventTime))
logging.debug("Bandwidth Total Bytes: " + str(event.data))
if self._upper_idx < len(self._timestamp_list):
time_delta = self._timestamp_list[self._upper_idx] - self._timestamp_list[
self._lower_idx]
bytes_delta = self._byte_list[self._upper_idx] - self._byte_list[self._lower_idx]
logging.debug("Bandwith Window Bytes Delta: " + str(bytes_delta))
logging.debug('Bandwidth Window Time Delta: ' + str(time_delta))
bandwidth_kilobits_per_second = int((bytes_delta * 8 / 1000) / (time_delta / 1000))
logging.debug('Bandwidth Kilobits Per Second: ' + str(bandwidth_kilobits_per_second))
self._max_hit_bandwidth_kilobits_per_second = \
max(self._max_hit_bandwidth_kilobits_per_second,
bandwidth_kilobits_per_second)
if bandwidth_kilobits_per_second >= self._maximum_bandwidth_kbits_per_second:
self._current_result.errorMessages.append(
str(bandwidth_kilobits_per_second)
+ ' is greater than the maximum bandwidth of '
+ str(self._maximum_bandwidth_kbits_per_second) + '!')
self._lower_idx += 1
self._upper_idx += 1
bandwidth_calculated_event = _create_bandwidth_calculated_event(
(bandwidth_kilobits_per_second / 8 * 1000),
event_time_ms=event.eventTime)
if bandwidth_calculated_event is not None:
ig.get_event_router().submit_asynchronously(
EventTags.AnalyticsEventServerNetworkTrafficCalculatedBytesPerSec,
bandwidth_calculated_event)
elif event_tag.event_type == EventType.ANALYSIS and event.eventSource in self._offline_clients:
self._offline_clients.remove(event.eventSource)
def attempt_validation(self, terminal_state: bool):
with self._lock:
result = self._attempt_validation(terminal_state=terminal_state)
if terminal_state:
ig.get_event_router().unsubscribe_listener(
EventType.ANALYSIS, self.receive_event)
results = ValidationResults(testDurationMS=(time.time() * 1000) -
self._startTimeMS,
results=[result])
ig.get_event_router().submit(EventTags.ValidationTestResultsProduced,
results)
def start(self):
ig.get_event_router().subscribe_listener(
EventTags.ValidationTestResultsProduced, self.receive_results)
if not os.path.exists(self._runner_configuration.deploymentDirectory +
'results/'):
os.mkdir(self._runner_configuration.deploymentDirectory +
'results/')
self._java_validator.start()
self._python_validator.start()
self._start_time_ms = time.time() * 1000
def perform_validation(state_manager: StateManager):
state_manager.set_scenario_runner_running(True)
src = triples_helper.load_phase01_scenario_runner_configuration(state_manager.get_deployment_model())
ig.get_event_router().archive_to_file(
str_to_write=src.to_json_str_pretty(include_metadata=False),
target_subpath=src.sessionIdentifier + '-scenariorunnerconfiguration.json',
clobber_existing=True)
s_runner = ScenarioRunner(runner_configuration=src, deployment_model=state_manager.get_deployment_model())
s_runner.execute_scenario()
state_manager.set_scenario_runner_running(False)
def adapt_and_validate_application(self):
body = bottle.request.body.read()
if isinstance(body, bytes):
body = body.decode()
print('##' + body)
ig.get_event_router().submit(EventTags.NetworkAcknowledgedPost,
'TA RECEIVED POST /action/adaptAndValidateApplication with BODY: ' +
prettify_body(body))
error_string, return_val = self._sm.initialize_execution(bottle.request.body.read(), True, 'validation')
if error_string is not None:
ig.get_event_router().submit(EventTags.THErrorGeneral, error_string)
ig.get_event_router().submit(EventTags.NetworkAcknowledgedPost,
'TA SENDING ACK /action/adaptAndValidateApplication of 400 with body: ' +
error_string)
return bottle.HTTPResponse(
status=400,
body=error_string
)
else:
ig.get_event_router().submit(EventTags.NetworkAcknowledgedPost,
'TA SENDING ACK /action/adaptAndValidateApplication with BODY: ' +
return_val)
return return_val
def start(self):
ig.get_event_router().submit(EventTags.ValidationStarted,
ValidationStartReturnData(
expectedDurationSeconds=self.get_duration(),
validatorIdentifiers=[k.identifier for k in self._validators]))
self._start_time_ms = time.time() * 1000
for v in list(self._running_pending_validators.values()):
v.start()
for m in list(self._monitors.values()):
m.start(duration_ms=self.get_duration())
tpr.start_timer(duration_seconds=self.get_duration() / 1000, shutdown_method=self._shutdown)
def perform_adaptation(state_manager: StateManager):
ig.get_event_router().submit(event_tag=EventTags.THStatusAdapting,
data=state_manager.get_test_adapter_state())
# TODO: This should be done outside of the FredScript, but requires the DAS supporting JSON triples
execute_ttl_generation(state_manager.get_ll_p1_input())
deployment_file = ph(True, get_configuration().immortalsRoot, 'models/scenario/deployment_model.ttl')
ig.get_event_router().archive_file(deployment_file)
payload = open(deployment_file, 'rb').read()
headers = {'Content-Type': 'text/plain'}
req = requests.post('http://localhost:8080/bbn/das/deployment-model', headers=headers, data=payload)
ar = AdaptationResult.from_dict(json.loads(req.text))
state_manager.set_adaptation_result(ar)
def __init__(self, host='127.0.0.1', port=8080, **config):
super(ImmortalsBokehBottleServer, self).__init__(host, port, **config)
if get_configuration().visualization.enabled:
self._io_loop = IOLoop.instance()
self._bokeh_server = VisualizationService(
io_loop=self._io_loop, event_router=ig.get_event_router())
else:
self._io_loop_thread = None
self._bokeh_server = None
def _attempt_finish_submission(self):
return_state = None
with self._lock:
if self._test_adapter_state is not None and not self._das_not_finished \
and not self._scenario_runner_not_finished \
and self._test_adapter_state.adaptation.adaptationStatus != Status.PENDING \
and self._test_adapter_state.validation.overallIntentStatus != Status.PENDING:
self._test_adapter_state.rawLogData = self._raw_events
return_state = self._test_adapter_state
self._das_not_finished = False
self._scenario_runner_not_finished = False
self._deployment_model = None
self._ll_p1_input = None
self._test_adapter_state = None
self._scenario_template_tag = None
self._raw_events = []
if return_state is not None:
ig.get_event_router().submit(event_tag=EventTags.THSubmitDone,
data=return_state)
def receive_results(self, event_tag: EventTag, results: ValidationResults):
for result in results.results:
self._validator_results.append(result)
if len(self._validator_results) == len(
self._runner_configuration.scenario.validatorIdentifiers):
ig.get_event_router().unsubscribe_listener(
EventTags.ValidationTestResultsProduced, self.receive_results)
v = ValidationResults((time.time() * 1000) - self._start_time_ms,
self._validator_results)
with open(
self._runner_configuration.deploymentDirectory +
'results/evaluation_result.json', 'w') as f:
json.dump(v.to_dict(include_metadata=False), f)
validation_state = _process_results(
test_results_list=self._validator_results, gif=self._gif)
ig.get_event_router().submit(EventTags.ValidationTestsFinished,
validation_state)
self._done_event.set()
def _result_listener(self, results: ValidationResults):
with self._lock:
ig.get_event_router().submit(EventTags.ValidationTestResultsProduced, results)
self._is_running = False
def __init__(self):
ig.get_olympus().route_add(path='/action/adaptAndValidateApplication', method='POST',
callback=self.adapt_and_validate_application)
ig.get_olympus().route_add(path='/action/validateBaselineApplication', method='POST',
callback=self.validate_baseline_application)
ig.get_event_router().subscribe_listener(event_tag_or_type=EventType.ERROR, listener=th_client.process_error)
ig.get_event_router().subscribe_listener(event_tag_or_type=EventType.STATUS, listener=th_client.process_status)
ig.get_event_router().set_log_events_to_file(event_tag_or_type=EventType.ERROR,
filepath=get_configuration().logFile,
transformer=th_client.THErrorStringTransformer)
ig.get_event_router().subscribe_listener(event_tag_or_type=EventTags.ValidationTestsFinished,
listener=self.receive_validation_finished)
if get_configuration().testAdapter.reportRawData:
ig.get_event_router().subscribe_listener(
event_tag_or_type=EventTags.AnalyticsEventServerNetworkTrafficMeasuredBytes,
listener=self._raw_event_appender)
ig.get_event_router().subscribe_listener(
event_tag_or_type=EventTags.AnalyticsEventServerNetworkTrafficCalculatedBytesPerSec,
listener=self._raw_event_appender)
self._sm = StateManager()
def initialize_execution(self, initialization_data: str, perform_adaptation: bool,
scenario_template_tag: str) -> Tuple[str, None] or Tuple[None, str]:
with self._lock:
if not self.ready_for_submission():
return ('A scenario with the identifier "' +
self._deployment_model.sessionIdentifier + ' is already running!')
err_msg, dm, llp1input = _parse_deployment_model(initialization_data, True)
if err_msg is not None:
return err_msg, None
self._das_not_finished = True
self._scenario_runner_not_finished = True
self._deployment_model = dm
self._ll_p1_input = llp1input
self._test_adapter_state: TestAdapterState = TestAdapterState(
identifier=self._deployment_model.sessionIdentifier,
adaptation=AdaptationState(
adaptationStatus=Status.PENDING if perform_adaptation else Status.NOT_APPLICABLE,
details=None
),
validation=ValidationState(
executedTests=None,
overallIntentStatus=Status.PENDING
),
rawLogData=[]
)
self._scenario_template_tag = scenario_template_tag
ig.get_event_router().submit(EventTags.DeploymentModelLoaded, data=self._deployment_model)
ig.get_event_router().submit(event_tag=EventTags.THStatusDasInfo,
data=self._test_adapter_state)
return_val = LLTestActionResult(RESULT=self._test_adapter_state)
if perform_adaptation:
ig.get_event_router().submit(event_tag=EventTags.THStatusPerturbationDetected,
data=self._test_adapter_state)
tpr.start_thread(
thread_method=LLRestEndpoint.perform_adaptation_and_validation,
thread_args=[self]
)
else:
self._das_not_finished = False
caid = CreateApplicationInstanceData(
sessionIdentifier=self._ll_p1_input.sessionIdentifier,
applicationType=ApplicationType.Client_ATAKLite
)
websocket.get_das_bridge().createApplicationInstance(caid)
# copy the existing app to where it belongs!
tpr.start_thread(
thread_method=LLRestEndpoint.perform_validation,
thread_args=[self]
)
return None, return_val.to_json_str(include_metadata=False)
def olympus_main(zargs=None):
from pymmortals.olympus import Olympus
from pymmortals import immortalsglobals
olympus = Olympus(immortalsglobals.get_event_router())
olympus.start()
