def __init__(self):
ig.get_olympus().route_add(path='/action/adaptAndValidateApplication', method='POST',
callback=self.adapt_and_validate_application)
ig.get_olympus().route_add(path='/action/validateBaselineApplication', method='POST',
callback=self.validate_baseline_application)
ig.get_event_router().subscribe_listener(event_tag_or_type=EventType.ERROR, listener=th_client.process_error)
ig.get_event_router().subscribe_listener(event_tag_or_type=EventType.STATUS, listener=th_client.process_status)
ig.get_event_router().set_log_events_to_file(event_tag_or_type=EventType.ERROR,
filepath=get_configuration().logFile,
transformer=th_client.THErrorStringTransformer)
ig.get_event_router().subscribe_listener(event_tag_or_type=EventTags.ValidationTestsFinished,
listener=self.receive_validation_finished)
if get_configuration().testAdapter.reportRawData:
ig.get_event_router().subscribe_listener(
event_tag_or_type=EventTags.AnalyticsEventServerNetworkTrafficMeasuredBytes,
listener=self._raw_event_appender)
ig.get_event_router().subscribe_listener(
event_tag_or_type=EventTags.AnalyticsEventServerNetworkTrafficCalculatedBytesPerSec,
listener=self._raw_event_appender)
self._sm = StateManager()
def __init__(self, event_router):
super(Olympus, self).__init__()
self.event_router = event_router
self._define_routes()
self._host = get_configuration().testAdapter.url
self._port = get_configuration().testAdapter.port
self._analytics_event_listeners = set()
self._validation_results_listeners = set()
self._das_status_message_listeners = set()
self.catchall = False
def archive_file(self, source_filepath: str, target_subpath: str = None):
artifact_root = get_configuration().artifactRoot
source = path_helper(True, get_configuration().immortalsRoot, source_filepath)
if target_subpath is None:
target = path_helper(False, artifact_root, source_filepath[source_filepath.rfind('/') + 1:])
else:
target_path = os.path.join(artifact_root, target_subpath)
if not os.path.exists(target_path):
os.makedirs(target_path)
target = path_helper(False,
artifact_root,
os.path.join(target_path, source_filepath[source_filepath.rfind('/') + 1:]))
shutil.copyfile(source, target)
def _start_das(self):
# target_override_filepath = _construct_override_file(test_scenario=test_scenario, overrides=overrides)
if self.immortals_root is None:
immortals_root = os.path.join('../')
else:
immortals_root = os.path.abspath(self.immortals_root)
print(immortals_root)
self.das_stdout = open(
os.path.join(
get_configuration().globals.globalLogDirectory,
self._current_test_scenario.scenarioIdentifier +
'-start-stdout.txt'), 'a')
self.das_stderr = open(
os.path.join(
get_configuration().globals.globalLogDirectory,
self._current_test_scenario.scenarioIdentifier +
'-start-stderr.txt'), 'a')
try:
self.das_process = Popen([
'bash',
os.path.join(
root_configuration.get_configuration().immortalsRoot,
'das/start.sh'), "-v", "DEBUG"
],
cwd=os.path.abspath(
os.path.join(immortals_root,
'harness')),
stdin=PIPE,
stderr=self.das_stderr,
stdout=self.das_stdout)
except ResourceWarning:
# Ignore this...
pass
def execute_ttl_generation(scenario_configuration, output_file=None):
client = scenario_configuration.clients[0]
res = client.presentResources
params = [
'python3', './py/mission_perturb.py', '--session',
scenario_configuration.sessionIdentifier, '--output',
'deployment_model.ttl', '--template', './template/gme-template.ttl',
'--pli-client-msg-rate',
str(60000 / int(client.latestSABroadcastIntervalMS)),
'--image-client-msg-rate',
str(60000 / int(client.imageBroadcastIntervalMS)),
'--server-bandwidth',
str(scenario_configuration.server.bandwidth), '--client-device-count',
str(client.count), '--android-bluetooth-resource',
'yes' if 'bluetooth' in res else 'no', '--android-usb-resource',
'yes' if 'usb' in res else 'no', '--android-internal-gps-resource',
'yes' if 'internalGps' in res else 'no', '--android-ui-resource',
'yes' if 'userInterface' in res else 'no', '--gps-satellite-resource',
'yes' if 'gpsSatellites' in res else 'no', '--mission-trusted-comms',
'yes' if 'trustedLocations' in client.requiredProperties else 'no'
]
if output_file is not None:
params.append('--output')
params.append(os.path.abspath(output_file))
tpr.global_subprocess.run(args=params,
cwd=get_configuration().immortalsRoot +
'models/scenario')
def start(self):
if get_configuration().debugMode:
logging.getLogger('tornado.access').setLevel(logging.WARNING)
else:
logging.getLogger('tornado.access').setLevel(logging.INFO)
ts = ImmortalsBokehBottleServer(host=self._host, port=self._port)
bottle.run(app=self, server=ts, debug=False)
def get_das_bridge() -> DasBridge:
global _das_bridge
with _das_bridge_init_lock:
if _das_bridge is None:
_das_bridge = _DasBridgeImpl(
target_addr='127.0.0.1',
target_port=get_configuration().dasService.websocketPort)
_das_bridge.start_in_thread()
return _das_bridge
def __init__(self, host='127.0.0.1', port=8080, **config):
super(ImmortalsBokehBottleServer, self).__init__(host, port, **config)
if get_configuration().visualization.enabled:
self._io_loop = IOLoop.instance()
self._bokeh_server = VisualizationService(
io_loop=self._io_loop, event_router=ig.get_event_router())
else:
self._io_loop_thread = None
self._bokeh_server = None
def archive_to_file(self, str_to_write: str, target_subpath: str, clobber_existing: bool = True):
target_path = os.path.join(get_configuration().artifactRoot, target_subpath)
full_target_path = target_path[:target_path.rfind('/')]
if not os.path.exists(full_target_path):
os.makedirs(full_target_path)
f = open(target_path, 'w' if clobber_existing else 'a')
f.write(str_to_write)
f.flush()
f.close()
def _submit(self, event_tag: EventTag, data: object, recurse_errors: bool):
with self._lock:
if get_configuration().debug.routing:
if isinstance(data, Serializable):
msg = 'EVENT ' + event_tag.identifier + ' submitted with data: \n' + data.to_json_str_pretty()
elif isinstance(data, dict):
msg = 'EVENT ' + event_tag.identifier + ' submitted with data: \n' + json.dumps(data,
indent=4,
separators=(
',', ': '))
else:
msg = 'EVENT ' + event_tag.identifier + ' submitted with data: ' + str(data)
logging.debug(msg)
submission_listeners = self._event_tag_submission_listeners
# Errors need to bubble up somehow at all cost, so care must be taken to make sure all listeners execute
if event_tag.event_type == EventType.ERROR:
try:
for l in submission_listeners[event_tag]:
try:
l(event_tag, data)
_FileLogger.flush()
except Exception:
if recurse_errors:
try:
self._submit(event_tag=EventTags.THErrorGeneral,
data=traceback.format_exc(),
recurse_errors=False)
except:
pass
except:
pass
else:
if event_tag in submission_listeners:
for l in submission_listeners[event_tag]:
if get_configuration().debug.routing:
logging.debug(' SENDING TO ' + str(l))
l(event_tag, data)
logging.debug(' ' + str(l) + ' Finished')
else:
l(event_tag, data)
def __init__(self, command_processor: 'ImmortalsSubprocess' or None,
log_tag: str, log_dirpath: str = None):
self.__command_processor = command_processor if command_processor is not None else subprocess
if log_dirpath is None:
log_dirpath = get_configuration().artifactRoot
endpoint = get_std_endpoint(log_dirpath=log_dirpath, file_tag=log_tag)
self.__err = endpoint.err
self.__out = endpoint.out
def __init__(self, gif: GmeInterchangeFormat, runner_configuration: ScenarioRunnerConfiguration):
super().__init__(gif=gif, runner_configuration=runner_configuration)
self._offline_clients: List[str] = [a.instanceIdentifier
for a in runner_configuration.scenario.deploymentApplications
if a.applicationIdentifier.lower() == 'ataklite']
self._timestamp_list: List[int] = []
self._byte_list: List[int] = []
self._max_hit_bandwidth_kilobits_per_second: int = -1
validation_reporting_interval_secs = \
get_configuration().validation.bandwidthMonitorReportingIntervalMS / 1000
self._sample_span_seconds: int = 0
client_count = triples_helper.get_android_client_count(gif)
image_broadcast_interval_ms = triples_helper.get_image_rate_ms(gif)
pli_broadcast_interval_ms = triples_helper.get_pli_rate_ms(gif)
for q in range(client_count):
self._sample_span_seconds = \
max(self._sample_span_seconds, image_broadcast_interval_ms, pli_broadcast_interval_ms)
self._sample_span_seconds /= 1000
sample_delay_seconds = max(len(self._offline_clients) * 2, self._sample_span_seconds // 2)
self._sample_span_seconds *= get_configuration().validation.bandwidthValidatorSampleDurationMultiplier
# Wait for half the sample span for things to settle before starting to measure data
self._lower_idx: int = sample_delay_seconds
self._upper_idx: int = int((self._lower_idx + (self._sample_span_seconds / validation_reporting_interval_secs)))
logging.debug('Bandwidth Sampling Starting Lower Idx: ' + str(self._lower_idx))
logging.info('Bandwidth Sampling Starting Upper Idx: ' + str(self._upper_idx))
self._maximum_bandwidth_kbits_per_second: int = triples_helper.get_bandwidth_constraint_kbit_per_second(gif)
self._image_send_count = 0
self._image_receive_count = 0
self._run_time_ms = max(60, 3 * self._sample_span_seconds)
def perform_adaptation(state_manager: StateManager):
ig.get_event_router().submit(event_tag=EventTags.THStatusAdapting,
data=state_manager.get_test_adapter_state())
# TODO: This should be done outside of the FredScript, but requires the DAS supporting JSON triples
execute_ttl_generation(state_manager.get_ll_p1_input())
deployment_file = ph(True, get_configuration().immortalsRoot, 'models/scenario/deployment_model.ttl')
ig.get_event_router().archive_file(deployment_file)
payload = open(deployment_file, 'rb').read()
headers = {'Content-Type': 'text/plain'}
req = requests.post('http://localhost:8080/bbn/das/deployment-model', headers=headers, data=payload)
ar = AdaptationResult.from_dict(json.loads(req.text))
state_manager.set_adaptation_result(ar)
def load_phase01_scenario_runner_configuration(deployment_model: GmeInterchangeFormat) -> ScenarioRunnerConfiguration:
client_count = get_android_client_count(deployment_model)
pli_rate_ms = get_pli_rate_ms(deployment_model)
image_rate_ms = get_image_rate_ms(deployment_model)
config_dict = resourcemanager.load_configuration_dict('scenario_runner_configuration')
vp = {
'sessionIdentifier': deployment_model.sessionIdentifier,
'runtimeRoot': get_configuration().runtimeRoot,
}
src: ScenarioRunnerConfiguration = ScenarioRunnerConfiguration.from_dict(d=config_dict, value_pool=vp)
vp['deploymentDirectory'] = src.deploymentDirectory
global_resources: List[Resource] = \
[k for k in deployment_model.availableResources if not isinstance(k, ExecutionPlatform)]
for resource in deployment_model.availableResources:
if isinstance(resource, ServerPlatform):
server_config: ApplicationConfig = JavaApplicationConfig.from_dict(
d=resourcemanager.load_configuration_dict('server_marti'),
value_pool=vp
)
src.scenario.deploymentApplications.append(server_config)
elif isinstance(resource, AndroidPlatform):
present_resources: List[str] = get_resource_identifiers(global_resources + resource.platformResources)
# Construct the clients and add them to the scenario runner configuration
client_j = resourcemanager.load_configuration_dict('client_ataklite')
ccid = 0
for q in range(client_count):
# client_configuration = scenario_configuration.clients[j] # type: ATAKLiteClient
i_str = str(q)
if len(i_str) == 1:
i_str = '00' + i_str
elif len(i_str) == 2:
i_str = '0' + i_str
client_vp = vp.copy()
client_vp.update({
'CCID': str(ccid),
'CID': i_str,
})
client: AndroidApplicationConfig = AndroidApplicationConfig.from_dict(client_j,
value_pool=client_vp)
for f in client.filesToCopy: # type: FileCopyConfiguration
if f.targetFilepath == '/sdcard/ataklite/ATAKLite-Config.json':
override_file_map = {k.sourceFilepath: k for k in client.configurationOverrides}
if f.sourceFilepath in override_file_map:
override_obj = override_file_map[f.sourceFilepath]
else:
override_obj = JsonFileOverrideConfiguration(f.sourceFilepath, {})
client.configurationOverrides.append(override_obj)
override_obj.overridePairs['callsign'] = client.instanceIdentifier
override_obj.overridePairs['latestSABroadcastIntervalMS'] = pli_rate_ms
override_obj.overridePairs['imageBroadcastIntervalMS'] = image_rate_ms
elif f.targetFilepath == '/sdcard/ataklite/env.json':
override_file_map = {k.sourceFilepath: k for k in client.configurationOverrides}
if f.sourceFilepath in override_file_map:
override_obj = override_file_map[f.sourceFilepath]
else:
override_obj = JsonFileOverrideConfiguration(f.sourceFilepath, {})
client.configurationOverrides.append(override_obj)
override_obj.overridePairs['availableResources'] = present_resources
src.scenario.deploymentApplications.append(client)
ccid += 1
return src
import json
import requests
import pymmortals.immortalsglobals as ig
from pymmortals.datatypes.root_configuration import get_configuration
from pymmortals.datatypes.routing import EventType, EventTag, EventTags
from pymmortals.datatypes.test_harness_api import LLTestActionDone, LLDasReady, LLDasStatusEvent, LLDasErrorEvent
from pymmortals.generated.mil.darpa.immortals.core.api.ll.phase1.testadapterstate import TestAdapterState
from pymmortals.routing.eventrouter import EventTransformer
from pymmortals.utils import get_th_timestamp
_BASE_URL = get_configuration().testHarness.protocol + get_configuration().testHarness.url + ':' + \
str(get_configuration().testHarness.port)
_JSON_HEADERS = {'Content-Type': 'application/json'}
URL_TH_ERROR = _BASE_URL + '/error'
URL_TH_READY = _BASE_URL + '/ready'
URL_TH_STATUS = _BASE_URL + '/status'
URL_TH_ACTION = _BASE_URL + '/action'
URL_TH_DONE = URL_TH_ACTION + '/done'
def _post_network_request(event_tag: EventTag, url: str, data: str):
details_str = 'TA SENDING POST ' + url.replace(_BASE_URL, '') + ' with '
if event_tag.event_type == EventType.STATUS:
if url.startswith(URL_TH_ACTION):
details_str += '/action/' + url[url.rfind('/') + 1:]
send_data = data
import logging
from threading import Thread
import bottle
from bottle import Bottle
from pymmortals import threadprocessrouter as tpr
from tornado.httpserver import HTTPServer
from tornado.ioloop import IOLoop
from tornado.wsgi import WSGIContainer
from pymmortals.datatypes.root_configuration import get_configuration
from pymmortals.generated.mil.darpa.immortals.core.api.ll.phase1.analyticsevent import AnalyticsEvent
from pymmortals.routing.eventrouter import EventRouter
from .datatypes.routing import EventTags
if get_configuration().visualization.enabled:
from pymmortals import immortalsglobals as ig
from .visualization.visualizationservice import VisualizationService
bottle.app.catchall = False
class ImmortalsBokehBottleServer(bottle.ServerAdapter):
def __init__(self, host='127.0.0.1', port=8080, **config):
super(ImmortalsBokehBottleServer, self).__init__(host, port, **config)
if get_configuration().visualization.enabled:
self._io_loop = IOLoop.instance()
self._bokeh_server = VisualizationService(
io_loop=self._io_loop, event_router=ig.get_event_router())
else:
self._io_loop_thread = None
def get_duration(self):
duration = get_configuration().validation.minimumTestDurationMS
for v in list(self._running_pending_validators.values()):
duration = max(duration, v.run_time_ms())
return duration
obj = _ThreadProcessInstance(thread=thread, shutdown_method=shutdown_method, shutdown_args=shutdown_args,
identifier=str(thread_method))
if shutdown_method is None:
thread.daemon = True
_thread_process_stack.append(obj)
thread.start()
return thread
def start_timer(duration_seconds, shutdown_method=None, shutdown_args=(), halt_on_shutdown=True):
t = Timer(
duration_seconds,
shutdown_method,
shutdown_args
)
obj = _ThreadProcessInstance(halt_on_shutdown=halt_on_shutdown, timer=t, identifier='Timer')
_thread_process_stack.append(obj)
t.start()
return t
global_subprocess = \
ImmortalsSubprocess(command_processor=subprocess, log_dirpath=get_configuration().artifactRoot, log_tag='global')
ig.add_exit_handler(exit_handler)
