def create_contexts(c):
contexts = []
cut = random.randint(1, c - 1)
# print('cut:', cut)
for i in range(c):
if i < cut: contexts.append(Context(2, [1, 0]))
else: contexts.append(Context(2, [0, 1]))
return contexts
def _on_apply_context(self, context_data):
reason = context_data['reason']
if reason == -1:
# set initial context
self._arch = context_data['arch']
self._platform = context_data['platform']
self._pointer_size = context_data['pointerSize']
self.java_available = context_data['java']
str_fmt = ('injected into := {0:d}'.format(self.pid))
self.log_event(str_fmt)
# unlock java on loads
if self.java_available:
self._app_window.hooks_panel.new_menu.addAction(
'Java class loading', self._app_window.hooks_panel._on_add_java_on_load)
elif 'context' in context_data:
context = Context(context_data['context'])
self.contexts[str(context_data['tid'])] = context
sym = ''
if 'pc' in context_data['context']:
name = context_data['ptr']
if 'symbol' in context_data['context']['pc'] and \
context_data['context']['pc']['symbol']['name'] is not None:
sym = context_data['context']['pc']['symbol']['moduleName']
sym += ' - '
sym += context_data['context']['pc']['symbol']['name']
else:
name = context_data['ptr']
if context_data['reason'] == 0:
self.log_event('hook %s %s @thread := %d' % (name, sym, context_data['tid']))
if not reason == -1 and self.context_tid == 0:
self.context_tid = context_data['tid']
def _on_apply_context(self, context_data):
if 'context' in context_data:
context = Context(context_data['context'])
self.contexts[str(context_data['tid'])] = context
sym = ''
if 'pc' in context_data['context']:
name = context_data['ptr']
if context_data['context']['pc']['symbol']['name'] is not None:
sym = context_data['context']['pc']['symbol']['moduleName']
sym += ' - '
sym += context_data['context']['pc']['symbol']['name']
else:
name = context_data['ptr']
self._app_window.threads.add_context(
context_data, library_onload=self.loading_library)
if self.loading_library is None and context_data['reason'] == 0:
self.log('hook %s %s @thread := %d' %
(name, sym, context_data['tid']))
# if len(self.contexts.keys()) > 1 and self._app_window.context_panel.have_context():
# return
else:
self._arch = context_data['arch']
self._platform = context_data['platform']
self._pointer_size = context_data['pointerSize']
self.java_available = context_data['java']
str_fmt = ('injected into := {0:d}'.format(self.pid))
self.log(str_fmt)
self.context_tid = context_data['tid']
if self._loading_library is not None:
self._loading_library = None
def _on_apply_context(self, context_data):
if 'context' in context_data:
context = Context(context_data['context'])
self.contexts[str(context_data['tid'])] = context
sym = ''
if 'pc' in context_data['context']:
name = context_data['ptr']
if 'symbol' in context_data['context']['pc'] and \
context_data['context']['pc']['symbol']['name'] is not None:
sym = context_data['context']['pc']['symbol']['moduleName']
sym += ' - '
sym += context_data['context']['pc']['symbol']['name']
else:
name = context_data['ptr']
self._app_window.threads.add_context(
context_data, library_onload=self.loading_library)
if self.loading_library is None and context_data['reason'] == 0:
self.log('hook %s %s @thread := %d' %
(name, sym, context_data['tid']))
if context_data['is_java']:
self._app_window.show_main_tab('java-explorer')
else:
self._arch = context_data['arch']
self._platform = context_data['platform']
self._pointer_size = context_data['pointerSize']
self.java_available = context_data['java']
str_fmt = ('injected into := {0:d}'.format(self.pid))
self.log(str_fmt)
# update current context tid
self.context_tid = context_data['tid']
if self._loading_library is not None:
self._loading_library = None
def main():
parser = argparse.ArgumentParser()
na = parser.add_argument_group("required named arguments")
na.add_argument("--workload-id", required=True, help="Workload ID")
na.add_argument(
"--context",
required=True,
help="S3 path to context (e.g. s3://bucket/path/to/context.json")
na.add_argument("--cache-dir",
required=True,
help="Local path for the context cache")
na.add_argument("--python-packages",
help="Resource ids of packages to build")
na.add_argument("--build",
action="store_true",
help="Flag to determine mode (build vs install)")
args, _ = parser.parse_known_args()
if args.build:
build(args)
else:
ctx = Context(s3_path=args.context,
cache_dir=args.cache_dir,
workload_id=args.workload_id)
install_packages(ctx.python_packages, ctx.bucket)
def start(args):
ctx = Context(s3_path=args.context,
cache_dir=args.cache_dir,
workload_id=args.workload_id)
package.install_packages(ctx.python_packages, ctx.bucket)
api = ctx.apis_id_map[args.api]
model = ctx.models[api["model_name"]]
tf_lib.set_logging_verbosity(ctx.environment["log_level"]["tensorflow"])
local_cache["ctx"] = ctx
local_cache["api"] = api
local_cache["model"] = model
if not os.path.isdir(args.model_dir):
aws.download_and_extract_zip(model["key"], args.model_dir, ctx.bucket)
for column_name in model["feature_columns"] + [model["target_column"]]:
if ctx.is_transformed_column(column_name):
trans_impl, _ = ctx.get_transformer_impl(column_name)
local_cache["trans_impls"][column_name] = trans_impl
transformed_column = ctx.transformed_columns[column_name]
input_args_schema = transformed_column["inputs"]["args"]
# cache aggregates and constants in memory
if input_args_schema is not None:
local_cache["transform_args_cache"][
column_name] = ctx.populate_args(input_args_schema)
channel = implementations.insecure_channel("localhost", args.tf_serve_port)
local_cache[
"stub"] = prediction_service_pb2.beta_create_PredictionService_stub(
channel)
local_cache["required_inputs"] = tf_lib.get_base_input_columns(
model["name"], ctx)
# wait a bit for tf serving to start before querying metadata
limit = 600
for i in range(limit):
try:
local_cache["metadata"] = run_get_model_metadata()
break
except Exception as e:
if i == limit - 1:
logger.exception(
"An error occurred, see `cx logs api {}` for more details."
.format(api["name"]))
sys.exit(1)
time.sleep(1)
logger.info("Serving model: {}".format(model["name"]))
serve(app, listen="*:{}".format(args.port))
def context(self, scope):
logins = self.__logins()
passwords = self.__passwords()
credentials = self.__credentials()
pipelines = self.__pipelines()
return Context(name=self.name,
scope=scope,
logins=self.__logins(),
passwords=self.__passwords(),
credentials=self.__credentials(),
pipelines=self.__pipelines(),
generators=self.__generators(),
outputs=self.outputs)
def run_job(args):
should_ingest, features_to_validate, features_to_aggregate, features_to_transform, training_datasets = parse_args(
args)
resource_id_list = (features_to_validate + features_to_aggregate +
features_to_transform + training_datasets)
try:
ctx = Context(s3_path=args.context,
cache_dir=args.cache_dir,
workload_id=args.workload_id)
except Exception as e:
logger.exception("An error occurred, see the logs for more details.")
sys.exit(1)
try:
spark = None # For the finally clause
spark = get_spark_session(ctx.workload_id)
raw_df = ingest_raw_dataset(spark, ctx, features_to_validate,
should_ingest)
if len(features_to_aggregate) > 0:
run_custom_aggregators(spark, ctx, features_to_aggregate, raw_df)
if len(features_to_transform) > 0:
validate_transformers(spark, ctx, features_to_transform, raw_df)
create_training_datasets(spark, ctx, training_datasets, raw_df)
util.log_job_finished(ctx.workload_id)
except CortexException as e:
e.wrap("error")
logger.error(str(e))
logger.exception(
"An error occurred, see `cx logs {} {}` for more details.".format(
ctx.id_map[resource_id_list[0]]["resource_type"],
ctx.id_map[resource_id_list[0]]["name"],
))
sys.exit(1)
except Exception as e:
logger.exception(
"An error occurred, see `cx logs {} {}` for more details.".format(
ctx.id_map[resource_id_list[0]]["resource_type"],
ctx.id_map[resource_id_list[0]]["name"],
))
sys.exit(1)
finally:
if spark is not None:
spark.stop()
def train(args):
ctx = Context(s3_path=args.context,
cache_dir=args.cache_dir,
workload_id=args.workload_id)
package.install_packages(ctx.python_packages, ctx.bucket)
model = ctx.models_id_map[args.model]
logger.info("Training")
with util.Tempdir(ctx.cache_dir) as temp_dir:
model_dir = os.path.join(temp_dir, "model_dir")
ctx.upload_resource_status_start(model)
try:
model_impl = ctx.get_model_impl(model["name"])
train_util.train(model["name"], model_impl, ctx, model_dir)
ctx.upload_resource_status_success(model)
logger.info("Caching")
logger.info("Caching model " + model["name"])
model_export_dir = os.path.join(model_dir, "export", "estimator")
model_zip_path = os.path.join(temp_dir, "model.zip")
util.zip_dir(model_export_dir, model_zip_path)
aws.upload_file_to_s3(local_path=model_zip_path,
key=model["key"],
bucket=ctx.bucket)
util.log_job_finished(ctx.workload_id)
except CortexException as e:
ctx.upload_resource_status_failed(model)
e.wrap("error")
logger.error(str(e))
logger.exception(
"An error occurred, see `cx logs model {}` for more details.".
format(model["name"]))
sys.exit(1)
except Exception as e:
ctx.upload_resource_status_failed(model)
logger.exception(
"An error occurred, see `cx logs model {}` for more details.".
format(model["name"]))
sys.exit(1)
def reverse_file(filename, symbol, options):
ctx = Context()
ctx.sectionsname = False
ctx.color = False
ctx.filename = filename
ctx.entry = symbol
for o in options:
if o == "--raw x86":
ctx.raw_type = "x86"
elif o == "--raw x64":
ctx.raw_type = "x64"
sio = StringIO()
with redirect_stdout(sio):
reverse(ctx)
postfix = '{0}.rev'.format('' if symbol is None else '_' + symbol)
with open(filename.replace('.bin', postfix)) as f:
assert_equal(sio.getvalue(), f.read())
def build(args):
ctx = Context(s3_path=args.context,
cache_dir=args.cache_dir,
workload_id=args.workload_id)
python_packages_list = [
ctx.pp_id_map[id] for id in args.python_packages.split(",")
]
python_packages = {
python_package["name"]: python_package
for python_package in python_packages_list
}
ctx.upload_resource_status_start(*python_packages_list)
try:
build_packages(python_packages, ctx.bucket)
util.log_job_finished(ctx.workload_id)
except Exception as e:
logger.exception(e)
ctx.upload_resource_status_failed(*python_packages_list)
else:
ctx.upload_resource_status_success(*python_packages_list)
def on_message(self, message, data):
if 'payload' not in message:
print(message)
return
what = message['payload']
parts = what.split(':::')
if len(parts) < 2:
print(what)
return
cmd = parts[0]
if cmd == 'backtrace':
if self.app.get_session_ui() is not None and self.app.get_backtrace_panel() is not None:
try:
self.app.get_backtrace_panel().set_backtrace(json.loads(parts[1]))
except:
pass
elif cmd == 'emulator':
# on a separate thread to allow js api recursion
Thread(target=self.emulator.api, args=(parts[1:],)).start()
elif cmd == 'enumerate_java_classes_start':
if self.app.get_java_classes_panel() is not None:
self.app.get_java_classes_panel().on_enumeration_start()
if self.app.get_java_trace_panel() is not None:
self.app.get_java_trace_panel().on_enumeration_start()
elif cmd == 'enumerate_java_classes_match':
if self.app.get_java_classes_panel() is not None:
self.app.get_java_classes_panel().on_enumeration_match(parts[1])
if self.app.get_java_trace_panel() is not None:
self.app.get_java_trace_panel().on_enumeration_match(parts[1])
elif cmd == 'enumerate_java_classes_complete':
self.app_window.get_menu().on_java_classes_enumeration_complete()
if self.app.get_java_classes_panel() is not None:
self.app.get_java_classes_panel().on_enumeration_complete()
if self.app.get_java_trace_panel() is not None:
self.app.get_java_trace_panel().on_enumeration_complete()
elif cmd == 'enumerate_java_methods_complete':
self.bus.emit(parts[1], json.loads(parts[2]), parts[1])
elif cmd == 'ftrace':
if self.app.get_ftrace_panel() is not None:
self.app.get_ftrace_panel().append_data(parts[1])
elif cmd == 'enable_kernel':
self.app_window.get_menu().enable_kernel_menu()
elif cmd == 'hook_java_callback':
h = Hook(Hook.HOOK_JAVA)
h.set_ptr(1)
h.set_input(parts[1])
if self.java_pending_args:
h.set_condition(self.java_pending_args['condition'])
h.set_logic(self.java_pending_args['logic'])
self.java_pending_args = None
self.java_hooks[h.get_input()] = h
self.app.get_hooks_panel().hook_java_callback(h)
elif cmd == 'hook_native_callback':
h = Hook(Hook.HOOK_NATIVE)
h.set_ptr(int(parts[1], 16))
h.set_input(self.temporary_input)
h.set_bytes(binascii.unhexlify(parts[2]))
self.temporary_input = ''
if self.native_pending_args:
h.set_condition(self.native_pending_args['condition'])
h.set_logic(self.native_pending_args['logic'])
self.native_pending_args = None
self.hooks[h.get_ptr()] = h
self.app.get_hooks_panel().hook_native_callback(h)
elif cmd == 'hook_onload_callback':
h = Hook(Hook.HOOK_ONLOAD)
h.set_ptr(0)
h.set_input(parts[1])
self.on_loads[parts[1]] = h
if self.app.session_ui is not None and self.app.get_hooks_panel() is not None:
self.app.get_hooks_panel().hook_onload_callback(h)
elif cmd == 'java_trace':
panel = self.app.get_java_trace_panel()
if panel is None:
panel = self.app.get_session_ui().add_dwarf_tab(SessionUi.TAB_JAVA_TRACE)
panel.on_event(parts[1], parts[2], parts[3])
elif cmd == 'log':
self.app.get_console_panel().get_js_console().log(parts[1])
elif cmd == 'memory_scan_match':
self.bus.emit(parts[1], parts[2], json.loads(parts[3]))
elif cmd == 'memory_scan_complete':
self.app_window.get_menu().on_bytes_search_complete()
self.bus.emit(parts[1] + ' complete', 0, 0)
elif cmd == 'onload_callback':
self.loading_library = parts[1]
self.app.get_console_panel().get_js_console().log('hook onload %s @thread := %s' % (
parts[1], parts[3]))
self.app.get_hooks_panel().hit_onload(parts[1], parts[2])
elif cmd == 'release':
if parts[1] in self.contexts:
del self.contexts[parts[1]]
self.app.on_tid_resumed(int(parts[1]))
elif cmd == 'set_context':
data = json.loads(parts[1])
if 'context' in data:
context = Context(data['context'])
self.contexts[str(data['tid'])] = context
sym = ''
if 'pc' in context.__dict__:
name = data['ptr']
if context.pc.symbol_name is not None:
sym = '(%s - %s)' % (context.pc.symbol_module_name, context.pc.symbol_name)
else:
name = data['ptr']
self.app.get_contexts_lists_panel().add_context(data, library_onload=self.loading_library)
# check if data['reason'] is 0 (REASON_HOOK)
if self.loading_library is None and data['reason'] == 0:
self.log('hook %s %s @thread := %d' % (name, sym, data['tid']))
if len(self.contexts.keys()) > 1 and self.app.get_context_panel().have_context():
return
self.app.get_session_ui().request_session_ui_focus()
else:
self.arch = data['arch']
self.pointer_size = data['pointerSize']
self.java_available = data['java']
self.app.get_console_panel().get_js_console().log('injected into := ' + str(self.pid))
self.app_window.on_context_info()
self.context_tid = data['tid']
self.app.apply_context(data)
if self.loading_library is not None:
self.loading_library = None
elif cmd == 'set_data':
key = parts[1]
if data:
self.app.get_data_panel().append_data(key, hexdump(data, result='return'))
else:
self.app.get_data_panel().append_data(key, str(parts[2]))
elif cmd == 'script_loaded':
if self._spawned:
self.device.resume(self.pid)
elif cmd == 'tracer':
panel = self.app.get_trace_panel()
if panel is None:
panel = self.app.get_session_ui().add_dwarf_tab(SessionUi.TAB_TRACE)
if panel is not None:
# safely checked later
panel.start()
trace_events_parts = parts[1].split(',')
while len(trace_events_parts) > 0:
t = TraceEvent(trace_events_parts.pop(0),
trace_events_parts.pop(0),
trace_events_parts.pop(0),
trace_events_parts.pop(0))
panel.event_queue.append(t)
elif cmd == 'unhandled_exception':
# todo
pass
elif cmd == 'update_modules':
self.app.apply_context({'tid': parts[1], 'modules': json.loads(parts[2])})
elif cmd == 'update_ranges':
self.app.apply_context({'tid': parts[1], 'ranges': json.loads(parts[2])})
elif cmd == 'watcher':
exception = json.loads(parts[1])
self.log('watcher hit op %s address %s @thread := %s' %
(exception['memory']['operation'], exception['memory']['address'], parts[2]))
elif cmd == 'watcher_added':
if self.app.get_watchers_panel() is not None:
self.app.get_watchers_panel().add_watcher_callback(parts[1])
elif cmd == 'watcher_removed':
if self.app.get_watchers_panel() is not None:
self.app.get_watchers_panel().remove_watcher_callback(parts[1])
else:
print(what)
async def handle_event(event: Dict, event_context, project_id_owner: Optional[str]):
if event_context is Dict:
context = LoggingContext(event_context.get("execution_id", None))
else:
context = LoggingContext(None)
selected_services = None
if "GCP_SERVICES" in os.environ:
selected_services_string = os.environ.get("GCP_SERVICES", "")
selected_services = selected_services_string.split(",") if selected_services_string else []
services = load_supported_services(context, selected_services)
async with aiohttp.ClientSession() as session:
setup_start_time = time.time()
token = await create_token(context, session)
if token is None:
context.log("Cannot proceed without authorization token, stopping the execution")
return
if not isinstance(token, str):
raise Exception(f"Failed to fetch access token, got non string value: {token}")
context.log("Successfully obtained access token")
if not project_id_owner:
project_id_owner = get_project_id_from_environment()
dynatrace_api_key = await fetch_dynatrace_api_key(session=session, project_id=project_id_owner, token=token)
dynatrace_url = await fetch_dynatrace_url(session=session, project_id=project_id_owner, token=token)
print_metric_ingest_input = \
"PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE"
context = Context(
session=session,
project_id_owner=project_id_owner,
token=token,
execution_time=datetime.utcnow(),
execution_interval_seconds=60 * 1,
dynatrace_api_key=dynatrace_api_key,
dynatrace_url=dynatrace_url,
print_metric_ingest_input=print_metric_ingest_input,
scheduled_execution_id=context.scheduled_execution_id
)
projects_ids = await get_all_accessible_projects(context, session)
context.setup_execution_time = (time.time() - setup_start_time)
fetch_gcp_data_start_time = time.time()
fetch_ingest_lines_tasks = [fetch_ingest_lines_task(context, project_id, services) for project_id in projects_ids]
ingest_lines_per_project = await asyncio.gather(*fetch_ingest_lines_tasks, return_exceptions=True)
ingest_lines = [ingest_line for sublist in ingest_lines_per_project for ingest_line in sublist]
context.fetch_gcp_data_execution_time = time.time() - fetch_gcp_data_start_time
context.log(f"Fetched GCP data in {context.fetch_gcp_data_execution_time} s")
await push_ingest_lines(context, ingest_lines)
await push_self_monitoring_time_series(context)
await session.close()
VERSION = "0.3"
import sys
try:
import settings
except ImportError as e:
print("Error: unable to load 'settings.py': %s" % e)
sys.exit(1)
from lib.context import Context
from lib.core import Core
from lib.webserver import Webserver
from lib.log import get_logger
import logging
if __name__ == '__main__':
get_logger().event("main", "started")
logging.info(
"Yelena, version {version} starting ...".format(version=VERSION))
context = Context()
core = Core(context=context)
webserver = Webserver(core=core, context=context)
webserver.start()
core.start()
if __name__=='__main__':
context_n = 10
arm_n = 2
ucb_alpha = 3 #alpha > 2
T = 10000
arms_mean = [
[0.6, 0.3],
[0.3, 0.6]
]
contexts = []
for i in range(context_n):
if i<int(context_n/2):
contexts.append(Context(arm_n,arms_mean[0]))
else:
contexts.append(Context(arm_n,arms_mean[1]))
X = [contexts[i].sample(T) for i in range(context_n)]
cucb = CUCB(context_n, arm_n, ucb_alpha)
ducb = DUCB(context_n, arm_n, ucb_alpha, 1000)
sc = strg1(X, context_n, cucb, arm_n, T)
sd = strg1(X, context_n, ducb, arm_n, T)
print(sc, sd)
# d = deltaDiff(arm_n, arms_mean[0])
# theory = theoryRegret(d, T, ucb_alpha)
def parse_args():
# Parse arguments
parser = ArgumentParser(
description=
'Reverse engineering for x86/ARM/MIPS binaries. Generation of pseudo-C. '
'Supported formats : ELF, PE. More commands available in the interactive'
' mode. https://github.com/joelpx/reverse')
parser.add_argument('filename', nargs='?', metavar='FILENAME')
parser.add_argument('-nc', '--nocolor', action='store_true')
parser.add_argument('-g',
'--graph',
action='store_true',
help='Generate an html flow graph. See d3/index.html.')
parser.add_argument('--nocomment',
action='store_true',
help="Don't print comments")
parser.add_argument('--noandif',
action='store_true',
help="Print normal 'if' instead of 'andif'")
parser.add_argument(
'--datasize',
type=int,
default=30,
metavar='N',
help=
'default 30, maximum of chars to display for strings or bytes array.')
parser.add_argument(
'-x',
'--entry',
metavar='SYMBOLNAME|0xXXXXX|EP',
help='Pseudo-decompilation, default is main. EP stands for entry point.'
)
parser.add_argument('--vim',
action='store_true',
help='Generate syntax colors for vim')
parser.add_argument('-s',
'--symbols',
action='store_true',
help='Print all symbols')
parser.add_argument('-c',
'--calls',
metavar='SECTION_NAME',
type=str,
help='Print all calls which are in the given section')
parser.add_argument('--sections',
action='store_true',
help='Print all sections')
parser.add_argument('--dump',
action='store_true',
help='Dump asm without decompilation')
parser.add_argument('-l',
'--lines',
type=int,
default=30,
metavar='N',
help='Max lines used with --dump')
parser.add_argument('--bytes',
action='store_true',
help='Print instruction bytes')
parser.add_argument('-i',
'--interactive',
action='store_true',
help='Interactive mode')
parser.add_argument('-d', '--opt_debug', action='store_true')
parser.add_argument('-ns', '--nosectionsname', action='store_true')
parser.add_argument('--raw',
metavar='x86|x64|arm|mips|mips64',
help='Consider the input file as a raw binary')
parser.add_argument('--rawbase',
metavar='0xXXXXX',
help='Set base address of a raw file (default=0)')
parser.add_argument('--rawbe',
action='store_true',
help='If not set it\'s in little endian')
args = parser.parse_args()
ctx = Context()
ctx.debug = args.opt_debug
ctx.print_andif = not args.noandif
ctx.color = not args.nocolor
ctx.comments = not args.nocomment
ctx.sectionsname = not args.nosectionsname
ctx.max_data_size = args.datasize
ctx.filename = args.filename
ctx.raw_type = args.raw
ctx.raw_base = args.rawbase
ctx.syms = args.symbols
ctx.calls_in_section = args.calls
ctx.entry = args.entry
ctx.dump = args.dump
ctx.vim = args.vim
ctx.interactive = args.interactive
ctx.lines = args.lines
ctx.graph = args.graph
ctx.raw_big_endian = args.rawbe
ctx.list_sections = args.sections
ctx.print_bytes = args.bytes
if ctx.raw_base is not None:
if ctx.raw_base.startswith("0x"):
ctx.raw_base = int(ctx.raw_base, 16)
else:
error("--rawbase must in hex format")
die()
else:
ctx.raw_base = 0
return ctx
def parse_args():
# Parse arguments
parser = ArgumentParser(
description=
'Reverse engineering for x86/ARM binaries. Generation of pseudo-C. '
'Supported formats : ELF, PE. https://github.com/joelpx/reverse')
parser.add_argument('filename', nargs='?', metavar='FILENAME')
parser.add_argument('-nc', '--nocolor', action='store_true')
parser.add_argument('-g',
'--graph',
action='store_true',
help='Generate an html flow graph. See d3/index.html.')
parser.add_argument('--nocomment',
action='store_true',
help="Don't print comments")
parser.add_argument('--noandif',
action='store_true',
help="Print normal 'if' instead of 'andif'")
parser.add_argument(
'--datasize',
type=int,
default=30,
metavar='N',
help=
'default 30, maximum of chars to display for strings or bytes array.')
parser.add_argument('-x',
'--entry',
metavar='SYMBOLNAME|0xXXXXX|EP',
help='default main. EP stands for entry point.')
parser.add_argument('--vim',
action='store_true',
help='Generate syntax colors for vim')
parser.add_argument('-s',
'--symbols',
action='store_true',
help='Print all symbols')
parser.add_argument(
'-c',
'--calls',
action='store_true',
help='Print all calls which are in the section containing the address '
'given with -x.')
parser.add_argument('--raw',
metavar='x86|x64|arm',
help='Consider the input file as a raw binary')
parser.add_argument('--dump',
action='store_true',
help='Dump asm without decompilation')
parser.add_argument('--lines',
type=int,
default=30,
metavar='N',
help='Max lines to dump')
parser.add_argument('-i',
'--interactive',
action='store_true',
help='Interactive mode')
parser.add_argument(
'--symfile',
metavar='FILENAME',
type=FileType('r'),
help=('Add user symbols for better readability of the analysis. '
'Line format: ADDRESS_HEXA SYMBOL_NAME'))
parser.add_argument('-d', '--opt_debug', action='store_true')
parser.add_argument('-ns', '--nosectionsname', action='store_true')
args = parser.parse_args()
ctx = Context()
ctx.debug = args.opt_debug
ctx.print_andif = not args.noandif
ctx.color = not args.nocolor
ctx.comments = not args.nocomment
ctx.sectionsname = not args.nosectionsname
ctx.max_data_size = args.datasize
ctx.filename = args.filename
ctx.raw_type = args.raw
ctx.symfile = args.symfile
ctx.syms = args.symbols
ctx.calls = args.calls
ctx.entry = args.entry
ctx.dump = args.dump
ctx.vim = args.vim
ctx.interactive = args.interactive
ctx.lines = args.lines
ctx.graph = args.graph
return ctx
async def handle_event(event: Dict, event_context, project_id_owner: Optional[str], projects_ids: Optional[List[str]] = None):
if isinstance(event_context, Dict):
context = LoggingContext(event_context.get("execution_id", None))
else:
context = LoggingContext(None)
selected_services = None
if "GCP_SERVICES" in os.environ:
selected_services_string = os.environ.get("GCP_SERVICES", "")
selected_services = selected_services_string.split(",") if selected_services_string else []
#set default featureset if featureset not present in env variable
for i, service in enumerate(selected_services):
if "/" not in service:
selected_services[i]=f"{service}/default"
services = load_supported_services(context, selected_services)
async with init_gcp_client_session() as gcp_session, init_dt_client_session() as dt_session:
setup_start_time = time.time()
token = await create_token(context, gcp_session)
if token is None:
context.log("Cannot proceed without authorization token, stopping the execution")
return
if not isinstance(token, str):
raise Exception(f"Failed to fetch access token, got non string value: {token}")
context.log("Successfully obtained access token")
if not project_id_owner:
project_id_owner = get_project_id_from_environment()
dynatrace_api_key = await fetch_dynatrace_api_key(gcp_session=gcp_session, project_id=project_id_owner, token=token)
dynatrace_url = await fetch_dynatrace_url(gcp_session=gcp_session, project_id=project_id_owner, token=token)
print_metric_ingest_input = \
"PRINT_METRIC_INGEST_INPUT" in os.environ and os.environ["PRINT_METRIC_INGEST_INPUT"].upper() == "TRUE"
context = Context(
gcp_session=gcp_session,
dt_session=dt_session,
project_id_owner=project_id_owner,
token=token,
execution_time=datetime.utcnow(),
execution_interval_seconds=60 * 1,
dynatrace_api_key=dynatrace_api_key,
dynatrace_url=dynatrace_url,
print_metric_ingest_input=print_metric_ingest_input,
scheduled_execution_id=context.scheduled_execution_id
)
if not projects_ids:
projects_ids = await get_all_accessible_projects(context, gcp_session, token)
setup_time = (time.time() - setup_start_time)
context.setup_execution_time = {project_id: setup_time for project_id in projects_ids}
context.start_processing_timestamp = time.time()
process_project_metrics_tasks = [
process_project_metrics(context, project_id, services)
for project_id
in projects_ids
]
await asyncio.gather(*process_project_metrics_tasks, return_exceptions=True)
context.log(f"Fetched and pushed GCP data in {time.time() - context.start_processing_timestamp} s")
context.log(f"Processed {sum(context.dynatrace_ingest_lines_ok_count.values())} lines")
await push_self_monitoring_time_series(context)
await gcp_session.close()
await dt_session.close()
def _get_context(d):
return Context(obj=d, cache_dir=".")
