def log_message(self, level='INFO', message=None, data=None, duration='', from_method=None, from_class=None): if type(data) is str: data = { 'str': data } # so that elastic doesn't make this field a string log_data = { "duration": duration, "from_class": from_class, "from_method": from_method, "level": level, "message": message, "data": data, self.time_field: datetime.utcnow() } # this is a python datatime object (which is well supported by elastic but doesn't serialise to json ok) if self.enabled: return self.elastic().add(data=log_data, refresh=self.refresh_index) # if elastic server is not available, log messages to console pprint(log_data)
def test_repository_create(self): pprint(self.result_create) assert self.ecr.repository_create(name=self.repository_name) == { 'message': f'repository {self.repository_name} already existed', 'status': 'warning' }
def test_query(self): query = self.py_query.query('head') assert type(query) == Py_Query assert query.outer_html() == '<head id="abc" answer="42">inside <b>head</b></head>' children = query.children() assert children[0].html() == '<b>head</b>' elements = query.elements() assert type(elements[0]) == Py_Query pprint(type(elements)) assert type(elements).mro() == [list, object] element = elements[0] assert type(element).mro() == [Py_Query, object] assert element.text() == 'inside head' assert element.attributes() == {'id': 'abc', 'answer': '42'} assert type(elements[0] ) == Py_Query assert type(element.items()[0]) is Py_Query # confirm that .items returns an Py_Query object # couple more pyquery tests assert query.set_pyquery_from_html('<form><input name="order" value="spam"></form>').serialize_array() == [{'name': 'order', 'value': 'spam'}]
def test__enter__leave__(self): image_name = 'hello-world' with Temp_ECS_Fargate_Task(image_name=image_name, delete_on_exit=True) as fargate_task: pprint(fargate_task.create()) assert 'Hello from Docker!\n' in fargate_task.logs_wait_for_data()
def test_setup_and_run_containers_in_ec2(self): container_instance = self.ecs.container_instances().pop() subnet_id = container_instance.get('attributes').get('ecs.subnet-id') security_group_id = self.ec2.security_group_default().get('GroupId') task_family = 'hello-world-in-ec2' image_name = 'hello-world' skip_if_exists = True task_definition_config = self.ecs.task_definition_setup( task_family=task_family, image_name=image_name, requires='EC2', network_mode='none') task_definition = self.ecs.task_definition_create( task_definition_config=task_definition_config, skip_if_exists=skip_if_exists) task_definition_arn = task_definition.get('taskDefinitionArn') # task_config = { "launch_type" : "EC2" , # "security_group_id" : security_group_id , # "subnet_id" : subnet_id , # "task_definition_arn" : task_definition_arn } #task = self.ecs.task_create(**task_config) pprint(task_definition_arn) task = self.ecs.task_create_ec2( task_definition_arn=task_definition_arn) task_arn = task.get('taskArn') self.ecs.wait_for_task_stopped(task_arn=task_arn) pprint( self.ecs.logs(task_arn=task_arn, task_definition_arn=task_definition_arn, image_name=image_name))
def test_metric_image(self): image_in_tmp = "/tmp/cloud_watch_metric.png" instance_id = "i-0a2a089f3dd878051" #"i-0f9b8338610fc96e7" options = { "id": "m1", "stat": "Average", "label": "Median value", "visible": True, "color": "#0000FF", "yAxis": "left", "period": 1800 } kwargs = { "namespace": "AWS/EC2", "metric_name": "CPUUtilization", "dimensions": { "InstanceId": instance_id }, "path_image_file": image_in_tmp, "options": options, "title": "Metric Title" } result = self.cloud_watch.metric_image(**kwargs) pprint(result)
def tearDown(self): if hasattr(self, 'result'): if self.result is not None: pprint(self.result) if hasattr(self, 'png_data'): if hasattr(self, 'png_data') is False: self.png_file = '/tmp/unit-test.png' self.save_png(self.png_data, self.png_file)
def test_upload_dependency(self): from osbot_aws.Dependencies import upload_dependency from osbot_aws.Dependencies import pip_install_dependency dependencies = ['websocket-client', 'syncer'] for dependency in dependencies: pip_install_dependency(dependency) result = upload_dependency(dependency) pprint(result)
def test_invoke_directly(self): assert run({}) == 'in lambda shell' auth_key = self.lambda_shell.get_lambda_shell_auth() pprint(auth_key) self.result = run( {'lambda_shell': { 'method_name': 'ping', 'auth_key': auth_key }})
def test_start_container_in_ec2__in_default_cluster(self): ecs = self.ecs task_family = 'testing-ec2' task_definition = ecs.task_definition(task_family=task_family) task_definition_arn = task_definition.get('taskDefinitionArn') task = ecs.task_create_ec2(task_definition_arn=task_definition_arn) task_arn = task.get('taskArn') ecs.wait_for_task_stopped(task_arn=task_arn) pprint(task_arn)
def test_simple_execution(self, region, account_id): event_data = {'answer': 42} queue_name = random_string(prefix="osbot_unit_test") with Temp_SQS_Queue(queue_name=queue_name) as queue: assert queue_name == queue.queue_name assert queue.exists() is True assert queue.queue_url == f'https://{region}.queue.amazonaws.com/{account_id}/{queue.queue_name}' assert queue.push(event_data).pop() == event_data pprint(queue.info())
async def open(self, path='', wait_until=None): url = urljoin(self.target_server, path) page = await self.page() try: await self.api_browser.open(url=url, page=page, wait_until=wait_until) except Exception as error: pprint(f'Error in Web_Base - open: {error}') return url
def test_update_status(self): temp_data_file = temp_file() with patch.object(Hash_Json, 'get_file_path', return_value=temp_data_file): self.hash_json.add_file(self.test_file_hash, self.test_file_name) assert self.hash_json.data()[self.test_file_hash]['file_status'] == 'Initial' self.hash_json.update_status(self.test_file_hash, 'BBBB') self.hash_json.save() assert self.hash_json.data()[self.test_file_hash]['file_status'] == 'BBBB' assert json_load_file(temp_data_file)[self.test_file_hash]['file_status'] == 'BBBB' pprint(self.hash_json.load())
def test_connect_to_ec2_instance(self): path_to_key = '/var/folders/_j/frqs70d93l328f307rw2jx5h0000gn/T/tmp8_tl8q5l/osbot-test_ec2_with_ssh_supporteyvjpbqb.pem' username = '******' server_ip = "3.251.63.86" ssh_config = {"server": server_ip, "ssh_key": path_to_key, "user": username} result = Ssh(ssh_config=ssh_config).ls('/') pprint(result)
def test_commands_list(self): instance_id = 'i-06298c4377973f455' # todo: refactor into ssm method command_id= '18f8de5e-116d-4245-89d0-f01f07310ef1' #result = self._.commands_list() #pprint(result) output = self._.client.get_command_invocation( CommandId=command_id, InstanceId=instance_id, ) pprint(output)
def test_user_privs(self): user_name = 'AlekhAnejaUpwork' access_key_id = 'AKIA3NUU5XSYZRQYXMP2' iam_user = IAM(user_name=user_name) assert iam_user.user_exists() assert list_set(iam_user.user_access_keys(index_by='AccessKeyId')) == [ access_key_id ] pprint(iam_user.user_polices())
async def wait_for_navigation(self, page=None, timeout_in_ms=10000, show_error=False): try: if page is None: page = await self.page() await page.waitForNavigation({'timeout': timeout_in_ms}) return True except Exception as error: if show_error: pprint(error) return False
def test_create(self, account_id, region): with Temp_SQS_Queue() as sqs_queue: assert self.event_rule.targets() == [] rule_name = self.event_rule.name() queue_name = sqs_queue.name() target_id = self.event_rule.add_target_sqs_queue(sqs_queue) assert self.event_rule.targets() == [{ 'Arn': f'arn:aws:sqs:{region}:{account_id}:{queue_name}', 'Id': f'{queue_name}', 'SqsParameters': { 'MessageGroupId': rule_name } }] pprint(self.event_rule.delete_target(target_id=target_id))
def test_invoke(self): #self.test_update_lambda_function() pprint(self.aws_lambda.invoke({'method_name': 'ping' }).get('return_value')) pprint(self.aws_lambda.invoke({'method_name': 'pwd' }).get('return_value')) pprint(self.aws_lambda.invoke({'method_name': 'disk_space' }).get('return_value')) pprint(self.aws_lambda.invoke({'method_name' : 'python_exec', 'method_kwargs': {'code':"result=40+2"}}))
def test_create_info_exists_delete(self): result = self.dashboard.create() pprint(result) return assert result.get('attributes').get('title') == self.pattern_name assert self.index_pattern.exists() is True assert list_set(self.index_pattern.info()) == [ 'fields', 'id', 'namespaces', 'references', 'score', 'title', 'type', 'updated_at' ] assert Index_Pattern(kibana=self.kibana, pattern_name=random_text()).info() == {} assert self.index_pattern.delete() is True
def test_tags_for_resource(self): resource_arn = None result = self.cloud_watch.tags_for_resource(resource_arn) pprint(result) # 'query': 'SOURCE ' # "'/aws/lambda/f2f_aws_lambdas_lambdas_deploy_fast_api__prod_v2' | " # 'fields @timestamp, @message\n' # '| sort @timestamp desc\n' # '| limit 20', # 'region': 'eu-west-1', # 'stacked': False, # 'title': 'Log group: ' # '/aws/lambda/f2f_aws_lambdas_lambdas_deploy_fast_api__prod_v2', # 'view': 'table'
def test_get_file_analysis(self): with patch.object(Metadata, 'get_from_file', return_value=self.meta_data): response=self.analysis_json.get_file_analysis(self.test_file_hash, self.report_data) pprint(response) assert "file_name" in response assert "original_hash" in response assert response["original_hash"] == self.test_file_hash assert "rebuild_hash" in response assert "file_type" in response assert "file_size" in response assert "remediated_item_count" in response assert "remediate_items_list" in response assert "sanitised_item_count" in response assert "sanitised_items_list" in response assert "issue_item_count" in response assert "issue_item_list" in response
def test_policy_add_sqs_permissions_to_lambda_role(self): policy_name = self.iam_utils.arn_aws_policy_service_sqs_lambda.split( '/').pop(-1) with Temp_Lambda() as temp_lambda: lambda_name = temp_lambda.lambda_name iam_role_name = self.iam_utils.policy_add_sqs_permissions_to_lambda_role( lambda_name) iam_role = IAM_Role(role_name=iam_role_name) pprint(iam_role.info()) assert policy_name in iam_role.policies_statements() assert iam_role.exists() is True self.iam_utils.policy_remove_sqs_permissions_to_lambda_role( lambda_name) assert policy_name not in iam_role.policies_statements()
def test_process_files(self): path_data = self.pre_processor.storage.hd2_data() path_status = self.pre_processor.storage.hd2_status() assert len(files_list(path_data)) == 0 assert len( files_list(path_status)) == 1 # should have the status.json file self.pre_processor.process_files() pprint('**********: DATA') pprint(files_list(path_data)) pprint('**********: STATUS') pprint(files_list(path_status)) pprint('------------------------') assert len(files_list(path_data)) > 0 assert len(files_list(path_status)) == 1 # should have: status.json
def test_import_dashboard(self): # dashboard_file = 'processed-files-v8.ndjson' # dashboard_file = 'KD1.ndjson' # url_dashboards = 'https://raw.githubusercontent.com/filetrust/cdr-plugin-folder-to-folder-test-data/main/kibana-dashboards/' # url_dashboard = url_dashboards + dashboard_file # dashboard_data = GET(url_dashboard) # # import_file = file_create(extension=dashboard_file, contents=dashboard_data) # dashboard = Dashboard(kibana=self.kibana) # #import_file = '/var/folders/_j/frqs70d93l328f307rw2jx5h0000gn/T/tmp6gz6sc76.tmp' # #import_data = file_contents(export_file) # pprint(self.dashboard.import_dashboard(import_file= import_file)) dashboard = Dashboard(kibana=self.kibana) pprint( dashboard.import_dashboard_from_github( dashboard_file_name='processed-files-v8.ndjson')) pprint( dashboard.import_dashboard_from_github( dashboard_file_name='KD1.ndjson'))
def test_run_containers_in_ec2(self): print() image_name = 'hello-world' task_definition_arn = 'arn:aws:ecs:eu-west-1:785217600689:task-definition/hello-world-in-ec2:3' for j in range(0, 2): for i in range(0, 12): result = self.ecs.task_create_ec2( task_definition_arn=task_definition_arn) failures = len(result.get('failures')) ok = len(result.get('tasks')) print(f'************** {j} {i} ok: {ok} failures: {failures}') return task_arn = task.get('taskArn') #self.ecs.wait_for_task_stopped(task_arn=task_arn) logs = self.ecs.logs_wait_for_data( task_arn=task_arn, task_definition_arn=task_definition_arn, image_name=image_name) pprint(logs)
def start(self): while self.enabled(): log_data = self.next_value() kwargs = { "level": log_data.get("level"), "message": log_data.get("message"), "data": log_data.get("data"), "duration": log_data.get("duration"), "from_method": log_data.get("from_method"), "from_class": log_data.get("from_class") } if print_log_messages: pprint(kwargs) self.log_message(**kwargs) # todo refactor into method focused on internal logging messages if kwargs.get('level' ) == 'DEBUG' and \ kwargs.get('message') == 'stop_logging' and \ kwargs.get('data' ) == {'when' : 'now'}: return
def test_role_create_assume_role(self): sts = STS() current_user_arn = sts.caller_identity_arn() original_policy = {'Statement': [ { 'Action' : 'sts:AssumeRole', 'Effect' : 'Allow', 'Principal': { 'Service': 'codebuild.amazonaws.com'}}]} new_policy = {'Statement': [{'Action' : 'sts:AssumeRole', 'Effect' : 'Allow', 'Principal': {'AWS': current_user_arn } }]} test_role = IAM(role_name="temp_role_to_test_assume_role") test_role.role_create(original_policy) role_arn = test_role.role_arn() current_assume_policy = test_role.role_assume_policy() test_role.role_assume_policy_update(new_policy) for i in range(0,15): with Catch(log_exception=False): sts.assume_role(role_arn=role_arn) sts.assume_role(role_arn=role_arn) sts.assume_role(role_arn=role_arn) sts.assume_role(role_arn=role_arn) sts.assume_role(role_arn=role_arn) sts.assume_role(role_arn=role_arn) sts.assume_role(role_arn=role_arn) sts.assume_role(role_arn=role_arn) pprint('got credentials') break print(f'after {i} seconds') wait(1) assert sts.assume_role(role_arn=role_arn).get('Credentials') is not None test_role.role_assume_policy_update(current_assume_policy) assert test_role.role_assume_policy() == current_assume_policy test_role.role_delete()
async def screenshot(self, url=None, page=None, full_page=True, file_screenshot=None, clip=None, viewport=None, js_code=None, delay=None): try: if url: await self.open(url, page=page) await self.js_execute(js_code) if delay: await asyncio.sleep(delay) if file_screenshot is None: file_screenshot = Files.temp_file('.png') page = await self.page() if viewport: await self.viewport(viewport) if clip: full_page = False await page.screenshot({ 'path': file_screenshot, 'fullPage': full_page, 'clip': clip }) return file_screenshot except Exception as error: pprint(f"Error in API_Browser.screenshot: f{error}" ) # todo: add support for logging
def test_alarms_for_metric(self): metric_name = None namespace = None result = self.cloud_watch.alarms_for_metric(metric_name=metric_name, namespace=namespace) pprint(result)