def test_url_argument(self): class FakeResponse: def getheader(self, name): return 'application/json;charset=SHIFT_JIS' def read(self): return '["Hello", "世界", "!"]'.encode('shift-jis') try: import urllib.request del urllib except ImportError: to_patch = 'urllib2.urlopen' else: to_patch = 'urllib.request.urlopen' to_patch = 'six.moves.urllib.request.urlopen' with patch(to_patch, return_value=FakeResponse()): self.assertEqual( pyjq.all('.[] | . + .', url='http://example.com'), ["HelloHello", "世界世界", "!!"] ) def opener(url): return "[1, 2, 3]" self.assertEqual( pyjq.all('.[] | . + .', url='http://example.com', opener=opener), [2, 4, 6] )
def image_list(self): full = json.loads(unicode(self.vList)) if self.vIsDebug: print json.dumps(full, indent=2) table = PrettyTable() table.field_names = ['Image_Name', 'Image_UUID', 'Image_Type', 'VM_Disk_ID'] table.align['Image_Name'] = "l" #if imageuuid is NULL, print summary for all images #else print summary for the special image and it's detail info if len(self.vUUID) == 0: #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR t = pyjq.all('.entities[] | {name, uuid, image_type, vm_disk_id}', full) i = 0 while i < len(t): table.add_row([t[i]['name'], t[i]['uuid'], t[i]['image_type'], t[i]['vm_disk_id']]) i = i + 1 else: for i in self.vUUID: #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR s = '.entities[] | select (.uuid==\"' + i + '\") | {name, uuid, image_type, vm_disk_id}' t = pyjq.all(s, full) table.add_row([t[0]['name'], t[0]['uuid'], t[0]['image_type'], t[0]['vm_disk_id']]) if self.vIsHTML: return table.get_html_string(sortby='Image_Name', attributes = self.table_style) else: return table.get_string(sortby='Image_Name')
def test_all(self): self.assertEqual( pyjq.all('.[] | . + $foo', ['val1', 'val2'], vars=dict(foo='bar')), ['val1bar', 'val2bar'] ) self.assertEqual( pyjq.all('. + $foo', 'val', vars=dict(foo='bar')), ['valbar'] )
def image_disktype(self): full = json.loads(unicode(self.vList)) d = pyjq.all('.entities[] | { uuid, image_type }', full) for u in d: if u['uuid'] == self.vUUID[0] and u['image_type'] == 'DISK_IMAGE': return True return False
def test_library_path(self): library_path = tempfile.mkdtemp() library_path2 = tempfile.mkdtemp() library_file = os.path.join(library_path, "greeting.jq") library_file2 = os.path.join(library_path2, "increment.jq") try: with io.open(library_file, 'w', encoding='ascii') as f: f.write('def hello: "HELLO";') f.write('def world: "WORLD";') with io.open(library_file2, 'w', encoding='ascii') as f: f.write('def increment: . + 1;\n') values = pyjq.all( 'include "greeting"; include "increment"; .[] | [. | increment, hello, world]', [1, 2, 3], library_paths = [library_path, library_path2] ) self.assertEquals( [[2, "HELLO", "WORLD"], [3, "HELLO", "WORLD"], [4, "HELLO", "WORLD"]], values ) finally: shutil.rmtree(library_path) shutil.rmtree(library_path2)
def doPcdmIndirect(self): self.log("Running doPcdmIndirect") self.log("Create a PCDM container") basic_headers = { 'Link': self.make_type(TestConstants.LDP_BASIC), 'Content-type': 'text/turtle' } r = self.do_post(self.getBaseUri(), headers=basic_headers, body=TestConstants.OBJECT_TTL) self.assertEqual(201, r.status_code, "Did not get expected status code") pcdm_container_location = self.get_location(r) self.log("Create a PCDM Collection") r = self.do_post(self.getBaseUri(), headers=basic_headers, body=TestConstants.OBJECT_TTL) self.assertEqual(201, r.status_code, "Did not get expected status code") pcdm_collection_location = self.get_location(r) self.log("Create indirect container inside PCDM collection") indirect_headers = { 'Link': self.make_type(TestConstants.LDP_INDIRECT), 'Content-type': 'text/turtle' } pcdm_indirect = "@prefix dc: <http://purl.org/dc/elements/1.1/> ." \ "@prefix pcdm: <http://pcdm.org/models#> ." \ "@prefix ore: <http://www.openarchives.org/ore/terms/> ." \ "@prefix ldp: <http://www.w3.org/ns/ldp#> ." \ "<> dc:title \"Members Container\" ;" \ " ldp:membershipResource <{0}> ;" \ " ldp:hasMemberRelation pcdm:hasMember ;" \ " ldp:insertedContentRelation ore:proxyFor .".format(pcdm_collection_location) r = self.do_post(pcdm_collection_location, headers=indirect_headers, body=pcdm_indirect) self.assertEqual(201, r.status_code, "Did not get expected status code") members_location = self.get_location(r) self.log("Create a proxy object") pcdm_proxy = "@prefix pcdm: <http://pcdm.org/models#>" \ "@prefix ore: <http://www.openarchives.org/ore/terms/>" \ "<> a pcdm:Object ;" \ "ore:proxyFor <{0}> .".format(pcdm_container_location) r = self.do_post(members_location, headers=basic_headers, body=pcdm_proxy) self.assertEqual(201, r.status_code, "Did not get expected status code") self.log("Check PCDM Collection for a new pcmd:hasMember property to PCDM container") headers = { 'Accept': TestConstants.JSONLD_MIMETYPE } r = self.do_get(pcdm_collection_location, headers=headers) self.assertEqual(200, r.status_code, "Did not get expected status code") body = r.content.decode('UTF-8') body_json = json.loads(body) hasmembers = pyjq.all('.[] | ."http://pcdm.org/models#hasMember" | .[]."@id"', body_json) found_member = False for member in hasmembers: if member == pcdm_container_location: found_member = True self.assertTrue(found_member, "Did not find hasMember property") self.log("Passed")
def image_config(self): if len(self.vUUID) == 0: print 'image uuid is NULL.' sys.exit(9) for i in self.vUUID: full = json.loads(unicode(self.vList)) t = pyjq.all('.entities[] | select (.uuid==\"' + i + '\") ', full) print json.dumps(t, indent=4)
def vm_config(self, uuid=''): if uuid == '': print 'vm uuid is NULL.' sys.exit(9) for i in uuid: if not self.vm_existed(uuid=i): print 'vm uuid not existed.' sys.exit(9) full = json.loads(unicode(self.vVMList)) t = pyjq.all('.entities[] | select (.uuid==\"' + i + '\") ', full) print json.dumps(t, indent=4)
def task_print(self, uuid=''): # # call this function with uuid parameter, or you will get full list # from prettytable import PrettyTable if not hasattr(self, 'vList'): self.task_list() full = json.loads(str(self.vList)) t = pyjq.all('. | { uuid, operation_type, progress_status }', full) t1 = pyjq.all('.entity_list[] | { entity_id, entity_type }', full) table = PrettyTable(['task uuid', 'operation type', 'progress status']) if len(t) != 0: table.add_row([t[0]['uuid'], t[0]['operation_type'], t[0]['progress_status']]) i = 0 while i < len(t1): #table.add_row([i, t1[i]['entity_type'], t1[i]['entity_id']]) c = 'entity_type:' + t1[i]['entity_type'] table.add_column(c, [t1[i]['entity_id']]) i = i + 1 print table
def vg_list(self): full = json.loads(unicode(self.vList)) if self.vIsDebug: print json.dumps(full, indent=2) if not self.vIsSummary: #self.vIsSummary is true, print detail vm list table = PrettyTable() table.field_names = ['VG_UUID', 'VG_Name', 'Disk_List', 'Attachment_List'] fields = '{ uuid, name, disk_list, attachment_list}' #if imageuuid is NULL, print summary for all images #else print summary for the special image and it's detail info if len(self.vUUID) == 0: #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR t = pyjq.all('.entities[] | ' + fields, full) i = 0 while i < len(t): try: disk_num = len(t[i]['disk_list']) except TypeError: disk_num = 0 try: attachment_num = len(t[i]['attachment_list']) except TypeError: attachment_num = 0 table.add_row([t[i]['uuid'], t[i]['name'], disk_num, attachment_num]) i = i + 1 else: for i in self.vUUID: #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR s = '.entities[] | select (.uuid==\"' + i + '\") | ' + fields t = pyjq.all(s, full) table.add_row([t[0]['uuid'], t[0]['name'], len(t[0]['disk_list']), len(t[0]['attachment_list'])]) if self.vIsHTML: html_str = table.get_html_string(attributes = self.table_style) count = html_str.count('\033[0;m') if count > 0: html_str = colorhtml(html_str) return html_str else: return table else: #self.vIsSummary is false, just print uuid pass
def vm_existed(self, uuid=''): if uuid == '': print 'vm uuid is NULL.' sys.exit(9) # check vm_list() loaded or not if not hasattr(self, 'vVMList'): self.vm_list() full = json.loads(unicode(self.vVMList)) d = pyjq.all('.entities[]', full) for u in d: if u['metadata']['uuid'] == uuid: return True return False
def assertTitleExists(self, expected, location): """ Check resource at {location} for the dc:title {expected} """ get_headers = { 'Accept': TestConstants.JSONLD_MIMETYPE } response = self.do_get(location, headers=get_headers) body = response.content.decode('UTF-8') json_body = json.loads(body) found_title = pyjq.all('.[] | ."http://purl.org/dc/elements/1.1/title" | .[]."@value"', json_body) for title in found_title: if title == expected: return self.fail("Did not find expected title \"{0}\" in response".format(expected))
def __init__(self, debug=False, hostname='', port='', token='', foo=False, color=True, html=False, summary=False, uuid=[]): self.vIsDebug = debug self.vHostname = hostname self.vPort = port self.vToken = token self.vIsColor = color self.vIsFoo = foo self.vIsHTML = html self.vIsSummary = summary self.vUUID = uuid # when you want to output comments and tables, you need this # called in vg_listdisk self.capture = StringIO() self.save_stdout = sys.stdout # table css style # called by attributes in get_html_string, when output of vg_list vg_listdisk self.table_style = {"name": "vg-table", "class": "table table-sm table-striped"} if not self.vIsFoo: url = '/PrismGateway/services/rest/v2.0/volume_groups/?include_disk_size=true' a = clsURL( debug=self.vIsDebug, hostname=self.vHostname, port=self.vPort, token=self.vToken, url=url ) self.vList = a.GetURL() #return(unicode(self.vList)) else: with open('test_clsVg.json', 'r') as f: self.vList = json.load(f) if len(self.vUUID) != 0: self.vExisted = list(self.vUUID) full = json.loads(unicode(self.vList)) d = pyjq.all('.entities[] | { uuid }', full) for i in self.vUUID: for u in d: if u['uuid'] == i: self.vExisted.remove(i) break if len(self.vExisted) != 0: print 'vg uuid %s is not existed.' % self.vExisted sys.exit(19)
def vm_in_net(self, netuuid=''): if netuuid == '': print 'net uuid is NULL.' sys.exit(9) if not hasattr(self, 'vVMList'): self.vm_list() full = json.loads(str(self.vVMList)) if self.vIsDebug: print json.dumps(full, indent=4) full_order = pyjq.all('.entities[] | { uuid, name, power_state, vm_nics }', full) vmlist = [] for vm in full_order: for nic in vm['vm_nics']: if nic['network_uuid'] == netuuid: vmlist.append(vm['uuid']) if len(vmlist) != 0: self.vm_print(uuid=vmlist)
def vg_listdisk(self): sys.stdout = self.capture if len(self.vUUID) == 0: print 'vg uuid is NULL.' sys.exit(9) full = json.loads(unicode(self.vList)) for h in self.vUUID: table = PrettyTable() table.field_names = ['disk index', 'disk uuid', 'size (GB)', 'Container'] t = pyjq.all('.entities[] | select (.uuid==\"' + h + '\") | { disk_list }', full) myset = list() for i in t[0]['disk_list']: a = clsContainer(debug=False, hostname=self.vHostname, port=self.vPort, token=self.vToken, foo=self.vIsFoo, uuid=[i['storage_container_uuid']]) container_name = a.container_getname() myset.append(container_name) if self.vIsColor: container_name = colourise('blue', container_name) else: container_name = container_name table.add_row([ i['index'], i['vmdisk_uuid'], i['vmdisk_size_bytes']/1024/1024/1024, container_name ]) sys.stdout.write('vguuid:' + h) # BPG if len(list(set(myset))) > 1: print " (%s container)" % colourise('red', 'In Different') else: print " (%s container)" % colourise('green', 'In Same') if self.vIsHTML: html_str = table.get_html_string(attributes = self.table_style) count = html_str.count('\033[0;m') if count > 0: html_str = colorhtml(html_str) print html_str else: print table if self.vIsDebug: print json.dumps(t, indent=2) sys.stdout = self.save_stdout return self.capture
def __init__(self, debug=False, foo=False, color=True, html=False, summary=False, uuid=[], msgtype='alerts', severity='', resolved='', count=10): self.vIsColor = color self.vIsDebug = debug self.vIsFoo = foo self.vIsHTML = html self.vIsSummary = summary self.vUUID = uuid self.vType = msgtype self.vSeverity = severity self.vResolved = resolved self.vCount = count if not self.vIsFoo: if self.vSeverity == '': severity_str = '' else: severity_str = '&severity=' + self.vSeverity if self.vResolved == '': resolved_str = '' else: resolved_str = '&resolved=' + self.vResolved url = '/PrismGateway/services/rest/v2.0/' + self.vType + '/?count=' + str(self.vCount) + severity_str + resolved_str a = clsURL( debug=self.vIsDebug, url=url ) self.vList = a.GetURL() #return(unicode(self.vList)) else: with open('test_clsEvent.json', 'r') as f: self.vList = json.load(f) if len(self.vUUID) != 0: self.vExisted = list(self.vUUID) full = json.loads(unicode(self.vList)) d = pyjq.all('.entities[] | { id }', full) for i in self.vUUID: for u in d: if u['id'] == i: self.vExisted.remove(i) break if len(self.vExisted) != 0: print 'event id %s is not existed.' % self.vExisted sys.exit(19)
def cluster_list(self): full = json.loads(unicode(self.vList)) if self.vIsDebug: print json.dumps(full, indent=4) table = PrettyTable() table.field_names = ['Cluster Name', 'Prism Central', 'Cluster IP', 'DataService IP', 'NTP Servers', 'Timezone', 'ShadowClone', 'Hypervisor', 'AOS Version', 'Cluster Functions'] #if imageuuid is NULL, print summary for all images #else print summary for the special image and it's detail info #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR s = '. | { name, multicluster, cluster_external_ipaddress, cluster_external_data_services_ipaddress, ntp_servers, timezone, enable_shadow_clones, hypervisor_types, version, cluster_functions }' t = pyjq.all(s, full) if self.vIsColor: # BGP if t[0]['enable_shadow_clones'] == True: shadowclone = colourise('red', 'Enabled') else: shadowclone = colourise('green', 'Disabled') else: if t[0]['enable_shadow_clones'] == True: shadowclone = 'Enabled' else: shadowclone = 'Disabled' table.add_row([ t[0]['name'], t[0]['multicluster'], t[0]['cluster_external_ipaddress'], t[0]['cluster_external_data_services_ipaddress'], ', '.join(t[0]['ntp_servers']), t[0]['timezone'], shadowclone, ', '.join(t[0]['hypervisor_types']), t[0]['version'], ', '.join(t[0]['cluster_functions']), ]) if self.vIsHTML: html_str = table.get_html_string(attributes = self.table_style) count = html_str.count('\033[0;m') if count > 0: html_str = colorhtml(html_str) return html_str else: return table
def __init__(self, debug=False, hostname='', port='', token='', foo=False, html=False, color=False, summary=False, uuid=[]): self.vIsDebug = debug self.vHostname = hostname self.vPort = port self.vToken = token self.vIsColor = color self.vIsFoo = foo self.vIsHTML = html self.vIsSummary = summary self.vUUID = uuid self.table_style = {"name": "vm-table", "class": "table table-sm table-striped"} if not self.vIsFoo: url = '/PrismGateway/services/rest/v2.0/images/' a = clsURL( debug=self.vIsDebug, hostname=self.vHostname, port=self.vPort, token=self.vToken, url=url ) self.vList = a.GetURL() if self.vIsDebug: print json.dumps(json.loads(str(self.vList)), indent=2) else: with open('test/test_clsImage.json', 'r') as f: self.vList = json.load(f) if len(self.vUUID) != 0: self.vExisted = list(self.vUUID) full = json.loads(unicode(self.vList)) d = pyjq.all('.entities[] | { uuid }', full) for i in self.vUUID: for u in d: if u['uuid'] == i: self.vExisted.remove(i) break if len(self.vExisted) != 0: print 'image uuid %s is not existed.' % self.vExisted sys.exit(19)
def policy_replicate(self): postdata = { "metadata": { "categories": {}, "kind": "network_security_rule", "spec_version": 0 } } url = '/api/nutanix/v3/network_security_rules' full = json.loads(unicode(self.vList)) s = '.entities[] | { spec, metadata }' t = pyjq.all(s, full) unique_str = str(random.randint(1,1000)) for i in self.vUUID: for j in t: rep_str = j.copy() if j['metadata']['uuid'] == i: rep_str['spec']['name'] = 'CLI ' + unique_str rep_str['spec']['description'] = 'CLI ' + unique_str rep_str['metadata'] = postdata['metadata'] #for k in rep_str['spec']['resources']['app_rule']['outbound_allow_list']: # print k #break #a = re.sub('"','\'',json.dumps(rep_str)) #print a #break if self.vIsDebug: print json.loads(rep_str) print url a = clsURL( debug=self.vIsDebug, hostname=self.vHostname, port=self.vPort, token=self.vToken, postdata=json.loads(json.dumps(rep_str)), url=url ) result = a.PostURL() print json.dumps(json.loads(result), indent=2) break
def main(): parser = ArgumentParser() parser.add_argument('host') parser.add_argument('port') parser.add_argument('job') parser.add_argument('build_number', nargs='?', default='lastSuccessfulBuild') parser.add_argument('-o', '--output-dir', action='store', dest='output_dir', default='artifacts') parser.add_argument('--console-text', '-c', action='store_true', dest='console_text') args = parser.parse_args() output_dir = Path(args.output_dir) create_empty_dir(output_dir) def api_url(url): return urljoin(url, 'api/json') build_api_url = api_url('http://{0.host}:{0.port}/job/{0.job}/{0.build_number}/'.format(args)) for run_url in jq.all('.runs[].url', url=build_api_url): subjob_url = urljoin(run_url, '../') subjob_name = jq.one('.displayName', url=api_url(subjob_url)) subjob_dir = output_dir / urllib.request.quote(subjob_name, '') if not subjob_dir.is_dir(): subjob_dir.mkdir(parents=True) with (subjob_dir / 'consoleText').open('wb') as local_fp, \ urlopen(urljoin(run_url, 'consoleText')) as http_fp: shutil.copyfileobj(http_fp, local_fp) zip_fp = io.BytesIO(urlopen(urljoin(run_url, 'artifact/*zip*/archive.zip')).read()) with ZipFile(zip_fp) as z: for name in z.namelist(): prefix = 'archive/' if not name.startswith(prefix): continue path = subjob_dir / name[len(prefix):] if not path.parent.is_dir(): path.parent.mkdir(parents=True) with path.open('wb') as fp: fp.write(z.read(name))
def policy_list(self): # The attribute vList just been define in this function. # check with "hasattr(self, 'vList')" in another function. postdata = { "filter": "" } a = clsURL( debug=self.vIsDebug, hostname=self.vHostname, port=self.vPort, token=self.vToken, postdata=postdata, url='/api/nutanix/v3/network_security_rules/list' ) self.vList = a.PostURL() #print json.dumps(unicode(self.vList)) full = json.loads(unicode(self.vList)) if self.vIsDebug: print json.dumps(full, indent=4) table = PrettyTable() table.field_names = ['Policy Name', 'UUID'] s = '.entities[] | { spec, metadata }' t = pyjq.all(s, full) for u in t: table.add_row([ u['spec']['name'], u['metadata']['uuid'] ]) if self.vIsHTML: html_str = table.get_html_string(attributes = self.table_style) count = html_str.count('\033[0;m') if count > 0: html_str = colorhtml(html_str) return html_str else: return table
def get_iam_trusts(account, nodes, connections, connections_to_get): # Get IAM iam = query_aws(account, "iam-get-account-authorization-details", Region(account, {'RegionName': 'us-east-1'})) for role in pyjq.all('.RoleDetailList[]', iam): principals = pyjq.all( '.AssumeRolePolicyDocument.Statement[].Principal', role) for principal in principals: assume_role_nodes = set() if principal.get('Federated', None): # TODO I should be using get-saml-provider to confirm this is really okta if "saml-provider/okta" in principal['Federated'].lower(): node = Account(json_blob={ 'id': 'okta', 'name': 'okta', 'type': 'Okta' }) assume_role_nodes.add(node) elif "saml-provider/adfs" in principal['Federated'].lower(): node = Account(json_blob={ 'id': 'adfs', 'name': 'adfs', 'type': 'ADFS' }) assume_role_nodes.add(node) elif principal[ 'Federated'] == 'cognito-identity.amazonaws.com': # TODO: Should show this somehow continue elif principal['Federated'] == 'www.amazon.com': node = Account(json_blob={ 'id': 'Amazon.com', 'name': 'Amazon.com', 'type': 'Amazon' }) continue else: raise Exception('Unknown federation provider: {}'.format( principal['Federated'])) if principal.get('AWS', None): principal = principal['AWS'] if not isinstance(principal, list): principal = [principal] for p in principal: if "arn:aws" not in p: # The role can simply be something like "AROA..." continue parts = p.split(':') account_id = parts[4] assume_role_nodes.add(Account(account_id=account_id)) for node in assume_role_nodes: if nodes.get(node.id, None) is None: nodes[node.id] = node access_type = 'iam' # TODO: Identify all admins better. Use code from find_admins.py for m in role['AttachedManagedPolicies']: for p in pyjq.all('.Policies[]', iam): if p['Arn'] == m['PolicyArn']: for policy_doc in p['PolicyVersionList']: if policy_doc['IsDefaultVersion'] == True: if is_admin_policy(policy_doc['Document']): access_type = 'admin' for policy in role['RolePolicyList']: policy_doc = policy['PolicyDocument'] if is_admin_policy(policy_doc): access_type = 'admin' if ((access_type == 'admin' and connections_to_get['admin']) or (access_type != 'admin' and connections_to_get['iam_nonadmin'])): connections[Connection(node, account, access_type)] = [] return
def extract_evidence(self, obj_json: str): parsed = json.loads(obj_json.lower()) findings = pyjq.all('to_entries[] | select((.key|startswith("name")) or (.key|endswith("id")))',parsed) return str(findings)
def cluster_timezone(self): full = json.loads(unicode(self.vList)) if self.vIsDebug: print json.dumps(full, indent=4) t = pyjq.all('. | { timezone }', full) return t[0]['timezone']
def audit_sg(findings, region): # TODO Check if security groups allow large CIDR range (ex. 1.2.3.4/3) # TODO Check if an SG restricts IPv4 and then opens IPv6 or vice versa. cidrs = {} sg_json = query_aws(region.account, "ec2-describe-security-groups", region) sgs = pyjq.all(".SecurityGroups[]", sg_json) for sg in sgs: cidr_and_name_list = pyjq.all( ".IpPermissions[]?.IpRanges[]|[.CidrIp,.Description]", sg) for cidr, name in cidr_and_name_list: if not is_external_cidr(cidr): continue if is_unblockable_cidr(cidr): findings.add( Finding( region, "SG_CIDR_UNNEEDED", sg["GroupId"], resource_details={"cidr": cidr}, )) continue if cidr.startswith("0.0.0.0") and not cidr.endswith("/0"): findings.add( Finding( region, "SG_CIDR_UNEXPECTED", sg["GroupId"], resource_details={"cidr": cidr}, )) continue if cidr == "0.0.0.0/0": continue cidrs[cidr] = cidrs.get(cidr, list()) cidrs[cidr].append(sg["GroupId"]) for ip_permissions in sg.get("IpPermissions", []): cidrs_seen = set() for ip_ranges in ip_permissions.get("IpRanges", []): if "CidrIp" not in ip_ranges: continue cidr = ip_ranges["CidrIp"] for cidr_seen in cidrs_seen: if IPNetwork(cidr_seen) in IPNetwork(cidr) or IPNetwork( cidr) in IPNetwork(cidr_seen): findings.add( Finding( region, "SG_CIDR_OVERLAPS", sg["GroupId"], resource_details={ "cidr1": cidr, "cidr2": cidr_seen }, )) cidrs_seen.add(cidr) for cidr in cidrs: ip = IPNetwork(cidr) if ip.size > 2048: findings.add( Finding( region, "SG_LARGE_CIDR", cidr, resource_details={ "size": ip.size, "security_groups": list(cidrs[cidr]) }, ))
def kafka_producer_common_news(producer): print("*" * 10, "COMMON NEWS", "*" * 10) page = -1 total_pages = 0 news_desk = ["Business Day", "Business", "Entrepreneurs","Financial", "Magazine", "Personal Investing", "Personal Tech", "Politics", "Retail", "Small Business", \ "Sunday Business", "Technology", "Washington", "Week", "Your Money", "World"] for news in news_desk: print("*" * 80) print('Fetching for the news :', news) page = -1 total_pages = 0 start_date = '20180101' end_date = '20200401' api = 'v9gJZYhGqTvAgpqfi4b6GxxtdiTJvxi8' while page <= total_pages and start_date < end_date: temp_end_date = "20200401" page += 1 mydict = query(start_date, end_date, page, news) total_pages = pyjq.all('.response .meta .hits', mydict)[0] if total_pages: total_pages = math.floor(total_pages / 10) else: total_pages = 0 time.sleep(8) doc_lis = mydict["response"]["docs"] print('len(doc_lis)', len(doc_lis)) for i, doc in enumerate(doc_lis): msg_str = "{}_{}||{}".format("Common", "Sector", str(doc)) producer.send(topic='common_news', value=bytes(msg_str, 'utf-8')) while (total_pages > 200): print("Inside while, total_pages = ", total_pages) date_object = monthdelta(end_date, -1) temp_end_date = date_object.strftime("%Y%m%d") mydict = query(start_date, temp_end_date, page, news) total_pages = pyjq.all('.response .meta .hits', mydict)[0] if total_pages: total_pages = math.floor(total_pages / 10) else: total_pages = 0 end_date = temp_end_date time.sleep(8) doc_lis = mydict["response"]["docs"] print('len(doc_lis)', len(doc_lis)) for i, doc in enumerate(doc_lis): doc["symbol"] = "Common" doc["sector"] = "Sector" msg_str = "{}".format(str(doc)) producer.send(topic='common_news', value=bytes(msg_str, 'utf-8')) start_date = temp_end_date end_date = '20200401'
def get_azs(vpc): azs = query_aws(vpc.account, "ec2-describe-availability-zones", vpc.region) resource_filter = ".AvailabilityZones[]" return pyjq.all(resource_filter, azs)
def get_subnets(az): subnets = query_aws(az.account, "ec2-describe-subnets", az.region) resource_filter = ( '.Subnets[] | select(.VpcId == "{}") | select(.AvailabilityZone == "{}")' ) return pyjq.all(resource_filter.format(az.vpc.local_id, az.local_id), subnets)
def get_lambda_functions(region): functions = query_aws(region.account, "lambda-list-functions", region.region) return pyjq.all(".Functions[]?|select(.VpcConfig!=null)", functions)
def get_sgs(vpc): sgs = query_aws(vpc.account, "ec2-describe-security-groups", vpc.region) return pyjq.all( '.SecurityGroups[]? | select(.VpcId == "{}")'.format(vpc.local_id), sgs )
def audit_s3_buckets(findings, region): buckets_json = query_aws(region.account, "s3-list-buckets", region) buckets = pyjq.all('.Buckets[].Name', buckets_json) for bucket in buckets: # Check policy try: policy_file_json = get_parameter_file(region, 's3', 'get-bucket-policy', bucket) if policy_file_json is not None: # Find the entity we need policy_string = policy_file_json['Policy'] # Load the string value as json policy = json.loads(policy_string) policy = Policy(policy) if policy.is_internet_accessible(): if len(policy.statements) == 1 and len( policy.statements[0].actions ) == 1 and 's3:GetObject' in policy.statements[0].actions: findings.add( Finding(region, 'S3_PUBLIC_POLICY_GETOBJECT_ONLY', bucket)) else: findings.add( Finding(region, 'S3_PUBLIC_POLICY', bucket, resource_details=policy_string)) region, issue_id, resource_id, resource_details except Exception as e: findings.add( Finding(region, 'EXCEPTION', bucket, resource_details={ 'policy': policy_string, 'exception': e, 'location': 'Exception checking policy of S3 bucket' })) # Check ACL try: file_json = get_parameter_file(region, 's3', 'get-bucket-acl', bucket) for grant in file_json['Grants']: uri = grant['Grantee'].get('URI', "") if (uri == 'http://acs.amazonaws.com/groups/global/AllUsers' or uri == 'http://acs.amazonaws.com/groups/global/AuthenticatedUsers' ): findings.add( Finding(region, 'S3_PUBLIC_ACL', bucket, resource_details=grant)) except Exception as e: findings.add( Finding(region, 'EXCEPTION', bucket, resource_details={ 'grant': grant, 'exception': e, 'location': 'Exception checking ACL of S3 bucket' }))
debug=param.vIsDebug, hostname=param.vHostname, port=param.vPort, username=param.vUsername, password=param.vPassword, snapshotuuid=param.vSnapshotUUID, url=param.vURL, urltype=param.vURLType ) d = json.loads(str(data)) if 'myerror' in d: print json.dumps(d, indent=4) else: table = PrettyTable() #print json.dumps(d['entities'][1]) #if vSnapshotUUID is NULL, print summary for all snapshots #else print summary for the special snapshot and it's vm's detail info if param.vSnapshotUUID == '': t = pyjq.all('.entities[] | {uuid, snapshot_name, vm_uuid}', d) #while i < d['metadata']['total_entities']: table.field_names = ['Snapshot_UUID', 'Snapshot_Name', 'VM_UUID'] i = 0 while i < len(t): #print '%s | %s | %s' % (t[i]['uuid'], t[i]['vm_uuid'], t[i]['snapshot_name']) table.add_row([t[i]['uuid'], t[i]['snapshot_name'], t[i]['vm_uuid']]) i = i + 1 else: t1 = pyjq.all('. | {uuid, snapshot_name, vm_uuid, vm_create_spec}', d) t2 = pyjq.all('.vm_create_spec | {name, num_vcpus, num_cores_per_vcpu, memory_mb}', d) table.field_names = ['Snapshot_UUID', 'Snapshot_Name', 'VM_UUID', 'VM_Name', 'Cores', 'Memory(MB)'] #print '%s | %s | %s | %s | %d | %d' % ( # t1[0]['uuid'], t1[0]['snapshot_name'], t1[0]['vm_uuid'], # t2[0]['name'], t2[0]['num_vcpus'] * t2[0]['num_cores_per_vcpu'], t2[0]['memory_mb'] # )
def calc_result(self, person_datas, event_ids, competiton_datas, rank_type): rank_data = [] with open(self.get_fixtures_path("result")) as file: json_object = json.loads(file.read()) index = 0 for event_id in event_ids: # イベントごとに順位でソートされた記録を取得 query = ( "[ [.[].fields ] | \ map(select ( .event_id == " + str(event_id) + " )) | \ group_by(.person_id) | .[] | sort_by(." + rank_type + ") | \ .[0] ] | \ sort_by(." + rank_type + ") | \ .[] | \ { id, event_id, person_id, " + rank_type + ", competition_id, value1, value2, value3, value4, value5 }" ) results = pyjq.all(query, json_object) rank = 0 gender_ranks = dict(map(lambda x: (x.value[0], 0), Gender)) generation_ranks = dict( map(lambda x: (x * GENERATION_MAX, 0), range(0, 10)) ) prefecture_ranks = dict( map(lambda x: (x.value, 0), PrefectureAndOversea) ) before_record = 0 skip_count = 1 for result in results: if result[rank_type] > 0: # 世代計算 generation = ( self.get_generation( person_datas[result["person_id"]].birth_at, competiton_datas[result["competition_id"]]["close_at"], ) * GENERATION_MAX ) if before_record == 0 or before_record < result[rank_type]: before_record = result[rank_type] gender_ranks[ person_datas[result["person_id"]].gender ] += skip_count generation_ranks[generation] += skip_count prefecture_ranks[ person_datas[result["person_id"]].prefecture_id ] += skip_count rank += skip_count skip_count = 1 else: skip_count += 1 result["competition_id"] = result["competition_id"] result["competition_name_id"] = competiton_datas[ result["competition_id"] ]["name_id"] result["competition_name"] = competiton_datas[ result["competition_id"] ]["name"] result["year"] = localtime( datetime.fromisoformat( competiton_datas[result["competition_id"]][ "open_at" ].replace("Z", "+00:00") ) ).year result["gender"] = person_datas[result["person_id"]].gender result["generation"] = generation result["prefecture_id"] = person_datas[ result["person_id"] ].prefecture_id result["rank"] = rank result["gender_rank"] = gender_ranks[ person_datas[result["person_id"]].gender ] result["generation_rank"] = generation_ranks[generation] result["prefecture_rank"] = prefecture_ranks[ person_datas[result["person_id"]].prefecture_id ] # 不要データ削除 result.pop("id") if rank_type == "average": result["best"] = result["average"] result.pop("average") if rank_type == "best": result.pop("value1") result.pop("value2") result.pop("value3") result.pop("value4") result.pop("value5") model_name = "bestrank" if rank_type == "average": model_name = "averagerank" index += 1 dictionary = {} dictionary["model"] = "app." + model_name dictionary["pk"] = index dictionary["fields"] = result rank_data.append(dictionary) return rank_data
def amis(args, accounts, config): # Loading the list of public images from disk takes a while, so we'll iterate by region regions_file = 'data/aws/us-east-1/ec2-describe-images.json' if not os.path.isfile(regions_file): raise Exception("You need to download the set of public AMI images. Run:\n" " mkdir -p data/aws\n" " cd data/aws\n" " aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} mkdir {}\n" " aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} sh -c 'aws --region {} ec2 describe-images --executable-users all > {}/ec2-describe-images.json'\n" ) print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( 'Account Name', 'Region Name', 'Instance Id', 'Instance Name', 'AMI ID', 'Is Public', 'AMI Description', 'AMI Owner')) for region in listdir('data/aws/'): # Get public images public_images_file = 'data/aws/{}/ec2-describe-images.json'.format(region) public_images = json.load(open(public_images_file)) resource_filter = '.Images[]' public_images = pyjq.all(resource_filter, public_images) for account in accounts: account = Account(None, account) region = Region(account, {'RegionName': region}) instances = query_aws(account, "ec2-describe-instances", region) resource_filter = '.Reservations[].Instances[] | select(.State.Name == "running")' if args.instance_filter != '': resource_filter += '|{}'.format(args.instance_filter) instances = pyjq.all(resource_filter, instances) account_images = query_aws(account, "ec2-describe-images", region) resource_filter = '.Images[]' account_images = pyjq.all(resource_filter, account_images) for instance in instances: image_id = instance['ImageId'] image_description = '' owner = '' image, is_public_image = find_image(image_id, public_images, account_images) if image: # Many images don't have all fields, so try the Name, then Description, then ImageLocation image_description = image.get('Name', '') if image_description == '': image_description = image.get('Description', '') if image_description == '': image_description = image.get('ImageLocation', '') owner = image.get('OwnerId', '') print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( account.name, region.name, instance['InstanceId'], get_instance_name(instance), image_id, is_public_image, image_description, owner))
def get_regional_direct_connects(region): direct_connects = query_aws(region.account, "/directconnect-describe-connections", region) resource_filter = ".connections[]?" return pyjq.all(resource_filter, direct_connects)
def get_iam_trusts(account, nodes, connections, connections_to_get): # Get IAM iam = query_aws( account, "iam-get-account-authorization-details", Region(account, {"RegionName": "us-east-1"}), ) saml_providers = query_aws( account, "iam-list-saml-providers", Region(account, {"RegionName": "us-east-1"}))["SAMLProviderList"] for role in pyjq.all(".RoleDetailList[]", iam): principals = pyjq.all( ".AssumeRolePolicyDocument.Statement[].Principal", role) for principal in principals: assume_role_nodes = set() federated_principals = principal.get("Federated", None) if federated_principals: if not isinstance(federated_principals, list): federated_principals = [federated_principals] for federated_principal in federated_principals: try: # Validate that the federated principal and the SAML provider is coming from known accounts. # WoT will show us the direction of that trust for further inspection. # this enables cross_account_admin_sts (STS between accounts) for saml in saml_providers: if saml["Arn"] == federated_principal: saml_provider_arn = saml["Arn"] elif get_account_by_id( account_id=federated_principal.split( ":")[4]): if get_account_by_id( account_id=saml["Arn"].split(":")[4]): saml_provider_arn = saml["Arn"] if "saml-provider/okta" in saml_provider_arn.lower(): node = Account(json_blob={ "id": "okta", "name": "okta", "type": "Okta" }) assume_role_nodes.add(node) elif "saml-provider/onelogin" in saml_provider_arn.lower( ): node = Account( json_blob={ "id": "onelogin", "name": "onelogin", "type": "Onelogin", }) assume_role_nodes.add(node) elif "saml-provider/waad" in saml_provider_arn.lower(): node = Account(json_blob={ "id": "WAAD", "name": "WAAD", "type": "waad", }) assume_role_nodes.add(node) elif "saml-provider/allcloud-sso" in saml_provider_arn.lower( ): node = Account( json_blob={ "id": "AllCloud-SSO", "name": "AllCloud-SSO", "type": "AllCloud-SSO", }) assume_role_nodes.add(node) elif "saml-provider/awssso" in saml_provider_arn.lower( ): node = Account( json_blob={ "id": "AWSSSO", "name": "AWS SSO", "type": "Amazon", }) assume_role_nodes.add(node) elif "saml-provider/adfs" in saml_provider_arn.lower(): node = Account(json_blob={ "id": "adfs", "name": "adfs", "type": "ADFS" }) assume_role_nodes.add(node) elif "saml-provider/auth0" in saml_provider_arn.lower( ): node = Account(json_blob={ "id": "auth0", "name": "auth0", "type": "auth0", }) assume_role_nodes.add(node) elif "saml-provider/google" in saml_provider_arn.lower( ): node = Account( json_blob={ "id": "google", "name": "google", "type": "google", }) assume_role_nodes.add(node) elif "saml-provider/gsuite" in saml_provider_arn.lower( ): node = Account( json_blob={ "id": "gsuite", "name": "gsuite", "type": "gsuite", }) assume_role_nodes.add(node) elif ("cognito-identity.amazonaws.com" in saml_provider_arn.lower()): continue elif "www.amazon.com" in saml_provider_arn.lower(): node = Account( json_blob={ "id": "Amazon.com", "name": "Amazon.com", "type": "Amazon", }) continue else: raise Exception( "Unknown federation provider: {}".format( saml_provider_arn.lower())) except (StopIteration, IndexError): if ("cognito-identity.amazonaws.com" in federated_principal.lower()): # TODO: Should show this somehow continue elif ":oidc-provider/" in federated_principal.lower(): # TODO: handle OpenID Connect identity providers # https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html continue raise Exception( "Principal {} is not a configured SAML provider". format(federated_principal)) if principal.get("AWS", None): principal = principal["AWS"] if not isinstance(principal, list): principal = [principal] for p in principal: if "arn:aws" not in p: # The role can simply be something like "AROA..." continue parts = p.split(":") account_id = parts[4] assume_role_nodes.add(Account(account_id=account_id)) for node in assume_role_nodes: if nodes.get(node.id, None) is None: nodes[node.id] = node access_type = "iam" # TODO: Identify all admins better. Use code from find_admins.py for m in role["AttachedManagedPolicies"]: for p in pyjq.all(".Policies[]", iam): if p["Arn"] == m["PolicyArn"]: for policy_doc in p["PolicyVersionList"]: if policy_doc["IsDefaultVersion"] == True: if is_admin_policy(policy_doc["Document"]): access_type = "admin" for policy in role["RolePolicyList"]: policy_doc = policy["PolicyDocument"] if is_admin_policy(policy_doc): access_type = "admin" if (access_type == "admin" and connections_to_get["admin"] ) or (access_type != "admin" and connections_to_get["iam_nonadmin"]): connections[Connection(node, account, access_type)] = [] return
def get_public_nodes(account, config, use_cache=False): # TODO Look for IPv6 also # TODO Look at more services from https://github.com/arkadiyt/aws_public_ips # TODO Integrate into something to more easily port scan and screenshot web services # Try reading from cache cache_file_path = 'account-data/{}/public_nodes.json'.format( account['name']) if use_cache: if os.path.isfile(cache_file_path): with open(cache_file_path) as f: return json.load(f), [] # Get the data from the `prepare` command outputfilter = { 'internal_edges': False, 'read_replicas': False, 'inter_rds_edges': False, 'azs': False, 'collapse_by_tag': None, 'collapse_asgs': True, 'mute': True } network = build_data_structure(account, config, outputfilter) public_nodes = [] warnings = [] # Look at all the edges for ones connected to the public Internet (0.0.0.0/0) for edge in pyjq.all( '.[].data|select(.type=="edge")|select(.source=="0.0.0.0/0")', network): # Find the node at the other end of this edge target = {'arn': edge['target'], 'account': account['name']} target_node = pyjq.first( '.[].data|select(.id=="{}")'.format(target['arn']), network, {}) # Depending on the type of node, identify what the IP or hostname is if target_node['type'] == 'elb': target['type'] = 'elb' target['hostname'] = target_node['node_data']['DNSName'] elif target_node['type'] == 'elbv2': target['type'] = 'elbv2' target['hostname'] = target_node['node_data']['DNSName'] elif target_node['type'] == 'autoscaling': target['type'] = 'autoscaling' target['hostname'] = target_node['node_data'].get( 'PublicIpAddress', '') if target['hostname'] == '': target['hostname'] = target_node['node_data']['PublicDnsName'] elif target_node['type'] == 'rds': target['type'] = 'rds' target['hostname'] = target_node['node_data']['Endpoint'][ 'Address'] elif target_node['type'] == 'ec2': target['type'] = 'ec2' dns_name = target_node['node_data'].get('PublicDnsName', '') target['hostname'] = target_node['node_data'].get( 'PublicIpAddress', dns_name) elif target_node['type'] == 'ecs': target['type'] = 'ecs' target['hostname'] = '' for ip in target_node['node_data']['ips']: if is_public_ip(ip): target['hostname'] = ip elif target_node['type'] == 'redshift': target['type'] = 'redshift' target['hostname'] = target_node['node_data'].get( 'Endpoint', {}).get('Address', '') else: # Unknown node raise Exception('Unknown type: {}'.format(target_node['type'])) # Check if any protocol is allowed (indicated by IpProtocol == -1) ingress = pyjq.all('.[]', edge.get('node_data', {})) sg_group_allowing_all_protocols = pyjq.first( 'select(.IpPermissions[]|.IpProtocol=="-1")|.GroupId', ingress, None) public_sgs = {} if sg_group_allowing_all_protocols is not None: warnings.append( 'All protocols allowed access to {} due to {}'.format( target, sg_group_allowing_all_protocols)) range_string = '0-65535' # I would need to redo this code in order to get the name of the security group public_sgs[sg_group_allowing_all_protocols] = { 'public_ports': '0-65535' } else: # from_port and to_port mean the beginning and end of a port range # We only care about TCP (6) and UDP (17) # For more info see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/security-group-rules-reference.html port_ranges = [] for sg in ingress: sg_port_ranges = [] for ip_permission in sg['IpPermissions']: selection = 'select((.IpProtocol=="tcp") or (.IpProtocol=="udp")) | select(.IpRanges[].CidrIp=="0.0.0.0/0")' sg_port_ranges.extend( pyjq.all('{}| [.FromPort,.ToPort]'.format(selection), ip_permission)) public_sgs[sg['GroupId']] = { 'GroupName': sg['GroupName'], 'public_ports': port_ranges_string(regroup_ranges(sg_port_ranges)) } port_ranges.extend(sg_port_ranges) range_string = port_ranges_string(regroup_ranges(port_ranges)) target['ports'] = range_string target['public_sgs'] = public_sgs if target['ports'] == "": issue_msg = 'No ports open for tcp or udp (probably can only be pinged). Rules that are not tcp or udp: {} -- {}' warnings.append( issue_msg.format( json.dumps( pyjq.all( '.[]|select((.IpProtocol!="tcp") and (.IpProtocol!="udp"))' .format(selection), ingress)), account)) public_nodes.append(target) # For the network diagram, if an ELB has availability across 3 subnets, I put one node in each subnet. # We don't care about that when we want to know what is public and it makes it confusing when you # see 3 resources with the same hostname, when you view your environment as only having one ELB. # This same issue exists for RDS. # Reduce these to single nodes. reduced_nodes = {} for node in public_nodes: reduced_nodes[node['hostname']] = node public_nodes = [] for _, node in reduced_nodes.items(): public_nodes.append(node) account = Account(None, account) for region_json in get_regions(account): region = Region(account, region_json) # Look for CloudFront if region.name == 'us-east-1': json_blob = query_aws(region.account, 'cloudfront-list-distributions', region) for distribution in json_blob.get('DistributionList', {}).get('Items', []): if not distribution['Enabled']: continue target = {'arn': distribution['ARN'], 'account': account.name} target['type'] = 'cloudfront' target['hostname'] = distribution['DomainName'] target['ports'] = '80,443' public_nodes.append(target) # Look for API Gateway json_blob = query_aws(region.account, 'apigateway-get-rest-apis', region) if json_blob is not None: for api in json_blob.get('items', []): target = {'arn': api['id'], 'account': account.name} target['type'] = 'apigateway' target['hostname'] = '{}.execute-api.{}.amazonaws.com'.format( api['id'], region.name) target['ports'] = '80,443' public_nodes.append(target) # Write cache file with open(cache_file_path, 'w') as f: f.write(json.dumps(public_nodes, indent=4, sort_keys=True)) return public_nodes, warnings
def get_redshift(region): clusters = query_aws(region.account, "redshift-describe-clusters", region.region) return pyjq.all(".Clusters[]?", clusters)
def vm_print(self, uuid=''): # call this function with uuid parameter, or you will get full list from prettytable import PrettyTable for i in uuid: if not self.vm_existed(uuid=i): print 'uuid %s does not exist' % (i) sys.exit(9) if not hasattr(self, 'vVMList'): self.vm_list() full = json.loads(unicode(self.vVMList)) if self.vIsDebug: print json.dumps(full, indent=2) if not self.vIsSummary: table = PrettyTable() table.field_names = ["uuid", "vm name", "ip addr", "power state", "vCPU (Cores)", "vMEM (GB)", "vdisk num", "vnic num"] if uuid == '': t1 = pyjq.all('.entities[]', full) for u in t1: t2 = pyjq.all('.spec.resources.nic_list[]', u) ip = '' for v in t2: t3 = pyjq.all('.ip_endpoint_list[]', v) for w in t3: if w['type'] == 'ASSIGNED': ip = w['ip'] + ',' + ip table.add_row([ u['metadata']['uuid'], u['spec']['name'], re.sub(',$','',ip), u['spec']['resources']['power_state'], str(u['spec']['resources']['num_sockets']) + ' * ' + str(u['spec']['resources']['num_vcpus_per_socket']), str(u['spec']['resources']['memory_size_mib']/1024), len(u['spec']['resources']['disk_list']), len(u['spec']['resources']['nic_list']) ]) #print u['spec']['resources']['nic_list'][0]['ip_endpoint_list'][0] else: for i in uuid: s = '.entities[] | select (.metadata.uuid==\"' + i + '\") ' t1 = pyjq.all(s, full) table.add_row([ t1[0]['metadata']['uuid'], t1[0]['spec']['name'], '', t1[0]['spec']['resources']['power_state'], str(t1[0]['spec']['resources']['num_sockets']) + ' * ' + str(t1[0]['spec']['resources']['num_vcpus_per_socket']), str(t1[0]['spec']['resources']['memory_size_mib']/1024), len(t1[0]['spec']['resources']['disk_list']), len(t1[0]['spec']['resources']['nic_list']) ]) print table else: t1 = pyjq.all('.entities[]', full) for u in t1: print u['metadata']['uuid'] #eliminate error like this, when output pipe to head command #close failed in file object destructor: #sys.excepthook is missing #lost sys.stderr try: sys.stdout.close() except: pass try: sys.stderr.close() except: pass
def get_connections(cidrs, vpc, outputfilter): """ For a VPC, for each instance, find all of the other instances that can connect to it, including those in peered VPCs. Note I do not consider subnet ACLs, routing tables, or some other network concepts. """ connections = {} # Get mapping of security group names to nodes that have that security group sg_to_instance_mapping = {} for instance in vpc.leaves: for sg in instance.security_groups: sg_to_instance_mapping.setdefault(sg, {})[instance] = True # For each security group, find all the instances that are allowed to connect to instances # within that group. for sg in get_sgs(vpc): # Get the CIDRs that are allowed to connect for cidr in pyjq.all(".IpPermissions[].IpRanges[].CidrIp", sg): if not is_external_cidr(cidr): # This is a private IP, ex. 10.0.0.0/16 # See if we should skip this if not outputfilter.get("internal_edges", True): continue # Find all instances in this VPC and peered VPCs that are in this CIDR for sourceVpc in itertools.chain(vpc.peers, (vpc,)): # Ensure it is possible for instances in this VPC to be in the CIDR if not ( IPNetwork(sourceVpc.cidr) in IPNetwork(cidr) or IPNetwork(cidr) in IPNetwork(sourceVpc.cidr) ): # The CIDR from the security group does not overlap with the CIDR of the VPC, # so skip it continue # For each instance, check if one of its IPs is within the CIDR for sourceInstance in sourceVpc.leaves: for ip in sourceInstance.ips: if IPAddress(ip) in IPNetwork(cidr): # Instance found that can connect to instances in the SG # So connect this instance (sourceInstance) to every instance # in the SG. for targetInstance in sg_to_instance_mapping.get( sg["GroupId"], {} ): add_connection( connections, sourceInstance, targetInstance, sg ) else: # This is an external IP (ie. not in a private range). for instance in sg_to_instance_mapping.get(sg["GroupId"], {}): # Ensure it has a public IP, as resources with only private IPs can't be reached if instance.is_public: cidrs[cidr].is_used = True add_connection(connections, cidrs[cidr], instance, sg) else: if cidr == "0.0.0.0/0": # Resource is not public, but allows anything to access it, # so mark set all the resources in the VPC as allowing access to it. for source_instance in vpc.leaves: add_connection( connections, source_instance, instance, sg ) if outputfilter.get("internal_edges", True): # Connect allowed in Security Groups for ingress_sg in pyjq.all( ".IpPermissions[].UserIdGroupPairs[].GroupId", sg ): # We have an SG and a list of SG's it allows in for target in sg_to_instance_mapping.get(sg["GroupId"], {}): # We have an instance and a list of SG's it allows in for source in sg_to_instance_mapping.get(ingress_sg, {}): if ( not outputfilter.get("inter_rds_edges", True) and ( source.node_type == "rds" or source.node_type == "rds_rr" ) and ( target.node_type == "rds" or target.node_type == "rds_rr" ) ): continue add_connection(connections, source, target, sg) # Connect everything to the Gateway endpoints for targetResource in vpc.leaves: if targetResource.has_unrestricted_ingress: for sourceVpc in itertools.chain(vpc.peers, (vpc,)): for sourceResource in sourceVpc.leaves: add_connection(connections, sourceResource, targetResource, []) # Remove connections for source nodes that cannot initiate traffic (ex. VPC endpoints) for connection in list(connections): if not connection.source.can_egress: del connections[connection] return connections
def collect(arguments): account_dir = './{}'.format(arguments.account_name) if arguments.clean and os.path.exists( 'account-data/{}'.format(account_dir)): rmtree('account-data/{}'.format(account_dir)) make_directory("account-data") make_directory("account-data/{}".format(account_dir)) print("* Getting region names") session_data = {'region_name': 'us-east-1'} if arguments.profile_name: session_data['profile_name'] = arguments.profile_name session = boto3.Session(**session_data) ec2 = session.client('ec2') region_list = ec2.describe_regions( RegionNames=['us-east-1', 'ap-northeast-1']) with open("account-data/{}/describe-regions.json".format(account_dir), 'w+') as f: f.write(json.dumps(region_list, indent=4, sort_keys=True)) print("* Creating directory for each region name") for region in region_list['Regions']: make_directory('account-data/{}/{}'.format( account_dir, region.get('RegionName', 'Unknown'))) # Services that will only be queried in us-east-1 universal_services = ['sts', 'iam', 'route53', 'route53domains', 's3'] with open("collect_commands.yaml", 'r') as f: collect_commands = yaml.safe_load(f) for runner in collect_commands: print('* Getting {}:{} info'.format(runner['Service'], runner['Request'])) parameters = {} for region in region_list['Regions']: dynamic_parameter = None # Only call universal services in us-east-1 if runner['Service'] in universal_services and region[ 'RegionName'] != 'us-east-1': continue handler = session.client(runner['Service'], region_name=region['RegionName']) filepath = "account-data/{}/{}/{}-{}".format( account_dir, region['RegionName'], runner['Service'], runner['Request']) method_to_call = snakecase(runner["Request"]) # Identify any parameters if runner.get('Parameters', False): for parameter in runner['Parameters']: parameters[parameter['Name']] = parameter['Value'] # Look for any dynamic values (ones that jq parse a file) if '|' in parameter['Value']: dynamic_parameter = parameter['Name'] if dynamic_parameter is not None: # Set up directory for the dynamic value make_directory(filepath) # The dynamic parameter must always be the first value parameter_file = parameters[dynamic_parameter].split('|')[0] parameter_file = "account-data/{}/{}/{}".format( account_dir, region['RegionName'], parameter_file) if not os.path.isfile(parameter_file): # The file where parameters are obtained from does not exist continue with open(parameter_file, 'r') as f: parameter_values = json.load(f) pyjq_parse_string = '|'.join( parameters[dynamic_parameter].split('|')[1:]) for parameter in pyjq.all(pyjq_parse_string, parameter_values): filename = get_filename_from_parameter(parameter) identifier = get_identifier_from_parameter(parameter) parameters[dynamic_parameter] = identifier outputfile = "{}/{}".format(filepath, filename) call_function(outputfile, handler, method_to_call, parameters) else: filepath = filepath + ".json" call_function(filepath, handler, method_to_call, parameters)
def get_vpc_peerings(region): vpc_peerings = query_aws( region.account, "ec2-describe-vpc-peering-connections", region ) resource_filter = ".VpcPeeringConnections[]?" return pyjq.all(resource_filter, vpc_peerings)
def get_ec2s(region): instances = query_aws(region.account, "ec2-describe-instances", region.region) resource_filter = '.Reservations[]?.Instances[] | select(.State.Name == "running")' return pyjq.all(resource_filter, instances)
def audit_s3_buckets(findings, region): buckets_json = query_aws(region.account, "s3-list-buckets", region) buckets = pyjq.all(".Buckets[].Name", buckets_json) for bucket in buckets: # Check policy try: policy_file_json = get_parameter_file(region, "s3", "get-bucket-policy", bucket) if policy_file_json is not None: # Find the entity we need policy_string = policy_file_json["Policy"] # Load the string value as json policy = json.loads(policy_string) policy = Policy(policy) if policy.is_internet_accessible(): if (len(policy.statements) == 1 and len(policy.statements[0].actions) == 1 and "s3:GetObject" in policy.statements[0].actions): findings.add( Finding(region, "S3_PUBLIC_POLICY_GETOBJECT_ONLY", bucket)) else: findings.add( Finding( region, "S3_PUBLIC_POLICY", bucket, resource_details=policy_string, )) except Exception as e: findings.add( Finding( region, "EXCEPTION", bucket, resource_details={ "policy": policy_string, "exception": str(e), "location": "Exception checking policy of S3 bucket", }, )) # Check ACL try: file_json = get_parameter_file(region, "s3", "get-bucket-acl", bucket) for grant in file_json["Grants"]: uri = grant["Grantee"].get("URI", "") if (uri == "http://acs.amazonaws.com/groups/global/AllUsers" or uri == "http://acs.amazonaws.com/groups/global/AuthenticatedUsers" ): findings.add( Finding(region, "S3_PUBLIC_ACL", bucket, resource_details=grant)) except Exception as e: findings.add( Finding( region, "EXCEPTION", bucket, resource_details={ "grant": grant, "exception": str(e), "location": "Exception checking ACL of S3 bucket", }, ))
# starting with the previous block, to look into the current block # and count how many validators missed it. # so this tool is always "one block behind" previousblock = block - 1 url = baseurl + str(previousblock) result = fetch_json(url) # jq -r '.result.block.header.proposer_address' proposer_address = ( dictor(result, 'result.block.header.proposer_address')) url = baseurl + str(block) # jq -r '.result.block.last_commit.precommits[].validator_address' result = fetch_json(url) validator_addresses = ( pyjq.all('.result.block.last_commit.precommits[].validator_address', result)) # Start out with the assumption that your val did miss the block missed = 1 # and count the number of validators that missed it starting from 0 valmissedcount = 0 # here you append the address of each validator that missed the block valset = "" # See if you missed the block, count how many validators missed, add them to a list for i in validator_addresses: if i == yourval: missed = 0 valset = valset + str(i) + "," elif i is None: valmissedcount = valmissedcount + 1
def amis(args, accounts, config): # Loading the list of public images from disk takes a while, so we'll iterate by region regions_file = "data/aws/us-east-1/ec2-describe-images.json" if not os.path.isfile(regions_file): raise Exception( "You need to download the set of public AMI images. Run:\n" " mkdir -p data/aws\n" " cd data/aws\n" " aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} mkdir {}\n" " aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} sh -c 'aws --region {} ec2 describe-images --executable-users all > {}/ec2-describe-images.json'\n" ) print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( "Account Name", "Region Name", "Instance Id", "Instance Name", "AMI ID", "Is Public", "AMI Description", "AMI Owner", )) for region_name in listdir("data/aws/"): # Get public images public_images_file = "data/aws/{}/ec2-describe-images.json".format( region_name) public_images = json.load(open(public_images_file)) resource_filter = ".Images[]" public_images = pyjq.all(resource_filter, public_images) for account in accounts: account = Account(None, account) region = Region(account, {"RegionName": region_name}) instances = query_aws(account, "ec2-describe-instances", region) resource_filter = ( '.Reservations[].Instances[] | select(.State.Name == "running")' ) if args.instance_filter != "": resource_filter += "|{}".format(args.instance_filter) if "Reservations" not in instances: print(f"** skipping: {account.name} in {region_name}") continue instances = pyjq.all(resource_filter, instances) account_images = query_aws(account, "ec2-describe-images", region) resource_filter = ".Images[]" if "Images" not in account_images: print(f"** skipping: {account.name} in {region_name}") continue account_images = pyjq.all(resource_filter, account_images) for instance in instances: image_id = instance["ImageId"] image_description = "" owner = "" image, is_public_image = find_image(image_id, public_images, account_images) if image: # Many images don't have all fields, so try the Name, then Description, then ImageLocation image_description = image.get("Name", "") if image_description == "": image_description = image.get("Description", "") if image_description == "": image_description = image.get("ImageLocation", "") owner = image.get("OwnerId", "") print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format( account.name, region.name, instance["InstanceId"], get_instance_name(instance), image_id, is_public_image, image_description, owner, ))
#Now let's replicate the twitch python script to generate a NYT #Define the key libraries import pandas as pd import numpy as np import requests import json import pyjq #Store the twitch api url and retrieve the raw data api_key = 'gcpa6XL4z1zLwQXSxW1Y4HniG3zWCWNM' url = 'https://api.nytimes.com/svc/archive/v1/2018/1.json?&api-key='+api_key #now let's extract the relevant fields only JSONdata = requests.get(url).json() JSONdoccount = pyjq.all('.response .docs | length',JSONdata)[0] JSONdataextract = f'.response .docs [] | {{the_snippet: .snippet, the_headline: .headline .main, the_date: .pub_date, the_news_desk: .news_desk}}' JSONdataextractoutput = pyjq.all(JSONdataextract, JSONdata) JSONdataextractoutput_dump = json.dumps(JSONdataextractoutput,indent=4) #print(JSONdataextractoutput_dump) #print("The total number of documents is " + str(JSONdoccount)) #creating a json extract and keeping it locally jsonurl = '/home/salexommer/Documents/yellow-submarine/extracts/nyt_month_data.json' with open(jsonurl,'w') as fi: fi.write(JSONdataextractoutput_dump) df = pd.read_json(jsonurl, orient='columns') #df = pd.read_json(jsonurl, orient='columns') #re-transforming the json into a csv table export_csv = df.to_csv(path_or_buf='/home/salexommer/Documents/yellow-submarine/extracts/nyt_snippet_sample.csv',index=True)
def jq_handler(ctx): r = requests.get(url) return jq.all(jq_string, r.json())
def get_vpc_endpoints(region): endpoints = query_aws(region.account, "ec2-describe-vpc-endpoints", region.region) return pyjq.all(".VpcEndpoints[]?", endpoints)
def event_list(self): full = json.loads(unicode(self.vList)) if self.vIsDebug: print json.dumps(full, indent=2) # get timezone from clsCluster.py a = clsCluster(debug=False) localtz = a.cluster_timezone() if not self.vIsSummary: #self.vIsSummary is true, print detail vm list table = PrettyTable() table.field_names = ['date', 'alert_type', 'severity', 'resolved', 'message'] table.align['message'] = "l" fields = '{ id, created_time_stamp_in_usecs, alert_type_uuid, severity, resolved, message, context_types, context_values }' #if imageuuid is NULL, print summary for all images #else print summary for the special image and it's detail info wrapper = TextWrapper(width=100) epoch = datetime(1970, 1, 1, tzinfo=timezone('UTC')) if len(self.vUUID) == 0: #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR t = pyjq.all('.entities[] | ' + fields, full) i = 0 while i < len(t): mymessage = t[i]['id'] + ' | ' + t[i]['message'] # convert usec to date date_str = epoch + timedelta(seconds=t[i]['created_time_stamp_in_usecs']/1000/1000) date_str_local = date_str.astimezone(timezone(localtz)) # replace {..} string in message to humanable readable. mytype = t[i]['context_types'] myvalue = t[i]['context_values'] mylist = re.findall(r'{[^}]+}', mymessage) for j in mylist: myindex = mytype.index(re.sub('[{}]','',j)) mystr = myvalue[myindex] mymessage = re.sub(j,mystr,mymessage) # create table table.add_row([ str(date_str_local.strftime(format)), t[i]['alert_type_uuid'], t[i]['severity'], t[i]['resolved'], wrapper.fill(mymessage), ]) i = i + 1 else: for i in self.vUUID: s = '.entities[] | select (.id==\"' + i + '\") | ' + fields t = pyjq.all(s, full) mymessage = t[0]['id'] + ' | ' + t[0]['message'] # convert usec to date date_str = epoch + timedelta(seconds=t[0]['created_time_stamp_in_usecs']/1000/1000) date_str_local = date_str.astimezone(timezone(localtz)) # replace {..} string in message to humanable readable. mytype = t[0]['context_types'] myvalue = t[0]['context_values'] mylist = re.findall(r'{[^}]+}', mymessage) for j in mylist: myindex = mytype.index(re.sub('[{}]','',j)) mystr = myvalue[myindex] mymessage = re.sub(j,mystr,mymessage) # create table table.add_row([ str(date_str_local), t[0]['alert_type_uuid'], t[0]['severity'], t[0]['resolved'], wrapper.fill(mymessage), ]) print table else: #self.vIsSummary is false, just print uuid pass
def get_rds_instances(region): instances = query_aws(region.account, "rds-describe-db-instances", region.region) return pyjq.all(".DBInstances[]?", instances)
def get_elbs(region): load_balancers = query_aws( region.account, "elb-describe-load-balancers", region.region ) return pyjq.all(".LoadBalancerDescriptions[]?", load_balancers)
def audit_sg(findings, region): # TODO Check if security groups allow large CIDR range (ex. 1.2.3.4/3) # TODO Check if an SG restricts IPv4 and then opens IPv6 or vice versa. cidrs = {} sg_json = query_aws(region.account, 'ec2-describe-security-groups', region) sgs = pyjq.all('.SecurityGroups[]', sg_json) for sg in sgs: cidr_and_name_list = pyjq.all( '.IpPermissions[].IpRanges[]|[.CidrIp,.Description]', sg) for cidr, name in cidr_and_name_list: if not is_external_cidr(cidr): continue if is_unblockable_cidr(cidr): findings.add( Finding(region, 'SG_CIDR_UNNEEDED', sg['GroupId'], resource_details={'cidr': cidr})) continue if cidr.startswith('0.0.0.0') and not cidr.endswith('/0'): findings.add( Finding(region, 'SG_CIDR_UNEXPECTED', sg['GroupId'], resource_details={'cidr': cidr})) continue if cidr == '0.0.0.0/0': continue cidrs[cidr] = cidrs.get(cidr, list()) cidrs[cidr].append(sg['GroupId']) for ip_permissions in sg['IpPermissions']: cidrs_seen = set() for ip_ranges in ip_permissions['IpRanges']: if 'CidrIp' not in ip_ranges: continue cidr = ip_ranges['CidrIp'] for cidr_seen in cidrs_seen: if (IPNetwork(cidr_seen) in IPNetwork(cidr) or IPNetwork(cidr) in IPNetwork(cidr_seen)): findings.add( Finding(region, 'SG_CIDR_OVERLAPS', sg['GroupId'], resource_details={ 'cidr1': cidr, 'cidr2': cidr_seen })) cidrs_seen.add(cidr) for cidr in cidrs: ip = IPNetwork(cidr) if ip.size > 2048: findings.add( Finding(region, 'SG_LARGE_CIDR', cidr, resource_details={ 'size': ip.size, 'security_groups': cidrs[cidr] }))
def get_account_stats(account, all_resources=False): """Returns stats for an account""" with open("stats_config.yaml", "r") as f: resources = yaml.safe_load(f) account = Account(None, account) log_debug("Collecting stats in account {} ({})".format( account.name, account.local_id)) stats = {} stats["keys"] = [] for resource in resources: # If the resource is marked as verbose, and we're not showing all resources, skip it. if resource.get("verbose", False) and not all_resources: continue stats["keys"].append(resource["name"]) stats[resource["name"]] = {} for region_json in get_regions(account): region = Region(account, region_json) for resource in resources: if resource.get("verbose", False) and not all_resources: continue # Skip global services (just CloudFront) if ("region" in resource) and (resource["region"] != region.name): continue # S3 buckets require special code to identify their location if resource["name"] == "S3 buckets": if region.name == "us-east-1": buckets = pyjq.all( ".Buckets[].Name", query_aws(region.account, "s3-list-buckets", region), ) for bucket in buckets: # Get the bucket's location bucket_region = get_parameter_file( region, "s3", "get-bucket-location", bucket)["LocationConstraint"] # Convert the value to a name. # See https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region if bucket_region is None: bucket_region = "us-east-1" elif bucket_region == "EU": bucket_region = "eu-west-1" # Increment the count tmp = stats[resource["name"]].get(bucket_region, 0) stats[resource["name"]][bucket_region] = tmp + 1 else: # Normal path stats[resource["name"]][region.name] = sum( pyjq.all( resource["query"], query_aws(region.account, resource["source"], region), )) return stats
def get_elbv2s(region): # ALBs and NLBs load_balancers = query_aws( region.account, "elbv2-describe-load-balancers", region.region ) return pyjq.all(".LoadBalancers[]?", load_balancers)
c = GetImage( hostname=param.vHostname, port=param.vPort, username=param.vUsername, password=param.vPassword, debug=False ) b = json.loads(str(c)) print json.dumps(b, indent=4) sys.exit(9) if 'myerror' in d: print json.dumps(d, indent=4) else: table = PrettyTable() table.field_names = ['Image_Name', 'Image_UUID', 'Image_Type', 'VM_Disk_ID'] #if imageuuid is NULL, print summary for all images #else print summary for the special image and it's detail info if param.vImageUUID == '': #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR t = pyjq.all('.entities[] | {name, uuid, image_type, vm_disk_id}', d) i = 0 while i < len(t): table.add_row([t[i]['name'], t[i]['uuid'], t[i]['image_type'], t[i]['vm_disk_id']]) i = i + 1 else: #cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR s = '.entities[] | select (.uuid==\"' + param.vImageUUID + '\") | {name, uuid, image_type, vm_disk_id}' t = pyjq.all(s, d) table.add_row([t[0]['name'], t[0]['uuid'], t[0]['image_type'], t[0]['vm_disk_id']]) print table
def vm_create_from_isoimage(self, vmsize='', imageuuid='', netuuid='', containeruuid=''): #check vmsize input if vmsize.lower() == 'large': vcpustring = 2 corestring = 2 memorystring = 8192 diskstring = 214748364800 elif vmsize.lower() == 'middle': vcpustring = 1 corestring = 2 memorystring = 4096 diskstring = 107374182400 elif vmsize.lower() == 'small': vcpustring = 1 corestring = 1 memorystring = 2048 diskstring = 53687091200 else: print 'vmsize invalid' sys.exit(9) #check diskimage uuid input test = clsImage() if test.image_disktype(uuid=imageuuid): print 'diskimage uuid is not ISO_IMAGE' sys.exit(9) else: full = json.loads(str(test.vList)) d = pyjq.all('.entities[] | { uuid, vm_disk_id }', full) #return True or False for u in d: if u['uuid'] == imageuuid: vmdiskuuid = u['vm_disk_id'] #check container uuid input test = clsContainer() if not test.container_existed(uuid=containeruuid): print 'container uuid is invalid' sys.exit(9) #check net uuid input test = clsNet() if not test.net_existed(uuid=netuuid): print 'network uuid is invalid' sys.exit(9) vmname = "VM-" + strftime("%Y%m%d-%H%M%S", localtime()) #make postdata postdata = { "name": vmname, "description": "", "memory_mb": memorystring, "num_cores_per_vcpu": corestring, "num_vcpus": vcpustring, "timezone": "Asia/Shanghai", "vm_disks": [ { "disk_address": { "device_bus": "IDE", "device_index": 0 }, "is_cdrom": "true", "is_empty": "false", "vm_disk_clone": { "disk_address": { "vmdisk_uuid": vmdiskuuid }} }, { "disk_address": { "device_bus": "SCSI", "device_index": 0 }, "is_cdrom": "false", "is_empty": "true", "vm_disk_create": { "size": diskstring, "storage_container_uuid": containeruuid } } ], "vm_nics": [ { "adapter_type": "E1000", "network_uuid": netuuid, "request_ip": "false" } ] } if self.vIsDebug: print json.dumps(postdata,indent=4) url = '/PrismGateway/services/rest/v2.0/vms/' a = clsURL( debug=self.vIsDebug, url=url, postdata=postdata ) result = json.loads(str(a.PostURL())) if 'myerror' in result: print ' `--error:', result['myerror'] else: print ' `-- ./getTask.py --uuid %s' % (result['task_uuid'])