def test_url_argument(self):
class FakeResponse:
def getheader(self, name):
return 'application/json;charset=SHIFT_JIS'
def read(self):
return '["Hello", "世界", "！"]'.encode('shift-jis')
try:
import urllib.request
del urllib
except ImportError:
to_patch = 'urllib2.urlopen'
else:
to_patch = 'urllib.request.urlopen'
to_patch = 'six.moves.urllib.request.urlopen'
with patch(to_patch, return_value=FakeResponse()):
self.assertEqual(
pyjq.all('.[] | . + .', url='http://example.com'),
["HelloHello", "世界世界", "！！"]
)
def opener(url):
return "[1, 2, 3]"
self.assertEqual(
pyjq.all('.[] | . + .', url='http://example.com', opener=opener),
[2, 4, 6]
)
def image_list(self):
full = json.loads(unicode(self.vList))
if self.vIsDebug:
print json.dumps(full, indent=2)
table = PrettyTable()
table.field_names = ['Image_Name', 'Image_UUID', 'Image_Type', 'VM_Disk_ID']
table.align['Image_Name'] = "l"
#if imageuuid is NULL, print summary for all images
#else print summary for the special image and it's detail info
if len(self.vUUID) == 0:
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR
t = pyjq.all('.entities[] | {name, uuid, image_type, vm_disk_id}', full)
i = 0
while i < len(t):
table.add_row([t[i]['name'], t[i]['uuid'], t[i]['image_type'], t[i]['vm_disk_id']])
i = i + 1
else:
for i in self.vUUID:
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR
s = '.entities[] | select (.uuid==\"' + i + '\") | {name, uuid, image_type, vm_disk_id}'
t = pyjq.all(s, full)
table.add_row([t[0]['name'], t[0]['uuid'], t[0]['image_type'], t[0]['vm_disk_id']])
if self.vIsHTML:
return table.get_html_string(sortby='Image_Name', attributes = self.table_style)
else:
return table.get_string(sortby='Image_Name')
def test_all(self):
self.assertEqual(
pyjq.all('.[] | . + $foo', ['val1', 'val2'], vars=dict(foo='bar')),
['val1bar', 'val2bar']
)
self.assertEqual(
pyjq.all('. + $foo', 'val', vars=dict(foo='bar')),
['valbar']
)
def image_disktype(self):
full = json.loads(unicode(self.vList))
d = pyjq.all('.entities[] | { uuid, image_type }', full)
for u in d:
if u['uuid'] == self.vUUID[0] and u['image_type'] == 'DISK_IMAGE':
return True
return False
def test_library_path(self):
library_path = tempfile.mkdtemp()
library_path2 = tempfile.mkdtemp()
library_file = os.path.join(library_path, "greeting.jq")
library_file2 = os.path.join(library_path2, "increment.jq")
try:
with io.open(library_file, 'w', encoding='ascii') as f:
f.write('def hello: "HELLO";')
f.write('def world: "WORLD";')
with io.open(library_file2, 'w', encoding='ascii') as f:
f.write('def increment: . + 1;\n')
values = pyjq.all(
'include "greeting"; include "increment"; .[] | [. | increment, hello, world]',
[1, 2, 3],
library_paths = [library_path, library_path2]
)
self.assertEquals(
[[2, "HELLO", "WORLD"], [3, "HELLO", "WORLD"], [4, "HELLO", "WORLD"]],
values
)
finally:
shutil.rmtree(library_path)
shutil.rmtree(library_path2)
def doPcdmIndirect(self):
self.log("Running doPcdmIndirect")
self.log("Create a PCDM container")
basic_headers = {
'Link': self.make_type(TestConstants.LDP_BASIC),
'Content-type': 'text/turtle'
}
r = self.do_post(self.getBaseUri(), headers=basic_headers, body=TestConstants.OBJECT_TTL)
self.assertEqual(201, r.status_code, "Did not get expected status code")
pcdm_container_location = self.get_location(r)
self.log("Create a PCDM Collection")
r = self.do_post(self.getBaseUri(), headers=basic_headers, body=TestConstants.OBJECT_TTL)
self.assertEqual(201, r.status_code, "Did not get expected status code")
pcdm_collection_location = self.get_location(r)
self.log("Create indirect container inside PCDM collection")
indirect_headers = {
'Link': self.make_type(TestConstants.LDP_INDIRECT),
'Content-type': 'text/turtle'
}
pcdm_indirect = "@prefix dc: <http://purl.org/dc/elements/1.1/> ." \
"@prefix pcdm: <http://pcdm.org/models#> ." \
"@prefix ore: <http://www.openarchives.org/ore/terms/> ." \
"@prefix ldp: <http://www.w3.org/ns/ldp#> ." \
"<> dc:title \"Members Container\" ;" \
" ldp:membershipResource <{0}> ;" \
" ldp:hasMemberRelation pcdm:hasMember ;" \
" ldp:insertedContentRelation ore:proxyFor .".format(pcdm_collection_location)
r = self.do_post(pcdm_collection_location, headers=indirect_headers, body=pcdm_indirect)
self.assertEqual(201, r.status_code, "Did not get expected status code")
members_location = self.get_location(r)
self.log("Create a proxy object")
pcdm_proxy = "@prefix pcdm: <http://pcdm.org/models#>" \
"@prefix ore: <http://www.openarchives.org/ore/terms/>" \
"<> a pcdm:Object ;" \
"ore:proxyFor <{0}> .".format(pcdm_container_location)
r = self.do_post(members_location, headers=basic_headers, body=pcdm_proxy)
self.assertEqual(201, r.status_code, "Did not get expected status code")
self.log("Check PCDM Collection for a new pcmd:hasMember property to PCDM container")
headers = {
'Accept': TestConstants.JSONLD_MIMETYPE
}
r = self.do_get(pcdm_collection_location, headers=headers)
self.assertEqual(200, r.status_code, "Did not get expected status code")
body = r.content.decode('UTF-8')
body_json = json.loads(body)
hasmembers = pyjq.all('.[] | ."http://pcdm.org/models#hasMember" | .[]."@id"', body_json)
found_member = False
for member in hasmembers:
if member == pcdm_container_location:
found_member = True
self.assertTrue(found_member, "Did not find hasMember property")
self.log("Passed")
def image_config(self):
if len(self.vUUID) == 0:
print 'image uuid is NULL.'
sys.exit(9)
for i in self.vUUID:
full = json.loads(unicode(self.vList))
t = pyjq.all('.entities[] | select (.uuid==\"' + i + '\") ', full)
print json.dumps(t, indent=4)
def vm_config(self, uuid=''):
if uuid == '':
print 'vm uuid is NULL.'
sys.exit(9)
for i in uuid:
if not self.vm_existed(uuid=i):
print 'vm uuid not existed.'
sys.exit(9)
full = json.loads(unicode(self.vVMList))
t = pyjq.all('.entities[] | select (.uuid==\"' + i + '\") ', full)
print json.dumps(t, indent=4)
def task_print(self, uuid=''):
#
# call this function with uuid parameter, or you will get full list
#
from prettytable import PrettyTable
if not hasattr(self, 'vList'):
self.task_list()
full = json.loads(str(self.vList))
t = pyjq.all('. | { uuid, operation_type, progress_status }', full)
t1 = pyjq.all('.entity_list[] | { entity_id, entity_type }', full)
table = PrettyTable(['task uuid', 'operation type', 'progress status'])
if len(t) != 0:
table.add_row([t[0]['uuid'], t[0]['operation_type'], t[0]['progress_status']])
i = 0
while i < len(t1):
#table.add_row([i, t1[i]['entity_type'], t1[i]['entity_id']])
c = 'entity_type:' + t1[i]['entity_type']
table.add_column(c, [t1[i]['entity_id']])
i = i + 1
print table
def vg_list(self):
full = json.loads(unicode(self.vList))
if self.vIsDebug:
print json.dumps(full, indent=2)
if not self.vIsSummary: #self.vIsSummary is true, print detail vm list
table = PrettyTable()
table.field_names = ['VG_UUID', 'VG_Name', 'Disk_List', 'Attachment_List']
fields = '{ uuid, name, disk_list, attachment_list}'
#if imageuuid is NULL, print summary for all images
#else print summary for the special image and it's detail info
if len(self.vUUID) == 0:
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR
t = pyjq.all('.entities[] | ' + fields, full)
i = 0
while i < len(t):
try:
disk_num = len(t[i]['disk_list'])
except TypeError:
disk_num = 0
try:
attachment_num = len(t[i]['attachment_list'])
except TypeError:
attachment_num = 0
table.add_row([t[i]['uuid'], t[i]['name'], disk_num, attachment_num])
i = i + 1
else:
for i in self.vUUID:
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR
s = '.entities[] | select (.uuid==\"' + i + '\") | ' + fields
t = pyjq.all(s, full)
table.add_row([t[0]['uuid'], t[0]['name'], len(t[0]['disk_list']), len(t[0]['attachment_list'])])
if self.vIsHTML:
html_str = table.get_html_string(attributes = self.table_style)
count = html_str.count('\033[0;m')
if count > 0:
html_str = colorhtml(html_str)
return html_str
else:
return table
else: #self.vIsSummary is false, just print uuid
pass
def vm_existed(self, uuid=''):
if uuid == '':
print 'vm uuid is NULL.'
sys.exit(9)
# check vm_list() loaded or not
if not hasattr(self, 'vVMList'):
self.vm_list()
full = json.loads(unicode(self.vVMList))
d = pyjq.all('.entities[]', full)
for u in d:
if u['metadata']['uuid'] == uuid:
return True
return False
def assertTitleExists(self, expected, location):
""" Check resource at {location} for the dc:title {expected} """
get_headers = {
'Accept': TestConstants.JSONLD_MIMETYPE
}
response = self.do_get(location, headers=get_headers)
body = response.content.decode('UTF-8')
json_body = json.loads(body)
found_title = pyjq.all('.[] | ."http://purl.org/dc/elements/1.1/title" | .[]."@value"', json_body)
for title in found_title:
if title == expected:
return
self.fail("Did not find expected title \"{0}\" in response".format(expected))
def __init__(self, debug=False, hostname='', port='', token='', foo=False, color=True, html=False, summary=False, uuid=[]):
self.vIsDebug = debug
self.vHostname = hostname
self.vPort = port
self.vToken = token
self.vIsColor = color
self.vIsFoo = foo
self.vIsHTML = html
self.vIsSummary = summary
self.vUUID = uuid
# when you want to output comments and tables, you need this
# called in vg_listdisk
self.capture = StringIO()
self.save_stdout = sys.stdout
# table css style
# called by attributes in get_html_string, when output of vg_list vg_listdisk
self.table_style = {"name": "vg-table", "class": "table table-sm table-striped"}
if not self.vIsFoo:
url = '/PrismGateway/services/rest/v2.0/volume_groups/?include_disk_size=true'
a = clsURL(
debug=self.vIsDebug,
hostname=self.vHostname,
port=self.vPort,
token=self.vToken,
url=url
)
self.vList = a.GetURL()
#return(unicode(self.vList))
else:
with open('test_clsVg.json', 'r') as f:
self.vList = json.load(f)
if len(self.vUUID) != 0:
self.vExisted = list(self.vUUID)
full = json.loads(unicode(self.vList))
d = pyjq.all('.entities[] | { uuid }', full)
for i in self.vUUID:
for u in d:
if u['uuid'] == i:
self.vExisted.remove(i)
break
if len(self.vExisted) != 0:
print 'vg uuid %s is not existed.' % self.vExisted
sys.exit(19)
def vm_in_net(self, netuuid=''):
if netuuid == '':
print 'net uuid is NULL.'
sys.exit(9)
if not hasattr(self, 'vVMList'):
self.vm_list()
full = json.loads(str(self.vVMList))
if self.vIsDebug:
print json.dumps(full, indent=4)
full_order = pyjq.all('.entities[] | { uuid, name, power_state, vm_nics }', full)
vmlist = []
for vm in full_order:
for nic in vm['vm_nics']:
if nic['network_uuid'] == netuuid:
vmlist.append(vm['uuid'])
if len(vmlist) != 0:
self.vm_print(uuid=vmlist)
def vg_listdisk(self):
sys.stdout = self.capture
if len(self.vUUID) == 0:
print 'vg uuid is NULL.'
sys.exit(9)
full = json.loads(unicode(self.vList))
for h in self.vUUID:
table = PrettyTable()
table.field_names = ['disk index', 'disk uuid', 'size (GB)', 'Container']
t = pyjq.all('.entities[] | select (.uuid==\"' + h + '\") | { disk_list }', full)
myset = list()
for i in t[0]['disk_list']:
a = clsContainer(debug=False, hostname=self.vHostname, port=self.vPort, token=self.vToken, foo=self.vIsFoo, uuid=[i['storage_container_uuid']])
container_name = a.container_getname()
myset.append(container_name)
if self.vIsColor:
container_name = colourise('blue', container_name)
else:
container_name = container_name
table.add_row([
i['index'],
i['vmdisk_uuid'],
i['vmdisk_size_bytes']/1024/1024/1024,
container_name
])
sys.stdout.write('vguuid:' + h)
# BPG
if len(list(set(myset))) > 1:
print " (%s container)" % colourise('red', 'In Different')
else:
print " (%s container)" % colourise('green', 'In Same')
if self.vIsHTML:
html_str = table.get_html_string(attributes = self.table_style)
count = html_str.count('\033[0;m')
if count > 0:
html_str = colorhtml(html_str)
print html_str
else:
print table
if self.vIsDebug:
print json.dumps(t, indent=2)
sys.stdout = self.save_stdout
return self.capture
def __init__(self, debug=False, foo=False, color=True, html=False, summary=False, uuid=[], msgtype='alerts', severity='', resolved='', count=10):
self.vIsColor = color
self.vIsDebug = debug
self.vIsFoo = foo
self.vIsHTML = html
self.vIsSummary = summary
self.vUUID = uuid
self.vType = msgtype
self.vSeverity = severity
self.vResolved = resolved
self.vCount = count
if not self.vIsFoo:
if self.vSeverity == '':
severity_str = ''
else:
severity_str = '&severity=' + self.vSeverity
if self.vResolved == '':
resolved_str = ''
else:
resolved_str = '&resolved=' + self.vResolved
url = '/PrismGateway/services/rest/v2.0/' + self.vType + '/?count=' + str(self.vCount) + severity_str + resolved_str
a = clsURL(
debug=self.vIsDebug,
url=url
)
self.vList = a.GetURL()
#return(unicode(self.vList))
else:
with open('test_clsEvent.json', 'r') as f:
self.vList = json.load(f)
if len(self.vUUID) != 0:
self.vExisted = list(self.vUUID)
full = json.loads(unicode(self.vList))
d = pyjq.all('.entities[] | { id }', full)
for i in self.vUUID:
for u in d:
if u['id'] == i:
self.vExisted.remove(i)
break
if len(self.vExisted) != 0:
print 'event id %s is not existed.' % self.vExisted
sys.exit(19)
def cluster_list(self):
full = json.loads(unicode(self.vList))
if self.vIsDebug:
print json.dumps(full, indent=4)
table = PrettyTable()
table.field_names = ['Cluster Name', 'Prism Central', 'Cluster IP', 'DataService IP', 'NTP Servers', 'Timezone', 'ShadowClone', 'Hypervisor', 'AOS Version', 'Cluster Functions']
#if imageuuid is NULL, print summary for all images
#else print summary for the special image and it's detail info
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR
s = '. | { name, multicluster, cluster_external_ipaddress, cluster_external_data_services_ipaddress, ntp_servers, timezone, enable_shadow_clones, hypervisor_types, version, cluster_functions }'
t = pyjq.all(s, full)
if self.vIsColor:
# BGP
if t[0]['enable_shadow_clones'] == True:
shadowclone = colourise('red', 'Enabled')
else:
shadowclone = colourise('green', 'Disabled')
else:
if t[0]['enable_shadow_clones'] == True:
shadowclone = 'Enabled'
else:
shadowclone = 'Disabled'
table.add_row([
t[0]['name'],
t[0]['multicluster'],
t[0]['cluster_external_ipaddress'],
t[0]['cluster_external_data_services_ipaddress'],
', '.join(t[0]['ntp_servers']),
t[0]['timezone'],
shadowclone,
', '.join(t[0]['hypervisor_types']),
t[0]['version'],
', '.join(t[0]['cluster_functions']),
])
if self.vIsHTML:
html_str = table.get_html_string(attributes = self.table_style)
count = html_str.count('\033[0;m')
if count > 0:
html_str = colorhtml(html_str)
return html_str
else:
return table
def __init__(self, debug=False, hostname='', port='', token='', foo=False, html=False, color=False, summary=False, uuid=[]):
self.vIsDebug = debug
self.vHostname = hostname
self.vPort = port
self.vToken = token
self.vIsColor = color
self.vIsFoo = foo
self.vIsHTML = html
self.vIsSummary = summary
self.vUUID = uuid
self.table_style = {"name": "vm-table", "class": "table table-sm table-striped"}
if not self.vIsFoo:
url = '/PrismGateway/services/rest/v2.0/images/'
a = clsURL(
debug=self.vIsDebug,
hostname=self.vHostname,
port=self.vPort,
token=self.vToken,
url=url
)
self.vList = a.GetURL()
if self.vIsDebug:
print json.dumps(json.loads(str(self.vList)), indent=2)
else:
with open('test/test_clsImage.json', 'r') as f:
self.vList = json.load(f)
if len(self.vUUID) != 0:
self.vExisted = list(self.vUUID)
full = json.loads(unicode(self.vList))
d = pyjq.all('.entities[] | { uuid }', full)
for i in self.vUUID:
for u in d:
if u['uuid'] == i:
self.vExisted.remove(i)
break
if len(self.vExisted) != 0:
print 'image uuid %s is not existed.' % self.vExisted
sys.exit(19)
def policy_replicate(self):
postdata = {
"metadata": {
"categories": {},
"kind": "network_security_rule",
"spec_version": 0
}
}
url = '/api/nutanix/v3/network_security_rules'
full = json.loads(unicode(self.vList))
s = '.entities[] | { spec, metadata }'
t = pyjq.all(s, full)
unique_str = str(random.randint(1,1000))
for i in self.vUUID:
for j in t:
rep_str = j.copy()
if j['metadata']['uuid'] == i:
rep_str['spec']['name'] = 'CLI ' + unique_str
rep_str['spec']['description'] = 'CLI ' + unique_str
rep_str['metadata'] = postdata['metadata']
#for k in rep_str['spec']['resources']['app_rule']['outbound_allow_list']:
# print k
#break
#a = re.sub('"','\'',json.dumps(rep_str))
#print a
#break
if self.vIsDebug:
print json.loads(rep_str)
print url
a = clsURL(
debug=self.vIsDebug,
hostname=self.vHostname,
port=self.vPort,
token=self.vToken,
postdata=json.loads(json.dumps(rep_str)),
url=url
)
result = a.PostURL()
print json.dumps(json.loads(result), indent=2)
break
def main():
parser = ArgumentParser()
parser.add_argument('host')
parser.add_argument('port')
parser.add_argument('job')
parser.add_argument('build_number', nargs='?', default='lastSuccessfulBuild')
parser.add_argument('-o', '--output-dir', action='store', dest='output_dir', default='artifacts')
parser.add_argument('--console-text', '-c', action='store_true', dest='console_text')
args = parser.parse_args()
output_dir = Path(args.output_dir)
create_empty_dir(output_dir)
def api_url(url):
return urljoin(url, 'api/json')
build_api_url = api_url('http://{0.host}:{0.port}/job/{0.job}/{0.build_number}/'.format(args))
for run_url in jq.all('.runs[].url', url=build_api_url):
subjob_url = urljoin(run_url, '../')
subjob_name = jq.one('.displayName', url=api_url(subjob_url))
subjob_dir = output_dir / urllib.request.quote(subjob_name, '')
if not subjob_dir.is_dir():
subjob_dir.mkdir(parents=True)
with (subjob_dir / 'consoleText').open('wb') as local_fp, \
urlopen(urljoin(run_url, 'consoleText')) as http_fp:
shutil.copyfileobj(http_fp, local_fp)
zip_fp = io.BytesIO(urlopen(urljoin(run_url, 'artifact/*zip*/archive.zip')).read())
with ZipFile(zip_fp) as z:
for name in z.namelist():
prefix = 'archive/'
if not name.startswith(prefix): continue
path = subjob_dir / name[len(prefix):]
if not path.parent.is_dir():
path.parent.mkdir(parents=True)
with path.open('wb') as fp:
fp.write(z.read(name))
def policy_list(self):
# The attribute vList just been define in this function.
# check with "hasattr(self, 'vList')" in another function.
postdata = {
"filter": ""
}
a = clsURL(
debug=self.vIsDebug,
hostname=self.vHostname,
port=self.vPort,
token=self.vToken,
postdata=postdata,
url='/api/nutanix/v3/network_security_rules/list'
)
self.vList = a.PostURL()
#print json.dumps(unicode(self.vList))
full = json.loads(unicode(self.vList))
if self.vIsDebug:
print json.dumps(full, indent=4)
table = PrettyTable()
table.field_names = ['Policy Name', 'UUID']
s = '.entities[] | { spec, metadata }'
t = pyjq.all(s, full)
for u in t:
table.add_row([
u['spec']['name'],
u['metadata']['uuid']
])
if self.vIsHTML:
html_str = table.get_html_string(attributes = self.table_style)
count = html_str.count('\033[0;m')
if count > 0:
html_str = colorhtml(html_str)
return html_str
else:
return table
def get_iam_trusts(account, nodes, connections, connections_to_get):
# Get IAM
iam = query_aws(account, "iam-get-account-authorization-details",
Region(account, {'RegionName': 'us-east-1'}))
for role in pyjq.all('.RoleDetailList[]', iam):
principals = pyjq.all(
'.AssumeRolePolicyDocument.Statement[].Principal', role)
for principal in principals:
assume_role_nodes = set()
if principal.get('Federated', None):
# TODO I should be using get-saml-provider to confirm this is really okta
if "saml-provider/okta" in principal['Federated'].lower():
node = Account(json_blob={
'id': 'okta',
'name': 'okta',
'type': 'Okta'
})
assume_role_nodes.add(node)
elif "saml-provider/adfs" in principal['Federated'].lower():
node = Account(json_blob={
'id': 'adfs',
'name': 'adfs',
'type': 'ADFS'
})
assume_role_nodes.add(node)
elif principal[
'Federated'] == 'cognito-identity.amazonaws.com':
# TODO: Should show this somehow
continue
elif principal['Federated'] == 'www.amazon.com':
node = Account(json_blob={
'id': 'Amazon.com',
'name': 'Amazon.com',
'type': 'Amazon'
})
continue
else:
raise Exception('Unknown federation provider: {}'.format(
principal['Federated']))
if principal.get('AWS', None):
principal = principal['AWS']
if not isinstance(principal, list):
principal = [principal]
for p in principal:
if "arn:aws" not in p:
# The role can simply be something like "AROA..."
continue
parts = p.split(':')
account_id = parts[4]
assume_role_nodes.add(Account(account_id=account_id))
for node in assume_role_nodes:
if nodes.get(node.id, None) is None:
nodes[node.id] = node
access_type = 'iam'
# TODO: Identify all admins better. Use code from find_admins.py
for m in role['AttachedManagedPolicies']:
for p in pyjq.all('.Policies[]', iam):
if p['Arn'] == m['PolicyArn']:
for policy_doc in p['PolicyVersionList']:
if policy_doc['IsDefaultVersion'] == True:
if is_admin_policy(policy_doc['Document']):
access_type = 'admin'
for policy in role['RolePolicyList']:
policy_doc = policy['PolicyDocument']
if is_admin_policy(policy_doc):
access_type = 'admin'
if ((access_type == 'admin' and connections_to_get['admin'])
or (access_type != 'admin'
and connections_to_get['iam_nonadmin'])):
connections[Connection(node, account, access_type)] = []
return
def extract_evidence(self, obj_json: str):
parsed = json.loads(obj_json.lower())
findings = pyjq.all('to_entries[] | select((.key|startswith("name")) or (.key|endswith("id")))',parsed)
return str(findings)
def cluster_timezone(self):
full = json.loads(unicode(self.vList))
if self.vIsDebug:
print json.dumps(full, indent=4)
t = pyjq.all('. | { timezone }', full)
return t[0]['timezone']
def audit_sg(findings, region):
# TODO Check if security groups allow large CIDR range (ex. 1.2.3.4/3)
# TODO Check if an SG restricts IPv4 and then opens IPv6 or vice versa.
cidrs = {}
sg_json = query_aws(region.account, "ec2-describe-security-groups", region)
sgs = pyjq.all(".SecurityGroups[]", sg_json)
for sg in sgs:
cidr_and_name_list = pyjq.all(
".IpPermissions[]?.IpRanges[]|[.CidrIp,.Description]", sg)
for cidr, name in cidr_and_name_list:
if not is_external_cidr(cidr):
continue
if is_unblockable_cidr(cidr):
findings.add(
Finding(
region,
"SG_CIDR_UNNEEDED",
sg["GroupId"],
resource_details={"cidr": cidr},
))
continue
if cidr.startswith("0.0.0.0") and not cidr.endswith("/0"):
findings.add(
Finding(
region,
"SG_CIDR_UNEXPECTED",
sg["GroupId"],
resource_details={"cidr": cidr},
))
continue
if cidr == "0.0.0.0/0":
continue
cidrs[cidr] = cidrs.get(cidr, list())
cidrs[cidr].append(sg["GroupId"])
for ip_permissions in sg.get("IpPermissions", []):
cidrs_seen = set()
for ip_ranges in ip_permissions.get("IpRanges", []):
if "CidrIp" not in ip_ranges:
continue
cidr = ip_ranges["CidrIp"]
for cidr_seen in cidrs_seen:
if IPNetwork(cidr_seen) in IPNetwork(cidr) or IPNetwork(
cidr) in IPNetwork(cidr_seen):
findings.add(
Finding(
region,
"SG_CIDR_OVERLAPS",
sg["GroupId"],
resource_details={
"cidr1": cidr,
"cidr2": cidr_seen
},
))
cidrs_seen.add(cidr)
for cidr in cidrs:
ip = IPNetwork(cidr)
if ip.size > 2048:
findings.add(
Finding(
region,
"SG_LARGE_CIDR",
cidr,
resource_details={
"size": ip.size,
"security_groups": list(cidrs[cidr])
},
))
def kafka_producer_common_news(producer):
print("*" * 10, "COMMON NEWS", "*" * 10)
page = -1
total_pages = 0
news_desk = ["Business Day", "Business", "Entrepreneurs","Financial", "Magazine", "Personal Investing", "Personal Tech", "Politics", "Retail", "Small Business", \
"Sunday Business", "Technology", "Washington", "Week", "Your Money", "World"]
for news in news_desk:
print("*" * 80)
print('Fetching for the news :', news)
page = -1
total_pages = 0
start_date = '20180101'
end_date = '20200401'
api = 'v9gJZYhGqTvAgpqfi4b6GxxtdiTJvxi8'
while page <= total_pages and start_date < end_date:
temp_end_date = "20200401"
page += 1
mydict = query(start_date, end_date, page, news)
total_pages = pyjq.all('.response .meta .hits', mydict)[0]
if total_pages:
total_pages = math.floor(total_pages / 10)
else:
total_pages = 0
time.sleep(8)
doc_lis = mydict["response"]["docs"]
print('len(doc_lis)', len(doc_lis))
for i, doc in enumerate(doc_lis):
msg_str = "{}_{}||{}".format("Common", "Sector", str(doc))
producer.send(topic='common_news',
value=bytes(msg_str, 'utf-8'))
while (total_pages > 200):
print("Inside while, total_pages = ", total_pages)
date_object = monthdelta(end_date, -1)
temp_end_date = date_object.strftime("%Y%m%d")
mydict = query(start_date, temp_end_date, page, news)
total_pages = pyjq.all('.response .meta .hits', mydict)[0]
if total_pages:
total_pages = math.floor(total_pages / 10)
else:
total_pages = 0
end_date = temp_end_date
time.sleep(8)
doc_lis = mydict["response"]["docs"]
print('len(doc_lis)', len(doc_lis))
for i, doc in enumerate(doc_lis):
doc["symbol"] = "Common"
doc["sector"] = "Sector"
msg_str = "{}".format(str(doc))
producer.send(topic='common_news',
value=bytes(msg_str, 'utf-8'))
start_date = temp_end_date
end_date = '20200401'
def get_azs(vpc):
azs = query_aws(vpc.account, "ec2-describe-availability-zones", vpc.region)
resource_filter = ".AvailabilityZones[]"
return pyjq.all(resource_filter, azs)
def get_subnets(az):
subnets = query_aws(az.account, "ec2-describe-subnets", az.region)
resource_filter = (
'.Subnets[] | select(.VpcId == "{}") | select(.AvailabilityZone == "{}")'
)
return pyjq.all(resource_filter.format(az.vpc.local_id, az.local_id), subnets)
def get_lambda_functions(region):
functions = query_aws(region.account, "lambda-list-functions", region.region)
return pyjq.all(".Functions[]?|select(.VpcConfig!=null)", functions)
def get_sgs(vpc):
sgs = query_aws(vpc.account, "ec2-describe-security-groups", vpc.region)
return pyjq.all(
'.SecurityGroups[]? | select(.VpcId == "{}")'.format(vpc.local_id), sgs
)
def audit_s3_buckets(findings, region):
buckets_json = query_aws(region.account, "s3-list-buckets", region)
buckets = pyjq.all('.Buckets[].Name', buckets_json)
for bucket in buckets:
# Check policy
try:
policy_file_json = get_parameter_file(region, 's3',
'get-bucket-policy', bucket)
if policy_file_json is not None:
# Find the entity we need
policy_string = policy_file_json['Policy']
# Load the string value as json
policy = json.loads(policy_string)
policy = Policy(policy)
if policy.is_internet_accessible():
if len(policy.statements) == 1 and len(
policy.statements[0].actions
) == 1 and 's3:GetObject' in policy.statements[0].actions:
findings.add(
Finding(region, 'S3_PUBLIC_POLICY_GETOBJECT_ONLY',
bucket))
else:
findings.add(
Finding(region,
'S3_PUBLIC_POLICY',
bucket,
resource_details=policy_string))
region, issue_id, resource_id, resource_details
except Exception as e:
findings.add(
Finding(region,
'EXCEPTION',
bucket,
resource_details={
'policy': policy_string,
'exception': e,
'location':
'Exception checking policy of S3 bucket'
}))
# Check ACL
try:
file_json = get_parameter_file(region, 's3', 'get-bucket-acl',
bucket)
for grant in file_json['Grants']:
uri = grant['Grantee'].get('URI', "")
if (uri == 'http://acs.amazonaws.com/groups/global/AllUsers'
or uri ==
'http://acs.amazonaws.com/groups/global/AuthenticatedUsers'
):
findings.add(
Finding(region,
'S3_PUBLIC_ACL',
bucket,
resource_details=grant))
except Exception as e:
findings.add(
Finding(region,
'EXCEPTION',
bucket,
resource_details={
'grant': grant,
'exception': e,
'location': 'Exception checking ACL of S3 bucket'
}))
debug=param.vIsDebug,
hostname=param.vHostname, port=param.vPort,
username=param.vUsername, password=param.vPassword,
snapshotuuid=param.vSnapshotUUID,
url=param.vURL, urltype=param.vURLType
)
d = json.loads(str(data))
if 'myerror' in d:
print json.dumps(d, indent=4)
else:
table = PrettyTable()
#print json.dumps(d['entities'][1])
#if vSnapshotUUID is NULL, print summary for all snapshots
#else print summary for the special snapshot and it's vm's detail info
if param.vSnapshotUUID == '':
t = pyjq.all('.entities[] | {uuid, snapshot_name, vm_uuid}', d)
#while i < d['metadata']['total_entities']:
table.field_names = ['Snapshot_UUID', 'Snapshot_Name', 'VM_UUID']
i = 0
while i < len(t):
#print '%s | %s | %s' % (t[i]['uuid'], t[i]['vm_uuid'], t[i]['snapshot_name'])
table.add_row([t[i]['uuid'], t[i]['snapshot_name'], t[i]['vm_uuid']])
i = i + 1
else:
t1 = pyjq.all('. | {uuid, snapshot_name, vm_uuid, vm_create_spec}', d)
t2 = pyjq.all('.vm_create_spec | {name, num_vcpus, num_cores_per_vcpu, memory_mb}', d)
table.field_names = ['Snapshot_UUID', 'Snapshot_Name', 'VM_UUID', 'VM_Name', 'Cores', 'Memory(MB)']
#print '%s | %s | %s | %s | %d | %d' % (
# t1[0]['uuid'], t1[0]['snapshot_name'], t1[0]['vm_uuid'],
# t2[0]['name'], t2[0]['num_vcpus'] * t2[0]['num_cores_per_vcpu'], t2[0]['memory_mb']
# )
def calc_result(self, person_datas, event_ids, competiton_datas, rank_type):
rank_data = []
with open(self.get_fixtures_path("result")) as file:
json_object = json.loads(file.read())
index = 0
for event_id in event_ids:
# イベントごとに順位でソートされた記録を取得
query = (
"[ [.[].fields ] | \
map(select ( .event_id == "
+ str(event_id)
+ " )) | \
group_by(.person_id) | .[] | sort_by(."
+ rank_type
+ ") | \
.[0] ] | \
sort_by(."
+ rank_type
+ ") | \
.[] | \
{ id, event_id, person_id, "
+ rank_type
+ ", competition_id, value1, value2, value3, value4, value5 }"
)
results = pyjq.all(query, json_object)
rank = 0
gender_ranks = dict(map(lambda x: (x.value[0], 0), Gender))
generation_ranks = dict(
map(lambda x: (x * GENERATION_MAX, 0), range(0, 10))
)
prefecture_ranks = dict(
map(lambda x: (x.value, 0), PrefectureAndOversea)
)
before_record = 0
skip_count = 1
for result in results:
if result[rank_type] > 0:
# 世代計算
generation = (
self.get_generation(
person_datas[result["person_id"]].birth_at,
competiton_datas[result["competition_id"]]["close_at"],
)
* GENERATION_MAX
)
if before_record == 0 or before_record < result[rank_type]:
before_record = result[rank_type]
gender_ranks[
person_datas[result["person_id"]].gender
] += skip_count
generation_ranks[generation] += skip_count
prefecture_ranks[
person_datas[result["person_id"]].prefecture_id
] += skip_count
rank += skip_count
skip_count = 1
else:
skip_count += 1
result["competition_id"] = result["competition_id"]
result["competition_name_id"] = competiton_datas[
result["competition_id"]
]["name_id"]
result["competition_name"] = competiton_datas[
result["competition_id"]
]["name"]
result["year"] = localtime(
datetime.fromisoformat(
competiton_datas[result["competition_id"]][
"open_at"
].replace("Z", "+00:00")
)
).year
result["gender"] = person_datas[result["person_id"]].gender
result["generation"] = generation
result["prefecture_id"] = person_datas[
result["person_id"]
].prefecture_id
result["rank"] = rank
result["gender_rank"] = gender_ranks[
person_datas[result["person_id"]].gender
]
result["generation_rank"] = generation_ranks[generation]
result["prefecture_rank"] = prefecture_ranks[
person_datas[result["person_id"]].prefecture_id
]
# 不要データ削除
result.pop("id")
if rank_type == "average":
result["best"] = result["average"]
result.pop("average")
if rank_type == "best":
result.pop("value1")
result.pop("value2")
result.pop("value3")
result.pop("value4")
result.pop("value5")
model_name = "bestrank"
if rank_type == "average":
model_name = "averagerank"
index += 1
dictionary = {}
dictionary["model"] = "app." + model_name
dictionary["pk"] = index
dictionary["fields"] = result
rank_data.append(dictionary)
return rank_data
def amis(args, accounts, config):
# Loading the list of public images from disk takes a while, so we'll iterate by region
regions_file = 'data/aws/us-east-1/ec2-describe-images.json'
if not os.path.isfile(regions_file):
raise Exception("You need to download the set of public AMI images. Run:\n"
" mkdir -p data/aws\n"
" cd data/aws\n"
" aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} mkdir {}\n"
" aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} sh -c 'aws --region {} ec2 describe-images --executable-users all > {}/ec2-describe-images.json'\n"
)
print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
'Account Name',
'Region Name',
'Instance Id',
'Instance Name',
'AMI ID',
'Is Public',
'AMI Description',
'AMI Owner'))
for region in listdir('data/aws/'):
# Get public images
public_images_file = 'data/aws/{}/ec2-describe-images.json'.format(region)
public_images = json.load(open(public_images_file))
resource_filter = '.Images[]'
public_images = pyjq.all(resource_filter, public_images)
for account in accounts:
account = Account(None, account)
region = Region(account, {'RegionName': region})
instances = query_aws(account, "ec2-describe-instances", region)
resource_filter = '.Reservations[].Instances[] | select(.State.Name == "running")'
if args.instance_filter != '':
resource_filter += '|{}'.format(args.instance_filter)
instances = pyjq.all(resource_filter, instances)
account_images = query_aws(account, "ec2-describe-images", region)
resource_filter = '.Images[]'
account_images = pyjq.all(resource_filter, account_images)
for instance in instances:
image_id = instance['ImageId']
image_description = ''
owner = ''
image, is_public_image = find_image(image_id, public_images, account_images)
if image:
# Many images don't have all fields, so try the Name, then Description, then ImageLocation
image_description = image.get('Name', '')
if image_description == '':
image_description = image.get('Description', '')
if image_description == '':
image_description = image.get('ImageLocation', '')
owner = image.get('OwnerId', '')
print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
account.name,
region.name,
instance['InstanceId'],
get_instance_name(instance),
image_id,
is_public_image,
image_description,
owner))
def get_regional_direct_connects(region):
direct_connects = query_aws(region.account,
"/directconnect-describe-connections", region)
resource_filter = ".connections[]?"
return pyjq.all(resource_filter, direct_connects)
def get_iam_trusts(account, nodes, connections, connections_to_get):
# Get IAM
iam = query_aws(
account,
"iam-get-account-authorization-details",
Region(account, {"RegionName": "us-east-1"}),
)
saml_providers = query_aws(
account, "iam-list-saml-providers",
Region(account, {"RegionName": "us-east-1"}))["SAMLProviderList"]
for role in pyjq.all(".RoleDetailList[]", iam):
principals = pyjq.all(
".AssumeRolePolicyDocument.Statement[].Principal", role)
for principal in principals:
assume_role_nodes = set()
federated_principals = principal.get("Federated", None)
if federated_principals:
if not isinstance(federated_principals, list):
federated_principals = [federated_principals]
for federated_principal in federated_principals:
try:
# Validate that the federated principal and the SAML provider is coming from known accounts.
# WoT will show us the direction of that trust for further inspection.
# this enables cross_account_admin_sts (STS between accounts)
for saml in saml_providers:
if saml["Arn"] == federated_principal:
saml_provider_arn = saml["Arn"]
elif get_account_by_id(
account_id=federated_principal.split(
":")[4]):
if get_account_by_id(
account_id=saml["Arn"].split(":")[4]):
saml_provider_arn = saml["Arn"]
if "saml-provider/okta" in saml_provider_arn.lower():
node = Account(json_blob={
"id": "okta",
"name": "okta",
"type": "Okta"
})
assume_role_nodes.add(node)
elif "saml-provider/onelogin" in saml_provider_arn.lower(
):
node = Account(
json_blob={
"id": "onelogin",
"name": "onelogin",
"type": "Onelogin",
})
assume_role_nodes.add(node)
elif "saml-provider/waad" in saml_provider_arn.lower():
node = Account(json_blob={
"id": "WAAD",
"name": "WAAD",
"type": "waad",
})
assume_role_nodes.add(node)
elif "saml-provider/allcloud-sso" in saml_provider_arn.lower(
):
node = Account(
json_blob={
"id": "AllCloud-SSO",
"name": "AllCloud-SSO",
"type": "AllCloud-SSO",
})
assume_role_nodes.add(node)
elif "saml-provider/awssso" in saml_provider_arn.lower(
):
node = Account(
json_blob={
"id": "AWSSSO",
"name": "AWS SSO",
"type": "Amazon",
})
assume_role_nodes.add(node)
elif "saml-provider/adfs" in saml_provider_arn.lower():
node = Account(json_blob={
"id": "adfs",
"name": "adfs",
"type": "ADFS"
})
assume_role_nodes.add(node)
elif "saml-provider/auth0" in saml_provider_arn.lower(
):
node = Account(json_blob={
"id": "auth0",
"name": "auth0",
"type": "auth0",
})
assume_role_nodes.add(node)
elif "saml-provider/google" in saml_provider_arn.lower(
):
node = Account(
json_blob={
"id": "google",
"name": "google",
"type": "google",
})
assume_role_nodes.add(node)
elif "saml-provider/gsuite" in saml_provider_arn.lower(
):
node = Account(
json_blob={
"id": "gsuite",
"name": "gsuite",
"type": "gsuite",
})
assume_role_nodes.add(node)
elif ("cognito-identity.amazonaws.com"
in saml_provider_arn.lower()):
continue
elif "www.amazon.com" in saml_provider_arn.lower():
node = Account(
json_blob={
"id": "Amazon.com",
"name": "Amazon.com",
"type": "Amazon",
})
continue
else:
raise Exception(
"Unknown federation provider: {}".format(
saml_provider_arn.lower()))
except (StopIteration, IndexError):
if ("cognito-identity.amazonaws.com"
in federated_principal.lower()):
# TODO: Should show this somehow
continue
elif ":oidc-provider/" in federated_principal.lower():
# TODO: handle OpenID Connect identity providers
# https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_providers_create_oidc.html
continue
raise Exception(
"Principal {} is not a configured SAML provider".
format(federated_principal))
if principal.get("AWS", None):
principal = principal["AWS"]
if not isinstance(principal, list):
principal = [principal]
for p in principal:
if "arn:aws" not in p:
# The role can simply be something like "AROA..."
continue
parts = p.split(":")
account_id = parts[4]
assume_role_nodes.add(Account(account_id=account_id))
for node in assume_role_nodes:
if nodes.get(node.id, None) is None:
nodes[node.id] = node
access_type = "iam"
# TODO: Identify all admins better. Use code from find_admins.py
for m in role["AttachedManagedPolicies"]:
for p in pyjq.all(".Policies[]", iam):
if p["Arn"] == m["PolicyArn"]:
for policy_doc in p["PolicyVersionList"]:
if policy_doc["IsDefaultVersion"] == True:
if is_admin_policy(policy_doc["Document"]):
access_type = "admin"
for policy in role["RolePolicyList"]:
policy_doc = policy["PolicyDocument"]
if is_admin_policy(policy_doc):
access_type = "admin"
if (access_type == "admin" and connections_to_get["admin"]
) or (access_type != "admin"
and connections_to_get["iam_nonadmin"]):
connections[Connection(node, account, access_type)] = []
return
def get_public_nodes(account, config, use_cache=False):
# TODO Look for IPv6 also
# TODO Look at more services from https://github.com/arkadiyt/aws_public_ips
# TODO Integrate into something to more easily port scan and screenshot web services
# Try reading from cache
cache_file_path = 'account-data/{}/public_nodes.json'.format(
account['name'])
if use_cache:
if os.path.isfile(cache_file_path):
with open(cache_file_path) as f:
return json.load(f), []
# Get the data from the `prepare` command
outputfilter = {
'internal_edges': False,
'read_replicas': False,
'inter_rds_edges': False,
'azs': False,
'collapse_by_tag': None,
'collapse_asgs': True,
'mute': True
}
network = build_data_structure(account, config, outputfilter)
public_nodes = []
warnings = []
# Look at all the edges for ones connected to the public Internet (0.0.0.0/0)
for edge in pyjq.all(
'.[].data|select(.type=="edge")|select(.source=="0.0.0.0/0")',
network):
# Find the node at the other end of this edge
target = {'arn': edge['target'], 'account': account['name']}
target_node = pyjq.first(
'.[].data|select(.id=="{}")'.format(target['arn']), network, {})
# Depending on the type of node, identify what the IP or hostname is
if target_node['type'] == 'elb':
target['type'] = 'elb'
target['hostname'] = target_node['node_data']['DNSName']
elif target_node['type'] == 'elbv2':
target['type'] = 'elbv2'
target['hostname'] = target_node['node_data']['DNSName']
elif target_node['type'] == 'autoscaling':
target['type'] = 'autoscaling'
target['hostname'] = target_node['node_data'].get(
'PublicIpAddress', '')
if target['hostname'] == '':
target['hostname'] = target_node['node_data']['PublicDnsName']
elif target_node['type'] == 'rds':
target['type'] = 'rds'
target['hostname'] = target_node['node_data']['Endpoint'][
'Address']
elif target_node['type'] == 'ec2':
target['type'] = 'ec2'
dns_name = target_node['node_data'].get('PublicDnsName', '')
target['hostname'] = target_node['node_data'].get(
'PublicIpAddress', dns_name)
elif target_node['type'] == 'ecs':
target['type'] = 'ecs'
target['hostname'] = ''
for ip in target_node['node_data']['ips']:
if is_public_ip(ip):
target['hostname'] = ip
elif target_node['type'] == 'redshift':
target['type'] = 'redshift'
target['hostname'] = target_node['node_data'].get(
'Endpoint', {}).get('Address', '')
else:
# Unknown node
raise Exception('Unknown type: {}'.format(target_node['type']))
# Check if any protocol is allowed (indicated by IpProtocol == -1)
ingress = pyjq.all('.[]', edge.get('node_data', {}))
sg_group_allowing_all_protocols = pyjq.first(
'select(.IpPermissions[]|.IpProtocol=="-1")|.GroupId', ingress,
None)
public_sgs = {}
if sg_group_allowing_all_protocols is not None:
warnings.append(
'All protocols allowed access to {} due to {}'.format(
target, sg_group_allowing_all_protocols))
range_string = '0-65535'
# I would need to redo this code in order to get the name of the security group
public_sgs[sg_group_allowing_all_protocols] = {
'public_ports': '0-65535'
}
else:
# from_port and to_port mean the beginning and end of a port range
# We only care about TCP (6) and UDP (17)
# For more info see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/security-group-rules-reference.html
port_ranges = []
for sg in ingress:
sg_port_ranges = []
for ip_permission in sg['IpPermissions']:
selection = 'select((.IpProtocol=="tcp") or (.IpProtocol=="udp")) | select(.IpRanges[].CidrIp=="0.0.0.0/0")'
sg_port_ranges.extend(
pyjq.all('{}| [.FromPort,.ToPort]'.format(selection),
ip_permission))
public_sgs[sg['GroupId']] = {
'GroupName':
sg['GroupName'],
'public_ports':
port_ranges_string(regroup_ranges(sg_port_ranges))
}
port_ranges.extend(sg_port_ranges)
range_string = port_ranges_string(regroup_ranges(port_ranges))
target['ports'] = range_string
target['public_sgs'] = public_sgs
if target['ports'] == "":
issue_msg = 'No ports open for tcp or udp (probably can only be pinged). Rules that are not tcp or udp: {} -- {}'
warnings.append(
issue_msg.format(
json.dumps(
pyjq.all(
'.[]|select((.IpProtocol!="tcp") and (.IpProtocol!="udp"))'
.format(selection), ingress)), account))
public_nodes.append(target)
# For the network diagram, if an ELB has availability across 3 subnets, I put one node in each subnet.
# We don't care about that when we want to know what is public and it makes it confusing when you
# see 3 resources with the same hostname, when you view your environment as only having one ELB.
# This same issue exists for RDS.
# Reduce these to single nodes.
reduced_nodes = {}
for node in public_nodes:
reduced_nodes[node['hostname']] = node
public_nodes = []
for _, node in reduced_nodes.items():
public_nodes.append(node)
account = Account(None, account)
for region_json in get_regions(account):
region = Region(account, region_json)
# Look for CloudFront
if region.name == 'us-east-1':
json_blob = query_aws(region.account,
'cloudfront-list-distributions', region)
for distribution in json_blob.get('DistributionList',
{}).get('Items', []):
if not distribution['Enabled']:
continue
target = {'arn': distribution['ARN'], 'account': account.name}
target['type'] = 'cloudfront'
target['hostname'] = distribution['DomainName']
target['ports'] = '80,443'
public_nodes.append(target)
# Look for API Gateway
json_blob = query_aws(region.account, 'apigateway-get-rest-apis',
region)
if json_blob is not None:
for api in json_blob.get('items', []):
target = {'arn': api['id'], 'account': account.name}
target['type'] = 'apigateway'
target['hostname'] = '{}.execute-api.{}.amazonaws.com'.format(
api['id'], region.name)
target['ports'] = '80,443'
public_nodes.append(target)
# Write cache file
with open(cache_file_path, 'w') as f:
f.write(json.dumps(public_nodes, indent=4, sort_keys=True))
return public_nodes, warnings
def get_redshift(region):
clusters = query_aws(region.account, "redshift-describe-clusters", region.region)
return pyjq.all(".Clusters[]?", clusters)
def vm_print(self, uuid=''):
# call this function with uuid parameter, or you will get full list
from prettytable import PrettyTable
for i in uuid:
if not self.vm_existed(uuid=i):
print 'uuid %s does not exist' % (i)
sys.exit(9)
if not hasattr(self, 'vVMList'):
self.vm_list()
full = json.loads(unicode(self.vVMList))
if self.vIsDebug:
print json.dumps(full, indent=2)
if not self.vIsSummary:
table = PrettyTable()
table.field_names = ["uuid", "vm name", "ip addr", "power state", "vCPU (Cores)", "vMEM (GB)", "vdisk num", "vnic num"]
if uuid == '':
t1 = pyjq.all('.entities[]', full)
for u in t1:
t2 = pyjq.all('.spec.resources.nic_list[]', u)
ip = ''
for v in t2:
t3 = pyjq.all('.ip_endpoint_list[]', v)
for w in t3:
if w['type'] == 'ASSIGNED':
ip = w['ip'] + ',' + ip
table.add_row([
u['metadata']['uuid'],
u['spec']['name'],
re.sub(',$','',ip),
u['spec']['resources']['power_state'],
str(u['spec']['resources']['num_sockets']) + ' * ' + str(u['spec']['resources']['num_vcpus_per_socket']),
str(u['spec']['resources']['memory_size_mib']/1024),
len(u['spec']['resources']['disk_list']),
len(u['spec']['resources']['nic_list'])
])
#print u['spec']['resources']['nic_list'][0]['ip_endpoint_list'][0]
else:
for i in uuid:
s = '.entities[] | select (.metadata.uuid==\"' + i + '\") '
t1 = pyjq.all(s, full)
table.add_row([
t1[0]['metadata']['uuid'],
t1[0]['spec']['name'],
'',
t1[0]['spec']['resources']['power_state'],
str(t1[0]['spec']['resources']['num_sockets']) + ' * ' + str(t1[0]['spec']['resources']['num_vcpus_per_socket']),
str(t1[0]['spec']['resources']['memory_size_mib']/1024),
len(t1[0]['spec']['resources']['disk_list']),
len(t1[0]['spec']['resources']['nic_list'])
])
print table
else:
t1 = pyjq.all('.entities[]', full)
for u in t1:
print u['metadata']['uuid']
#eliminate error like this, when output pipe to head command
#close failed in file object destructor:
#sys.excepthook is missing
#lost sys.stderr
try:
sys.stdout.close()
except:
pass
try:
sys.stderr.close()
except:
pass
def get_connections(cidrs, vpc, outputfilter):
"""
For a VPC, for each instance, find all of the other instances that can connect to it,
including those in peered VPCs.
Note I do not consider subnet ACLs, routing tables, or some other network concepts.
"""
connections = {}
# Get mapping of security group names to nodes that have that security group
sg_to_instance_mapping = {}
for instance in vpc.leaves:
for sg in instance.security_groups:
sg_to_instance_mapping.setdefault(sg, {})[instance] = True
# For each security group, find all the instances that are allowed to connect to instances
# within that group.
for sg in get_sgs(vpc):
# Get the CIDRs that are allowed to connect
for cidr in pyjq.all(".IpPermissions[].IpRanges[].CidrIp", sg):
if not is_external_cidr(cidr):
# This is a private IP, ex. 10.0.0.0/16
# See if we should skip this
if not outputfilter.get("internal_edges", True):
continue
# Find all instances in this VPC and peered VPCs that are in this CIDR
for sourceVpc in itertools.chain(vpc.peers, (vpc,)):
# Ensure it is possible for instances in this VPC to be in the CIDR
if not (
IPNetwork(sourceVpc.cidr) in IPNetwork(cidr)
or IPNetwork(cidr) in IPNetwork(sourceVpc.cidr)
):
# The CIDR from the security group does not overlap with the CIDR of the VPC,
# so skip it
continue
# For each instance, check if one of its IPs is within the CIDR
for sourceInstance in sourceVpc.leaves:
for ip in sourceInstance.ips:
if IPAddress(ip) in IPNetwork(cidr):
# Instance found that can connect to instances in the SG
# So connect this instance (sourceInstance) to every instance
# in the SG.
for targetInstance in sg_to_instance_mapping.get(
sg["GroupId"], {}
):
add_connection(
connections, sourceInstance, targetInstance, sg
)
else:
# This is an external IP (ie. not in a private range).
for instance in sg_to_instance_mapping.get(sg["GroupId"], {}):
# Ensure it has a public IP, as resources with only private IPs can't be reached
if instance.is_public:
cidrs[cidr].is_used = True
add_connection(connections, cidrs[cidr], instance, sg)
else:
if cidr == "0.0.0.0/0":
# Resource is not public, but allows anything to access it,
# so mark set all the resources in the VPC as allowing access to it.
for source_instance in vpc.leaves:
add_connection(
connections, source_instance, instance, sg
)
if outputfilter.get("internal_edges", True):
# Connect allowed in Security Groups
for ingress_sg in pyjq.all(
".IpPermissions[].UserIdGroupPairs[].GroupId", sg
):
# We have an SG and a list of SG's it allows in
for target in sg_to_instance_mapping.get(sg["GroupId"], {}):
# We have an instance and a list of SG's it allows in
for source in sg_to_instance_mapping.get(ingress_sg, {}):
if (
not outputfilter.get("inter_rds_edges", True)
and (
source.node_type == "rds"
or source.node_type == "rds_rr"
)
and (
target.node_type == "rds"
or target.node_type == "rds_rr"
)
):
continue
add_connection(connections, source, target, sg)
# Connect everything to the Gateway endpoints
for targetResource in vpc.leaves:
if targetResource.has_unrestricted_ingress:
for sourceVpc in itertools.chain(vpc.peers, (vpc,)):
for sourceResource in sourceVpc.leaves:
add_connection(connections, sourceResource, targetResource, [])
# Remove connections for source nodes that cannot initiate traffic (ex. VPC endpoints)
for connection in list(connections):
if not connection.source.can_egress:
del connections[connection]
return connections
def collect(arguments):
account_dir = './{}'.format(arguments.account_name)
if arguments.clean and os.path.exists(
'account-data/{}'.format(account_dir)):
rmtree('account-data/{}'.format(account_dir))
make_directory("account-data")
make_directory("account-data/{}".format(account_dir))
print("* Getting region names")
session_data = {'region_name': 'us-east-1'}
if arguments.profile_name:
session_data['profile_name'] = arguments.profile_name
session = boto3.Session(**session_data)
ec2 = session.client('ec2')
region_list = ec2.describe_regions(
RegionNames=['us-east-1', 'ap-northeast-1'])
with open("account-data/{}/describe-regions.json".format(account_dir),
'w+') as f:
f.write(json.dumps(region_list, indent=4, sort_keys=True))
print("* Creating directory for each region name")
for region in region_list['Regions']:
make_directory('account-data/{}/{}'.format(
account_dir, region.get('RegionName', 'Unknown')))
# Services that will only be queried in us-east-1
universal_services = ['sts', 'iam', 'route53', 'route53domains', 's3']
with open("collect_commands.yaml", 'r') as f:
collect_commands = yaml.safe_load(f)
for runner in collect_commands:
print('* Getting {}:{} info'.format(runner['Service'],
runner['Request']))
parameters = {}
for region in region_list['Regions']:
dynamic_parameter = None
# Only call universal services in us-east-1
if runner['Service'] in universal_services and region[
'RegionName'] != 'us-east-1':
continue
handler = session.client(runner['Service'],
region_name=region['RegionName'])
filepath = "account-data/{}/{}/{}-{}".format(
account_dir, region['RegionName'], runner['Service'],
runner['Request'])
method_to_call = snakecase(runner["Request"])
# Identify any parameters
if runner.get('Parameters', False):
for parameter in runner['Parameters']:
parameters[parameter['Name']] = parameter['Value']
# Look for any dynamic values (ones that jq parse a file)
if '|' in parameter['Value']:
dynamic_parameter = parameter['Name']
if dynamic_parameter is not None:
# Set up directory for the dynamic value
make_directory(filepath)
# The dynamic parameter must always be the first value
parameter_file = parameters[dynamic_parameter].split('|')[0]
parameter_file = "account-data/{}/{}/{}".format(
account_dir, region['RegionName'], parameter_file)
if not os.path.isfile(parameter_file):
# The file where parameters are obtained from does not exist
continue
with open(parameter_file, 'r') as f:
parameter_values = json.load(f)
pyjq_parse_string = '|'.join(
parameters[dynamic_parameter].split('|')[1:])
for parameter in pyjq.all(pyjq_parse_string,
parameter_values):
filename = get_filename_from_parameter(parameter)
identifier = get_identifier_from_parameter(parameter)
parameters[dynamic_parameter] = identifier
outputfile = "{}/{}".format(filepath, filename)
call_function(outputfile, handler, method_to_call,
parameters)
else:
filepath = filepath + ".json"
call_function(filepath, handler, method_to_call, parameters)
def get_vpc_peerings(region):
vpc_peerings = query_aws(
region.account, "ec2-describe-vpc-peering-connections", region
)
resource_filter = ".VpcPeeringConnections[]?"
return pyjq.all(resource_filter, vpc_peerings)
def get_ec2s(region):
instances = query_aws(region.account, "ec2-describe-instances", region.region)
resource_filter = '.Reservations[]?.Instances[] | select(.State.Name == "running")'
return pyjq.all(resource_filter, instances)
def audit_s3_buckets(findings, region):
buckets_json = query_aws(region.account, "s3-list-buckets", region)
buckets = pyjq.all(".Buckets[].Name", buckets_json)
for bucket in buckets:
# Check policy
try:
policy_file_json = get_parameter_file(region, "s3",
"get-bucket-policy", bucket)
if policy_file_json is not None:
# Find the entity we need
policy_string = policy_file_json["Policy"]
# Load the string value as json
policy = json.loads(policy_string)
policy = Policy(policy)
if policy.is_internet_accessible():
if (len(policy.statements) == 1
and len(policy.statements[0].actions) == 1 and
"s3:GetObject" in policy.statements[0].actions):
findings.add(
Finding(region, "S3_PUBLIC_POLICY_GETOBJECT_ONLY",
bucket))
else:
findings.add(
Finding(
region,
"S3_PUBLIC_POLICY",
bucket,
resource_details=policy_string,
))
except Exception as e:
findings.add(
Finding(
region,
"EXCEPTION",
bucket,
resource_details={
"policy": policy_string,
"exception": str(e),
"location": "Exception checking policy of S3 bucket",
},
))
# Check ACL
try:
file_json = get_parameter_file(region, "s3", "get-bucket-acl",
bucket)
for grant in file_json["Grants"]:
uri = grant["Grantee"].get("URI", "")
if (uri == "http://acs.amazonaws.com/groups/global/AllUsers"
or uri ==
"http://acs.amazonaws.com/groups/global/AuthenticatedUsers"
):
findings.add(
Finding(region,
"S3_PUBLIC_ACL",
bucket,
resource_details=grant))
except Exception as e:
findings.add(
Finding(
region,
"EXCEPTION",
bucket,
resource_details={
"grant": grant,
"exception": str(e),
"location": "Exception checking ACL of S3 bucket",
},
))
# starting with the previous block, to look into the current block
# and count how many validators missed it.
# so this tool is always "one block behind"
previousblock = block - 1
url = baseurl + str(previousblock)
result = fetch_json(url)
# jq -r '.result.block.header.proposer_address'
proposer_address = (
dictor(result, 'result.block.header.proposer_address'))
url = baseurl + str(block)
# jq -r '.result.block.last_commit.precommits[].validator_address'
result = fetch_json(url)
validator_addresses = (
pyjq.all('.result.block.last_commit.precommits[].validator_address', result))
# Start out with the assumption that your val did miss the block
missed = 1
# and count the number of validators that missed it starting from 0
valmissedcount = 0
# here you append the address of each validator that missed the block
valset = ""
# See if you missed the block, count how many validators missed, add them to a list
for i in validator_addresses:
if i == yourval:
missed = 0
valset = valset + str(i) + ","
elif i is None:
valmissedcount = valmissedcount + 1
def amis(args, accounts, config):
# Loading the list of public images from disk takes a while, so we'll iterate by region
regions_file = "data/aws/us-east-1/ec2-describe-images.json"
if not os.path.isfile(regions_file):
raise Exception(
"You need to download the set of public AMI images. Run:\n"
" mkdir -p data/aws\n"
" cd data/aws\n"
" aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} mkdir {}\n"
" aws ec2 describe-regions | jq -r '.Regions[].RegionName' | xargs -I{} sh -c 'aws --region {} ec2 describe-images --executable-users all > {}/ec2-describe-images.json'\n"
)
print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
"Account Name",
"Region Name",
"Instance Id",
"Instance Name",
"AMI ID",
"Is Public",
"AMI Description",
"AMI Owner",
))
for region_name in listdir("data/aws/"):
# Get public images
public_images_file = "data/aws/{}/ec2-describe-images.json".format(
region_name)
public_images = json.load(open(public_images_file))
resource_filter = ".Images[]"
public_images = pyjq.all(resource_filter, public_images)
for account in accounts:
account = Account(None, account)
region = Region(account, {"RegionName": region_name})
instances = query_aws(account, "ec2-describe-instances", region)
resource_filter = (
'.Reservations[].Instances[] | select(.State.Name == "running")'
)
if args.instance_filter != "":
resource_filter += "|{}".format(args.instance_filter)
if "Reservations" not in instances:
print(f"** skipping: {account.name} in {region_name}")
continue
instances = pyjq.all(resource_filter, instances)
account_images = query_aws(account, "ec2-describe-images", region)
resource_filter = ".Images[]"
if "Images" not in account_images:
print(f"** skipping: {account.name} in {region_name}")
continue
account_images = pyjq.all(resource_filter, account_images)
for instance in instances:
image_id = instance["ImageId"]
image_description = ""
owner = ""
image, is_public_image = find_image(image_id, public_images,
account_images)
if image:
# Many images don't have all fields, so try the Name, then Description, then ImageLocation
image_description = image.get("Name", "")
if image_description == "":
image_description = image.get("Description", "")
if image_description == "":
image_description = image.get("ImageLocation", "")
owner = image.get("OwnerId", "")
print("{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}".format(
account.name,
region.name,
instance["InstanceId"],
get_instance_name(instance),
image_id,
is_public_image,
image_description,
owner,
))
#Now let's replicate the twitch python script to generate a NYT
#Define the key libraries
import pandas as pd
import numpy as np
import requests
import json
import pyjq
#Store the twitch api url and retrieve the raw data
api_key = 'gcpa6XL4z1zLwQXSxW1Y4HniG3zWCWNM'
url = 'https://api.nytimes.com/svc/archive/v1/2018/1.json?&api-key='+api_key
#now let's extract the relevant fields only
JSONdata = requests.get(url).json()
JSONdoccount = pyjq.all('.response .docs | length',JSONdata)[0]
JSONdataextract = f'.response .docs [] | {{the_snippet: .snippet, the_headline: .headline .main, the_date: .pub_date, the_news_desk: .news_desk}}'
JSONdataextractoutput = pyjq.all(JSONdataextract, JSONdata)
JSONdataextractoutput_dump = json.dumps(JSONdataextractoutput,indent=4)
#print(JSONdataextractoutput_dump)
#print("The total number of documents is " + str(JSONdoccount))
#creating a json extract and keeping it locally
jsonurl = '/home/salexommer/Documents/yellow-submarine/extracts/nyt_month_data.json'
with open(jsonurl,'w') as fi:
fi.write(JSONdataextractoutput_dump)
df = pd.read_json(jsonurl, orient='columns')
#df = pd.read_json(jsonurl, orient='columns')
#re-transforming the json into a csv table
export_csv = df.to_csv(path_or_buf='/home/salexommer/Documents/yellow-submarine/extracts/nyt_snippet_sample.csv',index=True)
def jq_handler(ctx):
r = requests.get(url)
return jq.all(jq_string, r.json())
def get_vpc_endpoints(region):
endpoints = query_aws(region.account, "ec2-describe-vpc-endpoints", region.region)
return pyjq.all(".VpcEndpoints[]?", endpoints)
def event_list(self):
full = json.loads(unicode(self.vList))
if self.vIsDebug:
print json.dumps(full, indent=2)
# get timezone from clsCluster.py
a = clsCluster(debug=False)
localtz = a.cluster_timezone()
if not self.vIsSummary: #self.vIsSummary is true, print detail vm list
table = PrettyTable()
table.field_names = ['date', 'alert_type', 'severity', 'resolved', 'message']
table.align['message'] = "l"
fields = '{ id, created_time_stamp_in_usecs, alert_type_uuid, severity, resolved, message, context_types, context_values }'
#if imageuuid is NULL, print summary for all images
#else print summary for the special image and it's detail info
wrapper = TextWrapper(width=100)
epoch = datetime(1970, 1, 1, tzinfo=timezone('UTC'))
if len(self.vUUID) == 0:
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR
t = pyjq.all('.entities[] | ' + fields, full)
i = 0
while i < len(t):
mymessage = t[i]['id'] + ' | ' + t[i]['message']
# convert usec to date
date_str = epoch + timedelta(seconds=t[i]['created_time_stamp_in_usecs']/1000/1000)
date_str_local = date_str.astimezone(timezone(localtz))
# replace {..} string in message to humanable readable.
mytype = t[i]['context_types']
myvalue = t[i]['context_values']
mylist = re.findall(r'{[^}]+}', mymessage)
for j in mylist:
myindex = mytype.index(re.sub('[{}]','',j))
mystr = myvalue[myindex]
mymessage = re.sub(j,mystr,mymessage)
# create table
table.add_row([
str(date_str_local.strftime(format)),
t[i]['alert_type_uuid'],
t[i]['severity'],
t[i]['resolved'],
wrapper.fill(mymessage),
])
i = i + 1
else:
for i in self.vUUID:
s = '.entities[] | select (.id==\"' + i + '\") | ' + fields
t = pyjq.all(s, full)
mymessage = t[0]['id'] + ' | ' + t[0]['message']
# convert usec to date
date_str = epoch + timedelta(seconds=t[0]['created_time_stamp_in_usecs']/1000/1000)
date_str_local = date_str.astimezone(timezone(localtz))
# replace {..} string in message to humanable readable.
mytype = t[0]['context_types']
myvalue = t[0]['context_values']
mylist = re.findall(r'{[^}]+}', mymessage)
for j in mylist:
myindex = mytype.index(re.sub('[{}]','',j))
mystr = myvalue[myindex]
mymessage = re.sub(j,mystr,mymessage)
# create table
table.add_row([
str(date_str_local),
t[0]['alert_type_uuid'],
t[0]['severity'],
t[0]['resolved'],
wrapper.fill(mymessage),
])
print table
else: #self.vIsSummary is false, just print uuid
pass
def get_rds_instances(region):
instances = query_aws(region.account, "rds-describe-db-instances", region.region)
return pyjq.all(".DBInstances[]?", instances)
def get_elbs(region):
load_balancers = query_aws(
region.account, "elb-describe-load-balancers", region.region
)
return pyjq.all(".LoadBalancerDescriptions[]?", load_balancers)
def audit_sg(findings, region):
# TODO Check if security groups allow large CIDR range (ex. 1.2.3.4/3)
# TODO Check if an SG restricts IPv4 and then opens IPv6 or vice versa.
cidrs = {}
sg_json = query_aws(region.account, 'ec2-describe-security-groups', region)
sgs = pyjq.all('.SecurityGroups[]', sg_json)
for sg in sgs:
cidr_and_name_list = pyjq.all(
'.IpPermissions[].IpRanges[]|[.CidrIp,.Description]', sg)
for cidr, name in cidr_and_name_list:
if not is_external_cidr(cidr):
continue
if is_unblockable_cidr(cidr):
findings.add(
Finding(region,
'SG_CIDR_UNNEEDED',
sg['GroupId'],
resource_details={'cidr': cidr}))
continue
if cidr.startswith('0.0.0.0') and not cidr.endswith('/0'):
findings.add(
Finding(region,
'SG_CIDR_UNEXPECTED',
sg['GroupId'],
resource_details={'cidr': cidr}))
continue
if cidr == '0.0.0.0/0':
continue
cidrs[cidr] = cidrs.get(cidr, list())
cidrs[cidr].append(sg['GroupId'])
for ip_permissions in sg['IpPermissions']:
cidrs_seen = set()
for ip_ranges in ip_permissions['IpRanges']:
if 'CidrIp' not in ip_ranges:
continue
cidr = ip_ranges['CidrIp']
for cidr_seen in cidrs_seen:
if (IPNetwork(cidr_seen) in IPNetwork(cidr)
or IPNetwork(cidr) in IPNetwork(cidr_seen)):
findings.add(
Finding(region,
'SG_CIDR_OVERLAPS',
sg['GroupId'],
resource_details={
'cidr1': cidr,
'cidr2': cidr_seen
}))
cidrs_seen.add(cidr)
for cidr in cidrs:
ip = IPNetwork(cidr)
if ip.size > 2048:
findings.add(
Finding(region,
'SG_LARGE_CIDR',
cidr,
resource_details={
'size': ip.size,
'security_groups': cidrs[cidr]
}))
def get_account_stats(account, all_resources=False):
"""Returns stats for an account"""
with open("stats_config.yaml", "r") as f:
resources = yaml.safe_load(f)
account = Account(None, account)
log_debug("Collecting stats in account {} ({})".format(
account.name, account.local_id))
stats = {}
stats["keys"] = []
for resource in resources:
# If the resource is marked as verbose, and we're not showing all resources, skip it.
if resource.get("verbose", False) and not all_resources:
continue
stats["keys"].append(resource["name"])
stats[resource["name"]] = {}
for region_json in get_regions(account):
region = Region(account, region_json)
for resource in resources:
if resource.get("verbose", False) and not all_resources:
continue
# Skip global services (just CloudFront)
if ("region" in resource) and (resource["region"] != region.name):
continue
# S3 buckets require special code to identify their location
if resource["name"] == "S3 buckets":
if region.name == "us-east-1":
buckets = pyjq.all(
".Buckets[].Name",
query_aws(region.account, "s3-list-buckets", region),
)
for bucket in buckets:
# Get the bucket's location
bucket_region = get_parameter_file(
region, "s3", "get-bucket-location",
bucket)["LocationConstraint"]
# Convert the value to a name.
# See https://docs.aws.amazon.com/general/latest/gr/rande.html#s3_region
if bucket_region is None:
bucket_region = "us-east-1"
elif bucket_region == "EU":
bucket_region = "eu-west-1"
# Increment the count
tmp = stats[resource["name"]].get(bucket_region, 0)
stats[resource["name"]][bucket_region] = tmp + 1
else:
# Normal path
stats[resource["name"]][region.name] = sum(
pyjq.all(
resource["query"],
query_aws(region.account, resource["source"], region),
))
return stats
def get_elbv2s(region):
# ALBs and NLBs
load_balancers = query_aws(
region.account, "elbv2-describe-load-balancers", region.region
)
return pyjq.all(".LoadBalancers[]?", load_balancers)
c = GetImage(
hostname=param.vHostname, port=param.vPort,
username=param.vUsername, password=param.vPassword,
debug=False
)
b = json.loads(str(c))
print json.dumps(b, indent=4)
sys.exit(9)
if 'myerror' in d:
print json.dumps(d, indent=4)
else:
table = PrettyTable()
table.field_names = ['Image_Name', 'Image_UUID', 'Image_Type', 'VM_Disk_ID']
#if imageuuid is NULL, print summary for all images
#else print summary for the special image and it's detail info
if param.vImageUUID == '':
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | {name, uuid, image_type, vm_disk_id}'"$STR
t = pyjq.all('.entities[] | {name, uuid, image_type, vm_disk_id}', d)
i = 0
while i < len(t):
table.add_row([t[i]['name'], t[i]['uuid'], t[i]['image_type'], t[i]['vm_disk_id']])
i = i + 1
else:
#cmd="./$runfile 2>/dev/null |jq -r '.entities[] | select (.uuid=="'"$1"'") | {name, uuid, image_type, vm_disk_id}'"$STR
s = '.entities[] | select (.uuid==\"' + param.vImageUUID + '\") | {name, uuid, image_type, vm_disk_id}'
t = pyjq.all(s, d)
table.add_row([t[0]['name'], t[0]['uuid'], t[0]['image_type'], t[0]['vm_disk_id']])
print table
def vm_create_from_isoimage(self, vmsize='', imageuuid='', netuuid='', containeruuid=''):
#check vmsize input
if vmsize.lower() == 'large':
vcpustring = 2
corestring = 2
memorystring = 8192
diskstring = 214748364800
elif vmsize.lower() == 'middle':
vcpustring = 1
corestring = 2
memorystring = 4096
diskstring = 107374182400
elif vmsize.lower() == 'small':
vcpustring = 1
corestring = 1
memorystring = 2048
diskstring = 53687091200
else:
print 'vmsize invalid'
sys.exit(9)
#check diskimage uuid input
test = clsImage()
if test.image_disktype(uuid=imageuuid):
print 'diskimage uuid is not ISO_IMAGE'
sys.exit(9)
else:
full = json.loads(str(test.vList))
d = pyjq.all('.entities[] | { uuid, vm_disk_id }', full)
#return True or False
for u in d:
if u['uuid'] == imageuuid:
vmdiskuuid = u['vm_disk_id']
#check container uuid input
test = clsContainer()
if not test.container_existed(uuid=containeruuid):
print 'container uuid is invalid'
sys.exit(9)
#check net uuid input
test = clsNet()
if not test.net_existed(uuid=netuuid):
print 'network uuid is invalid'
sys.exit(9)
vmname = "VM-" + strftime("%Y%m%d-%H%M%S", localtime())
#make postdata
postdata = {
"name": vmname,
"description": "",
"memory_mb": memorystring,
"num_cores_per_vcpu": corestring,
"num_vcpus": vcpustring,
"timezone": "Asia/Shanghai",
"vm_disks": [
{
"disk_address": {
"device_bus": "IDE",
"device_index": 0
},
"is_cdrom": "true",
"is_empty": "false",
"vm_disk_clone": {
"disk_address": {
"vmdisk_uuid": vmdiskuuid
}}
},
{
"disk_address": {
"device_bus": "SCSI",
"device_index": 0
},
"is_cdrom": "false",
"is_empty": "true",
"vm_disk_create": {
"size": diskstring,
"storage_container_uuid": containeruuid
}
}
],
"vm_nics": [
{
"adapter_type": "E1000",
"network_uuid": netuuid,
"request_ip": "false"
}
]
}
if self.vIsDebug:
print json.dumps(postdata,indent=4)
url = '/PrismGateway/services/rest/v2.0/vms/'
a = clsURL(
debug=self.vIsDebug,
url=url,
postdata=postdata
)
result = json.loads(str(a.PostURL()))
if 'myerror' in result:
print ' `--error:', result['myerror']
else:
print ' `-- ./getTask.py --uuid %s' % (result['task_uuid'])
