def make_data(self): """ """ if self.does_table_exist(): shlog.normal("untagged_list already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Prepare table for untagged onject list sql = """CREATE TABLE untagged_list ( service TEXT, request TEXT, json TEXT ) """ shlog.verbose(sql) self.q.q(sql) # analyze data from all_json sql = """SELECT resource_name, id, record as json FROM all_json a WHERE a.record like '%"Tags": []%' -- empty tags OR a.record like '%"TagSet": []%' -- empty tagset OR (a.record not like '%Tags":%' and a.record not like '%TagSet":%') -- no tag dicts """ df = self.q.q_to_df(sql) # pyqj relevant information df['json'] = df['json'].apply(lambda x: self.extract_evidence(x)) self.q.df_to_db(self.table_name, df)
def make_data(self): """ Make a table called TAGS based on tagging data. This collection of data is based on the resourcetaggingapi If the tags table exists, then we take it data collection would result in duplicate rows. """ if self.does_table_exist(): shlog.normal("tags data already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Make a flattened table for the tag data. # one tag, value pair in each record. sql = "create table tags (short_arn text, tag text, value text, arn text)" shlog.verbose(sql) self.q.q(sql) for page, _ in self._pages_all_regions('resourcegroupstaggingapi', 'get_resources'): ResourceMappingList = page['ResourceTagMappingList'] for d in ResourceMappingList: full_arn = d['ResourceARN'] short_arn = aws_utils.shortened_arn(d['ResourceARN']) for kvdict in d['Tags']: sql = 'insert into tags values ("%s","%s","%s", "%s")' % ( short_arn, kvdict["Key"], kvdict["Value"], full_arn) self.q.q(sql)
def main(args): """visualize IAM entities and policies attached to them x_report needs to be run first""" from security_scripts.information.lib import vanilla_utils from security_scripts.information.lib import L1_L2 from security_scripts.information.lib import Liam # decode full path to dbfile so imports from other directories don't get confused if args.dbfile != ':memory:': args.dbfile = os.path.abspath(args.dbfile) shlog.verbose(args) shlog.verbose("only tests matching %s will be considered",(args.only)) q=vanilla_utils.Q(args.dbfile, args.flush) # switch in custom to_self and to_others L1_replacer() # run L1_L2 L1_L2.Acquire(args, "L1_L2", q) # cluster by type (label minus name minus /n Liam.Acquire(args, "Liam", q) exit(0)
def L1_replacer(): from security_scripts.information import L1_preport l1dir = os.getcwd() + "/report_files/L1/" shlog.verbose("Overwriting L1 files") with open(l1dir + 'to_self.json', 'w') as file: file.write(L1_preport.t_self) with open(l1dir + 'to_others.json', 'w') as file: file.write(L1_preport.t_others)
def make_data(self): """ """ if self.does_table_exist(): shlog.normal("tag_counter already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Prepare table for tag count analysis output sql = """CREATE TABLE tag_counter ( service TEXT, request TEXT, total TEXT, untagged TEXT, tagged TEXT ) """ shlog.verbose(sql) self.q.q(sql) # analyze data from all_json sql = """WITH untagged(resource,id, cnt) AS ( SELECT resource_name, id, count(record) as cnt FROM all_json a WHERE a.record like '%"Tags": []%' -- empty tags OR a.record like '%"TagSet": []%' -- empty tagset OR (a.record not like '%Tags":%' and a.record not like '%TagSet":%') -- no tag dicts GROUP BY resource_name, id ), tagged(resource, id, cnt) AS ( SELECT resource_name, id, count(record) as cnt FROM all_json a WHERE a.record like '%"Tags": [{%' -- empty tags OR a.record like '%"TagSet": [{%' -- empty tagset GROUP BY resource_name, id ), all_count(resource, id, cnt) AS ( SELECT resource_name, id, count(record) as cnt FROM all_json a GROUP BY resource_name, id ) SELECT a.resource, a.id, a.cnt as total, ifnull(u.cnt, 0) as untagged, ifnull(t.cnt,0) as tagged FROM all_count a LEFT JOIN untagged u on u.resource = a.resource and u.id = a.id LEFT JOIN tagged t on t.resource = a.resource and t.id = a.id GROUP BY a.resource, a.id """ df = self.q.q_to_df(sql) self.q.df_to_db(self.table_name, df)
def make_data(self): """ Make a table called TAGS based on tagging data. This collection of data is based on the resourcetaggingapi If the tags table exists, then we take it data collection would result in duplicate rows. """ if self.does_table_exist(): shlog.normal("secretsdata already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Make a flattened table for the tag data. # one tag, value pair in each record. sql = """CREATE TABLE secrets ( short_arn TEXT, region TEXT, name TEXT, description TEXT, lastchangeddate TEXT, arn TEXT, record JSON ) """ shlog.verbose(sql) self.q.q(sql) # Get the tags for each region. # accomidate the boto3 API can retul data in pages. for page, _ in self._pages_all_regions('secretsmanager', 'list_secrets'): for secret in page['SecretList']: #import pdb; pdb.set_trace() arn = secret['ARN'] short_arn = aws_utils.shortened_arn(arn) region = secret["ARN"].split(':')[3] name = secret['Name'] description = secret['Description'] lastchangeddate = datetime.datetime.isoformat( secret['LastChangedDate']) record = secret record["FlatTags"] = vanilla_utils.flatten_tags(secret["Tags"]) record = self._json_clean_dumps(record) sql = """ INSERT INTO secrets VALUES (?, ?, ?, ?, ?, ?, ?) """ list = (short_arn, region, name, description, lastchangeddate, arn, record) shlog.verbose(sql) self.q.executemany(sql, [list]) # populate the all_json table self._insert_all_json("secrets", short_arn, record)
def main(args): """Run tag, s3, secret, certificate, repo inventory reports""" from security_scripts.information.lib import vanilla_utils from security_scripts.information.lib import tags from security_scripts.information.lib import s3 from security_scripts.information.lib import secrets from security_scripts.information.lib import certificates from security_scripts.information.lib import repos from security_scripts.information.lib import load_balancer from security_scripts.information.lib import instances from security_scripts.information.lib import assets # decode full path to dbfile so imports from other directories don't get confused if args.dbfile != ':memory:': args.dbfile = os.path.abspath(args.dbfile) shlog.verbose(args) shlog.verbose("only tests matching %s will be considered", (args.only)) q = vanilla_utils.Q(args.dbfile, args.flush) tag_acquire = tags.Acquire(args, "TAGS", q) repos_acquire = repos.Acquire(args, "repos", q) s3_acquire = s3.Acquire(args, "s3", q) secret_acquire = secrets.Acquire(args, "secrets", q) certificate_acquire = certificates.Acquire(args, "TAGS", q) load_balancer_acquire = load_balancer.Acquire(args, "load_balancer", q) instances_acquire = instances.Acquire(args, "instances", q) # at this point data is in the relattion DB if args.dump: tag_acquire.print_data() s3_acquire.print_data() secret_acquire.print_data() certificate_acquire.print_data() load_balancer_acquire.print_data() repos_acquire.print_data() # instances_acquire.print_data() exit() # reporting actions are driven by instanitating the classes. tag_reports = tags.Report(args, "Tagging Rule Check", q) s3_reports = s3.Report(args, "s3", q) secret_reports = secrets.Report(args, "secrets", q) cert_reports = certificates.Report(args, "Certificates", q) load_balancer_reports = load_balancer.Report(args, "load_balancers", q) repo_reports = repos.Report(args, "repos", q) instances_reports = instances.Report(args, "Tagging Rule Check", q) assets.Acquire(args, "assets", q) assets.Report(args, "assets", q)
def make_data(self): """ Make a table called CERTIFICATES based on certificates held in AWS """ if self.does_table_exist(): shlog.normal("{} data already collected".format(self.table_name)) return shlog.normal("beginning to make {} data".format(self.table_name)) #import pdb; pdb.set_trace() # Make a flattened table for the tag data. # one tag, value pair in each record. sql = """CREATE TABLE certificates (asset text, domain text, arn text, short_arn text, inuseby text, hash text, record json)""" shlog.verbose(sql) self.q.q(sql) # Get the tags for each region. # accomidate the boto3 API can retul data in pages for page, _ in self._pages_all_regions('acm', 'list_certificates'): for certificate in page['CertificateSummaryList']: arn = certificate["CertificateArn"] domain = certificate["DomainName"] short_arn = aws_utils.shortened_arn(arn) try: for page, _ in self._pages_all_regions('acm','describe_certificate',parameter={'CertificateArn':arn}): # for response in page: if 'Nothing' not in page.keys(): record = page record = record["Certificate"] inuseby = record["InUseBy"] inuseby = [aws_utils.shortened_arn(arn) for arn in inuseby] inuseby = ",".join(inuseby) asset = "Cert:{} for use by {}".format(domain, inuseby) hash = vanilla_utils.tiny_hash(arn) record = self._json_clean_dumps(record) sql = "INSERT INTO certificates VALUES (?, ?, ?, ?, ?, ?, ?)" params = (asset, domain, arn, short_arn, inuseby, hash, record) self.q.executemany(sql, [params]) # populate the all_json table self._insert_all_json("certificate", short_arn, record) except Exception as e: if "ResourceNotFoundException" in str(e): print("certificate not found!")
def ec2stop(args, dryrun=False): """ Make a request to stop all ec2 instances :return: """ from botocore.exceptions import ClientError from security_scripts.information.lib import aws_utils as au # only works in plugin and IDE args.session = boto3.Session(profile_name=args.profile) regions = au.decribe_regions_df(args) # use for deployment # regions = {'RegionName':['us-east-2']} # test mode for region in regions['RegionName']: shlog.normal('Stopping region ' + region) # init connection to region and get instances there client = boto3.client('ec2', region_name=region) response = client.describe_instances()['Reservations'] # go through intance ids for inst in response: # ...and allow termination... instance = inst['Instances'][0]['InstanceId'] shlog.verbose('Allowing API termination for instance ' + instance + ' in region ' + region) response = client.modify_instance_attribute( InstanceId=instance, DisableApiTermination={'Value': False}) shlog.debug(response) # ...and perform halt shlog.normal('Stopping instance ' + instance + ' in region ' + region) try: response = client.stop_instances(InstanceIds=[instance], DryRun=dryrun, Force=True) except ClientError as ce: if dryrun: # client error is expected when simulating shlog.normal('Stop simulation succeeded for instance ' + instance) shlog.verbose('Success code: ' + str(ce)) else: # we might actually want to catch real exceptions raise ce pass
def env_control(): import platform import os var = 'SCIMMA_SECURITY_CFG' val = os.environ.get(var) if val: shlog.verbose('Reading custom config file location from $' + var + ' as ' + val) return val else: if platform.system() in ['Linux', 'Darwin']: # *nixes val = '~/.scimma-security.cfg' with open(os.path.expanduser("~/.bash_profile"), "a") as outfile: outfile.write("export {0}={1}".format(var, val)) # pass with open(os.path.expanduser("~/.zshenv"), "a") as outfile: # pass outfile.write("export {0}={1}".format(var, val)) shlog.verbose('$' + var + ' written to ~/.bash_profile and ~/.zshenv as ' + val) else: # windows val = '$HOME\\scimma-security.cfg' os.system('SETX {0} "{1}" /M'.format(var, val)) shlog.verbose('$' + var + ' written as system variable with value ' + val) return val
def make_data(self): """ Make a table called ec2 based on tagging data. This collection of data is based on the resourcetaggingapi If the tags table exists, then we take it data collection would result in duplicate rows. """ if self.does_table_exist(): shlog.normal("ec2 data already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Make a flattened table for the tag data. # one tag, value pair in each record. sql = """CREATE TABLE ec2 ( instance TEXT, vpc TEXT, subnet TEXT, publicdnsname TEXT, privatednsame TEXT, privateipaddress TEXT, keyname TEXT, record JSON)""" shlog.verbose(sql) self.q.q(sql) # Get the tags for each region. # accomidate the boto3 API can retul data in pages for page, _ in self._pages_all_regions('ec2', 'describe_instances'): reservations = page["Reservations"] for r in reservations: for i in r["Instances"]: record = self._json_clean_dumps(i) instance = i.get("InstanceId",""); vpc = i.get("VpcId",""); subnet = i.get("SubnetId",""); publicdnsname =i.get("PublicDnsName",""); privatednsame =i.get('PrivateDnsName',""); privateipaddress = i.get("PrivateIpAddress",""); keyname= i.get("KeyName","") sql = '''INSERT INTO ec2 VALUES (?, ?, ?, ?, ?, ?, ?,?)''' list = (instance, vpc, subnet, publicdnsname, privatednsame, privateipaddress, keyname, record) self.q.executemany(sql,[list]) # populate the all_json table self._insert_all_json("ec2", instance, record)
def make_data(self): """ Make a table called load_balancer based on tagging data. This collection of data is based on the resourcetaggingapi If the tags table exists, then we take it data collection would result in duplicate rows. """ if self.does_table_exist(): shlog.normal("load_balancers already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Make a flattened table for the tag data. # one tag, value pair in each record. sql = """CREATE TABLE load_balancers ( name TEXT, vpc TEXT, record JSON ) """ shlog.verbose(sql) self.q.q(sql) # classic load balancers for page, _ in self._pages_all_regions('elb', 'describe_load_balancers'): for elb in page['LoadBalancerDescriptions']: # import pdb ; pdb.set_trace() name = elb['LoadBalancerName'] vpc = elb['VPCId'] record = elb record = self._json_clean_dumps(record) sql = """ INSERT INTO load_balancers VALUES (?, ?, ?) """ list = (name, vpc, record) shlog.verbose(sql) self.q.executemany(sql, [list]) # application balancers for page, _ in self._pages_all_regions('elbv2', 'describe_load_balancers'): for elb in page['LoadBalancers']: #import pdb ; pdb.set_trace() name = elb['LoadBalancerName'] vpc = elb['VpcId'] type = elb['Type'] record = elb record = self._json_clean_dumps(record) sql = """ INSERT INTO load_balancers VALUES (?, ?, ?) """ list = (name, vpc, record) shlog.verbose(sql) self.q.executemany(sql, [list])
def make_data(self): """ Make a table called TAGS based on tagging data. This collection of data is based on the resourcetaggingapi If the tags table exists, then we take it data collection would result in duplicate rows. """ if self.does_table_exist(): shlog.normal("repos already collected") return if self.netrc_has_credentials(): shlog.normal('using api.github.com login+token from ~/.netrc') else: shlog.normal('api.github.com credentials not found in ~/.netrc') exit(0) shlog.normal("beginning to make {} data".format(self.name)) # Make a flattened table for the tag data. # one tag, value pair in each record. sql = "CREATE TABLE repos (name TEXT, description TEXT, url TEXT, who TEXT, " \ "can_pull text, can_push text, admins text, hash TEXT, record JSON)" shlog.verbose(sql) self.q.q(sql) #n.b. beta interface when this was coded cmd = 'curl -n -H "Accept: application/vnd.github.inertia-preview+json" ' \ ' https://api.github.com/orgs/scimma/repos' import subprocess #n.b check will throw an execption if curl exits with non 0 #rik is that we get valid output, lokin of like a "404" page. shlog.verbose(cmd) result = subprocess.run(cmd, text=True, capture_output=True, shell=True, check=True) stdout = result.stdout stderr = result.stderr if len(result.stdout) < 200: shlog.verbose("github curl error: stdout:{} stderr:{}".format( stdout, stderr)) exit(1) result = json.loads(result.stdout) for repo in result: # import pdb; pdb.set_trace() name = repo['full_name'] description = "{}(Private={})".format(repo['description'], repo['private']) where = repo['url'] #url who = repo['owner']['login'] # is really the account. hash = vanilla_utils.tiny_hash(name) record = json.dumps(result) # get user list collaborators_url = repo['collaborators_url'].split('{')[0] cmd = 'curl -n ' + collaborators_url result = subprocess.run(cmd, text=True, capture_output=True, shell=True, check=True) result = json.loads(result.stdout) try: # attempt to sort the users members = { 'can_pull': self.get_members(result, 'pull'), 'can_push': self.get_members(result, 'push'), 'admins': self.get_members(result, 'admin') } except subprocess.CalledProcessError: # repo is unreadable cmd = 'curl -n ' + collaborators_url + ' | jq ".message"' CPE = subprocess.run(cmd, text=True, capture_output=True, shell=True, check=True) CPE = json.loads(result.stdout) admins = result sql = "INSERT INTO repos VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" self.q.executemany( sql, [(name, description, where, who, members['can_pull'], members['can_push'], members['admins'], hash, record)])
def make_data(self): """ Make a table called TAGS based on tagging data. This collection of data is based on the resourcetaggingapi If the tags table exists, then we take it data collection would result in duplicate rows. """ if self.does_table_exist(): shlog.normal("tags data already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Make a flattened table for the tag data. # one tag, value pair in each record. sql = "create table s3 (asset text, bucket text, arn text, region text, npolicies text, ngrants text, grants json, policy_status text, bucket_policy JSON, record JSON)" shlog.verbose(sql) self.q.q(sql) # get buckets from local storage response = self.json_from_file(self.s_path)[0] # still need to init session = boto3.Session(profile_name=self.args.profile) client = session.client('s3') # the rest is untouched - this module still makes boto3 requests. shoudl we keep this info locally? # should we pull it with L0A? #ignore the 'ResponseMetadata' == seems to be no pagination context in there. buckets = response["Buckets"] for bucket in buckets: name = bucket["Name"] # arn can be computed from aws partition (e.g aws, aws-us-gov) and bucket name arn = "arn:{}:s3:::{}".format("aws", name) region = client.head_bucket( Bucket=name )['ResponseMetadata']['HTTPHeaders']['x-amz-bucket-region'] grants = client.get_bucket_acl(Bucket=name)["Grants"] ngrants = len(grants) asset = "aws bucket:{}".format(name) try: result = client.get_bucket_policy_status(Bucket=name) policy_status = result["PolicyStatus"] bucket_policy = client.get_bucket_policy(Bucket=name)["Policy"] npolicies = len(bucket_policy) except botocore.exceptions.ClientError: policy_status = {"IsPublic": False} bucket_policy = [] npolicies = 0 except: raise record = bucket record['BucketPolicy'] = policy_status record['PolicyStatus'] = bucket_policy record['Region'] = region record = self._json_clean_dumps(record) sql = '''INSERT INTO s3 VALUES (?,?,?,?,?,?,?,?,?,?)''' npolicies = "{}".format(npolicies) ngrants = "{}".format(ngrants) list = (asset, name, arn, region, npolicies, ngrants, json.dumps(grants), json.dumps(policy_status), json.dumps(bucket_policy), record) self.q.executemany(sql, [list]) # populate the all_json table self._insert_all_json("s3", name, record)
def make_data(self): """ MAKE DATA FOR GIT REPOS. """ if self.does_table_exist(): shlog.normal("Assets already collected") return shlog.normal("beginning to make {} data".format(self.name)) # Master Table for other reports to deposit asset data they # one tag, value pair in each record. sql = "CREATE TABLE {} ({} TEXT)".format( self.table_name, " TEXT, ".join(self.asset_columns)) shlog.vverbose(sql) self.q.q(sql) # get all the tables. sql = """ SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name NOT LIKE 'sqlite_%' AND name LIKE 'asset_data%' ORDER BY 1; """ for table in self.q.q(sql).fetchall(): table = table[0] #one item selects are retuned in a list. # # Below is a Massive query fond on the web to ger # column meta-data out of sqlite I'd like to # finds somemething simpler, but don;t want to touch this sql = """ SELECT m.name as tableName, p.name as columnName, p.type as columnType FROM sqlite_master m left outer join pragma_table_info((m.name)) p on m.name <> p.name WHERE m.name is '{}' AND columnName in {} order by tableName, columnName """.format(table, self.asset_columns) shlog.vverbose(sql) # # Now get a string with col, cl, cos # cols = [col for (_, col, _) in self.q.q(sql)] if not cols: shlog.verbose( "{} has no matching columns for asset table".format(table)) continue cols_text = ", ".join(cols) # # and populate the assets table with # fields that match # sql = """ INSERT INTO assets ({}) SELECT {} from {} """.format(cols_text, cols_text, table) shlog.vverbose(sql) self.q.q(sql)
def main(args): """Run reports pulling in existing infrastructure and mapping them to services""" from security_scripts.information.lib import vanilla_utils from security_scripts.information.lib import xs3 from security_scripts.information.lib import xtags from security_scripts.information.lib import L0A from security_scripts.information.lib import L0A_L0B from security_scripts.information.lib import tag_counter from security_scripts.information.lib import untagged_lister from security_scripts.information.lib import L1_L2 from security_scripts.information.lib import L0B_L1 from security_scripts.information.lib import L2_L3 # decode full path to dbfile so imports from other directories don't get confused if args.dbfile != ':memory:': args.dbfile = os.path.abspath(args.dbfile) shlog.verbose(args) shlog.verbose("only tests matching %s will be considered", (args.only)) q = vanilla_utils.Q(args.dbfile, args.flush) # dict structure: lvl name: [import name, priority] xreports_dict = { 'L0A': ['L0A', 1], 'L0B': ['L0A_L0B', 2], 'L1': ['L0B_L1', 3], 'L2': ['L1_L2', 4], 'L3': ['L2_L3', 5] } if args.start not in xreports_dict.keys(): shlog.normal('illegal start parameter: {}'.format(args.start)) exit(0) if args.end not in xreports_dict.keys(): shlog.normal('illegal end parameter: {}'.format(args.start)) exit(0) for x in xreports_dict: # compare x's priority to arg's priority if xreports_dict[x][1] >= xreports_dict[args.start][ 1] and xreports_dict[x][1] <= xreports_dict[args.end][1]: exec_string = '{}.Acquire(args, "{}", q)'.format( xreports_dict[x][0], xreports_dict[x][0]) exec(exec_string) # there's more stuff down there, think about implementing it exit(0) s3_acquire = xs3.Acquire(args, "s3", q) tags_acquire = xtags.Acquire(args, "tags", q) tag_c_acquire = tag_counter.Acquire(args, "TAG_COUNTER", q) untagged = untagged_lister.Acquire(args, "UNTAGGED_LISTER", q) # at this point data is in the relattion DB if args.dump: # tag_acquire.print_data() # s3_acquire.print_data() # secret_acquire.print_data() # certificate_acquire.print_data() # load_balancer_acquire.print_data() # repos_acquire.print_data() # instances_acquire.print_data() exit() # reporting actions are driven by instanitating the classes. tag_reports = xtags.Report(args, "Tagging Rule Check", q) tag_c_report = tag_counter.Report(args, "Tagging Count Check", q) untagged_report = untagged_lister.Report(args, "Untagged Resources", q) exit()