def time_ordered_event_archives(args, template_path):
"""
Return a list of paths, sorted by increasing date.
Construct the list based the template (ie havig wildcards),
Considering only the first timestamp in the file holding the events.
Given the vault files are small timeslices, this should produce an
event strean that is "pretty good" nearly time ordered.
"""
import os
import glob
import gzip
template_path = os.path.expanduser(template_path)
list = []
for path in glob.glob(template_path):
with gzip.open(path, 'rb') as f:
data = json.loads(f.read())
list.append("{} {}".format(data["Records"][0]["eventTime"], path))
list.sort() #sort on date
list = [l.split(" ") for l in list]
if list != []:
shlog.vverbose("first date, file available is {}".format(list[0]))
shlog.vverbose("last date, file available is {}".format(list[-1]))
return [l[1] for l in list]
def match(obj, arr, valuematch, c):
"""Recursively search for values"""
if isinstance(obj, dict):
for k, v in obj.items():
if isinstance(v, (dict, list)):
#recurr if composite type
match(v, arr, valuematch, c)
elif c.match(v):
# save base type that matches
arr.append([k, c.matched])
else:
# no match, let go
pass
shlog.vverbose("skipping: %s", v)
elif isinstance(obj, list):
for item in obj:
if isinstance(item, (dict, list)):
#recurr if composite type
match(item, arr, valuematch, c)
elif c.match(item):
# is it a basetype that matches?
arr.append([c.matched])
else:
# no match, let it go
pass
shlog.vverbose("skipping: %s", item)
return arr
def time_ordered_event_archives(args, template_path):
"""
Return a list of paths, sorted by increasing date.
Construct the list based the template (ie havig wildcards),
Considering only the first timestamp in the file holding the events.
Given the vault files are small timeslices, this should produce an
event strean that is "pretty good" nearly time ordered.
"""
import os
import glob
template_path = os.path.expanduser(template_path)
list = []
for path in glob.glob(template_path):
# get stamp from file name
# with gzip.open(path, 'rb') as f:
filedate = re.findall('[^\/]+$', path)[0].split('_')[4]
list.append("{} {}".format(filedate, path))
list.sort() # sort on date
list = [l.split(" ") for l in list]
if list != []:
shlog.vverbose("first date, file available is {}".format(list[0]))
shlog.vverbose("last date, file available is {}".format(list[-1]))
return [l[1] for l in list]
def make_asset_format(self):
"""
Create asset format from fundamental descriptive data
"""
sql = '''
CREATE TABLE asset_data_certificates AS
SELECT
'R:AWS users w/ authority to read/modify X509 cert for:'||domain asset,
'R:Protects via:'||inuseby description,
'D:Assures endpoint is Scimma endpoint' business_value,
'D:Attacker might spoof Scimma Service' impact_c,
'C' type,
'R:'||short_arn "where"
FROM
certificates
'''
shlog.vverbose(sql)
self.q.q(sql)
def make_asset_data(self):
"""
Make table(s) for interetion into the master asset table.
"""
sql = '''
CREATE TABLE asset_data_repos AS
SELECT
"git_repo_"||hash tag,
"R:Contents of: "||name asset,
"R:Source Materials for "||description description,
"D:Maintains source files for topic" business_value,
"D:None, open to public" impact_c,
"D:contents corrupted" impact_i,
"D:users of data disrupted" impact_a,
"D" type,
"R:"||url "where",
"R:"||who who
FROM repos
'''
shlog.vverbose(sql)
r = self.q.q(sql)
sql = '''
CREATE TABLE asset_data_repo_credentials AS
SELECT
"git_repo_"||hash tag,
"R:Credentials to administer/read/write Git repo: "||name asset,
"R:Defines who can drop/write/read the repo "||description description,
"D:PRovides access control to repo for staff and community " business_value,
"D:disruption by repo write or rpo admin creds" impact_c,
"D:Lost? user uses github to replace credential" impact_a,
"C" type,
"R: Personal (not SCiMMA controlled github identity " "where",
"R: Staff authorized to administer/read/write repo" who
FROM repos
'''
shlog.vverbose(sql)
r = self.q.q(sql)
def make_data(self):
"""
MAKE DATA FOR GIT REPOS.
"""
if self.does_table_exist():
shlog.normal("Assets already collected")
return
shlog.normal("beginning to make {} data".format(self.name))
# Master Table for other reports to deposit asset data they
# one tag, value pair in each record.
sql = "CREATE TABLE {} ({} TEXT)".format(
self.table_name, " TEXT, ".join(self.asset_columns))
shlog.vverbose(sql)
self.q.q(sql)
# get all the tables.
sql = """
SELECT name FROM sqlite_master
WHERE type IN ('table','view')
AND name NOT LIKE 'sqlite_%'
AND name LIKE 'asset_data%'
ORDER BY 1;
"""
for table in self.q.q(sql).fetchall():
table = table[0] #one item selects are retuned in a list.
#
# Below is a Massive query fond on the web to ger
# column meta-data out of sqlite I'd like to
# finds somemething simpler, but don;t want to touch this
sql = """
SELECT m.name as tableName,
p.name as columnName,
p.type as columnType
FROM sqlite_master m
left outer join pragma_table_info((m.name)) p
on m.name <> p.name
WHERE m.name is '{}'
AND columnName in {}
order by tableName, columnName
""".format(table, self.asset_columns)
shlog.vverbose(sql)
#
# Now get a string with col, cl, cos
#
cols = [col for (_, col, _) in self.q.q(sql)]
if not cols:
shlog.verbose(
"{} has no matching columns for asset table".format(table))
continue
cols_text = ", ".join(cols)
#
# and populate the assets table with
# fields that match
#
sql = """
INSERT INTO assets ({})
SELECT {} from {}
""".format(cols_text, cols_text, table)
shlog.vverbose(sql)
self.q.q(sql)