def loads(self):
'''
loads a raw file of bug report
:return:
'''
fileConnt = self.getFileCounts(self.SourceBugPath)
bugitems = []
# show progress
progress = Progress(u'[%s] Loading bug reports' % self.__name__, 2, 10,
True)
progress.set_upperbound(fileConnt)
progress.start()
for root, dirs, files in os.walk(self.SourceBugPath):
for f in files:
if f[:f.find(u'-')].strip().lower() != self.ProjectName.lower(
):
continue
#shutil.copy(os.path.join(root, f), os.path.join(_dest, f))
bugitem = self.get_bugitem(os.path.join(root, f))
if bugitem is not None:
bugitems.append(bugitem)
progress.check()
progress.done()
return bugitems
def make_childmap(self):
visited = set([])
q = Queue()
q.put((self.git.head.commit, None)) # (commit, child_hash)
progress = Progress(u'[%s] making git childmap' % self.__name__, 500,
10000, False)
progress.set_point(0)
progress.start()
while q.empty() is False:
progress.check()
commit, child_hash = q.get()
commit_hash = str(commit)[:7]
# create child map
if commit_hash not in self.childmap:
self.childmap[commit_hash] = set([])
if child_hash is not None:
self.childmap[commit_hash].add(child_hash)
if commit_hash in visited: continue
visited.add(commit_hash)
# pass itself to parent
for parent in commit.parents:
q.put((parent, commit_hash))
# add ancestors if this commit has no parents
if len(commit.parents) == 0:
self.ancestors.add(commit_hash)
progress.done()
pass
def load(self, _force=False):
'''
Load commit info from GitLogPath
:return: {bugID:[{'hash':u'', 'author':u'', 'commit_date':u'', 'message':u'', 'fixedFiles':{}}, {}, ...], ...}
'''
if os.path.exists(self.GitLogPath) is False or _force is True:
self.make()
logfile = codecs.open(self.GitLogPath, 'r', 'utf-8')
progress = Progress(u'[%s] loading git log data' % self.__name__, 1000,
20000, False)
progress.set_point(0)
progress.start()
for logitem in self.file_loader(logfile, _with_filter=False):
# filter unuseful logs
if len(logitem['fixedFiles']) == 0: continue
# We only use bug report id in log message
# mapping bug report ID
logitem['linked_bug'] = re.findall(
r'%s-[0-9]+' % self.ProjectName.upper(), logitem['message'])
logitem['linked_bug'] = set(logitem['linked_bug'])
for linked_id in logitem['linked_bug']:
if linked_id not in self.logs:
self.logs[linked_id] = [logitem]
else:
self.logs[linked_id].append(logitem)
progress.check()
progress.done()
logfile.close()
return self.logs
def setup_subscriptions(account_id, environment_id, trails_configuration, sourceAccountSession, targetAccountSession):
prefix = "Setting up CloudTrails subscriptions for environment %s:" % environment_id
regions = trails_configuration.keys()
if not len(regions): return trails_configuration
progress = Progress(
len(regions),
prefix + "\t\t")
for region in regions:
trail = trails_configuration.pop(region)
try:
trails_configuration[region] = aws_setup_subscription(
account_id,
environment_id,
trail,
sourceAccountSession,
targetAccountSession,
progress)
except Exception as e:
print "Error: %s" % (e)
# details = e.args[0]
# trails_configuration[u'invalid_trails'][region] = details['reason']
progress.report()
progress.done()
return trails_configuration
def fill_SubjectSheet(self, _sheet, _group, _srcCounts, _bugCounts, _dupCounts): projects = _bugCounts.keys() projects.sort() size = sum([len(_bugCounts[project]) for project in projects]) progress = Progress(u'[%s] fill subject' % self.__name__, 2, 10, True) progress.set_point(0).set_upperbound(size) progress.start() styles = [self.base_format, self.base_format, self.base_format, self.number_format, self.number_format] for project in projects: for version in _bugCounts[project].keys(): if version == 'all': continue values = [_group, project, version.upper(), _bugCounts[project][version], _srcCounts[project][version]] self.input_row(_sheet, self.subj_data_row, 6, values, styles) self.subj_data_row += 1 progress.check() progress.done() #summary styles = [self.subtitle_format, self.subtitle_format, self.number_format, self.number_format, self.number_format] for project in projects: values = [_group, project.upper(), _bugCounts[project]['all'], _dupCounts[project], _srcCounts[project]['all']] self.input_row(_sheet, self.subj_summary_row, 0, values, styles) self.subj_summary_row += 1 pass
def source_counting(self, _group, _project):
statistics = {}
progress = Progress('source counting', 2, 10, True)
progress.set_upperbound(len(self.S.versions[_project].keys()))
progress.start()
for version in self.S.versions[_project].keys():
vname = VersionUtil.get_versionName(version, _project)
repo = os.path.join(self.S.getPath_source(_group, _project,
vname), )
result = self.getCodeCount(repo)
if result is None: continue
statistics[vname] = result
progress.check()
progress.done()
maxValue = 0
for vname in statistics:
if maxValue < statistics[vname]:
maxValue = statistics[vname]
statistics['max'] = maxValue
pretty = PrettyStringBuilder(_indent_depth=2)
text = pretty.get_dicttext({_project: statistics})
f = open(
os.path.join(self.S.getPath_base(_group, _project),
u'sources.txt'), 'w')
f.write(text)
f.close()
def unhash_folder(_src, _dest):
'''
hashed folder ==> unshed folder
example) path/aa/00/filename ==> path/filename
:param _src:
:param _dest:
:return:
'''
if os.path.exists(_dest) is False:
os.makedirs(_dest)
progress = Progress(u'Bug reports is merging', 20, 1000, False)
progress.start()
for root, dirs, files in os.walk(_src):
for f in files:
shutil.copy(os.path.join(root, f), os.path.join(_dest, f))
progress.check()
progress.done()
def make_tagmap(self, ):
q = Queue()
visited = set([])
# root node find (queue init)
for item in list(self.ancestors):
q.put((item, None)) # (commit_hash, tagname)
# For each item in queue
progress = Progress(u'[%s] making git tagmaps' % self.__name__, 500,
10000, False)
progress.set_point(0)
progress.start()
while q.empty() is False:
commit_hash, parent_tag = q.get()
# If this commit in tags, map with commit_hash and tag
if commit_hash in self.tags:
commit_tag = self.tags[commit_hash]
self.tagmap[commit_hash] = commit_tag
# if this commit not in tags, map with child commit_hash and tag
else:
if commit_hash not in self.tagmap:
self.tagmap[commit_hash] = parent_tag
else:
# compare time previous_tag and parent_tag
previous_tag = self.tagmap[commit_hash]
pre_time = self.tagtimes[previous_tag]
par_time = self.tagtimes[parent_tag]
if par_time > pre_time:
self.tagmap[commit_hash] = parent_tag
commit_tag = parent_tag
if commit_hash not in visited:
visited.add(commit_hash)
for child_hash in self.childmap[commit_hash]:
q.put((child_hash, commit_tag))
progress.check()
progress.done()
pass
def load_raw(self, _force=False):
'''
Load commit info from GitLogPath
:return: {bugID:[{'hash':u'', 'author':u'', 'commit_date':u'', 'message':u'', 'fixedFiles':{}}, {}, ...], ...}
'''
if os.path.exists(self.GitLogPath) is False or _force is True:
self.make()
self.logs = []
logfile = codecs.open(self.GitLogPath, 'r', 'utf-8')
progress = Progress(u'[%s] loading git log data' % self.__name__, 1000,
20000, False)
progress.set_point(0)
progress.start()
for logitem in self.file_loader(logfile):
# filter unuseful logs
#if len(logitem['fixedFiles'])==0: continue
if logitem['hash'] == '': continue
self.logs.insert(0, logitem)
progress.check()
progress.done()
logfile.close()
return self.logs
def main():
args = get_user_input()
targetAccountSession = args.profile and boto3.session.Session(profile_name = args.profile) or None
sourceAccountSession = args.source_profile and boto3.session.Session(profile_name = args.source_profile) or None
#
# Connect to CloudInsight
#
ci = CI_API(args.user, args.password, account_id = args.account, locality = args.locality)
print "Successfully logged in into CloudInsight. Account: %s(%s), User: %s" % \
(ci.auth_account_name, ci.auth_account_id, ci.auth_user_name)
#
# Load configuration file
#
config = {}
environments = []
with open(args.config) as data_file:
config = json.load(data_file)
if u'role' not in config:
raise Exception("Missing 'role' attribute in '%s' configuration file" % (args.config))
if u'external_id' not in config:
raise Exception("Missing 'external_id' attribute in '%s' configuration file" % (args.config))
if u'trails' not in config and u'regions' not in config :
raise Exception("Missing 'trails' and 'regions' configuration in '%s' configuration file" % (args.config))
role_arn = config[u'role']
external_id = config[u'external_id']
if u'environments' in config:
environments = config[u'environments']
elif u'aws_account_id' in config:
environments = ci.get_environments(config[u'aws_account_id'])
#
# Get CloudInsight Credential ID for the specified role
#
credential_id = get_credential(ci, role_arn, external_id)[u'credential'][u'id']
print "Obtained credential id for '%s' role" % (role_arn)
#
# Get sources for environments specified in the configuration file
#
sources = []
trails = {}
progress = Progress(
len(config[u'regions']),
"Validating configuration.\t\t\t\t\t\t\t\t\t\t")
for region_name, region_config in config[u'regions'].iteritems():
progress.report()
if region_config[u'type'] == u'queue':
if not u'queue' in region_config:
raise Exception("Invalid config file. 'queue' property is missing for '%s' region" % region_name)
if targetAccountSession and not validate_queue(region_name, region_config[u'queue'], targetAccountSession):
raise Exception("Invalid config file. '%s' queue doesn't exist in '%s' region in '%s' AWS Account." %\
(region_config[u'queue'], region_name, get_account_id(targetAccountSession) ) )
bucket_region = u'bucket_region' in region_config and region_config[u'bucket_region'] or u'us-east-1'
for environment_id in environments:
result = ci.get_sources(environment_id = environment_id, region = region_name)
sources.append(update_source_config(
len(result) and result[0] or None,
ci.account_id,
environment_id,
region_name,
credential_id = credential_id,
bucket_region = bucket_region,
queue = get_queue_name(region_config[u'queue'])))
elif region_config[u'type'] == u'trail':
if u'trail' not in region_config or not region_config[u'trail']:
raise Exception("Invalid config file. 'trail' property is missing '%s' region" % region_name)
trail = get_cloud_trail_configuration(
region_name,
region_config[u'trail'],
sourceAccountSession,
targetAccountSession)
if trail:
trails[region_name] = trail
progress.done()
#
# Setup CloudTrail subscriptions
#
for environment_id in environments:
trails_configuration = setup_subscriptions(
args.account,
environment_id,
trails,
sourceAccountSession,
targetAccountSession)
for region_name, trail_configuration in trails_configuration.iteritems():
result = ci.get_sources(environment = environment_id, region = region_name)
sources.append(update_source_config(
len(result) and result[0] or None,
ci.account_id,
environment_id,
region_name,
credential_id = credential_id,
bucket_region = trail_configuration[u'bucket_region'],
queue = trail_configuration[u'sqs_queue_name']))
#
# Create CloudInsight sources
#
for source in sources:
print "Updating '%s' source in '%s' environment." %\
(source[u'source'][u'name'], source[u'source'][u'environment'])
ci.create_source(source)
print "Successfully updated CloudInsight configuration."
print_instructions(role_arn)
