def loads(self): ''' loads a raw file of bug report :return: ''' fileConnt = self.getFileCounts(self.SourceBugPath) bugitems = [] # show progress progress = Progress(u'[%s] Loading bug reports' % self.__name__, 2, 10, True) progress.set_upperbound(fileConnt) progress.start() for root, dirs, files in os.walk(self.SourceBugPath): for f in files: if f[:f.find(u'-')].strip().lower() != self.ProjectName.lower( ): continue #shutil.copy(os.path.join(root, f), os.path.join(_dest, f)) bugitem = self.get_bugitem(os.path.join(root, f)) if bugitem is not None: bugitems.append(bugitem) progress.check() progress.done() return bugitems
def make_childmap(self): visited = set([]) q = Queue() q.put((self.git.head.commit, None)) # (commit, child_hash) progress = Progress(u'[%s] making git childmap' % self.__name__, 500, 10000, False) progress.set_point(0) progress.start() while q.empty() is False: progress.check() commit, child_hash = q.get() commit_hash = str(commit)[:7] # create child map if commit_hash not in self.childmap: self.childmap[commit_hash] = set([]) if child_hash is not None: self.childmap[commit_hash].add(child_hash) if commit_hash in visited: continue visited.add(commit_hash) # pass itself to parent for parent in commit.parents: q.put((parent, commit_hash)) # add ancestors if this commit has no parents if len(commit.parents) == 0: self.ancestors.add(commit_hash) progress.done() pass
def load(self, _force=False): ''' Load commit info from GitLogPath :return: {bugID:[{'hash':u'', 'author':u'', 'commit_date':u'', 'message':u'', 'fixedFiles':{}}, {}, ...], ...} ''' if os.path.exists(self.GitLogPath) is False or _force is True: self.make() logfile = codecs.open(self.GitLogPath, 'r', 'utf-8') progress = Progress(u'[%s] loading git log data' % self.__name__, 1000, 20000, False) progress.set_point(0) progress.start() for logitem in self.file_loader(logfile, _with_filter=False): # filter unuseful logs if len(logitem['fixedFiles']) == 0: continue # We only use bug report id in log message # mapping bug report ID logitem['linked_bug'] = re.findall( r'%s-[0-9]+' % self.ProjectName.upper(), logitem['message']) logitem['linked_bug'] = set(logitem['linked_bug']) for linked_id in logitem['linked_bug']: if linked_id not in self.logs: self.logs[linked_id] = [logitem] else: self.logs[linked_id].append(logitem) progress.check() progress.done() logfile.close() return self.logs
def setup_subscriptions(account_id, environment_id, trails_configuration, sourceAccountSession, targetAccountSession): prefix = "Setting up CloudTrails subscriptions for environment %s:" % environment_id regions = trails_configuration.keys() if not len(regions): return trails_configuration progress = Progress( len(regions), prefix + "\t\t") for region in regions: trail = trails_configuration.pop(region) try: trails_configuration[region] = aws_setup_subscription( account_id, environment_id, trail, sourceAccountSession, targetAccountSession, progress) except Exception as e: print "Error: %s" % (e) # details = e.args[0] # trails_configuration[u'invalid_trails'][region] = details['reason'] progress.report() progress.done() return trails_configuration
def fill_SubjectSheet(self, _sheet, _group, _srcCounts, _bugCounts, _dupCounts): projects = _bugCounts.keys() projects.sort() size = sum([len(_bugCounts[project]) for project in projects]) progress = Progress(u'[%s] fill subject' % self.__name__, 2, 10, True) progress.set_point(0).set_upperbound(size) progress.start() styles = [self.base_format, self.base_format, self.base_format, self.number_format, self.number_format] for project in projects: for version in _bugCounts[project].keys(): if version == 'all': continue values = [_group, project, version.upper(), _bugCounts[project][version], _srcCounts[project][version]] self.input_row(_sheet, self.subj_data_row, 6, values, styles) self.subj_data_row += 1 progress.check() progress.done() #summary styles = [self.subtitle_format, self.subtitle_format, self.number_format, self.number_format, self.number_format] for project in projects: values = [_group, project.upper(), _bugCounts[project]['all'], _dupCounts[project], _srcCounts[project]['all']] self.input_row(_sheet, self.subj_summary_row, 0, values, styles) self.subj_summary_row += 1 pass
def source_counting(self, _group, _project): statistics = {} progress = Progress('source counting', 2, 10, True) progress.set_upperbound(len(self.S.versions[_project].keys())) progress.start() for version in self.S.versions[_project].keys(): vname = VersionUtil.get_versionName(version, _project) repo = os.path.join(self.S.getPath_source(_group, _project, vname), ) result = self.getCodeCount(repo) if result is None: continue statistics[vname] = result progress.check() progress.done() maxValue = 0 for vname in statistics: if maxValue < statistics[vname]: maxValue = statistics[vname] statistics['max'] = maxValue pretty = PrettyStringBuilder(_indent_depth=2) text = pretty.get_dicttext({_project: statistics}) f = open( os.path.join(self.S.getPath_base(_group, _project), u'sources.txt'), 'w') f.write(text) f.close()
def unhash_folder(_src, _dest): ''' hashed folder ==> unshed folder example) path/aa/00/filename ==> path/filename :param _src: :param _dest: :return: ''' if os.path.exists(_dest) is False: os.makedirs(_dest) progress = Progress(u'Bug reports is merging', 20, 1000, False) progress.start() for root, dirs, files in os.walk(_src): for f in files: shutil.copy(os.path.join(root, f), os.path.join(_dest, f)) progress.check() progress.done()
def make_tagmap(self, ): q = Queue() visited = set([]) # root node find (queue init) for item in list(self.ancestors): q.put((item, None)) # (commit_hash, tagname) # For each item in queue progress = Progress(u'[%s] making git tagmaps' % self.__name__, 500, 10000, False) progress.set_point(0) progress.start() while q.empty() is False: commit_hash, parent_tag = q.get() # If this commit in tags, map with commit_hash and tag if commit_hash in self.tags: commit_tag = self.tags[commit_hash] self.tagmap[commit_hash] = commit_tag # if this commit not in tags, map with child commit_hash and tag else: if commit_hash not in self.tagmap: self.tagmap[commit_hash] = parent_tag else: # compare time previous_tag and parent_tag previous_tag = self.tagmap[commit_hash] pre_time = self.tagtimes[previous_tag] par_time = self.tagtimes[parent_tag] if par_time > pre_time: self.tagmap[commit_hash] = parent_tag commit_tag = parent_tag if commit_hash not in visited: visited.add(commit_hash) for child_hash in self.childmap[commit_hash]: q.put((child_hash, commit_tag)) progress.check() progress.done() pass
def load_raw(self, _force=False): ''' Load commit info from GitLogPath :return: {bugID:[{'hash':u'', 'author':u'', 'commit_date':u'', 'message':u'', 'fixedFiles':{}}, {}, ...], ...} ''' if os.path.exists(self.GitLogPath) is False or _force is True: self.make() self.logs = [] logfile = codecs.open(self.GitLogPath, 'r', 'utf-8') progress = Progress(u'[%s] loading git log data' % self.__name__, 1000, 20000, False) progress.set_point(0) progress.start() for logitem in self.file_loader(logfile): # filter unuseful logs #if len(logitem['fixedFiles'])==0: continue if logitem['hash'] == '': continue self.logs.insert(0, logitem) progress.check() progress.done() logfile.close() return self.logs
def main(): args = get_user_input() targetAccountSession = args.profile and boto3.session.Session(profile_name = args.profile) or None sourceAccountSession = args.source_profile and boto3.session.Session(profile_name = args.source_profile) or None # # Connect to CloudInsight # ci = CI_API(args.user, args.password, account_id = args.account, locality = args.locality) print "Successfully logged in into CloudInsight. Account: %s(%s), User: %s" % \ (ci.auth_account_name, ci.auth_account_id, ci.auth_user_name) # # Load configuration file # config = {} environments = [] with open(args.config) as data_file: config = json.load(data_file) if u'role' not in config: raise Exception("Missing 'role' attribute in '%s' configuration file" % (args.config)) if u'external_id' not in config: raise Exception("Missing 'external_id' attribute in '%s' configuration file" % (args.config)) if u'trails' not in config and u'regions' not in config : raise Exception("Missing 'trails' and 'regions' configuration in '%s' configuration file" % (args.config)) role_arn = config[u'role'] external_id = config[u'external_id'] if u'environments' in config: environments = config[u'environments'] elif u'aws_account_id' in config: environments = ci.get_environments(config[u'aws_account_id']) # # Get CloudInsight Credential ID for the specified role # credential_id = get_credential(ci, role_arn, external_id)[u'credential'][u'id'] print "Obtained credential id for '%s' role" % (role_arn) # # Get sources for environments specified in the configuration file # sources = [] trails = {} progress = Progress( len(config[u'regions']), "Validating configuration.\t\t\t\t\t\t\t\t\t\t") for region_name, region_config in config[u'regions'].iteritems(): progress.report() if region_config[u'type'] == u'queue': if not u'queue' in region_config: raise Exception("Invalid config file. 'queue' property is missing for '%s' region" % region_name) if targetAccountSession and not validate_queue(region_name, region_config[u'queue'], targetAccountSession): raise Exception("Invalid config file. '%s' queue doesn't exist in '%s' region in '%s' AWS Account." %\ (region_config[u'queue'], region_name, get_account_id(targetAccountSession) ) ) bucket_region = u'bucket_region' in region_config and region_config[u'bucket_region'] or u'us-east-1' for environment_id in environments: result = ci.get_sources(environment_id = environment_id, region = region_name) sources.append(update_source_config( len(result) and result[0] or None, ci.account_id, environment_id, region_name, credential_id = credential_id, bucket_region = bucket_region, queue = get_queue_name(region_config[u'queue']))) elif region_config[u'type'] == u'trail': if u'trail' not in region_config or not region_config[u'trail']: raise Exception("Invalid config file. 'trail' property is missing '%s' region" % region_name) trail = get_cloud_trail_configuration( region_name, region_config[u'trail'], sourceAccountSession, targetAccountSession) if trail: trails[region_name] = trail progress.done() # # Setup CloudTrail subscriptions # for environment_id in environments: trails_configuration = setup_subscriptions( args.account, environment_id, trails, sourceAccountSession, targetAccountSession) for region_name, trail_configuration in trails_configuration.iteritems(): result = ci.get_sources(environment = environment_id, region = region_name) sources.append(update_source_config( len(result) and result[0] or None, ci.account_id, environment_id, region_name, credential_id = credential_id, bucket_region = trail_configuration[u'bucket_region'], queue = trail_configuration[u'sqs_queue_name'])) # # Create CloudInsight sources # for source in sources: print "Updating '%s' source in '%s' environment." %\ (source[u'source'][u'name'], source[u'source'][u'environment']) ci.create_source(source) print "Successfully updated CloudInsight configuration." print_instructions(role_arn)