def _load(self): dir_info = self._data['connect_info'].get('directory') file_n = self._data['connect_info'].get('file_name') full_name = os.path.join(dir_info, file_n) i = 0 with open(full_name, 'r') as txt_file: csv_d_rdr = csv.DictReader(txt_file) for r in csv_d_rdr: if i == 0: if not Helper.is_empty(self._data['key_columns']): if not Helper.is_column_list_valid( self._data['key_columns'], r.keys()): self._logger.error( 'Specified primary keys don\'t match table columns' ) raise Exception self._add_row(r) i = i + 1 if not Helper.is_empty(self._data['key_columns']): if not self._check_primary_key_constraint_for_first_load(): self._rows = [] self._logger.error( 'The specified primary keys don\'t comply with primary key constraint' ) raise Exception self._logger.debug('CSVDataTable._load: Loaded ' + str(len(self._rows)) + ' rows')
def delete_by_template(self, template, limit=None): ''' :param template: Template to determine rows to delete. :return: Number of rows deleted. ''' if not Helper.is_empty(template): if not Helper.is_template_valid(template, self.get_columns()): self._logger.error( 'Some columns in the specified template don\'t match table columns' ) raise Exception rows = self.get_rows() r_indexes = [] for i in range(0, len(rows)): if limit is not None: if len(r_indexes) == limit: break if Helper.matches_template(rows[i], template): r_indexes.append(i) if len(r_indexes) == 0: return 0 elif len(r_indexes) == 1: self._delete_row(r_indexes[0]) return 1 else: self._delete_rows(r_indexes) return len(r_indexes)
def delete_by_template(self, template): ''' :param template: Template to determine rows to delete. :return: Number of rows deleted. ''' template_string = '' if not Helper.is_empty(template): if not Helper.is_template_valid(template, self.get_columns()): self._logger.error( 'Some columns in the specified template don\'t match table columns' ) raise Exception template_string = 'WHERE ' + self._compose_template_string( template) query = 'DELETE FROM ' + '`' + self._data[ 'table_name'] + '` ' + template_string with self._connection.cursor() as cursor: try: cursor.execute(query) if cursor.rowcount > 0: if self._auto_commit: self.commit() return cursor.rowcount except pymysql.Error as error: self._logger.error( 'Failed to delete record(s) in the table {}'.format(error)) raise Exception
def find_by_primary_key(self, key_fields, field_list=None): ''' :param key_fields: The list with the values for the key_columns, in order, to use to find a record. :param field_list: A subset of the fields of the record to return. :return: None, or a dictionary containing the requested fields for the record identified by the key. ''' if Helper.is_empty(self._data['key_columns']): self._logger.error('Table has no primary keys') raise Exception if not Helper.are_key_fields_valid(key_fields, self._data['key_columns']): self._logger.error('Key fields are not valid') raise Exception template = Helper.convert_key_fields_to_template( key_fields, self._data['key_columns']) result = self.find_by_template(template, field_list, 1) if Helper.is_empty(result): return None return result[0]
def main(): arguments = DictObj(vars(ArgsParser.getArgv(ArgsParser.createParser()))) # arguments = DictObj({"command": "dir", "file": "D:\PyProject\setterdog\src\\user_config.json", "debug": 0}) # print(arguments) debug.getInstance(arguments.debug, "./logger.log", 0, 0) Configuration.getInstance(arguments.file, arguments.command) # print(Configuration.GI().config) """ debug.getInstance(logging.getLevelName(Configuration.GI().config.debug.logLevel), Configuration.GI().config.debug.fileName, 0, 0)""" Helper.createFolder(Configuration.GI().config.output.outputDir) Helper.createFolder(Configuration.GI().config.report.reportDir) outputFile = os.path.join(Configuration.GI().config.output.outputDir, Configuration.GI().config.output.outputFileNamePattern. format(date=Helper.getCurrentDate())) stdout = StdoutData(Configuration.GI().config.launchCommand, True, outputFile) a = Analyzer(Configuration.GI().config.pattern) a.calculatePatternMatches(stdout.getOutputData()) report = Reporter(Configuration.GI().config.report.reportDir, Configuration.GI().config.report.reportFileNamePattern.format(date=Helper.getCurrentDate()), a.expandedPattern) report.writeReport() a.returnCode(stdout.returnCode)
def getOutputData(self): """ @return {String} - returns output of command """ try: process = Popen(self.command, stdout=PIPE) # output = str(process.stdout.read()) # Use for big data of stdout (output, error) = process.communicate() # Store data in clipboard self.returnCode = process.returncode if self.returnCode == 0: res = None if output: res = str(output) if self.writeToFile and self.fileName: lines = res.split('\\n') Helper.createNewFile(self.fileName, os.linesep.join(lines), 'w') return res else: debug.log().error( "Command '{}' failed, exit-code = {}, error = {}".format( self.command, self.returnCode, str(error))) except (ValueError, OSError, TypeError) as err: debug.log().critical(err) Helper.systemExit()
def checkValidConfigKey(self): """ Check the availability of required keys """ for obj in self.config.pattern: for key in self.validKeys: if key not in obj: debug.log().critical( "Not valid config pattern {}".format(obj)) Helper.systemExit()
def _violate_primary_key_constraint(self, new_keys_template): if Helper.is_empty(self._data['key_columns']): return False key_fields = Helper.extract_key_fields_from_template( new_keys_template, self._data['key_columns']) records = self.find_by_primary_key(key_fields) if Helper.is_empty(records): return False return True
def insert(self, new_record): ''' :param new_record: A dictionary representing a row to add to the set of records. :return: None ''' if not Helper.is_new_record_valid(new_record, self.get_columns()): self._logger.error('new_record must contains all columns') raise Exception column_strings = '(' value_strings = '(' for key in new_record: column_strings = column_strings + '`' + key + '`' + ', ' value_strings = value_strings + '\'' + new_record[key] + '\'' + ', ' column_strings = column_strings[:-2] + ')' value_strings = value_strings[:-2] + ')' with self._connection.cursor() as cursor: query = 'INSERT INTO ' + '`' + self._data[ 'table_name'] + '` ' + column_strings + ' VALUES ' + value_strings + ';' try: cursor.execute(query) if cursor.rowcount > 0: if self._auto_commit: self.commit() except pymysql.Error as error: self._logger.error( 'Failed to insert record(s) into the table {}'.format( error)) raise Exception
def find_by_template(self, template, field_list=None, limit=None, offset=None, order_by=None): ''' :param template: A dictionary of the form { 'field1' : value1, 'field2': value2, ...} :param field_list: A list of request fields of the form, ['fielda', 'fieldb', ...] :param limit: Do not worry about this for now. :param offset: Do not worry about this for now. :param order_by: Do not worry about this for now. :return: A list containing dictionaries. A dictionary is in the list representing each record that matches the template. The dictionary only contains the requested fields. ''' template_string = '' if not Helper.is_empty(template): if not Helper.is_template_valid(template, self.get_columns()): self._logger.error( 'Some columns in the specified template don\'t match table columns' ) raise Exception template_string = 'WHERE ' + self._compose_template_string( template) if Helper.is_empty(field_list): field_list_string = '*' else: field_list_string = self._compose_field_list_string(field_list) query = 'SELECT ' + field_list_string + ' FROM ' + '`' + self._data[ 'table_name'] + '`' + template_string + ';' with self._connection.cursor() as cursor: try: cursor.execute(query) result = cursor.fetchall() if len(result) == 0: return [] return result except pymysql.Error as error: self._logger.error( 'Failed to find record(s) in the table {}'.format(error)) raise Exception
def delete_by_key(self, key_fields): ''' Deletes the record that matches the key. :param template: A template. :return: A count of the rows deleted. ''' if Helper.is_empty(self._data['key_columns']): self._logger.error('Table has no primary keys') raise Exception if not Helper.are_key_fields_valid(key_fields, self._data['key_columns']): self._logger.error('Key fields are not valid') raise Exception template = Helper.convert_key_fields_to_template( key_fields, self._data['key_columns']) return self.delete_by_template(template)
def update_by_template(self, template, new_values): ''' :param template: Template for rows to match. :param new_values: New values to set for matching fields. :return: Number of rows updated. ''' if Helper.is_empty(new_values): return 0 template_string = '' if not Helper.is_empty(template): if not Helper.is_template_valid(template, self.get_columns()): self._logger.error( 'Some columns in the specified template don\'t match table columns' ) raise Exception template_string = 'WHERE ' + self._compose_template_string( template) update_string = '' for key in new_values: update_string = update_string + '`' + key + '`=\'' + new_values[ key] + '\', ' update_string = update_string[:-2] query = 'UPDATE ' + '`' + self._data[ 'table_name'] + '` SET ' + update_string + template_string + ';' with self._connection.cursor() as cursor: try: cursor.execute(query) if cursor.rowcount > 0: if self._auto_commit: self.commit() return cursor.rowcount except pymysql.Error as error: self._logger.error( 'Failed to update record(s) in the table {}'.format(error)) raise Exception
def update_by_key(self, key_fields, new_values): ''' :param key_fields: List of value for the key fields. :param new_values: A dict of field:value to set for updated row. :return: Number of rows updated. ''' if Helper.is_empty(self._data['key_columns']): self._logger.error('Table has no primary keys') raise Exception if not Helper.are_key_fields_valid(key_fields, self._data['key_columns']): self._logger.error('Key fields are not valid') raise Exception if Helper.is_empty(new_values): return 0 template = Helper.convert_key_fields_to_template( key_fields, self._data['key_columns']) return self.update_by_template(template, new_values)
def insert(self, new_record): ''' :param new_record: A dictionary representing a row to add to the set of records. :return: None ''' if not Helper.is_new_record_valid(new_record, self.get_columns()): self._logger.error('new_record must contains all columns') raise Exception if self._violate_primary_key_constraint(new_record): self._logger.error('Violates primary key constraint') raise Exception self._add_row(new_record)
def update_by_template(self, template, new_values, limit=None): ''' :param template: Template for rows to match. :param new_values: New values to set for matching fields. :return: Number of rows updated. ''' if Helper.is_empty(new_values): return 0 if not Helper.is_empty(template): if not Helper.is_template_valid(template, self.get_columns()): self._logger.error( 'Some columns in the specified template don\'t match table columns' ) raise Exception # Extract key_fields from template if any changed_keys = Helper.extract_key_columns_and_values_from_template( new_values, self._data['key_columns']) rows = self.get_rows() r_indexes = [] for i in range(0, len(rows)): if limit is not None: if len(r_indexes) == limit: break if Helper.matches_template(rows[i], template): # Apply changed_keys and check if modification would result in duplicate primary key if not Helper.is_empty(changed_keys): # Very important to make copy of rows[i] so that it will not be altered new_keys_template = Helper.change_keys( copy.copy(rows[i]), changed_keys) if self._violate_primary_key_constraint(new_keys_template): self._logger.error('Violates primary key constraint') raise Exception r_indexes.append(i) if len(r_indexes) == 0: return 0 elif len(r_indexes) == 1: self._modify_row(r_indexes[0], new_values) return 1 else: self._modify_rows(r_indexes, new_values) return len(r_indexes)
def find_by_template(self, template, field_list=None, limit=None, offset=None, order_by=None): ''' :param template: A dictionary of the form { 'field1' : value1, 'field2': value2, ...} :param field_list: A list of request fields of the form, ['fielda', 'fieldb', ...] :param limit: Do not worry about this for now. :param offset: Do not worry about this for now. :param order_by: Do not worry about this for now. :return: A list containing dictionaries. A dictionary is in the list representing each record that matches the template. The dictionary only contains the requested fields. ''' if not Helper.is_empty(template): if not Helper.is_template_valid(template, self.get_columns()): self._logger.error( 'Some columns in the specified template don\'t match table columns' ) raise Exception if not Helper.is_empty(field_list): if not Helper.is_column_list_valid(field_list, self.get_columns()): self._logger.error( 'Some columns in the specified field_list don\'t match table columns' ) raise Exception matching_rows = [] for row in self.get_rows(): if limit is not None: if len(matching_rows) == limit: break if Helper.matches_template(row, template): matching_rows.append( Helper.extract_needed_fields(field_list, row)) return matching_rows
def __init__(self, userConfig=None, launchCommand=None): """ Class constructor @param userConfig {Dict|None} - [optional]:None @param launchCommand {String|None} - [optional]:None """ super().__init__() self._defaultConfig = Helper.parseJson("src/default_config.json") if not self._defaultConfig: debug.log().critical("Default config doesn't exist") Helper.systemExit() # Config in current folder, which program was run. self._currentConfig = Helper.parseJson( os.path.join(os.getcwd(), "current.json")) self.config = Helper.MergeDict(self._defaultConfig, self._currentConfig) if userConfig: self._userConfig = Helper.parseJson(userConfig) self.config = Helper.MergeDict(self.config, self._userConfig) if launchCommand: self.config.launchCommand = launchCommand # Keys, which must be in each pattern self.validKeys = ["code", "regex", "name"] self.checkValidConfigKey()
def writeReport(self): Helper.toJsonFile(self.file, self.obj, 'w')
def test_find_by_template(appearances_csv): # Function fails if there is incorrect columns in template or field_list with pytest.raises(Exception): assert appearances_csv.find_by_template({'wrong': '150'}, ['playerID']) with pytest.raises(Exception): assert appearances_csv.find_by_template({'playerID': 'aardsda01'}, ['wrong']) # Example when some rows match the template label = [{ 'playerID': 'millake01', 'teamID': 'BOS', 'yearID': '2004' }, { 'playerID': 'staubru01', 'teamID': 'HOU', 'yearID': '1963' }, { 'playerID': 'wongko01', 'teamID': 'SLN', 'yearID': '2015' }] assert Helper.compare_two_list_of_dicts( appearances_csv.find_by_template( { 'G_all': '150', 'GS': '140', 'G_ph': '7' }, ['playerID', 'teamID', 'yearID']), label) # Example when some rows matches the template but no field_list is provided # All the fields are returned in this case, this mimics the database behavior of SELECT * label = [{ 'yearID': '2004', 'teamID': 'BOS', 'lgID': 'AL', 'playerID': 'millake01', 'G_all': '150', 'GS': '140', 'G_batting': '150', 'G_defense': '137', 'G_p': '0', 'G_c': '0', 'G_1b': '69', 'G_2b': '0', 'G_3b': '0', 'G_ss': '0', 'G_lf': '20', 'G_cf': '0', 'G_rf': '55', 'G_of': '74', 'G_dh': '8', 'G_ph': '7', 'G_pr': '0' }, { 'yearID': '1963', 'teamID': 'HOU', 'lgID': 'NL', 'playerID': 'staubru01', 'G_all': '150', 'GS': '140', 'G_batting': '150', 'G_defense': '144', 'G_p': '0', 'G_c': '0', 'G_1b': '109', 'G_2b': '0', 'G_3b': '0', 'G_ss': '0', 'G_lf': '0', 'G_cf': '0', 'G_rf': '49', 'G_of': '49', 'G_dh': '0', 'G_ph': '7', 'G_pr': '1' }, { 'yearID': '2015', 'teamID': 'SLN', 'lgID': 'NL', 'playerID': 'wongko01', 'G_all': '150', 'GS': '140', 'G_batting': '150', 'G_defense': '147', 'G_p': '0', 'G_c': '0', 'G_1b': '0', 'G_2b': '147', 'G_3b': '0', 'G_ss': '0', 'G_lf': '0', 'G_cf': '0', 'G_rf': '0', 'G_of': '0', 'G_dh': '0', 'G_ph': '7', 'G_pr': '0' }] assert Helper.compare_two_list_of_dicts( appearances_csv.find_by_template({ 'G_all': '150', 'GS': '140', 'G_ph': '7' }), label) # No rows match the template, returning an empty list assert appearances_csv.find_by_template({ 'G_all': '123', 'GS': '250' }, ['playerID', 'yearID']) == []
def collect(self, dir, conf): Helper.collect(self, dir, conf) try: self.total_authors = int( getpipeoutput(['git log', 'git shortlog -s', 'wc -l'])) except: self.total_authors = 0 # self.total_lines = int(getoutput('git-ls-files -z |xargs -0 cat |wc -l')) self.activity_by_hour_of_day = {} # hour -> commits self.activity_by_day_of_week = {} # day -> commits self.activity_by_month_of_year = {} # month [1-12] -> commits self.activity_by_hour_of_week = {} # weekday -> hour -> commits self.activity_by_hour_of_day_busiest = 0 self.activity_by_hour_of_week_busiest = 0 self.activity_by_year_week = {} # yy_wNN -> commits self.activity_by_year_week_peak = 0 self.activity_by_every_day = {} # self.authors = { } # name -> {commits, first_commit_stamp, last_commit_stamp, last_active_day, active_days, lines_added, lines_removed} # domains self.domains = {} # domain -> commits # author of the month self.author_of_month = {} # month -> author -> commits self.author_of_year = {} # year -> author -> commits self.author_of_day = {} # year -> author -> commits self.commits_by_month = {} # month -> commits self.commits_by_year = {} # year -> commits self.commits_by_day = {} # year -> commits self.first_commit_stamp = 0 self.last_commit_stamp = 0 self.last_active_day = None self.active_days = set() # lines self.total_lines = 0 self.total_lines_added = 0 self.total_lines_removed = 0 # timezone self.commits_by_timezone = {} # timezone -> commits # tags self.tags = {} lines = getpipeoutput(['git show-ref --tags']).split('\n') for line in lines: if len(line) == 0: continue (hash, tag) = line.split(' ') tag = tag.replace('refs/tags/', '') output = getpipeoutput( ['git log "%s" --pretty=format:"%%at %%an" -n 1' % hash]) if len(output) > 0: parts = output.split(' ') stamp = 0 try: stamp = int(parts[0]) except ValueError: stamp = 0 self.tags[tag] = { 'stamp': stamp, 'hash': hash, 'date': datetime.datetime.fromtimestamp(stamp).strftime( '%Y-%m-%d'), 'commits': 0, 'authors': {} } # collect info on tags, starting from latest tags_sorted_by_date_desc = list( map( lambda el: el[1], reversed( sorted( map(lambda el: (el[1]['date'], el[0]), self.tags.items()))))) prev = None for tag in reversed(tags_sorted_by_date_desc): cmd = 'git shortlog -s "%s"' % tag if prev != None: cmd += ' "^%s"' % prev output = getpipeoutput([cmd]) if len(output) == 0: continue prev = tag for line in output.split('\n'): parts = re.split('\s+', line, 2) commits = int(parts[1]) author = parts[2] self.tags[tag]['commits'] += commits self.tags[tag]['authors'][author] = commits # Collect revision statistics # Outputs "<stamp> <date> <time> <timezone> <author> '<' <mail> '>'" lines = getpipeoutput([ 'git rev-list --pretty=format:"%at %ai %an <%aE>" HEAD', 'grep -v ^commit' ]).split('\n') for line in lines: parts = line.split(' ', 4) author = '' try: stamp = int(parts[0]) except ValueError: stamp = 0 timezone = parts[3] author, mail = parts[4].split('<', 1) author = author.rstrip() mail = mail.rstrip('>') domain = '?' if mail.find('@') != -1: domain = mail.rsplit('@', 1)[1] date = datetime.datetime.fromtimestamp(float(stamp)) # First and last commit stamp if self.last_commit_stamp == 0: self.last_commit_stamp = stamp self.first_commit_stamp = stamp # activity # hour hour = date.hour self.activity_by_hour_of_day[ hour] = self.activity_by_hour_of_day.get(hour, 0) + 1 # most active hour? if self.activity_by_hour_of_day[ hour] > self.activity_by_hour_of_day_busiest: self.activity_by_hour_of_day_busiest = self.activity_by_hour_of_day[ hour] # day of week day = date.weekday() self.activity_by_day_of_week[ day] = self.activity_by_day_of_week.get(day, 0) + 1 # domain stats if domain not in self.domains: self.domains[domain] = {} # commits self.domains[domain]['commits'] = self.domains[domain].get( 'commits', 0) + 1 # hour of week if day not in self.activity_by_hour_of_week: self.activity_by_hour_of_week[day] = {} self.activity_by_hour_of_week[day][ hour] = self.activity_by_hour_of_week[day].get(hour, 0) + 1 # most active hour? if self.activity_by_hour_of_week[day][ hour] > self.activity_by_hour_of_week_busiest: self.activity_by_hour_of_week_busiest = self.activity_by_hour_of_week[ day][hour] # month of year month = date.month self.activity_by_month_of_year[ month] = self.activity_by_month_of_year.get(month, 0) + 1 # yearly/weekly activity yyw = date.strftime('%Y-%W') self.activity_by_year_week[yyw] = self.activity_by_year_week.get( yyw, 0) + 1 if self.activity_by_year_week_peak < self.activity_by_year_week[ yyw]: self.activity_by_year_week_peak = self.activity_by_year_week[ yyw] # author stats if author not in self.authors: self.authors[author] = {} # commits if 'last_commit_stamp' not in self.authors[author]: self.authors[author]['last_commit_stamp'] = stamp self.authors[author]['first_commit_stamp'] = stamp self.authors[author]['commits'] = self.authors[author].get( 'commits', 0) + 1 # author of the month/year yymm = date.strftime('%Y-%m') if yymm in self.author_of_month: self.author_of_month[yymm][ author] = self.author_of_month[yymm].get(author, 0) + 1 else: self.author_of_month[yymm] = {} self.author_of_month[yymm][author] = 1 self.commits_by_month[yymm] = self.commits_by_month.get(yymm, 0) + 1 yy = date.year if yy in self.author_of_year: self.author_of_year[yy][author] = self.author_of_year[yy].get( author, 0) + 1 else: self.author_of_year[yy] = {} self.author_of_year[yy][author] = 1 self.commits_by_year[yy] = self.commits_by_year.get(yy, 0) + 1 # authors: active days yymmdd = date.strftime('%Y-%m-%d') if 'last_active_day' not in self.authors[author]: self.authors[author]['last_active_day'] = yymmdd self.authors[author]['active_days'] = 1 elif yymmdd != self.authors[author]['last_active_day']: self.authors[author]['last_active_day'] = yymmdd self.authors[author]['active_days'] += 1 # project: active days if yymmdd != self.last_active_day: self.last_active_day = yymmdd self.active_days.add(yymmdd) # timezone self.commits_by_timezone[timezone] = self.commits_by_timezone.get( timezone, 0) + 1 # TODO Optimize this, it's the worst bottleneck # outputs "<stamp> <files>" for each revision self.files_by_stamp = {} # stamp -> files revlines = getpipeoutput( ['git rev-list --pretty=format:"%at %T" HEAD', 'grep -v ^commit']).strip().split('\n') lines = [] for revline in revlines: time, rev = revline.split(' ') linecount = self.getFilesInCommit(rev) lines.append('%d %d' % (int(time), linecount)) self.total_commits = len(lines) for line in lines: parts = line.split(' ') if len(parts) != 2: continue (stamp, files) = parts[0:2] try: self.files_by_stamp[int(stamp)] = int(files) except ValueError: Log.warning('Warning: failed to parse line "%s"' % line) # extensions self.extensions = {} # extension -> files, lines lines = getpipeoutput(['git ls-tree -r -z HEAD']).split('\000') self.total_files = len(lines) for line in lines: if len(line) == 0: continue parts = re.split('\s+', line, 4) sha1 = parts[2] filename = parts[3] if filename.find('.') == -1 or filename.rfind('.') == 0: ext = '' else: ext = filename[(filename.rfind('.') + 1):] if len(ext) > conf['max_ext_length']: ext = '' if ext not in self.extensions: self.extensions[ext] = {'files': 0, 'lines': 0} self.extensions[ext]['files'] += 1 try: self.extensions[ext]['lines'] += self.getLinesInBlob(sha1) except: Log.warning('Warning: Could not count lines for file "%s"' % line) # line statistics # outputs: # N files changed, N insertions (+), N deletions(-) # <stamp> <author> self.changes_by_date = {} # stamp -> { files, ins, del } # self.changes_by_date[stamp] = { 'files': files, 'ins': inserted, 'del': deleted } lines = self.getLinesByTime() files = 0 inserted = 0 deleted = 0 total_lines = 0 author = None for line in lines: if len(line) == 0: continue # <stamp> <author> if line.find('files changed,') == -1: pos = line.find(' ') if pos != -1: try: (stamp, author) = (int(line[:pos]), line[pos + 1:]) self.changes_by_date[stamp] = { 'files': files, 'ins': inserted, 'del': deleted, 'lines': total_lines } if author not in self.authors: self.authors[author] = { 'lines_added': 0, 'lines_removed': 0 } self.authors[author][ 'lines_added'] = self.authors[author].get( 'lines_added', 0) + inserted self.authors[author][ 'lines_removed'] = self.authors[author].get( 'lines_removed', 0) + deleted except ValueError: Log.warning('Warning: unexpected line "%s"' % line) else: Log.warning('Warning: unexpected line "%s"' % line) else: numbers = re.findall('\d+', line) if len(numbers) == 3: (files, inserted, deleted) = map(lambda el: int(el), numbers) total_lines += inserted total_lines -= deleted self.total_lines_added += inserted self.total_lines_removed += deleted else: Log.warning('Warning: failed to handle line "%s"' % line) (files, inserted, deleted) = (0, 0, 0) self.total_lines = total_lines # 统计每个人每天的任务量 # 初始化author_of_day数组 for name in self.authors.keys(): self.author_of_day[name] = {} self.author_of_day[name]['lines_added'] = 0 self.author_of_day[name]['lines_removed'] = 0 self.author_of_day[name]['commit'] = 0 today = datetime.date.today() yesterday = today - datetime.timedelta(days=1) lines = self.getLinesByTime(yesterday.strftime('%Y-%m-%d'), today.strftime('%Y-%m-%d')) for line in lines: if len(line) == 0: continue # <stamp> <author> if line.find('files changed,') == -1: pos = line.find(' ') if pos != -1: try: (stamp, author) = (int(line[:pos]), line[pos + 1:]) if author in self.authors: self.author_of_day[author][ 'commit'] = self.author_of_day[author].get( 'commit', 0) + 1 self.author_of_day[author][ 'lines_added'] = self.author_of_day[ author].get('lines_added', 0) + inserted self.author_of_day[author][ 'lines_removed'] = self.author_of_day[ author].get('lines_removed', 0) + deleted except ValueError: Log.warning('Warning: unexpected line "%s"' % line) else: Log.warning('Warning: unexpected line "%s"' % line) self.total_lines = total_lines
""" Created on Tue Jul 16 15:11:30 2019 @author: d2gu53 """ #### Neural Net in Python import numpy as np from sklearn import datasets from sklearn import linear_model import sklearn from matplotlib import pyplot as plt from src.Helper import Helper helper = Helper() np.random.seed(0) X, y = datasets.make_moons(1000, noise=0.3) clf = sklearn.linear_model.LogisticRegression() clf.fit(X, y) ### überschreibt das clf Objekt plt.scatter(X[:, 0], X[:, 1], c=y, cmap=plt.cm.Spectral) X = helper.plot_decision_boundary(lambda x: clf.predict(x), X) plt.show() ### implement back propagation num_examples = len(X) # size training set nn_input_dim = 2 # input dimensionality nn_output_dim = 2 # output dimensionality