コード例 #1
0
def process_json_files(global_config, competition, output_file, input_dir, reprocess_files=False):
    
    # Initialize the database session connection
    db_name  = global_config['db_name'] + global_config['this_season']
    session  = DbSession.open_db_session(db_name)

    # get all the verified files from the input directory. These files are
    # candidates to be processed
    verified_files = FileSync.get_file_list(input_dir, ext='.verified', recurse=True )
    verified_files.sort(key=match_sort)
    
    # For the normal case, get all the processed files, too. We'll use the processed list to 
    # determine which files are actually newly verified and need to be processed. If the 
    # reprocess flag is true, then we'll process all verified files.
    if reprocess_files is not True:
        processed_files = FileSync.get_file_list(input_dir, ext='.processed', recurse=True )
        for processed_file in processed_files:
            verified_files.remove( processed_file.replace('processed','verified') )
    
    xlsx_workbook = None
    excel_intf_ctrl = global_config.get('excel_sheets_intf', 'Disabled')
    if excel_intf_ctrl == 'Enabled':
        # read in the output file, which is expected to be an XLSX file
        try:
            xlsx_workbook = openpyxl.load_workbook(output_file)
        except:
            print 'Error Reading Spreadsheet %s For Input' % output_file
    
    google_intf_ctrl = global_config.get('google_sheets_intf', 'Disabled')

    '''
    # took out for now until we have local dictionary storage
    events = global_config.get('events')
    if events is None:
        events = {}
        global_config['events'] = events
    event_data = events.get( competition )
    if event_data is None:
        events[competition] = { 'ScoutingData': { 'TeamData': {} } }
        event_data = events[competition]

    event_scouting_data = event_data['ScoutingData']['TeamData']
    '''

    for verified_file in verified_files:
        filename = verified_file.split('/')[-1]

        # read the file into a dictionary
        with open(input_dir+verified_file) as fd:
            scouting_data = json.load(fd)
            
            if filename.startswith('Match'):
                team = scouting_data['Setup'].get('Team')
                category = 'Match'
            elif filename.startswith('Pit'):
                team = scouting_data['Pit'].get('Team')
                category = 'Pit'                
            else:
                category = 'Unknown'

            if team is not None and len(team) > 0:
                # ######################################################### #
                # store the scouting data to the local database

                DataModel.addTeamToEvent(session, int(team), competition)

                attr_definitions = AttributeDefinitions.AttrDefinitions(global_config)
                for section_name, section_data in scouting_data.iteritems():
                    if isinstance(section_data,dict):
                        for attr_name, attr_value in section_data.iteritems():
                            
                            # use the attribute definitions to control whether information gets 
                            # stored to the database rather than the hard coded stuff here.
                            # also need to consider the section/category name as the attributes
                            # and definitions are processed
                            
                            # don't store the team number in the database
                            if attr_name == 'Team':
                                continue
                            
                            # augment the attribute name with the section name in order to make the attribute
                            # unique
                            attr_name = '%s:%s' % (section_name, attr_name)
                            
                            attribute_def = {}
                            attribute_def['Name'] = attr_name
                            if attr_value.isdigit():
                                attribute_def['Type'] = 'Integer'
                                attribute_def['Weight'] = 1.0
                            else:
                                attribute_def['Type'] = 'String'
                                attribute_def['Weight'] = 0.0
                            attribute_def['Statistic_Type'] = 'Average'
                            attr_definitions.add_definition(attribute_def)

                            try:
                                DataModel.createOrUpdateAttribute(session, int(team), competition, category, 
                                                                  attr_name, attr_value, attribute_def)
                            except Exception, exception:
                                traceback.print_exc(file=sys.stdout)
                                exc_type, exc_value, exc_traceback = sys.exc_info()
                                exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
                                for line in exception_info:
                                    line = line.replace('\n','')
                                    global_config['logger'].debug(line)
                    else:
                        print 'Unexpected entry in scouting data file, name: %s, value: %s' % (section_name,section_data)
                
                score = DataModel.calculateTeamScore(session, int(team), competition, attr_definitions)
                DataModel.setTeamScore(session, int(team), competition, score)
                session.commit()

                # ######################################################### #
                # Google spreadsheet update
                if google_intf_ctrl == 'Enabled':
                    row_data = []
                    for section_name, section_data in scouting_data.iteritems():
                        if isinstance(section_data,dict):
                            for attr_name, attr_value in section_data.iteritems():
    
                                # augment the attribute name with the section name in order to make the attribute
                                # unique
                                attr_name = '%s_%s' % (section_name, attr_name)
    
                                row_data.append((attr_name,attr_value))
                                
                        else:
                            print 'Unexpected entry in scouting data file, name: %s, value: %s' % (section_name,section_data)
    
                    sheet_name = '%s_%s_Data' % (competition,category)
                    GoogleSheetsIntf.add_scouting_data_row( sheet_name, row_data )
                # ######################################################### #

                # ######################################################### #
                '''
                # ######################################################### #
                # store the scouting data to a local dictionary
                team_data = event_scouting_data.get(team)
                if team_data is None:
                    team_data = { 'Summary': {}, 'MatchData': [] }
                    event_scouting_data[team] = team_data

                team_match_data = team_data['MatchData']

                # if this match has already been scored, then update the data by removing the existing
                # match data and then add the updated data
                update_match = False
                for match_data in team_match_data:
                    if scouting_data['Setup']['Match'] == match_data['Setup']['Match']:
                        update_match = True
                        break
                if update_match is True:
                    team_match_data.remove(match_data)
                team_match_data.append(scouting_data)
                # ######################################################### #
                '''

                # ######################################################### #
                # store the match scouting data information to the spreadsheet
                if excel_intf_ctrl == 'Enabled' and xlsx_workbook is not None:
                    if category == 'Match':
                        team_name = 'Team %s' % team
                        try:
                            team_sheet = xlsx_workbook.get_sheet_by_name(team_name)
                        except:
                            team_sheet = create_team_sheet( xlsx_workbook, team_name )
        
                        curr_matches = team_sheet['B2'].value
                        if curr_matches is None:
                            curr_matches = 0
                        
                        # get max row and column count and iterate over the sheet
                        max_row= team_sheet.max_row
                        
                        for i in range(1,max_row+1):
                             # scan for a row that has Match in the first column to identify rows where data will be stored
                             cell_value = team_sheet.cell(row=i,column=1).value
                             if team_sheet.cell(row=i,column=1).value == 'Match':
                                 attr_row = i
                                 data_row = i+1
                                 data_cell = team_sheet.cell(row=i+1,column=1).value
                                 if data_cell is None:
                                     team_sheet = update_data_row( team_sheet, attr_row, data_row, scouting_data )
                                     team_sheet['B2'].value = curr_matches+1
                                     break
                                 elif data_cell == int(scouting_data['Setup']['Match']):
                                     # Update an existing row
                                     team_sheet = update_data_row( team_sheet, attr_row, data_row, scouting_data )
                                     break
                                     
                                 # Jump over the next two rows
                                 i += 2
                    # ######################################################### #
                
                shutil.copyfile(input_dir+verified_file, input_dir+verified_file.replace('verified','processed'))
                
            if xlsx_workbook is not None:             
                xlsx_workbook.save(output_file)
コード例 #2
0
def process_file(session, attr_definitions, data_filename):
    print 'processing %s'%data_filename
    
    # Initialize the file_attributes dictionary in preparation for the
    # parsing of the data file
    file_attributes = {}
    
    # Parse the data file, storing all the information in the file_attributes
    # dictionary
    FileParser.FileParser(data_filename).parse(file_attributes)

    # The team number can be retrieved from the Team attribute, one of the
    # mandatory attributes within the data file
    team = file_attributes['Team']
    
    # Also, extract the competition name, too, if it has been included in
    # the data file
    if file_attributes.has_key('Competition'):
        competition = file_attributes['Competition']
    else:
        competition = global_config['this_competition'] + global_config['this_season']

        if competition == None:
            raise Exception( 'Competition Not Specified!')

    DataModel.addTeamToEvent(session, team, competition)
    
    if file_attributes.has_key('Scouter'):
        scouter = file_attributes['Scouter']
    else:
        scouter = 'Unknown'
        
    if file_attributes.has_key('Match'):
        category = 'Match'
    else:
        if '_Pit_' in data_filename:
            category = 'Pit'
        else:
            category = 'Other'

    # Loop through the attributes from the data file and post them to the
    # database
    for attribute, value in file_attributes.iteritems():
        if value is None:
            value = ''
        try:
            attr_definition = attr_definitions.get_definition(attribute)
            if attr_definition == None:
                err_str = 'ERROR: No Attribute Defined For Attribute: %s' % attribute
                print err_str
            elif attr_definition['Database_Store']=='Yes':
                try:
                    DataModel.createOrUpdateAttribute(session, team, competition, category, attribute, value, attr_definition)
                except Exception, exception:
                    traceback.print_exc(file=sys.stdout)
                    exc_type, exc_value, exc_traceback = sys.exc_info()
                    exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
                    for line in exception_info:
                        line = line.replace('\n','')
                        global_config['logger'].debug(line)
        except Exception:
            err_str = 'ERROR: Attribute Could Not Be Processed: %s' % attribute
            traceback.print_exc(file=sys.stdout)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
            for line in exception_info:
                line = line.replace('\n','')
                global_config['logger'].debug(line)
            print err_str
            
        if category == 'Match':
            try:
                match = file_attributes['Match']
                DataModel.createOrUpdateMatchDataAttribute(session, team, competition, match, scouter, attribute, value)
            except:
                err_str = 'ERROR: Match Data Attribute Could Not Be Processed: %s' % attribute
                traceback.print_exc(file=sys.stdout)
                exc_type, exc_value, exc_traceback = sys.exc_info()
                exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
                for line in exception_info:
                    line = line.replace('\n','')
                    global_config['logger'].debug(line)
                print err_str
            
    score = DataModel.calculateTeamScore(session, team, competition, attr_definitions)
    DataModel.setTeamScore(session, team, competition, score)
コード例 #3
0
def process_file(global_config, session, attr_definitions, data_filename):
    print "processing %s" % data_filename

    # Initialize the file_attributes dictionary in preparation for the
    # parsing of the data file
    file_attributes = {}

    # Parse the data file, storing all the information in the file_attributes
    # dictionary
    FileParser.FileParser(data_filename).parse(file_attributes)

    # The team number can be retrieved from the Team attribute, one of the
    # mandatory attributes within the data file
    team = file_attributes["Team"]

    # Also, extract the competition name, too, if it has been included in
    # the data file
    if file_attributes.has_key("Competition"):
        # check if the global_config indicates that we're working with the 2014-era tablet UI that may not
        # format the competition string as we expect it to be. If we are in 'legacy' mode, then ignore the
        # competition setting in the file and apply the competition/season from the config
        if global_config.has_key("legacy_tablet_ui") and global_config["legacy_tablet_ui"].lower() == "yes":
            competition = global_config["this_competition"] + global_config["this_season"]
        else:
            competition = file_attributes["Competition"]
    else:
        # if no competition setting attribute is in the file, then apply the competition/season from the config
        competition = global_config["this_competition"] + global_config["this_season"]

        if competition == None:
            raise Exception("Competition Not Specified!")

    DataModel.addTeamToEvent(session, team, competition)

    if file_attributes.has_key("Scouter"):
        scouter = file_attributes["Scouter"]
    else:
        scouter = "Unknown"

    if "_Match" in data_filename:
        category = "Match"
        if not file_attributes.has_key("Match"):
            file_attributes["Match"] = "0"
    elif "_Pit_" in data_filename:
        category = "Pit"
    else:
        category = "Other"

    # Loop through the attributes from the data file and post them to the
    # database
    for attribute, value in file_attributes.iteritems():
        if value is None:
            value = ""
        try:
            attr_definition = attr_definitions.get_definition(attribute)
            if attr_definition == None:
                err_str = "ERROR: No Attribute Defined For Attribute: %s" % attribute
                print err_str
            elif attr_definition["Database_Store"] == "Yes":
                try:
                    DataModel.createOrUpdateAttribute(
                        session, team, competition, category, attribute, value, attr_definition
                    )
                except Exception, exception:
                    traceback.print_exc(file=sys.stdout)
                    exc_type, exc_value, exc_traceback = sys.exc_info()
                    exception_info = traceback.format_exception(exc_type, exc_value, exc_traceback)
                    for line in exception_info:
                        line = line.replace("\n", "")
                        global_config["logger"].debug(line)
        except Exception:
            err_str = "ERROR: Attribute Could Not Be Processed: %s" % attribute
            traceback.print_exc(file=sys.stdout)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exception_info = traceback.format_exception(exc_type, exc_value, exc_traceback)
            for line in exception_info:
                line = line.replace("\n", "")
                global_config["logger"].debug(line)
            print err_str

        if category == "Match":
            try:
                match = file_attributes["Match"]
                DataModel.createOrUpdateMatchDataAttribute(session, team, competition, match, scouter, attribute, value)
            except:
                err_str = "ERROR: Match Data Attribute Could Not Be Processed: %s" % attribute
                traceback.print_exc(file=sys.stdout)
                exc_type, exc_value, exc_traceback = sys.exc_info()
                exception_info = traceback.format_exception(exc_type, exc_value, exc_traceback)
                for line in exception_info:
                    line = line.replace("\n", "")
                    global_config["logger"].debug(line)
                print err_str

    score = DataModel.calculateTeamScore(session, team, competition, attr_definitions)
    DataModel.setTeamScore(session, team, competition, score)