def process_json_files(global_config, competition, output_file, input_dir, reprocess_files=False):
# Initialize the database session connection
db_name = global_config['db_name'] + global_config['this_season']
session = DbSession.open_db_session(db_name)
# get all the verified files from the input directory. These files are
# candidates to be processed
verified_files = FileSync.get_file_list(input_dir, ext='.verified', recurse=True )
verified_files.sort(key=match_sort)
# For the normal case, get all the processed files, too. We'll use the processed list to
# determine which files are actually newly verified and need to be processed. If the
# reprocess flag is true, then we'll process all verified files.
if reprocess_files is not True:
processed_files = FileSync.get_file_list(input_dir, ext='.processed', recurse=True )
for processed_file in processed_files:
verified_files.remove( processed_file.replace('processed','verified') )
xlsx_workbook = None
excel_intf_ctrl = global_config.get('excel_sheets_intf', 'Disabled')
if excel_intf_ctrl == 'Enabled':
# read in the output file, which is expected to be an XLSX file
try:
xlsx_workbook = openpyxl.load_workbook(output_file)
except:
print 'Error Reading Spreadsheet %s For Input' % output_file
google_intf_ctrl = global_config.get('google_sheets_intf', 'Disabled')
'''
# took out for now until we have local dictionary storage
events = global_config.get('events')
if events is None:
events = {}
global_config['events'] = events
event_data = events.get( competition )
if event_data is None:
events[competition] = { 'ScoutingData': { 'TeamData': {} } }
event_data = events[competition]
event_scouting_data = event_data['ScoutingData']['TeamData']
'''
for verified_file in verified_files:
filename = verified_file.split('/')[-1]
# read the file into a dictionary
with open(input_dir+verified_file) as fd:
scouting_data = json.load(fd)
if filename.startswith('Match'):
team = scouting_data['Setup'].get('Team')
category = 'Match'
elif filename.startswith('Pit'):
team = scouting_data['Pit'].get('Team')
category = 'Pit'
else:
category = 'Unknown'
if team is not None and len(team) > 0:
# ######################################################### #
# store the scouting data to the local database
DataModel.addTeamToEvent(session, int(team), competition)
attr_definitions = AttributeDefinitions.AttrDefinitions(global_config)
for section_name, section_data in scouting_data.iteritems():
if isinstance(section_data,dict):
for attr_name, attr_value in section_data.iteritems():
# use the attribute definitions to control whether information gets
# stored to the database rather than the hard coded stuff here.
# also need to consider the section/category name as the attributes
# and definitions are processed
# don't store the team number in the database
if attr_name == 'Team':
continue
# augment the attribute name with the section name in order to make the attribute
# unique
attr_name = '%s:%s' % (section_name, attr_name)
attribute_def = {}
attribute_def['Name'] = attr_name
if attr_value.isdigit():
attribute_def['Type'] = 'Integer'
attribute_def['Weight'] = 1.0
else:
attribute_def['Type'] = 'String'
attribute_def['Weight'] = 0.0
attribute_def['Statistic_Type'] = 'Average'
attr_definitions.add_definition(attribute_def)
try:
DataModel.createOrUpdateAttribute(session, int(team), competition, category,
attr_name, attr_value, attribute_def)
except Exception, exception:
traceback.print_exc(file=sys.stdout)
exc_type, exc_value, exc_traceback = sys.exc_info()
exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
for line in exception_info:
line = line.replace('\n','')
global_config['logger'].debug(line)
else:
print 'Unexpected entry in scouting data file, name: %s, value: %s' % (section_name,section_data)
score = DataModel.calculateTeamScore(session, int(team), competition, attr_definitions)
DataModel.setTeamScore(session, int(team), competition, score)
session.commit()
# ######################################################### #
# Google spreadsheet update
if google_intf_ctrl == 'Enabled':
row_data = []
for section_name, section_data in scouting_data.iteritems():
if isinstance(section_data,dict):
for attr_name, attr_value in section_data.iteritems():
# augment the attribute name with the section name in order to make the attribute
# unique
attr_name = '%s_%s' % (section_name, attr_name)
row_data.append((attr_name,attr_value))
else:
print 'Unexpected entry in scouting data file, name: %s, value: %s' % (section_name,section_data)
sheet_name = '%s_%s_Data' % (competition,category)
GoogleSheetsIntf.add_scouting_data_row( sheet_name, row_data )
# ######################################################### #
# ######################################################### #
'''
# ######################################################### #
# store the scouting data to a local dictionary
team_data = event_scouting_data.get(team)
if team_data is None:
team_data = { 'Summary': {}, 'MatchData': [] }
event_scouting_data[team] = team_data
team_match_data = team_data['MatchData']
# if this match has already been scored, then update the data by removing the existing
# match data and then add the updated data
update_match = False
for match_data in team_match_data:
if scouting_data['Setup']['Match'] == match_data['Setup']['Match']:
update_match = True
break
if update_match is True:
team_match_data.remove(match_data)
team_match_data.append(scouting_data)
# ######################################################### #
'''
# ######################################################### #
# store the match scouting data information to the spreadsheet
if excel_intf_ctrl == 'Enabled' and xlsx_workbook is not None:
if category == 'Match':
team_name = 'Team %s' % team
try:
team_sheet = xlsx_workbook.get_sheet_by_name(team_name)
except:
team_sheet = create_team_sheet( xlsx_workbook, team_name )
curr_matches = team_sheet['B2'].value
if curr_matches is None:
curr_matches = 0
# get max row and column count and iterate over the sheet
max_row= team_sheet.max_row
for i in range(1,max_row+1):
# scan for a row that has Match in the first column to identify rows where data will be stored
cell_value = team_sheet.cell(row=i,column=1).value
if team_sheet.cell(row=i,column=1).value == 'Match':
attr_row = i
data_row = i+1
data_cell = team_sheet.cell(row=i+1,column=1).value
if data_cell is None:
team_sheet = update_data_row( team_sheet, attr_row, data_row, scouting_data )
team_sheet['B2'].value = curr_matches+1
break
elif data_cell == int(scouting_data['Setup']['Match']):
# Update an existing row
team_sheet = update_data_row( team_sheet, attr_row, data_row, scouting_data )
break
# Jump over the next two rows
i += 2
# ######################################################### #
shutil.copyfile(input_dir+verified_file, input_dir+verified_file.replace('verified','processed'))
if xlsx_workbook is not None:
xlsx_workbook.save(output_file)
def process_file(session, attr_definitions, data_filename):
print 'processing %s'%data_filename
# Initialize the file_attributes dictionary in preparation for the
# parsing of the data file
file_attributes = {}
# Parse the data file, storing all the information in the file_attributes
# dictionary
FileParser.FileParser(data_filename).parse(file_attributes)
# The team number can be retrieved from the Team attribute, one of the
# mandatory attributes within the data file
team = file_attributes['Team']
# Also, extract the competition name, too, if it has been included in
# the data file
if file_attributes.has_key('Competition'):
competition = file_attributes['Competition']
else:
competition = global_config['this_competition'] + global_config['this_season']
if competition == None:
raise Exception( 'Competition Not Specified!')
DataModel.addTeamToEvent(session, team, competition)
if file_attributes.has_key('Scouter'):
scouter = file_attributes['Scouter']
else:
scouter = 'Unknown'
if file_attributes.has_key('Match'):
category = 'Match'
else:
if '_Pit_' in data_filename:
category = 'Pit'
else:
category = 'Other'
# Loop through the attributes from the data file and post them to the
# database
for attribute, value in file_attributes.iteritems():
if value is None:
value = ''
try:
attr_definition = attr_definitions.get_definition(attribute)
if attr_definition == None:
err_str = 'ERROR: No Attribute Defined For Attribute: %s' % attribute
print err_str
elif attr_definition['Database_Store']=='Yes':
try:
DataModel.createOrUpdateAttribute(session, team, competition, category, attribute, value, attr_definition)
except Exception, exception:
traceback.print_exc(file=sys.stdout)
exc_type, exc_value, exc_traceback = sys.exc_info()
exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
for line in exception_info:
line = line.replace('\n','')
global_config['logger'].debug(line)
except Exception:
err_str = 'ERROR: Attribute Could Not Be Processed: %s' % attribute
traceback.print_exc(file=sys.stdout)
exc_type, exc_value, exc_traceback = sys.exc_info()
exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
for line in exception_info:
line = line.replace('\n','')
global_config['logger'].debug(line)
print err_str
if category == 'Match':
try:
match = file_attributes['Match']
DataModel.createOrUpdateMatchDataAttribute(session, team, competition, match, scouter, attribute, value)
except:
err_str = 'ERROR: Match Data Attribute Could Not Be Processed: %s' % attribute
traceback.print_exc(file=sys.stdout)
exc_type, exc_value, exc_traceback = sys.exc_info()
exception_info = traceback.format_exception(exc_type, exc_value,exc_traceback)
for line in exception_info:
line = line.replace('\n','')
global_config['logger'].debug(line)
print err_str
score = DataModel.calculateTeamScore(session, team, competition, attr_definitions)
DataModel.setTeamScore(session, team, competition, score)
def process_file(global_config, session, attr_definitions, data_filename):
print "processing %s" % data_filename
# Initialize the file_attributes dictionary in preparation for the
# parsing of the data file
file_attributes = {}
# Parse the data file, storing all the information in the file_attributes
# dictionary
FileParser.FileParser(data_filename).parse(file_attributes)
# The team number can be retrieved from the Team attribute, one of the
# mandatory attributes within the data file
team = file_attributes["Team"]
# Also, extract the competition name, too, if it has been included in
# the data file
if file_attributes.has_key("Competition"):
# check if the global_config indicates that we're working with the 2014-era tablet UI that may not
# format the competition string as we expect it to be. If we are in 'legacy' mode, then ignore the
# competition setting in the file and apply the competition/season from the config
if global_config.has_key("legacy_tablet_ui") and global_config["legacy_tablet_ui"].lower() == "yes":
competition = global_config["this_competition"] + global_config["this_season"]
else:
competition = file_attributes["Competition"]
else:
# if no competition setting attribute is in the file, then apply the competition/season from the config
competition = global_config["this_competition"] + global_config["this_season"]
if competition == None:
raise Exception("Competition Not Specified!")
DataModel.addTeamToEvent(session, team, competition)
if file_attributes.has_key("Scouter"):
scouter = file_attributes["Scouter"]
else:
scouter = "Unknown"
if "_Match" in data_filename:
category = "Match"
if not file_attributes.has_key("Match"):
file_attributes["Match"] = "0"
elif "_Pit_" in data_filename:
category = "Pit"
else:
category = "Other"
# Loop through the attributes from the data file and post them to the
# database
for attribute, value in file_attributes.iteritems():
if value is None:
value = ""
try:
attr_definition = attr_definitions.get_definition(attribute)
if attr_definition == None:
err_str = "ERROR: No Attribute Defined For Attribute: %s" % attribute
print err_str
elif attr_definition["Database_Store"] == "Yes":
try:
DataModel.createOrUpdateAttribute(
session, team, competition, category, attribute, value, attr_definition
)
except Exception, exception:
traceback.print_exc(file=sys.stdout)
exc_type, exc_value, exc_traceback = sys.exc_info()
exception_info = traceback.format_exception(exc_type, exc_value, exc_traceback)
for line in exception_info:
line = line.replace("\n", "")
global_config["logger"].debug(line)
except Exception:
err_str = "ERROR: Attribute Could Not Be Processed: %s" % attribute
traceback.print_exc(file=sys.stdout)
exc_type, exc_value, exc_traceback = sys.exc_info()
exception_info = traceback.format_exception(exc_type, exc_value, exc_traceback)
for line in exception_info:
line = line.replace("\n", "")
global_config["logger"].debug(line)
print err_str
if category == "Match":
try:
match = file_attributes["Match"]
DataModel.createOrUpdateMatchDataAttribute(session, team, competition, match, scouter, attribute, value)
except:
err_str = "ERROR: Match Data Attribute Could Not Be Processed: %s" % attribute
traceback.print_exc(file=sys.stdout)
exc_type, exc_value, exc_traceback = sys.exc_info()
exception_info = traceback.format_exception(exc_type, exc_value, exc_traceback)
for line in exception_info:
line = line.replace("\n", "")
global_config["logger"].debug(line)
print err_str
score = DataModel.calculateTeamScore(session, team, competition, attr_definitions)
DataModel.setTeamScore(session, team, competition, score)