def process_debrief_files(global_config, input_dir, recursive, test):
# Initialize the database session connections
issues_db_name = global_config['issues_db_name']
debrief_db_name = global_config['debriefs_db_name']
debrief_session = DbSession.open_db_session(debrief_db_name)
issues_session = DbSession.open_db_session(issues_db_name)
# Create the database if it doesn't already exist
#if not os.path.exists('./' + db_name):
# DebriefDataModel.create_db_tables(my_db)
# The following regular expression will select all files that conform to
# the file naming format Debrief*.txt. Build a list of all datafiles that match
# the naming format within the directory passed in via command line
# arguments.
file_regex = re.compile('Debrief[a-zA-Z0-9_-]+.txt')
files = get_files(debrief_session, debrief_db_name, input_dir, file_regex, recursive, test)
# Process data files
for data_filename in files:
print 'processing %s'%data_filename
try:
# Initialize the debrief_attributes dictionary in preparation for the
# parsing of the data file
debrief_attributes = {}
# Parse the data file, storing all the information in the attributes
# dictionary
FileParser.FileParser(data_filename).parse(debrief_attributes)
DebriefDataModel.addDebriefFromAttributes(debrief_session, debrief_attributes)
# Also, extract the competition name, too, if it has been included in
# the data file
if debrief_attributes.has_key('Competition'):
competition = debrief_attributes['Competition']
else:
competition = global_config['this_competition'] + global_config['this_season']
if competition == None:
raise Exception( 'Competition Not Specified!')
# At competition, we will likely have multiple laptops manging the data, but we want
# only one machine to be responsible for the issues database. In all likelihood,
# that machine will be the one in the pits, or possibly the application running
# in the cloud.
if global_config['issues_db_master'] == 'Yes':
match_id = debrief_attributes['Match']
submitter = debrief_attributes['Scouter']
timestamp = str(int(time.time()))
subgroup = 'Unassigned'
status = 'Open'
owner = 'Unassigned'
if debrief_attributes.has_key('Issue1_Summary'):
# look to see if there is already a debrief issue, and if so, do not attempt to create/update
# an issue, as there are already other issue files that would then conflict with this one
issue_key = 'Issue1'
if DebriefDataModel.getDebriefIssue(debrief_session, match_id, issue_key) == None:
summary = debrief_attributes['Issue1_Summary']
if debrief_attributes.has_key('Issue1_Priority'):
priority = debrief_attributes['Issue1_Priority']
else:
priority = 'Priority_3'
if debrief_attributes.has_key('Issue1_Taskgroup'):
component = debrief_attributes['Issue1_Taskgroup']
else:
component = ''
if debrief_attributes.has_key('Issue1_Description'):
description = debrief_attributes['Issue1_Description']
else:
description = ''
debrief_key = str(match_id) + '_' + issue_key
issue_id = IssueTrackerDataModel.getIssueId(issues_session, 'Robot')
issue = IssueTrackerDataModel.addOrUpdateIssue(issues_session, issue_id, summary, status, priority,
subgroup, component, submitter, owner, description, timestamp, debrief_key)
if issue != None:
issue.create_file('./static/data/%s/ScoutingData' % competition)
DebriefDataModel.addOrUpdateDebriefIssue(debrief_session, match_id, competition, issue_id, issue_key)
if debrief_attributes.has_key('Issue2_Summary'):
issue_key = 'Issue2'
if DebriefDataModel.getDebriefIssue(debrief_session, match_id, issue_key) == None:
summary = debrief_attributes['Issue2_Summary']
if debrief_attributes.has_key('Issue2_Priority'):
priority = debrief_attributes['Issue2_Priority']
else:
priority = 'Priority_3'
if debrief_attributes.has_key('Issue2_Taskgroup'):
component = debrief_attributes['Issue2_Taskgroup']
else:
component = ''
if debrief_attributes.has_key('Issue3_Description'):
description = debrief_attributes['Issue3_Description']
else:
description = ''
debrief_key = str(match_id) + '_' + issue_key
issue_id = IssueTrackerDataModel.getIssueId(issues_session, 'Robot')
issue = IssueTrackerDataModel.addOrUpdateIssue(issues_session, issue_id, summary, status, priority,
subgroup, component, submitter, owner, description, timestamp, debrief_key)
if issue != None:
issue.create_file('./static/data/%s/ScoutingData' % competition)
DebriefDataModel.addOrUpdateDebriefIssue(debrief_session, match_id, competition, issue_id, issue_key)
if debrief_attributes.has_key('Issue3_Summary'):
issue_key = 'Issue3'
if DebriefDataModel.getDebriefIssue(debrief_session, match_id, issue_key) == None:
summary = debrief_attributes['Issue3_Summary']
if debrief_attributes.has_key('Issue3_Priority'):
priority = debrief_attributes['Issue3_Priority']
else:
priority = 'Priority_3'
if debrief_attributes.has_key('Issue3_Taskgroup'):
component = debrief_attributes['Issue3_Taskgroup']
else:
component = ''
if debrief_attributes.has_key('Issue3_Description'):
description = debrief_attributes['Issue3_Description']
else:
description = ''
debrief_key = str(match_id) + '_' + issue_key
issue_id = IssueTrackerDataModel.getIssueId(issues_session, 'Robot')
issue = IssueTrackerDataModel.addOrUpdateIssue(issues_session, issue_id, summary, status, priority,
subgroup, component, submitter, owner, description, timestamp, debrief_key)
if issue != None:
issue.create_file('./static/data/%s/ScoutingData' % competition)
DebriefDataModel.addOrUpdateDebriefIssue(debrief_session, match_id, competition, issue_id, issue_key)
except Exception, e:
# log the exception but continue processing other files
log_exception(e)
# add the file to the set of processed files so that we don't process it again. Do it outside the
# try/except block so that we don't try to process a bogus file over and over again.
DebriefDataModel.addProcessedFile(debrief_session, data_filename)
def process_debrief_files(global_config, input_dir, recursive=True):
# Initialize the database session connections
issues_db_name = global_config["issues_db_name"] + global_config["this_season"]
debrief_db_name = global_config["debriefs_db_name"] + global_config["this_season"]
debrief_session = DbSession.open_db_session(debrief_db_name)
issues_session = DbSession.open_db_session(issues_db_name)
# Create the database if it doesn't already exist
# if not os.path.exists('./' + db_name):
# DebriefDataModel.create_db_tables(my_db)
# The following regular expression will select all files that conform to
# the file naming format Debrief*.txt. Build a list of all datafiles that match
# the naming format within the directory passed in via command line
# arguments.
file_regex = re.compile("Debrief[a-zA-Z0-9_-]+.txt")
files = get_files(global_config, debrief_session, debrief_db_name, input_dir, file_regex, recursive)
# Process data files
for data_filename in files:
print "processing %s" % data_filename
try:
# Initialize the debrief_attributes dictionary in preparation for the
# parsing of the data file
debrief_attributes = {}
# Parse the data file, storing all the information in the attributes
# dictionary
FileParser.FileParser(data_filename).parse(debrief_attributes)
DebriefDataModel.addDebriefFromAttributes(debrief_session, debrief_attributes)
# Also, extract the competition name, too, if it has been included in
# the data file
if debrief_attributes.has_key("Competition"):
competition = debrief_attributes["Competition"]
issue_base_name = WebCommonUtils.split_comp_str(competition)[0]
else:
competition = global_config["this_competition"] + global_config["this_season"]
issue_base_name = global_config["this_competition"]
if competition == None:
raise Exception("Competition Not Specified!")
# At competition, we will likely have multiple laptops manging the data, but we want
# only one machine to be responsible for the issues database. In all likelihood,
# that machine will be the one in the pits, or possibly the application running
# in the cloud.
if global_config["issues_db_master"] == "Yes":
match_id = debrief_attributes["Match"]
submitter = debrief_attributes["Scouter"]
timestamp = str(int(time.time()))
subgroup = "Unassigned"
status = "Open"
owner = "Unassigned"
if debrief_attributes.has_key("Issue1_Summary") or debrief_attributes.has_key("Issue1_Description"):
# look to see if there is already a debrief issue, and if so, do not attempt to create/update
# an issue, as there are already other issue files that would then conflict with this one
issue_key = "Issue1"
if DebriefDataModel.getDebriefIssue(debrief_session, competition, match_id, issue_key) == None:
# if no summary is provided, then use the description as the summary. Likewise, if no description
# is provided then use the summary as the description. Keep in mind that we need at least the
# summary or description to be provided.
if debrief_attributes.has_key("Issue1_Summary"):
summary = debrief_attributes["Issue1_Summary"]
else:
summary = debrief_attributes["Issue1_Description"]
if debrief_attributes.has_key("Issue1_Description"):
description = debrief_attributes["Issue1_Description"]
else:
description = debrief_attributes["Issue1_Summary"]
if debrief_attributes.has_key("Issue1_Priority"):
priority = debrief_attributes["Issue1_Priority"]
else:
priority = "Priority_3"
if debrief_attributes.has_key("Issue1_Taskgroup"):
component = debrief_attributes["Issue1_Taskgroup"]
else:
component = ""
debrief_key = str(match_id) + "_" + issue_key
issue_id = IssueTrackerDataModel.getIssueId(issues_session, issue_base_name)
issue = IssueTrackerDataModel.addOrUpdateIssue(
issues_session,
issue_id,
summary,
status,
priority,
subgroup,
component,
submitter,
owner,
description,
timestamp,
debrief_key,
)
if issue != None:
issue.create_file("./static/data/%s/ScoutingData" % competition)
DebriefDataModel.addOrUpdateDebriefIssue(
debrief_session, match_id, competition, issue_id, issue_key
)
if debrief_attributes.has_key("Issue2_Summary") or debrief_attributes.has_key("Issue2_Description"):
# look to see if there is already a debrief issue, and if so, do not attempt to create/update
# an issue, as there are already other issue files that would then conflict with this one
issue_key = "Issue2"
if DebriefDataModel.getDebriefIssue(debrief_session, competition, match_id, issue_key) == None:
# if no summary is provided, then use the description as the summary. Likewise, if no description
# is provided then use the summary as the description. Keep in mind that we need at least the
# summary or description to be provided.
if debrief_attributes.has_key("Issue2_Summary"):
summary = debrief_attributes["Issue2_Summary"]
else:
summary = debrief_attributes["Issue2_Description"]
if debrief_attributes.has_key("Issue2_Description"):
description = debrief_attributes["Issue2_Description"]
else:
description = debrief_attributes["Issue2_Summary"]
if debrief_attributes.has_key("Issue2_Priority"):
priority = debrief_attributes["Issue2_Priority"]
else:
priority = "Priority_3"
if debrief_attributes.has_key("Issue2_Taskgroup"):
component = debrief_attributes["Issue2_Taskgroup"]
else:
component = ""
debrief_key = str(match_id) + "_" + issue_key
issue_id = IssueTrackerDataModel.getIssueId(issues_session, issue_base_name)
issue = IssueTrackerDataModel.addOrUpdateIssue(
issues_session,
issue_id,
summary,
status,
priority,
subgroup,
component,
submitter,
owner,
description,
timestamp,
debrief_key,
)
if issue != None:
issue.create_file("./static/data/%s/ScoutingData" % competition)
DebriefDataModel.addOrUpdateDebriefIssue(
debrief_session, match_id, competition, issue_id, issue_key
)
if debrief_attributes.has_key("Issue3_Summary") or debrief_attributes.has_key("Issue3_Description"):
# look to see if there is already a debrief issue, and if so, do not attempt to create/update
# an issue, as there are already other issue files that would then conflict with this one
issue_key = "Issue3"
if DebriefDataModel.getDebriefIssue(debrief_session, competition, match_id, issue_key) == None:
# if no summary is provided, then use the description as the summary. Likewise, if no description
# is provided then use the summary as the description. Keep in mind that we need at least the
# summary or description to be provided.
if debrief_attributes.has_key("Issue3_Summary"):
summary = debrief_attributes["Issue3_Summary"]
else:
summary = debrief_attributes["Issue3_Description"]
if debrief_attributes.has_key("Issue3_Description"):
description = debrief_attributes["Issue3_Description"]
else:
description = debrief_attributes["Issue3_Summary"]
if debrief_attributes.has_key("Issue3_Priority"):
priority = debrief_attributes["Issue3_Priority"]
else:
priority = "Priority_3"
if debrief_attributes.has_key("Issue3_Taskgroup"):
component = debrief_attributes["Issue3_Taskgroup"]
else:
component = ""
debrief_key = str(match_id) + "_" + issue_key
issue_id = IssueTrackerDataModel.getIssueId(issues_session, issue_base_name)
issue = IssueTrackerDataModel.addOrUpdateIssue(
issues_session,
issue_id,
summary,
status,
priority,
subgroup,
component,
submitter,
owner,
description,
timestamp,
debrief_key,
)
if issue != None:
issue.create_file("./static/data/%s/ScoutingData" % competition)
DebriefDataModel.addOrUpdateDebriefIssue(
debrief_session, match_id, competition, issue_id, issue_key
)
except Exception, e:
# log the exception but continue processing other files
log_exception(global_config["logger"], e)
# add the file to the set of processed files so that we don't process it again. Do it outside the
# try/except block so that we don't try to process a bogus file over and over again.
DebriefDataModel.addProcessedFile(debrief_session, data_filename)