def add_awards_from_csv(cls, awards_csv):
events = {} # for reducing datastore fetches of events and teams
awards = []
for award in CSVAwardsParser.parse(awards_csv):
event_key_name = '{}{}'.format(award['year'], award['event_short'])
if event_key_name in events:
event = events[event_key_name]
else:
event = Event.get_by_id(event_key_name)
if event is None:
logging.warning("Event: {} doesn't exist!".format(event_key_name))
continue
events[event_key_name] = event
awards.append(Award(
id=Award.render_key_name(event.key_name, award['award_type_enum']),
name_str=award['name_str'],
award_type_enum=award['award_type_enum'],
year=event.year,
event=event.key,
event_type_enum=event.event_type_enum,
team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']],
recipient_json_list=award['recipient_json_list']
))
new_awards = AwardManipulator.createOrUpdate(awards)
if type(new_awards) != list:
new_awards = [new_awards]
return new_awards
def test_parse_regional_awards(self):
with open("test_data/pre_2002_regional_awards.csv", "r") as f:
awards = CSVAwardsParser.parse(f.read())
# Check number of parsed awards
num_awards = 0
for award in awards:
num_awards += len(award["recipient_json_list"])
self.assertEqual(num_awards, 885)
self.assertEqual(len(awards), 601)
awards = convert_to_comparable(awards)
award = {
"year": 1995,
"event_short": "nh",
"name_str": "Team Spirit Award",
"award_type_enum": AwardType.SPIRIT,
"team_number_list": [213],
"recipient_json_list": [{"team_number": 213, "awardee": None}],
}
self.assertTrue(award in awards)
award = {
"year": 2000,
"event_short": "mi",
"name_str": "Best Offensive Round",
"award_type_enum": AwardType.BEST_OFFENSIVE_ROUND,
"team_number_list": [1, 240],
"recipient_json_list": [{"team_number": 1, "awardee": None}, {"team_number": 240, "awardee": None}],
}
self.assertTrue(award in awards)
def add_awards_from_csv(cls, awards_csv):
events = {} # for reducing datastore fetches of events and teams
awards = []
for award in CSVAwardsParser.parse(awards_csv):
event_key_name = '{}{}'.format(award['year'], award['event_short'])
if event_key_name in events:
event = events[event_key_name]
else:
event = Event.get_by_id(event_key_name)
if event is None:
logging.warning(
"Event: {} doesn't exist!".format(event_key_name))
continue
events[event_key_name] = event
awards.append(
Award(id=Award.render_key_name(event.key_name,
award['award_type_enum']),
name_str=award['name_str'],
award_type_enum=award['award_type_enum'],
year=event.year,
event=event.key,
event_type_enum=event.event_type_enum,
team_list=[
ndb.Key(Team, 'frc{}'.format(team_number))
for team_number in award['team_number_list']
],
recipient_json_list=award['recipient_json_list']))
new_awards = AwardManipulator.createOrUpdate(awards)
if type(new_awards) != list:
new_awards = [new_awards]
return new_awards
def test_parse_cmp_awards(self):
with open("test_data/pre_2007_cmp_awards.csv", "r") as f:
awards = CSVAwardsParser.parse(f.read())
# Check number of parsed awards
num_awards = 0
for award in awards:
num_awards += len(award["recipient_json_list"])
self.assertEqual(num_awards, 560)
self.assertEqual(len(awards), 330)
awards = convert_to_comparable(awards)
award = {
"year": 2003,
"event_short": "cmp",
"name_str": "Championship Finalist",
"award_type_enum": AwardType.FINALIST,
"team_number_list": [25, 343, 494],
"recipient_json_list": [
{"team_number": 25, "awardee": None},
{"team_number": 343, "awardee": None},
{"team_number": 494, "awardee": None},
],
}
self.assertTrue(award in awards)
award = {
"year": 2002,
"event_short": "arc",
"name_str": "#1 Seed",
"award_type_enum": AwardType.NUM_1_SEED,
"team_number_list": [121],
"recipient_json_list": [{"team_number": 121, "awardee": None}],
}
self.assertTrue(award in awards)
award = {
"year": 1992,
"event_short": "cmp",
"name_str": "Play of the Day",
"award_type_enum": AwardType.PLAY_OF_THE_DAY,
"team_number_list": [131],
"recipient_json_list": [{"team_number": 131, "awardee": None}],
}
self.assertTrue(award in awards)
def test_parse_regional_awards(self):
with open('test_data/pre_2002_regional_awards.csv', 'r') as f:
awards = CSVAwardsParser.parse(f.read())
# Check number of parsed awards
num_awards = 0
for award in awards:
num_awards += len(award['recipient_json_list'])
self.assertEqual(num_awards, 885)
self.assertEqual(len(awards), 601)
awards = convert_to_comparable(awards)
award = {
'year': 1995,
'event_short': 'nh',
'name_str': "Team Spirit Award",
'award_type_enum': AwardType.SPIRIT,
'team_number_list': [213],
'recipient_json_list': [{
'team_number': 213,
'awardee': None
}],
}
self.assertTrue(award in awards)
award = {
'year':
2000,
'event_short':
'mi',
'name_str':
"Best Offensive Round",
'award_type_enum':
AwardType.BEST_OFFENSIVE_ROUND,
'team_number_list': [1, 240],
'recipient_json_list': [{
'team_number': 1,
'awardee': None
}, {
'team_number': 240,
'awardee': None
}],
}
self.assertTrue(award in awards)
def post(self):
self._require_admin()
awards_csv = self.request.get('awards_csv')
events = {} # for reducing datastore fetches of events and teams
awards = []
for award in CSVAwardsParser.parse(awards_csv):
event_key_name = '{}{}'.format(award['year'], award['event_short'])
if event_key_name in events:
event = events[event_key_name]
else:
event = Event.get_by_id(event_key_name)
if event is None:
logging.warning(
"Event: {} doesn't exist!".format(event_key_name))
continue
events[event_key_name] = event
awards.append(
Award(id=Award.render_key_name(event.key_name,
award['award_type_enum']),
name_str=award['name_str'],
award_type_enum=award['award_type_enum'],
year=event.year,
event=event.key,
event_type_enum=event.event_type_enum,
team_list=[
ndb.Key(Team, 'frc{}'.format(team_number))
for team_number in award['team_number_list']
],
recipient_json_list=award['recipient_json_list']))
new_awards = AwardManipulator.createOrUpdate(awards)
if type(new_awards) != list:
new_awards = [new_awards]
self.template_values = {
'awards': new_awards,
}
path = os.path.join(os.path.dirname(__file__),
'../../templates/admin/awards_add.html')
self.response.out.write(template.render(path, self.template_values))
def post(self):
self._require_admin()
awards_csv = self.request.get('awards_csv')
events = {} # for reducing datastore fetches of events and teams
awards = []
for award in CSVAwardsParser.parse(awards_csv):
event_key_name = '{}{}'.format(award['year'], award['event_short'])
if event_key_name in events:
event = events[event_key_name]
else:
event = Event.get_by_id(event_key_name)
if event is None:
logging.warning("Event: {} doesn't exist!".format(event_key_name))
continue
events[event_key_name] = event
awards.append(Award(
id=Award.render_key_name(event.key_name, award['award_type_enum']),
name_str=award['name_str'],
award_type_enum=award['award_type_enum'],
year=event.year,
event=event.key,
event_type_enum=event.event_type_enum,
team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']],
recipient_json_list=award['recipient_json_list']
))
new_awards = AwardManipulator.createOrUpdate(awards)
if type(new_awards) != list:
new_awards = [new_awards]
self.template_values = {
'awards': new_awards,
}
path = os.path.join(os.path.dirname(__file__), '../../templates/admin/awards_add.html')
self.response.out.write(template.render(path, self.template_values))
def get(self, event_key):
if tba_config.CONFIG["env"] == "prod": # disable in prod for now
logging.error("Tried to restore {} from CSV in prod! No can do.".format(event_key))
return
event = Event.get_by_id(event_key)
# alliances
result = urlfetch.fetch(self.ALLIANCES_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.ALLIANCES_URL.format(event.year, event_key, event_key)))
else:
data = result.content.replace('frc', '')
alliance_selections = CSVAllianceSelectionsParser.parse(data)
event_details = EventDetails(
id=event_key,
alliance_selections=alliance_selections
)
EventDetailsManipulator.createOrUpdate(event_details)
# awards
result = urlfetch.fetch(self.AWARDS_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.AWARDS_URL.format(event.year, event_key, event_key)))
else:
# convert into expected input format
data = StringIO.StringIO()
writer = csv.writer(data, delimiter=',')
for row in csv.reader(StringIO.StringIO(result.content), delimiter=','):
writer.writerow([event.year, event.event_short, row[1], row[2].replace('frc', ''), row[3]])
awards = []
for award in CSVAwardsParser.parse(data.getvalue()):
awards.append(Award(
id=Award.render_key_name(event.key_name, award['award_type_enum']),
name_str=award['name_str'],
award_type_enum=award['award_type_enum'],
year=event.year,
event=event.key,
event_type_enum=event.event_type_enum,
team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']],
recipient_json_list=award['recipient_json_list']
))
AwardManipulator.createOrUpdate(awards)
# matches
result = urlfetch.fetch(self.MATCHES_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.MATCHES_URL.format(event.year, event_key, event_key)))
else:
data = result.content.replace('frc', '').replace('{}_'.format(event_key), '')
match_dicts, _ = OffseasonMatchesParser.parse(data)
matches = [
Match(
id=Match.renderKeyName(
event.key.id(),
match.get("comp_level", None),
match.get("set_number", 0),
match.get("match_number", 0)),
event=event.key,
year=event.year,
set_number=match.get("set_number", 0),
match_number=match.get("match_number", 0),
comp_level=match.get("comp_level", None),
team_key_names=match.get("team_key_names", None),
alliances_json=match.get("alliances_json", None)
)
for match in match_dicts]
MatchManipulator.createOrUpdate(matches)
# rankings
result = urlfetch.fetch(self.RANKINGS_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.RANKINGS_URL.format(event.year, event_key, event_key)))
else:
# convert into expected input format
rankings = list(csv.reader(StringIO.StringIO(result.content), delimiter=','))
event_details = EventDetails(
id=event_key,
rankings=rankings
)
EventDetailsManipulator.createOrUpdate(event_details)
self.response.out.write("Done restoring {}!".format(event_key))
def get(self, event_key):
if tba_config.CONFIG["env"] == "prod": # disable in prod for now
logging.error("Tried to restore {} from CSV in prod! No can do.".format(event_key))
return
event = Event.get_by_id(event_key)
# alliances
result = urlfetch.fetch(self.ALLIANCES_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.ALLIANCES_URL.format(event.year, event_key, event_key)))
else:
data = result.content.replace('frc', '')
alliance_selections = CSVAllianceSelectionsParser.parse(data)
event_details = EventDetails(
id=event_key,
alliance_selections=alliance_selections
)
EventDetailsManipulator.createOrUpdate(event_details)
# awards
result = urlfetch.fetch(self.AWARDS_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.AWARDS_URL.format(event.year, event_key, event_key)))
else:
# convert into expected input format
data = StringIO.StringIO()
writer = csv.writer(data, delimiter=',')
for row in csv.reader(StringIO.StringIO(result.content), delimiter=','):
writer.writerow([event.year, event.event_short, row[1], row[2].replace('frc', ''), row[3]])
awards = []
for award in CSVAwardsParser.parse(data.getvalue()):
awards.append(Award(
id=Award.render_key_name(event.key_name, award['award_type_enum']),
name_str=award['name_str'],
award_type_enum=award['award_type_enum'],
year=event.year,
event=event.key,
event_type_enum=event.event_type_enum,
team_list=[ndb.Key(Team, 'frc{}'.format(team_number)) for team_number in award['team_number_list']],
recipient_json_list=award['recipient_json_list']
))
AwardManipulator.createOrUpdate(awards)
# matches
result = urlfetch.fetch(self.MATCHES_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.MATCHES_URL.format(event.year, event_key, event_key)))
else:
data = result.content.replace('frc', '').replace('{}_'.format(event_key), '')
match_dicts, _ = OffseasonMatchesParser.parse(data)
matches = [
Match(
id=Match.renderKeyName(
event.key.id(),
match.get("comp_level", None),
match.get("set_number", 0),
match.get("match_number", 0)),
event=event.key,
year=event.year,
set_number=match.get("set_number", 0),
match_number=match.get("match_number", 0),
comp_level=match.get("comp_level", None),
team_key_names=match.get("team_key_names", None),
alliances_json=match.get("alliances_json", None)
)
for match in match_dicts]
MatchManipulator.createOrUpdate(matches)
# rankings
result = urlfetch.fetch(self.RANKINGS_URL.format(event.year, event_key, event_key))
if result.status_code != 200:
logging.warning('Unable to retreive url: ' + (self.RANKINGS_URL.format(event.year, event_key, event_key)))
else:
# convert into expected input format
rankings = list(csv.reader(StringIO.StringIO(result.content), delimiter=','))
event_details = EventDetails(
id=event_key,
rankings=rankings
)
EventDetailsManipulator.createOrUpdate(event_details)
self.response.out.write("Done restoring {}!".format(event_key))
def test_parse_cmp_awards(self):
with open('test_data/pre_2007_cmp_awards.csv', 'r') as f:
awards = CSVAwardsParser.parse(f.read())
# Check number of parsed awards
num_awards = 0
for award in awards:
num_awards += len(award['recipient_json_list'])
self.assertEqual(num_awards, 560)
self.assertEqual(len(awards), 330)
awards = convert_to_comparable(awards)
award = {
'year':
2003,
'event_short':
'cmp',
'name_str':
"Championship Finalist",
'award_type_enum':
AwardType.FINALIST,
'team_number_list': [25, 343, 494],
'recipient_json_list': [{
'team_number': 25,
'awardee': None
}, {
'team_number': 343,
'awardee': None
}, {
'team_number': 494,
'awardee': None
}],
}
self.assertTrue(award in awards)
award = {
'year': 2002,
'event_short': 'arc',
'name_str': "#1 Seed",
'award_type_enum': AwardType.NUM_1_SEED,
'team_number_list': [121],
'recipient_json_list': [{
'team_number': 121,
'awardee': None
}],
}
self.assertTrue(award in awards)
award = {
'year': 1992,
'event_short': 'cmp',
'name_str': "Play of the Day",
'award_type_enum': AwardType.PLAY_OF_THE_DAY,
'team_number_list': [131],
'recipient_json_list': [{
'team_number': 131,
'awardee': None
}],
}
self.assertTrue(award in awards)
