def __iter__(self): # flake8: noqa
path = self.options.get('filename', 'output.csv').strip()
fieldnames = get_words(self.options.get('fieldnames'))
if path != '-' and not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
if sys.version_info[0] < 3:
fp = BytesIO()
else:
fp = StringIO()
writer = DictWriter(fp, list(fieldnames))
counter = 0
for item in self.previous:
if self.condition(item) and is_mapping(item):
if not writer.fieldnames:
writer.fieldnames = [key for key in item.keys()
if not key.startswith('_')]
if counter == 0:
header = dict(zip(writer.fieldnames, writer.fieldnames))
writer.writerow(header)
clone = dict(item)
for fieldname in writer.fieldnames:
clone.setdefault(fieldname, None)
writer.writerow(dict([
(key, value) for key, value in clone.items()
if key in writer.fieldnames
]))
counter += 1
yield item
fp.seek(0)
if path == '-':
sys.stdout.write(fp.read())
else:
with open(path, 'w') as fp2:
fp2.write(fp.read())
logger.info('{0:s}:{1:s} wrote {2:d} items to {3:s}'.format(
self.__class__.__name__, self.name, counter, path,
self.options.get('filename', 'output.csv')
))
def __iter__(self): # flake8: noqa
path = self.options.get('filename', 'output.csv').strip()
fieldnames = get_words(self.options.get('fieldnames'))
if path != '-' and not os.path.isabs(path):
path = os.path.join(os.getcwd(), path)
if sys.version_info[0] < 3:
fp = BytesIO()
else:
fp = StringIO()
writer = DictWriter(fp, list(fieldnames))
counter = 0
for item in self.previous:
if self.condition(item) and is_mapping(item):
if not writer.fieldnames:
writer.fieldnames = [
key for key in item.keys() if not key.startswith('_')
]
if counter == 0:
header = dict(zip(writer.fieldnames, writer.fieldnames))
writer.writerow(header)
clone = dict(item)
for fieldname in writer.fieldnames:
clone.setdefault(fieldname, None)
writer.writerow(
dict([(key, value) for key, value in clone.items()
if key in writer.fieldnames]))
counter += 1
yield item
fp.seek(0)
if path == '-':
sys.stdout.write(fp.read())
else:
with open(path, 'w') as fp2:
fp2.write(fp.read())
logger.info('{0:s}:{1:s} wrote {2:d} items to {3:s}'.format(
self.__class__.__name__, self.name, counter, path,
self.options.get('filename', 'output.csv')))
def build_training_data(self):
csv_headers_written = False
shutil.copy('training_data/training-latest.csv', 'training_data/training-data-backup.csv')
with open(f'training_data/training-latest.csv', 'w+') as csv_out:
dw = DictWriter(csv_out, fieldnames=[])
all_stats = []
for ctr in range(self.num_primer_races, self.system.num_actual_races):
if ctr % 100 == 0:
print(f'processed {ctr} races')
if not any(mn for mn in self.system.latest_race.mice_names if mn in self.system.dead_mice):
output_dict = OrderedDict()
race_stats = self.get_mice_stats(self.system.latest_race.mice_names)
output_dict['mice'] = self.system.latest_race.mice_names
output_dict['winner_name'] = self.system.latest_race.winner_name
output_dict['winner_name_id'] = self.system.latest_race.winner_name_id
output_dict['winner_position_ndx'] = self.system.latest_race.winner_position_ndx
output_dict['race_id'] = self.system.latest_race.id
output_dict['completed_at_year'] = self.system.latest_race.completed_at.year
output_dict['completed_at_month'] = self.system.latest_race.completed_at.month
output_dict['completed_at_day'] = self.system.latest_race.completed_at.day
output_dict['completed_at_weekday'] = self.system.latest_race.completed_at.weekday()
output_dict['completed_at_hour'] = self.system.latest_race.completed_at.hour
output_dict['completed_at_minute'] = self.system.latest_race.completed_at.minute
for mouse_num, mouse_stats in enumerate(race_stats):
for k, v in mouse_stats.items():
output_dict[f"mouse_{mouse_num}_{k}"] = v
if not csv_headers_written:
dw.fieldnames = list(output_dict.keys())
dw.writeheader()
dw.writerow(output_dict)
csv_headers_written = True
all_stats.append(output_dict)
else:
all_stats.append(output_dict)
dw.writerow(output_dict)
self.system.ingest_new_race()