def import_teams_csv(db_session, csv_folder):
team_csv_file = csv_folder.joinpath(TEAMS_CSV_FILENAME)
csv_text = team_csv_file.read_text()
total_rows = len([row for row in csv_text.split("\n") if row])
try:
with open(team_csv_file) as team_csv:
reader = DataclassReader(team_csv, TeamCsvRow)
reader.map("2B").to("doubles")
reader.map("3B").to("triples")
with tqdm(
total=total_rows,
desc="Populating team table..........",
unit="row",
mininterval=0.12,
maxinterval=5,
unit_scale=True,
ncols=90,
) as pbar:
for row in reader:
t = db.Team(
year=row.yearID,
league=row.lgID,
team_id=row.teamID,
franch_id=row.franchID,
division=row.divID,
games=row.G,
games_at_home=row.Ghome,
wins=row.W,
losses=row.L,
runs=row.R,
at_bats=row.AB,
hits=row.H,
doubles=row.doubles,
triples=row.triples,
homeruns=row.HR,
base_on_balls=row.BB,
strikeouts=row.SO,
stolen_bases=row.SB,
caught_stealing=row.CS,
runs_against=row.RA,
earned_runs=row.ER,
saves=row.SV,
ip_outs=row.IPouts,
errors=row.E,
name=row.name,
park=row.park,
batting_park_factor=row.BPF,
pitching_park_factor=row.PPF,
team_id_br=row.teamIDBR,
team_id_retro=row.teamIDretro,
)
db_session.add(t)
pbar.update()
return Result.Ok()
except Exception as e:
error = f"Error: {repr(e)}"
db_session.rollback()
return Result.Fail(error)
def get_tiingo_tickers(tiingo_ticker_path: str) -> Iterable[TiingoTicker]:
with open(tiingo_ticker_path) as tiingo_tickers_csv:
tickers_datareader = DataclassReader(tiingo_tickers_csv, TiingoTicker)
tickers_datareader.map("assetType").to("asset_type")
tickers_datareader.map("priceCurrency").to("price_currency")
tickers_datareader.map("startDate").to("start_date")
tickers_datareader.map("endDate").to("end_date")
for ticker in tickers_datareader:
yield ticker
def test_raise_error_when_mapped_column_not_found(create_csv):
csv_file = create_csv({'name': 'User1', 'e-mail': '*****@*****.**'})
with csv_file.open() as f:
with pytest.raises(
KeyError,
match=
'The value for the mapped column `e_mail` is missing in the CSV file'
):
reader = DataclassReader(f, UserWithEmail)
reader.map('e_mail').to('email')
list(reader)
def test_raise_error_when_mapped_column_not_found(create_csv):
csv_file = create_csv({"name": "User1", "e-mail": "*****@*****.**"})
with csv_file.open() as f:
with pytest.raises(
KeyError,
match=
"The value for the mapped column `e_mail` is missing in the CSV file",
):
reader = DataclassReader(f, UserWithEmail)
reader.map("e_mail").to("email")
list(reader)
def test_csv_header_items_with_spaces_with_prop_with_wrong_type(create_csv):
csv_file = create_csv({' name': 'User1', 'age ': 'this should be an int'})
with csv_file.open() as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, User)
items = list(reader)
def test_should_raise_error_str_to_int_prop(create_csv):
csv_file = create_csv({'name': 'User1', 'age': 'wrong type'})
with csv_file.open() as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, User)
list(reader)
def test_csv_header_items_with_spaces_with_missing_value(create_csv):
csv_file = create_csv({' name': 'User1', 'age ': None})
with csv_file.open() as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, User)
items = list(reader)
def test_csv_header_items_with_spaces_with_missing_props_raises_keyerror(create_csv):
csv_file = create_csv({' name': 'User1'})
with csv_file.open() as f:
with pytest.raises(KeyError):
reader = DataclassReader(f, User)
items = list(reader)
def test_should_raise_error_when_required_value_is_emptyspaces(create_csv):
csv_file = create_csv({'name': ' ', 'age': 40})
with csv_file.open() as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, User)
list(reader)
def test_should_raise_error_when_required_column_is_missing(create_csv):
csv_file = create_csv({'name': 'User1'})
with csv_file.open() as f:
with pytest.raises(KeyError):
reader = DataclassReader(f, User)
list(reader)
def test_should_raise_error_with_incorrect_dateformat(create_csv):
csv_file = create_csv({'name': 'User1', 'create_date': '2018-12-07 10:00'})
with csv_file.open() as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, UserWithDateFormatDecorator)
list(reader)
def test_should_raise_error_when_value_is_whitespaces(create_csv):
csv_file = create_csv({'name': ' '})
with csv_file.open('r') as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, UserWithoutAcceptWhiteSpacesDecorator)
list(reader)
def test_should_raise_error_without_dateformat(create_csv):
csv_file = create_csv({'name': 'Test', 'create_date': '2018-12-09'})
with csv_file.open('r') as f:
with pytest.raises(AttributeError):
reader = DataclassReader(f, UserWithoutDateFormatDecorator)
list(reader)
def test_should_raise_error_str_to_int_prop(create_csv):
csv_file = create_csv({"name": "User1", "age": "wrong type"})
with csv_file.open() as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, User)
list(reader)
def test_should_raise_error_when_required_value_is_missing(create_csv):
csv_file = create_csv({"name": "User1", "age": None})
with csv_file.open() as f:
with pytest.raises(CsvValueError):
reader = DataclassReader(f, User)
list(reader)
def test_reader_values(create_csv):
csv_file = create_csv([{
'name': 'User1',
'age': 40
}, {
'name': 'User2',
'age': 30
}])
with csv_file.open() as f:
reader = DataclassReader(f, User)
items = list(reader)
assert items and len(items) == 2
for item in items:
assert dataclasses.is_dataclass(item)
user1, user2 = items[0], items[1]
assert user1.name == 'User1'
assert user1.age == 40
assert user2.name == 'User2'
assert user2.age == 30
def extract_towers(tower_csv):
towers = []
with open(tower_csv) as tower_csv:
csv_reader = DataclassReader(tower_csv, FluxTower)
for tower in csv_reader:
towers.append(tower)
return towers
def test_try_to_access_not_initialized_prop_with_default_value(create_csv):
csv_file = create_csv({"firstname": "User1", "lastname": "TestUser"})
with csv_file.open() as f:
reader = DataclassReader(f, UserWithInitFalseAndDefaultValue)
items = list(reader)
user = items[0]
assert user.age == 0
def test_reader_values(create_csv):
csv_file = create_csv([{
"name": "User1",
"age": 40
}, {
"name": "User2",
"age": 30
}])
with csv_file.open() as f:
reader = DataclassReader(f, User)
items = list(reader)
assert items and len(items) == 2
for item in items:
assert dataclasses.is_dataclass(item)
user1, user2 = items[0], items[1]
assert user1.name == "User1"
assert user1.age == 40
assert user2.name == "User2"
assert user2.age == 30
def test_parse_bool_value_none_default(create_csv):
csv_file = create_csv({"boolValue": ""})
with csv_file.open() as f:
reader = DataclassReader(f, DataclassWithBooleanValueNoneDefault)
items = list(reader)
dataclass_instance = items[0]
assert dataclass_instance.boolValue is None
def test_parse_bool_value_none_default(create_csv):
"""Verify that blank/null values are parsed as None for optional fields"""
csv_file = create_csv({'boolValue': ''})
with csv_file.open() as f:
reader = DataclassReader(f, DataclassWithBooleanValueNoneDefault)
items = list(reader)
dataclass_instance = items[0]
assert dataclass_instance.boolValue is None
def test_parse_bool_value_false(create_csv):
for false_value in ['no', 'false', 'f', 'n', '0']:
csv_file = create_csv({'boolValue': f'{false_value}'})
with csv_file.open() as f:
reader = DataclassReader(f, DataclassWithBooleanValue)
items = list(reader)
dataclass_instance = items[0]
assert dataclass_instance.boolValue is False
def test_try_to_access_not_initialized_prop_raise_attr_error(create_csv):
csv_file = create_csv({'firstname': 'User1', 'lastname': 'TestUser'})
with csv_file.open() as f:
reader = DataclassReader(f, UserWithInitFalse)
items = list(reader)
with pytest.raises(AttributeError):
user = items[0]
user_age = user.age
def test_reader_with_datetime_default_value(create_csv):
csv_file = create_csv({"name": "User", "birthday": ""})
with csv_file.open() as f:
reader = DataclassReader(f, UserWithDefaultDatetimeField)
items = list(reader)
assert len(items) > 0
assert isinstance(items[0].birthday, datetime)
def test_try_to_access_not_initialized_prop_raise_attr_error(create_csv):
csv_file = create_csv({"firstname": "User1", "lastname": "TestUser"})
with csv_file.open() as f:
reader = DataclassReader(f, UserWithInitFalse)
items = list(reader)
with pytest.raises(AttributeError):
user = items[0]
assert user.age is not None
def test_parse_bool_value_false(create_csv):
for false_value in ["no", "false", "f", "n", "0"]:
csv_file = create_csv({"boolValue": f"{false_value}"})
with csv_file.open() as f:
reader = DataclassReader(f, DataclassWithBooleanValue)
items = list(reader)
dataclass_instance = items[0]
assert dataclass_instance.boolValue is False
def test_parse_bool_value_true(create_csv):
for true_value in ["yes", "true", "t", "y", "1"]:
csv_file = create_csv({"boolValue": f"{true_value}"})
with csv_file.open() as f:
reader = DataclassReader(f, DataclassWithBooleanValue)
items = list(reader)
dataclass_instance = items[0]
assert dataclass_instance.boolValue is True
def test_parse_bool_value_true(create_csv):
for true_value in ['yes', 'true', 't', 'y', '1']:
csv_file = create_csv({'boolValue': f'{true_value}'})
with csv_file.open() as f:
reader = DataclassReader(f, DataclassWithBooleanValue)
items = list(reader)
dataclass_instance = items[0]
assert dataclass_instance.boolValue is True
def __init__(self) -> None:
with urllib.request.urlopen(
'https://raw.githubusercontent.com/otomarukanta/stocks-data-public/main/stocks.csv'
) as response:
data = response.read()
reader = DataclassReader(data.decode('utf-8').splitlines(), Company)
self.companies: List[Company] = [row for row in reader]
def setup_csv_stuff():
base_file = pathlib.Path(__file__).resolve().parent
filename = pathlib.Path.joinpath(base_file, 'data', 'nfl_tickets.csv')
with open(filename, 'r', encoding='utf-8') as file_out:
text = DataclassReader(file_out, Tickets)
for rows in text:
data.append(rows)
