def import_teams_csv(db_session, csv_folder): team_csv_file = csv_folder.joinpath(TEAMS_CSV_FILENAME) csv_text = team_csv_file.read_text() total_rows = len([row for row in csv_text.split("\n") if row]) try: with open(team_csv_file) as team_csv: reader = DataclassReader(team_csv, TeamCsvRow) reader.map("2B").to("doubles") reader.map("3B").to("triples") with tqdm( total=total_rows, desc="Populating team table..........", unit="row", mininterval=0.12, maxinterval=5, unit_scale=True, ncols=90, ) as pbar: for row in reader: t = db.Team( year=row.yearID, league=row.lgID, team_id=row.teamID, franch_id=row.franchID, division=row.divID, games=row.G, games_at_home=row.Ghome, wins=row.W, losses=row.L, runs=row.R, at_bats=row.AB, hits=row.H, doubles=row.doubles, triples=row.triples, homeruns=row.HR, base_on_balls=row.BB, strikeouts=row.SO, stolen_bases=row.SB, caught_stealing=row.CS, runs_against=row.RA, earned_runs=row.ER, saves=row.SV, ip_outs=row.IPouts, errors=row.E, name=row.name, park=row.park, batting_park_factor=row.BPF, pitching_park_factor=row.PPF, team_id_br=row.teamIDBR, team_id_retro=row.teamIDretro, ) db_session.add(t) pbar.update() return Result.Ok() except Exception as e: error = f"Error: {repr(e)}" db_session.rollback() return Result.Fail(error)
def get_tiingo_tickers(tiingo_ticker_path: str) -> Iterable[TiingoTicker]: with open(tiingo_ticker_path) as tiingo_tickers_csv: tickers_datareader = DataclassReader(tiingo_tickers_csv, TiingoTicker) tickers_datareader.map("assetType").to("asset_type") tickers_datareader.map("priceCurrency").to("price_currency") tickers_datareader.map("startDate").to("start_date") tickers_datareader.map("endDate").to("end_date") for ticker in tickers_datareader: yield ticker
def test_raise_error_when_mapped_column_not_found(create_csv): csv_file = create_csv({'name': 'User1', 'e-mail': '*****@*****.**'}) with csv_file.open() as f: with pytest.raises( KeyError, match= 'The value for the mapped column `e_mail` is missing in the CSV file' ): reader = DataclassReader(f, UserWithEmail) reader.map('e_mail').to('email') list(reader)
def test_raise_error_when_mapped_column_not_found(create_csv): csv_file = create_csv({"name": "User1", "e-mail": "*****@*****.**"}) with csv_file.open() as f: with pytest.raises( KeyError, match= "The value for the mapped column `e_mail` is missing in the CSV file", ): reader = DataclassReader(f, UserWithEmail) reader.map("e_mail").to("email") list(reader)
def test_csv_header_items_with_spaces_with_prop_with_wrong_type(create_csv): csv_file = create_csv({' name': 'User1', 'age ': 'this should be an int'}) with csv_file.open() as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, User) items = list(reader)
def test_should_raise_error_str_to_int_prop(create_csv): csv_file = create_csv({'name': 'User1', 'age': 'wrong type'}) with csv_file.open() as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, User) list(reader)
def test_csv_header_items_with_spaces_with_missing_value(create_csv): csv_file = create_csv({' name': 'User1', 'age ': None}) with csv_file.open() as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, User) items = list(reader)
def test_csv_header_items_with_spaces_with_missing_props_raises_keyerror(create_csv): csv_file = create_csv({' name': 'User1'}) with csv_file.open() as f: with pytest.raises(KeyError): reader = DataclassReader(f, User) items = list(reader)
def test_should_raise_error_when_required_value_is_emptyspaces(create_csv): csv_file = create_csv({'name': ' ', 'age': 40}) with csv_file.open() as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, User) list(reader)
def test_should_raise_error_when_required_column_is_missing(create_csv): csv_file = create_csv({'name': 'User1'}) with csv_file.open() as f: with pytest.raises(KeyError): reader = DataclassReader(f, User) list(reader)
def test_should_raise_error_with_incorrect_dateformat(create_csv): csv_file = create_csv({'name': 'User1', 'create_date': '2018-12-07 10:00'}) with csv_file.open() as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, UserWithDateFormatDecorator) list(reader)
def test_should_raise_error_when_value_is_whitespaces(create_csv): csv_file = create_csv({'name': ' '}) with csv_file.open('r') as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, UserWithoutAcceptWhiteSpacesDecorator) list(reader)
def test_should_raise_error_without_dateformat(create_csv): csv_file = create_csv({'name': 'Test', 'create_date': '2018-12-09'}) with csv_file.open('r') as f: with pytest.raises(AttributeError): reader = DataclassReader(f, UserWithoutDateFormatDecorator) list(reader)
def test_should_raise_error_str_to_int_prop(create_csv): csv_file = create_csv({"name": "User1", "age": "wrong type"}) with csv_file.open() as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, User) list(reader)
def test_should_raise_error_when_required_value_is_missing(create_csv): csv_file = create_csv({"name": "User1", "age": None}) with csv_file.open() as f: with pytest.raises(CsvValueError): reader = DataclassReader(f, User) list(reader)
def test_reader_values(create_csv): csv_file = create_csv([{ 'name': 'User1', 'age': 40 }, { 'name': 'User2', 'age': 30 }]) with csv_file.open() as f: reader = DataclassReader(f, User) items = list(reader) assert items and len(items) == 2 for item in items: assert dataclasses.is_dataclass(item) user1, user2 = items[0], items[1] assert user1.name == 'User1' assert user1.age == 40 assert user2.name == 'User2' assert user2.age == 30
def extract_towers(tower_csv): towers = [] with open(tower_csv) as tower_csv: csv_reader = DataclassReader(tower_csv, FluxTower) for tower in csv_reader: towers.append(tower) return towers
def test_try_to_access_not_initialized_prop_with_default_value(create_csv): csv_file = create_csv({"firstname": "User1", "lastname": "TestUser"}) with csv_file.open() as f: reader = DataclassReader(f, UserWithInitFalseAndDefaultValue) items = list(reader) user = items[0] assert user.age == 0
def test_reader_values(create_csv): csv_file = create_csv([{ "name": "User1", "age": 40 }, { "name": "User2", "age": 30 }]) with csv_file.open() as f: reader = DataclassReader(f, User) items = list(reader) assert items and len(items) == 2 for item in items: assert dataclasses.is_dataclass(item) user1, user2 = items[0], items[1] assert user1.name == "User1" assert user1.age == 40 assert user2.name == "User2" assert user2.age == 30
def test_parse_bool_value_none_default(create_csv): csv_file = create_csv({"boolValue": ""}) with csv_file.open() as f: reader = DataclassReader(f, DataclassWithBooleanValueNoneDefault) items = list(reader) dataclass_instance = items[0] assert dataclass_instance.boolValue is None
def test_parse_bool_value_none_default(create_csv): """Verify that blank/null values are parsed as None for optional fields""" csv_file = create_csv({'boolValue': ''}) with csv_file.open() as f: reader = DataclassReader(f, DataclassWithBooleanValueNoneDefault) items = list(reader) dataclass_instance = items[0] assert dataclass_instance.boolValue is None
def test_parse_bool_value_false(create_csv): for false_value in ['no', 'false', 'f', 'n', '0']: csv_file = create_csv({'boolValue': f'{false_value}'}) with csv_file.open() as f: reader = DataclassReader(f, DataclassWithBooleanValue) items = list(reader) dataclass_instance = items[0] assert dataclass_instance.boolValue is False
def test_try_to_access_not_initialized_prop_raise_attr_error(create_csv): csv_file = create_csv({'firstname': 'User1', 'lastname': 'TestUser'}) with csv_file.open() as f: reader = DataclassReader(f, UserWithInitFalse) items = list(reader) with pytest.raises(AttributeError): user = items[0] user_age = user.age
def test_reader_with_datetime_default_value(create_csv): csv_file = create_csv({"name": "User", "birthday": ""}) with csv_file.open() as f: reader = DataclassReader(f, UserWithDefaultDatetimeField) items = list(reader) assert len(items) > 0 assert isinstance(items[0].birthday, datetime)
def test_try_to_access_not_initialized_prop_raise_attr_error(create_csv): csv_file = create_csv({"firstname": "User1", "lastname": "TestUser"}) with csv_file.open() as f: reader = DataclassReader(f, UserWithInitFalse) items = list(reader) with pytest.raises(AttributeError): user = items[0] assert user.age is not None
def test_parse_bool_value_false(create_csv): for false_value in ["no", "false", "f", "n", "0"]: csv_file = create_csv({"boolValue": f"{false_value}"}) with csv_file.open() as f: reader = DataclassReader(f, DataclassWithBooleanValue) items = list(reader) dataclass_instance = items[0] assert dataclass_instance.boolValue is False
def test_parse_bool_value_true(create_csv): for true_value in ["yes", "true", "t", "y", "1"]: csv_file = create_csv({"boolValue": f"{true_value}"}) with csv_file.open() as f: reader = DataclassReader(f, DataclassWithBooleanValue) items = list(reader) dataclass_instance = items[0] assert dataclass_instance.boolValue is True
def test_parse_bool_value_true(create_csv): for true_value in ['yes', 'true', 't', 'y', '1']: csv_file = create_csv({'boolValue': f'{true_value}'}) with csv_file.open() as f: reader = DataclassReader(f, DataclassWithBooleanValue) items = list(reader) dataclass_instance = items[0] assert dataclass_instance.boolValue is True
def __init__(self) -> None: with urllib.request.urlopen( 'https://raw.githubusercontent.com/otomarukanta/stocks-data-public/main/stocks.csv' ) as response: data = response.read() reader = DataclassReader(data.decode('utf-8').splitlines(), Company) self.companies: List[Company] = [row for row in reader]
def setup_csv_stuff(): base_file = pathlib.Path(__file__).resolve().parent filename = pathlib.Path.joinpath(base_file, 'data', 'nfl_tickets.csv') with open(filename, 'r', encoding='utf-8') as file_out: text = DataclassReader(file_out, Tickets) for rows in text: data.append(rows)