def get_data_source(self, data_client_params):
_source = None
if data_client_params["is_local_source"]:
_source = LocalSource(
data_spec=data_client_params["source_data_spec"],
local_cache=data_client_params["source_local_cache"],
)
elif data_client_params["is_gcs_source"]:
if isinstance(
data_client_params["source_data_spec"], DonateYourDataSpec
):
_source = GCSDYDSource(
gcp_project=data_client_params["gcp_project"],
gcs_uri_base=data_client_params["gcs_uri_base"],
local_cache=data_client_params["source_local_cache"],
)
elif isinstance(
data_client_params["source_data_spec"], FlatFilesSpec
):
_source = GCSFlatFilesSource(
gcp_project=data_client_params["gcp_project"],
gcs_uri_base=data_client_params["gcs_uri_base"],
local_cache=data_client_params["source_local_cache"],
)
elif data_client_params["is_gbq_source"]:
_source = GBQDataSource(
data_spec=data_client_params["source_data_spec"],
gcp_project=data_client_params["gcp_project"],
gbq_table=data_client_params["gbq_table"],
local_cache=data_client_params["source_local_cache"],
)
return _source
def setup_class(cls):
# initialize with data to avoid pulling multiple times
cls.sim_config = Config.make_sim_config(
identifier=[
os.environ.get("TEST_FLATFILES_IDENTIFIER_MISSING"), # missing
os.environ.get("TEST_FLATFILES_IDENTIFIER_FULL"), # full
"9999999", # file not found
],
latitude=33.481136,
longitude=-112.078232,
start_utc="2018-01-01 00:00:00",
end_utc="2018-12-31 00:00:00",
min_sim_period="7D",
min_chunk_period="30D",
sim_step_size_seconds=300,
output_step_size_seconds=300,
)
cls.data_clients = []
# set local_cache=None to test connection with GCS
cls.data_client = DataClient(
source=GCSFlatFilesSource(
gcp_project=os.environ.get("FLATFILE_GOOGLE_CLOUD_PROJECT"),
gcs_uri_base=os.environ.get("FLATFILES_GCS_URI_BASE"),
local_cache=None,
),
destination=LocalDestination(
local_cache=os.environ.get("LOCAL_CACHE_DIR"),
data_spec=FlatFilesSpec(),
),
nrel_dev_api_key=os.environ.get("NREL_DEV_API_KEY"),
nrel_dev_email=os.environ.get("NREL_DEV_EMAIL"),
archive_tmy3_dir=os.environ.get("ARCHIVE_TMY3_DIR"),
archive_tmy3_meta=os.environ.get("ARCHIVE_TMY3_META"),
archive_tmy3_data_dir=os.environ.get("ARCHIVE_TMY3_DATA_DIR"),
ep_tmy3_cache_dir=os.environ.get("EP_TMY3_CACHE_DIR"),
simulation_epw_dir=os.environ.get("SIMULATION_EPW_DIR"),
)
for _idx, _sim_config in cls.sim_config.iterrows():
dc = copy.deepcopy(cls.data_client)
dc.sim_config = _sim_config
if _sim_config["identifier"] in [
"9999999",
]:
with pytest.raises(ValueError):
dc.get_data()
else:
dc.get_data()
cls.data_clients.append(dc)