def test_no_create_fields_field_exists_has_columns(config):
del config["datasources"]["testdb2"]
table_config = config["datasources"]["testdb1"]["tables"]["main.partners"]
table_config["create_fields"] = False
wh = Warehouse(config=config)
# ds_partner_name was already defined, make sure it doesnt get overwritten
dim = wh.get_dimension("ds_partner_name")
assert dim.sa_type.length == 50
def test_warehouse_remote_json_table(adhoc_config):
url = (
"https://raw.githubusercontent.com/totalhack/zillion/master/tests/dma_zip.json"
)
adhoc_config["datasources"]["test_adhoc_db"]["tables"]["main.dma_zip"][
"data_url"] = url
wh = Warehouse(config=adhoc_config)
assert wh.has_dimension("Zip_Code")
def test_example_wh_report2():
wh = Warehouse(config=config)
result = wh.execute(
metrics=["sales", "leads", "revenue"],
dimensions=["campaign_name"],
criteria=[("partner_name", "=", "Partner A")],
)
assert result
info(result.df)
def test_example_wh_report3():
wh = Warehouse(config=config)
result = wh.execute(
metrics=["sales", "leads", "revenue"],
dimensions=["partner_name", "campaign_name"],
rollup=RollupTypes.ALL,
)
assert result
info(result.df)
def test_field_description(config):
wh = Warehouse(config=config)
partner_name = wh.get_field("partner_name") # Regular dim/field
assert partner_name.description is None
print(partner_name.display_name, partner_name.description)
rpl = wh.get_field("rpl") # Formula Metric
assert rpl.description is not None
print(rpl.display_name, rpl.description)
sale_hour = wh.get_field("sale_hour") # Auto conversion field
assert sale_hour.description is not None
print(sale_hour.display_name, sale_hour.description)
def test_no_create_fields_has_columns(config):
del config["datasources"]["testdb2"]
campaigns_config = config["datasources"]["testdb1"]["tables"][
"main.campaigns"]
campaigns_config["create_fields"] = False
# This will raise an error because the fields referenced in the columns'
# field lists don't exist
with pytest.raises(WarehouseException):
wh = Warehouse(config=config)
def test_warehouse_technical_within_formula(config):
config["metrics"].append({
"name": "revenue_ma_5_sum_5",
"aggregation": AggregationTypes.SUM,
"rounding": 2,
"formula": "{revenue_ma_5}/{revenue_sum_5}",
})
with pytest.raises(InvalidFieldException):
wh = Warehouse(config=config)
def test_create_fields_no_columns(config):
del config["datasources"]["testdb2"]
table_config = config["datasources"]["testdb1"]["tables"]["main.partners"]
table_config["create_fields"] = True
del table_config["columns"]
table_config["primary_key"] = ["fake_field"]
# Primary key mismatch in parent/child relationship with partners/campaigns
with pytest.raises(ZillionException):
wh = Warehouse(config=config)
def test_reserved_field_name(config):
config["datasources"]["testdb1"]["metrics"].append({
"name":
"row_hash",
"type":
"integer",
"aggregation":
AggregationTypes.SUM
})
ds = DataSource("testdb1", config=config["datasources"]["testdb1"])
with pytest.raises(WarehouseException):
wh = Warehouse(config=config, datasources=[ds])
def test_wh_save_and_load():
name = "test_warehouse_%s" % time.time()
config_url = REMOTE_CONFIG_URL
wh = Warehouse(config=config_url)
wh_id = wh.save(name, config_url, meta=dict(test=1))
wh = Warehouse.load(wh_id)
assert wh.name == name
Warehouse.delete(wh_id)
def init_warehouse_data(db: Session) -> None:
whs_json_file = os.getenv("INITIAL_WAREHOUSES_FILE", None)
if not whs_json_file:
print("#### No INITIAL_WAREHOUSES_FILE set")
return
print(f"#### Adding Initial Warehouses from {whs_json_file}")
whs = json.loads(read_filepath_or_buffer(whs_json_file))
engine = db.get_bind()
for wh in whs:
print("Adding Warehouse...")
print(wh)
engine.execute(
Warehouses.insert(),
id=wh["id"],
name=wh["name"],
params=json.dumps(wh["params"]),
meta=None,
)
# Load to make sure any required data syncs occur before app start
Warehouse.load(wh["id"])
def test_field_display_name(config):
wh = Warehouse(config=config)
partner_name = wh.get_field("partner_name") # Regular dim/field
assert partner_name.display_name == "Partner Name"
rpl = wh.get_field("rpl") # Formula Metric
assert rpl.display_name == "Revenue/Lead"
main_sales_created_at = wh.get_field(
"main_sales_created_at") # Field from column
assert main_sales_created_at.display_name == "Main Sales Created At"
sale_hour = wh.get_field("sale_hour") # Auto conversion field
assert sale_hour.display_name == "Sale Hour"
def test_create_fields_has_columns(config):
del config["datasources"]["testdb2"]
table_config = config["datasources"]["testdb1"]["tables"]["main.partners"]
table_config["create_fields"] = True
wh = Warehouse(config=config)
assert wh.has_dimension("partner_id")
assert wh.has_dimension("partner_name")
assert not wh.has_dimension("partners_name")
# This one is auto-generated with create_fields, so it has a default naming
# style:
assert wh.has_dimension("main_partners_created_at")
def get_warehouses() -> Dict[str, Any]:
"""NOTE: this assumes Zillion Web DB is same as Zillion DB"""
global warehouses
if warehouses:
# TODO: cache control?
return warehouses
print("Building warehouses...")
db = SessionLocal()
try:
result = db.query(Warehouses).all()
for row in result:
warehouses[row.id] = Warehouse.load(row.id)
pp(warehouses)
return warehouses
finally:
db.close()
def test_metric_copy(config):
wh = Warehouse(config=config)
field = wh.get_metric("revenue")
info(field.copy())
def test_warehouse_remote_datasource_config(config):
config["datasources"][
"testdb2"] = "https://raw.githubusercontent.com/totalhack/zillion/master/tests/test_sqlite_ds_config.json"
wh = Warehouse(config=config)
assert wh.has_metric("aggr_sales")
def test_formula_metric_copy(config):
wh = Warehouse(config=config)
field = wh.get_metric("rpl")
info(field.copy())
def test_warehouse_remote_csv_table(adhoc_config):
table_config = adhoc_config["datasources"]["test_adhoc_db"]["tables"][
"main.dma_zip"]
table_config["adhoc_table_options"] = {"nrows": 30}
wh = Warehouse(config=adhoc_config)
assert wh.has_dimension("Zip_Code")
def test_wh_from_url_wh_config():
f = "test_wh_config.json"
wh = Warehouse(config=f)
def test_warehouse_remote_google_sheet(adhoc_config):
url = "https://docs.google.com/spreadsheets/d/1iCzY4av_tinUpG2Q0mQhbxd77XOREwfbPAVZPObzFeE/edit?usp=sharing"
adhoc_config["datasources"]["test_adhoc_db"]["tables"]["main.dma_zip"][
"data_url"] = url
wh = Warehouse(config=adhoc_config)
assert wh.has_dimension("Zip_Code")
def test_dimension_copy(config):
wh = Warehouse(config=config)
field = wh.get_dimension("partner_name")
info(field.copy())
def test_example_wh_report1():
wh = Warehouse(config=config)
result = wh.execute(metrics=["sales", "leads", "revenue"],
dimensions=["partner_name"])
assert result
info(result.df)
import os
from zillion.warehouse import Warehouse
FILEDIR = os.path.dirname(os.path.abspath(__file__))
wh = Warehouse(config=FILEDIR + "/covid_warehouse.json")
if __name__ == "__main__":
wh.print_info()
def test_get_dimension_configs(config):
wh = Warehouse(config=config)
dims = wh.get_dimension_configs()
assert "partner_name" in dims
def test_example_wh_init():
wh = Warehouse(config=config)
wh.print_info()
def test_column_config_override(config):
table_config = config["datasources"]["testdb1"]["tables"]["main.sales"]
table_config["columns"]["revenue"]["active"] = False
wh = Warehouse(config=config)
assert not "sales" in wh.get_datasource("testdb1").get_tables_with_field(
"revenue")
import os
from zillion.warehouse import Warehouse
FILEDIR = os.path.dirname(os.path.abspath(__file__))
wh = Warehouse(config=FILEDIR + "/baseball_warehouse.json")
if __name__ == "__main__":
wh.print_info()
def test_table_config_override(config):
table_config = config["datasources"]["testdb1"]["tables"]["main.sales"]
table_config["active"] = False
wh = Warehouse(config=config)
assert not wh.get_datasource("testdb1").has_table("main.sales")
Example:
COMPOSE_FILE=docker-compose.light.dev.yml ./scripts/run-local.sh python /app/scripts/add_warehouse.py My_Warehouse /app/custom/my_warehouse.jdon
"""
import json
import sys
from zillion.model import Warehouses
from zillion.warehouse import Warehouse
from app.db.session import SessionLocal
if __name__ == "__main__":
db = SessionLocal()
name = sys.argv[1]
assert name
config_url = sys.argv[2]
assert config_url
print(f"Loading from url {config_url}")
wh = Warehouse(config=config_url)
engine = db.get_bind()
qr = engine.execute(
Warehouses.insert(),
name=name,
params=json.dumps({"config": config_url}),
)
wh_id = qr.inserted_primary_key[0]
print(f"Created warehouse ID {wh_id}")
def test_no_create_fields_no_columns(config):
table_config = config["datasources"]["testdb1"]["tables"]["main.partners"]
del table_config["columns"]
table_config["create_fields"] = False
with pytest.raises(ZillionException):
wh = Warehouse(config=config)
