def test_run_performance_job(auth0_id, nocommit_transaction, mockup_modelchain):
    si = storage.StorageInterface(user=auth0_id)
    stored_job, save, df = mockup_modelchain

    compute.run_performance_job(stored_job, si)
    assert save.call_count == 1
    reslist = save.call_args[0][1]
    assert len(reslist) == 3

    perf = reslist[-1]
    assert perf.type == "performance data"
    iob = BytesIO(perf.data)
    iob.seek(0)
    perf_df = pd.read_feather(iob).set_index("time")
    assert perf_df.loc[df.index[0], "performance"] == 2.0  # sum of 2 inverters
    pd.testing.assert_index_equal(
        perf_df.index, stored_job.definition.parameters.time_parameters._time_range
    )

    month_avg = reslist[0]
    assert month_avg.type == "monthly summary"
    iob = BytesIO(month_avg.data)
    iob.seek(0)
    month_df = pd.read_feather(iob)
    assert len(month_df.index) == 12
    ser = month_df.iloc[0]
    assert len(ser) == 5
    assert ser.loc["month"] == "January"
    assert abs(ser.loc["total_energy"] - 2.0) < 1e-8
    assert abs(ser.loc["plane_of_array_insolation"] - 1.0) < 1e-8
def test_generate_job_weather_data_inverter_multi_array(stored_job, auth0_id, mocker):
    si = storage.StorageInterface(user=auth0_id)

    def mockgetdata(job_id, data_id, si):
        return (job_id, data_id)

    mocker.patch("solarperformanceinsight_api.compute._get_data", new=mockgetdata)

    do = stored_job.data_objects[0]
    new_do = []
    ids = []
    for i in range(3):
        ndo = deepcopy(do)
        ndo.object_id = uuid1()
        ids.append((stored_job.object_id, ndo.object_id))
        ndo.definition.schema_path = f"/inverters/{i}"
        new_do.append(ndo)
    inv = deepcopy(stored_job.definition.system_definition.inverters[0])
    inv.arrays = [inv.arrays[0], inv.arrays[0]]
    stored_job.definition.system_definition.inverters = [inv, inv, inv]
    stored_job.definition.parameters.weather_granularity = "inverter"
    stored_job.data_objects = new_do
    gen = compute.generate_job_weather_data(stored_job, si)
    assert str(type(gen)) == "<class 'generator'>"
    genlist = list(gen)
    # returns list of dataframes for each item
    assert genlist == [[i, i] for i in ids]
def test_generate_job_weather_data_array(stored_job, auth0_id, mocker):
    si = storage.StorageInterface(user=auth0_id)

    def mockgetdata(job_id, data_id, si):
        return (job_id, data_id)

    mocker.patch("solarperformanceinsight_api.compute._get_data", new=mockgetdata)

    do = stored_job.data_objects[0]
    new_do = []
    ids = []
    for i in range(3):
        arid = []
        for j in range(2):
            ndo = deepcopy(do)
            ndo.object_id = f"{i}_{j}"
            arid.append((stored_job.object_id, ndo.object_id))
            ndo.definition.schema_path = f"/inverters/{i}/arrays/{j}"
            new_do.append(ndo)
        ids.append(arid)
    inv = deepcopy(stored_job.definition.system_definition.inverters[0])
    arr = deepcopy(inv.arrays[0])
    inv.arrays = [arr, arr]
    stored_job.definition.system_definition.inverters = [inv, inv, inv]
    stored_job.definition.parameters.weather_granularity = "array"
    stored_job.data_objects = new_do
    gen = compute.generate_job_weather_data(stored_job, si)
    assert str(type(gen)) == "<class 'generator'>"
    genlist = list(gen)
    assert len(genlist[0]) == 2
    assert genlist[0][0][1] == "0_0"
    assert genlist[0][1][1] == "0_1"
    assert genlist[1][1][1] == "1_1"
    # returns list of dataframes for each inverter
    assert genlist == ids
def test_create_upload_delete_compute(client, nocommit_transaction, new_job,
                                      weather_df, async_queue, mocker,
                                      auth0_id):
    cr = client.post("/jobs/", data=new_job.json())
    assert cr.status_code == 201
    new_id = cr.json()["object_id"]
    response = client.get(f"/jobs/{new_id}")
    assert response.status_code == 200
    stored_job = response.json()
    assert len(stored_job["data_objects"]) == 1
    data_id = stored_job["data_objects"][0]["object_id"]
    iob = BytesIO()
    weather_df.to_feather(iob)
    iob.seek(0)
    response = client.post(
        f"/jobs/{new_id}/data/{data_id}",
        files={
            "file": ("test.arrow", iob, "application/vnd.apache.arrow.file")
        },
    )
    assert response.status_code == 200
    response = client.get(f"/jobs/{new_id}/status")
    assert response.json()["status"] == "prepared"
    response = client.post(f"/jobs/{new_id}/compute")
    assert response.status_code == 202
    response = client.get(f"/jobs/{new_id}/status")
    assert response.json()["status"] == "queued"
    with storage.StorageInterface(user=auth0_id).start_transaction() as st:
        st.delete_job(new_id)

    ww = SimpleWorker([async_queue], connection=async_queue.connection)
    log = mocker.spy(ww, "log")
    ww.work(burst=True)
    # worker logs error when exception raised in job
    assert log.error.call_count == 0
def test_start_transaction_commit(mocker):
    si = storage.StorageInterface()
    conn = mocker.MagicMock()
    mocker.patch.object(storage.engine, "connect", return_value=conn)
    with si.start_transaction() as st:
        st.cursor.execute("select 1")
    conn.commit.assert_called()
def test_start_transaction_rollback(mocker, err):
    si = storage.StorageInterface()
    conn = mocker.MagicMock()
    mocker.patch.object(storage.engine, "connect", return_value=conn)

    with pytest.raises(err):
        with si.start_transaction():
            raise err(400)
    conn.rollback.assert_called()
    conn.commit.assert_not_called()
def test_run_job_job_fail(job_id, auth0_id, mocker, msg, nocommit_transaction):
    if msg:
        new = mocker.MagicMock(side_effect=ValueError("message"))
    else:
        new = mocker.MagicMock(side_effect=ValueError)
    mocker.patch.object(compute, "lookup_job_compute_function", return_value=new)
    compute.run_job(job_id, auth0_id)
    assert new.call_count == 1
    with storage.StorageInterface(user=auth0_id).start_transaction() as st:
        results = st.list_job_results(job_id)
    assert len(results) == 1
    assert results[0].definition.type == "error message"
def test_save_results_to_db(job_id, nocommit_transaction, auth0_id):
    si = storage.StorageInterface(user=auth0_id)
    with si.start_transaction() as st:
        prev_results = st.list_job_results(job_id)
    assert len(prev_results) == 0
    df = pd.DataFrame(
        {"a": 0.0},
        index=pd.date_range("2020-01-01T00:00Z", freq="10min", periods=3, name="time"),
    )
    dbrs = [
        compute.DBResult(schema_path="/", type="performance data", data=df),
        compute.DBResult(schema_path="/", type="weather data", data=df),
    ]
    compute.save_results_to_db(job_id, dbrs, si)
    with si.start_transaction() as st:
        new_results = st.list_job_results(job_id)
        status = st.get_job_status(job_id)
    assert len(new_results) == 2
    assert status.status == "complete"
def test_generate_job_weather_data_system_multi_array(stored_job, auth0_id, mocker):
    si = storage.StorageInterface(user=auth0_id)
    arr = stored_job.definition.system_definition.inverters[0].arrays[0]
    stored_job.definition.system_definition.inverters[0].arrays = [arr, arr]

    def mockgetdata(job_id, data_id, si):
        return pd.DataFrame({"jid": job_id, "did": data_id}, index=[0])

    mocker.patch("solarperformanceinsight_api.compute._get_data", new=mockgetdata)

    ndo = deepcopy(stored_job.data_objects[0])
    ndo.object_id = uuid1()
    ndo.definition.schema_path = "/"

    stored_job.definition.parameters.weather_granularity = "system"
    stored_job.data_objects = [ndo]
    gen = compute.generate_job_weather_data(stored_job, si)
    assert str(type(gen)) == "<class 'generator'>"
    genlist = list(gen)
    assert len(genlist) == 1
    assert len(genlist[0]) == 2
def test_compare_expected_and_actual(mockup_modelchain, auth0_id, nocommit_transaction):
    si = storage.StorageInterface(user=auth0_id)
    stored_job, save, df = mockup_modelchain

    compute.compare_expected_and_actual(stored_job, si)
    assert save.call_count == 1
    reslist = save.call_args[0][1]
    assert len(reslist) == 4

    summary = reslist[-1]
    assert summary.type == "actual vs expected energy"
    iob = BytesIO(summary.data)
    iob.seek(0)
    summary_df = pd.read_feather(iob)
    assert len(summary_df.index) == 12
    ser = summary_df.iloc[0]
    assert len(ser) == 5
    assert ser.loc["month"] == "January"
    assert (ser.loc["expected_energy"] - 2.0) < 1e-7
    assert ser.loc["actual_energy"] == 1.0
    assert (ser.loc["difference"] - -1.0) < 1e-7
    assert (ser.loc["ratio"] - 1.0 / 2.0) < 1e-7
def test_generate_job_weather_data_system(stored_job, auth0_id, mocker):
    si = storage.StorageInterface(user=auth0_id)

    def mockgetdata(job_id, data_id, si):
        return pd.DataFrame({"jid": job_id, "did": data_id}, index=[0])

    mocker.patch("solarperformanceinsight_api.compute._get_data", new=mockgetdata)

    ndo = deepcopy(stored_job.data_objects[0])
    ndo.object_id = uuid1()
    ndo.definition.schema_path = "/"

    stored_job.definition.parameters.weather_granularity = "system"
    stored_job.data_objects = [ndo]
    gen = compute.generate_job_weather_data(stored_job, si)
    assert str(type(gen)) == "<class 'generator'>"
    genlist = list(gen)
    assert len(genlist) == 1
    # returns list of dataframes for each item
    pd.testing.assert_frame_equal(
        genlist[0][0],
        pd.DataFrame({"jid": stored_job.object_id, "did": ndo.object_id}, index=[0]),
    )
def test_generate_job_weather_data_fail(stored_job, auth0_id, mocker):
    si = storage.StorageInterface(user=auth0_id)

    stored_job.definition.parameters.weather_granularity = "unknown"
    with pytest.raises(ValueError):
        list(compute.generate_job_weather_data(stored_job, si))
def test_get_data_bad(job_id, job_data_ids, auth0_id):
    si = storage.StorageInterface(user=auth0_id)
    with pytest.raises(TypeError):
        compute._get_data(job_id, job_data_ids[0], si)
def test_get_data(complete_job_id, complete_job_data_id, auth0_id):
    si = storage.StorageInterface(user=auth0_id)
    assert isinstance(
        compute._get_data(complete_job_id, complete_job_data_id, si),
        pd.DataFrame,
    )
def storage_interface(auth0_id):
    out = storage.StorageInterface()
    out.commit = False
    out.user = auth0_id
    return out