def test_multi_report_not_run_raises_value_error():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3], n_iterations=1)
with pytest.raises(ValueError):
multi_report.plot(show=False)
def test_multi_plot_show(plt, plt1):
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3], n_iterations=1)
results = multi_report.run(print_report=False)
multi_report.plot(show=True, save_location="test.jpg")
def test_multi_plot():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3], n_iterations=1)
results = multi_report.run(print_report=False)
multi_report.plot(show=False)
def test_multi_plot_matplotlib_not_installed():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3], n_iterations=1)
results = multi_report.run(print_report=False)
with patch.dict(sys.modules, {"matplotlib": None}):
with pytest.raises(MatplotlibNotInstalledError):
multi_report.plot()
def test_multi_run_print_report():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3], n_iterations=1)
results = multi_report.run()
assert isinstance(results, list)
for result in results:
for expected in EXPECTED_METRICS:
assert expected in result.keys()
def test_multi_init_no_names_given():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3], n_iterations=1)
assert multi_report.model_names == ["Model 0", "Model 1"]
def test_multi_init_list_inputs_len_less_than_models_len():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [1], n_iterations=1)
assert multi_report.inputs == [1, 1]
def test_multi_init_model_not_callable():
model = MockModel()
model1 = MockModel()
with pytest.raises(ModelIsNotCallableError):
multi_report = MultiInferenceReport([model1, model.infer_callable],
1,
n_seconds=10)
def test_multi_no_intervals_given_raises_error():
model = MockModel()
model1 = MockModel()
with pytest.raises(MeasurementIntervalNotSetError):
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3])
def test_multi_init_seconds():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], 1, n_seconds=10)
assert multi_report.n_iterations == None
assert multi_report.n_seconds == 10
def test_multi_init_names_given_greater_than_number_of_models():
model = MockModel()
model1 = MockModel()
with pytest.raises(NamesNotEqualsModelsLengthError):
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable],
[3],
n_iterations=1,
model_names=["MyModel1", "MyModel2", "MyModel3"],
)
def test_multi_two_intervals_given_defaults_to_seconds():
model = MockModel()
model1 = MockModel()
multi_report = MultiInferenceReport(
[model.infer_callable, model1.infer_callable], [3],
n_iterations=1,
n_seconds=1)
assert multi_report.n_iterations == None
assert multi_report.n_seconds == 1
return input * pow(input, 2)
pipe = pipeline(task="sentiment-analysis")
class MockModel:
def run(self, input):
time.sleep(random.choice([0.041, 0.012, 0.067, 0.036, 0.001, 0.002]))
return (input**2 / 3)**math.pi
mock_model = MockModel()
mock_model1 = MockModel()
_input = "The light released from around the first massive black holes in the universe is so intense that it is able to reach telescopes across the entire expanse of the universe. Incredibly, the light from the most distant black holes (or quasars) has been traveling to us for more than 13 billion light years. However, we do not know how these monster black holes formed. The light released from around the first massive black holes in the universe is so intense that it is able to reach telescopes across the entire expanse of the universe. Incredibly, the light from the most distant black holes (or quasars) has been traveling to us for more than 13 billion light years. However, we do not know how these monster black holes formed. New research led by researchers from Georgia Institute of Technology, Dublin City University, Michigan State University, the University of California at San Diego, the San Diego Supercomputer Center and IBM, provides a new and extremely promising avenue for solving this cosmic riddle. The team showed that when galaxies assemble extremely rapidly -- and sometimes violently -- that can lead to the formation of very massive black holes. In these rare galaxies, normal star formation is disrupted and black hole formation takes over. The new study finds that massive black holes form in dense starless regions that are growing rapidly, turning upside down the long-accepted belief that massive black hole formation was limited to regions bombarded by the powerful radiation of nearby galaxies. Conclusions of the study, reported on January 23rd in the journal Nature and supported by funding from the National Science Foundation, the European Union and NASA, also finds that massive black holes are much more common in the universe than previously thought. The key criteria for determining where massive black holes formed during the universe's infancy relates to the rapid growth of pre-galactic gas clouds that are the forerunners of all present-day galaxies, meaning that most supermassive black holes have a common origin forming in this newly discovered scenario, said John Wise, an associate professor in the Center for Relativistic Astrophysics at Georgia Tech and the paper's corresponding author. Dark matter collapses into halos that are the gravitational glue for all galaxies. Early rapid growth of these halos prevented the formation of stars that would have competed with black holes for gaseous matter flowing into the area."
# report = InferenceReport(
# model=pipe, inputs=_input, n_seconds=30, infer_failure_point=0.480
# )
# report.run()
# report.plot(report.runs)
multi_report = MultiInferenceReport(
[pipe, pipe, pipe, pipe],
[_input, _input[:2000], _input[:1500], _input[:1000]],
n_iterations=30,
infer_failure_point=1,
)
results_list = multi_report.run()
multi_report.plot()
)
electra_model = AutoModelForSequenceClassification.from_pretrained(
"monologg/electra-small-finetuned-imdb"
)
electra_tokenizer = AutoTokenizer.from_pretrained(
"monologg/electra-small-finetuned-imdb"
)
bert_model = AutoModelForSequenceClassification.from_pretrained(
"textattack/bert-base-uncased-imdb"
)
bert_tokenizer = AutoTokenizer.from_pretrained("textattack/bert-base-uncased-imdb")
TEXT = "It was the best of times, it was the worst of times, it was the age of wisdom, it was the age of foolishness, it was the epoch of belief, it was the epoch of incredulity, it was the season of Light, it was the season of Darkness"
distilbert_text = distilbert_tokenizer(TEXT, return_tensors="pt")
electra_text = electra_tokenizer(TEXT, return_tensors="pt")
bert_text = bert_tokenizer(TEXT, return_tensors="pt")
multi_report = MultiInferenceReport(
[distilbert_model, electra_model, bert_model],
[distilbert_text["input_ids"], electra_text["input_ids"], bert_text["input_ids"]],
n_iterations=100,
model_names=["Distilbert", "Electra", "Bert"],
infer_failure_point=0.170,
)
multi_report.run()
multi_report.plot()