def test_refund(self):
data = np.array([0, 2, 0])
correct_class = _ground_truth(data)
orig_address = "Orig"
bal = 1E5
self.balances.initialize(orig_address, bal)
msg = Msg(orig_address, 1E3)
self.time_method.set_time(self.time_method() + 1)
added_time = self.time_method()
self.decai.add_data(msg, data, correct_class)
self.assertLess(self.balances[orig_address], bal)
# Add same data from another address.
msg = Msg(self.good_address, 1E3)
self.time_method.set_time(self.time_method() + 1)
bal = self.balances[self.good_address]
self.decai.add_data(msg, data, correct_class)
self.assertLess(self.balances[self.good_address], bal)
# Original address refunds.
msg = Msg(orig_address, 1E3)
bal = self.balances[orig_address]
self.time_method.set_time(self.time_method() +
self.decai.im.refund_time_s + 1)
self.decai.refund(msg, data, correct_class, added_time)
self.assertGreater(self.balances[orig_address], bal)
def test_report_take_all(self):
data = np.array([0, 0, 0])
correct_class = _ground_truth(data)
submitted_classification = 1 - correct_class
# Add bad data.
malicious_address = 'malicious_take_backer'
self.balances.initialize(malicious_address, 1E6)
bal = self.balances[malicious_address]
msg = Msg(malicious_address, bal)
self.time_method.set_time(self.time_method() + 1)
added_time = self.time_method()
self.decai.add_data(msg, data, submitted_classification)
self.assertLess(self.balances[malicious_address], bal,
"Adding data should have a cost.")
self.time_method.set_time(self.time_method() +
self.decai.im.any_address_claim_wait_time_s +
1)
# Can't refund.
msg = Msg(malicious_address, self.balances[malicious_address])
try:
self.decai.refund(msg, data, submitted_classification, added_time)
self.fail("Should have failed.")
except RejectException as e:
self.assertEqual("The model doesn't agree with your contribution.",
e.args[0])
bal = self.balances[malicious_address]
msg = Msg(malicious_address, bal)
self.decai.report(msg, data, submitted_classification, added_time,
malicious_address)
self.assertGreater(self.balances[malicious_address], bal)
def test_report(self):
inj = Injector([
SimpleDataModule,
LoggingModule,
PerceptronModule,
PredictionMarketImModule(
allow_greater_deposit=True,
group_contributions=True,
reset_model_during_reward_phase=True,
),
])
balances = inj.get(Balances)
data = inj.get(DataLoader)
im = cast(PredictionMarket, inj.get(IncentiveMechanism))
im.owner = 'owner'
time_method = inj.get(TimeMock)
assert isinstance(im, PredictionMarket)
init_train_data_portion = 0.2
initializer_address = 'initializer'
total_bounty = 100_000
balances.initialize(initializer_address, total_bounty)
good_contributor_address = 'good_contributor'
initial_good_balance = 10_000
balances.initialize(good_contributor_address, initial_good_balance)
bad_contributor_address = 'bad_contributor'
initial_bad_balance = 10_000
balances.initialize(bad_contributor_address, initial_bad_balance)
(x_train, y_train), (x_test, y_test) = data.load_data()
init_idx = int(len(x_train) * init_train_data_portion)
assert init_idx > 0
x_init_data, y_init_data = x_train[:init_idx], y_train[:init_idx]
x_remaining, y_remaining = x_train[init_idx:], y_train[init_idx:]
# Split test set into pieces.
num_pieces = 10
test_dataset_hashes, test_sets = im.get_test_set_hashes(num_pieces, x_test, y_test)
# Ending criteria:
min_length_s = 100
min_num_contributions = min(len(x_remaining), 100)
# Commitment Phase
self.assertIsNone(im.state)
im.model.init_model(x_init_data, y_init_data)
test_reveal_index = im.initialize_market(Msg(initializer_address, total_bounty),
test_dataset_hashes,
min_length_s, min_num_contributions)
self.assertEqual(MarketPhase.INITIALIZATION, im.state)
assert 0 <= test_reveal_index < len(test_dataset_hashes)
im.reveal_init_test_set(test_sets[test_reveal_index])
self.assertEqual(MarketPhase.PARTICIPATION, im.state)
# Participation Phase
value = 100
total_deposits = defaultdict(float)
stored_data = None
for i in range(min_num_contributions):
time_method.add_time(60)
data = x_remaining[i]
classification = y_remaining[i]
if i % 2 == 0:
contributor = good_contributor_address
else:
contributor = bad_contributor_address
classification = 1 - classification
cost, _ = im.handle_add_data(contributor, value, data, classification)
if stored_data is None:
stored_data = StoredData(classification, time_method(), contributor, cost, cost)
balances.send(contributor, im.owner, cost)
total_deposits[contributor] += cost
# Reward Phase
self.assertEqual(MarketPhase.PARTICIPATION, im.state)
im.end_market()
time_method.add_time(60)
self.assertEqual(MarketPhase.REVEAL_TEST_SET, im.state)
for i, test_set_portion in enumerate(test_sets):
if i != test_reveal_index:
im.verify_next_test_set(test_set_portion)
self.assertEqual(MarketPhase.REWARD_RESTART, im.state)
while im.remaining_bounty_rounds > 0:
time_method.add_time(60)
im.process_contribution()
# Collect rewards.
self.assertEqual(MarketPhase.REWARD_COLLECT, im.state)
# Get some stored data.
# Make sure reporting doesn't work yet.
reward = im.handle_report(bad_contributor_address, stored_data, False, None)
self.assertEqual(0, reward, "There should be no reward yet.")
time_method.add_time(im.any_address_claim_wait_time_s)
reward = im.handle_report(bad_contributor_address, stored_data, False, None)
balances.send(im.owner, bad_contributor_address, reward)
# Don't need to pass the right StoredData.
# noinspection PyTypeChecker
reward = im.handle_refund(bad_contributor_address, None, 0, False, None)
balances.send(im.owner, bad_contributor_address, reward)
# General checks that should be true for a market with a reasonably sensitive model.
self.assertLess(balances[im.owner], total_bounty,
f"Some of the bounty should be distributed.\n"
f"Balances: {balances.get_all()}")
self.assertLess(0, balances[im.owner])
self.assertGreater(total_deposits[good_contributor_address], 0)
self.assertGreater(total_deposits[bad_contributor_address], 0)
# The bad contributor profited because they reported the good contributor.
self.assertGreater(balances[bad_contributor_address], initial_bad_balance)
self.assertLess(balances[good_contributor_address], initial_good_balance)
self.assertLess(balances[good_contributor_address], balances[bad_contributor_address])
self.assertLessEqual(balances[bad_contributor_address] - balances[good_contributor_address],
total_bounty)
self.assertEqual(initial_good_balance + initial_bad_balance + total_bounty,
balances[good_contributor_address] + balances[bad_contributor_address] +
balances[im.owner],
"Should be a zero-sum.")
self.assertEqual(initial_good_balance - total_deposits[good_contributor_address],
balances[good_contributor_address],
"The good contributor should lose all of their deposits.")
def test_market_like_original_paper(self):
inj = Injector([
SimpleDataModule,
LoggingModule,
PerceptronModule,
PredictionMarketImModule(
allow_greater_deposit=False,
group_contributions=False,
reset_model_during_reward_phase=False,
),
])
balances = inj.get(Balances)
data = inj.get(DataLoader)
im = cast(PredictionMarket, inj.get(IncentiveMechanism))
im.owner = 'owner'
assert isinstance(im, PredictionMarket)
init_train_data_portion = 0.2
initializer_address = 'initializer'
total_bounty = 100_000
balances.initialize(initializer_address, total_bounty)
good_contributor_address = 'good_contributor'
initial_good_balance = 10_000
balances.initialize(good_contributor_address, initial_good_balance)
bad_contributor_address = 'bad_contributor'
initial_bad_balance = 10_000
balances.initialize(bad_contributor_address, initial_bad_balance)
(x_train, y_train), (x_test, y_test) = data.load_data()
init_idx = int(len(x_train) * init_train_data_portion)
assert init_idx > 0
x_init_data, y_init_data = x_train[:init_idx], y_train[:init_idx]
x_remaining, y_remaining = x_train[init_idx:], y_train[init_idx:]
# Split test set into pieces.
num_pieces = 10
test_dataset_hashes, test_sets = im.get_test_set_hashes(num_pieces, x_test, y_test)
# Ending criteria:
min_length_s = 100
min_num_contributions = min(len(x_remaining), 100)
# Commitment Phase
self.assertIsNone(im.state)
im.model.init_model(x_init_data, y_init_data)
hashes_split = 3
test_reveal_index = im.initialize_market(Msg(initializer_address, total_bounty),
test_dataset_hashes[:hashes_split],
min_length_s, min_num_contributions)
assert 0 <= test_reveal_index < len(test_dataset_hashes)
self.assertEqual(MarketPhase.INITIALIZATION, im.state)
test_reveal_index = im.add_test_set_hashes(Msg(initializer_address, 0), test_dataset_hashes[hashes_split:])
assert 0 <= test_reveal_index < len(test_dataset_hashes)
self.assertEqual(MarketPhase.INITIALIZATION, im.state)
im.reveal_init_test_set(test_sets[test_reveal_index])
self.assertEqual(MarketPhase.PARTICIPATION, im.state)
# Participation Phase
value = 100
total_deposits = defaultdict(float)
for i in range(min_num_contributions):
data = x_remaining[i]
classification = y_remaining[i]
if i % 2 == 0:
contributor = good_contributor_address
else:
contributor = bad_contributor_address
classification = 1 - classification
cost, _ = im.handle_add_data(contributor, value, data, classification)
self.assertEqual(im.min_stake, cost, "Cost should be the minimum stake because of the options passed in.")
balances.send(contributor, im.owner, cost)
total_deposits[contributor] += cost
# Reward Phase
self.assertEqual(MarketPhase.PARTICIPATION, im.state)
im.end_market()
self.assertEqual(MarketPhase.REVEAL_TEST_SET, im.state)
for i, test_set_portion in enumerate(test_sets):
if i != test_reveal_index:
im.verify_next_test_set(test_set_portion)
self.assertEqual(MarketPhase.REWARD_RESTART, im.state)
while im.remaining_bounty_rounds > 0:
im.process_contribution()
# Collect rewards.
self.assertEqual(MarketPhase.REWARD_COLLECT, im.state)
for contributor in [good_contributor_address, bad_contributor_address]:
# Don't need to pass the right StoredData.
# noinspection PyTypeChecker
reward = im.handle_refund(contributor, None, 0, False, None)
balances.send(im.owner, contributor, reward)
self.assertGreater(total_deposits[good_contributor_address], 0)
self.assertGreater(total_deposits[bad_contributor_address], 0)
# General checks that should be true for a market with a reasonably sensitive model.
self.assertLess(balances[im.owner], total_bounty,
f"Some of the bounty should be distributed.\n"
f"Balances: {balances.get_all()}")
self.assertLess(0, balances[im.owner])
# Sometimes the bad contributor happens to get some value but not much.
self.assertAlmostEqual(balances[bad_contributor_address], initial_bad_balance, delta=2,
msg=f"The bad contributor should lose funds.\n"
f"Balances: {balances.get_all()}")
self.assertGreater(balances[good_contributor_address], initial_good_balance)
self.assertLess(balances[bad_contributor_address], balances[good_contributor_address])
self.assertLessEqual(balances[good_contributor_address] - balances[bad_contributor_address],
total_bounty)
self.assertEqual(initial_good_balance + initial_bad_balance + total_bounty,
balances[good_contributor_address] + balances[bad_contributor_address] +
balances[im.owner],
"Should be a zero-sum.")
def setUpClass(cls):
inj = Injector([
DefaultCollaborativeTrainerModule,
LoggingModule,
PerceptronModule,
StakeableImModule,
])
cls.balances = inj.get(Balances)
cls.decai = inj.get(CollaborativeTrainer)
cls.time_method = inj.get(TimeMock)
cls.good_address = 'sender'
initial_balance = 1E6
cls.balances.initialize(cls.good_address, initial_balance)
msg = Msg(cls.good_address, cls.balances[cls.good_address])
X = np.array([
# Initialization Data
[0, 0, 0],
[1, 1, 1],
# Data to Add
[0, 0, 1],
[0, 1, 0],
[0, 1, 1],
[1, 0, 0],
[1, 0, 1],
[1, 1, 0],
])
y = np.array([_ground_truth(x) for x in X])
cls.decai.model.init_model(np.array([X[0, :], X[1, :]]),
np.array([y[0], y[1]]))
score = cls.decai.model.evaluate(X, y)
assert score != 1, "Model shouldn't fit the data yet."
# Add all data.
first_added_time = None
for i in range(X.shape[0]):
x = X[i]
cls.time_method.set_time(cls.time_method() + 1)
if first_added_time is None:
first_added_time = cls.time_method()
cls.decai.add_data(msg, x, y[i])
for _ in range(1000):
score = cls.decai.model.evaluate(X, y)
if score >= 1:
break
i = random.randint(0, X.shape[0] - 1)
x = X[i]
cls.time_method.set_time(cls.time_method() + 1)
cls.decai.add_data(msg, x, y[i])
assert score == 1, "Model didn't fit the data."
bal = cls.balances[msg.sender]
assert bal < initial_balance, "Adding data should have a cost."
# Make sure sender has some good data refunded so that they can report data later.
cls.time_method.set_time(cls.time_method() +
cls.decai.im.refund_time_s + 1)
cls.decai.refund(msg, X[0], y[0], first_added_time)
assert cls.balances[msg.sender] > bal, "Refunding should return value."
def task():
(x_train, y_train), (x_test, y_test) = \
self._data_loader.load_data(train_size=train_size, test_size=test_size)
init_idx = int(len(x_train) * init_train_data_portion)
self._logger.info("Initializing model with %d out of %d samples.",
init_idx, len(x_train))
x_init_data, y_init_data = x_train[:init_idx], y_train[:init_idx]
x_remaining, y_remaining = x_train[init_idx:], y_train[init_idx:]
self._decai.model.init_model(x_init_data, y_init_data)
if self._logger.isEnabledFor(logging.DEBUG):
s = self._decai.model.evaluate(x_init_data, y_init_data)
self._logger.debug("Initial training data evaluation: %s", s)
s = self._decai.model.evaluate(x_remaining, y_remaining)
self._logger.debug("Remaining training data evaluation: %s", s)
self._logger.info("Evaluating initial model.")
accuracy = self._decai.model.evaluate(x_test, y_test)
self._logger.info("Initial test set accuracy: %0.2f%%", accuracy * 100)
t = self._time()
doc.add_next_tick_callback(
partial(plot_accuracy_cb, t=t, a=accuracy))
q = PriorityQueue()
random.shuffle(agents)
for agent in agents:
self._balances.initialize(agent.address, agent.start_balance)
q.put((self._time() + agent.get_next_wait_s(), agent))
doc.add_next_tick_callback(
partial(plot_cb, agent=agent, t=t, b=agent.start_balance))
unclaimed_data = []
next_data_index = 0
next_accuracy_plot_time = 1E4
desc = "Processing agent requests"
with tqdm(desc=desc,
unit_scale=True, mininterval=2, unit=" requests",
total=len(x_remaining),
) as pbar:
while not q.empty():
# For now assume sending a transaction (editing) is free (no gas)
# since it should be relatively cheaper than the deposit required to add data.
# It may not be cheaper than calling `report`.
if next_data_index >= len(x_remaining):
if not continuous_evaluation or len(unclaimed_data) == 0:
break
current_time, agent = q.get()
update_balance_plot = False
if current_time > next_accuracy_plot_time:
self._logger.debug("Evaluating.")
next_accuracy_plot_time += accuracy_plot_wait_s
accuracy = self._decai.model.evaluate(x_test, y_test)
doc.add_next_tick_callback(
partial(plot_accuracy_cb, t=current_time, a=accuracy))
if continuous_evaluation:
self._logger.debug("Unclaimed data: %d", len(unclaimed_data))
pbar.set_description(f"{desc} ({len(unclaimed_data)} unclaimed)")
with open(save_path, 'w') as f:
json.dump(save_data, f, separators=(',', ':'))
if os.path.exists(plot_save_path):
os.remove(plot_save_path)
export_png(plot, plot_save_path)
self._time.set_time(current_time)
balance = self._balances[agent.address]
if balance > 0 and next_data_index < len(x_remaining):
# Pick data.
x, y = x_remaining[next_data_index], y_remaining[next_data_index]
if agent.calls_model:
# Only call the model if it's good.
if random.random() < accuracy:
update_balance_plot = True
self._decai.predict(Msg(agent.address, agent.pay_to_call), x)
else:
if not agent.good:
y = 1 - y
if agent.prob_mistake > 0 and random.random() < agent.prob_mistake:
y = 1 - y
# Bad agents always contribute.
# Good agents will only work if the model is doing well.
# Add a bit of chance they will contribute since 0.85 accuracy is okay.
if not agent.good or random.random() < accuracy + 0.15:
value = agent.get_next_deposit()
if value > balance:
value = balance
msg = Msg(agent.address, value)
try:
self._decai.add_data(msg, x, y)
# Don't need to plot every time. Plot less as we get more data.
update_balance_plot = next_data_index / len(x_remaining) + 0.1 < random.random()
balance = self._balances[agent.address]
if continuous_evaluation:
unclaimed_data.append((current_time, agent, x, y))
next_data_index += 1
pbar.update()
except RejectException:
# Probably failed because they didn't pay enough which is okay.
# Or if not enough time has passed since data was attempted to be added
# which is okay too because a real contract would reject this
# because the smallest unit of time we can use is 1s.
if self._logger.isEnabledFor(logging.DEBUG):
self._logger.exception("Error adding data.")
if balance > 0:
q.put((current_time + agent.get_next_wait_s(), agent))
claimed_indices = []
for i in range(len(unclaimed_data)):
added_time, adding_agent, x, classification = unclaimed_data[i]
if current_time - added_time < self._decai.im.refund_time_s:
break
if next_data_index >= len(x_remaining) \
and current_time - added_time < self._decai.im.any_address_claim_wait_time_s:
break
balance = self._balances[agent.address]
msg = Msg(agent.address, balance)
if current_time - added_time > self._decai.im.any_address_claim_wait_time_s:
# Attempt to take the entire deposit.
try:
self._decai.report(msg, x, classification, added_time, adding_agent.address)
update_balance_plot = True
except RejectException:
if self._logger.isEnabledFor(logging.DEBUG):
self._logger.exception("Error taking reward.")
elif adding_agent.address == agent.address:
try:
self._decai.refund(msg, x, classification, added_time)
update_balance_plot = True
except RejectException:
if self._logger.isEnabledFor(logging.DEBUG):
self._logger.exception("Error getting refund.")
else:
try:
self._decai.report(msg, x, classification, added_time, adding_agent.address)
update_balance_plot = True
except RejectException:
if self._logger.isEnabledFor(logging.DEBUG):
self._logger.exception("Error taking reward.")
stored_data = self._decai.data_handler.get_data(x, classification,
added_time, adding_agent.address)
if stored_data.claimable_amount <= 0:
claimed_indices.append(i)
for i in claimed_indices[::-1]:
unclaimed_data.pop(i)
if update_balance_plot:
balance = self._balances[agent.address]
doc.add_next_tick_callback(
partial(plot_cb, agent=agent, t=current_time, b=balance))
self._logger.info("Done going through data.")
if continuous_evaluation:
pbar.set_description(f"{desc} ({len(unclaimed_data)} unclaimed)")
if isinstance(self._decai.im, PredictionMarket):
self._time.add_time(agents[0].get_next_wait_s())
self._decai.im.end_market()
for i, test_set_portion in enumerate(pm_test_sets):
if i != self._decai.im.test_reveal_index:
self._decai.im.verify_next_test_set(test_set_portion)
with tqdm(desc="Processing contributions",
unit_scale=True, mininterval=2, unit=" contributions",
total=self._decai.im.get_num_contributions_in_market(),
) as pbar:
finished_first_round_of_rewards = False
while self._decai.im.remaining_bounty_rounds > 0:
self._time.add_time(agents[0].get_next_wait_s())
self._decai.im.process_contribution()
pbar.update()
if not finished_first_round_of_rewards:
accuracy = self._decai.im.prev_acc
# If we plot too often then we end up with a blob instead of a line.
if random.random() < 0.1:
doc.add_next_tick_callback(
partial(plot_accuracy_cb, t=self._time(), a=accuracy))
if self._decai.im.state == MarketPhase.REWARD_RESTART:
finished_first_round_of_rewards = True
if self._decai.im.reset_model_during_reward_phase:
# Update the accuracy after resetting all data.
accuracy = self._decai.im.prev_acc
else:
# Use the accuracy after training with all data.
pass
doc.add_next_tick_callback(
partial(plot_accuracy_cb, t=self._time(), a=accuracy))
pbar.total += self._decai.im.get_num_contributions_in_market()
self._time.add_time(self._time() * 0.001)
for agent in agents:
balance = self._balances[agent.address]
market_bal = self._decai.im._market_balances[agent.address]
self._logger.debug("\"%s\" market balance: %0.2f Balance: %0.2f",
agent.address, market_bal, balance)
doc.add_next_tick_callback(
partial(plot_cb, agent=agent, t=self._time(), b=max(balance + market_bal, 0)))
self._time.add_time(self._time() * 0.02)
for agent in agents:
msg = Msg(agent.address, 0)
# Find data submitted by them.
data = None
for key, stored_data in self._decai.data_handler:
if stored_data.sender == agent.address:
data = key[0]
break
if data is not None:
self._decai.refund(msg, np.array(data), stored_data.classification, stored_data.time)
balance = self._balances[agent.address]
doc.add_next_tick_callback(
partial(plot_cb, agent=agent, t=self._time(), b=balance))
self._logger.info("Balance for \"%s\": %.2f (%+.2f%%)",
agent.address, balance,
(balance - agent.start_balance) / agent.start_balance * 100)
else:
self._logger.warning("No data submitted by \"%s\" was found."
"\nWill not update it's balance.", agent.address)
self._logger.info("Done issuing rewards.")
accuracy = self._decai.model.evaluate(x_test, y_test)
doc.add_next_tick_callback(
partial(plot_accuracy_cb, t=current_time, a=accuracy))
with open(save_path, 'w') as f:
json.dump(save_data, f, separators=(',', ':'))
if os.path.exists(plot_save_path):
os.remove(plot_save_path)
export_png(plot, plot_save_path)
assert init_idx > 0
x_init_data, y_init_data = x_train[:init_idx], y_train[:init_idx]
x_remaining, y_remaining = x_train[init_idx:], y_train[init_idx:]
# Split test set into pieces.
num_pieces = 10
test_dataset_hashes, test_sets = self._im.get_test_set_hashes(
num_pieces, x_test, y_test)
# Ending criteria:
min_length_s = 1_000
min_num_contributions = len(x_remaining)
self._im.model.init_model(x_init_data, y_init_data)
test_reveal_index = self._im.initialize_market(
Msg(initializer_address, total_bounty), test_dataset_hashes,
min_length_s, min_num_contributions)
assert 0 <= test_reveal_index < len(test_dataset_hashes)
self._im.reveal_init_test_set(test_sets[test_reveal_index])
# Accuracy on hidden test set after training with all training data:
baseline_accuracies = {
100: 0.6210,
200: 0.6173,
1000: 0.7945,
10000: 0.84692,
20000: 0.8484,
}
# Start the simulation.
self._s.simulate(
