def test_register_producer():
# Make a manager with no neighbourhood (not needed for testing)
sim = Simulator(dict(), None)
manager = Manager(sim)
producer = Producer("id", manager)
manager.register_producer("id", producer)
assert producer in manager.producers
def test_terminate_consumers():
sim = Simulator(dict(), None)
manager = Manager(sim)
job = Job(est=0, lst=0, id="job_id", load_profile=pd.Series())
consumer = Consumer.start(producers=[], job=job, manager=manager)
manager.consumers = [consumer]
manager.terminate_consumers()
assert manager.consumers == []
def __init__(self, config, number):
self.logger = logging.getLogger("src.Simulator#" + str(number))
self.save_name = os.path.join(ROOT_DIR, "tmp", "run" + str(number))
# Default 1000 simulated seconds per second. 1 day = 86 seconds to run.
factor = config["timefactor"] if "timefactor" in config else 0.001
self.neighbourhood = simpy.rt.RealtimeEnvironment(factor=factor,
strict=False)
# Default to one day
self.end_time = config["length"] if "length" in config else 86400
# A dictionary over every timeout event containing a contract and the id to fetch that event
self.active_contracts = {}
# A list of completed contracts
self.fulfilled_contracts = []
# A dict of production profiles
self.production_profiles = {}
# Name of the configuration to be ran. Defaults to "test"
config_name = config[
"neighbourhood"] if "neighbourhood" in config else "test"
self.DATA_DIR = os.path.join(ROOT_DIR, "input", config_name)
# The manager that is simulated. Every new load and prediction should be sent to it.
self.manager = Manager(self, config)
# Loads in events and normalizes them
events, predictions = self.load_files_from_csv(config_name)
events, predictions = normalize_times(events, predictions)
# Schedule everything
for event in events:
schedule = self.schedule_load(event["timestamp"], event["job"])
simpy.events.Process(self.neighbourhood, schedule)
for prediction in predictions:
schedule = self.schedule_prediction(prediction["timestamp"],
prediction)
simpy.events.Process(self.neighbourhood, schedule)
simpy.events.Process(self.neighbourhood, self.end_simulation())
def test_cancel():
sim = Simulator(dict(), None)
manager = Manager(sim)
producer = Producer("producer_id", manager)
job = Job(est=0, lst=0, id="job_id", load_profile=pd.Series())
consumer = Consumer.start(producers=[producer], job=job, manager=manager)
schedule_object = dict(consumer=consumer, job=job)
producer.schedule.append(schedule_object)
assert (schedule_object in producer.schedule)
producer.cancel(schedule_object)
assert producer.schedule == []
consumer.stop()
def test_request_job_integration():
simulator = MockSimulator()
manager = Manager(simulator, options=dict(algo="SLSQP"))
Producer(id="p1", manager=manager)
manager.new_producer("p1")
prediction = pd.Series(index=[0, 3600, 7200], data=[0.0, 10.0, 20.0])
manager.send_new_prediction(prediction, manager.producers["p1"])
job = Job("j1", 0, 0, pd.Series(index=[0, 1800, 3600],
data=[0.0, 2.5, 5.0]))
manager.new_job(job)
consumer = manager.consumers[0]
consumer._actor.request_producer()
manager.producers["p1"].stop()
assert manager.producer_rankings["p1"] == 9
def test_fulfill_contract():
sim = Simulator(dict(), None)
manager = Manager(sim)
producer = Producer("producer_id", manager)
job = Job(est=0, lst=0, id="job_id", load_profile=pd.Series())
consumer = Consumer.start(producers=[producer], job=job, manager=manager)
schedule_object = dict(consumer=consumer, job=job)
producer.schedule.append(schedule_object)
contract = producer.create_contract(job)
consumer.stop()
assert producer.schedule == [schedule_object]
producer.fulfill_contract(contract)
assert producer.schedule == []
def test_optimize_fifty_fifty_negative():
sim = Simulator(dict(), None)
man = Manager(sim)
producer = Producer("id", man)
producer.optimizer = Optimizer(producer, options=dict(algo="fifty_fifty"))
producer.prediction = pd.Series(index=[0, 1], data=[0.0, 0.0])
for i in range(10):
job = dict(consumer=None, job=Job("id" + str(i), 0, 0, pd.Series(index=[0, 1], data=[0.0, 1.0])))
producer.schedule.append(job)
schedule_time, should_keep = producer.optimizer.optimize()
if not should_keep[-1]:
producer.schedule.pop()
num_accepted = len(producer.schedule)
assert 0 == num_accepted
def test_terminate_producers():
sim = Simulator(dict(), None)
manager = Manager(sim)
p1 = Producer.start("P1", manager)
p2 = Producer.start("P2", manager)
manager.register_producer(p1, "P1")
manager.register_producer(p2, "P2")
manager.terminate_producers()
assert manager.producers == {}
def test_create_contract():
sim = Simulator(dict(), None)
manager = Manager(sim)
producer = Producer("producer_id", manager)
job = Job(est=0, lst=0, id="job_id", load_profile=pd.Series())
expected = dict(id='producer_id;job_id;0',
time=0,
time_of_agreement=0,
load_profile=pd.Series(),
job_id='job_id',
producer_id='producer_id')
actual = producer.create_contract(job)
assert expected['id'] == actual['id']
assert expected['time'] == actual['time']
assert expected['time_of_agreement'] == actual['time_of_agreement']
assert expected['job_id'] == actual['job_id']
assert expected['producer_id'] == actual['producer_id']
def test_update_power_profile():
sim = Simulator(dict(), None)
manager = Manager(sim)
producer = Producer("producer_id", manager)
p1 = pd.Series(index=[0, 3600, 7200, 10800], data=[0.0, 1.0, 2.0, 3.0])
p2 = pd.Series(index=[7200, 10800, 14400, 18000],
data=[5.0, 6.0, 7.0, 8.0])
expected1 = p1
expected2 = pd.Series(index=[0, 3600, 7200, 10800, 14400, 18000],
data=[0.0, 1.0, 6.0, 7.0, 8.0, 9.0])
assert producer.prediction is None
producer.update_power_profile(p1)
assert producer.prediction.equals(expected1)
producer.update_power_profile(p2)
assert producer.prediction.equals(expected2)
def test_optimize_basinhopping():
sim = Simulator(dict(), None)
man = Manager(sim)
producer = Producer("id", man)
job0 = dict(consumer=None, job=Job("id", 0, 500, pd.Series(index=[0, 60, 120], data=[0.0, 5.0, 10.0])))
job1 = dict(consumer=None, job=Job("id", 0, 500, pd.Series(index=[0, 60, 120], data=[0.0, 5.0, 10.0])))
job2 = dict(consumer=None, job=Job("id", 0, 500, pd.Series(index=[0, 60, 120], data=[0.0, 5.0, 10.0])))
job3 = dict(consumer=None, job=Job("id", 0, 500, pd.Series(index=[0, 60, 120], data=[0.0, 5.0, 10.0])))
job4 = dict(consumer=None, job=Job("id", 0, 500, pd.Series(index=[0, 60, 120], data=[0.0, 5.0, 10.0])))
producer.optimizer = Optimizer(producer, options=dict(algo="SLSQP", pen=1.0, tol=1.0, eps=0.01))
producer.schedule = [job0, job1, job2, job3, job4]
producer.prediction = pd.Series(index=[0, 300, 600], data=[0.0, 25.0, 50.0])
schedule_time, should_keep = producer.optimizer.optimize()
schedule = set([int(utils.round_to_nearest_60(x)) for x in schedule_time])
# Assert that we have at least one scheduled time.
assert len(schedule) >= 1
class Simulator:
def __init__(self, config, number):
self.logger = logging.getLogger("src.Simulator#" + str(number))
self.save_name = os.path.join(ROOT_DIR, "tmp", "run" + str(number))
# Default 1000 simulated seconds per second. 1 day = 86 seconds to run.
factor = config["timefactor"] if "timefactor" in config else 0.001
self.neighbourhood = simpy.rt.RealtimeEnvironment(factor=factor,
strict=False)
# Default to one day
self.end_time = config["length"] if "length" in config else 86400
# A dictionary over every timeout event containing a contract and the id to fetch that event
self.active_contracts = {}
# A list of completed contracts
self.fulfilled_contracts = []
# A dict of production profiles
self.production_profiles = {}
# Name of the configuration to be ran. Defaults to "test"
config_name = config[
"neighbourhood"] if "neighbourhood" in config else "test"
self.DATA_DIR = os.path.join(ROOT_DIR, "input", config_name)
# The manager that is simulated. Every new load and prediction should be sent to it.
self.manager = Manager(self, config)
# Loads in events and normalizes them
events, predictions = self.load_files_from_csv(config_name)
events, predictions = normalize_times(events, predictions)
# Schedule everything
for event in events:
schedule = self.schedule_load(event["timestamp"], event["job"])
simpy.events.Process(self.neighbourhood, schedule)
for prediction in predictions:
schedule = self.schedule_prediction(prediction["timestamp"],
prediction)
simpy.events.Process(self.neighbourhood, schedule)
simpy.events.Process(self.neighbourhood, self.end_simulation())
# Functions that schedule the events. Simpy-specific
def schedule_load(self, delay, job):
event = simpy.events.Timeout(self.neighbourhood,
delay=delay,
value=job)
yield event
self.new_load(job)
def schedule_prediction(self, delay, prediction):
event = simpy.events.Timeout(self.neighbourhood,
delay=delay,
value=prediction)
yield event
self.new_prediction(prediction)
def stop(self):
self.manager.save_producer_scores()
self.manager.terminate_producers()
self.manager.terminate_consumers()
# Starts a new consumer event
# Will call corresponding function in manager
def new_load(self, job):
self.logger.info("A new load happened! Time=" +
str(self.neighbourhood.now))
self.manager.new_job(job)
# Sends out a new weather prediction (every 6 hours)
# Will call corresponding function in manager
def new_prediction(self, prediction):
self.logger.info("A new prediction happened! Time=" +
str(self.neighbourhood.now))
self.manager.new_prediction(prediction)
# Register a contract between a consumer and producer
# It is added as a Simpy event with a timeout until it should start
def register_contract(self, contract):
self.neighbourhood.process(self.register_new_contract(contract))
def register_new_contract(self, contract):
self.logger.info("Registering contract. Time = " +
str(self.neighbourhood.now))
id = contract["id"]
contract_time = contract["time"]
delay = contract_time - self.neighbourhood.now
try:
event = self.neighbourhood.timeout(delay)
self.active_contracts[id] = event
yield event
self.fulfill_contract(contract)
except simpy.Interrupt as i:
self.logger.info("Contract with id " + str(id) +
" was interrupted.")
# Cancel a contract between a consumer and producer
# Makes sure that invalid contracts won't be executed
def cancel_contract(self, contract):
id = contract["id"]
event = self.active_contracts[id]
event.interrupt()
# Call when a contract is fulfilled.
def fulfill_contract(self, contract):
self.logger.info("Contract fulfilled. Contract id: " +
str(contract["id"]) + " Time: " +
str(self.neighbourhood.now))
if contract['producer_id'] != 'grid':
producer = self.manager.producers[contract['producer_id']]
producer._actor.fulfill_contract(contract)
self.fulfilled_contracts.append(contract)
# Loading functions
def load_files_from_csv(self, name):
events = get_events_from_csv(name)
predictions = get_predictions_from_csv(name)
return events, predictions
# Starts the simulation
def start(self):
self.logger.info("Starting simulation")
self.neighbourhood.run(until=self.end_time)
def get_output(self):
self.update_production_profiles()
return self.fulfilled_contracts, self.production_profiles
def update_production_profiles(self):
self.production_profiles = self.manager.get_production_profiles()
def end_simulation(self):
event = simpy.events.Timeout(self.neighbourhood,
delay=self.end_time - 1)
yield event
self.logger.info("Finishing run")
self.save_output()
def save_output(self):
contracts, profiles = self.get_output()
with open(self.save_name + '.pkl', 'wb') as f:
pickle.dump((contracts, profiles), f)
self.stop()
from backend import run_example
from backend.db_connection2 import *
from backend.manager import Manager
from python_query_aws import script_turk_performance
from python_query_aws import script_current_task_results
app = Flask(__name__, static_folder="./dist/static", template_folder="./dist")
#Task_Crowd_Source_Test
#conn, meta, session = connect("postgres", "1234", db="match_girl_suggestions") #temp2
conn, meta, session = connect("postgres", "1234",
db="Task_Crowd_Source_Test") #temp2
#meta.drop_all(bind=conn) # clear everything
#Base.metadata.create_all(conn)
manager = Manager(session, max_time=6) #in minutes
if type(os.environ.get('DATABASE_URL')) != type(None):
manager._minimum_work_time = 4
manager._effort_ratio = 0
@app.route('/', defaults={'path': ''})
@app.route('/<path:path>')
def index(path):
return render_template("index.html")
@app.route('/api/get_results', methods=["POST"])
def dashboard():
'''