def simulate(cycle=1, phase=0): db = DBAccess(env.DB_RESULT_NAME) db.clear_collection(env.DB_GLOBAL_RESULT_COLLECTION_NAME) db.clear_collection(env.DB_DETAILED_RESULT_COLLECTION_NAME) sim = Simulator() granulation_solver = GranulationSolver(sim.nodes, sim.sales_plan) granulation_results = granulation_solver.launch_granulation_solver()
def JESA_DropAll(): db_access = DBAccess(env.DB_NAME) for collection in COLLECTIONS_CACHE: db_access.clear_collection(collection) collections = list(COLLECTIONS_CACHE) COLLECTIONS_CACHE.clear() return "|".join(collections)
def save_data(): try: reset_db_name(request.json['db_name']) records = request.json['table'] db = DBAccess(env.DB_NAME) name_ = trim_collection_name(request.json['name']) db.clear_collection(name_) db.save_to_db(name_, records) return jsonify(status=env.HTML_STATUS.OK) except Exception as e: logger.error("Cannot save data: %s" % e) return jsonify(status=env.HTML_STATUS.ERROR)
def get_best_global_scenarios(quantile_step): db = DBAccess(env.DB_RESULT_NAME) db.clear_collection(env.DB_DETAILED_BEST_RESULT_COLLECTION_NAME) scenarios = db.get_records(env.DB_GLOBAL_RESULT_COLLECTION_NAME, {}).sort([("Cost PV", DESCENDING)]) step = int(quantile_step * scenarios.count()) representative_scenarios = [ scenarios.skip(step * i)[0] for i in range(0, int(scenarios.count() / step)) ] db.save_to_db_no_check(env.DB_GLOBAL_BEST_RESULT_COLLECTION_NAME, representative_scenarios)
def JESA_UploadTable(name, table, db_name="mine2farm"): records = [] header = list(table) for row in table.iterrows(): record = {} for h in header: record[h] = row[1][h] records.append(record) env.DB_NAME = db_name db_access = DBAccess(env.DB_NAME) name_ = trim_collection_name(name) db_access.clear_collection(name_) db_access.save_to_db(name_, records) COLLECTIONS_CACHE.add(name_) return "%s Saved! @%s" % (name_, datetime.now().strftime("%H:%M:%S"))
def simulate(cycle=1, phase=0, use_db=False): if use_db: db = DBAccess(env.DB_RESULT_NAME) db.clear_collection(env.DB_GLOBAL_RESULT_COLLECTION_NAME) db.clear_collection(env.DB_DETAILED_RESULT_COLLECTION_NAME) scenarios_global, scenarios_details = Simulator().simulate( cycle, phase, logistics_lp=False) if use_db: for scenario in scenarios_global: db.save_to_db_no_check(env.DB_GLOBAL_RESULT_COLLECTION_NAME, scenarios_global[scenario]) for scenario in scenarios_details: json_data = json.dumps(NodeJSONEncoder().encode( scenarios_details[scenario])) data = json.loads(json.loads(json_data)) db.save_to_db_no_check(env.DB_DETAILED_RESULT_COLLECTION_NAME, data)
def get_best_detailed_scenarios(quantile_step): db = DBAccess(env.DB_RESULT_NAME) db.clear_collection(env.DB_DETAILED_BEST_RESULT_COLLECTION_NAME) scenarios = db.get_fields(env.DB_GLOBAL_RESULT_COLLECTION_NAME, { "Cost PV": 1, "Scenario": 1 }, [("Cost PV", DESCENDING)]) step = int(quantile_step * scenarios.count()) points = [ scenarios.skip(step * i)[0]["Scenario"] for i in range(0, int(scenarios.count() / step)) ] representative_scenarios = db.get_records( env.DB_DETAILED_RESULT_COLLECTION_NAME, {"Scenario": { "$in": points }}) db.save_to_db_no_check(env.DB_DETAILED_BEST_RESULT_COLLECTION_NAME, representative_scenarios)
def serve(self, cycle): """ Crating tasks and sending to broker :param cycle: :return: """ # reset scenarios table db = DBAccess(env.DB_RESULT_NAME) db.clear_collection(env.DB_GLOBAL_RESULT_COLLECTION_NAME) db.clear_collection(env.DB_DETAILED_RESULT_COLLECTION_NAME) db.clear_collection(env.DB_SENSITIVITY_COLLECTION_NAME) db.create_index( env.DB_GLOBAL_RESULT_COLLECTION_NAME, [("Cost PV", pymongo.DESCENDING), ("Scenario", pymongo.ASCENDING)] ) db.create_index( env.DB_DETAILED_RESULT_COLLECTION_NAME, [("Scenario", pymongo.ASCENDING)] ) db.save_to_db_no_check(env.DB_SENSITIVITY_COLLECTION_NAME, {"NH3": 0, "ACS": 0, "HCl": 0, "Raw water": 0, "Electricity": 0, "K09": 0, "Rock": 0, "Scenario": -1}) data = [] for i in range(cycle): data.append(json.dumps({ "cycle": cycle, "phase": i, "db_name": env.DB_NAME, "logistics_lp": env.LOGISTICS_LP })) broker = Broker(env.RABBITMQ_SIMULATOR_QUEUE_NAME) broker.publish(data)
from app.model.Simulator import * import cProfile from multiprocessing import Pool, TimeoutError, Process from app.data.DBAccess import DBAccess from flask import Response, render_template from flask import Flask import dash import dash_bootstrap_components as dbc import dash_core_components as dcc import dash_html_components as html from dash.dependencies import Input, Output server = Flask(__name__) db = DBAccess(env.DB_RESULT_NAME) db.clear_collection(env.DB_GLOBAL_RESULT_COLLECTION_NAME) db.clear_collection(env.DB_DETAILED_RESULT_COLLECTION_NAME) simulator = Simulator() @server.route('/') def inddex(): return 'Test' app = dash.Dash(__name__, server=server, routes_pathname_prefix='/dash/', external_stylesheets=[dbc.themes.BOOTSTRAP] ) app.layout = html.Div( [ dbc.Progress(id="progress", value=0, striped=True, animated=True),
def get_best_scenarios(quantile_step, db_name="mine2farm"): update_cache(db_name, -1) try: time_start = datetime.datetime.now().strftime("%d/%m/%y %H:%M:%S") # insert status of best scenarios "running" db_history = DBAccess(env.MONITORING_DB_NAME) query_insert = { 'time_start': time_start, 'db_name': db_name, 'quantile_step': quantile_step, 'status': -1 } _id = db_history.save_to_db_no_check( env.MONITORING_COLLECTION_HISTORY_BEST_NAME, query_insert) # get best representative scenarios quantile_step = quantile_step / 100. reset_db_name(db_name) db = DBAccess(env.DB_RESULT_NAME) logger.info("Deleting best collections from DB") db.clear_collection(env.DB_GLOBAL_BEST_RESULT_COLLECTION_NAME) db.clear_collection(env.DB_DETAILED_BEST_RESULT_COLLECTION_NAME) scenarios = db.get_records(env.DB_GLOBAL_RESULT_COLLECTION_NAME, {}).sort([("Cost PV", DESCENDING)]) scenarios_count = scenarios.count() step = int(quantile_step * scenarios_count) # save to db if step == 0: # all scenarios are concerned logger.info("Moving all scenarios to best collections") db.copy_to_collection(env.DB_GLOBAL_RESULT_COLLECTION_NAME, env.DB_GLOBAL_BEST_RESULT_COLLECTION_NAME) db.copy_to_collection(env.DB_DETAILED_RESULT_COLLECTION_NAME, env.DB_DETAILED_BEST_RESULT_COLLECTION_NAME) details_count = db.count( env.DB_DETAILED_BEST_RESULT_COLLECTION_NAME) else: # filter on specific scenarios representative_scenario_ids = [ scenarios.skip(step * i)[0]["Scenario"] for i in range(0, int(scenarios_count / step)) ] logger.info("List of selected best scenarios: %s" % representative_scenario_ids) # simulate scenarios_global, scenarios_details = \ Simulator().simulate(scenarios_filter=representative_scenario_ids, logistics_lp=env.LOGISTICS_LP) # save for scenario in scenarios_global: db.save_to_db_no_check( env.DB_GLOBAL_BEST_RESULT_COLLECTION_NAME, scenarios_global[scenario]) for scenario in scenarios_details: json_data = json.dumps(NodeJSONEncoder().encode( scenarios_details[scenario])) data = json.loads(json.loads(json_data)) db.save_to_db_no_check( env.DB_DETAILED_BEST_RESULT_COLLECTION_NAME, data) details_count = len(scenarios_details) # status update query_insert['global_count'] = scenarios_count query_insert['detailed_count'] = details_count filter_ = {'_id': ObjectId(_id)} db_history.update_record( collection=env.MONITORING_COLLECTION_HISTORY_BEST_NAME, filter_=filter_, data=query_insert) # raw materials sensitivities logger.info("Running sensitivity over raw materials") db.clear_collection(env.DB_SENSITIVITY_COLLECTION_NAME) raw_materials_df = Driver().get_data("raw_materials") shocks = {} for raw_material in raw_materials_df: item = raw_material["Item"] shocks[item] = 1 scenarios_df = pd.DataFrame(Driver().get_results( env.DB_GLOBAL_BEST_RESULT_COLLECTION_NAME)) scenarios_dic = Utils.get_scenario_from_df(scenarios_df) risk_engine = RiskEngine() for scenario_id in scenarios_dic: deltas = risk_engine.compute_delta(scenarios_dic[scenario_id], shocks, with_logistics=env.LOGISTICS_LP) deltas['Scenario'] = int(scenario_id) db.save_to_db_no_check(env.DB_SENSITIVITY_COLLECTION_NAME, deltas) # status update query_insert['time_end'] = datetime.datetime.now().strftime( "%d/%m/%y %H:%M:%S") query_insert['status'] = 0 filter_ = {'_id': ObjectId(_id)} db_history.update_record( collection=env.MONITORING_COLLECTION_HISTORY_BEST_NAME, filter_=filter_, data=query_insert) update_cache(db_name, 0) except Exception as e: logger.error("Best scenarios failed") update_cache(db_name, 0)
from app.config.env_func import reset_db_name from app.config.env import DB_SENSITIVITY_COLLECTION_NAME from app.data.DBAccess import DBAccess import pandas as pd import app.config.env as env from app.data.Client import Driver from app.risk.RiskEngine import RiskEngine from tqdm import tqdm import json from app.tools import Utils if __name__ == "__main__": reset_db_name('mine2farm') db = DBAccess(env.DB_RESULT_NAME) db.clear_collection(DB_SENSITIVITY_COLLECTION_NAME) raw_materials_sensitivity = [] raw_materials_df = Driver().get_data("raw_materials") shocks = {} for raw_material in raw_materials_df: item = raw_material["Item"] shocks[item] = 1 #scenarios_df = pd.DataFrame(Driver().get_results(DB_GLOBAL_BEST_RESULT_COLLECTION_NAME)) scenarios_df = pd.read_csv(env.APP_FOLDER + "outputs/global.csv") scenarios_dic = Utils.get_scenario_from_df(scenarios_df) for scenario_id in scenarios_dic: risk_engine = RiskEngine() deltas = risk_engine.compute_delta(scenarios_dic[scenario_id], shocks) deltas['Scenario'] = int(scenario_id) db.save_to_db_no_check(DB_SENSITIVITY_COLLECTION_NAME, deltas)