def runExperiment(filter, removedWordFile, modelfile, resultFile, vocabFile):
data = FileReader.getData(fileToRead, types, 2018)
vocabDict = data[0]
catDict = data[1]
sortedVocabDict = OrderedDict(sorted(vocabDict.items()))
removedWords = Processing.filterAndRemove(sortedVocabDict, catDict, filter)
Utility.outputListToFile(sortedVocabDict.keys(), vocabFile)
Utility.outputListToFile(removedWords, removedWordFile)
Processing.smoothAllCategories(vocabDict, catDict, smoothingValue)
ModelGenerator.outputModelToFile(modelfile, sortedVocabDict, catDict,
smoothingValue)
model = FileReader.getModel(modelfile)
testData = FileReader.getTestData(fileToRead, 2019)
return ModelGenerator.outputBaysianClassification(resultFile, testData,
model, types,
sortedVocabDict, catDict)
def main():
model_path = os.path.join(directory, f"%s.py")
criteria_path = os.path.join(directory, f"%sCriteria.py")
mapper_path = os.path.join(directory, f"%sMapper.py")
for table_name in tables:
model_name = ''.join(
[word[0].upper() + word[1:] for word in table_name.split('_')])
if switch[0]:
ModelGenerator.generate(
mysql=mysql,
table_name=table_name,
model_name=model_name,
model_path=model_path,
)
if switch[1]:
CriteriaGenerator.generate(
model_name=model_name,
model_path=model_path,
criteria_path=criteria_path,
)
if switch[2]:
MapperGenerator.generate(table_name=table_name,
model_name=model_name,
mapper_path=mapper_path)
with open(os.path.join(directory, '__init__.py'), 'a') as f:
if switch[0]:
f.write(f"from .{model_name} import {model_name}\n")
if switch[1]:
f.write(
f"from .{model_name}Criteria import {model_name}Criteria\n"
)
if switch[2]:
f.write(
f"from .{model_name}Mapper import {model_name}Mapper\n")
def RuntTest(seed):
#Init random generator
net.NetworkGraph.SetRandomSeed(seed)
#Define flow bounds
FLOW_BOUNDS = (0.001, 3)
CAPACITY_BOUNDS = (0.3, 1)
#Network
world_init_data = [{
'NodesNumber': 3,
'EdgesNumber': 8,
'ExternalEdgesNumber': 3
}, {
'NodesNumber': 6,
'EdgesNumber': 4,
'ExternalEdgesNumber': 2
}, {
'NodesNumber': 4,
'EdgesNumber': 6,
'ExternalEdgesNumber': 1
}, {
'NodesNumber': 1,
'EdgesNumber': 3,
'ExternalEdgesNumber': 1
}]
network = net.NetworkGraph.GenerateSmallWorld(world_init_data,
*CAPACITY_BOUNDS)
network.GenerateRandomSrcDst(4)
bnb_solver = bb.BranchAndBoundSolverD(network, FLOW_BOUNDS)
bnb_res = bnb_solver.Solve(log=True)
print('BnB:')
print(bnb_res)
#Get network params
n_list = network.GetNodeList()
f_list = network.GetFlowList()
sd_dict = network.GetSrcDstDict()
a_list = network.GetArcList()
c_dict = network.GetCapacityParam()
nmg = mg.RsModelGenerator(mg.QuadObjectiveGenerator(),
mg.RouteConstraintsGenerator(),
mg.LinearCapacityConstraintsGenerator())
rs_model = nmg.CreateCompletRsModel(f_list, sd_dict, n_list, a_list,
c_dict, FLOW_BOUNDS)
solver = sm.CplexSolver()
solver_result = solver.Solve(rs_model.cmodel)
print("SOLVER:")
print(
f"OBJECTIVE: {solver_result['Objective']}, TIME: {solver_result['Time']}"
)
def EstimateObjectvie(rs_model, preserve_feasibility=True):
"""Find shortest pathes and appropriate strains for them."""
#copy model for less interference
cmodel = cp.deepcopy(rs_model.cmodel)
#create and solve shortest path model
sp_amodel = mg.RsModelGenerator(
mg.RouteConstraintsGenerator()).CreateAbstractModel()
def ObjectiveShortestPathRule(model):
return sum(model.FlowRoute[flow, arc] for flow in model.Flows
for arc in model.Arcs)
sp_amodel.RouteObj = pyo.Objective(rule=ObjectiveShortestPathRule,
sense=pyo.minimize)
sp_cmodel = sp_amodel.create_instance(data=rs_model.init_data)
sp_solver = sm.GlpkSolver()
sp_solution = sp_solver.Solve(sp_cmodel, False)
if sp_solution == False:
return None
sp_route = [{
arc: pyo.value(sp_cmodel.FlowRoute[flow, arc])
for arc in sp_cmodel.Arcs
} for flow in sp_cmodel.Flows]
#find feasible flow strains
if preserve_feasibility:
feasible_solution = RecoverFeasibleStrain(rs_model, sp_route,
sm.IpoptSolver())
feasible_solution['Time'] += sp_solver.time
return feasible_solution
#find maximum flow
minimum_capacity = defaultdict(lambda: float('inf'))
for flow, node_s, node_d in cmodel.FlowRoute:
route_val = sp_route[flow][(node_s, node_d)]
cmodel.FlowRoute[(flow, node_s, node_d)].fix(route_val)
if route_val >= (1 - 1e-3):
minimum_capacity[flow] = min(
[minimum_capacity[flow], cmodel.Capacity[node_s, node_d]])
for flow in cmodel.FlowStrain:
cmodel.FlowStrain[flow].fix(minimum_capacity[flow])
obj_val, strain_val, route_val = sm.isolver.ISolver.ExtractSolution(cmodel)
max_solution = {
'Objective': obj_val,
'Strain': strain_val,
'Route': route_val,
'Time': sp_solver.time
}
return max_solution
def __init__(self, network, flow_bounds):
self.log = True
self.BranchingCutsName = 'BranchingCuts'
self.required_precision = 1e-4
# get network params
n_list = network.GetNodeList()
f_list = network.GetFlowList()
sd_dict = network.GetSrcDstDict()
a_list = network.GetArcList()
c_dict = network.GetCapacityParam()
# create model
nmg = mg.RsModelGenerator(mg.QuadObjectiveGenerator(),
mg.RouteConstraintsGenerator(),
mg.NonlinearCapacityConstraintsGenerator())
self.rs_model = nmg.CreateCompletRsModel(f_list, sd_dict, n_list,
a_list, c_dict, flow_bounds)
def RecoverFeasibleStrain(rs_model, routes, solver):
"""Recover optimal and feasible solution for the given routes"""
#update route for required format
routes_updated = {}
for indx, route in enumerate(routes):
ru = {(indx, *k): v for k, v in route.items()}
routes_updated.update(ru)
#copy model for less interference
amodel = cp.deepcopy(rs_model.amodel)
#create basic model
rec_amodel = mg.RsModelGenerator(
mg.NonlinearCapacityConstraintsGenerator()).CreateAbstractModel()
#add objective from the original model
objective_name = amodel.Suffix[amodel.Obj]
if objective_name == 'Linear':
obj_maker = mg.objmk.LinearObjectiveGenerator()
if objective_name == 'Quadratic':
obj_maker = mg.objmk.QuadObjectiveGenerator()
if objective_name == 'Logarithmic':
obj_maker = mg.objmk.LogObjectiveGenerator()
obj_maker(rec_amodel)
#create concrete instance
rec_cmodel = rec_amodel.create_instance(data=rs_model.init_data)
rec_cmodel.FlowUbMax = rs_model.cmodel.FlowUbMax.value
rec_cmodel.FlowLbMin = rs_model.cmodel.FlowLbMin.value
#fix variables
for indx in routes_updated:
rec_cmodel.FlowRoute[indx].fix(routes_updated[indx])
#solve recovery model
ret_val = solver.Solve(rec_cmodel)
#add generated model to the output
if ret_val is not None:
ret_val['Cmodel'] = rec_cmodel
return ret_val
def deserialize(json_str, destdb):
json_objs = json.loads(json_str)
objs = []
for obj in json_objs:
o = ModelGenerator.getModel(destdb, obj['class'])()
for f, v in obj['fields'].iteritems():
o.__dict__[f] = v
objs.append(o)
return objs
def json2schema(schema_json, commit = True, destdb = None):
"""
Creates Database, Table, and Column objects as needed to satisfy the incoming schema.
If the table is already present, assume we are updating: delete all columns and recreate from the schema.
Unless commit is false, call the required sql to create the incoming tables in the destination database.
"""
schema = json.loads(schema_json)
for dbname, table_schema in schema.iteritems():
if destdb:
dbname = destdb
try:
db = Database.objects.get(name=dbname)
except Database.DoesNotExist:
db = Database(name=dbname)
db.save()
for tablename, column_schema in table_schema.iteritems():
try:
table = Table.objects.get(db=db, name=tablename)
for column in Column.objects.filter(table=table):
column.delete()
except Table.DoesNotExist:
table = Table(db=db, name=tablename)
table.save()
for columnname, columntype in column_schema.iteritems():
column = Column(table=table, name=columnname, type=columntype)
column.save()
if commit:
model = ModelGenerator.getModel(dbname, tablename)
cursor = connections[dbname].cursor()
for sql in ModelGenerator.getSQL(model):
cursor.execute(sql)
return None
from flask import Flask
from os import path
from flask.ext.admin import Admin
from flask.ext.sqlalchemy import SQLAlchemy
app = Flask(__name__)
app.config.from_object('config')
db = SQLAlchemy(app)
from views import *
from documents import *
from ModelGenerator import *
modelGenerator = ModelGenerator(app, db)
init_login()
admin = Admin(app, name = 'Admin', index_view=LoginView(name='Login', category='Home'))
admin.add_view(LoggedUserIndexView(name='Home', endpoint='home', category='Home'))
admin.add_view(LogoutView(name='Logout', endpoint='logout', category='Home'))
for key, tupple in modelGenerator.compile().items():
admin.add_view(tupple[1](tupple[0], db.session))
'''
admin.add_view(RaceView(Race, db.session))
admin.add_view(HighlightView(Highlight, db.session))
'''
import InputExtractor as IE
import ModelGenerator as MG
import pandas as pd
if __name__ == '__main__':
#Paso 1: Conseguir las columnas (words + songs)
words = []
songs = []
path = r'H:\Workshop\Machine Learning and AI\Datasets\spotify\jsons'
data = IE.InputExtractor(path)
songs, words = data.Extract_Input()
print("Total songs: " + str(len(songs)))
print("Total words: " + str(len(words)))
#Paso 2: Crear el dataframe de Pandas
model = MG.ModelGenerator(songs, words, path)
df = model.CreateFrame()
#Paso 3: Crear la Neural Network
def RunTest(seed):
#desribe run
print('#######################################################')
print(f'Seed is equal: {seed}')
#define flow bounds
FLOW_BOUNDS = (0.001, 3)
CAPACITY_BOUNDS = (0.3, 1)
#random network
net.NetworkGraph.SetRandomSeed(seed)
#Network small
world_init_data = [ {'NodesNumber': 3, 'EdgesNumber': 8, 'ExternalEdgesNumber': 3 },
{'NodesNumber': 12, 'EdgesNumber': 32, 'ExternalEdgesNumber': 3 },
{'NodesNumber': 6, 'EdgesNumber': 22, 'ExternalEdgesNumber': 2 },
{'NodesNumber': 4, 'EdgesNumber': 12, 'ExternalEdgesNumber': 2 }
]
network = net.NetworkGraph.GenerateSmallWorld(world_init_data, *CAPACITY_BOUNDS)
network.GenerateRandomSrcDst(12)
# #Network medium
# world_init_data = [ {'NodesNumber': 3, 'EdgesNumber': 8, 'ExternalEdgesNumber': 3 },
# {'NodesNumber': 12, 'EdgesNumber': 32, 'ExternalEdgesNumber': 3 },
# {'NodesNumber': 6, 'EdgesNumber': 22, 'ExternalEdgesNumber': 2 },
# {'NodesNumber': 4, 'EdgesNumber': 12, 'ExternalEdgesNumber': 2 },
# {'NodesNumber': 9, 'EdgesNumber': 16, 'ExternalEdgesNumber': 1 },
# {'NodesNumber': 8, 'EdgesNumber': 20, 'ExternalEdgesNumber': 2 },
# {'NodesNumber': 4, 'EdgesNumber': 12, 'ExternalEdgesNumber': 1 },
# {'NodesNumber': 3, 'EdgesNumber': 6, 'ExternalEdgesNumber': 1 },
# ]
# network = net.NetworkGraph.GenerateSmallWorld(world_init_data, *CAPACITY_BOUNDS)
# network.GenerateRandomSrcDst(32)
#get network params
n_list = network.GetNodeList()
f_list = network.GetFlowList()
sd_dict = network.GetSrcDstDict()
a_list = network.GetArcList()
c_dict = network.GetCapacityParam()
nmg = mg.RsModelGenerator(mg.QuadObjectiveGenerator(), mg.RouteConstraintsGenerator(), mg.NonlinearCapacityConstraintsGenerator())
rs_model = nmg.CreateCompletRsModel(f_list, sd_dict, n_list, a_list, c_dict, FLOW_BOUNDS)
start_time_lb = t.perf_counter()
estimate_result_lb = mp.EstimateObjectvie(rs_model, False)
elapsed_time_lb = t.perf_counter() - start_time_lb
objective_lb, strains_lb, routes_lb, time_lb = (estimate_result_lb['Objective'], estimate_result_lb['Strain'],
estimate_result_lb['Route'], estimate_result_lb['Time'] )
print('###################!RESULTS!#############################')
print(f'LB:\nObjective: {objective_lb}, Time: {time_lb}, Total time: {elapsed_time_lb}')
violations = mp.FindConstraintsViolation(rs_model.cmodel, strains_lb, routes_lb)
cc_vn = violations['capacity_constraints'][1]
rc_vn = violations['route_constraints'][1]
if cc_vn == 0 and rc_vn == 0:
print('Feasible')
else:
print(f'Capacity constraint violations number: {cc_vn}')
print(f'Route constraint violations number: {rc_vn}')
print('__________________________________________________________')
start_time_ub = t.perf_counter()
estimate_result_ub = mp.EstimateObjectvie(rs_model, True)
elapsed_time_ub = t.perf_counter() - start_time_ub
objective_ub, strains_ub, routes_ub, time_ub = (estimate_result_ub['Objective'], estimate_result_ub['Strain'],
estimate_result_ub['Route'], estimate_result_ub['Time'] )
print('###################!RESULTS!#############################')
print(f'UB:\nObjective: {objective_ub}, Time: {time_ub}, Total time: {elapsed_time_ub}')
violations = mp.FindConstraintsViolation(rs_model.cmodel, strains_ub, routes_ub)
cc_vn = violations['capacity_constraints'][1]
rc_vn = violations['route_constraints'][1]
if cc_vn == 0 and rc_vn == 0:
print('Feasible')
else:
print(f'Capacity constraint violations number: {cc_vn}')
print(f'Route constraint violations number: {rc_vn}')
print('__________________________________________________________')
def RunTest(seed, network_size, formulation, decomposition=0, coordination=0):
#desribe run
network_description = ['Medium', 'Large']
formulations_description = [
'Original', 'Linearized Constraints', 'Alternative'
]
decomposition_description = [
'Solve undecomposed', 'Demands', 'Subnetworks', 'Variable separating'
]
coordination_description = [
'Solve unrelaxed', 'Proximal cutting planes',
'Subgradient level evaluation'
]
print('#######################################################')
print(f'Seed is equal: {seed}')
print(f'Network size: {network_description[network_size]}')
print(f'Formulation: {formulations_description[formulation]}')
print(f'Decomposition: {decomposition_description[decomposition]}')
print(f'Coordination: {coordination_description[coordination]}')
#define flow bounds
FLOW_BOUNDS = (0.001, 3)
CAPACITY_BOUNDS = (0.3, 1)
#random network
net.NetworkGraph.SetRandomSeed(seed)
networks = [None, None]
#Network medium
world_init_data = [{
'NodesNumber': 3,
'EdgesNumber': 8,
'ExternalEdgesNumber': 3
}, {
'NodesNumber': 12,
'EdgesNumber': 32,
'ExternalEdgesNumber': 3
}, {
'NodesNumber': 6,
'EdgesNumber': 22,
'ExternalEdgesNumber': 2
}, {
'NodesNumber': 4,
'EdgesNumber': 12,
'ExternalEdgesNumber': 2
}]
networks[0] = net.NetworkGraph.GenerateSmallWorld(world_init_data,
*CAPACITY_BOUNDS)
networks[0].GenerateRandomSrcDst(12)
#Network large
world_init_data = [
{
'NodesNumber': 3,
'EdgesNumber': 8,
'ExternalEdgesNumber': 3
},
{
'NodesNumber': 12,
'EdgesNumber': 32,
'ExternalEdgesNumber': 3
},
{
'NodesNumber': 6,
'EdgesNumber': 22,
'ExternalEdgesNumber': 2
},
{
'NodesNumber': 4,
'EdgesNumber': 12,
'ExternalEdgesNumber': 2
},
{
'NodesNumber': 9,
'EdgesNumber': 16,
'ExternalEdgesNumber': 1
},
{
'NodesNumber': 8,
'EdgesNumber': 20,
'ExternalEdgesNumber': 2
},
{
'NodesNumber': 4,
'EdgesNumber': 12,
'ExternalEdgesNumber': 1
},
{
'NodesNumber': 3,
'EdgesNumber': 6,
'ExternalEdgesNumber': 1
},
]
networks[1] = net.NetworkGraph.GenerateSmallWorld(world_init_data,
*CAPACITY_BOUNDS)
networks[1].GenerateRandomSrcDst(32)
network = networks[network_size]
#get network params
n_list = network.GetNodeList()
f_list = network.GetFlowList()
sd_dict = network.GetSrcDstDict()
a_list = network.GetArcList()
c_dict = network.GetCapacityParam()
#select formulation
if formulation == 0:
nmg = mg.RsModelGenerator(mg.QuadObjectiveGenerator(),
mg.RouteConstraintsGenerator(),
mg.NonlinearCapacityConstraintsGenerator())
if formulation == 1:
nmg = mg.RsModelGenerator(mg.QuadObjectiveGenerator(),
mg.RouteConstraintsGenerator(),
mg.LinearCapacityConstraintsGenerator())
if formulation == 2:
nmg = mg.RsModelGenerator(mg.QuadObjectiveGenerator(),
mg.ReformulatedConstraintsGenerator())
rs_model = nmg.CreateCompletRsModel(f_list, sd_dict, n_list, a_list,
c_dict, FLOW_BOUNDS)
#initialize solvers
opt_solver = sm.CplexSolver()
mstr_solver = sm.IpoptSolver()
rec_solver = sm.IpoptSolver()
#select coordinator
coordinator = None
if coordination == 0:
coordinator = None
if coordination == 1:
coordinator = cr.CoordinatorCuttingPlaneProximal(lm_min=-4.6,
lm_max=6.6)
if coordination == 2:
coordinator = cr.CoordinatorGradient(
step_rule=cr.gradstep.ObjectiveLevelStepRule(3.2, 1.4))
#select decomposer
decomposer = None
decomposed_solvers = []
if decomposition == 0:
decomposer = None
if decomposition == 1:
decomposed_solvers = [opt_solver for _ in f_list]
decomposer = dec.FlowDecomposer(rs_model, coordinator)
if decomposition == 2:
decomposed_solvers = [opt_solver for _ in world_init_data]
decomposer = dec.SubnetsDecomposer(rs_model, coordinator,
network.GetWorldNodes())
if decomposition == 3:
decomposed_solvers = [opt_solver, opt_solver]
decomposer = dec.SepVarDecomposer(rs_model, coordinator)
#deduce run parameter
run_decomposition = (coordination != 0) and (decomposition != 0)
run_original = not run_decomposition
#run computation
tf.RunTest(network,
rs_model,
decomposer, {
'Original': opt_solver,
'Recovered': rec_solver,
'Master': mstr_solver,
'Decomposed': decomposed_solvers
},
solve_original=run_original,
solve_decomposed=run_decomposition,
max_iter=25,
validate_feasability=True,
recover_feasible=True,
draw_progress=True,
draw_solution=False)
