def __init__(self, variables, clauses, verbose=True):
self.verbose = verbose
self.variables = variables
self.clauses = ClauseSet(variables, clauses)
self.assignment = Assignment(variables)
self.queue = self.variables[:]
self.counts = [dict(), dict()]
def gather(self, verbose=True):
os.system("rm -f " + BulkGatherer.BUFFER_FILE)
for i, part in tqdm.tqdm(list(enumerate(os.scandir(self.folder))),
disable=not verbose):
if i == 0:
os.system("head -n 1 {path} >> {out}".format(
path=os.path.join(part.path, Inputs.ASSIGNMENT_PATH),
out=BulkGatherer.BUFFER_FILE))
os.system("tail -n +2 {path} >> {out}".format(
path=os.path.join(part.path, Inputs.ASSIGNMENT_PATH),
out=BulkGatherer.BUFFER_FILE))
assignment = Assignment([])
assignment.load(BulkGatherer.BUFFER_FILE)
os.system("rm -f " + BulkGatherer.BUFFER_FILE)
return assignment
def solve(self, variables_path=None, constraints_path=None):
open(self.gurobi_log_file, "w").close()
self.model.params.Threads = 12
self.model.optimize()
self.assignment = Assignment(self.variables)
f_vars, f_cstr = None, None
if constraints_path is not None:
f_cstr = open(constraints_path, "w")
f_cstr.write("\t".join(Ilp.cstr_attrs) + "\n")
if f_cstr is not None:
for constraint in self.model.getConstrs():
for attr in Ilp.cstr_attrs:
value = ""
try:
value = constraint.getAttr(attr)
except:
pass
f_cstr.write(str(value) + "\t")
f_cstr.write("\n")
f_cstr.close()
if variables_path is not None:
f_vars = open(variables_path, "w")
f_vars.write("\t".join(Ilp.vars_attrs) + "\n")
inner_confidence = []
for gurobi_var in self.model.getVars():
if gurobi_var.varName[0] not in "PTRS":
continue
if gurobi_var.rc == 0:
up = min(2, gurobi_var.SAObjUp)
low = max(-2, gurobi_var.SAObjLow)
inner_confidence.append((gurobi_var.x - .51) * (up - low))
if len(inner_confidence) == 0:
a, b = 0, 1
else:
a, b = min(inner_confidence), max(inner_confidence)
for gurobi_var in self.model.getVars():
if f_vars is not None:
for attr in Ilp.vars_attrs:
value = ""
try:
value = gurobi_var.getAttr(attr)
except:
pass
f_vars.write(str(value) + "\t")
f_vars.write("\n")
letter = gurobi_var.varName[0]
if letter not in "PTRS":
continue
index = int(gurobi_var.varName[2:-1])
confidence = 0
if gurobi_var.rc != 0:
confidence = gurobi_var.rc
else:
confidence = gurobi_var.x + gurobi_var.obj + Parameters.EVIDENCE_OFFSET
self.assignment.assign(
Variable(index, Dimensions.from_letter(letter)),
gurobi_var.x >= .5,
confidence,
)
if f_vars is not None:
f_vars.close()
return self.assignment
class Ilp:
"""
Attributes and their signification @
http://www.gurobi.com/documentation/8.1/refman/attributes.html#sec:Attributes
"""
gurobi_log_file = "gurobi.log"
vars_attrs = [
"VarName",
"VType",
"LB",
"UB",
"Obj",
"X",
"Xn",
"RC",
"BarX",
"Start",
"VarHintVal",
"VarHintPri",
"BranchPriority",
"Partition",
"VBasis",
"PStart",
"IISLB",
"IISUB",
"PWLObjCvx",
"SAObjLow",
"SAObjUp",
"SALBLow",
"SALBUp",
"SAUBLow",
"SAUBUp",
"UnbdRay",
]
cstr_attrs = [
"ConstrName",
"Sense",
"RHS",
"Pi",
"Slack",
"CBasis",
"DStart",
"Lazy",
"IISConstr",
"SARHSLow",
"SARHSUp",
"FarkasDual",
]
def __init__(self, variables, model, verbose=True):
self.variables = variables
self.model = model
self.verbose = verbose
self.assignment = None
def solve(self, variables_path=None, constraints_path=None):
open(self.gurobi_log_file, "w").close()
self.model.params.Threads = 12
self.model.optimize()
self.assignment = Assignment(self.variables)
f_vars, f_cstr = None, None
if constraints_path is not None:
f_cstr = open(constraints_path, "w")
f_cstr.write("\t".join(Ilp.cstr_attrs) + "\n")
if f_cstr is not None:
for constraint in self.model.getConstrs():
for attr in Ilp.cstr_attrs:
value = ""
try:
value = constraint.getAttr(attr)
except:
pass
f_cstr.write(str(value) + "\t")
f_cstr.write("\n")
f_cstr.close()
if variables_path is not None:
f_vars = open(variables_path, "w")
f_vars.write("\t".join(Ilp.vars_attrs) + "\n")
inner_confidence = []
for gurobi_var in self.model.getVars():
if gurobi_var.varName[0] not in "PTRS":
continue
if gurobi_var.rc == 0:
up = min(2, gurobi_var.SAObjUp)
low = max(-2, gurobi_var.SAObjLow)
inner_confidence.append((gurobi_var.x - .51) * (up - low))
if len(inner_confidence) == 0:
a, b = 0, 1
else:
a, b = min(inner_confidence), max(inner_confidence)
for gurobi_var in self.model.getVars():
if f_vars is not None:
for attr in Ilp.vars_attrs:
value = ""
try:
value = gurobi_var.getAttr(attr)
except:
pass
f_vars.write(str(value) + "\t")
f_vars.write("\n")
letter = gurobi_var.varName[0]
if letter not in "PTRS":
continue
index = int(gurobi_var.varName[2:-1])
confidence = 0
if gurobi_var.rc != 0:
confidence = gurobi_var.rc
else:
confidence = gurobi_var.x + gurobi_var.obj + Parameters.EVIDENCE_OFFSET
self.assignment.assign(
Variable(index, Dimensions.from_letter(letter)),
gurobi_var.x >= .5,
confidence,
)
if f_vars is not None:
f_vars.close()
return self.assignment
def report(self):
with open(self.gurobi_log_file) as file:
text = file.read()
report = Report()
report.add(text)
return report
class MaxSat:
def __init__(self, variables, clauses, verbose=True):
self.verbose = verbose
self.variables = variables
self.clauses = ClauseSet(variables, clauses)
self.assignment = Assignment(variables)
self.queue = self.variables[:]
self.counts = [dict(), dict()]
def update_counts(self, mask=None):
clauses = self.clauses.get_units()
if mask is not None:
clauses = clauses.intersection(mask)
for x in self.assignment.get_unassigned():
self.counts[1][x] = 0.
self.counts[0][x] = 0.
for c in clauses:
if x in c.positives:
self.counts[1][x] += c.get_weight()
if x in c.negatives:
self.counts[0][x] += c.get_weight()
def update_queue(self):
self.queue.sort(
key=lambda x: abs(self.counts[1][x] - self.counts[0][x]))
def find_safe_var(self):
for x in self.assignment.get_unassigned():
for truth_value in [True, False]:
if truth_value:
units = self.clauses.get_pos_units(x)
unsatisfieds = self.clauses.get_neg_unsatisfieds(x)
else:
units = self.clauses.get_neg_units(x)
unsatisfieds = self.clauses.get_pos_unsatisfieds(x)
confidence = sum([c.get_weight() for c in units]) - sum(
[c.get_weight() for c in unsatisfieds])
if confidence >= 0:
return x, truth_value, {
True: 1.,
False: -1.
}[truth_value] * confidence
return None, None, None
def solve(self):
self.update_counts()
self.update_queue()
if self.verbose:
bar = ProgressBar(len(self.queue))
print("Starting reasoning...")
bar.start()
while len(self.queue) > 0:
x = self.queue.pop()
if self.verbose:
bar.increment()
self.assignment.assign(x, self.counts[1][x] > self.counts[0][x],
self.counts[1][x] - self.counts[0][x])
self.clauses.update(x, self.counts[1][x] > self.counts[0][x])
while True:
safe_var, truth_value, confidence = self.find_safe_var()
if safe_var is None:
break
self.assignment.assign(safe_var, truth_value, confidence)
self.clauses.update(safe_var, truth_value)
self.queue.remove(safe_var)
if self.verbose:
bar.increment()
self.update_counts(self.clauses.get_occurences(x))
self.update_queue()
if self.verbose:
bar.stop()
return self.assignment
def report(self):
report = Report()
unsatisfieds = self.clauses.get_unsatisfieds()
n_clauses = len(self.clauses)
n_satisfieds = n_clauses - len(unsatisfieds)
w_total = sum([c.get_weight() for c in self.clauses])
w_satisfieds = w_total - sum([c.get_weight() for c in unsatisfieds])
report.add_value_ratio("Satisfied weight", w_satisfieds, w_total)
report.add_value_ratio("Satisfied clauses", n_satisfieds, n_clauses)
return report
def load_assignment(self):
if os.path.isfile(self.path(self.ASSIGNMENT_PATH)):
from dice.reason import Assignment
self._assignment = Assignment([])
self._assignment.load(self.path(self.ASSIGNMENT_PATH))
class Inputs(Output):
KB_PATH = "kb.tsv"
TAXONOMY_PATH = "taxonomy.zip"
EMBEDDING_PATH = "embedding"
SIMILARITY_MATRIX_PATH = "similarity"
PROBABILITY_PATH = "probability"
TRANSITION_PATH = "transition.npy"
ENTAILER_PATH = "entailer.tsv"
DETECTIVE_PATH = "detective.tsv"
ASSIGNMENT_PATH = "assignment.tsv"
def __init__(self, folder, load=False):
Output.__init__(self, folder)
self._kb = None
self._taxonomy = None
self._embedding = None
self._similarity_matrix = None
self._probability = None
self._transition = None
self._entailer = None
self._detective = None
self._assignment = None
if load:
self.load()
def clone(self, clone_folder):
inputs = Inputs(clone_folder)
if self._kb is not None:
inputs.set_kb(self._kb)
if self._taxonomy is not None:
inputs.set_taxonomy(self._taxonomy)
if self._embedding is not None:
inputs.set_embedding(self._embedding)
if self._similarity_matrix is not None:
inputs.set_similarity_matrix(self._similarity_matrix)
if self._probability is not None:
inputs.set_probability(self._probability)
if self._transition is not None:
inputs.set_transition(self._transition)
if self._entailer is not None:
inputs.set_entailer(self._entailer)
if self._detective is not None:
inputs.set_detective(self._detective)
if self._assignment is not None:
inputs.set_assignment(self._assignment)
return inputs
def merge_kb(self, index_offset, other):
if self._kb is not None:
if other._kb is not None:
for fact in other._kb.values():
fact.index += index_offset
self._kb[fact.index] = fact
else:
self._kb = other._kb
def merge_taxonomy(self, index_offset, other):
if self._taxonomy is not None:
if other._taxonomy is not None:
for index, key in other._taxonomy.relation._map.items():
self._taxonomy.relation.map(index + index_offset, key)
for u in other._taxonomy.nodes():
if self._taxonomy.has_node(u):
self._taxonomy.nodes[u].setdefault("weight", 0)
self._taxonomy.nodes[u][
"weight"] += other._taxonomy.nodes[u].get(
"weight", 0)
else:
self._taxonomy.add_node(
u,
weight=other._taxonomy.nodes[u].get("weight", 0))
for u, v in other._taxonomy.edges():
if self._taxonomy.has_edge(u, v):
pass
else:
self._taxonomy.add_edge(
u,
v,
weight=other._taxonomy.get_edge_data(u, v).get(
"weight", 0))
else:
self._taxonomy = other._taxonomy
def merge_embedding(self, index_offset, other):
if self._embedding is not None:
if other._embedding is not None:
import numpy as np
shape = self._embedding._matrix.shape[0]
for key, index in other._embedding._map.items():
self._embedding.add_to_map(key + index_offset,
index + shape)
self._embedding._matrix = np.vstack(
(self._embedding._matrix, other._embedding._matrix))
else:
self._embedding = other._embedding
def merge_detective(self, index_offset, other):
if self._detective is not None:
if other._detective is not None:
from dice.evidence import EvidenceWrapper
for cue_cls, cue_dict in other._detective.cues.items():
for index, value in cue_dict.items():
self._detective.cues[cue_cls][index +
index_offset] = value
for index in other._detective.keys():
self._detective[index + index_offset] = EvidenceWrapper(
index + index_offset, self._detective.cues.values())
else:
self._detective = other._detective
def merge(self, other):
index_offset = 0
if self._kb is not None:
index_offset = max(self._kb.keys()) + 1
self.merge_kb(index_offset, other)
self.merge_taxonomy(index_offset, other)
self.merge_embedding(index_offset, other)
self.merge_detective(index_offset, other)
def save(self):
self.save_kb()
self.save_taxonomy()
self.save_embedding()
self.save_similarity_matrix()
self.save_probability()
self.save_transition()
self.save_entailer()
self.save_detective()
self.save_assignment()
def load(self):
self.load_kb()
self.load_taxonomy()
self.load_embedding()
self.load_similarity_matrix()
self.load_probability()
self.load_transition()
self.load_entailer()
self.load_detective()
self.load_assignment()
def save_kb(self):
if self._kb is not None:
self._kb.export(self.path(self.KB_PATH))
def save_taxonomy(self):
if self._taxonomy is not None:
self._taxonomy.save(self.path(self.TAXONOMY_PATH))
def save_embedding(self):
if self._embedding is not None:
self._embedding.save(self.path(self.EMBEDDING_PATH))
def save_similarity_matrix(self):
if self._similarity_matrix is not None:
self._similarity_matrix.save(self.path(
self.SIMILARITY_MATRIX_PATH))
def save_probability(self):
if self._probability is not None:
self._probability.save(self.path(self.PROBABILITY_PATH))
def save_transition(self):
if self._transition is not None:
np.save(self.path(self.TRANSITION_PATH), self._transition)
def save_entailer(self):
if self._entailer is not None:
self._entailer.save(self.path(self.ENTAILER_PATH))
def save_detective(self):
if self._detective is not None:
self._detective.save(self.path(self.DETECTIVE_PATH))
def save_assignment(self):
if self._assignment is not None:
self._assignment.save(self.path(self.ASSIGNMENT_PATH))
def load_kb(self):
if os.path.isfile(self.path(self.KB_PATH)):
from dice.kb import KnowledgeBase
self._kb = KnowledgeBase(self.path(self.KB_PATH))
def load_taxonomy(self):
if os.path.isfile(self.path(self.TAXONOMY_PATH)):
from dice.taxonomy import Taxonomy
self._taxonomy = Taxonomy()
self._taxonomy.load(self.path(self.TAXONOMY_PATH))
def load_embedding(self):
if os.path.isfile(self.path(self.EMBEDDING_PATH + ".tsv"))\
and os.path.isfile(self.path(self.EMBEDDING_PATH + ".npy")):
from dice.similarity import Embedding
self._embedding = Embedding()
self._embedding.load(self.path(self.EMBEDDING_PATH))
def load_similarity_matrix(self):
if os.path.isfile(self.path(self.SIMILARITY_MATRIX_PATH + "_index.pickle"))\
and os.path.isfile(self.path(self.SIMILARITY_MATRIX_PATH + "_matrix.npz")):
from dice.similarity import SimilarityMatrix
self._similarity_matrix = SimilarityMatrix()
self._similarity_matrix.load(self.path(
self.SIMILARITY_MATRIX_PATH))
def load_probability(self):
if os.path.isfile(self.path(self.PROBABILITY_PATH + "_sets.pickle"))\
and os.path.isfile(self.path(self.PROBABILITY_PATH + "_law.npz")):
from dice.evidence import Probability
self._probability = Probability()
self._probability.load(self.path(self.PROBABILITY_PATH))
def load_transition(self):
if os.path.isfile(self.path(self.TRANSITION_PATH)):
self._transition = np.load(self.path(self.TRANSITION_PATH))
def load_entailer(self):
if os.path.isfile(self.path(self.ENTAILER_PATH)):
from dice.evidence import Entailer
self._entailer = Entailer(self)
self._entailer.load(self.path(self.ENTAILER_PATH))
def load_detective(self):
if os.path.isfile(self.path(self.DETECTIVE_PATH)):
from dice.evidence import Detective
self._detective = Detective(self)
self._detective.load(self.path(self.DETECTIVE_PATH))
def load_assignment(self):
if os.path.isfile(self.path(self.ASSIGNMENT_PATH)):
from dice.reason import Assignment
self._assignment = Assignment([])
self._assignment.load(self.path(self.ASSIGNMENT_PATH))
def get_kb(self):
if self._kb is None:
self.load_kb()
if self._kb is None:
raise InputsException("KB")
return self._kb
def get_taxonomy(self):
if self._taxonomy is None:
self.load_taxonomy()
if self._taxonomy is None:
raise InputsException("Taxonomy")
return self._taxonomy
def get_embedding(self):
if self._embedding is None:
self.load_embedding()
if self._embedding is None:
raise InputsException("Property embedding")
return self._embedding
def get_similarity_matrix(self):
if self._similarity_matrix is None:
self.load_similarity_matrix()
if self._similarity_matrix is None:
raise InputsException("Similarity matrix")
return self._similarity_matrix
def get_probability(self):
if self._probability is None:
self.load_probability()
if self._probability is None:
raise InputsException("Probability")
return self._probability
def get_transition(self):
if self._transition is None:
self.load_transition()
if self._transition is None:
raise InputsException("Transition")
return self._transition
def get_entailer(self):
if self._entailer is None:
self.load_entailer()
if self._entailer is None:
raise InputsException("Entailer")
return self._entailer
def get_detective(self):
if self._detective is None:
self.load_detective()
if self._detective is None:
raise InputsException("Detective")
return self._detective
def get_assignment(self):
if self._assignment is None:
self.load_assignment()
if self._assignment is None:
raise InputsException("Assignment")
return self._assignment
def set_kb(self, kb, save=True):
self._kb = kb
if save:
self.save_kb()
def set_taxonomy(self, taxonomy, save=True):
self._taxonomy = taxonomy
if save:
self.save_taxonomy()
def set_embedding(self, embedding, save=True):
self._embedding = embedding
if save:
self.save_embedding()
def set_similarity_matrix(self, similarity_matrix, save=True):
self._similarity_matrix = similarity_matrix
if save:
self.save_similarity_matrix()
def set_probability(self, probability, save=True):
self._probability = probability
if save:
self.save_probability()
def set_transition(self, transition, save=True):
self._transition = transition
if save:
self.save_transition()
def set_entailer(self, entailer, save=True):
self._entailer = entailer
if save:
self.save_entailer()
def set_detective(self, detective, save=True):
self._detective = detective
if save:
self.save_detective()
def set_assignment(self, assignment, save=True):
self._assignment = assignment
if save:
self.save_assignment()