def build_iceberg_lattice(filename, lattice, threshold): irreducable = [] for i, (intent, extent) in enumerate(lattice): coverage = list(intent) if (len(intent) < threshold): continue is_irreducable = True for j, (intent1, extent1) in enumerate(lattice): if (j == i or len(intent1) < threshold or len(intent) <= len(intent1)): continue is_subset = True for obj in intent1: if (not (obj in intent)): is_subset = False break if is_subset: for obj in intent1: if obj in coverage: coverage.remove(obj) if (len(coverage) == 0): is_irreducable = False break if is_irreducable: irreducable.append((intent, extent)) #print intent, extent #print '\n' df = Definition() for intent, extent in irreducable: obj_name = ';'.join(intent) df.add_object(obj_name, list(extent)) conc = Context(*df) conc.tofile(filename='iceberg.' + filename, frmat='csv')
def test_fromstring_serialized(tmp_path, source, filename, includes_lattice): if filename is None: context = Context.fromstring(source, frmat='python-literal') else: target = tmp_path / filename kwargs = {'encoding': 'utf-8'} target.write_text(source, **kwargs) context = Context.fromfile(str(target), frmat='python-literal', **kwargs) assert context.objects == SERIALIZED['objects'] assert context.properties == SERIALIZED['properties'] assert context.bools == [ (True, False, False, True, False, True, True, False, False, True), (True, False, False, True, False, True, False, True, True, False), (False, True, True, False, False, True, True, False, False, True), (False, True, True, False, False, True, False, True, True, False), (False, True, False, True, True, False, True, False, False, True), (False, True, False, True, True, False, False, True, True, False) ] if includes_lattice: assert 'lattice' in context.__dict__ else: assert 'lattice' not in context.__dict__
def test_dict_roundtrip(context, ignore_lattice): context = Context(context.objects, context.properties, context.bools) assert 'lattice' not in context.__dict__ d = context.todict(ignore_lattice=ignore_lattice) assert isinstance(d, dict) and d assert all(d[k] for k in ('objects', 'properties', 'context')) if ignore_lattice or ignore_lattice is None: assert 'lattice' not in context.__dict__ assert 'lattice' not in d else: assert 'lattice' in context.__dict__ assert d['lattice'] result = Context.fromdict(d) assert isinstance(result, Context) assert result == context if ignore_lattice or ignore_lattice is None: assert 'lattice' not in result.__dict__ else: assert 'lattice' in result.__dict__ assert result.lattice._eq(context.lattice)
def __init__(self, csv_location): # the Frame-capability lattice self.context = Context.fromfile(csv_location, frmat='csv') self.lattice = self.context.lattice # the Frame-uncapability lattice self.context_inv = Context(*self.context.definition().inverted()) self.lattice_inv = self.context_inv.lattice # the list of all capabilities and frames self.capabilities = self.context.properties self.frames = self.context.objects
def test_todict(context, d): assert 'lattice' not in context.__dict__ if 'lattice' in d: context = Context(context.objects, context.properties, context.bools) assert 'lattice' not in context.__dict__ for ignore_lattice in (False, None): assert context.todict(ignore_lattice=ignore_lattice) == d assert 'lattice' in context.__dict__ else: for ignore_lattice in (True, None): assert context.todict(ignore_lattice=ignore_lattice) == d assert 'lattice' not in context.__dict__
def main(): #print sys.argv filename = str(sys.argv[1]) iceberg_threshold = int(sys.argv[2]) draw_iceberg = False cols_to_use = [] i = 3 cols_started = False while (i < len(sys.argv)): if (sys.argv[i][0] == '-'): cols_started = False if (sys.argv[i] == '-draw'): draw_iceberg = True elif (sys.argv[i] == '-columns'): cols_started = True elif (cols_started): cols_to_use.append(sys.argv[i]) i += 1 #print cols_to_use dataframe = pd.read_csv(filename) if (len(cols_to_use) > 0): dataframe = dataframe[[dataframe.columns[0]] + cols_to_use] col_info = pd.read_csv('cols.' + filename) transform_columns(dataframe, col_info) dataframe = dataframe.drop_duplicates(subset=list(dataframe.columns[0:1]), keep='first') dataframe.to_csv('transformed.' + filename, index_label=False, index=False) context = Context.fromfile('transformed.' + filename, frmat='csv') lattice_str = str(context.lattice.graphviz()) f = open('lattice.dot', 'w') f.write(lattice_str) f.close() #context.lattice.graphviz() build_iceberg_lattice(filename, context.lattice, iceberg_threshold) iceberg_context = Context.fromfile(filename='iceberg.' + filename, frmat='csv') if draw_iceberg: iceberg_context.lattice.graphviz(view=True) lattice_str = str(iceberg_context.lattice.graphviz()) f = open('iceberg.dot', 'w') f.write(lattice_str) f.close() implication_basis = find_implication_basis(iceberg_context) print "Implication basis: " for i, e in implication_basis: print str(i) + " => " + str(e)
def test_json_roundtrip(context, path_or_fileobj, encoding): context = Context(context.objects, context.properties, context.bools) assert 'lattice' not in context.__dict__ is_fileobj = hasattr(path_or_fileobj, 'seek') kwargs = {'encoding': encoding} if encoding is not None else {} context.tojson(path_or_fileobj, ignore_lattice=True, **kwargs) if is_fileobj: path_or_fileobj.seek(0) assert 'lattice' not in context.__dict__ deserialized = Context.fromjson(path_or_fileobj, **kwargs) if is_fileobj: path_or_fileobj.seek(0) assert 'lattice' not in deserialized.__dict__ assert deserialized == context assert isinstance(context.lattice, Lattice) assert 'lattice' in context.__dict__ context.tojson(path_or_fileobj, ignore_lattice=None, **kwargs) if is_fileobj: path_or_fileobj.seek(0) deserialized = Context.fromjson(path_or_fileobj, **kwargs) assert 'lattice' in deserialized.__dict__ assert deserialized == context assert deserialized.lattice._eq(context.lattice)
def extract_concepts_in_order(objs, order:Order) -> [(set, set)]: var_idx = list(set(itertools.chain.from_iterable(variables for variables in objs.values()))) context_def = [ [var_idx.index(variable) for variable in variables] for (obj, variables) in objs.items() ] ctx = Context.fromdict({'objects': list(objs), 'properties': var_idx, 'context': context_def}) def linksof(c) -> set: "edges covered by the given concept" return set(itertools.product(c.extent, c.intent)) concepts_cover = {c: linksof(c) for c in iter(ctx.lattice)} treated = set() # set of edges already described def def_Random(concepts): return random.choice(tuple(concepts)) def def_LargestCoverFirst(concepts): return max(concepts.keys(), key=lambda c: len(linksof(c) - treated)) def def_LargestExtentFirst(concepts): return max(concepts.keys(), key=lambda c: len(c.extent)) def def_LargestIntentFirst(concepts): return max(concepts.keys(), key=lambda c: len(c.intent)) def def_LargestExtentOrIntentFirst(concepts): return max(concepts.keys(), key=lambda c: max(len(c.extent), len(c.intent))) while concepts_cover: best = locals()['def_' + order.name](concepts_cover) simplified_best = simplify_concept(best, treated) treated |= linksof(best) concepts_cover = {c: linksof(c) - treated for c in concepts_cover} concepts_cover = {c: links for c, links in concepts_cover.items() if len(links) > 0} if not simplified_best[0] or not simplified_best[1]: continue # ignore the extremums yield simplified_best
def render_all(filepattern='*.cxt', frmat=None, encoding=None, directory=None, out_format=None): from concepts import Context if directory is not None: get_name = lambda filename: os.path.basename(filename) else: get_name = lambda filename: filename if frmat is None: from concepts.formats import Format get_frmat = Format.by_extension.get else: get_frmat = lambda filename: frmat for cxtfile in glob.glob(filepattern): name, ext = os.path.splitext(cxtfile) filename = '%s.gv' % get_name(name) c = Context.fromfile(cxtfile, get_frmat(ext), encoding=encoding) l = c.lattice dot = l.graphviz(filename, directory) if out_format is not None: dot.format = out_format dot.render()
def render_all(filepattern='*.cxt', frmat=None, encoding=None, directory=None, out_format=None): from concepts import Context if directory is not None: get_name = lambda filename: os.path.basename(filename) else: get_name = lambda filename: filename if frmat is None: from concepts.formats import Format get_frmat = Format.by_extension.get else: get_frmat = lambda filename: frmat for cxtfile in glob.glob(filepattern): name, ext = os.path.splitext(cxtfile) filename = '%s.gv' % get_name(name) c = Context.fromfile(cxtfile, get_frmat(ext), encoding=encoding) l = c.lattice dot = l.graphviz(filename, directory) if out_format is not None: dot.format = out_format dot.render()
def local_fca(self, molecules): props = self.get_properties(molecules) props = list(set(props)) sub = list(molecules[0].subjects())[0] molecule_properties = [ str(prop.encode('utf-8')) + '->' + str(value.encode('utf-8')) for prop, value in props ] molecule_names = [ "{}_{}".format(str(sub.encode('utf-8')), y) for y in [2014, 2015, 2016] ] mat = [] for molecule in molecules: row = [False] * len(props) for idx, (prop, val) in enumerate(props): if (sub, prop, val) in molecule: row[idx] = True mat.append(row) c = Context(molecule_names, molecule_properties, mat) res = c.lattice result = [] for (extent, intent) in res: result.append((list(extent), list(intent))) return result
def test_fromdict_raw(context, lattice, d, raw): def shuffled(items): result = list(items) random.shuffle(result) return result _lattice = d.get('lattice') d = { 'objects': d['objects'], 'properties': d['properties'], 'context': [shuffled(intent) for intent in d['context']] } if _lattice is not None: pairs = shuffled(enumerate(_lattice)) index_map = {old: new for new, (old, _) in enumerate(pairs)} d['lattice'] = [(shuffled(ex), shuffled(in_), shuffled(index_map[i] for i in up), shuffled(index_map[i] for i in lo)) for _, (ex, in_, up, lo) in pairs] result = Context.fromdict(d, raw=raw) assert isinstance(result, Context) assert result == context if _lattice is not None: if raw: assert result.lattice._eq(lattice) else: # instance broken by shuffled(d['lattice']) assert not result.lattice._eq(lattice)
def dictToConcept(data_matrix): """ From dictionnary to concepts """ definition = concepts.Definition() for (current_obj, current_values) in data_matrix.items(): definition.add_object(current_obj, current_values) context = Context(*definition) lattice = context.lattice return context, lattice
def nonascii_context(abba=(u'Agneta F\xe4ltskog', u'Anni-Frid Lyngstat', u'Benny Andersson', u'Bj\xf6rn Ulvaeus')): d = Definition() for o in abba: d.add_object(o, [u'human', u'singer']) d.add_property(u'female', abba[:2]) d.add_property(u'male', abba[2:]) d.add_property(u'keyboarder', [abba[2]]) d.add_property(u'guitarrist', [abba[3]]) d.add_property(u'sch\xf6n', abba[::2]) return Context(*d)
def generate_concept_matrix(filename, skill_list=None, render=False): # applying fca c = Context.fromfile(filename, frmat="csv") if render: c.lattice.graphviz(filename=filename.rstrip(".csv"), view=True) # reading csv headers csvfile = open(filename) csvreader = csv.reader(csvfile) # reading skills if skill_list is None: skill_list = csvreader.__next__() skill_list.pop(0) else: csvreader.__next__() # reading abstract names row_header = list() for row in csvreader: row_header.append(row[0]) csvfile.close() # matrix to return mat = list() for i, concept in enumerate(c.lattice): extent, intent = concept # skip for non-significant concept if len(extent) == 0 or len(intent) == 0: continue print("c{} = {} > {}".format(i, extent, intent)) row = list() for skill in skill_list: if skill in intent: row.append(1) else: row.append(0) for header in row_header: if header in extent: row.append(1) else: row.append(0) mat.append(row) return mat, row_header, skill_list
def test_fromdict(context, lattice, d, require_lattice, ignore_lattice, raw): if require_lattice and 'lattice' not in d: return result = Context.fromdict(d, require_lattice=require_lattice, ignore_lattice=ignore_lattice, raw=raw) assert result == context if ignore_lattice or 'lattice' not in d: assert 'lattice' not in result.__dict__ else: assert 'lattice' in result.__dict__ assert result.lattice._eq(lattice)
def predict_fca(s): properties = s.index.values objects = [str(s.name)] bools = tuple(s.astype(bool)) s_lattice = Context(objects, properties, [bools]) s_intents = set() for extent_s, intent_s in s_lattice.lattice: s_intents.add(intent_s) sets = set(list(s_intents)[1]) probs = [] for i in range(0, no_of_classes): for intent_c in class_intents_sets[i]: setc = set(intent_c) if sets.issubset(setc): probs.append(i) if len(probs) == 0: return -1 return max(probs, key=probs.count)
def test_json_roundtrip_nonascii_context(nonascii_context, encoding): assert isinstance(nonascii_context.lattice, Lattice) assert 'lattice' in nonascii_context.__dict__ kwargs = {'encoding': encoding} if encoding is not None else {} with io.StringIO() as f: nonascii_context.tojson(f, **kwargs) serialized = f.getvalue() f.seek(0) deserialized = Context.fromjson(f, **kwargs) assert 'lattice' in deserialized.__dict__ assert deserialized == nonascii_context assert deserialized.lattice._eq(nonascii_context.lattice) assert '"Agneta F\\u00e4ltskog"' in serialized assert '"Bj\\u00f6rn Ulvaeus"' in serialized assert '"sch\\u00f6n"' in serialized
def generate_concept_matrix(filename, skill_list=None): # applying fca c = Context.fromfile(filename, frmat="csv") # reading csv headers csvfile = open(filename) csvreader = csv.reader(csvfile) # reading skills if skill_list is None: skill_list = csvreader.__next__() skill_list.pop(0) else: csvreader.__next__() # reading abstract names row_header = list() for row in csvreader: row_header.append(row[0]) csvfile.close() # matrix to return mat = list() for extent, intent in c.lattice: print("{} > {}".format(extent, intent)) row = list() for skill in skill_list: if skill in intent: row.append(1) else: row.append(0) for header in row_header: if header in extent: row.append(1) else: row.append(0) mat.append(row) return mat, row_header, skill_list
# Creating and save context for implication rules X_train_one_hot['Class'] = y_train X_train_Class_split = pd.concat([ X_train_one_hot, pd.get_dummies(X_train_one_hot['Class'], prefix='Class') ], axis=1) X_train_Class_split = X_train_Class_split.drop(["Class"], axis=1).drop_duplicates() objects = X_train_Class_split.index.values objects = [str(oi) for oi in objects] properties = X_train_Class_split.columns.values bools = list( X_train_Class_split.astype(bool).itertuples(index=False, name=None)) cxt = Context(objects, properties, bools) cxt.tofile('diabetes_context.cxt', frmat='cxt', encoding='utf-8') ## Create concepts lattices for each class c = {} l = {} no_of_classes = 2 X_train_one_hot['Class'] = y_train X_train_one_hot = X_train_one_hot.drop_duplicates() for i in range(0, no_of_classes): X_temp = X_train_one_hot.copy(deep=True) X_temp = X_temp[X_temp['Class'] == i].drop(["Class"], axis=1) objects = X_temp.index.values objects = [str(oi) for oi in objects] properties = X_temp.columns.values
def test_fromdict_empty_lattice(d_invalid): d_invalid['lattice'] = [] with pytest.raises(ValueError, match='empty lattice'): Context.fromdict(d_invalid)
def test_fromdict_context_invalid_index(d_invalid): first = d_invalid['context'][0] d_invalid['context'][0] = (42, ) + first[1:] with pytest.raises(ValueError, match='invalid index'): Context.fromdict(d_invalid)
def test_fromdict_context_duplicates(d_invalid): first = d_invalid['context'][0] d_invalid['context'][0] = (first[0], first[0]) + first[2:] with pytest.raises(ValueError, match='duplicate'): Context.fromdict(d_invalid)
def test_fromdict_mismatch(d_invalid, short): d_invalid[short] = d_invalid[short][1:] lens = (5, 6) if short == 'objects' else (6, 5) match = r'mismatch: %d objects with %d context' % lens with pytest.raises(ValueError, match=match): Context.fromdict(d_invalid)
def test_fromdict_nonstring(d_invalid, nonstring): d_invalid[nonstring] = (42, ) + d_invalid[nonstring][1:] with pytest.raises(ValueError, match=r'non-string %s' % nonstring): Context.fromdict(d_invalid)
def buildLattice(pattern = True, inputFiles = "dec", inputAttributes = "arts"): if pattern == True: name = "WithPS" else: name = "WithoutPS" print inputFiles, inputAttributes, name #Le contexte a construire matrixAttribute = [] # listFiles = [] #Liste des fichiers lus pour construire le contexte if(inputFiles == "dec"): listAllFiles = fg.getAllDecisions() elif(inputFiles == "avis"): listAllFiles = fg.getAllAvis() elif(inputFiles == "all"): listAllFiles = fg.getAllFiles() else: print "choix non reconnu. Choix possibles : 'dec' 'avis' 'all'" listAllFiles = fg.getAllDecisions() #Nombre de fichiers lus lengthAllFiles = len(listAllFiles) #L'ensemble des attributs du contexte setOfAttributes = set() #L'ensemble des attributs modifiés du contexte setFormated = set() #L'expression régulière des attributs possibles des différents textes if (inputAttributes == "arts"): expre = expreAttribute() elif(inputAttributes == "artsdocs"): expre = expreAttribute()+'|'+regex.exprReguliereDecision() else: print "choix non reconnu. Choix possibles : 'arts' 'docs' 'artsdocs'" expre = expreAttribute() #Compteur de fichiers lus i = 0 #Lecture des fichiers pour lister les attributs for dfile in listAllFiles: f = open(dfile,'r') #Enlever les sauts de lignes dûs au copier/coller du pdf data = ' '.join([line.rstrip() for line in f]) #Pour chaque expression trouvée dans le texte for m in re.finditer(expre, data): #Expression réguliere attributFormated = m.group(0) #Lissage de l'expression : #Enlever les accents attributFormated = regex.removeAccent(attributFormated) #Corriger les erreurs potentielles attributFormated = correctSyntaxe(attributFormated) attributFormated = regex.supprNumero(attributFormated) setOfAttributes.add(attributFormated) i = i + 0.5 if i%100==0: print str(int(i))+' fichiers lus sur '+str(lengthAllFiles) #Modification des attributs pour éviter les doublons setOfAttributes = list(setOfAttributes) for item in setOfAttributes: setFormated.add(regex.formatArticle(item)) if pattern == True: developAttributes = buildAttributes(setFormated) setFormated = list(developAttributes) else: setFormated = list(setFormated) #Nombre d'attributs dans le contexte lenset = len(setFormated) print str(lenset) writeAttributes(setFormated,name) #Construction du contexte for dfile in listAllFiles: f = open(dfile, 'r') data = ' '.join([line.rstrip() for line in f]) #Lister les documents pour la construction du contexte listFiles.append(regex.nomDocument(dfile)) #Construction d'une ligne du contexte nuplet = (False,)*lenset listuple = list(nuplet) #Pour chaque expression for m in re.finditer(expre, data): attributFormated = m.group(0) #Formater l'expression régulière attributFormated = regex.removeAccent(attributFormated) attributFormated = correctSyntaxe(attributFormated) attributFormated = regex.supprNumero(attributFormated) attributFormated = regex.formatArticle(attributFormated) #Si pattern, on découpe chaque attribut if pattern == True: listAtt = developAttribute(attributFormated) for item in listAtt: #Trouver l'indice de l'attribut index = setFormated.index(item) #Mettre à jour la valeur listuple[index] = True #Sinon on cherche juste les attributs else: index = setFormated.index(attributFormated) listuple[index] = True i = i + 0.5 if i%100==0: print str(int(i))+' fichiers lus sur '+str(lengthAllFiles) nuplet = tuple(listuple) #Ajoute le nouvel objet au contexte matrixAttribute.append(nuplet) print str(int(i))+' fichiers lus sur '+str(lengthAllFiles) #Sauvegarde les attributs dans un txt #sauvegarde le contexte dans un json exportContext(listFiles,setFormated,matrixAttribute,name) c = Context(listFiles,setFormated,matrixAttribute) print "construction de la lattice. Cela peut prendre quelques instants" c.lattice.graphviz(view=True) #Sauvegarde le contexte dans un txt writeConcepts(c.lattice,name) c.tofile('latticeEtContext/saveLatticeWithPS.txt',frmat='cxt',encoding='utf-8')
if concept.properties: # dot.edge(name, name, taillabel=' '.join(concept.properties), labelangle='90', color='transparent') print("properties >", ' | '.join(concept.properties)) # dot.edges((name, node_name(c)) for c in sorted(concept.lower_neighbors, key=sortkey)) print("edges :") for i in sorted(concept.lower_neighbors, key=sortkey): print(name, "->", node_name(i)) edgecount += 1 print() print("nodes:", nodecount, "edges:", edgecount) # if render or view: # dot.render(view=view) # pragma: no cover # return dot # c = Context.fromfile("test_files/tech_formal_context.csv",frmat="csv") c = Context.fromfile("test_files/student_formal_context.csv", frmat="csv") # max_e_len = 0 # for e,i in c.lattice: # if len(e) > max_e_len: # max_e_len = len(e) for i, exin in enumerate(c.lattice): extent, intent = exin print("c" + str(i), ">", extent, "\t->", intent) # # c.lattice.graphviz(view=True,filename="temp_show.pdf") # show_graph(c.lattice,filename="temp_show.pdf",directory="output_trees",view=True)
outputCSVFile = open('train_output.csv', 'w+') wtr = csv.writer(outputCSVFile, delimiter=',', lineterminator='\n') for i in range(number_of_objects + 1): for j in range(number_of_columns + 1): if i == 0 and j == 0: output_matrix[i][j] = '' elif i == 0 and j > 0: output_matrix[i][j] = 'c' + str(j - 1) elif i > 0 and j == 0: output_matrix[i][j] = str(i - 1) else: output_matrix[i][j] = str(context_matrix[i - 1][j - 1]) wtr.writerow(output_matrix[i]) outputCSVFile.close() train_dict = {} c = Context.fromfile('train_output.csv', 'csv') # sys.stdout = open('output1.txt', 'w+') for extent, intent in c.lattice: #print('%r %r' % (extent, intent)) # attribute_combinations = np.asarray(intent) if intent not in train_dict: count = 0 extent_array = np.asarray(extent) for row in extent_array: if count == 0: train_dict[intent] = [ int(float(tableCells[int(row)][number_of_columns])) ] count = count + 1 else:
def __init__(self, dataframe, leaves, annotate=None, dummy_formatter=None, keep_names=True, comp_prefix=None, col_formatter=None, na_value=None, AOC=False, collections=False, verbose=True): """ Arguments: dataframe (:class:`pandas:pandas.DataFrame`): A dataframe leaves (dict): Dictionnaire de microclasses annotate (dict): Extra annotations to add on lattice. Of the form: {<object label>:<annotation>} dummy_formatter (func): Function to make dummies from the table. (default to panda's) keep_names (bool): whether to keep original column names when dropping duplicate dummy columns. comp_prefix (str): If there are two sets of properties, the prefix used to distinguish column names. AOC (bool): Whether to limit ourselves to Attribute or Object Concepts. col_formatter (func): Function to format columns in the context table. na_value : A value tu use as "Na". Defaults to `None` collections (bool): Whether the table contains :class:`representations.patterns.PatternCollection` objects. """ self.comp = comp_prefix # whether there are two sets of properties. if na_value is not None: dataframe = dataframe.applymap(lambda x: None if x == na_value else x) if collections: dummies = to_dummies(dataframe, keep_names=keep_names) elif dummy_formatter: dummies = dummy_formatter(dataframe) else: dummies = pd.get_dummies(dataframe, prefix_sep="=") dummies = dummies.applymap(lambda x: "X" if x == 1 else "") if col_formatter: dummies.columns = col_formatter(dummies.columns) if verbose: print("Reading the context and building the lattice...") context_str = dummies.to_csv() c1 = Context.fromstring(context_str, frmat='csv') self.context = c1 self.lattice = c1.lattice if annotate: for label in annotate: if label in self.lattice.supremum.extent: self.lattice[[label ]]._extra_qumin_annotation = annotate[label] self.leaves = leaves if verbose: print("Converting to qumin node...") if AOC: self.nodes = self._lattice_to_nodeAOC() else: self.nodes = self._lattice_to_node() font = {'family': 'DejaVu Sans', 'weight': 'normal', 'size': 9} matplotlib.rc('font', **font)
def main(): import os import sys import csv if not os.path.isdir(esbm_benchmark_path): print 'The esbm benchmark directory is required.' sys.exit(1) given_entities = esbm_benchmark_path + 'elist.txt' target_entities = set([]) for row in open(given_entities): target_entities.add('<' + row.strip().split('\t')[2] + '>') for entity_idx in range(1, 141): if entity_idx > 100: targetKB = 'lmdb' else: targetKB = 'dbpedia' # One given entity description file entity_decriptions = esbm_benchmark_path + targetKB + '/' + str( entity_idx) + '/' + str(entity_idx) + '_desc.nt' # Creating a grid of formal concepts and save it as a CSV file if not os.path.isdir(fca_lattice_path): os.mkdir(fca_lattice_path) fcs_lattice_filename = fca_lattice_path + 'FCA_' + str( entity_idx) + '.csv' fcs_lattice_file = open(fcs_lattice_filename, 'w') sep = ':-:' property_set = set([]) target_facts = set([]) for row in open(entity_decriptions, 'r'): s = row.strip().split()[0] p = row.strip().split()[1] o = ' '.join(row.strip().split()[2:])[:-2] if s not in target_entities and o in target_entities: _s = s s = o + '[FALSE]' o = _s property_set.add(p) target_facts.add(s + sep + p + sep + o) property_list = list(property_set) property_list.insert(0, '') fca_csv = [property_list] final_rank = {} attribute_map = {} for spo in target_facts: default_score = 1 s, p, o = spo.split(sep) s = s.replace('[FALSE]', '') # If there is less information available from the surface information, the score will be lower. for uninform_str in uninformative_values: if uninform_str in o: default_score = 0 if default_score > 0: # building attribute-token dict try: attribute_map[p] = attribute_map[p] | extract_key_tokens(o) except KeyError: attribute_map[p] = extract_key_tokens(o) final_rank[s + sep + p + sep + o] = default_score for spo, v in sorted(final_rank.items(), key=lambda x: x[1], reverse=True): tmp_fca_list = [''] * len(property_list) s, p, o = spo.split(sep) tmp_fca_list[0] = p + sep + o tmp_fca_list[property_list.index(p)] = 'X' for prop, tokens in attribute_map.items(): for token in tokens: if token in o.lower(): tmp_fca_list[property_list.index(prop)] = 'X' # print tmp_fca_list fca_csv.append(tmp_fca_list) with fcs_lattice_file: writer = csv.writer(fcs_lattice_file) writer.writerows(fca_csv) # Formal concept analysis from concepts import Context c = Context.fromfile(fcs_lattice_filename, frmat='csv') hierarchical_layer = 0 for extents, intents in c.lattice: # print extents, intents for extent in extents: if final_rank[s + sep + extent] == 1: final_rank[s + sep + extent] = len(target_facts) - hierarchical_layer hierarchical_layer += 1 # Generating result file if not os.path.isdir(kafca_final_result_path): os.mkdir(kafca_final_result_path) if not os.path.isdir(kafca_final_result_path + targetKB): os.mkdir(kafca_final_result_path + targetKB) output_filepath = kafca_final_result_path + targetKB + '/' + str( entity_idx) + '/' if not os.path.isdir(output_filepath): os.mkdir(output_filepath) fo_top5 = open(output_filepath + str(entity_idx) + '_top5.nt', 'wb') fo_top10 = open(output_filepath + str(entity_idx) + '_top10.nt', 'wb') fo_rank = open(output_filepath + str(entity_idx) + '_rank.nt', 'wb') chkcount = 0 for spo, score in sorted(final_rank.items(), key=lambda x: x[1], reverse=True): s, p, o = spo.split(sep) if spo not in target_facts: _s = s s = o o = _s chkcount += 1 try: fo_rank.write("%s %s %s .\n" % (s, p, o)) fo_top10.write("%s %s %s .\n" % (s, p, o)) fo_top5.write("%s %s %s .\n" % (s, p, o)) except ValueError: pass if chkcount == 5: fo_top5.close() if chkcount == 10: fo_top10.close() fo_rank.close()
def test_fromdict_missing(d_invalid, missing): del d_invalid[missing] with pytest.raises(ValueError, match=r'missing .*%s' % missing): Context.fromdict(d_invalid, require_lattice=(missing == 'lattice'))
def extract_sumz(): import os import csv import copy print 'summary' input_json = request.get_json(force=True) #print input_json print type(input_json) input_entity = input_json['entity'] input_KB = input_json['KB'] target_entity = set([]) target_entity.add(input_entity) if not os.path.isdir(fca_lattice_path): os.mkdir(fca_lattice_path) fcs_lattice_filename = fca_lattice_path + 'FCA_' + input_entity + '.csv' fcs_lattice_file = open(fcs_lattice_filename, 'w') sep = ':-:' property_set = set([]) target_facts = set([]) for row in input_KB: s = row.strip().split()[0] p = row.strip().split()[1] o = ' '.join(row.strip().split()[2:])[:-2] if s not in target_entity and o in target_entity: _s = s s = o + '[FALSE]' o = _s property_set.add(p) target_facts.add(s + sep + p + sep + o) property_list = list(property_set) property_list.insert(0, '') fca_csv = [property_list] final_rank = {} attribute_map = {} for spo in target_facts: default_score = 1 s, p, o = spo.split(sep) s = s.replace('[FALSE]', '') # If there is less information available from the surface information, the score will be lower. for uninform_str in uninformative_values: if uninform_str in o: default_score = 0 if default_score > 0: # building attribute-token dict try: attribute_map[p] = attribute_map[p] | extract_key_tokens(o) except KeyError: attribute_map[p] = extract_key_tokens(o) final_rank[s + sep + p + sep + o] = default_score for spo, v in sorted(final_rank.items(), key=lambda x: x[1], reverse=True): tmp_fca_list = [''] * len(property_list) s, p, o = spo.split(sep) tmp_fca_list[0] = p + sep + o tmp_fca_list[property_list.index(p)] = 'X' for prop, tokens in attribute_map.items(): for token in tokens: if token in o.lower(): tmp_fca_list[property_list.index(prop)] = 'X' # print tmp_fca_list fca_csv.append(tmp_fca_list) tmp_list = copy.deepcopy(fca_csv) with fcs_lattice_file: writer = csv.writer(fcs_lattice_file) for index, row in enumerate(fca_csv): for index_se, ele in enumerate(row): #print ele tmp_list[index][index_se] = ele.encode('utf-8') writer.writerows(tmp_list) # Formal concept analysis from concepts import Context #fcs_lattice_filename = './KAFCA_lattice/FCA_141.csv' c = Context.fromfile(fcs_lattice_filename, frmat='csv') hierarchical_layer = 0 for extents, intents in c.lattice: #print extents, intents #f = open('text2.json', 'w') #json.dump(final_rank, f, ensure_ascii=False, indent=4) for extent in extents: try: extent_de = extent.decode('utf-8') if final_rank[s+sep+extent_de] == 1: final_rank[s+sep+extent_de] = len(target_facts) - hierarchical_layer except KeyError: print s+sep+extent_de continue hierarchical_layer += 1 #print '-'*10 #print final_rank.keys() os.remove(fcs_lattice_filename) result_top5 = [] chkcount = 0 for spo, score in sorted(final_rank.items(), key=lambda x: x[1], reverse=True): s, p, o = spo.split(sep) if spo not in target_facts: _s = s s = o o = _s chkcount += 1 result_top5.append(s+'\t'+p+'\t'+o) if chkcount == 5: break result = {} result['top5'] = result_top5 return jsonify(result)
print res return res if __name__ == '__main__': animaux = ["Bat","Eagle","Monkey","Parrot fish","Penguin","Shark","Lantern fish"] proprietes = ["breathes in water","can fly","has beak","has hands","has skeleton","has wings","lives in water","is viviparous","produces light"] matrix = [ (False, True, False, False, True, True, False, True, False), # Bat (False, True, True, False, True, True, False, False, False), # Eagle (False, False, False, True, True, False, False, True, False), # Monkey (True, False, True, False, True, False, True, False, False), # Parrot Fish (False, False, True, False, True, True, True, False, False), # Penguin (True, False, False, False, True, False, True, False, False), # Shark (True, False, False, False, True, False, True, False, True)] # Lantern Fish exportContext(animaux,proprietes,matrix) c = Context(animaux, proprietes, matrix) # doctest: +ELLIPSIS '''clients = ['Anne','Basile','Carole'] articles = ['fromage','vin','lait','lessive'] matrix = [ (True, False, True, False), #A (True, True, False, True), #B (True,False,True,True)] #C c = Context(clients, articles, matrix)''' #print c #c.lattice.graphviz(view=True) #for intent, extent in c.lattice: #print intent, extent c.tofile('animaux.txt',frmat='cxt',encoding='utf-8') writeConcepts(c.lattice)
from concepts import Context c = Context() #c = Context.fromfile('examples/digits.cxt') c.additem('cat',['a','b','eats fish']) #c.additem('cat',['eats fish']) print c.tostring()