def patternStringGenerator(seed): if seed.isdigit(): # 纯数字 length = len(seed) # print(length) strr = r'\b[0-9]{' + str(length) + r'}\b' # print(strr) return strr else: aaa = re.sub("\D", "", seed) if len(aaa) > 0: # seed 中包含数字 restr = [r'\b'] for i in seed: if i.isalpha(): restr.append(r'[a-z]') else: if i.isdigit(): restr.append(r'\d') else: restr.append(r'.') restr.append(r'\b') restrr = ''.join(restr) # print(restr) # print(restrr) return restrr elif seed.isalpha(): print("is alpha") return None else: if len(seed) > 30: # 增加长字串模糊匹配? return None cleaningseed = processhelper.simpledatacleaning(seed) splists = cleaningseed.split(' ') if len(splists) > 0: restr = [r'\b', splists[0]] if len(splists) < 2: a = r'[a-zA-Z]+' restr.append(r'[^a-zA-Z0-9]*') restr.append(a) else: for substring in range(1, len(splists) - 1): a = r'[a-zA-Z]+' restr.append(r'[^a-zA-Z0-9]*') restr.append(a) restr.append(r'[^a-zA-Z0-9]*') restr.append(splists[len(splists) - 1]) restr.append(r'\b') restrr = ''.join(restr) print(restrr) return restrr
def AttributeTrainingDataMixGenerator(username, attr_id): print(username) traingdata = [] testingdata = [] validationdata = [] dataset = Cora_labeled.objects.all() entitys = models.sigirCoraToAttrEntity.objects.filter(user=username) for record in dataset: syns = entitys.filter(cora_id=record.id) if syns: attrids = [syn.attrsynonym.value.attr_id for syn in syns] if attr_id in attrids: text = record.cleantext dict = {} texttemp = [] for syn in syns: print(syn) synoym = syn.attrsynonym.synonym aa = processhelper.simpledatacleaning(synoym) aalist = aa.split(' ') for item in aalist: if syn.attrsynonym.value.attr_id == attr_id: dict[item] = syn.attrsynonym.value_id else: dict[item] = 'ooo' # text = text.replace(synoym,' '+synoym+'###'+syn.attrsynonym.value.attr.attrname+' ') # text = processhelper.simpledatacleaning(text) textlist = text.split(' ') left = list(set(textlist).difference(set(dict.keys()))) for item in textlist: if item in dict.keys(): texttemp.append(item + '/' + str(dict[item])) else: # texttemp.append(item + '/NULL') print(item + '/NULL') for i in range(len(left)): if i < 3: texttemp.append(left[i] + '/NULL') traingdata.append(' '.join(texttemp)) else: testingdata.append(record.cleantext) else: testingdata.append(record.cleantext) validationdata = traingdata return traingdata, testingdata, validationdata
def hbfaffinegapOnlyclusterSimple(corahbf, dataset, acr_threshold, username, dis_threshold): subrecords = [] dataset = dataset.order_by('id') record_ids = [data.id for data in dataset] for data in dataset: # print(data.cleantext) subrecord = data.cleantext coraent = models.sigirCoraToAttrEntity.objects.filter(user=username, cora_id=data.id) for cc in coraent: syn = cc.attrsynonym.synonym cleansyn = processhelper.simpledatacleaning(syn) subrecord = subrecord.replace(cleansyn, cc.attrsynonym.value.value) subrecords.append(subrecord) subgroups = dextrapreclustering.affinegaphClustering(data=subrecords) cluster_dict = {} for i in range(len(subgroups)): cluster_dict[record_ids[i]] = subgroups[i] return cluster_dict
def basicTrainingDataGenerator(username): print(username) traingdata = [] testingdata = [] validationdata = [] dataset = Cora_labeled.objects.all() entitys = models.sigirCoraToAttrEntity.objects.filter(user=username) for record in dataset: syns = entitys.filter(cora_id=record.id) temp = syns.count() if temp > 2: print(temp) text = record.cleantext dict = {} texttemp = [] for syn in syns: print(syn) synoym = syn.attrsynonym.synonym aa = processhelper.simpledatacleaning(synoym) aalist = aa.split(' ') for item in aalist: dict[item] = syn.attrsynonym.value.attr.attrname # text = text.replace(synoym,' '+synoym+'###'+syn.attrsynonym.value.attr.attrname+' ') # text = processhelper.simpledatacleaning(text) textlist = text.split(' ') for item in textlist: if item in dict.keys(): texttemp.append(item + '/' + dict[item]) else: texttemp.append(item + '/NULL') if temp < 4: validationdata.append(' '.join(texttemp)) else: traingdata.append(' '.join(texttemp)) else: testingdata.append(record.cleantext) return traingdata, testingdata, validationdata
def collectSubrecords(node, dataset, username): # dataset = Cora_labeled.objects.all() nodeRecords = node.get_index_recordids() edges_dict = node.get_edges_dict() # find max common values commonValues = [] for layer in edges_dict.keys(): layer_edges_dict = edges_dict[layer] for nodevalue in layer_edges_dict.keys(): recordsids = layer_edges_dict[nodevalue] if set(nodeRecords).issubset(set(recordsids)): commonValues.append(nodevalue) print(node.get_attributeValue(), ':', nodeRecords) commonValues.append(node.get_attributeValue()) # print(commonValues) datas = dataset.filter(id__in=nodeRecords) # record_id_dict = {} # for i in range(len(nodeRecords)): # record_id_dict[i] = nodeRecords[i] # collect subrecords subrecords = [] for data in datas: # print(data.cleantext) subrecord = data.cleantext coraent = models.sigirCoraToAttrEntity.objects.filter(user=username, cora_id=data.id) for cc in coraent: syn = cc.attrsynonym.synonym cleansyn = processhelper.simpledatacleaning(syn) if cc.attrsynonym.value.value in commonValues: # print(cleansyn) subrecord = subrecord.replace(cleansyn, '') # print(subrecord) else: subrecord = subrecord.replace(cleansyn, cc.attrsynonym.value.value) subrecords.append(subrecord) return subrecords