def process_sentences(dt, purpose, usr):
purps = purpose # purposes-> layouts definidos dentro dos facts dos ractionlines escalados
to_run_c = []
print 'Process load layout...'
if True:
all_layouts = []
for pur_p in purps:
print 'Start purpose-load-layout:', pur_p, '--------------------------------------------------------------'
layouts_f = get_layouts(usr, pur_p)
layouts_f2 = get_layouts2(usr, pur_p)
onto_basis2 = []
for onto_basisk in layouts_f:
l2 = Identify.prepare_layout(usr, onto_basisk)
onto_basis2.append(l2)
onto_basis22 = []
for onto_basisk in layouts_f2:
l2 = Identify.prepare_layout(usr, onto_basisk)
#print 'Prepare layout(2):',onto_basisk,'->',l2.fzs
onto_basis22.append(l2)
all_layouts.append([onto_basis2, onto_basis22, pur_p])
print 'End purpose:', pur_p, '--------------------------------------------------------------'
#
return process_termo(dt, usr, pur_p, 0, '', all_layouts)
def entry(param, data_parse):
l = Identify.prepare_layout('igor.moraes', 'common-indentify')
l2 = Identify.prepare_layout('igor.moraes', 'common-indentify')
onto = Identify.prepare_data_by_ask(l, param, 'igor.moraes',
'common-indentify', [])
for dta in data_parse:
#print 'process data:',dta
ir = Identify.process_data(l2, dta, onto, 'common-indentify',
'igor.moraes')
for topico in ir[0].topicos:
if len(topico.dt) > 0:
print '==========================================='
print topico.dt
print 'sins:--'
def pr(topico):
for p in topico.sinapses:
print p.nr.dt
if p.opcode == 'Relaction':
print '?????????????????????????????????'
pr(p.nr)
print '?????????????????????????????????'
elif p.opcode == "cmpp":
print ';;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;'
pr(topico)
print '-------'
print '==========================================='
def process_sentences(start_c, usr):
resultSet = conn.sql(
"SELECT USERNAME,TERMO,TRIGGER_AS FROM clipping_info where USERNAME='******' ") # 50 rows por vez
r1 = []
for results in resultSet:
username = results[0]
termo = results[1]
trigger_as = results[2]
r1.append([username, termo, trigger_as])
purps = get_purposes(
usr
) # purposes-> layouts definidos dentro dos facts dos ractionlines escalados
to_run_c = []
print 'Process load layout...'
for r in r1:
[username, termo, trigger_as] = r
#===
print 'Process termo:', termo
all_layouts = []
for pur_p in purps:
print 'Start purpose-load-layout:', pur_p, '--------------------------------------------------------------'
layouts_f = get_layouts(usr, pur_p)
layouts_f2 = get_layouts2(usr, pur_p)
onto_basis2 = []
for onto_basisk in layouts_f:
l2 = Identify.prepare_layout(usr, onto_basisk)
onto_basis2.append(l2)
onto_basis22 = []
for onto_basisk in layouts_f2:
l2 = Identify.prepare_layout(usr, onto_basisk)
#print 'Prepare layout(2):',onto_basisk,'->',l2.fzs
onto_basis22.append(l2)
all_layouts.append([onto_basis2, onto_basis22, pur_p])
print 'End purpose:', pur_p, '--------------------------------------------------------------'
#
print 'Start process page:---'
process_termo(termo, username, pur_p, start_c, '', all_layouts)
#=========================
if len(entry_doc) > 0: return
def mount_node(term,id,purpose): conn= MySQLdb.connect(host='dbmy0023.whservidor.com', user='******' , passwd='acc159753', db='mindnet') l=Identify.prepare_layout(conn,id,purpose) allp=[] onto=Identify.prepare_data_by_ask(l, term,id,purpose,allp ) conn.close() return [onto,allp]
def mount_node(term, id, purposes):
rets = []
for purpose in purposes:
l = Identify.prepare_layout(id, purpose)
allp = []
onto = Identify.prepare_data_by_ask(l, term, id, purpose, allp)
rets.append([onto, allp])
return rets
def process_sentences(start_c,usr,purps):
r1=[]
just_r=False
if True:
username=usr
termo='SYSTEM'
trigger_as=''
r1.append([username,termo,trigger_as])
to_run_c=[]
print 'Process load layout...'
for r in r1:
[username,termo,trigger_as]=r
#===
print 'Process termo:',termo
all_layouts=[]
for pur_p in purps:
print 'Start purpose-load-layout:',pur_p,'--------------------------------------------------------------'
layouts_f =get_layouts(usr,pur_p)
layouts_f2=get_layouts2(usr,pur_p)
onto_basis2=[]
for onto_basisk in layouts_f:
l2=Identify.prepare_layout(usr,onto_basisk)
onto_basis2.append(l2)
onto_basis22=[]
for onto_basisk in layouts_f2:
l2=Identify.prepare_layout(usr,onto_basisk)
#print 'Prepare layout(2):',onto_basisk,'->',l2.fzs
onto_basis22.append(l2)
all_layouts.append([onto_basis2,onto_basis22,pur_p])
print 'End purpose:',pur_p, '--------------------------------------------------------------'
#
print 'Start process page:---'
process_termo(termo,username,pur_p,start_c,'',all_layouts,just_r )
#=========================
if len(entry_doc) > 0 : return
def entry_process(param, data_parse, usr):
layouts_f = get_layouts(usr, 'simple-search')
layouts_f2 = get_layouts2(usr, 'simple-search')
rets = []
str_ret = []
l = Identify.prepare_layout(usr, 'simple-search')
l2 = Identify.prepare_layout(usr, 'simple-search')
onto = Identify.prepare_data_by_ask(l, param, usr, 'simple-search', [])
for dta in data_parse:
ir = Identify.process_data(l2, dta, onto, 'simple-search', usr)
if ir[0] == None: continue
for topico in ir[0].topicos:
if len(topico.dt) > 0:
it = topico.dt
its = []
for p in topico.sinapses:
its.append([p.opcode, p.nr.dt])
rets.append([it, its])
for tp in rets:
if 'identificador' in tp[0]:
c2 = tp[1]
for tp2 in c2:
for dt in tp2[1]:
str_ret.append(dt)
#========================================
sret = '%'
for s in str_ret:
sret += (s + '%')
#==================
if sret == '%':
sret += (param + '%')
return sret
def entry_process(data_parse, usuario, purpose):
rets = []
t_threads = []
t_threads.append(thread_cntl())
onto_basis2 = []
ontos = get_layouts(usuario, purpose)
for onto_basisk in ontos:
l2 = Identify.prepare_layout(usuario, onto_basisk)
onto_basis2.append(l2)
onto_basis22 = []
ontos2 = get_layouts2(usuario, purpose)
for onto_basisk in ontos2:
l2 = Identify.prepare_layout(usuario, onto_basisk)
onto_basis22.append(l2)
onto = Identify.pre_process_data2(onto_basis2, onto_basis22, data_parse,
purpose, id,
t_threads[len(t_threads) - 1], [])
for s in onto:
rets.append(s)
return rets
def process_page(lines_doc2,id,purpose,pgs,finish,th):
ln_o=''
try:
l2=Identify.prepare_layout(id,purpose)
if True :
for s in lines_doc2:
ln_o+=' '+s
if umisc.trim(ln_o) == '':
finish.finished=True
return
ir=Identify.process_data(l2,ln_o,onto_basis,purpose,id,th)
if ir[0] != None :
result_onto_tree_er.append(ir[0])
if ir[1] != None:
result_onto_tree_bpm.append(ir[1])
finish.finished=True
except:
finish.finished=True
print 'Thread ',pgs,' was finished.','Len:',len(ln_o),' process:',start_c/10
def mount_node(term,id,purpose): l=Identify.prepare_layout(id,purpose) allp=[] onto=Identify.prepare_data_by_ask(l, term,id,purpose,allp ) return [onto,allp]
def process_page(lines_doc2,id,purpose,pgs,finish,th):
ln_o=''
conn= MySQLdb.connect(host='dbmy0023.whservidor.com', user='******' , passwd='acc159753', db='mindnet')
if True:
#try:
l2=Identify.prepare_layout(conn,id,purpose)
if True :
for s in lines_doc2:
ln_o=s
if umisc.trim(ln_o) == '':
finish.finished=True
return
ir=Identify.process_data(conn,l2,ln_o,onto_basis,purpose,id,th)
if ir[0] != None :
# procura identificador ---
fnd_ident=False
for es in ir[0].topicos:
if ir[0].es_compare_dt(es,'identificador'):
fnd_ident=True
if not fnd_ident:
ind=len(result_onto_tree_er)-1
fond_cs=False
while ind >=0 and not fond_cs:
for es2 in result_onto_tree_er[ind].topicos:
if ir[0].es_compare_dt(es2,'identificador'):
ir[0].set_topico_nr(es2)
fond_cs=True
break
ind-=1
# verificar se nao tem somente identificadores(elemento fact invalido)
oth=False
for es in ir[0].topicos:
if ir[0].es_compare_dt(es,'identificador'):
pass
else:
oth=True
if not oth:
continue
result_onto_tree_er.append(ir[0])
# procurar group
ind=len(result_onto_tree_er)-1
while ind >=0 :
if ir[0] != result_onto_tree_er[ind]:
for es2 in result_onto_tree_er[ind].topicos:
if ir[0].es_compare_dt(es2,'identificador'):
for top in ir[0].topicos:
if ir[0].compare_dt_depend(conn,usr,purpose,es2,top,['']):
# encontrou referencias do mesmo identificador, incluir nos objetos linkados
rt=None
fnd_new=False
for k1 in result_linked:
for k2 in k1:
if k1 == ir[0]:
fnd_new=True
rt=k1
#=
if not fnd_new:
result_linked.append([ir[0]])
rt= result_linked[len(result_linked)-1 ]
#=======================================
fnd_new=False
for k2 in rt:
if k2 == result_onto_tree_er[ind]:
fnd_new=True
if not fnd_new:
rt.append(result_onto_tree_er[ind])
ind-=1
#==========================
if ir[1] != None:
result_onto_tree_bpm.append(ir[1])
#except Exception ,err:
# print 'Except:',err
# finish.finished=True
finish.finished=True
conn.close()
print 'Thread ',pgs,' was finished.','Len:',len(ln_o),' process:',start_c/10
def process_page(all_ps, id, purpose, pgs, finish, th, pg_index_rs,
all_size_pg, job_index, addresses, result_onto_tree_er,
onto_basis):
#try:
if True:
ln_o = ''
endereco_url = all_ps[0]
#===
progress = int(pg_index_rs / all_size_pg)
l2 = Identify.prepare_layout(id, purpose)
for lines_doc2_ in all_ps:
#try:
print 'Start page:', pg_index_rs, ' of total:', all_size_pg
if True:
endereco_url = lines_doc2_[0]
lines_doc2 = lines_doc2_[1]
if True:
#============= parse fuzzy ===========================================
t_threads = []
ret_ps = []
indice_linha = 0
for s in lines_doc2:
indice_linha += 1
ln_o = s
addresses.append(endereco_url)
if umisc.trim(ln_o) == '':
continue
ret_ps.append([])
t_threads.append(thread_cntl())
print 'Preprocessdata in page:', pg_index_rs, ' of total:', all_size_pg, ' line:', indice_linha, ' of:', len(
lines_doc2)
Identify.pre_process_data(
l2, ln_o, onto_basis, purpose, id,
t_threads[len(t_threads) - 1],
ret_ps[len(ret_ps) - 1])
print 'Preprocessdata END in page:', pg_index_rs, ' of total:', all_size_pg, ' line:', indice_linha, ' of:', len(
lines_doc2)
#thread.start_new_thread(Identify_pre_process_data,(l2,ln_o,onto_basis,purpose,id,t_threads[len(t_threads)-1],ret_ps[len(ret_ps)-1]) )
#====================================================================
#run rcts linkadas
sentence_index = 0
for s_ps in ret_ps:
sentence_index += 1
print 'Process sentence:', sentence_index, ' of :', len(
ret_ps
), ' page:', pg_index_rs, ' of total:', all_size_pg
ir = Identify.resume_process_data(
s_ps, onto_basis, purpose, id)
if ir[0] != None:
# procura identificador ---
fnd_ident = False
for es in ir[0].topicos:
if ir[0].es_compare_dt(es, 'identificador'):
fnd_ident = True
if not fnd_ident:
ind = len(result_onto_tree_er) - 1
fond_cs = False
while ind >= 0 and not fond_cs:
for es2 in result_onto_tree_er[
ind].topicos:
if ir[0].es_compare_dt(
es2, 'identificador'):
ir[0].set_topico_nr(es2)
fond_cs = True
break
ind -= 1
# verificar se nao tem somente identificadores(elemento fact invalido)
oth = False
for es in ir[0].topicos:
if ir[0].es_compare_dt(es, 'identificador'):
pass
else:
oth = True
if not oth:
continue
result_onto_tree_er.append(ir[0])
# procurar group
ind = len(result_onto_tree_er) - 1
while ind >= 0:
if ir[0] != result_onto_tree_er[ind]:
for es2 in result_onto_tree_er[
ind].topicos:
if ir[0].es_compare_dt(
es2, 'identificador'):
for top in ir[0].topicos:
if ir[0].compare_dt_depend(
conn, usr, purpose,
es2, top, ['']):
# encontrou referencias do mesmo identificador, incluir nos objetos linkados
rt = None
fnd_new = False
for k1 in result_linked:
for k2 in k1:
if k1 == ir[0]:
fnd_new = True
rt = k1
#=
if not fnd_new:
result_linked.append(
[ir[0]])
rt = result_linked[len(
result_linked) - 1]
#=======================================
fnd_new = False
for k2 in rt:
if k2 == result_onto_tree_er[
ind]:
fnd_new = True
if not fnd_new:
rt.append(
result_onto_tree_er[
ind])
ind -= 1
#==========================
if ir[1] != None:
result_onto_tree_bpm.append(ir[1])
#except Exception ,err:
# print 'Except on process pages:',err,'pg:',pg_index_rs
traceQ(progress, id, pg_index_rs, (start_c / 10), endereco_url,
'Processed page:')
def process_termo(termo,usr,purp,start_c,path_j):
# montar ontlogia
layouts_f=get_layouts(usr,purp)
layouts_f2=get_layouts2(usr,purp)
t_threads=[]
t_threads.append(thread_cntl())
ret_ps=[]
ret_ps.append([])
lines_doc2=termo
id=usr
onto_basis2=[]
for onto_basisk in layouts_f:
l2=Identify.prepare_layout(id,onto_basisk)
onto_basis2.append(l2)
onto_basis22=[]
for onto_basisk in layouts_f2:
l2=Identify.prepare_layout(id,onto_basisk)
onto_basis22.append(l2)
ret_ps[len(ret_ps)-1]=Identify.pre_process_data2(onto_basis2,onto_basis22,lines_doc2,purp,id,t_threads[len(t_threads)-1],[])
objs_search2=[]
complements=[]
#=
objs_search=[]
purposes=get_purpsz(usr,purp)
print 'purposes:[',purposes,']'
for lays in ret_ps:
for lay in lays:
for top in lay.topicos:
for dts in top.dt:
if dts.upper() == "IDENTIFICADOR" or dts.lower() in ['action' ] :
for s in top.sinapses:
for dts2 in s.nr.dt:
if dts2 not in ['.',':','\'','"','?','?']:
objs_search.append(dts2)
print 'Identify:',dts2
if dts.upper() == "REALID" or dts.upper() == "REALID2":
for s in top.sinapses:
for dts2 in s.nr.dt:
if dts2 not in ['.',':','\'','"','?','?']:
objs_search2.append(dts2)
print 'RealID:',dts2
#==================================
print 'Objs:Search:',objs_search
if len(objs_search2) > 0 or len(objs_search) > 0 :
opener = urllib2.build_opener()
def pg_open(addresss,th,pages,pgind,ind_emit,start_c,total_p):
try:
ind=0
inds=0
acumul=0
print 'Process init open page:',len(addresss)
total_p=total_p/len(addresss)
for address in addresss:
try:
ind+=1
inds+=1
lines_doc=[]
if address != 'debug-url':
if inds > 2:
print 'traceq i:',total_p
if traceQ(acumul,usr) :
acumul=0
inds=0
else:
acumul+=total_p
#======================
address=urllib.quote(address)
address=address.replace('%3A',':')
url='http://www.mind-net.com/get_Text.php?q='+address
print 'Open page:',url
content = opener.open(url, '' ).read()
tmpd=content.replace('\n',' ')
lines_doc.append(tmpd)
#tmpd=''
#for d in content:
# if d == '\n':
# lines_doc.append(tmpd)
# tmpd=''
# else:
# tmpd+=d
#============
pages.append(Task_C(pg_add,lines_doc))
print 'Get content for page:',pgind,' was finished.Len:',len(lines_doc)
pgind+=1
else:
for line_deb in entry_doc:
lines_doc.append(line_deb)
pages.append(Task_C(pg_add,lines_doc))
print 'Get content for page:',pgind,' was finished.Len:',len(lines_doc)
pgind+=1
except Exception,e:
print 'Error PG_OPEN_I',e,'..'
th.finished=True
except Exception,e:
print 'Error PG_OPEN',e,'..'
#logs.exception( 'Error process sentences->pg_open:' )
#logs.exception('[Layout(p) Exec Error]Stack execution---------------------------------')
th.finished=True