def __init__(self, dhf: DHTFacade, ert: EthearnalProfileController):
self.dhf = dhf
self.ert = ert
self.dhf.indexer = self
self.db_idx = InvIndexTimestampSQLite(
db_name=self.ert.db_gigs_index_fn, table_name='gigs_index')
self.idx = ResourceIndexingEngine(self.db_idx)
def __init__(
self,
db_name,
table_name,
logger=None,
):
self.db = InvIndexTimestampSQLite(
db_name=db_name,
table_name=table_name,
)
self.idx = ResourceIndexingEngine(self.db)
self.logger = default_value(logger, ErtLogger(logger=Print()))
class DocumentIndexer(object):
def __init__(
self,
db_name,
table_name,
logger=None,
):
self.db = InvIndexTimestampSQLite(
db_name=db_name,
table_name=table_name,
)
self.idx = ResourceIndexingEngine(self.db)
self.logger = default_value(logger, ErtLogger(logger=Print()))
def index_field(
self,
container_hk_hex,
specifier,
text_data,
prefixes=True,
ert_tokens_major=0,
ert_tokens_minor=0,
q1: int = 0,
q2: int = 0,
):
try:
pk_bin = guid_hex_to_bin(container_hk_hex)
self.idx.index_bag_of_spec_text(container_hash=pk_bin,
specifier=specifier,
text_data=text_data,
prefixes=prefixes,
ert_tokens_major=ert_tokens_major,
ert_tokens_minor=ert_tokens_minor,
q1=q1,
q2=q2)
self.logger('INDEXED', container_hk_hex, specifier)
except Exception as e:
self.logger('ERR IDX FAILED', specifier, text_data, str(e))
def unindex(self, hk_hex):
try:
pk_bin = guid_hex_to_bin(hk_hex)
self.idx.unindex(pk_bin)
except Exception as e:
self.logger('UNINDEX FAILED', e)
class Indexer(object):
def __init__(self, dhf: DHTFacade, ert: EthearnalProfileController):
self.dhf = dhf
self.ert = ert
self.dhf.indexer = self
self.db_idx = InvIndexTimestampSQLite(
db_name=self.ert.db_gigs_index_fn,
table_name='gigs_index'
)
self.idx = ResourceIndexingEngine(self.db_idx)
def unindex(self, hk_hex):
try:
pk_bin = guid_hex_to_bin(hk_hex)
self.idx.unindex(pk_bin)
except:
print('UNINDEX FAILED')
def index_field(self, pk_bin, specifier, text_data, prefixes=True, q1: int = 0, q2: int = 0,):
print('\n\n INDEX FIELD q1,q2', q1,q2)
try:
self.idx.index_bag_of_spec_text(
container_hash=pk_bin,
specifier=specifier, text_data=text_data, prefixes=prefixes, q1=q1, q2=q2)
except Exception as e:
print('ERR IDX FAILED', specifier, text_data, str(e))
def query_terms(self, terms: dict, prefixes=True, limit=30):
cur = self.idx.qry_terms(terms=terms, prefixes=prefixes, limit=limit)
if cur:
ll = [guid_bin_to_hex2(t[0]) for t in cur.fetchall()]
return ll
def query_all(self, limit=10, ret_t=False):
cur = self.db_idx.no_component(limit=limit)
if cur:
if ret_t:
ll = [(guid_bin_to_hex2(t[0]), t[1], t[2], t[3], t[4]) for t in cur.fetchall()]
return ll
else:
ll = [guid_bin_to_hex2(t[0]) for t in cur.fetchall()]
return ll
def query_terms_d(self, terms_d: dict, limit=30):
# print("\n\nQUERY_TERMS_D", terms_d)
cur = self.idx.qry_terms_d(terms_d, limit=limit)
if cur:
# print(list(cur.fetchall()))
# ll = None
l1 = list(cur.fetchall())
# print('---->', l1)
ll = [guid_bin_to_hex2(t[0]) for t in l1]
return ll
def query_text(self, text, limit=30):
return self.query_terms({'text': text}, limit=limit)
def query_tags(self, tags, limit=30):
return self.query_terms({'tags': tags}, prefixes=False, limit=limit)
def query_owner(self, guid_owner, limit=30):
return self.query_terms({'owner_guid': guid_owner}, prefixes=False, limit=limit)
def index_gig_document(self, hk_hex: str, doc: dict):
text = ''
q1 = 0
q2 = 0
if 'price' in doc:
q1 = int(doc['price'])
if 'title' in doc:
text += doc['title'] + ' '
if 'description' in doc:
text += doc['description']
if text.strip() != '':
# pass
self.index_field(guid_hex_to_bin(hk_hex), 'text', text_data=text, q1=q1, q2=q2)
if 'tags' in doc:
self.index_field(guid_hex_to_bin(hk_hex), 'tags', text_data=' '.join(doc['tags']), prefixes=False,
q1=q1, q2=q2)
if 'owner_guid' in doc:
self.index_field(guid_hex_to_bin(hk_hex), 'owner_guid',
text_data=doc['owner_guid'], prefixes=False, q1=q1, q2=q2)
if 'category' in doc:
self.index_field(guid_hex_to_bin(hk_hex), 'category',
text_data=doc['category'], prefixes=False, q1=q1, q2=q2)
def index_on(self, hk_hex: str, data: dict):
if hk_hex and data:
if 'value' in data:
doc = data['value']
if 'model' in doc:
if doc['model'] == 'Gig':
if 'meta_gig_deleted' in doc:
if doc['meta_gig_deleted']:
# remove from index
self.unindex(hk_hex)
return
# index it
self.index_gig_document(hk_hex, doc)
def unindex_on(self, hk_hex):
try:
self.unindex(guid_hex_to_bin(hk_hex))
except:
print('UNINDEX FAILED')
class Indexer(object):
def __init__(self, dhf: DHTFacade, ert: EthearnalProfileController):
self.dhf = dhf
self.ert = ert
self.dhf.indexer = self
self.db_idx = InvIndexTimestampSQLite(
db_name=self.ert.db_gigs_index_fn, table_name='gigs_index')
self.idx = ResourceIndexingEngine(self.db_idx)
def index_field(self, pk_bin, specifier, text_data, prefixes=True):
try:
self.idx.index_bag_of_spec_text(container_hash=pk_bin,
specifier=specifier,
text_data=text_data,
prefixes=prefixes)
except Exception as e:
print('ERR IDX FAILED', specifier, text_data, str(e))
def query_terms(self, terms: dict, prefixes=True):
cur = self.idx.qry_terms(terms=terms, prefixes=prefixes)
if cur:
ll = [guid_bin_to_hex2(t[2]) for t in cur.fetchall()]
return ll
def query_all(self):
cur = self.db_idx.no_component()
if cur:
ll = [guid_bin_to_hex2(t[2]) for t in cur.fetchall()]
return ll
def query_terms_d(self, terms_d: dict):
cur = self.idx.qry_terms_d(terms_d)
if cur:
ll = [guid_bin_to_hex2(t[2]) for t in cur.fetchall()]
return ll
def query_text(self, text):
return self.query_terms({'text': text})
def query_tags(self, tags):
return self.query_terms({'tags': tags}, prefixes=False)
def query_owner(self, guid_owner):
return self.query_terms({'owner_guid': guid_owner}, prefixes=False)
def index_gig_document(self, hk_hex: str, doc: dict):
text = ''
if 'title' in doc:
text += doc['title'] + ' '
if 'description' in doc:
text += doc['description']
if text.strip() != '':
# pass
self.index_field(guid_hex_to_bin(hk_hex), 'text', text_data=text)
if 'tags' in doc:
self.index_field(guid_hex_to_bin(hk_hex),
'tags',
text_data=' '.join(doc['tags']),
prefixes=False)
if 'owner_guid' in doc:
self.index_field(guid_hex_to_bin(hk_hex),
'owner_guid',
text_data=doc['owner_guid'],
prefixes=False)
def index_on(self, hk_hex: str, data: dict):
if hk_hex and data:
if 'value' in data:
doc = data['value']
if 'model' in doc:
if doc['model'] == 'Gig':
self.index_gig_document(hk_hex, doc)