def fetch(self, **kwargs):
"""Fetch resource from database"""
def extract_x(obj):
if not obj.get('description'):
return
val = obj.get('description')
if val and isinstance(val, Binary):
val = gzip.decompress(val.value)
in_x.append(val)
iid.append(obj.get('entity_id'))
config = get_config()
chunksize = int(
getattr(config, 'producer_chunksize', Default.PRODUCER_CHUNKSIZE))
brk = get_brk(OrganizationTable.ID)
gen = brk._iscan(chunksize=chunksize, \
filter_expr=Attr('data_type').eq(OrganizationTable.DT.value))
for chunk in gen:
if not chunk:
continue
chunk = chunk.get('Items')
iid, in_x = [], []
list(map(extract_x, chunk))
if iid and in_x:
self.total += len(in_x)
print(self, self.total)
yield {
'id': iid,
'input_x': in_x,
}
def fetch(self, **kwargs):
"""Fetch resource from database"""
def extract_x(obj):
if not obj.get('abstract'):
return
iid.append(obj.get('npl_id'))
in_x.append(obj.get('abstract'))
config = get_config()
chunksize = int(
getattr(config, 'producer_chunksize', Default.PRODUCER_CHUNKSIZE))
brk = get_brk(JournalArticle.ID)
gen = brk._iscan(chunksize=chunksize, \
filter_expr=Attr('data_type').eq(JournalArticle.DT.value))
for chunk in gen:
if not chunk:
continue
chunk = chunk.get('Items')
iid, in_x = [], []
list(map(extract_x, chunk))
if iid and in_x:
self.total += len(in_x)
print(self, self.total)
yield {
'id': iid,
'input_x': in_x,
}
def get_result(self, raw_code):
brk_dict = get_brk(TechClassi.ID)
cla = brk_dict._query(
Key(TechClassi.ID.value).eq(
TechClassi.make_id(raw_code))).get('Items')
if not cla:
print('-- [error] TechClassi items not found for code:{}'.format(
raw_code))
return
cla = cla[0]
return {
'tech_domain_classifications': [{
TechClassi.ID.value: raw_code,
"name": cla['name'],
}]
}
def send(self, **kwargs):
self.recv_cnt += 1
iid = kwargs.get('id')
d6_code = kwargs['d6code']
try:
brk = get_brk(OrganizationTable.ID)
res = self.get_result(d6_code)
if not res:
return
print('++ [classi] {} {}'.format(res, self.recv_cnt))
brk._update({
OrganizationTable.ID.value: iid,
}, res)
print('++ [updated] {} {}'.format(iid, d6_code))
except Exception as ex:
print('-- [error] send failed iid:{} code:{}: {}'.format(
iid, d6_code, ex))
def send(self, **kwargs):
self.recv_cnt += 1
iid = kwargs.get('id')
cip = kwargs['cip']
try:
table, _ = JournalArticle.rebuild(id=iid)
brk = get_brk(JournalArticle.ID)
res = self.get_result(cip)
if not res:
return
print('++ [classi] {} {}'.format(res, self.recv_cnt))
brk._update({
JournalArticle.ID.value: iid,
}, res)
print('++ [updated] {} {}'.format(iid, cip))
except Exception as ex:
print('-- [error] send failed iid:{} code:{}: {}'.format(
iid, cip, ex))