def is_similar(self, columns, new, old):
res = True
for key in columns:
LOG.debug("Comparing[%s],new is [%s],old is [%s]" % (key, new[key], old[key]))
if new[key] == old[key]:
continue
if new[key] is None or old[key] is None:
res = False
break
if new[key] in old[key] or old[key] in new[key]:
continue
if StrSim.get_sim(str(new[key]), str(old[key])) < THRESHOLD:
res = False
break
return res
def deal_resp(self,resp):
entrys = resp.strip('\n').split("\n")
new_entrys = []
LOG.info("[%s] Entrys We Totally Get From DMG"%len(entrys))
for entry in entrys:
new_entry = {}
try:
entry = json.loads(entry)
specials = entry.get('specials')
origins = entry.get('contentOrigns')
if origins != None:
origins = origins[0]
if specials != None:
new_entry.update(specials)
if origins != None:
new_entry.update(origins)
#LOG.debug(new_entry)
new_entry = self.convertor.dmg_to_db(new_entry)
except Exception,e:
LOG.debug(e)
LOG.debug("broken json is:[%s]"%entry)
if len(new_entry) != 0:
new_entrys.append(new_entry)
