def is_similar(self, columns, new, old): res = True for key in columns: LOG.debug("Comparing[%s],new is [%s],old is [%s]" % (key, new[key], old[key])) if new[key] == old[key]: continue if new[key] is None or old[key] is None: res = False break if new[key] in old[key] or old[key] in new[key]: continue if StrSim.get_sim(str(new[key]), str(old[key])) < THRESHOLD: res = False break return res
def deal_resp(self,resp): entrys = resp.strip('\n').split("\n") new_entrys = [] LOG.info("[%s] Entrys We Totally Get From DMG"%len(entrys)) for entry in entrys: new_entry = {} try: entry = json.loads(entry) specials = entry.get('specials') origins = entry.get('contentOrigns') if origins != None: origins = origins[0] if specials != None: new_entry.update(specials) if origins != None: new_entry.update(origins) #LOG.debug(new_entry) new_entry = self.convertor.dmg_to_db(new_entry) except Exception,e: LOG.debug(e) LOG.debug("broken json is:[%s]"%entry) if len(new_entry) != 0: new_entrys.append(new_entry)