def s3puc_dumpurls(pucfile): ends = '\.(cn|com|org|net|me)/?$' with open(pucfile, 'r') as fp: for line in fp.readlines(): try: if constant.DEBUG_FLAG == constant.DEBUG_FLAG_WINDOWS: url = line.strip() else: url = Common.urldec(json.loads(line.strip())['url']) if re.search(ends, url): continue NewsStorage.storeurl(url) except: Logger.printexception()
def __storeurllist__(self, urllist): for url in urllist: temp = url.strip().split('\t') if temp: url = temp[-1] NewsStorage.storeurl(url, True)