def s3puc_dumpurls(pucfile):
ends = '\.(cn|com|org|net|me)/?$'
with open(pucfile, 'r') as fp:
for line in fp.readlines():
try:
if constant.DEBUG_FLAG == constant.DEBUG_FLAG_WINDOWS:
url = line.strip()
else:
url = Common.urldec(json.loads(line.strip())['url'])
if re.search(ends, url):
continue
NewsStorage.storeurl(url)
except:
Logger.printexception()
def __storeurllist__(self, urllist):
for url in urllist:
temp = url.strip().split('\t')
if temp:
url = temp[-1]
NewsStorage.storeurl(url, True)
