def storeurls(self, urls, request=constant.REQUEST_TYPE_COMMON):
urlfile = URLFileManager.getinstance().geturlfilepath(request)
if FileUtility.geturlfilelines(urlfile) + len(
urls) > URLFileManager.URL_FILE_LINES_MAX_NUMBER:
URLFileManager.getinstance().generateurlfilepath()
urlfile = URLFileManager.getinstance().geturlfilepath(request)
FileUtility.writelines(urlfile, urls)
def storeurl(self, url, urlcontext, request=constant.REQUEST_TYPE_COMMON):
if url.strip():
urlfile = URLFileManager.getinstance().geturlfilepath(request)
if FileUtility.geturlfilelines(
urlfile) + 1 > URLFileManager.URL_FILE_LINES_MAX_NUMBER:
URLFileManager.getinstance().generateurlfilepath()
urlfile = URLFileManager.getinstance().geturlfilepath(request)
FileUtility.writeline(urlfile, url)
key = Common.md5(url.strip())
if key not in self.urlcontextdict:
self.urlcontextdict[key] = []
self.urlcontextdict[key].append(urlcontext)