def storeurls(self, urls, request=constant.REQUEST_TYPE_COMMON): urlfile = URLFileManager.getinstance().geturlfilepath(request) if FileUtility.geturlfilelines(urlfile) + len( urls) > URLFileManager.URL_FILE_LINES_MAX_NUMBER: URLFileManager.getinstance().generateurlfilepath() urlfile = URLFileManager.getinstance().geturlfilepath(request) FileUtility.writelines(urlfile, urls)
def storeurl(self, url, urlcontext, request=constant.REQUEST_TYPE_COMMON): if url.strip(): urlfile = URLFileManager.getinstance().geturlfilepath(request) if FileUtility.geturlfilelines( urlfile) + 1 > URLFileManager.URL_FILE_LINES_MAX_NUMBER: URLFileManager.getinstance().generateurlfilepath() urlfile = URLFileManager.getinstance().geturlfilepath(request) FileUtility.writeline(urlfile, url) key = Common.md5(url.strip()) if key not in self.urlcontextdict: self.urlcontextdict[key] = [] self.urlcontextdict[key].append(urlcontext)