def __init__(self, enable_debug): self.log = logging.getLogger(__name__) today = datetime.date.today() previousMonth = today - datetime.timedelta(days=31) prevoiusMonthSubDir = previousMonth.strftime("%Y-%m") cache_dir_to_remove = os.path.join(os.path.normpath(os.path.expanduser("~/.epguide/cache")), prevoiusMonthSubDir) if os.path.exists(cache_dir_to_remove): self.log.info("Removing previous month cache dir: " + cache_dir_to_remove) try: shutil.rmtree(cache_dir_to_remove) self.log.info("Removed.") except Exception: self.log.exception("Exception while removing dir " + cache_dir_to_remove) else: self.log.info("Previous month cache dir not exists: " + cache_dir_to_remove) thisMonthSubDir = today.strftime("%Y-%m") cache_dir = os.path.join(os.path.normpath(os.path.expanduser("~/.epguide/cache")), thisMonthSubDir) self.log.info("Cache dir: " + cache_dir) self.cache = FileCache(cache_dir) self.http = Http(cache = self.cache) self.user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0" if(enable_debug): httplib2.debuglevel = 255
def test_epgguide(self): cache_dir = os.getcwd() + "/.epguide" # os.makedirs(cache_dir) print cache_dir cache = FileCache(cache_dir) h = Http(cache=cache) user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0" httplib2.debuglevel = 255 resp, content = h.request( "http://www.teleman.pl/program-tv/stacje/TVP-2", headers={'user-agent': user_agent}) print "content:" print content print "response:" print resp print "fromcache:" + str(resp.fromcache) print "status:" + str(resp.status) #url = "http://www.teleman.pl/program-tv/stacje/TVP-2" url = "http://www.teleman.pl/tv/Dr-House-7-152-885990" resp, content = h.request(url, headers={'user-agent': user_agent}) print "fromcache:" + str(resp.fromcache) print "status:" + str(resp.status) safe = httplib2.safename(url) print "safe:" + safe cached_value = cache.get(url) info, cached = cached_value.split('\r\n\r\n', 1) print "====" print content print "====" print cached print "====" self.assertEqual(content, cached)
def test_epgguide(self): cache_dir = os.getcwd() + "/.epguide" # os.makedirs(cache_dir) print cache_dir cache = FileCache(cache_dir) h = Http(cache = cache) user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0" httplib2.debuglevel = 255 resp, content = h.request("http://www.teleman.pl/program-tv/stacje/TVP-2", headers={'user-agent':user_agent}) print "content:" print content print "response:" print resp print "fromcache:" + str(resp.fromcache) print "status:" + str(resp.status) #url = "http://www.teleman.pl/program-tv/stacje/TVP-2" url = "http://www.teleman.pl/tv/Dr-House-7-152-885990" resp, content = h.request(url, headers={'user-agent':user_agent}) print "fromcache:" + str(resp.fromcache) print "status:" + str(resp.status) safe = httplib2.safename(url) print "safe:" + safe cached_value = cache.get(url) info, cached = cached_value.split('\r\n\r\n', 1) print "====" print content print "====" print cached print "====" self.assertEqual(content, cached)
def do_GET(self): from httplib2 import FileCache logging.basicConfig(level=logging.DEBUG) store = FileCache("cache4") opener = urllib2.build_opener(CacheHandle(store)) urllib2.install_opener(opener) response = opener.open(self.path) (scm, netloc, path, params, query, fragment) = urlparse.urlparse(self.path, 'http') if scm not in ('http', 'ftp') or fragment or not netloc: self.send_error(400, "bad url %s" % self.path) return soc = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: if scm == 'http': if self._connect_to(netloc, soc): self.log_request() soc.send("%s %s %s\r\n" % (self.command, urlparse.urlunparse(('', '', path, params, query, '')), self.request_version)) self.headers['Connection'] = 'close' del self.headers['Proxy-Connection'] for key_val in self.headers.items(): soc.send("%s: %s\r\n" % key_val) soc.send("\r\n") self._read_write(soc) finally: soc.close() self.connection.close()
class HttpHelper(object): ''' classdocs ''' def __init__(self, enable_debug): self.log = logging.getLogger(__name__) today = datetime.date.today() previousMonth = today - datetime.timedelta(days=31) prevoiusMonthSubDir = previousMonth.strftime("%Y-%m") cache_dir_to_remove = os.path.join(os.path.normpath(os.path.expanduser("~/.epguide/cache")), prevoiusMonthSubDir) if os.path.exists(cache_dir_to_remove): self.log.info("Removing previous month cache dir: " + cache_dir_to_remove) try: shutil.rmtree(cache_dir_to_remove) self.log.info("Removed.") except Exception: self.log.exception("Exception while removing dir " + cache_dir_to_remove) else: self.log.info("Previous month cache dir not exists: " + cache_dir_to_remove) thisMonthSubDir = today.strftime("%Y-%m") cache_dir = os.path.join(os.path.normpath(os.path.expanduser("~/.epguide/cache")), thisMonthSubDir) self.log.info("Cache dir: " + cache_dir) self.cache = FileCache(cache_dir) self.http = Http(cache = self.cache) self.user_agent = "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0" if(enable_debug): httplib2.debuglevel = 255 def get(self, url, force_cache = False, charset = "UTF-8"): if(force_cache): cached_value = self.cache.get(url) if cached_value: try: info, content = cached_value.split('\r\n\r\n', 1) if(content): return content.decode(charset) else: return self.get(url, False, charset) except (IndexError, ValueError): return self.get(url, False, charset) else: return self.get(url, False, charset) else: resp, content = self.http.request(url, headers={'user-agent':self.user_agent}) self.log.debug("result content: %s response: %s fromcache: %s status: %s)" % (content, resp, str(resp.fromcache), str(resp.status))) c = content.decode(charset) return c
def __init__(self, credentials): super().__init__() # do the authentication outside assert credentials is not None and credentials.invalid is False, "Invalid or misssing credentials" cache = FileCache(osJoin(gettempdir(), ".httpcache"), safe=_SafeCacheName) http = Http(cache, timeout=60) http = credentials.authorize(http) self.drive = build("drive", "v3", http=http) _meta = self._meta = { "case_insensitive": False, # it will even let you have 2 identical filenames in the same directory! "invalid_path_chars": ":", # not sure what else "max_path_length": None, # don't know what the limit is "max_sys_path_length": None, # there's no syspath "network": True, "read_only": False, "supports_rename": False # since we don't have a syspath... }