def reset_cache(self, cache_duration=None): """Remove any cached singles or albums charts Because the UK Top40 charts only change once per week, :py:class:`Top40` will cache the results of singles and albums. This means that during the execution of a program, repeated calls to retrieve singles and albums chart information will only actually call the remote API once. If, for whatever reason you need to ensure that an attempt to access single or album information actually results in a call to the remote API, then calling the :py:meth:`Top40.reset_cache` method will do this, by clearing down any existing cached chart information. If a cache is in place, then the results will also be cached across python runtime executions. Params: cache_duration (:py:class:`int`): If ``None`` we will uninstall the requests cache and the next read from the API will cause a remote call to be executed. Otherwise it specifies the number of seconds before the persistent cache will expire. """ if cache_duration is None: # We are disabling the existing persistent_cache requests_cache.uninstall_cache() else: # We are setting a persistent cache so insert the duration into our cache config self.cache_config['expire_after'] = cache_duration # and then install the cache with this configuration requests_cache.install_cache(**self.cache_config) # Remember the new duration self.cache_duration = cache_duration # Rest the in-memory caches to force a read from remote site self._albums_chart = None self._singles_chart = None
def test_hybrid_dois(self, test_data): (doi, fulltext_url, license, color) = test_data # because cookies breaks the cache pickling # for doi_start in ["10.1109", "10.1161", "10.1093", "10.1007", "10.1039"]: # if doi.startswith(doi_start): requests_cache.uninstall_cache() my_pub = pub.lookup_product_by_doi(doi) my_pub.refresh() logger.info(u"\n\nwas looking for {}, got {}".format(fulltext_url, my_pub.fulltext_url)) logger.info(u"https://api.unpaywall.org/v2/{}?email=me".format(doi)) logger.info(u"doi: https://doi.org/{}".format(doi)) logger.info(u"license: {}".format(my_pub.license)) logger.info(u"oa_color: {}".format(my_pub.oa_color)) logger.info(u"evidence: {}".format(my_pub.evidence)) if my_pub.error: logger.info(my_pub.error) assert_equals(my_pub.error, "") assert_equals(my_pub.fulltext_url, fulltext_url) # assert_equals(my_pub.license, license) assert_equals(my_pub.error, "")
def compare_to(ctx, cfr_title, cfr_part, api_base, path): """Compare local JSON to a remote server. This is useful for verifying changes to the parser. API_BASE is the uri of the root of the API. Use what would be the last parameter in the `write_to` command. PATH parameters will filter the files we're trying to compare. For example, if we only want to see the difference between trees, one of the PATH parameters should be "regulation". """ if not api_base.endswith("/"): api_base += "/" tmppath = tempfile.mkdtemp() ctx.invoke(write_to, cfr_title=cfr_title, cfr_part=cfr_part, output=tmppath) # @todo: ugly to uninstall the cache after installing it in eregs.py. # Remove the globalness requests_cache.uninstall_cache() for file_name in files_to_compare(tmppath, path or ['']): local_name = os.path.join(tmppath, file_name) remote_name = api_base + file_name.replace(os.path.sep, "/") compare(local_name, remote_name) shutil.rmtree(tmppath)
def request(self, method, url, params=None, headers=None, to_json=True, data=None, **kwargs): """ Make request to TC API. """ url, params, headers, data = self.prepare(url, params, headers, data) if self.options['cache']: rc.install_cache(self.options['cache']) elif type(self).cache_installed: rc.uninstall_cache() type(self).cache_installed = bool(self.options['cache']) try: response = rs.api.request( method, url, params=params, headers=headers, data=data, **kwargs) logger.debug(response.content) response.raise_for_status() if to_json: response = response.json() except (ValueError, rs.HTTPError): if locals().get('response') is not None: message = "%s: %s" % (response.status_code, response.content) raise TCException(message) raise return response
def test_chorus_dois(self, test_data): doi = test_data # because cookies breaks the cache pickling # for doi_start in ["10.1109", "10.1161", "10.1093", "10.1007", "10.1039"]: # if doi.startswith(doi_start): requests_cache.uninstall_cache() my_pub = pub.lookup_product_by_doi(doi) if not my_pub: logger.info(u"doi {} not in db, skipping".format(doi)) return my_pub.refresh() logger.info(u"https://api.unpaywall.org/v2/{}?email=me".format(doi)) logger.info(u"doi: https://doi.org/{}".format(doi)) logger.info(u"license: {}".format(my_pub.best_license)) logger.info(u"evidence: {}".format(my_pub.best_evidence)) logger.info(u"host: {}".format(my_pub.best_host)) if my_pub.error: logger.info(my_pub.error) assert_equals(my_pub.error, "") assert_is_not_none(my_pub.fulltext_url)
def after_scenario(context, scenario): requests_cache.uninstall_cache() _clean_up_marathon_apps(context) _clean_up_chronos_jobs(context) _clean_up_mesos_cli_config(context) _clean_up_soa_dir(context) _clean_up_etc_paasta(context) _clean_up_zookeeper_autoscaling(context)
def full_tests(): import nose mymods = ["", "tests"] # nose essentially ignores the first arg to argv. mymods.extend(get_stevedore_module_names("eregs_ns.parser.test_suite")) requests_cache.uninstall_cache() nose.run(argv=mymods)
def test_install_uninstall(self): for _ in range(2): requests_cache.install_cache(name=CACHE_NAME, backend=CACHE_BACKEND) self.assertTrue(isinstance(requests.Session(), CachedSession)) self.assertTrue(isinstance(requests.sessions.Session(), CachedSession)) self.assertTrue(isinstance(requests.session(), CachedSession)) requests_cache.uninstall_cache() self.assertFalse(isinstance(requests.Session(), CachedSession)) self.assertFalse(isinstance(requests.sessions.Session(), CachedSession)) self.assertFalse(isinstance(requests.session(), CachedSession))
def ctx(self, **options): """ Redefine context. """ _opts = dict(self.options) try: self.options.update(options) yield self finally: self.options = _opts if not self.options['cache'] and type(self).cache_installed: rc.uninstall_cache()
def full_tests(): import pytest mymods = ["tests"] mymods.extend(get_stevedore_module_names("eregs_ns.parser.test_suite")) requests_cache.uninstall_cache() errno = pytest.main(["--pyargs"] + mymods) sys.exit(errno)
def main(): """Attempt to set up a list of marathon service instances given. Exits 1 if any service.instance deployment failed. This is done in the following order: - Load the marathon configuration - Connect to marathon - Do the following for each service.instance: - Load the service instance's configuration - Create the complete marathon job configuration - Deploy/bounce the service - Emit an event about the deployment to sensu""" args = parse_args() soa_dir = args.soa_dir if args.verbose: logging.basicConfig(level=logging.DEBUG) else: logging.basicConfig(level=logging.WARNING) # Setting up transparent cache for http API calls requests_cache.install_cache("setup_marathon_jobs", backend="memory") marathon_config = get_main_marathon_config() client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(), marathon_config.get_password()) marathon_apps = marathon_tools.get_all_marathon_apps(client, embed_failures=True) num_failed_deployments = 0 for service_instance in args.service_instance_list: try: service, instance, _, __ = decompose_job_id(service_instance) except InvalidJobNameError: log.error("Invalid service instance specified. Format is service%sinstance." % SPACER) num_failed_deployments = num_failed_deployments + 1 else: if deploy_marathon_service(service, instance, client, soa_dir, marathon_config, marathon_apps): num_failed_deployments = num_failed_deployments + 1 requests_cache.uninstall_cache() log.debug("%d out of %d service.instances failed to deploy." % (num_failed_deployments, len(args.service_instance_list))) sys.exit(1 if num_failed_deployments else 0)
def compare_to(api_base, paths, prompt): """Compare local JSON to a remote server. This is useful for verifying changes to the parser. API_BASE is the uri of the root of the API. Use what would be the last parameter in the `write_to` command. PATH parameters indicate specific files or directories to use when comparing. For example, use `/some/path/to/regulation/555` to compare all versions of 555. Glob syntax works if your shell supports it""" if not api_base.endswith("/"): api_base += "/" # @todo: ugly to uninstall the cache after installing it in eregs.py. # Remove the globalness requests_cache.uninstall_cache() pairs = local_and_remote_generator(api_base, paths) return any([compare(local, remote, prompt) for local, remote in pairs])
def main(): # The real request will only be made once; afterward, the cached response is used for i in range(5): response = requests.get('http://httpbin.org/get') # This is more obvious when calling a slow endpoint for i in range(5): response = requests.get('http://httpbin.org/delay/2') # Caching can be disabled if we want to get a fresh page and not cache it with requests_cache.disabled(): print(requests.get('http://httpbin.org/ip').text) # Get some debugging info about the cache print(requests_cache.get_cache()) print('Cached URLS:', requests_cache.get_cache().urls) # Uninstall to remove caching from all requests functions requests_cache.uninstall_cache()
def scan_documents(self, cached=False): if cached: requests_cache.install_cache() r = requests.get(URL_BASE + '/files/') if r.status_code != 200: raise Exception("Directory read failed") root = html.fromstring(r.content) links = root.xpath("//a") info_list = [] for link_el in links: link = link_el.attrib['href'] if not link.startswith('/files'): continue policymaker_id = link_el.text.split('_')[-1].strip() dir_list = self.scan_dir(link, policymaker_id) info_list = info_list + dir_list self.doc_list = info_list if cached: requests_cache.uninstall_cache() return info_list
def generate_csl_items(args, citation_df): """ General CSL (citeproc) items for standard_citations in citation_df. Writes references.json to disk and logs warnings for potential problems. """ # Read manual references (overrides) in JSON CSL manual_refs = read_manual_references(args.manual_references_path) requests_cache.install_cache(args.requests_cache_path, include_get_headers=True) cache = requests_cache.get_cache() if args.clear_requests_cache: logging.info('Clearing requests-cache') requests_cache.clear() logging.info(f'requests-cache starting with {len(cache.responses)} cached responses') csl_items = list() failures = list() for citation in citation_df.standard_citation.unique(): if citation in manual_refs: csl_items.append(manual_refs[citation]) continue try: citeproc = citation_to_citeproc(citation) csl_items.append(citeproc) except Exception as error: logging.exception(f'Citeproc retrieval failure for {citation}') failures.append(citation) logging.info(f'requests-cache finished with {len(cache.responses)} cached responses') requests_cache.uninstall_cache() if failures: message = 'Citeproc retrieval failed for:\n{}'.format( '\n'.join(failures)) logging.error(message) # Write JSON CSL bibliography for Pandoc. with args.references_path.open('w') as write_file: json.dump(csl_items, write_file, indent=2, ensure_ascii=False) write_file.write('\n') return csl_items
def disable_cache(): """ If the requests_cache package is available, uninstall the existing installed cache. Returns True if disable happened. """ global _CACHE_INSTALLED if not _CACHE_INSTALLED: return False try: from requests_cache import uninstall_cache uninstall_cache() except ImportError: return False else: _CACHE_INSTALLED = False return True
def request(self, method, url, params=None, headers=None, **kwargs): """ Make request to Github API. """ loglevel = self.options.get('loglevel', 'info') logger.setLevel(loglevel.upper()) rs_logger.setLevel(loglevel.upper()) if self.options['cache']: rc.install_cache(self.options['cache']) elif type(self).cache_installed: rc.uninstall_cache() type(self).cache_installed = bool(self.options['cache']) if self.options['mock'] and url in self.options['mock']: return self.__load_mock(self.options['mock'][url]) url = 'https://%s/%s' % (self.options['domain'], url.strip('/')) _params = self.params if params is not None: _params.update(params) _headers = self.headers if headers is not None: _headers.update(headers) try: response = rs.api.request( method, url, params=_params, headers=_headers, **kwargs) logger.debug(response.content) response.raise_for_status() response = response.json() except (rs.HTTPError, ValueError): message = "%s: %s" % (response.status_code, response.content) raise OctocatException(message) return response
def test_policy(rulefiles): requests_cache.uninstall_cache() defense_api = CbDefenseAPI(profile="test") default_policies = [policy for policy in defense_api.select(Policy) if policy.name == "default"] new_policy = defense_api.create(Policy) new_policy.policy = default_policies[0].policy new_policy.name = "cbapi-python-test-%d" % time.time() new_policy.priorityLevel = "LOW" new_policy.description = "Test policy" new_policy.version = 2 new_policy.save() for t in rulefiles: try: test_rule(new_policy, t) print("Added rule %s" % t) except Exception as e: print("Exception adding rule %s: %s" % (t, e)) new_policy.delete()
def _import_depfinder(): import os import yaml # https://github.com/ericdill/depfinder/pull/64/ if not hasattr(yaml, "CSafeLoader"): yaml.CSafeLoader = yaml.SafeLoader if os.getenv("CI"): import depfinder else: if "depfinder" not in sys.modules: import requests_cache dir = Path(__file__).parent requests_cache.install_cache(str(options.cache / "requests_cache")) dir.mkdir(parents=True, exist_ok=True) import depfinder requests_cache.uninstall_cache() else: import depfinder return depfinder
def cache_enabled(*args, **kwargs): requests_cache.install_cache(*args, **kwargs) try: yield finally: requests_cache.uninstall_cache()
def stop_caching(self): ''' Stop caching.''' if self._cached and caching_avail: requests_cache.uninstall_cache() self._cached = False return
def cache_requests(): testdir = os.path.dirname(os.path.abspath(__file__)) location = os.path.join(testdir, 'data/requests_cache') requests_cache.install_cache(cache_name=location) yield requests_cache.uninstall_cache()
def setUp(self): self.s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE) self.s.cache.clear() requests_cache.uninstall_cache()
def caching( cache=False, name=None, backend="sqlite", expire_after=86400, allowable_codes=(200, ), allowable_methods=("GET", ), ): """ pygbif caching management :param cache: [bool] if ``True`` all http requests are cached. if ``False`` (default), no http requests are cached. :param name: [str] the cache name. when backend=sqlite, this is the path for the sqlite file, ignored if sqlite not used. if not set, the file is put in your temporary directory, and therefore is cleaned up/deleted after closing your python session :param backend: [str] the backend, one of: - ``sqlite`` sqlite database (default) - ``memory`` not persistent, stores all data in Python dict in memory - ``mongodb`` (experimental) MongoDB database (pymongo < 3.0 required) - ``redis`` stores all data on a redis data store (redis required) :param expire_after: [str] timedelta or number of seconds after cache will be expired or None (default) to ignore expiration. default: 86400 seconds (24 hrs) :param allowable_codes: [tuple] limit caching only for response with this codes (default: 200) :param allowable_methods: [tuple] cache only requests of this methods (default: ‘GET’) :return: sets options to be used by pygbif, returns the options you selected in a hash Note: setting cache=False will turn off caching, but the backend data still persists. thus, you can turn caching back on without losing your cache. this also means if you want to delete your cache you have to do it yourself. Note: on loading pygbif, we clean up expired responses Usage:: import pygbif # caching is off by default from pygbif import occurrences %time z=occurrences.search(taxonKey = 3329049) %time w=occurrences.search(taxonKey = 3329049) # turn caching on pygbif.caching(True) %time z=occurrences.search(taxonKey = 3329049) %time w=occurrences.search(taxonKey = 3329049) # set a different backend pygbif.caching(cache=True, backend="redis") %time z=occurrences.search(taxonKey = 3329049) %time w=occurrences.search(taxonKey = 3329049) # set a different backend pygbif.caching(cache=True, backend="mongodb") %time z=occurrences.search(taxonKey = 3329049) %time w=occurrences.search(taxonKey = 3329049) # set path to a sqlite file pygbif.caching(name = "some/path/my_file") """ default_name = "pygbif_requests_cache" if not cache: requests_cache.uninstall_cache() CACHE_NAME = None else: if name is None and backend == "sqlite": CACHE_NAME = os.path.join(tempfile.gettempdir(), default_name) else: CACHE_NAME = default_name requests_cache.install_cache(cache_name=CACHE_NAME, backend=backend, expire_after=expire_after) remove_expired_responses() cache_settings = { "cache": cache, "name": CACHE_NAME, "backend": backend, "expire_after": expire_after, "allowable_codes": allowable_codes, "allowable_methods": allowable_methods, } return cache_settings
#Specify parameters for api request params = { 'key': 'shkEPTAKIwt6G92CdZAUEuw8U9pWbwfZ', } #Use computer memory for api caching requests_cache.install_cache(backend='memory') #Read input excel file and create a list of urls read_df = pd.read_excel(sys.argv[1], engine='openpyxl') urlList = [( f'https://api.tomtom.com/search/2/reverseGeocode/{read_df.loc[x,["Latitude"]].values[0]},' f'{read_df.loc[x,["Longitude"]].values[0]}.JSON') for x in range(len(read_df.index))] #Use multithreading to excute the requests in parellel. with concurrent.futures.ThreadPoolExecutor(max_workers=3) as executor: results = executor.map(get_address, urlList) data = [f'{val["streetNumber"]} {val["streetName"]}, {val["municipality"]} {val["postalCode"]}'\ for val in results] #Print addresses for address in data: sys.stdout.write(address + '\n') #Remove cache requests_cache.uninstall_cache()
def use_cache(self, use): if use: requests_cache.install_cache('kosapy_cache', expire_after=24*60*60) else: requests_cache.uninstall_cache()
def generate_csl_items( citekeys: list, manual_refs: dict = {}, requests_cache_path: Optional[str] = None, clear_requests_cache: Optional[bool] = False, ) -> list: """ General CSL (citeproc) items for standard_citekeys in citekeys_df. Parameters: - citekeys: list of standard_citekeys - manual_refs: mapping from standard_citekey to csl_item for manual references - requests_cache_path: path for the requests cache database. Passed as cache_name to `requests_cache.install_cache`. requests_cache may append an extension to this path, so it is not always the exact path to the cache. If None, do not use requests_cache. - clear_requests_cache: If True, clear the requests cache before generating citekey metadata. """ # Deduplicate citations citekeys = list(dict.fromkeys(citekeys)) # Install cache if requests_cache_path is not None: requests # require `import requests` in case this is essential for monkey patching by requests_cache. requests_cache.install_cache(requests_cache_path, include_get_headers=True) cache = requests_cache.get_cache() if clear_requests_cache: logging.info("Clearing requests-cache") requests_cache.clear() logging.info( f"requests-cache starting with {len(cache.responses)} cached responses" ) csl_items = list() failures = list() for standard_citekey in citekeys: if standard_citekey in manual_refs: csl_items.append(manual_refs[standard_citekey]) continue elif standard_citekey.startswith("raw:"): logging.error( f"CSL JSON Data with a standard_citekey of {standard_citekey!r} not found in manual-references.json. " "Metadata must be provided for raw citekeys.") failures.append(standard_citekey) try: csl_item = citekey_to_csl_item(standard_citekey) csl_items.append(csl_item) except Exception: logging.exception( f"Citeproc retrieval failure for {standard_citekey!r}") failures.append(standard_citekey) # Uninstall cache if requests_cache_path is not None: logging.info( f"requests-cache finished with {len(cache.responses)} cached responses" ) requests_cache.uninstall_cache() if failures: message = "CSL JSON Data retrieval failed for the following standardized citation keys:\n{}".format( "\n".join(failures)) logging.error(message) return csl_items
def setUp(self): requests_cache.uninstall_cache() self.tls_adapter = CbAPISessionAdapter(force_tls_1_2=True) self.session = requests.Session() self.session.mount("https://", self.tls_adapter)
def use_cache(self, use): if use: requests_cache.install_cache('kosapy_cache', expire_after=24 * 60 * 60) else: requests_cache.uninstall_cache()
def _stop_caching(self): '''Stop caching.''' if self._cached and caching_avail: requests_cache.uninstall_cache() self._cached = False return
def setUp(self): requests_cache.install_cache(name=CACHE_NAME, backend=CACHE_BACKEND) requests.Session().cache.clear() requests_cache.uninstall_cache()
def tearDown(self): requests_cache.uninstall_cache()
def __exit__(self, type, value, traceback): requests_cache.uninstall_cache()
def test_remove_expired_responses(remove_expired_responses): requests_cache.install_cache(expire_after=360) requests_cache.remove_expired_responses() assert remove_expired_responses.called is True requests_cache.uninstall_cache()
def close(self): """uninstall cache""" logging.info( f"requests-cache finished with {len(self.cache.responses)} cached responses" ) requests_cache.uninstall_cache()
def disable_cache(self) -> None: requests_cache.disable_cache() requests_cache.uninstall_cache() self.cache_enabled = False