def compare_to(ctx, cfr_title, cfr_part, api_base, path):
    """Compare local JSON to a remote server. This is useful for verifying
    changes to the parser.

    API_BASE is the uri of the root of the API. Use what would be the last
    parameter in the `write_to` command.

    PATH parameters will filter the files we're trying to compare. For
    example, if we only want to see the difference between trees, one of the
    PATH parameters should be "regulation".
    """
    if not api_base.endswith("/"):
        api_base += "/"

    tmppath = tempfile.mkdtemp()
    ctx.invoke(write_to, cfr_title=cfr_title, cfr_part=cfr_part,
               output=tmppath)

    # @todo: ugly to uninstall the cache after installing it in eregs.py.
    # Remove the globalness
    requests_cache.uninstall_cache()

    for file_name in files_to_compare(tmppath, path or ['']):
        local_name = os.path.join(tmppath, file_name)
        remote_name = api_base + file_name.replace(os.path.sep, "/")
        compare(local_name, remote_name)
    shutil.rmtree(tmppath)
Example #2
0
    def request(self, method, url, params=None, headers=None, to_json=True, data=None, **kwargs):
        """ Make request to TC API. """

        url, params, headers, data = self.prepare(url, params, headers, data)

        if self.options['cache']:
            rc.install_cache(self.options['cache'])

        elif type(self).cache_installed:
            rc.uninstall_cache()

        type(self).cache_installed = bool(self.options['cache'])

        try:
            response = rs.api.request(
                method, url, params=params, headers=headers, data=data, **kwargs)
            logger.debug(response.content)
            response.raise_for_status()
            if to_json:
                response = response.json()

        except (ValueError, rs.HTTPError):
            if locals().get('response') is not None:
                message = "%s: %s" % (response.status_code, response.content)
                raise TCException(message)
            raise

        return response
Example #3
0
    def reset_cache(self, cache_duration=None):
        """Remove any cached singles or albums charts

        Because the UK Top40 charts only change once per week, :py:class:`Top40` will cache the results of singles and
        albums. This means that during the execution of a program, repeated calls to retrieve singles and albums chart
        information will only actually call the remote API once. If, for whatever reason you need to ensure that an
        attempt to access single or album information actually results in a call to the remote API, then calling the
        :py:meth:`Top40.reset_cache` method will do this, by clearing down any existing cached chart information.

        If a cache is in place, then the results will also be cached across python runtime executions.

        Params:
            cache_duration (:py:class:`int`): If ``None`` we will uninstall the requests cache and the next
                read from the API will cause a remote call to be executed. Otherwise it specifies the number of
                seconds before the persistent cache will expire.
        """

        if cache_duration is None:
            # We are disabling the existing persistent_cache
            requests_cache.uninstall_cache()
        else:
            # We are setting a persistent cache so insert the duration into our cache config
            self.cache_config['expire_after'] = cache_duration

            # and then install the cache with this configuration
            requests_cache.install_cache(**self.cache_config)

        # Remember the new duration
        self.cache_duration = cache_duration

        # Rest the in-memory caches to force a read from remote site
        self._albums_chart = None
        self._singles_chart = None
    def test_hybrid_dois(self, test_data):

        (doi, fulltext_url, license, color) = test_data

        # because cookies breaks the cache pickling
        # for doi_start in ["10.1109", "10.1161", "10.1093", "10.1007", "10.1039"]:
        #     if doi.startswith(doi_start):
        requests_cache.uninstall_cache()

        my_pub = pub.lookup_product_by_doi(doi)
        my_pub.refresh()

        logger.info(u"\n\nwas looking for {}, got {}".format(fulltext_url, my_pub.fulltext_url))
        logger.info(u"https://api.unpaywall.org/v2/{}?email=me".format(doi))
        logger.info(u"doi: https://doi.org/{}".format(doi))
        logger.info(u"license: {}".format(my_pub.license))
        logger.info(u"oa_color: {}".format(my_pub.oa_color))
        logger.info(u"evidence: {}".format(my_pub.evidence))
        if my_pub.error:
            logger.info(my_pub.error)

        assert_equals(my_pub.error, "")
        assert_equals(my_pub.fulltext_url, fulltext_url)
        # assert_equals(my_pub.license, license)
        assert_equals(my_pub.error, "")
    def test_chorus_dois(self, test_data):

        doi = test_data

        # because cookies breaks the cache pickling
        # for doi_start in ["10.1109", "10.1161", "10.1093", "10.1007", "10.1039"]:
        #     if doi.startswith(doi_start):
        requests_cache.uninstall_cache()

        my_pub = pub.lookup_product_by_doi(doi)
        if not my_pub:
            logger.info(u"doi {} not in db, skipping".format(doi))
            return
        my_pub.refresh()

        logger.info(u"https://api.unpaywall.org/v2/{}?email=me".format(doi))
        logger.info(u"doi: https://doi.org/{}".format(doi))
        logger.info(u"license: {}".format(my_pub.best_license))
        logger.info(u"evidence: {}".format(my_pub.best_evidence))
        logger.info(u"host: {}".format(my_pub.best_host))
        if my_pub.error:
            logger.info(my_pub.error)

        assert_equals(my_pub.error, "")
        assert_is_not_none(my_pub.fulltext_url)
Example #6
0
def after_scenario(context, scenario):
    requests_cache.uninstall_cache()
    _clean_up_marathon_apps(context)
    _clean_up_chronos_jobs(context)
    _clean_up_mesos_cli_config(context)
    _clean_up_soa_dir(context)
    _clean_up_etc_paasta(context)
    _clean_up_zookeeper_autoscaling(context)
def full_tests():
    import nose

    mymods = ["", "tests"]  # nose essentially ignores the first arg to argv.
    mymods.extend(get_stevedore_module_names("eregs_ns.parser.test_suite"))

    requests_cache.uninstall_cache()

    nose.run(argv=mymods)
 def test_install_uninstall(self):
     for _ in range(2):
         requests_cache.install_cache(name=CACHE_NAME, backend=CACHE_BACKEND)
         self.assertTrue(isinstance(requests.Session(), CachedSession))
         self.assertTrue(isinstance(requests.sessions.Session(), CachedSession))
         self.assertTrue(isinstance(requests.session(), CachedSession))
         requests_cache.uninstall_cache()
         self.assertFalse(isinstance(requests.Session(), CachedSession))
         self.assertFalse(isinstance(requests.sessions.Session(), CachedSession))
         self.assertFalse(isinstance(requests.session(), CachedSession))
def full_tests():
    import pytest

    mymods = ["tests"]
    mymods.extend(get_stevedore_module_names("eregs_ns.parser.test_suite"))

    requests_cache.uninstall_cache()

    errno = pytest.main(["--pyargs"] + mymods)
    sys.exit(errno)
Example #10
0
 def ctx(self, **options):
     """ Redefine context. """
     _opts = dict(self.options)
     try:
         self.options.update(options)
         yield self
     finally:
         self.options = _opts
         if not self.options['cache'] and type(self).cache_installed:
             rc.uninstall_cache()
Example #11
0
def main():
    """Attempt to set up a list of marathon service instances given.
    Exits 1 if any service.instance deployment failed.
    This is done in the following order:

    - Load the marathon configuration
    - Connect to marathon
    - Do the following for each service.instance:
        - Load the service instance's configuration
        - Create the complete marathon job configuration
        - Deploy/bounce the service
        - Emit an event about the deployment to sensu"""

    args = parse_args()
    soa_dir = args.soa_dir
    if args.verbose:
        logging.basicConfig(level=logging.DEBUG)
    else:
        logging.basicConfig(level=logging.WARNING)

    # Setting up transparent cache for http API calls
    requests_cache.install_cache("setup_marathon_jobs", backend="memory")

    marathon_config = get_main_marathon_config()
    client = marathon_tools.get_marathon_client(marathon_config.get_url(), marathon_config.get_username(),
                                                marathon_config.get_password())
    marathon_apps = marathon_tools.get_all_marathon_apps(client, embed_failures=True)

    num_failed_deployments = 0
    for service_instance in args.service_instance_list:
        try:
            service, instance, _, __ = decompose_job_id(service_instance)
        except InvalidJobNameError:
            log.error("Invalid service instance specified. Format is service%sinstance." % SPACER)
            num_failed_deployments = num_failed_deployments + 1
        else:
            if deploy_marathon_service(service, instance, client, soa_dir, marathon_config, marathon_apps):
                num_failed_deployments = num_failed_deployments + 1

    requests_cache.uninstall_cache()

    log.debug("%d out of %d service.instances failed to deploy." %
              (num_failed_deployments, len(args.service_instance_list)))

    sys.exit(1 if num_failed_deployments else 0)
def compare_to(api_base, paths, prompt):
    """Compare local JSON to a remote server. This is useful for verifying
    changes to the parser.

    API_BASE is the uri of the root of the API. Use what would be the last
    parameter in the `write_to` command.

    PATH parameters indicate specific files or directories to use when
    comparing. For example, use `/some/path/to/regulation/555` to compare all
    versions of 555. Glob syntax works if your shell supports it"""
    if not api_base.endswith("/"):
        api_base += "/"

    # @todo: ugly to uninstall the cache after installing it in eregs.py.
    # Remove the globalness
    requests_cache.uninstall_cache()

    pairs = local_and_remote_generator(api_base, paths)
    return any([compare(local, remote, prompt) for local, remote in pairs])
Example #13
0
 def scan_documents(self, cached=False):
     if cached:
         requests_cache.install_cache()
     r = requests.get(URL_BASE + '/files/')
     if r.status_code != 200:
         raise Exception("Directory read failed")
     root = html.fromstring(r.content)
     links = root.xpath("//a")
     info_list = []
     for link_el in links:
         link = link_el.attrib['href']
         if not link.startswith('/files'):
             continue
         policymaker_id = link_el.text.split('_')[-1].strip()
         dir_list = self.scan_dir(link, policymaker_id)
         info_list = info_list + dir_list
     self.doc_list = info_list
     if cached:
         requests_cache.uninstall_cache()
     return info_list
Example #14
0
def disable_cache():
    """
    If the requests_cache package is available, uninstall the existing
    installed cache. Returns True if disable happened.
    """

    global _CACHE_INSTALLED

    if not _CACHE_INSTALLED:
        return False

    try:
        from requests_cache import uninstall_cache
        uninstall_cache()

    except ImportError:
        return False

    else:
        _CACHE_INSTALLED = False
        return True
Example #15
0
    def request(self, method, url, params=None, headers=None, **kwargs):
        """ Make request to Github API. """

        loglevel = self.options.get('loglevel', 'info')
        logger.setLevel(loglevel.upper())
        rs_logger.setLevel(loglevel.upper())

        if self.options['cache']:
            rc.install_cache(self.options['cache'])

        elif type(self).cache_installed:
            rc.uninstall_cache()

        type(self).cache_installed = bool(self.options['cache'])

        if self.options['mock'] and url in self.options['mock']:
            return self.__load_mock(self.options['mock'][url])

        url = 'https://%s/%s' % (self.options['domain'], url.strip('/'))

        _params = self.params
        if params is not None:
            _params.update(params)

        _headers = self.headers
        if headers is not None:
            _headers.update(headers)

        try:
            response = rs.api.request(
                method, url, params=_params, headers=_headers, **kwargs)
            logger.debug(response.content)
            response.raise_for_status()
            response = response.json()

        except (rs.HTTPError, ValueError):
            message = "%s: %s" % (response.status_code, response.content)
            raise OctocatException(message)

        return response
def test_policy(rulefiles):
    requests_cache.uninstall_cache()
    defense_api = CbDefenseAPI(profile="test")


    default_policies = [policy for policy in defense_api.select(Policy) if policy.name == "default"]
    new_policy = defense_api.create(Policy)
    new_policy.policy = default_policies[0].policy
    new_policy.name = "cbapi-python-test-%d" % time.time()
    new_policy.priorityLevel = "LOW"
    new_policy.description = "Test policy"
    new_policy.version = 2
    new_policy.save()

    for t in rulefiles:
        try:
            test_rule(new_policy, t)
            print("Added rule %s" % t)
        except Exception as e:
            print("Exception adding rule %s: %s" % (t, e))

    new_policy.delete()
 def setUp(self):
     requests_cache.install_cache(name=CACHE_NAME, backend=CACHE_BACKEND)
     requests.Session().cache.clear()
     requests_cache.uninstall_cache()
Example #18
0
 def stop_caching(self):
     ''' Stop caching.'''
     if self._cached and caching_avail:
         requests_cache.uninstall_cache()
         self._cached = False
     return
Example #19
0
 def use_cache(self, use):
     if use:
         requests_cache.install_cache('kosapy_cache', expire_after=24*60*60)
     else:
         requests_cache.uninstall_cache()
Example #20
0
def cache_enabled(*args, **kwargs):
    requests_cache.install_cache(*args, **kwargs)
    try:
        yield
    finally:
        requests_cache.uninstall_cache()
Example #21
0
 def setUp(self):
     self.s = CachedSession(CACHE_NAME, backend=CACHE_BACKEND, fast_save=FAST_SAVE)
     self.s.cache.clear()
     requests_cache.uninstall_cache()