class GetTimestampTest(unittest.TestCase): def test_use_timestamp(self): class CustomRequest: remote_user = '******' timestamp = NOW - 15*SECOND self.limiter = RateLimiter() with freeze_time() as mock_datetime: ts = self.limiter.get_timestamp(CustomRequest()) mock_datetime.utcnow.assert_called_once() self.assertEqual(ts, NOW - 15*SECOND) def test_optional_timestamp(self): class CustomRequest: remote_user = '******' self.limiter = RateLimiter() with freeze_time() as mock_datetime: ts = self.limiter.get_timestamp(CustomRequest()) mock_datetime.utcnow.assert_called_once() self.assertEqual(ts, mock_datetime.utcnow.return_value)
def _get_eutils_rate_limiter() -> "RateLimiter": """ Rate limiter to cap NCBI E-utilities queries to <= 3 per second as per https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/ """ from ratelimiter import RateLimiter if "CI" in os.environ: # multiple CI jobs might be running concurrently return RateLimiter(max_calls=1, period=1.5) return RateLimiter(max_calls=2, period=1)
def test_optional_timestamp(self): class CustomRequest: remote_user = '******' self.limiter = RateLimiter() with freeze_time() as mock_datetime: ts = self.limiter.get_timestamp(CustomRequest()) mock_datetime.utcnow.assert_called_once() self.assertEqual(ts, mock_datetime.utcnow.return_value)
def test_use_timestamp(self): class CustomRequest: remote_user = '******' timestamp = NOW - 15*SECOND self.limiter = RateLimiter() with freeze_time() as mock_datetime: ts = self.limiter.get_timestamp(CustomRequest()) mock_datetime.utcnow.assert_called_once() self.assertEqual(ts, NOW - 15*SECOND)
def _get_eutils_rate_limiter() -> "RateLimiter": """ Rate limiter to cap NCBI E-utilities queries to <= 3 per second as per https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/ """ with warnings.catch_warnings(): # https://github.com/RazerM/ratelimiter/issues/10 # https://github.com/manubot/manubot/issues/257 warnings.filterwarnings("ignore", category=DeprecationWarning) from ratelimiter import RateLimiter if "CI" in os.environ: # multiple CI jobs might be running concurrently return RateLimiter(max_calls=1, period=1.5) return RateLimiter(max_calls=2, period=1)
def test_use_cost(self): ratelimter = RateLimiter(InMemoryContext, max_calls=50, period=1.0) calls = [] count = 100 for i in range(count): with ratelimter.use_cost(2): pass calls.append(time.time()) elapsed = calls[-1] - calls[0] rate = count / elapsed expectedRate = 100
def main(): """Find all the projects in an organization.""" parser = argparse.ArgumentParser() parser.add_argument('--organization_id', help='Organization id') parser.add_argument('--lifecycle_state', help='Lifecycle State') args = parser.parse_args() rate_limiter = RateLimiter(400, 100) filter_params = {} if args.organization_id: filter_params['parent.type'] = 'organization' filter_params['parent.id'] = args.organization_id if args.lifecycle_state: filter_params['lifecycleState'] = args.lifecycle_state projects_api = resources_projects(resource_mgr_service(version='v1')) folders_api = resources_folders(resource_mgr_service(version='v2alpha1')) with rate_limiter: for (project, num_bindings) in _find_resources(projects_api, folders_api, **filter_params): print '{} ({}) - {}; {} IAM bindings, parent={}'.format( project['projectId'], project['projectNumber'], project['lifecycleState'], num_bindings, project.get('parent'))
def __init__(self, credentials=None, rate_limiter=None): super(CloudResourceManagerClient, self).__init__(credentials=credentials, api_name=self.API_NAME) if rate_limiter: self.rate_limiter = rate_limiter else: self.rate_limiter = RateLimiter(self.DEFAULT_MAX_QUERIES, 100)
def synchronize(): # update wfirma database credentials = Credentials.objects.all().first() username, password = credentials.login, credentials.password client = WFirmaClient(username, password) client.get_all_goods() rate_limiter = RateLimiter(max_calls=3, period=1) products_to_sync = Product.objects.all().filter(enabled=True).exclude( code__exact='') shops = Shop.objects.all() for shop in shops: click = ClickShopClient(shop) print(f"[INFO] SHOP: {shop.url}") products_in_shop = click.get_all_products() products_variants = [x for x in products_in_shop if x.options] variants = [] for product in products_variants: variants.extend(product.options) # TODO: SOMETHING GOES WRONG HERE MY MAN !!! product_sync = [ x for x in products_to_sync if x.code in [y.code for y in products_in_shop] ] products_in_shop = [ x for x in products_in_shop if x.code in [y.code for y in product_sync] ] print( f"[DEBUG] Sklep: {len(products_in_shop)} | WFIRMA: {len(product_sync)}" ) keyfun = operator.attrgetter("code") product_sync.sort(key=keyfun, reverse=True) products_in_shop.sort(key=keyfun, reverse=True) pairs = list(zip(product_sync, products_in_shop)) pairs = [(f, c) for f, c in pairs if int(f.available) != int(c.stock.stock)] for firma, clickshop in pairs: with rate_limiter: print( f"{firma.name}:{firma.code} -> {clickshop.code}:{clickshop.translations['pl_PL']['name']}" ) clickshop.stock.stock = int(firma.available) click.put_product(clickshop) # SELENIUM bot = Bot(shop.url, shop.username, shop.password, products_to_sync) length = len(variants) for index, variant in enumerate(variants): print(f"[INFO] {index}/{length}") bot.edit_variant(variant) bot.close()
def _get_eutils_rate_limiter(): """ Rate limiter to cap NCBI E-utilities queries to 3 per second as per https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/ """ from ratelimiter import RateLimiter return RateLimiter(max_calls=2, period=1)
def __iter__(self): rate_limiter = RateLimiter(max_calls=1, period=2) for params in self.job_params(): with metrics.job_timer('insights'): job = self.run_job(params) min_date_start_for_job = None count = 0 res = self.get_job_result(job) for obj in res: with rate_limiter: count += 1 rec = obj.export_all_data() if not min_date_start_for_job or rec[ 'date_stop'] < min_date_start_for_job: min_date_start_for_job = rec['date_stop'] yield {'record': rec} LOGGER.info('Got %d results for insights job', count) # when min_date_start_for_job stays None, we should # still update the bookmark using 'until' in time_ranges if min_date_start_for_job is None: for time_range in params['time_ranges']: if time_range['until']: min_date_start_for_job = time_range['until'] yield { 'state': advance_bookmark(self, self.bookmark_key, min_date_start_for_job) } # pylint: disable=no-member
def __init__(self, credentials=None, quota_max_calls=None, quota_period=None, use_rate_limiter=False, project_id=None, **kwargs): """Constructor. Args: api_name (str): The API name to wrap. More details here: https://developers.google.com/api-client-library/python/apis/ versions (list): A list of version strings to initialize. credentials (object): GoogleCredentials. quota_max_calls (int): Allowed requests per <quota_period> for the API. quota_period (float): The time period to track requests over. use_rate_limiter (bool): Set to false to disable the use of a rate limiter for this service. **kwargs (dict): Additional args such as version. """ self._use_cached_http = False if not credentials: # Only share the http object when using the default credentials. self._use_cached_http = True credentials = client.GoogleCredentials.get_application_default() self._credentials = _set_ua_and_scopes(credentials, 'custodian-gcp', '0.1') if use_rate_limiter: self._rate_limiter = RateLimiter(max_calls=quota_max_calls, period=quota_period) else: self._rate_limiter = None self.project_id = project_id
def get_user_repo_branches(): extract_json = { 'data_source_access_time': strftime("%Y%m%dT%H%M%SZ", gmtime()), 'data_source_name': 'GitHub API' } g = Github(get_config_value('github_access_token')) user = g.get_user() user_json = { 'login': user.login, 'public_repos': user.public_repos, 'repos': {} } repos = user.get_repos() rate_limiter = RateLimiter(max_calls=5000, period=3600) for i, repo in enumerate(repos): with rate_limiter: if i >= get_config_value('max_repos_to_download'): pass else: repo_name = repo.name repo_json = { 'name': repo_name, 'private': repo.private, 'forks_count': repo.forks_count, 'stargazers_count': repo.stargazers_count, 'watchers_count': repo.watchers_count, 'size': repo.size, 'open_issues_count': repo.open_issues_count, 'has_issues': repo.has_issues, 'has_projects': repo.has_projects, 'has_wiki': repo.has_wiki, 'has_downloads': repo.has_downloads, 'pushed_at': repo.pushed_at, 'created_at': repo.created_at, 'updated_at': repo.updated_at, 'branches': {} } branches = repo.get_branches() for branch in branches: branch_name = branch.name branch_commit = branch.commit branch_json = { 'name': branch_name, 'commit': { 'sha': branch_commit.sha, 'url': branch_commit.url }, 'protected': branch.protected, } repo_json['branches'][branch_name] = branch_json user_json['repos'][repo_name] = repo_json extract_json['data'] = user_json with open('data/user_repo_branches.json', 'w') as output_file: json.dump(extract_json, output_file, indent=4, default=str)
def get_data(in_size): url = "https://poloniex.com/public" currency_pairs = keys start_stamp = 1410158341 end_stamp = time.time() chunk_size = 600000 start_stamps = np.arange(start_stamp, end_stamp - chunk_size, chunk_size) end_stamps = np.arange(start_stamp + chunk_size, end_stamp, chunk_size) data_frames = {} ratelimiter = RateLimiter(max_calls=1, period=1 / 5) for pair in currency_pairs: frame = None all_data_lists = [] for start, end in zip(start_stamps, end_stamps): start_chunk = start end_chunk = end chunk_chunk = chunk_size while True: payload = {"command": "returnTradeHistory", "currencyPair": pair, "start": start_chunk, "end": end_chunk} with ratelimiter: req = get_poloniex_data(url, payload) data_list_dict = req.json() if len(data_list_dict) < 50000 and end_chunk == end: all_data_lists += data_list_dict print("Got all the items") break elif len(data_list_dict) < 50000: print("Appending {} items to list".format(len(data_list_dict))) all_data_lists += data_list_dict start_chunk = end_chunk end_chunk = start_chunk + chunk_chunk if end_chunk > end: end_chunk = end else: chunk_chunk = int(chunk_chunk / 2) start_chunk = start_chunk end_chunk = start_chunk + chunk_chunk print("Redoing the call.., chunk size is now: {}".format(chunk_chunk)) if end_chunk > end: end_chunk = end if all_data_lists: print("{} now has {} items".format(pair, len(all_data_lists))) print("For {} got {} items".format(pair, len(data_list_dict))) frame = pd.DataFrame(all_data_lists) print("Got data frame with {} items".format(len(frame))) data_frames[pair] = frame
def __init__(self): super(AdminDirectoryClient, self).__init__(credentials=self._build_credentials(), api_name=self.API_NAME) self.rate_limiter = RateLimiter( FLAGS.max_admin_api_calls_per_day, self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def get_rate_limiter(self): """Return an appriopriate rate limiter. Returns: object: The rate limiter. """ return RateLimiter(FLAGS.max_admin_api_calls_per_day, self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def __init__(self, username=None, password=None, session=None): self._session = self._session_class() self._session.headers.update(HEADERS) self.setup(username, password, session) self.logger = _make_logger(self.username) self.hold_requests = Lock() self._mutex = Lock() # todo add path regex self.limits = { 'a': RateLimiter(100, 3600), # login/logout 'b': RateLimiter(60, 3600), # like/follow/comment '.*': RateLimiter(5000, 3600) # all other requests }
def main(): """Find all the organizations.""" rate_limiter = RateLimiter(400, 100) orgs_api = resources_orgs(resource_mgr_service()) with rate_limiter: for response in _find_resources(orgs_api): print response
def __init__(self, credentials=None, version=None): # The beta api provides more complete firewall rules data. # TODO: Remove beta when it becomes GA. super(ComputeClient, self).__init__(credentials=credentials, api_name=self.API_NAME, version=version) self.rate_limiter = RateLimiter(FLAGS.max_compute_api_calls_per_second, 1)
def test_limit_2(self): calls = [] obj = RateLimiter(self.max_calls, self.period) for i in range(3 * self.max_calls): with obj: calls.append(time.time()) self.assertEqual(len(calls), 3 * self.max_calls) self.validate_call_times(calls, self.max_calls, self.period)
def __init__(self, **kwargs): super(CloudResourceManagerClient, self).__init__(api_name=self.API_NAME, **kwargs) # TODO: we will need multiple rate limiters when we need to invoke # the CRM write API for enforcement. self.rate_limiter = RateLimiter( FLAGS.max_crm_api_calls_per_100_seconds, self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def test_limit_3(self): with Timer() as timer: obj = RateLimiter(self.max_calls, self.period, consume=2) for _ in range(self.max_calls + 1): with obj: pass self.assertGreaterEqual(timer.duration, self.period * obj.consume) self.assertLessEqual(timer.duration, self.period * (obj.consume + 1))
def __init__(self, token: str, **kwargs): self.base_url = "https://api.dagpi.xyz" self.token = token self.loop = loop = kwargs.get('loop', None) or asyncio.get_event_loop() self.client = kwargs.get('session') or aiohttp.ClientSession(loop=loop) self.ratelimiter = RateLimiter(max_calls=60, period=60, callback=limited) self.user_agent = "AsyncDagpi v{__version__} Python/Python/ \
async def test_async(self): import asyncio ratelimter = RateLimiter(InMemoryContext, max_calls=50, period=1.0) calls = [] count = 100 for i in range(count): async with ratelimter.use_cost(2): await asyncio.sleep(0.001) calls.append(time.time()) elapsed = calls[-1] - calls[0] rate = count / elapsed expectedRate = 100
def test_limit_2(self): calls = [] obj = RateLimiter(self.max_calls, self.period) for _ in range(3 * self.max_calls): with obj: calls.extend([time.time()] * obj.consume) self.assertEqual(len(calls), 3 * self.max_calls) self.validate_call_times(calls, self.max_calls, self.period)
def __init__(self, config: CollectorConfig): self.metrics = config.metrics self.info_metrics = config.info_metrics self.client = AcsClient(ak=config.credential['access_key_id'], secret=config.credential['access_key_secret'], region_id=config.credential['region_id']) self.pool = ThreadPoolExecutor(max_workers=config.pool_size) self.rateLimiter = RateLimiter(max_calls=config.rate_limit) self.info_provider = InfoProvider(self.client)
def get_rate_limiter(self): """Return an appriopriate rate limiter. Returns: object: The rate limiter. """ return RateLimiter( self.global_configs.get('max_admin_api_calls_per_day'), self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def test_random(self): for _ in range(10): calls = [] obj = RateLimiter(self.max_calls, self.period) for i in range(random.randint(10, 50)): with obj: calls.append(time.time()) self.validate_call_times(calls, self.max_calls, self.period)
def get_rate_limiter(self): """Return an appropriate rate limiter. Returns: RateLimiter: The rate limiter. """ return RateLimiter( self.global_configs.get('max_bigquery_api_calls_per_100_seconds'), self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def __init__(self, config_path): self.data_dir = get_data_dir() self.log_dir = get_log_dir() self.config = ConfigReader(config_path) self.rate_limit_config = self.config["RateLimiter"] max_calls = int(self.rate_limit_config["max_calls"]) period = int(self.rate_limit_config["period"]) self.rate_limiter = RateLimiter(max_calls=max_calls, period=period)