class GetTimestampTest(unittest.TestCase):
def test_use_timestamp(self):
class CustomRequest:
remote_user = '******'
timestamp = NOW - 15*SECOND
self.limiter = RateLimiter()
with freeze_time() as mock_datetime:
ts = self.limiter.get_timestamp(CustomRequest())
mock_datetime.utcnow.assert_called_once()
self.assertEqual(ts, NOW - 15*SECOND)
def test_optional_timestamp(self):
class CustomRequest:
remote_user = '******'
self.limiter = RateLimiter()
with freeze_time() as mock_datetime:
ts = self.limiter.get_timestamp(CustomRequest())
mock_datetime.utcnow.assert_called_once()
self.assertEqual(ts, mock_datetime.utcnow.return_value)
def _get_eutils_rate_limiter() -> "RateLimiter":
"""
Rate limiter to cap NCBI E-utilities queries to <= 3 per second as per
https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/
"""
from ratelimiter import RateLimiter
if "CI" in os.environ:
# multiple CI jobs might be running concurrently
return RateLimiter(max_calls=1, period=1.5)
return RateLimiter(max_calls=2, period=1)
def test_optional_timestamp(self):
class CustomRequest:
remote_user = '******'
self.limiter = RateLimiter()
with freeze_time() as mock_datetime:
ts = self.limiter.get_timestamp(CustomRequest())
mock_datetime.utcnow.assert_called_once()
self.assertEqual(ts, mock_datetime.utcnow.return_value)
def test_use_timestamp(self):
class CustomRequest:
remote_user = '******'
timestamp = NOW - 15*SECOND
self.limiter = RateLimiter()
with freeze_time() as mock_datetime:
ts = self.limiter.get_timestamp(CustomRequest())
mock_datetime.utcnow.assert_called_once()
self.assertEqual(ts, NOW - 15*SECOND)
def _get_eutils_rate_limiter() -> "RateLimiter":
"""
Rate limiter to cap NCBI E-utilities queries to <= 3 per second as per
https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/
"""
with warnings.catch_warnings():
# https://github.com/RazerM/ratelimiter/issues/10
# https://github.com/manubot/manubot/issues/257
warnings.filterwarnings("ignore", category=DeprecationWarning)
from ratelimiter import RateLimiter
if "CI" in os.environ:
# multiple CI jobs might be running concurrently
return RateLimiter(max_calls=1, period=1.5)
return RateLimiter(max_calls=2, period=1)
def test_use_cost(self):
ratelimter = RateLimiter(InMemoryContext, max_calls=50, period=1.0)
calls = []
count = 100
for i in range(count):
with ratelimter.use_cost(2):
pass
calls.append(time.time())
elapsed = calls[-1] - calls[0]
rate = count / elapsed
expectedRate = 100
def main():
"""Find all the projects in an organization."""
parser = argparse.ArgumentParser()
parser.add_argument('--organization_id', help='Organization id')
parser.add_argument('--lifecycle_state', help='Lifecycle State')
args = parser.parse_args()
rate_limiter = RateLimiter(400, 100)
filter_params = {}
if args.organization_id:
filter_params['parent.type'] = 'organization'
filter_params['parent.id'] = args.organization_id
if args.lifecycle_state:
filter_params['lifecycleState'] = args.lifecycle_state
projects_api = resources_projects(resource_mgr_service(version='v1'))
folders_api = resources_folders(resource_mgr_service(version='v2alpha1'))
with rate_limiter:
for (project, num_bindings) in _find_resources(projects_api,
folders_api,
**filter_params):
print '{} ({}) - {}; {} IAM bindings, parent={}'.format(
project['projectId'], project['projectNumber'],
project['lifecycleState'], num_bindings, project.get('parent'))
def __init__(self, credentials=None, rate_limiter=None):
super(CloudResourceManagerClient,
self).__init__(credentials=credentials, api_name=self.API_NAME)
if rate_limiter:
self.rate_limiter = rate_limiter
else:
self.rate_limiter = RateLimiter(self.DEFAULT_MAX_QUERIES, 100)
def synchronize():
# update wfirma database
credentials = Credentials.objects.all().first()
username, password = credentials.login, credentials.password
client = WFirmaClient(username, password)
client.get_all_goods()
rate_limiter = RateLimiter(max_calls=3, period=1)
products_to_sync = Product.objects.all().filter(enabled=True).exclude(
code__exact='')
shops = Shop.objects.all()
for shop in shops:
click = ClickShopClient(shop)
print(f"[INFO] SHOP: {shop.url}")
products_in_shop = click.get_all_products()
products_variants = [x for x in products_in_shop if x.options]
variants = []
for product in products_variants:
variants.extend(product.options)
# TODO: SOMETHING GOES WRONG HERE MY MAN !!!
product_sync = [
x for x in products_to_sync
if x.code in [y.code for y in products_in_shop]
]
products_in_shop = [
x for x in products_in_shop
if x.code in [y.code for y in product_sync]
]
print(
f"[DEBUG] Sklep: {len(products_in_shop)} | WFIRMA: {len(product_sync)}"
)
keyfun = operator.attrgetter("code")
product_sync.sort(key=keyfun, reverse=True)
products_in_shop.sort(key=keyfun, reverse=True)
pairs = list(zip(product_sync, products_in_shop))
pairs = [(f, c) for f, c in pairs
if int(f.available) != int(c.stock.stock)]
for firma, clickshop in pairs:
with rate_limiter:
print(
f"{firma.name}:{firma.code} -> {clickshop.code}:{clickshop.translations['pl_PL']['name']}"
)
clickshop.stock.stock = int(firma.available)
click.put_product(clickshop)
# SELENIUM
bot = Bot(shop.url, shop.username, shop.password, products_to_sync)
length = len(variants)
for index, variant in enumerate(variants):
print(f"[INFO] {index}/{length}")
bot.edit_variant(variant)
bot.close()
def _get_eutils_rate_limiter():
"""
Rate limiter to cap NCBI E-utilities queries to 3 per second as per
https://ncbiinsights.ncbi.nlm.nih.gov/2017/11/02/new-api-keys-for-the-e-utilities/
"""
from ratelimiter import RateLimiter
return RateLimiter(max_calls=2, period=1)
def __iter__(self):
rate_limiter = RateLimiter(max_calls=1, period=2)
for params in self.job_params():
with metrics.job_timer('insights'):
job = self.run_job(params)
min_date_start_for_job = None
count = 0
res = self.get_job_result(job)
for obj in res:
with rate_limiter:
count += 1
rec = obj.export_all_data()
if not min_date_start_for_job or rec[
'date_stop'] < min_date_start_for_job:
min_date_start_for_job = rec['date_stop']
yield {'record': rec}
LOGGER.info('Got %d results for insights job', count)
# when min_date_start_for_job stays None, we should
# still update the bookmark using 'until' in time_ranges
if min_date_start_for_job is None:
for time_range in params['time_ranges']:
if time_range['until']:
min_date_start_for_job = time_range['until']
yield {
'state':
advance_bookmark(self, self.bookmark_key,
min_date_start_for_job)
} # pylint: disable=no-member
def __init__(self,
credentials=None,
quota_max_calls=None,
quota_period=None,
use_rate_limiter=False,
project_id=None,
**kwargs):
"""Constructor.
Args:
api_name (str): The API name to wrap. More details here:
https://developers.google.com/api-client-library/python/apis/
versions (list): A list of version strings to initialize.
credentials (object): GoogleCredentials.
quota_max_calls (int): Allowed requests per <quota_period> for the
API.
quota_period (float): The time period to track requests over.
use_rate_limiter (bool): Set to false to disable the use of a rate
limiter for this service.
**kwargs (dict): Additional args such as version.
"""
self._use_cached_http = False
if not credentials:
# Only share the http object when using the default credentials.
self._use_cached_http = True
credentials = client.GoogleCredentials.get_application_default()
self._credentials = _set_ua_and_scopes(credentials, 'custodian-gcp',
'0.1')
if use_rate_limiter:
self._rate_limiter = RateLimiter(max_calls=quota_max_calls,
period=quota_period)
else:
self._rate_limiter = None
self.project_id = project_id
def get_user_repo_branches():
extract_json = {
'data_source_access_time': strftime("%Y%m%dT%H%M%SZ", gmtime()),
'data_source_name': 'GitHub API'
}
g = Github(get_config_value('github_access_token'))
user = g.get_user()
user_json = {
'login': user.login,
'public_repos': user.public_repos,
'repos': {}
}
repos = user.get_repos()
rate_limiter = RateLimiter(max_calls=5000, period=3600)
for i, repo in enumerate(repos):
with rate_limiter:
if i >= get_config_value('max_repos_to_download'):
pass
else:
repo_name = repo.name
repo_json = {
'name': repo_name,
'private': repo.private,
'forks_count': repo.forks_count,
'stargazers_count': repo.stargazers_count,
'watchers_count': repo.watchers_count,
'size': repo.size,
'open_issues_count': repo.open_issues_count,
'has_issues': repo.has_issues,
'has_projects': repo.has_projects,
'has_wiki': repo.has_wiki,
'has_downloads': repo.has_downloads,
'pushed_at': repo.pushed_at,
'created_at': repo.created_at,
'updated_at': repo.updated_at,
'branches': {}
}
branches = repo.get_branches()
for branch in branches:
branch_name = branch.name
branch_commit = branch.commit
branch_json = {
'name': branch_name,
'commit': {
'sha': branch_commit.sha,
'url': branch_commit.url
},
'protected': branch.protected,
}
repo_json['branches'][branch_name] = branch_json
user_json['repos'][repo_name] = repo_json
extract_json['data'] = user_json
with open('data/user_repo_branches.json', 'w') as output_file:
json.dump(extract_json, output_file, indent=4, default=str)
def get_data(in_size):
url = "https://poloniex.com/public"
currency_pairs = keys
start_stamp = 1410158341
end_stamp = time.time()
chunk_size = 600000
start_stamps = np.arange(start_stamp, end_stamp - chunk_size, chunk_size)
end_stamps = np.arange(start_stamp + chunk_size, end_stamp, chunk_size)
data_frames = {}
ratelimiter = RateLimiter(max_calls=1, period=1 / 5)
for pair in currency_pairs:
frame = None
all_data_lists = []
for start, end in zip(start_stamps, end_stamps):
start_chunk = start
end_chunk = end
chunk_chunk = chunk_size
while True:
payload = {"command": "returnTradeHistory", "currencyPair": pair,
"start": start_chunk, "end": end_chunk}
with ratelimiter:
req = get_poloniex_data(url, payload)
data_list_dict = req.json()
if len(data_list_dict) < 50000 and end_chunk == end:
all_data_lists += data_list_dict
print("Got all the items")
break
elif len(data_list_dict) < 50000:
print("Appending {} items to list".format(len(data_list_dict)))
all_data_lists += data_list_dict
start_chunk = end_chunk
end_chunk = start_chunk + chunk_chunk
if end_chunk > end:
end_chunk = end
else:
chunk_chunk = int(chunk_chunk / 2)
start_chunk = start_chunk
end_chunk = start_chunk + chunk_chunk
print("Redoing the call.., chunk size is now: {}".format(chunk_chunk))
if end_chunk > end:
end_chunk = end
if all_data_lists:
print("{} now has {} items".format(pair,
len(all_data_lists)))
print("For {} got {} items".format(pair, len(data_list_dict)))
frame = pd.DataFrame(all_data_lists)
print("Got data frame with {} items".format(len(frame)))
data_frames[pair] = frame
def __init__(self):
super(AdminDirectoryClient,
self).__init__(credentials=self._build_credentials(),
api_name=self.API_NAME)
self.rate_limiter = RateLimiter(
FLAGS.max_admin_api_calls_per_day,
self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def get_rate_limiter(self):
"""Return an appriopriate rate limiter.
Returns:
object: The rate limiter.
"""
return RateLimiter(FLAGS.max_admin_api_calls_per_day,
self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def __init__(self, username=None, password=None, session=None):
self._session = self._session_class()
self._session.headers.update(HEADERS)
self.setup(username, password, session)
self.logger = _make_logger(self.username)
self.hold_requests = Lock()
self._mutex = Lock()
# todo add path regex
self.limits = {
'a': RateLimiter(100, 3600), # login/logout
'b': RateLimiter(60, 3600), # like/follow/comment
'.*': RateLimiter(5000, 3600) # all other requests
}
def main():
"""Find all the organizations."""
rate_limiter = RateLimiter(400, 100)
orgs_api = resources_orgs(resource_mgr_service())
with rate_limiter:
for response in _find_resources(orgs_api):
print response
def __init__(self, credentials=None, version=None):
# The beta api provides more complete firewall rules data.
# TODO: Remove beta when it becomes GA.
super(ComputeClient, self).__init__(credentials=credentials,
api_name=self.API_NAME,
version=version)
self.rate_limiter = RateLimiter(FLAGS.max_compute_api_calls_per_second,
1)
def test_limit_2(self):
calls = []
obj = RateLimiter(self.max_calls, self.period)
for i in range(3 * self.max_calls):
with obj:
calls.append(time.time())
self.assertEqual(len(calls), 3 * self.max_calls)
self.validate_call_times(calls, self.max_calls, self.period)
def __init__(self, **kwargs):
super(CloudResourceManagerClient,
self).__init__(api_name=self.API_NAME, **kwargs)
# TODO: we will need multiple rate limiters when we need to invoke
# the CRM write API for enforcement.
self.rate_limiter = RateLimiter(
FLAGS.max_crm_api_calls_per_100_seconds,
self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def test_limit_3(self):
with Timer() as timer:
obj = RateLimiter(self.max_calls, self.period, consume=2)
for _ in range(self.max_calls + 1):
with obj:
pass
self.assertGreaterEqual(timer.duration, self.period * obj.consume)
self.assertLessEqual(timer.duration, self.period * (obj.consume + 1))
def __init__(self, token: str, **kwargs):
self.base_url = "https://api.dagpi.xyz"
self.token = token
self.loop = loop = kwargs.get('loop', None) or asyncio.get_event_loop()
self.client = kwargs.get('session') or aiohttp.ClientSession(loop=loop)
self.ratelimiter = RateLimiter(max_calls=60,
period=60,
callback=limited)
self.user_agent = "AsyncDagpi v{__version__} Python/Python/ \
async def test_async(self):
import asyncio
ratelimter = RateLimiter(InMemoryContext, max_calls=50, period=1.0)
calls = []
count = 100
for i in range(count):
async with ratelimter.use_cost(2):
await asyncio.sleep(0.001)
calls.append(time.time())
elapsed = calls[-1] - calls[0]
rate = count / elapsed
expectedRate = 100
def test_limit_2(self):
calls = []
obj = RateLimiter(self.max_calls, self.period)
for _ in range(3 * self.max_calls):
with obj:
calls.extend([time.time()] * obj.consume)
self.assertEqual(len(calls), 3 * self.max_calls)
self.validate_call_times(calls, self.max_calls, self.period)
def __init__(self, config: CollectorConfig):
self.metrics = config.metrics
self.info_metrics = config.info_metrics
self.client = AcsClient(ak=config.credential['access_key_id'],
secret=config.credential['access_key_secret'],
region_id=config.credential['region_id'])
self.pool = ThreadPoolExecutor(max_workers=config.pool_size)
self.rateLimiter = RateLimiter(max_calls=config.rate_limit)
self.info_provider = InfoProvider(self.client)
def get_rate_limiter(self):
"""Return an appriopriate rate limiter.
Returns:
object: The rate limiter.
"""
return RateLimiter(
self.global_configs.get('max_admin_api_calls_per_day'),
self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def test_random(self):
for _ in range(10):
calls = []
obj = RateLimiter(self.max_calls, self.period)
for i in range(random.randint(10, 50)):
with obj:
calls.append(time.time())
self.validate_call_times(calls, self.max_calls, self.period)
def get_rate_limiter(self):
"""Return an appropriate rate limiter.
Returns:
RateLimiter: The rate limiter.
"""
return RateLimiter(
self.global_configs.get('max_bigquery_api_calls_per_100_seconds'),
self.DEFAULT_QUOTA_TIMESPAN_PER_SECONDS)
def __init__(self, config_path):
self.data_dir = get_data_dir()
self.log_dir = get_log_dir()
self.config = ConfigReader(config_path)
self.rate_limit_config = self.config["RateLimiter"]
max_calls = int(self.rate_limit_config["max_calls"])
period = int(self.rate_limit_config["period"])
self.rate_limiter = RateLimiter(max_calls=max_calls, period=period)
