def run_fetches(self,
dataset,
limit,
query_params=None) -> Tuple[List[dict], List[dict]]:
"""sets up event loop and fetches for 'calls' and 'users' datasets"""
loop = get_loop()
future = asyncio.ensure_future(
self._get_data(dataset, limit, query_params))
return loop.run_until_complete(future)
def _run_fetch(self, url):
"""Run loop."""
access_token = self.get_access_token()
if not access_token:
raise NoCredentialsError('No credentials')
headers = {'Authorization': f'Bearer {access_token}'}
loop = get_loop()
future = asyncio.ensure_future(self._fetch(url, headers))
return loop.run_until_complete(future)
def _retrieve_data(self, data_source: GithubDataSource) -> pd.DataFrame:
"""
:param data_source: GithubDataSource, the GithubDataSource to query
:return: a Pandas DataFrame of pull requests or team memberships
"""
dataset = data_source.dataset
access_token = self.get_access_token()
organization = data_source.organization
if not access_token:
raise NoCredentialsError(NO_CREDENTIALS_ERROR)
headers = {'Authorization': f'token {access_token}'}
client = GraphqlClient(BASE_ROUTE, headers)
loop = get_loop()
return loop.run_until_complete(
self._fetch_data(
dataset=dataset,
organization=organization,
client=client,
page_limit=data_source.page_limit,
names_limit=data_source.entities_limit,
))
def _run_fetch(self, endpoint, jq_filter):
"""Event loop handler"""
loop = get_loop()
future = asyncio.ensure_future(self._get_data(endpoint, jq_filter))
return loop.run_until_complete(future)
def run_fetches_for_tags(self, dataset, limit):
"""sets up event loop and fetches for 'tags' dataset"""
loop = get_loop()
future = asyncio.ensure_future(self._get_tags(dataset, limit))
return loop.run_until_complete(future)
def batch_fetch(urls):
"""fetch asyncrhonously `urls` in a single batch"""
loop = get_loop()
future = asyncio.ensure_future(_batch_fetch(urls))
return loop.run_until_complete(future)
