def run_fetches(self, dataset, limit, query_params=None) -> Tuple[List[dict], List[dict]]: """sets up event loop and fetches for 'calls' and 'users' datasets""" loop = get_loop() future = asyncio.ensure_future( self._get_data(dataset, limit, query_params)) return loop.run_until_complete(future)
def _run_fetch(self, url): """Run loop.""" access_token = self.get_access_token() if not access_token: raise NoCredentialsError('No credentials') headers = {'Authorization': f'Bearer {access_token}'} loop = get_loop() future = asyncio.ensure_future(self._fetch(url, headers)) return loop.run_until_complete(future)
def _retrieve_data(self, data_source: GithubDataSource) -> pd.DataFrame: """ :param data_source: GithubDataSource, the GithubDataSource to query :return: a Pandas DataFrame of pull requests or team memberships """ dataset = data_source.dataset access_token = self.get_access_token() organization = data_source.organization if not access_token: raise NoCredentialsError(NO_CREDENTIALS_ERROR) headers = {'Authorization': f'token {access_token}'} client = GraphqlClient(BASE_ROUTE, headers) loop = get_loop() return loop.run_until_complete( self._fetch_data( dataset=dataset, organization=organization, client=client, page_limit=data_source.page_limit, names_limit=data_source.entities_limit, ))
def _run_fetch(self, endpoint, jq_filter): """Event loop handler""" loop = get_loop() future = asyncio.ensure_future(self._get_data(endpoint, jq_filter)) return loop.run_until_complete(future)
def run_fetches_for_tags(self, dataset, limit): """sets up event loop and fetches for 'tags' dataset""" loop = get_loop() future = asyncio.ensure_future(self._get_tags(dataset, limit)) return loop.run_until_complete(future)
def batch_fetch(urls): """fetch asyncrhonously `urls` in a single batch""" loop = get_loop() future = asyncio.ensure_future(_batch_fetch(urls)) return loop.run_until_complete(future)