def get_all(self, newer_than: datetime, state: PullRequestState = None, limit: int = None): variables = {} received = 0 ends_at = newer_than while limit is None or received < limit: if limit is None: variables['prFirst'] = self.MAX_PRS else: variables['prFirst'] = min(limit - received, self.MAX_PRS) if state is not None: variables['prState'] = state.value formatted_ends_at = ends_at.replace(microsecond=0).astimezone(pytz.utc).isoformat() variables['searchQuery'] = f'type:pr updated:>={formatted_ends_at} repo:bitcoin/bitcoin sort:updated-asc' log.debug('Variables for graphql pull requests query', variables=variables) json_object = { 'query': pull_requests_graphql_query, 'variables': variables } data = self.graphql_post(json_object=json_object).json() search_data = data['data']['search'] pull_requests_graphql_data = search_data['edges'] results_count = len(search_data['edges']) log.debug( 'response from github graphql', results_count=results_count ) if not results_count: break starts_at = pull_requests_graphql_data[0]['node']['updatedAt'] previous_ends_at = ends_at ends_at = dateutil.parser.parse(pull_requests_graphql_data[-1]['node']['updatedAt']) if previous_ends_at == ends_at: break log.debug( 'Pull requests fetched', starts_at=starts_at, ends_at=ends_at ) pull_requests_graphql_data = [r['node'] for r in pull_requests_graphql_data if r['node']] for pull_request_graphql in pull_requests_graphql_data: validated_pull_request_data = pull_request_schema.load(pull_request_graphql) self.parse_into_queue(validated_pull_request_data) if limit is not None and received == limit: break received += 1 self.flush_queue_to_database()
def get_one(self, number: int): json_object = { 'query': pull_request_graphql_query, 'variables': {'prNumber': number} } data = self.graphql_post(json_object=json_object).json() pull_request = data['data']['repository']['pullRequest'] validated_pull_request_data = pull_request_schema.load(pull_request) self.parse_into_queue(validated_pull_request_data)
def get(self, number: int) -> dict: json_object = { 'query': pull_request_graphql_query, 'variables': {'prNumber': number} } data = self.graphql_post(json_object=json_object).json() pull_request = data['data']['repository']['pullRequest'] deserialized_data = pull_request_schema.load(pull_request) return deserialized_data
def get_all(self, state: PullRequestState = None, limit: int = None, newer_than: date = None): """ Generator: starting from the most recent to the oldest, yield one PR at a time """ last_cursor = None variables = {} received = 0 while limit is None or received < limit: if limit is None: variables['prFirst'] = self.MAX_PRS else: variables['prFirst'] = min(limit - received, self.MAX_PRS) if state is not None: variables['prState'] = state.value if last_cursor is not None: variables['prCursorAfter'] = last_cursor if newer_than: variables[ 'searchQuery'] = f'updated:>{newer_than} repo:bitcoin/bitcoin' else: variables['searchQuery'] = 'repo:bitcoin/bitcoin' print(variables) json_object = { 'query': pull_requests_graphql_query, 'variables': variables } data = self.graphql_post(json_object=json_object).json() results = data['data']['search']['edges'] if not len(results): break last_cursor = results[-1]['cursor'] results = [r['node'] for r in results] for pull_request in results: if limit is not None and received == limit: break deserialized_data = pull_request_schema.load(pull_request) if deserialized_data: yield deserialized_data received += 1
def test_pull_requests_data_validation(self, pull_requests_data): deserialized_data, errors = pull_request_schema.load( pull_requests_data, many=True) assert not errors