def get_all(self,
newer_than: datetime,
state: PullRequestState = None,
limit: int = None):
variables = {}
received = 0
ends_at = newer_than
while limit is None or received < limit:
if limit is None:
variables['prFirst'] = self.MAX_PRS
else:
variables['prFirst'] = min(limit - received, self.MAX_PRS)
if state is not None:
variables['prState'] = state.value
formatted_ends_at = ends_at.replace(microsecond=0).astimezone(pytz.utc).isoformat()
variables['searchQuery'] = f'type:pr updated:>={formatted_ends_at} repo:bitcoin/bitcoin sort:updated-asc'
log.debug('Variables for graphql pull requests query', variables=variables)
json_object = {
'query': pull_requests_graphql_query,
'variables': variables
}
data = self.graphql_post(json_object=json_object).json()
search_data = data['data']['search']
pull_requests_graphql_data = search_data['edges']
results_count = len(search_data['edges'])
log.debug(
'response from github graphql',
results_count=results_count
)
if not results_count:
break
starts_at = pull_requests_graphql_data[0]['node']['updatedAt']
previous_ends_at = ends_at
ends_at = dateutil.parser.parse(pull_requests_graphql_data[-1]['node']['updatedAt'])
if previous_ends_at == ends_at:
break
log.debug(
'Pull requests fetched',
starts_at=starts_at,
ends_at=ends_at
)
pull_requests_graphql_data = [r['node'] for r in pull_requests_graphql_data if r['node']]
for pull_request_graphql in pull_requests_graphql_data:
validated_pull_request_data = pull_request_schema.load(pull_request_graphql)
self.parse_into_queue(validated_pull_request_data)
if limit is not None and received == limit:
break
received += 1
self.flush_queue_to_database()
def get_one(self, number: int):
json_object = {
'query': pull_request_graphql_query,
'variables': {'prNumber': number}
}
data = self.graphql_post(json_object=json_object).json()
pull_request = data['data']['repository']['pullRequest']
validated_pull_request_data = pull_request_schema.load(pull_request)
self.parse_into_queue(validated_pull_request_data)
def get(self, number: int) -> dict:
json_object = {
'query': pull_request_graphql_query,
'variables': {'prNumber': number}
}
data = self.graphql_post(json_object=json_object).json()
pull_request = data['data']['repository']['pullRequest']
deserialized_data = pull_request_schema.load(pull_request)
return deserialized_data
def get_all(self,
state: PullRequestState = None,
limit: int = None,
newer_than: date = None):
"""
Generator: starting from the most recent to the oldest, yield one PR at a time
"""
last_cursor = None
variables = {}
received = 0
while limit is None or received < limit:
if limit is None:
variables['prFirst'] = self.MAX_PRS
else:
variables['prFirst'] = min(limit - received, self.MAX_PRS)
if state is not None:
variables['prState'] = state.value
if last_cursor is not None:
variables['prCursorAfter'] = last_cursor
if newer_than:
variables[
'searchQuery'] = f'updated:>{newer_than} repo:bitcoin/bitcoin'
else:
variables['searchQuery'] = 'repo:bitcoin/bitcoin'
print(variables)
json_object = {
'query': pull_requests_graphql_query,
'variables': variables
}
data = self.graphql_post(json_object=json_object).json()
results = data['data']['search']['edges']
if not len(results):
break
last_cursor = results[-1]['cursor']
results = [r['node'] for r in results]
for pull_request in results:
if limit is not None and received == limit:
break
deserialized_data = pull_request_schema.load(pull_request)
if deserialized_data:
yield deserialized_data
received += 1
def test_pull_requests_data_validation(self, pull_requests_data):
deserialized_data, errors = pull_request_schema.load(
pull_requests_data, many=True)
assert not errors