def guess_definition(
attrs: T.Dict[str, str],
definitions: T.Dict[str, T.Dict[str, str]],
log: structlog.BoundLogger = LOGGER,
) -> T.Dict[str, str]:
standard_name = attrs.get("standard_name")
if standard_name is not None:
log = log.bind(standard_name=standard_name)
matching_variables = []
for var_name, var_def in definitions.items():
if var_def.get("standard_name") == standard_name:
matching_variables.append(var_name)
if len(matching_variables) == 0:
log.warning("'standard_name' attribute not valid")
elif len(matching_variables) == 1:
expected_name = matching_variables[0]
log.warning("wrong name for variable", expected_name=expected_name)
return definitions[expected_name]
else:
log.warning(
"variables with matching 'standard_name':",
matching_variables=matching_variables,
)
else:
log.warning("missing recommended attribute 'standard_name'")
return {}
def check_variable_attrs(
variable_attrs: T.Mapping[T.Hashable, T.Any],
definition: T.Dict[str, str],
dtype: T.Optional[str] = None,
log: structlog.BoundLogger = LOGGER,
) -> None:
attrs = sanitise_mapping(variable_attrs, log)
if "long_name" not in attrs:
log.warning("missing recommended attribute 'long_name'")
if "units" not in attrs:
if dtype not in TIME_DTYPE_NAMES:
log.warning("missing recommended attribute 'units'")
else:
units = attrs.get("units")
expected_units = definition.get("units")
if expected_units is not None:
log = log.bind(expected_units=expected_units)
cf_units = cfunits.Units(units)
if not cf_units.isvalid:
log.warning("'units' attribute not valid", units=units)
else:
expected_cf_units = cfunits.Units(expected_units)
log = log.bind(units=units, expected_units=expected_units)
if not cf_units.equivalent(expected_cf_units):
log.warning(
"'units' attribute not equivalent to the expected")
elif not cf_units.equals(expected_cf_units):
log.warning("'units' attribute not equal to the expected")
standard_name = attrs.get("standard_name")
expected_standard_name = definition.get("standard_name")
if expected_standard_name is not None:
log = log.bind(expected_standard_name=expected_standard_name)
if standard_name is None:
log.warning("missing expected attribute 'standard_name'")
elif standard_name != expected_standard_name:
log.warning("'standard_name' attribute not valid",
standard_name=standard_name)
def check_dataset_data_vars(
dataset_data_vars: T.Mapping[T.Hashable, xr.DataArray],
log: structlog.BoundLogger = LOGGER,
) -> T.Tuple[T.Dict[str, xr.DataArray], T.Dict[str, xr.DataArray]]:
data_vars = sanitise_mapping(dataset_data_vars, log=log)
payload_vars = {}
ancillary_vars = {}
for name, var in data_vars.items():
if name in CDM_ANCILLARY_VARS:
ancillary_vars[name] = var
else:
payload_vars[name] = var
if len(payload_vars) > 1:
log.error(
"file must have at most one non-auxiliary variable",
data_vars=list(payload_vars),
)
for data_var_name, data_var in payload_vars.items():
log = log.bind(data_var_name=data_var_name)
check_variable(data_var_name, data_var, CDM_DATA_VARS, log=log)
return payload_vars, ancillary_vars
def check_dataset_coords(dataset_coords: T.Mapping[T.Hashable, T.Any],
log: structlog.BoundLogger = LOGGER) -> None:
coords = sanitise_mapping(dataset_coords, log=log)
for coord_name, coord in coords.items():
log = log.bind(coord_name=coord_name)
check_variable(coord_name, coord, CDM_COORDS, log=log)
async def evaluate_pr(
install: str,
owner: str,
repo: str,
number: int,
merging: bool,
dequeue_callback: Callable[[], Awaitable[None]],
requeue_callback: Callable[[], Awaitable[None]],
queue_for_merge_callback: QueueForMergeCallback,
is_active_merging: bool,
log: structlog.BoundLogger,
) -> None:
skippable_check_timeout = 4
api_call_retries_remaining = 5
api_call_errors = [] # type: List[APICallError]
log = log.bind(owner=owner, repo=repo, number=number, merging=merging)
while True:
log.info("get_pr")
try:
pr = await asyncio.wait_for(
get_pr(
install=install,
owner=owner,
repo=repo,
number=number,
dequeue_callback=dequeue_callback,
requeue_callback=requeue_callback,
queue_for_merge_callback=queue_for_merge_callback,
),
timeout=60,
)
if pr is None:
log.info("failed to get_pr")
return
try:
await asyncio.wait_for(
mergeable(
api=pr,
subscription=pr.event.subscription,
config=pr.event.config,
config_str=pr.event.config_str,
config_path=pr.event.config_file_expression,
app_id=conf.GITHUB_APP_ID,
repository=pr.event.repository,
pull_request=pr.event.pull_request,
branch_protection=pr.event.branch_protection,
review_requests=pr.event.review_requests,
bot_reviews=pr.event.bot_reviews,
contexts=pr.event.status_contexts,
check_runs=pr.event.check_runs,
commits=pr.event.commits,
valid_merge_methods=pr.event.valid_merge_methods,
merging=merging,
is_active_merge=is_active_merging,
skippable_check_timeout=skippable_check_timeout,
api_call_errors=api_call_errors,
api_call_retries_remaining=api_call_retries_remaining,
),
timeout=60,
)
log.info("evaluate_pr successful")
except RetryForSkippableChecks:
if skippable_check_timeout > 0:
skippable_check_timeout -= 1
log.info("waiting for skippable checks to pass")
await asyncio.sleep(RETRY_RATE_SECONDS)
continue
except PollForever:
log.info("polling")
await asyncio.sleep(POLL_RATE_SECONDS)
continue
except ApiCallException as e:
# if we have some api exception, it's likely a temporary error that
# can be resolved by calling GitHub again.
if api_call_retries_remaining:
api_call_errors.append(
APICallError(
api_name=e.method,
http_status=str(e.status_code),
response_body=str(e.response),
)
)
api_call_retries_remaining -= 1
log.info("problem contacting remote api. retrying")
continue
log.warning("api_call_retries_remaining", exc_info=True)
return
except asyncio.TimeoutError:
# On timeout we add the PR to the back of the queue to try again.
log.warning("mergeable_timeout", exc_info=True)
await requeue_callback()
async def process_repo_queue(log: structlog.BoundLogger,
connection: RedisConnection,
queue_name: str) -> None:
log.info("block for new repo event")
webhook_event_json: BlockingZPopReply = await connection.bzpopmin(
[queue_name])
webhook_event = WebhookEvent.parse_raw(webhook_event_json.value)
# mark this PR as being merged currently. we check this elsewhere to set proper status codes
await connection.set(webhook_event.get_merge_target_queue_name(),
webhook_event.json())
async with Client(
owner=webhook_event.repo_owner,
repo=webhook_event.repo_name,
installation_id=webhook_event.installation_id,
) as api_client:
pull_request = PR(
owner=webhook_event.repo_owner,
repo=webhook_event.repo_name,
number=webhook_event.pull_request_number,
installation_id=webhook_event.installation_id,
client=api_client,
)
# mark that we're working on this PR
await pull_request.set_status(summary="⛴ attempting to merge PR")
skippable_check_timeout = 4
while True:
# there are two exits to this loop:
# - OK MergeabilityResponse
# - NOT_MERGEABLE MergeabilityResponse
#
# otherwise we continue to poll the Github API for a status change
# from the other states: NEEDS_UPDATE, NEED_REFRESH, WAIT
# TODO(chdsbd): Replace enum response with exceptions
m_res, event = await pull_request.mergeability(merging=True)
log = log.bind(res=m_res)
if event is None or m_res == MergeabilityResponse.NOT_MERGEABLE:
log.info("cannot merge")
break
if m_res == MergeabilityResponse.SKIPPABLE_CHECKS:
if skippable_check_timeout:
skippable_check_timeout -= 1
await asyncio.sleep(RETRY_RATE_SECONDS)
continue
await pull_request.set_status(
summary="⌛️ waiting a bit for skippable checks")
break
if m_res == MergeabilityResponse.NEEDS_UPDATE:
log.info("update pull request and don't attempt to merge")
if await update_pr_with_retry(pull_request):
continue
log.error("failed to update branch")
await pull_request.set_status(
summary="🛑 could not update branch")
# break to find next PR to try and merge
break
elif m_res == MergeabilityResponse.NEED_REFRESH:
# trigger a git mergeability check on Github's end and poll for result
log.info("needs refresh")
await pull_request.trigger_mergeability_check()
continue
elif m_res == MergeabilityResponse.WAIT:
# continuously poll until we either get an OK or a failure for mergeability
log.info("waiting for status checks")
continue
elif m_res == MergeabilityResponse.OK:
# continue to try and merge
pass
else:
raise Exception("Unknown MergeabilityResponse")
retries = 5
while retries:
log.info("merge")
if await pull_request.merge(event):
# success merging
break
retries -= 1
log.info("retry merge")
await asyncio.sleep(RETRY_RATE_SECONDS)
else:
log.error("Exhausted attempts to merge pull request")