def _download(source_url):
response = requests.get(source_url)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.error("Request failed: %s", response.text)
raise
return response.content
def _make_request(url, access_token, params):
response = requests.get(
url,
params=params,
headers={'Authorization': f'Bearer {access_token}'},
)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.error('Request failed: %s', response.text)
raise
return response.json()
def _hawk_api_request(
url: str,
credentials: dict,
results_key: Optional[str],
next_key: Optional[str],
validate_response: Optional[bool] = True,
force_http: Optional[bool] = False,
):
sender = Sender(
credentials,
# Currently data workspace denies hawk requests signed with https urls.
# Once fixed the protocol replacement can be removed.
url.replace('https', 'http') if force_http else url,
"get",
content="",
content_type="",
always_hash_content=True,
)
logger.info(f"Fetching page {url}")
response = requests.get(
url,
headers={
"Authorization": sender.request_header,
"Content-Type": ""
},
timeout=300,
)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.warning(f"Request failed: {response.text}")
raise
if validate_response:
try:
sender.accept_response(
response.headers["Server-Authorization"],
content=response.content,
content_type=response.headers["Content-Type"],
)
except HawkFail as e:
logger.error(f"HAWK Authentication failed {str(e)}")
raise
response_json = response.json()
if (next_key and next_key not in response_json) or (
results_key and results_key not in response_json):
raise ValueError("Unexpected response structure")
return response_json
def _ons_sparql_request(url: str,
query: str,
page: int = 1,
per_page: int = 10000):
query += f" LIMIT {per_page} OFFSET {per_page * (page - 1)}"
response = requests.request("POST",
url,
data={"query": query},
headers={"Accept": "application/json"})
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.error(f"Request failed: {response.text}")
raise
response_json = response.json()
if "results" not in response_json:
raise ValueError("Unexpected response structure")
return response_json
def fetch_from_gtr_api(table_name: str, resource_type: str, **kwargs):
source_url = 'https://gtr.ukri.org/gtr/api'
s3 = S3Data(table_name, kwargs["ts_nodash"])
page = 1
while True:
response = requests.get(
f'{source_url}/{resource_type}s',
params={
'p': page,
's': 100
},
headers={'Accept': 'application/json'},
)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.error(f"Request failed: {response.text}")
raise
response_json = response.json()
total_pages = response_json['totalPages']
total_number_of_results = response_json['totalSize']
results = response_json[resource_type]
s3.write_key(f"{page:010}.json", results)
logger.info(
f"Fetched {len(results*page)} out of {total_number_of_results} {resource_type} records"
)
page += 1
if page > total_pages:
break
logger.info("Fetching from source completed")
assert not tp_repo.is_dirty()
df_repo = git.Repo(DF_ROOT)
assert not df_repo.is_dirty()
df_commits = df_repo.iter_commits(paths='dataflow')
df_latest_commit = next(df_commits)
logger.info("DataFlow commit to match: \n" + show_commit(df_latest_commit))
unsynced_commits = []
for cmt in tp_repo.iter_commits(paths=['tensorpack/dataflow', 'tensorpack/utils']):
if match_commit(cmt, df_latest_commit):
logger.info("Matched tensorpack commit: \n" + show_commit(cmt))
break
unsynced_commits.append(cmt)
else:
logger.error("Cannot find tensorpack commit that matches the above commit.")
sys.exit(1)
logger.info("{} more commits to sync".format(len(unsynced_commits)))
unsynced_commits = unsynced_commits[::-1]
for commit_to_sync in unsynced_commits:
tp_repo.git.checkout(commit_to_sync.hexsha)
logger.info("-" * 60)
logger.info("Syncing commit '{}' at {}".format(
commit_to_sync.message.strip(), show_date(commit_to_sync.authored_date)))
# sync files
dst = os.path.join(DF_ROOT, 'dataflow', 'dataflow')
logger.info("Syncing {} ...".format(dst))
shutil.rmtree(dst)
def fetch_from_api_endpoint(
table_name: str,
source_url: str,
auth_token: Optional[str] = None,
auth_token_builder: Optional[Callable] = None,
results_key: Optional[str] = "results",
next_key: Optional[str] = "next",
auth_type: Optional[str] = "Token",
extra_headers: Optional[Mapping] = None,
**kwargs,
):
if auth_token is not None and auth_token_builder is not None:
raise ValueError(
"You can provide at most one of `auth_token` and `auth_token_builder`"
)
s3 = S3Data(table_name, kwargs["ts_nodash"])
total_records = 0
page = 1
while True:
if auth_token:
request_headers = {"Authorization": f'{auth_type} {auth_token}'}
elif auth_token_builder:
request_headers = {
"Authorization": f'{auth_type} {auth_token_builder()}'
}
else:
request_headers = {}
if extra_headers:
request_headers = {**request_headers, **extra_headers}
response = requests.get(source_url, headers=request_headers)
try:
response.raise_for_status()
except requests.exceptions.HTTPError:
logger.error(f"Request failed: {response.text}")
raise
response_json = response.json()
if (next_key and next_key not in response_json) or (
results_key and results_key not in response_json):
raise ValueError("Unexpected response structure")
if results_key is not None:
results = get_nested_key(response_json, results_key)
else:
results = response_json
s3.write_key(f"{page:010}.json", results)
total_records += len(results)
logger.info(f"Fetched {total_records} records")
source_url = get_nested_key(response_json,
next_key) if next_key else None
if not source_url:
break
page += 1
logger.info("Fetching from source completed")