def import_missing_roles(self, results: List[GalaxyAPIPage]) -> List[GalaxyAPIPage]:
role_ids: Set[int] = set()
role_search_ids: Set[int] = set()
highest_role_page_num = 0
for page in results:
if page.page_type == 'roles':
highest_role_page_num = max(
highest_role_page_num, page.page_num)
for role in cast(List[Dict[str, Any]], page.response['results']):
role_ids.add(role['id'])
if page.page_type == 'role_search':
for role in cast(List[Dict[str, Any]], page.response['results']):
role_search_ids.add(role['id'])
missing_ids = role_search_ids - role_ids
new_pages: List[Any] = []
api = GalaxyAPI()
for role_id in tqdm(missing_ids, desc='Loading missing roles'):
role_page = api.load_role(role_id)
if role_page is not None:
new_pages.append(role_page)
# Imitate the JSON of the role page.
page_content = {'results': new_pages}
results.append(GalaxyAPIPage(
'roles', highest_role_page_num + 1, json.dumps(page_content)))
return results
def v2_role_namespaces(p: Path) -> bool: # noqa: C901
"""Add namespaces to Galaxy roles."""
with (p / 'roles.json').open('r') as json_roles_f:
orig_roles = json.load(json_roles_f)
roles = dict(orig_roles) # Copy
galaxy_api = GalaxyAPI()
# Include deprecated roles in the results: Some of the roles we gathered
# earlier may now be deprecated
galaxy_it = galaxy_api.search_roles(deprecated=None)
todo_role_ids = set(roles.keys())
unmatched: List[GalaxyRole] = []
for galaxy_role in tqdm(galaxy_it, unit=' roles', desc='Update roles'):
try:
roles[galaxy_role.id]['namespace'] = galaxy_role.namespace
todo_role_ids.remove(galaxy_role.id)
if not todo_role_ids:
# Done
break
except KeyError:
# OK, new role not in dataset
unmatched.append(galaxy_role)
# Perform a deep comparison (heuristically) of any roles whose ID has
# changed
for role_id in list(todo_role_ids):
missing_role = roles[role_id]
candidates: List[GalaxyRole] = []
for new_role in unmatched:
if (_deep_match(new_role, missing_role)):
candidates.append(new_role)
if not candidates:
continue
if len(candidates) > 1:
_log(f'Unmatched role {role_id} matches multiple candidates: ' +
', '.join(r.id for r in candidates))
continue
missing_role['namespace'] = candidates[0].namespace
todo_role_ids.remove(role_id)
# Verify
if todo_role_ids:
_log('Failed to update following roles: ' + ', '.join(todo_role_ids))
_log('Heuristically setting to GitHub owner name')
for role_id in todo_role_ids:
roles[role_id]['namespace'] = roles[role_id]['github_user']
if len(orig_roles) != len(roles):
raise MigrationException('Lengths of updated roles not equal')
try:
for role_id, updated_role in roles.items():
m = CONVERTER.structure(updated_role, GalaxyRole)
assert m.id == role_id
except Exception as exc:
raise MigrationException('Failed to structure role') from exc
# Verification passed, write new file
with (p / 'roles.json').open('w') as json_roles_f:
json.dump(roles, json_roles_f, indent=4, sort_keys=True)
return True
def run(self) -> ResultMap[GalaxyRole]:
"""Run the stage."""
galaxy_api = GalaxyAPI()
it_roles = galaxy_api.search_roles(limit=self.config.count)
if not self.config.progress:
return ResultMap(it_roles)
return ResultMap(tqdm(
it_roles, desc='Searching roles', total=self.config.count))
def api(
request: _pytest.fixtures.SubRequest,
betamax_parametrized_session: requests.Session
) -> GalaxyAPI: # pragma: no cover
"""Fixture to create the API with or without a recorded session.
If the param is absent or True, the recorder is created.
If the param is False, all requests are carried out.
"""
if (not hasattr(request, 'param')) or request.param:
return GalaxyAPI(betamax_parametrized_session)
return GalaxyAPI()
def test_role_order_asc(api: GalaxyAPI) -> None:
results = api.search_roles(
limit=5, order_by=RoleOrder.CREATED,
order=OrderDirection.ASCENDING)
first = next(results)
second = next(results)
assert first.created <= second.created
def test_role_order_desc(api: GalaxyAPI) -> None:
results = list(api.search_roles(
limit=5, order_by=RoleOrder.DOWNLOAD_RANK,
order=OrderDirection.DESCENDING))
first = results[0]
last = results[-1]
assert first.download_count >= last.download_count
def load_pages(self, page_name: str, page_url: str) -> List[GalaxyAPIPage]:
cached_results = self.try_load_pages(page_name)
if cached_results is not None:
return cached_results
api = GalaxyAPI()
page_size = PAGE_SIZES.get(page_name, 500)
it_pages = api.load_pages(page_name, page_url, page_size=page_size)
pbar = tqdm(
desc=f'Loading {page_name} pages', unit='pages', leave=False)
results: List[GalaxyAPIPage] = []
total_set = False
for page in it_pages:
if not total_set:
pbar.total = (cast(int, page.response['count']) // page_size) + 1
total_set = True
pbar.update(1)
results.append(page)
pbar.close()
self.save_pages(results)
return results
def test_role_search_multi_page_integration() -> None:
api = GalaxyAPI()
roles = api.search_roles(15)
names = [role.name for role in roles]
assert len(names) == 15
def test_role_search_multi_page(api: GalaxyAPI) -> None:
roles = api.search_roles(15)
names = [role.name for role in roles]
assert names == ROLE_NAMES
def test_role_search_limit(api: GalaxyAPI) -> None:
results = api.search_roles(limit=5)
assert len(list(results)) <= 5