def get_workunits(self) -> Iterable[ApiWorkUnit]: # noqa: C901
config = self.config
sw_dict = self.config.get_swagger()
self.url_basepath = get_url_basepath(sw_dict)
# Getting all the URLs accepting the "GET" method
with warnings.catch_warnings(record=True) as warn_c:
url_endpoints = get_endpoints(sw_dict)
for w in warn_c:
w_msg = w.message
w_spl = w_msg.args[0].split(" --- ") # type: ignore
self.report.report_warning(key=w_spl[1], reason=w_spl[0])
# here we put a sample from the "listing endpoint". To be used for later guessing of comosed endpoints.
root_dataset_samples = {}
# looping on all the urls
for endpoint_k, endpoint_dets in url_endpoints.items():
if endpoint_k in config.ignore_endpoints:
continue
dataset_snapshot, dataset_name = self.init_dataset(
endpoint_k, endpoint_dets)
# adding dataset fields
if "data" in endpoint_dets.keys():
# we are lucky! data is defined in the swagger for this endpoint
schema_metadata = set_metadata(dataset_name,
endpoint_dets["data"])
dataset_snapshot.aspects.append(schema_metadata)
yield from self.build_wu(dataset_snapshot, dataset_name)
elif ("{" not in endpoint_k
): # if the API does not explicitely require parameters
tot_url = clean_url(config.url + self.url_basepath +
endpoint_k)
if config.token:
response = request_call(tot_url, token=config.token)
else:
response = request_call(tot_url,
username=config.username,
password=config.password)
if response.status_code == 200:
fields2add, root_dataset_samples[
dataset_name] = extract_fields(response, dataset_name)
if not fields2add:
self.report.report_warning(key=endpoint_k,
reason="No Fields")
schema_metadata = set_metadata(dataset_name, fields2add)
dataset_snapshot.aspects.append(schema_metadata)
yield from self.build_wu(dataset_snapshot, dataset_name)
else:
self.report_bad_responses(response.status_code,
key=endpoint_k)
else:
if endpoint_k not in config.forced_examples.keys():
# start guessing...
url_guess = try_guessing(endpoint_k, root_dataset_samples)
tot_url = clean_url(config.url + self.url_basepath +
url_guess)
if config.token:
response = request_call(tot_url, token=config.token)
else:
response = request_call(tot_url,
username=config.username,
password=config.password)
if response.status_code == 200:
fields2add, _ = extract_fields(response, dataset_name)
if not fields2add:
self.report.report_warning(key=endpoint_k,
reason="No Fields")
schema_metadata = set_metadata(dataset_name,
fields2add)
dataset_snapshot.aspects.append(schema_metadata)
yield from self.build_wu(dataset_snapshot,
dataset_name)
else:
self.report_bad_responses(response.status_code,
key=endpoint_k)
else:
composed_url = compose_url_attr(
raw_url=endpoint_k,
attr_list=config.forced_examples[endpoint_k])
tot_url = clean_url(config.url + self.url_basepath +
composed_url)
if config.token:
response = request_call(tot_url, token=config.token)
else:
response = request_call(tot_url,
username=config.username,
password=config.password)
if response.status_code == 200:
fields2add, _ = extract_fields(response, dataset_name)
if not fields2add:
self.report.report_warning(key=endpoint_k,
reason="No Fields")
schema_metadata = set_metadata(dataset_name,
fields2add)
dataset_snapshot.aspects.append(schema_metadata)
yield from self.build_wu(dataset_snapshot,
dataset_name)
else:
self.report_bad_responses(response.status_code,
key=endpoint_k)
def test_no_good_guesses(self):
url2complete = "/advancedcomputersearches/name/{nasde}/asd/{asd}/jhg"
guessed_url = try_guessing(url2complete, self.extr_data2)
self.assertEqual(guessed_url, url2complete)