class SWDS:
def __init__(self):
self.db_Manager = DbManager()
def get_json(url):
resp = requests.get(url)
if resp.status_code != 200:
return None
return json.loads(resp.text)
def get_entity(url, obj):
json_data = get_json(url)
entity = obj()
entity.parse_json(json_data)
self.db_Manager.save(entity)
def populate(obj, next_url):
while next_url:
group_data = get_json(next_url)
results = group_data['results']
for row in results:
url = row['url']
results = self.db_Manager.query(obj).filter(
obj.url == url).all()
if len(results) == 0:
save_entity(url, obj)
# print(url)
next_url = group_data['next']
class DCK():
def __init__(self):
self.db_Manager = DbManager()
def get_json(self, url):
resp = requests.get(url)
if resp.status_code!=200:
return None
return json.loads(resp.text)
def get_entity(self, obj, url):
results = self.db_Manager.open().query(obj).filter(obj.api == url).all()
json_data= self.get_json(url)
entity = obj()
entity.parse_json(json_data)
if len(results)==0:
self.db_Manager.save(entity)
return entity
def find_end(self, endChar, url):
for index, c in enumerate(url[::-1]):
if c==endChar:
return len(url)-index-1
return None
def find_base_url(self, url):
index=self.find_end('/', url)
if index:
return url[: index+1]
else:
return None
def populate_table(self, obj, next_url):
base_url=self.find_base_url(next_url)
if base_url:
group_data=self.get_json(next_url)
page_id=1
while group_data:
print(next_url)
for result in group_data:
single_url = result['api']
print(single_url)
self.get_entity(obj, single_url)
page_id=page_id+1
next_url=base_url+str(page_id)
group_data=self.get_json(next_url)
# group_data=None
class CDC():
def __init__(self):
self.db_Manager = DbManager()
def get_data(self, url):
resp = requests.get(url, headers=DEFAULT_HEADER)
if resp.status_code != 200:
print(resp.status_code)
return None
return json.loads(resp.text)
def get_entity(self, obj, url):
json_data = self.get_data(url)
entity = obj()
try:
entity.parse_json(json_data)
except:
entity = None
print("bad link: {}".format(url))
return entity
results = self.db_Manager.open().query(obj).filter(
obj.url == url).all()
if len(results) == 0:
self.db_Manager.save(entity)
print("new add: {}".format(entity.url))
else:
print("already added: {}".format(entity.url))
return entity
def populate_committee(self, congress, chamber):
chamber = chamber.lower()
url = API_BASE + "{}/{}/committees.json".format(congress, chamber)
json_data = self.get_data(url)
try:
results = json_data['results'][0]
except:
print("Bad Link")
return "Bad Link"
committee_list = [c['api_uri'] for c in results['committees']]
subcommittee_list = [
c['subcommittees'] for c in results['committees']
if c.get('subcommittees')
]
sc_url_list = [sc for sc in subcommittee_list if len(sc) > 0]
for sc_url in sc_url_list:
for sc in sc_url:
pprint(sc)
self.get_entity(SubCommittee, sc['api_uri'])
for com_url in committee_list:
self.get_entity(Committee, com_url)
return "Good Link"
def populate_members(self, congress, chamber):
chamber = chamber.lower()
url = API_BASE + "{}/{}/members.json".format(congress, chamber)
json_data = self.get_data(url)
try:
results = json_data['results'][0]
except:
return "Bad Link"
member_list = [m['id'] for m in results['members']]
for mid in member_list:
member_url = API_BASE + "members/{}.json".format(mid)
self.get_entity(Member, member_url)
return "Good Link"
def populate_house(self, congress):
url = API_BASE + "{}/{}/members.json".format(congress, "house")
self.get_entity(Congress, url)
def populate_senate(self, congress):
url = API_BASE + "{}/{}/members.json".format(congress, "senate")
self.get_entity(Congress, url)
class SWDS:
def __init__(self):
self.db = DbManager()
def get_json(self, url):
print("GET\t<{}>".format(url))
response = requests.get(url)
return json.loads(response.text)
def get_person(self, i):
person_url = 'https://swapi.co/api/people/{}/'.format(i)
results = self.db.open().query(Person).filter(
Person.url == person_url).all()
if len(results) == 0:
json_data = self.get_json(person_url)
person = Person()
if 'detail' not in json_data:
person.parse_person(json_data)
return self.db.save(person)
else:
return results[0]
def get_planet(self, i):
planet_url = 'https://swapi.co/api/planets/{}/'.format(i)
results = self.db.open().query(Planet).filter(
Planet.url == planet_url).all()
if len(results) == 0:
json_data = self.get_json(planet_url)
planet = Planet()
if 'detail' not in json_data:
planet.parse_planet(json_data)
self.db.save(planet)
else:
return results[0]
def get_vehicle(self, i):
vehicle_url = 'https://swapi.co/api/vehicles/{}/'.format(i)
results = self.db.open().query(Vehicle).filter(
Vehicle.url == vehicle_url).all()
if len(results) == 0:
json_data = self.get_json(vehicle_url)
vehicle = Vehicle()
if 'detail' not in json_data:
vehicle.parse_vehicle(json_data)
self.db.save(vehicle)
else:
return results[0]
def get_starship(self, i):
starship_url = 'https://swapi.co/api/starships/{}/'.format(i)
results = self.db.open().query(Starship).filter(
Starship.url == starship_url).all()
if len(results) == 0:
json_data = self.get_json(starship_url)
starship = Starship()
if 'detail' not in json_data:
starship.parse_starship(json_data)
self.db.save(starship)
else:
return results[0]
def get_species(self, i):
species_url = 'https://swapi.co/api/species/{}/'.format(i)
results = self.db.open().query(Species).filter(
Species.url == species_url).all()
if len(results) == 0:
json_data = self.get_json(species_url)
species = Species()
if 'detail' not in json_data:
species.parse_species(json_data)
self.db.save(species)
else:
return results[0]