def setUp(self):
config_path = os.path.join(os.path.dirname(__file__), "../../.secrets")
config = configparser.ConfigParser()
config.read(config_path)
self.sms_emitter = EmitSms.new(
account_sid=config["twilio-test"]["account_sid"],
auth_token=config["twilio-test"]["auth_token"],
from_phone=config["twilio-test"]["from_phone"],
to_phone=config["twilio-test"]["to_phone"],
)
self.tweet_emitter = EmitTweet.new(
consumer_key=config["twitter"]["consumer_key"],
consumer_secret=config["twitter"]["consumer_secret"],
access_token_key=config["twitter"]["access_token_key"],
access_token_secret=config["twitter"]["access_token_secret"],
)
self.stuff = Stuff(
title="Furniture",
url="https://somewhere.com/",
time=datetime(2019, 4, 20),
price=0,
neighborhood="Clinton Hill",
coordinates=None,
image_urls=None,
)
def get_inventory(self) -> List[Stuff]:
soup = BeautifulSoup(self.get_text(), features="html.parser")
ul = soup.find("ul", {"class": "rows"})
inventory = []
for list_item in ul.find_all("li"):
stuff = Stuff.parse_item(list_item, self.region.value)
stuff.city = self.region.value
inventory.append(stuff)
return inventory
def test_client_insert_and_fetch_by_url(self):
stuff_id = self.client.insert_stuff(
Stuff(
title="My Title", url="https://somewhere.com",
time=datetime(2019, 4, 20), price=0,
neighborhood="Clinton Hill", city="newyork",
coordinates=None, image_urls=None,
)
)
actual = self.client.get_stuff_by_url("https://somewhere.com")
self.assertEqual(actual.title, "My Title")
self.assertEqual(actual.id, stuff_id)
self.assertEqual(actual.url, "https://somewhere.com")
def setUp(self):
self.stuffs = [
Stuff(
url="https://newyork.craigslist.org/brk/zip/d/free-couch/1234556.html",
title="free couch",
time=datetime(2019, 9, 13, 13, 31),
price=0,
neighborhood="Clinton Hill",
city="newyork",
coordinates=Coordinates(longitude='-73.947000', latitude='40.546700'),
image_urls=['https://images.craigslist.org/someother.jpg']
),
Stuff(
url="https://newyork.craigslist.org/brk/zip/d/free-boxes-and-packing-supplies/6978063787.html",
title="FREE BOXES and PACKING SUPPLIES",
time=datetime(2019, 9, 13, 16, 31),
price=0,
city="newyork",
neighborhood="Clinton Hill",
coordinates=Coordinates(longitude='-73.957000', latitude='40.646700'),
image_urls=['https://images.craigslist.org/00L0L_5e2M7zY0JYR_600x450.jpg']
)
]
def test_client_update_stuff(self):
stuff_id = self.client.insert_stuff(
Stuff(
title="My Title", url="https://somewhere.com/",
time=datetime(2019, 4, 20), price=0,
neighborhood="Clinton Hill", city="newyork",
coordinates=None, image_urls=None,
)
)
stuff = self.client.get_stuff_by_id(stuff_id)
self.assertEqual(stuff.delivered, False)
stuff.delivered = True
self.client.update_stuff(stuff)
updated_stuff = self.client.get_stuff_by_id(stuff.id)
self.assertEqual(updated_stuff.delivered, True)
def test_client_roundtrip_id(self):
stuff_id = self.client.insert_stuff(
Stuff(
title="My Title", url="https://somewhere.com/",
time=datetime(2019, 4, 20), price=0,
neighborhood="Clinton Hill", city="newyork",
coordinates=Coordinates(10.0, 10.0),
image_urls=["https://somewhere.com/item/1", "https://somewhere.com/item/2"],
)
)
api_model = self.client.get_stuff_by_id(stuff_id)
self.assertEqual(api_model.id, stuff_id)
db_stuff = DBStuff.from_api_model(api_model)
self.assertEqual(db_stuff.id, stuff_id)
roundtrip_stuff = db_stuff.to_api_model()
# One of the urls are "lost here"
self.assertEqual(roundtrip_stuff.image_urls, ['https://somewhere.com/item/1'])
def test_search_get_inventory(self):
search = Search()
responses.add(responses.GET,
search.build_url(),
body=_data("craigslist_zip.html"))
inventory = search.get_inventory()
expected = Stuff(
url=
'https://newyork.craigslist.org/brk/zip/d/brooklyn-2-round-blue-outdoor-side/6979177488.html',
title='2 Round Blue Outdoor Side Tables',
time=datetime(2019, 9, 15, 12, 15),
price=0,
neighborhood='Ditmas Park Area, Brooklyn',
city="newyork",
image_urls=None,
coordinates=None,
)
self.assertEqual(expected, inventory[0])
self.assertEqual(120, len(inventory))
def test_search_enrich_inventory_can_run_enrich_async_in_sync_thread(self):
inventory = [
Stuff(
url=
'https://newyork.craigslist.org/brk/zip/d/brooklyn-free-insulation/6977996917.html',
title='FREE Insulation',
time=datetime(2019, 9, 13, 15, 24),
price=0,
city="newyork",
neighborhood='Bay Ridge, Brooklyn',
image_urls=None,
coordinates=None),
Stuff(
url=
'https://newyork.craigslist.org/brk/zip/d/brooklyn-large-navy-blue-area-rug-10-14/6977979965.html',
title='Large navy blue area rug 10â\x80\x99 x 14â\x80\x99',
time=datetime(2019, 9, 13, 14, 59),
city="newyork",
price=0,
neighborhood='Greenpoint, Brooklyn',
image_urls=None,
coordinates=None),
Stuff(
url=
'https://newyork.craigslist.org/brk/zip/d/brooklyn-10-foot-round-pool/6977959276.html',
title='10 foot round pool',
time=datetime(2019, 9, 13, 14, 38),
price=0,
neighborhood='bklyn',
image_urls=None,
coordinates=None,
city="newyork"),
Stuff(
url=
'https://newyork.craigslist.org/brk/zip/d/brooklyn-microwave-popcorn-bags-free/6977920662.html',
title='MICROWAVE POPCORN --- 8 BAGS --- FREE',
time=datetime(2019, 9, 13, 13, 58),
price=0,
city="newyork",
neighborhood='Bay Ridge',
image_urls=None,
coordinates=None),
]
self.assertFalse(any([inv.coordinates for inv in inventory]))
self.assertFalse(any([inv.image_urls for inv in inventory]))
responses.add(
responses.GET,
"https://newyork.craigslist.org/brk/zip/d/brooklyn-free-insulation/6977996917.html",
body=_data("craigslist_zip_item_page.html"),
)
responses.add(
responses.GET,
"https://newyork.craigslist.org/brk/zip/d/brooklyn-large-navy-blue-area-rug-10-14/6977979965.html",
body=_data("craigslist_zip_item_page.html"),
)
responses.add(
responses.GET,
"https://newyork.craigslist.org/brk/zip/d/brooklyn-10-foot-round-pool/6977959276.html",
body=_data("craigslist_zip_item_page.html"),
)
responses.add(
responses.GET,
"https://newyork.craigslist.org/brk/zip/d/brooklyn-microwave-popcorn-bags-free/6977920662.html",
body=_data("craigslist_zip_item_page.html"),
)
inventory = Search.enrich_inventory(inventory)
self.assertTrue(any([inv.coordinates for inv in inventory]))
self.assertTrue(any([inv.image_urls for inv in inventory]))
self.assertEqual(4, len(inventory))
expected = Stuff(
url=
'https://newyork.craigslist.org/brk/zip/d/brooklyn-free-insulation/6977996917.html',
title='FREE Insulation',
time=datetime(2019, 9, 13, 15, 24),
price=0,
neighborhood='Bay Ridge, Brooklyn',
city="newyork",
image_urls=[
'https://images.craigslist.org/00L0L_5e2M7zY0JYR_600x450.jpg'
],
coordinates=Coordinates(longitude='-73.957000',
latitude='40.646700'))
self.assertEqual(inventory[0], expected)
def enrich_item(cls, item: Stuff, proxies: Optional[dict]):
text = requests.get(item.url, proxies=proxies).text
item.parse_details(BeautifulSoup(text, features="html.parser"))
return item