def test_insert_update_deleteData(self):
# Insert
self.db.pushAllDataToDB(self.data)
retriveData = self.db.getAllDataFromTable()
ok = True
for item in self.data:
if item not in retriveData:
ok = False
break
self.assertTrue(ok, "Push data failed")
# Update
self.phone.price = 12345
self.phone.info["currency"] = "Yolo Dollar"
self.db.updateItemToDB(self.phone)
try:
temp = self.db.getSpecificItemFromDB(brand=self.phone.getBrand(),
model=self.phone.getModel(),
vendor=self.phone.getVendor())
result = phoneDBEngine.convertAllDataToPhone(temp)
self.assertEqual(result.getPrice(), self.phone.getPrice())
self.assertEqual(result.getInfo().get("currency"),
self.phone.getInfo().get("currency"))
except ClientError as e:
self.fail("update Data failed")
# Delete
self.db.deleteItemFromDB(self.phone)
temp = self.db.getSpecificItemFromDB(brand=self.phone.getBrand(),
model=self.phone.getModel(),
vendor=self.phone.getVendor())
self.assertIsNone(temp)
def test_getAllPhones_withBrands(self):
phoneAdapter = phoneDBEngine(constants.dynamoDBTableName)
data = phoneAdapter.getItemsWithBrandAndModel(brand="Samsung")
phoneData = phoneDBEngine.convertAllDataToPhone(data)
for item in phoneData:
if "Samsung" not in item.getBrand():
self.fail("Phones not from correct brand." + str(item))
def all_mobiles(request, page):
# Retrieve all data from amazonDB
phoneDB = phoneDBEngine(tableName=constants.dynamoDBTableName)
data = phoneDB.getAllDevicesWithType('Mobile')
phones = phoneDBEngine.convertAllDataToPhone(data)
collections = phoneDBEngine.orderPhonesToCollections(phones)
paginator = Paginator(collections, 12)
allData = paginator.get_page(page)
return render(request, 'mobile/all_mobiles.html',
{'allData': allData})
def search_PriceRange(request, dev, low, high, page):
phoneDB = phoneDBEngine(tableName=constants.dynamoDBTableName)
phones = phoneDBEngine.convertAllDataToPhone(phoneDB.getItemWithBrandAndPriceAndType(devType=dev, lowerLim=low, higherLim=high))
phones.sort(key=lambda phone: phone.price)
collections = phoneDBEngine.orderPhonesToCollections(phones)
paginator = Paginator(collections, 12)
allData = paginator.get_page(page)
return render(request, 'mobile/all_mobiles.html',
{'allData': allData, 'dev': dev, 'low': low, 'high': high})
def masterUpdate():
ChangesToNotify = {}
phoneDBAdaper = phoneDBEngine(constant.dynamoDBTableName)
all_brands = set()
# Get all phones from DB
dataFromDB = phoneDBAdaper.getAllDevicesWithType('Mobile')
phonesFromDB = phoneDBEngine.convertAllDataToPhone(dataFromDB)
# Loop through each source to update information
for src in constant.scrapingSources:
if src.name in scrap_parser:
parser = scrap_parser[src.name](src.info.ignoreTerm, src.url,
src.info.param)
phonesFromScraper = parser.getAllPages()
if phonesFromScraper is None or len(phonesFromScraper) == 0:
continue
# Update data for each source
priceChange = []
infoChange = []
newItem = []
# Update data
for item in phonesFromScraper:
all_brands.add(item.getBrand())
existed = False
for phone in phonesFromDB:
if item == phone:
if item.checkPriceChange(oldData=phone):
priceChange.append((item, phone))
elif item.checkInfoChange(oldData=phone):
infoChange.append((item, phone))
existed = True
break
if not existed:
newItem.append(item)
# Delete old items that not there anymore
toDelete = []
for phone in phonesFromDB:
existed = False
for item in phonesFromScraper:
if item == phone:
existed = True
break
if not existed and phone.getVendor() == src.name:
toDelete.append(phone)
# push new data to database
for item, _ in priceChange:
phoneDBAdaper.updateItemToDB(item)
for item, _ in infoChange:
phoneDBAdaper.updateItemToDB(item)
for item in toDelete:
phoneDBAdaper.deleteItemFromDB(item)
if len(newItem) > 0:
phoneDBAdaper.pushAllDataToDB(newItem)
# Add changes to notify list
ChangesToNotify[src.name] = (newItem, priceChange, toDelete)
else:
print("Parser for " + src.name + " is not available. Skip")
# Update all brands
if len(all_brands) > 0:
phoneDBAdaper.updateAllBrandData(list(all_brands))
fullpath = "{}/Core/brands.txt".format(
os.path.dirname(os.path.realpath(__file__)))
f = open(fullpath, "+w")
f.write(', '.join(list(all_brands)))
f.close()
return ChangesToNotify