def test_CuriosityImage_DataStoreGet(self):
#Insert a basic CuriosityImage
ci = CuriosityImage(key_name=str(42))
ci.imageid = 42
ci.description = u'previously “N165”'
ci.put()
dbCI = CuriosityImage.get_by_key_name(str(42))
self.assertIsNotNone(dbCI)
self.assertEquals(ci.description, dbCI.description)
def test_CuriosityImageExists_ExistingImageID_ReturnsTrue(self):
#Insert a basic CuriosityImage
ci = CuriosityImage(key_name=str(42))
ci.imageid = 42
ci.description = 'DESC123'
ci.put()
exists = getfeeds.CuriosityImageExists(42)
self.assertTrue(exists)
def test_SaveCuriosityImage_InsertIfNotExists(self):
self.assertIsNone(CuriosityImage.get_by_key_name(str(42)))
newCi = CuriosityImage(key_name=str(42))
newCi.imageid = 42
newCi.description = 'DESC123'
#Save to DB
getfeeds.SaveCuriosityImage(newCi)
self.assertIsNotNone(CuriosityImage.get_by_key_name(str(42)))
def ParseAllPagesAndFollowNextLink(self, imageIndexPageUrl):
if imageIndexPageUrl is None or len(imageIndexPageUrl) == 0:
return
self.log("Processing %s" % imageIndexPageUrl)
imageIndexPageHTML = self.GetUrlContents(imageIndexPageUrl)
if not imageIndexPageHTML:
self.log("imageIndexPageHTML is empty for " + imageIndexPageUrl)
return
imageIds = GetImageIDs(imageIndexPageHTML)
for i,j in imageIds:
if CuriosityImageExists(i):
self.log("Image ID %s already exists in the datastore." % i)
if not self.firstRun:
self.log("Stopping processing.")
return
else:
continue
imagePageHtml = self.GetUrlContents(GetImagePageUrl(i))
if not imagePageHtml:
self.log("imagePageHtml is empty for " + str(i))
break
ci = CuriosityImage(key_name=str(i))
ci.imageid = i
ci.title = GetImageTitle(imagePageHtml)
ci.description = GetImageDescription(imagePageHtml)
ci.date = GetImageDate(imagePageHtml)
ci.imageurl = GetMediumImageUrl(imagePageHtml)
SaveCuriosityImage(ci)
self.log("Sleeping for 5 seconds")
sleep(5)
#App engine doesn't like long running pages,
#so we redirect to same page with different querystring
self.ProcessNextPage(imageIndexPageHTML)
def CuriosityImageExists(imageId):
return CuriosityImage.get_by_key_name(str(imageId)) is not None
def SaveCuriosityImage(curiosityImage):
dbCI = CuriosityImage.get_by_key_name(curiosityImage.key().name())
if dbCI is None:
memcache.flush_all()
curiosityImage.put()
def GetImagesFromDataStore(self, numberOfImages):
q = CuriosityImage.all()
q.order('-date')
return q.fetch(limit=numberOfImages)
def test_SaveCuriosityImage_DoNotUpdateExistingCuriosityImage(self):
#Insert a basic CuriosityImage
ci = CuriosityImage(key_name=str(42))
ci.imageid = 42
ci.description = 'DESC123'
ci.put()
newCi = CuriosityImage(key_name=str(42))
newCi.imageid = 42
newCi.description = 'NEWDESC123'
#Save to DB
getfeeds.SaveCuriosityImage(newCi)
#Get back from DB
dbCI = CuriosityImage.get_by_key_name(str(42))
self.assertEquals(u'DESC123', dbCI.description)
self.assertNotEquals(u'NEWDESC123', ci.description)
def GetImagesFromDataStore(self, numberOfImages):
q = CuriosityImage.all()
q.order('-date')
return q.fetch(limit=numberOfImages)