def test_index_can_store_photo(self):
"""Test index can store a photo."""
self.index.es.index = MagicMock()
time.time = MagicMock(return_value=time.time())
url = Url.from_string('http://example.com')
path = PhotoPath(self.datadir)
path.filesize = MagicMock(return_value=10000)
photo = LoadingPhoto(url=url, path=path, refresh_rate=refresh.Hourly)
self.index.save_photo(photo)
self.index.es.index.assert_called_with(
index='photos',
doc_type='photo',
id=path.uuid,
body={
'url_id': url.hash(),
'refresh_rate': refresh.Hourly.lock_format(),
'captured_at': refresh.Hourly().lock(),
'filesize': photo.filesize(),
'filename': photo.filename(),
'directory': photo.directory(),
'domain': photo.domain(),
'timestamp': int(time.time())
})
def test_loading_photo_can_be_saved_to_datadir(self):
"""Test a loading photo can be saved to data directory."""
path = PhotoPath(self.datadir)
url = Url.from_string('https://example.com')
photo = LoadingPhoto(url=url, path=path, refresh_rate=refresh.Hourly)
photo.save_loading_text()
self.assertTrue(isfile(path.full_path()))
self.assertEqual('loading', photo.get_raw())
def test_camera_can_save_screenshot(self):
"""Test camera can save screenshot."""
self.creates_webdriver()
self.camera.webdriver.get_window_size = MagicMock()
self.camera.webdriver.save = MagicMock()
self.camera.webdriver.save_screenshot = MagicMock()
path = PhotoPath(self.datadir)
self.camera._save(path)
self.camera.webdriver.save_screenshot.assert_called_with(
path.full_path())
def test_filesystem_can_translate_path_to_file_in_datadir(self):
"""Test filesystem can translate path to file in datadir."""
datadir_path = PhotoPath(self.datadir)
url = Url.from_string('https://example.com/foo/bar')
photo = Screenshot(url, datadir_path, self.refresh_rate)
self.index.es.index = MagicMock()
photo.path.filesize = MagicMock(return_value=10000)
self.index.save_photo(photo)
self.index.photos_file_exists = MagicMock(return_value=123000)
self.index.photos_get_photo = MagicMock(return_value=photo)
path = self.filesystem._translate_path(
'/example.com/2019-01-13H20:00/foo/bar.png')
self.assertEqual(datadir_path.full_path(), path)
def tick(self):
"""Tick.
Checkout a url from index, take photo of url,
save to datadir and update index with photo
metadata
"""
try:
timer = time.time()
url = self._checkout_url()
console.dp(f'taking photo of {url.to_string()}')
path = PhotoPath(self.datadir)
photo = LoadingPhoto(
url=url,
path=path,
refresh_rate=self.refresh_rate
)
photo.save_loading_text()
self.index.save_photo(photo)
camera = c.Camera(
viewport_width=self.viewport_width,
viewport_height=self.viewport_height,
viewport_max_height=self.viewport_max_height,
addons={
'IDCAC': Addons.IDCAC,
'REFERER_HEADER': Addons.REFERER_HEADER,
'UBLOCK_ORIGIN': Addons.UBLOCK_ORIGIN,
}
)
photo = camera.take_picture(url, path, self.refresh_rate)
self.index.save_photo(photo)
timer = int(time.time() - timer)
console.p(
f'photo was taken of {url.to_string()} took: {timer}s'
)
except EmptySearchResultException as e:
pass
finally:
time.sleep(1)
def photos_get_photo(self, domain: str, captured_at: str,
full_filename: str,
refresh_rate: Type[RefreshRate]) -> Photo:
"""Get photo from photos index.
Args:
domain: domain photo belongs to
captured_at: when photo was captured
full_filename: full filename of photo
eg. /some/path/some-filename.png
refresh_rate: Given refresh rate photo was taken with
Returns:
Requested photo metadata
Photo
Raises:
PhotoNotFoundException: If photo was not found
"""
directory = '/'.join(full_filename.split('/')[:-1])
directory = directory.rstrip('/') + '/'
filename = full_filename.split('/')[-1:][0]
captured_at = file.LastCapture.translate(captured_at, domain, self,
refresh_rate)
res = self.es.search(index=Index.PHOTOS,
size=1,
body={
'query': {
'bool': {
'must': [{
'term': {
'domain': domain
}
}, {
'term': {
'refresh_rate':
refresh_rate.lock_format()
}
}, {
'term': {
'captured_at': captured_at
}
}, {
'term': {
'directory': directory
}
}, {
'term': {
'filename': filename
}
}]
}
},
})
if res['hits']['total'] == 0:
raise PhotoNotFoundException('no photo was found')
res = res['hits']['hits'][0]
uuid = res['_id']
if self.datadir is None:
raise Exception('Cannot get photo from Index without a data dir')
path = PhotoPath(self.datadir, uuid=uuid)
photo = Screenshot(url=UrlId(res['_source']['url_id']),
path=path,
refresh_rate=refresh_rate,
index_filesize=res['_source']['filesize'])
return photo
def test_photo_path_can_be_created(self):
"""Pass."""
path = PhotoPath(self.datadir)
self.assertGreater(len(path.uuid), 20)
self.assertIn(member=self.datadir.root, container=path.full_path())