def speak(person_name, location_name, ignore_motion, ignore_confirmation,
voice_id, no_audio, simulate, fail_confirm):
try:
while True:
click.echo("Checking for messages . . .")
lm = LocationManager()
loc = lm.get_location(location_name)
if not ignore_motion and not loc.is_motion:
print 'Exiting. No motion detected at ' + location_name
exit(1)
speaker = Speaker(NoAudio=no_audio)
message_manager = MessageManager(LocationName=location_name)
click.echo('getting files')
cache_manager = CacheManager(BucketName='pollexy-media',
CacheName='chimes')
cache_manager.sync_remote_folder()
print 'Writing speech'
vid, speech = message_manager.write_speech(PersonName=person_name)
if vid:
voice_id = vid
if not speech:
print "I have nothing to say"
message_manager.delete_sqs_msgs()
else:
print "Speech:\n\n%s" % speech
try:
pm = PersonManager()
p = pm.get_person(person_name)
do_speech = True
if fail_confirm:
logging.warning("FORCE FAILING confirmation")
reason, do_speech = "NoResponse", False
elif not no_audio and p.require_physical_confirmation and \
not ignore_confirmation:
lv = LocationVerification(PersonName=person_name,
LocationName=location_name,
VoiceId=voice_id)
do_speech, retry_count, timeout = \
lv.verify_person_at_location(SpeechMethod=say)
if do_speech:
speaker = Speaker(NoAudio=no_audio)
speaker.generate_audio(Message=speech,
TextType='ssml',
VoiceId=voice_id)
speaker.speak(IncludeChime=True)
message_manager.succeed_speech(dont_delete=simulate)
else:
message_manager.fail_speech(Reason=reason)
finally:
speaker.cleanup()
except Exception as exc:
exc_type, exc_value, exc_traceback = sys.exc_info()
print repr(
traceback.format_exception(exc_type, exc_value, exc_traceback))
click.echo("Error: %s" % str(exc))
exit(2)
def startup_non_main_thread():
# type: () -> None
"""
:return:
"""
if module_logger.isEnabledFor(LazyLogger.DEBUG):
module_logger.debug('%s', 'Enter', lazy_logger=False)
Settings.save_settings()
Monitor.register_settings_changed_listener(
Settings.on_settings_changed)
Monitor.register_settings_changed_listener(
LazyLogger.on_settings_changed)
try:
Settings.get_locale()
except AbortException:
reraise(*sys.exc_info())
except Exception:
pass
load_trailers()
# Start the periodic garbage collector
CacheManager.get_instance().start_cache_garbage_collection_thread()
Monitor.register_settings_changed_listener(load_trailers)
async def _extract_meta_data(
self,
allow_list: dict,
cache_manager: CacheManager,
shared_memory_name: str,
) -> dict:
data = {}
tasks = []
shared_status = shared_memory.ShareableList(name=shared_memory_name)
shared_status[0] = 0
for metadata_extractor in self.metadata_extractors:
if allow_list[metadata_extractor.key]:
if (not cache_manager.bypass
and self.is_feature_whitelisted_for_cache(
metadata_extractor)
and cache_manager.is_host_predefined()
and cache_manager.is_enough_cached_data_present(
metadata_extractor.key)):
extracted_metadata: dict = (
cache_manager.get_predefined_metadata(
metadata_extractor.key))
data.update(extracted_metadata)
shared_status[0] += 1
else:
tasks.append(metadata_extractor.start())
extracted_metadata: tuple[dict] = await asyncio.gather(*tasks)
shared_status[0] += len(tasks)
data = {**data, **dict(ChainMap(*extracted_metadata))}
return data
def test_is_host_predefined(cache_manager: CacheManager):
assert cache_manager.is_host_predefined() is False
test_host = "test_host"
cache_manager._hosts = [test_host]
cache_manager.domain = test_host
assert cache_manager.is_host_predefined()
def test_convert_cached_data(cache_manager: CacheManager):
meta_data = []
feature = ACCESSIBILITY
empty_cache = cache_manager.convert_cached_data(meta_data, feature)
assert empty_cache[EXPLANATION] == [Explanation.Cached]
assert empty_cache[STAR_CASE] == StarCase.ONE
assert empty_cache[VALUES] == []
meta_data = [
f'{{"{VALUES}":[], "{EXPLANATION}": ["{Explanation.AccessibilityServiceReturnedFailure}"], '
f'"{STAR_CASE}": "{StarCase.ONE}", '
f'"{TIMESTAMP}": {get_utc_now()}}}'
]
cache_output = cache_manager.convert_cached_data(meta_data, feature)
assert isinstance(cache_output, dict)
assert len(cache_output.keys()) == 3
assert cache_output[EXPLANATION] == [
Explanation.Cached,
Explanation.AccessibilityServiceReturnedFailure,
]
assert cache_output[STAR_CASE] == StarCase.ONE
assert cache_output[VALUES] == []
meta_data = [
f'{{"{VALUES}":[], "{EXPLANATION}": [], '
f'"{STAR_CASE}": "{StarCase.ONE}", '
f'"{TIMESTAMP}": {get_utc_now()}}}'
]
cache_output = cache_manager.convert_cached_data(meta_data, feature)
assert cache_output[STAR_CASE] == StarCase.ONE
meta_data = [
f'{{"{VALUES}":[0.85, 0.95], "{EXPLANATION}": ["{Explanation.AccessibilitySuitable}"], '
f'"{STAR_CASE}": "{StarCase.ONE}", '
f'"{TIMESTAMP}": {get_utc_now()}}}'
]
cache_output = cache_manager.convert_cached_data(meta_data, feature)
assert cache_output[VALUES] == [0.9]
assert cache_output[EXPLANATION] == [
Explanation.Cached,
Explanation.AccessibilitySuitable,
]
meta_data = [
f'{{"{VALUES}":[], "{EXPLANATION}": [], '
f'"{STAR_CASE}": "{StarCase.FIVE}", '
f'"{TIMESTAMP}": {get_utc_now()}}}'
]
cache_output = cache_manager.convert_cached_data(meta_data, feature)
assert cache_output[STAR_CASE] == StarCase.FIVE
def test_is_enough_cached_data_present(cache_manager: CacheManager):
cache_entries = [f'{{"timestamp": {get_utc_now()}}}'] * 1
with mock.patch(
"cache.cache_manager.read_cached_values_by_feature") as read_cache:
read_cache.return_value = cache_entries
assert (cache_manager.is_enough_cached_data_present(ACCESSIBILITY) is
False)
cache_entries = [f'{{"timestamp": {get_utc_now()}}}'] * 5
with mock.patch(
"cache.cache_manager.read_cached_values_by_feature") as read_cache:
read_cache.return_value = cache_entries
assert (cache_manager.is_enough_cached_data_present(ACCESSIBILITY) is
True)
def test_can_pull_down_cache_folder():
s3 = boto3.resource('s3')
s3.create_bucket(Bucket="test-bucket")
with patch('__builtin__.open', mock_open(read_data="TestData"),
create=True) as m:
with open('file') as h:
s3.Bucket('test-bucket').put_object(Key='chimes/file1', Body=h)
s3.Bucket('test-bucket').put_object(Key='chimes/file2', Body=h)
s3.Bucket('test-bucket').put_object(Key='chimes/file3', Body=h)
cache_manager = CacheManager(BucketName="test-bucket", CacheName="chimes")
with patch.object(boto3.s3.transfer.S3Transfer,
"download_file",
return_value=None) as m:
cache_manager.sync_remote_folder()
assert len(m.mock_calls) == 3
def cache_data(
self,
extracted_metadata: dict,
cache_manager: CacheManager,
allow_list: dict,
):
for feature, meta_data in extracted_metadata.items():
if (allow_list[feature]
and Explanation.Cached not in meta_data[EXPLANATION]):
values = []
if feature == ACCESSIBILITY:
values = meta_data[VALUES]
data_to_be_cached = {
VALUES: values,
STAR_CASE: meta_data[STAR_CASE],
TIMESTAMP: get_utc_now(),
EXPLANATION: meta_data[EXPLANATION],
}
create_cache_entry(
cache_manager.get_domain(),
feature,
data_to_be_cached,
self._logger,
)
def test_init(mocker):
with mock.patch("cache.cache_manager.get_logger"):
with mock.patch(
"cache.cache_manager.CacheManager._class._prepare_cache_manager"
) as prepare_cache_manager:
manager = CacheManager.get_instance()
assert manager._logger.debug.call_count == 1
assert prepare_cache_manager.call_count == 1
def cache_manager(mocker):
with mock.patch("cache.cache_manager.get_logger"):
with mock.patch(
"cache.cache_manager.CacheManager._class._prepare_cache_manager"
):
manager = CacheManager.get_instance()
return manager
def teach(no_audio, voice_id):
try:
c = ConfigHelper().config
cache_manager = CacheManager(BucketName='pollexy-media',
CacheName='chimes')
cache_manager.sync_remote_folder()
chime = c['teach_chime']
speech = random.choice(c['teach_phrases'])
speaker = Speaker(NoAudio=no_audio)
logging.info('Teach phrase={}'.format(speech))
speaker.generate_audio(Message=speech, TextType='text',
VoiceId=voice_id)
speaker.speak(IncludeChime=True, Chime=chime)
speaker.cleanup()
except Exception as exc:
exc_type, exc_value, exc_traceback = sys.exc_info()
print repr(traceback.format_exception(exc_type, exc_value,
exc_traceback))
click.echo("Error: %s" % str(exc))
exit(2)
def test_get_predefined_metadata(mocker, cache_manager: CacheManager):
meta_data = []
feature = ACCESSIBILITY
empty_cache = cache_manager.convert_cached_data(meta_data, feature)
assert empty_cache[EXPLANATION] == [Explanation.Cached]
assert empty_cache[STAR_CASE] == StarCase.ONE
assert empty_cache[VALUES] == []
meta_data = (
f'{{"{VALUES}":[], "{EXPLANATION}": ["{Explanation.AccessibilityServiceReturnedFailure}"], '
+ f'"{STAR_CASE}": "{StarCase.ONE}"}}')
with mock.patch("cache.cache_manager.read_cached_values_by_feature"):
cache_manager.convert_cached_data = mocker.MagicMock()
cache_manager.convert_cached_data.return_value = json.loads(meta_data)
cache_output = cache_manager.get_predefined_metadata(feature)
assert isinstance(cache_output, dict)
assert len(cache_output.keys()) == 1
assert len(cache_output[feature].keys()) == 4
def start(self, message: dict) -> dict:
self._logger.debug(
f"Start metadata_manager at {time.perf_counter() - global_start} since start"
)
shared_status = shared_memory.ShareableList(
name=message[MESSAGE_SHARED_MEMORY_NAME])
url = message[MESSAGE_URL]
if len(url) > 1024:
url = url[0:1024]
shared_status[1] = url
website_manager = WebsiteManager.get_instance()
self._logger.debug(
f"WebsiteManager initialized at {time.perf_counter() - global_start} since start"
)
website_manager.load_website_data(message=message)
self._logger.debug(
f"WebsiteManager loaded at {time.perf_counter() - global_start} since start"
)
cache_manager = CacheManager.get_instance()
cache_manager.update_to_current_domain(
website_manager.website_data.domain,
bypass=message[MESSAGE_BYPASS_CACHE],
)
now = time.perf_counter()
self._logger.debug(
f"starting_extraction at {now - global_start} since start")
starting_extraction = get_utc_now()
if website_manager.website_data.html == "":
exception = "Empty html. Potentially, splash failed."
extracted_meta_data = {MESSAGE_EXCEPTION: exception}
else:
try:
extracted_meta_data = asyncio.run(
self._extract_meta_data(
allow_list=message[MESSAGE_ALLOW_LIST],
cache_manager=cache_manager,
shared_memory_name=message[MESSAGE_SHARED_MEMORY_NAME],
))
self.cache_data(
extracted_meta_data,
cache_manager,
allow_list=message[MESSAGE_ALLOW_LIST],
)
except ConnectionError as e:
exception = f"Connection error extracting metadata: '{e.args}'"
self._logger.exception(
exception,
exc_info=True,
)
extracted_meta_data = {MESSAGE_EXCEPTION: exception}
except Exception as e:
exception = (
f"Unknown exception from extracting metadata: '{e.args}'. "
f"{''.join(traceback.format_exception(None, e, e.__traceback__))}"
)
self._logger.exception(
exception,
exc_info=True,
)
extracted_meta_data = {MESSAGE_EXCEPTION: exception}
self._logger.debug(
f"extracted_meta_data at {time.perf_counter() - global_start} since start"
)
extracted_meta_data.update({
"time_for_extraction": get_utc_now() - starting_extraction,
**website_manager.get_website_data_to_log(),
})
website_manager.reset()
cache_manager.reset()
shared_status[1] = ""
self._logger.debug(
f"website_manager.reset() at {time.perf_counter() - global_start} since start"
)
return extracted_meta_data
def get_cache_manager():
with mock.patch("cache.cache_manager.get_logger"):
with mock.patch(
"cache.cache_manager.CacheManager._class._prepare_cache_manager"
):
return CacheManager.get_instance()
def delete_cache(reset_input: DeleteCacheInput):
cache_manager = CacheManager.get_instance()
row_count = cache_manager.reset_cache(reset_input.domain)
return {"deleted_rows": row_count}
def speak(person_name,
location_name,
ignore_motion,
ignore_confirmation,
voice_id,
no_audio,
simulate,
fail_confirm,
verbose):
log = logging.getLogger('PollexyCli')
if verbose:
os.environ['LOG_LEVEL'] = 'DEBUG'
log.setLevel(logging.DEBUG)
try:
while True:
lm = LocationManager()
loc = lm.get_location(location_name)
if not ignore_motion and not loc.is_motion:
print 'Exiting. No motion detected at ' + location_name
exit(1)
speaker = Speaker(NoAudio=no_audio)
message_manager = MessageManager(LocationName=location_name)
bm = message_manager.get_messages(MessageType='Bot',
PersonName=person_name)
if bm and len(bm) > 0:
log.debug('Bot count = {}'.format(len(bm)))
for bot in bm:
username = str(uuid.uuid4())
try:
lp = LexPlayer(
BotNames=bot.bot_names,
Alias="$LATEST",
Username=username,
VoiceId=voice_id,
IceBreaker=bot.ice_breaker,
Introduction=bot.introduction,
NoAudio=no_audio,
BotsRequired=bot.required_bots)
while (not lp.is_done):
lp.get_user_input()
except Exception as e:
print 'Bot failed: {}'.format(e)
raise
message_manager.succeed_messages(dont_delete=simulate)
cache_manager = CacheManager(BucketName='pollexy-media',
CacheName='chimes')
cache_manager.sync_remote_folder()
vid, speech = message_manager.write_speech(PersonName=person_name)
if vid:
voice_id = vid
if not speech:
message_manager.succeed_messages(dont_delete=simulate)
message_manager.delete_sqs_msgs()
else:
try:
pm = PersonManager()
p = pm.get_person(person_name)
do_speech = True
if fail_confirm:
log.warn("FORCE FAILING confirmation")
reason, do_speech = "NoResponse", False
elif not no_audio and p.require_physical_confirmation and \
not ignore_confirmation:
lv = LocationVerification(PersonName=person_name,
LocationName=location_name,
VoiceId=voice_id)
do_speech, retry_count, timeout = \
lv.verify_person_at_location(SpeechMethod=say)
log.debug('do_speech={}'.format(bool(do_speech)))
if fail_confirm:
message_manager.fail_messages(Reason=reason)
else:
if do_speech:
log.debug('starting speech')
speaker = Speaker(NoAudio=no_audio)
speaker.generate_audio(Message=speech,
TextType='ssml',
VoiceId=voice_id)
speaker.speak(IncludeChime=True)
log.debug('Succeeding messages')
message_manager.succeed_messages(dont_delete=simulate)
finally:
speaker.cleanup()
except Exception as exc:
exc_type, exc_value, exc_traceback = sys.exc_info()
print repr(traceback.format_exception(exc_type, exc_value,
exc_traceback))
click.echo("Error: %s" % str(exc))
exit(2)
def test_missing_cache_name_throws_error():
with pytest.raises(ValueError) as exception:
CacheManager(BucketName="Test")
assert 'No cache name provided' in str(exception.value)
def test_empty_bucket_name_throws_error():
with pytest.raises(ValueError) as exception:
CacheManager(BucketName="", CacheName="test")
assert 'No S3 bucket name provided' in str(exception.value)