def build_message(self, kind, utt, message, speaker=None):
"""
Build a message for user input or neon response
:param kind: "neon speak" or "execute"
:param utt: string to emit
:param message: incoming message object
:param speaker: speaker data dictionary
:return: Message object
"""
LOG.debug(speaker)
default_speech = self.preference_speech(message)
# Override user preference for all script responses
if not speaker:
speaker = {"name": "Neon",
"language": default_speech["tts_language"],
"gender": default_speech["tts_gender"],
"voice": default_speech["neon_voice"],
"override_user": True}
elif speaker and speaker.get("language"):
speaker["override_user"] = True
else:
speaker = None
LOG.debug(f"data={message.data}")
# LOG.debug(f"context={message.context}")
emit_response = False
if kind == "skill_data":
emit_response = True
kind = "execute"
try:
if kind == "execute":
# This is picked up in the intent handler
return message.reply("skills:execute.utterance", {
"utterances": [utt.lower()],
"lang": message.data.get("lang", "en-US"),
"session": None,
"ident": None,
"speaker": speaker
}, {
"neon_should_respond": True,
"cc_data": {"request": utt,
"emit_response": emit_response,
"execute_from_script": True
}
})
elif kind == "neon speak":
added_context = {"cc_data": message.context.get("cc_data", {})}
added_context["cc_data"]["request"] = utt
return message.reply("speak", {"lang": message.data.get("lang", "en-US"),
"speaker": speaker
}, added_context)
except Exception as x:
LOG.error(x)
def __init__(self):
super().__init__()
try:
from langdetect import detect, detect_langs
except ImportError:
LOG.error("Run pip install langdetect")
raise
self._detect = detect
self._detect_prob = detect_langs
def async_volume_handler(self, vol):
LOG.error("ASYNC SET VOL PASSED IN %s" % (vol, ))
if vol > 1.0:
vol = vol / 10
self.current_volume = vol
LOG.error("ASYNC SET VOL TO %s" % (self.current_volume, ))
# notify anybody listening on the bus who cares
self.bus.emit(
Message("hardware.volume", {"volume": self.current_volume},
context={"source": ["enclosure"]}))
def tearDownClass(cls) -> None:
super(TestAPIMethods, cls).tearDownClass()
cls.messagebus.shutdown()
cls.speech_thread.terminate()
try:
if cls.speech_thread.is_alive():
LOG.error("Bus still alive")
cls.speech_thread.kill()
except Exception as e:
LOG.error(e)
def show_settings_gui(self):
"""
Function to update and
:return:
"""
try:
# TODO: Conditionalize register, only needs to happen once but only after skill init DM
self.gui.register_settings()
self.gui.show_settings()
except Exception as e:
LOG.error(e)
def _confirmation_timeout(self, message):
user = message.data.get("user", "local")
try:
if user in self.actions_to_confirm.keys():
removed = self.actions_to_confirm.pop(user)
LOG.info(f"confirmation timed out ({time.time()}): {removed}")
except Exception as e:
# Catches if the item was already popped
LOG.error(e)
if len(self.actions_to_confirm.keys()) == 0:
self.reload_skill = True
def __init__(self, name=None, bus=None, use_settings=True):
self.user_config = get_neon_user_config()
self.local_config = get_neon_local_config()
self._ngi_settings: Optional[NGIConfig] = None
super(NeonSkill, self).__init__(name, bus, use_settings)
self.cache_loc = os.path.expanduser(self.local_config.get('dirVars', {}).get('cacheDir') or
"~/.local/share/neon/cache")
self.lru_cache = LRUCache()
# TODO: Depreciate these references, signal use is discouraged DM
self.create_signal = create_signal
self.check_for_signal = check_for_signal
self.sys_tz = gettz()
self.gui_enabled = self.local_config.get("prefFlags", {}).get("guiEvents", False)
# if use_settings:
# self.settings = {}
# self._initial_settings = None
# self.init_settings()
# else:
# LOG.error(f"{name} Skill requested no settings!")
# self.settings = None
self.scheduled_repeats = []
# Server-specific imports and timeout setting
# A server is a device that hosts the core and skills to serve clients,
# but that a user will not interact with directly.
# A server will likely serve multiple users and devices concurrently.
if self.local_config.get("devVars", {}).get("devType", "generic") == "server":
self.server = True
self.default_intent_timeout = 90
else:
self.server = False
self.default_intent_timeout = 60
self.neon_core = True # TODO: This should be depreciated DM
self.actions_to_confirm = dict()
self.skill_mode = self.user_config.content.get('response_mode', {}).get('speed_mode') or DEFAULT_SPEED_MODE
self.extension_time = SPEED_MODE_EXTENSION_TIME.get(self.skill_mode)
try:
# Lang support
self.language_config = get_neon_lang_config()
self.lang_detector = DetectorFactory.create() # Default fastlang
self.translator = TranslatorFactory.create() # Default Amazon
except Exception as e:
LOG.error(e)
self.language_config, self.language_detector, self.translator = None, None, None
def find_neon_aws_keys(base_path: str = "~/") -> dict:
"""
Searches standard locations for AWS credentials
Args:
base_path: Base directory to check in addition to XDG directories (default ~/)
Returns:
dict containing 'aws_access_key_id' and 'aws_secret_access_key'
"""
path_to_check = os.path.expanduser(base_path)
csv_paths = (path_to_check, os.path.join(path_to_check, "accessKeys.csv"))
sys_path = os.path.expanduser("~/.aws/credentials")
json_path = os.path.expanduser("~/.local/share/neon/aws.json")
amazon_creds = None
for path in csv_paths:
if os.path.isfile(path):
try:
with open(path, "r") as f:
aws_id, aws_key = f.readlines()[1].rstrip('\n').split(
',', 1)
amazon_creds = {
"aws_access_key_id": aws_id,
"aws_secret_access_key": aws_key
}
except Exception as e:
LOG.error(e)
LOG.error(path)
if not amazon_creds:
if os.path.isfile(json_path):
with open(json_path, "r") as f:
amazon_creds = json.load(f)
elif os.path.isfile(sys_path):
with open(sys_path, "r") as f:
for line in f.read().split("\n"):
if line.startswith("aws_access_key_id"):
aws_id = line.split("=", 1)[1].strip()
elif line.startswith("aws_secret_access_key"):
aws_key = line.split("=", 1)[1].strip()
amazon_creds = {
"aws_access_key_id": aws_id,
"aws_secret_access_key": aws_key
}
if not amazon_creds:
raise FileNotFoundError(
f"No aws credentials found in default locations or path: {path_to_check}"
)
return amazon_creds
def _emit_utterance_to_skills(self, message_to_emit: Message) -> bool:
"""
Emits a message containing a user utterance to skills for intent
processing and checks that it is received by the skills module.
:param message_to_emit: utterance message to send
:return: True if skills module received input, else False
"""
# Emit single intent request
ident = message_to_emit.context['ident']
resp = self.bus.wait_for_response(message_to_emit, timeout=10)
if not resp:
LOG.error(f"Skills didn't handle {ident}!")
return False
return True
def _init_settings(self):
"""
Initializes yml-based skill config settings, updating from default dict as necessary for added parameters
"""
# TODO: This should just use the underlying Mycroft methods DM
super()._init_settings()
if os.path.isfile(os.path.join(self.root_dir, "settingsmeta.yml")):
skill_meta = NGIConfig("settingsmeta", self.root_dir).content
elif os.path.isfile(os.path.join(self.root_dir, "settingsmeta.json")):
with open(os.path.join(self.root_dir, "settingsmeta.json")) as f:
skill_meta = json.load(f)
else:
skill_meta = None
# Load defaults from settingsmeta
default = {}
if skill_meta:
# LOG.info(skill_meta)
LOG.info(skill_meta["skillMetadata"]["sections"])
for section in skill_meta["skillMetadata"]["sections"]:
for pref in section.get("fields", []):
if not pref.get("name"):
LOG.debug(f"non-data skill meta: {pref}")
else:
if pref.get("value") == "true":
value = True
elif pref.get("value") == "false":
value = False
elif isinstance(pref.get("value"), CommentedMap):
value = dict(pref.get("value"))
else:
value = pref.get("value")
default[pref["name"]] = value
# Load or init configuration
self._ngi_settings = NGIConfig(self.name, self.settings_write_path)
# Load any new or updated keys
try:
LOG.debug(self._ngi_settings.content)
LOG.debug(default)
if self._ngi_settings.content and len(self._ngi_settings.content.keys()) > 0 and len(default.keys()) > 0:
self._ngi_settings.make_equal_by_keys(default, recursive=False)
elif len(default.keys()) > 0:
LOG.info("No settings to load, use default")
self._ngi_settings.populate(default)
except Exception as e:
LOG.error(e)
self._ngi_settings.populate(default)
def request_check_timeout(self, time_wait, intent_to_check):
LOG.info("request received")
LOG.info(time_wait)
LOG.info(len(intent_to_check))
try:
if isinstance(intent_to_check, str):
intent_to_check = [intent_to_check]
for intent in intent_to_check:
data = {'time_out': time_wait,
'intent_to_check': f"{self.skill_id}:{intent}"}
LOG.debug(f"DM: Set Timeout: {data}")
self.bus.emit(Message("set_timeout", data))
except Exception as x:
LOG.error(x)
def run(self):
LOG.debug("chase thread started")
chase_ctr = 0
while not self.exit_flag:
chase_ctr += 1
LOG.error("chase thread %s" % (chase_ctr, ))
for x in range(0, 10):
self.led_obj.set_led(x, self.fgnd_col)
time.sleep(self.delay)
self.led_obj.set_led(x, self.bkgnd_col)
if chase_ctr > 10:
self.exit_flag = True
LOG.debug("chase thread stopped")
self.led_obj.fill((0, 0, 0))
def handle_audio_input(self, message):
"""
Handler for `neon.audio_input`.
Handles remote audio input to Neon and replies with confirmation
:param message: Message associated with request
"""
def build_context(msg: Message):
ctx: dict = message.context
defaults = {'client_name': 'mycroft_listener',
'client': 'api',
'source': 'speech_api',
'ident': time(),
'username': self.user_config["user"]["username"] or
"local",
'user_profiles': [self.user_config.content]}
ctx = {**defaults, **ctx, 'destination': ['skills'],
'timing': {'start': msg.data.get('time'),
'transcribed': time()}}
return ctx
ident = message.context.get("ident") or "neon.audio_input.response"
LOG.info(f"Handling audio input: {ident}")
if message.data.get("audio_data"):
wav_file_path = self._write_encoded_file(
message.data.pop("audio_data"))
else:
wav_file_path = message.data.get("audio_file")
lang = message.data.get("lang")
try:
_, parser_data, transcriptions = \
self._get_stt_from_file(wav_file_path, lang)
message.context["audio_parser_data"] = parser_data
context = build_context(message)
data = {
"utterances": transcriptions,
"lang": message.data.get("lang", "en-us")
}
handled = self._emit_utterance_to_skills(Message(
'recognizer_loop:utterance', data, context))
self.bus.emit(message.reply(ident,
data={"parser_data": parser_data,
"transcripts": transcriptions,
"skills_recv": handled}))
except Exception as e:
LOG.error(e)
self.bus.emit(message.reply(ident, data={"error": repr(e)}))
def create(module=None):
module = module or "fastlang"
config = get_neon_lang_config()
module = module or config.get("detection_module", "fastlang")
try:
clazz = DetectorFactory.CLASSES.get(module)
return clazz()
except Exception as e:
# The translate backend failed to start. Report it and fall back to
# default.
LOG.exception(
'The selected language detector backend could not be loaded, '
'falling back to default...')
LOG.error(e)
if module != 'fastlang':
return FastLangDetector()
else:
raise
def clean_quotes(raw_utt: str) -> str:
"""
Method for stripping quotes from fully quoted strings in different languages
:param raw_utt: Input string to be cleaned
:return: string with all paired quote characters removed
"""
if not raw_utt:
raise ValueError("Expected a string and got None")
if not isinstance(raw_utt, str):
raise TypeError(f"{raw_utt} is not a string!")
chars_to_remove = ['“', '"', '«', u'\u201d', u'\u00bb', u'\u201e', '「', '」', u'u\xa0', u'\u00a0']
raw_utt = raw_utt.strip()
utt = raw_utt
trailing_punctuation = False
if utt.endswith("."):
trailing_punctuation = True
utt = utt.rstrip(".")
quotes_cleaned = False
try:
# Checks if utterance starts AND ends with some form of quotations and removes them accordingly
while (utt.startswith('“') or utt.startswith(u'\u201d') or utt.startswith('"') or utt.startswith('«')
or utt.startswith(u'\u00bb') or utt.startswith(u'\u201e') or utt.startswith('「') or
utt.startswith(u'u\xa0') or utt.startswith(u'\u00a0')) and \
(utt.endswith('“') or utt.endswith(u'\u201d') or utt.endswith('"') or utt.endswith(u'\u00bb') or
utt.endswith(u'\u201e') or utt.endswith('」') or utt.endswith(u'u\xa0') or
utt.endswith(u'\u00a0') or utt.endswith('»')):
quotes_cleaned = True
removed_left, removed_right = False, False
for c in chars_to_remove:
if not removed_left and utt.startswith(c):
utt = utt[1:]
removed_left = True
if not removed_right and utt.endswith(c):
utt = utt[:-1]
removed_right = True
if quotes_cleaned:
if trailing_punctuation:
return f"{utt}."
return utt
else:
return raw_utt
except Exception as x:
LOG.error(x)
return raw_utt
def handle_goodbye_intent(self, message):
"""
Note: now the "reply" intents are deactivated,
since the user specified the end of the skill
by saying "goodbye"
"""
# Remove any awaiting confirmation
try:
user = self.get_utterance_user(message)
self.actions_to_confirm.pop(user)
except Exception as e:
LOG.error(e)
self.disable_intent('CaffeineContentGoodbyeIntent')
# self.disable_intent('CaffeineYesIDoIntent')
# self.disable_intent('Caffeine_no_intent')
# LOG.debug('3- Goodbye')
self.speak_dialog("StayCaffeinated")
def find_neon_google_keys(base_path: str = "~/") -> dict:
"""
Locates google json credentials and returns the parsed credentials as a dict
Args:
base_path: Base directory to check in addition to XDG directories (default ~/)
Returns:
dict Google json credential
"""
path_to_check = os.path.expanduser(base_path)
paths_to_check = (path_to_check,
os.path.join(path_to_check, "google.json"),
os.path.expanduser("~/.local/share/neon/google.json"))
for path in paths_to_check:
if os.path.isfile(path):
try:
with open(path, "r") as f:
credential = json.load(f)
return credential
except Exception as e:
LOG.error(f"Invalid google credential found at: {path}")
raise e
raise FileNotFoundError(f"No google credentials found in default locations or path: {path_to_check}")
def preference_skill(self, message=None) -> dict:
"""
Returns the skill settings configuration
Equivalent to self.settings for non-server
:param message: Message associated with request
:return: dict of skill preferences
"""
nick = get_message_user(message) if message else None
if self.server and nick:
try:
skill = self.skill_id
LOG.info(f"Get server prefs for skill={skill}")
user_overrides = message.context["nick_profiles"][nick]["skills"].get(self.skill_id, dict())
LOG.debug(user_overrides)
merged_settings = {**self.settings, **user_overrides}
if user_overrides.keys() != merged_settings.keys():
LOG.info(f"New settings keys: user={nick}|skill={self.skill_id}|user={user_overrides}")
self.update_skill_settings(merged_settings, message)
return merged_settings
except Exception as e:
LOG.error(e)
return self.settings
def get_utterance_user(self, message: Optional[Message]) -> str:
"""
Gets the user associated with the given message. Returns default 'local' or 'server' if no user specified.
Args:
message: Message associated with request
Returns:
Username associated with the message or a default value of 'local' or 'server'.
"""
if self.server:
default_user = "******"
else:
default_user = self.preference_user(message).get("username", "local")
if not message:
return default_user
try:
return get_message_user(message) or default_user
except Exception as e:
LOG.error(e)
# TODO: Depreciate this and fix underlying error DM
return default_user
def preference_unit(self, message=None) -> dict:
"""
Returns the units dictionary that contains time, date, measure formatting preferences
Equivalent to self.user_config["units"] for non-server use
"""
try:
nick = get_message_user(message) if message else None
if self.server:
if not message or not nick:
LOG.warning("No message given!")
return self.user_config['units']
if message.context.get("nick_profiles"):
return message.context["nick_profiles"][nick]["units"]
else:
LOG.error(
f"Unable to get user settings! message={message.data}")
else:
return self.user_config['units']
except Exception as x:
LOG.error(x)
return {'time': 12, 'date': 'MDY', 'measure': 'imperial'}
def get_forecast(lat: Union[str, float],
lng: Union[str, float],
units: str = "metric",
**kwargs) -> dict:
"""
Queries Open Weather Map for weather data at the specified location
:param lat: latitude
:param lng: longitude
:param units: Temperature and Speed units "metric", "imperial", or "standard"
:param kwargs:
'api_key' - optional str api_key to use for query (None to force remote lookup)
'language' - optional language param (default english)
:return: dict weather data
"""
api_key = kwargs.get("api_key", AUTH_CONFIG.get("owm", {}).get("api_key"))
if api_key:
query_params = {
"lat": lat,
"lon": lng,
"units": units,
"appid": api_key
}
resp = query_owm_api(
f"http://api.openweathermap.org/data/2.5/onecall?{urllib.parse.urlencode(query_params)}"
)
else:
query_params = {"lat": lat, "lon": lng, "units": units}
resp = request_neon_api(NeonAPI.OPEN_WEATHER_MAP, query_params)
data = json.loads(resp["content"])
if data.get('cod'):
data['cod'] = str(
data['cod']
) # 400 is str, 401 is int; cast all to str for safe refs
LOG.error(f"Error return: {data}")
# TODO: Handle failures
return data
def preference_brands(self, message=None) -> dict:
"""
Returns a brands dictionary for the user
Equivalent to self.user_config["speech"] for non-server use
"""
try:
nick = get_message_user(message) if message else None
if self.server:
if not message or not nick:
LOG.warning("No message given!")
return self.user_config['brands']
if message.context.get("nick_profiles"):
return message.context["nick_profiles"][nick]["brands"]
else:
LOG.error(f"Unable to get user settings! message={message.data}")
else:
return self.user_config['brands']
except Exception as x:
LOG.error(x)
return {'ignored_brands': {},
'favorite_brands': {},
'specially_requested': {}}
def find_generic_keyfile(base_path: str, filename: str) -> str:
"""
Locates a generic text keyfile
Args:
base_path: Base directory to check in addition to XDG directories (default ~/)
filename: File basename to read
Returns:
str contents of located file
"""
path_to_check = os.path.expanduser(base_path)
paths_to_check = (path_to_check, os.path.join(path_to_check, filename),
os.path.expanduser(f"~/.local/share/neon/{filename}"))
for path in paths_to_check:
if os.path.isfile(path):
try:
with open(path, "r") as f:
credential = f.read().strip()
return credential
except Exception as e:
LOG.error(f"Invalid credential found at: {path}")
raise e
raise FileNotFoundError(
f"No credentials found in default locations or path: {path_to_check}")
def preference_user(self, message=None) -> dict:
"""
Returns the user dictionary with name, email
Equivalent to self.user_config["user"] for non-server use
"""
try:
nick = get_message_user(message) if message else None
if self.server:
if not message or not nick:
LOG.warning("No message given!")
return self.user_config['user']
if message.context.get("nick_profiles"):
return message.context["nick_profiles"][nick]["user"]
else:
LOG.error(
f"Unable to get user settings! message={message.data}")
else:
return self.user_config['user']
except Exception as x:
LOG.error(x)
return {
'first_name': '',
'middle_name': '',
'last_name': '',
'preferred_name': '',
'full_name': '',
'dob': 'YYYY/MM/DD',
'age': '',
'email': '',
'username': '',
'password': '',
'picture': '',
'about': '',
'phone': '',
'email_verified': False,
'phone_verified': False
}
def send_email(self,
title,
body,
message=None,
email_addr=None,
attachments=None):
"""
Send an email to the registered user's email.
Email address priority: email_addr, user prefs from message, fallback to DeviceApi for Mycroft method
Arguments:
title (str): Title of email
body (str): HTML body of email. This supports
simple HTML like bold and italics
email_addr (str): Optional email address to use
attachments (dict): Optional dict of file names to Base64 encoded files
message (Message): Optional message to get email from
"""
if not email_addr and message:
email_addr = self.preference_user(message).get("email")
if email_addr:
LOG.info("Send email via Neon Server")
try:
LOG.debug(f"body={body}")
self.bus.emit(
Message(
"neon.send_email", {
"title": title,
"email": email_addr,
"body": body,
"attachments": attachments
}))
except Exception as e:
LOG.error(e)
else:
super().send_email(title, body)
def on_error(e='Unknown'):
LOG.error('Enclosure failed: {}'.format(repr(e)))
def build_message(kind, utt, message, signal_to_check=None, speaker=None):
"""
Build a message for user input or neon response
:param kind: "neon speak" or "execute"
:param utt: string to emit
:param message: incoming message object
:param signal_to_check: signal to check in speech
:param speaker: speaker data dictionary
:return: Message object
"""
from copy import deepcopy
from neon_utils import SKILL
# utt = utt.strip('"') This is done before calling build_message now
# Use utt as default signal to check
if not signal_to_check:
signal_to_check = utt
LOG.debug(speaker)
default_speech = SKILL.preference_speech(message)
# Override user preference for all script responses
if not speaker:
speaker = {
"name": "Neon",
"language": default_speech["tts_language"],
"gender": default_speech["tts_gender"],
"voice": default_speech["neon_voice"],
"override_user": True
}
else:
speaker["override_user"] = True
LOG.debug(f"data={message.data}")
LOG.debug(f"context={message.context}")
emit_response = False
if kind == "skill_data":
emit_response = True
kind = "execute"
try:
if kind in ("execute", "skill"):
message.context["cc_data"] = message.context.get("cc_data", {})
# This is picked up in the intent handler
# return message.reply("skills:execute.utterance", {
return message.reply(
"recognizer_loop:utterance",
{
"utterances": [utt.lower()],
"lang": message.data.get("lang", "en-US"),
"session": None,
"ident": None,
"speaker": speaker
},
{
# "mobile": message.context.get("mobile", False),
# "client": message.context.get("client", None),
# "flac_filename": message.context.get("flac_filename", ''),
# "nick_profiles": message.context.get("nick_profiles", {}),
"neon_should_respond": True,
"cc_data": {
"signal_to_check":
signal_to_check,
"request":
utt,
"emit_response":
emit_response,
# "Neon": True,
"execute_from_script":
True,
"audio_file":
message.context["cc_data"].get("audio_file", None),
"raw_utterance":
utt
}
})
elif kind == "neon speak":
context = deepcopy(message.context)
LOG.info(f"CONTEXT IS {context}")
context["cc_data"] = context.get("cc_data", {})
context["cc_data"]["signal_to_check"] = signal_to_check
context["cc_data"]["request"] = utt
return message.reply("speak", {
"lang": message.data.get("lang", "en-US"),
"speaker": speaker
}, context)
except Exception as x:
LOG.error(x)
def on_error(e='Unknown'):
LOG.error('Audio service failed to launch ({}).'.format(repr(e)))
def build_new_auth_config(key_path: str = "~/") -> dict:
"""
Constructs a dict of authentication key data by locating credential files in the specified path
:param key_path: path to locate key files (default locations checked in addition)
:return: dict of located authentication keys
"""
key_path = key_path or "~/"
auth_config = dict()
try:
auth_config["github"] = {"token": find_neon_git_token(key_path)}
except Exception as e:
LOG.error(e)
try:
auth_config["amazon"] = find_neon_aws_keys(key_path)
except Exception as e:
LOG.error(e)
try:
auth_config["wolfram"] = {"app_id": find_neon_wolfram_key(key_path)}
except Exception as e:
LOG.error(e)
try:
auth_config["google"] = find_neon_google_keys(key_path)
except Exception as e:
LOG.error(e)
try:
auth_config["alpha_vantage"] = {
"api_key": find_neon_alpha_vantage_key(key_path)
}
except Exception as e:
LOG.error(e)
try:
auth_config["owm"] = {"api_key": find_neon_owm_key(key_path)}
except Exception as e:
LOG.error(e)
return auth_config
def scrape_page_for_links(url: str) -> dict:
"""
Scrapes the passed url for any links and returns a dictionary of link labels to URLs
:param url: Web page to scrape
:return: Lowercase names to links on page
"""
import unicodedata
available_links = {}
retry_count = 0
def _get_links(url):
LOG.debug(url)
if not str(url).startswith("http"):
url = f"http://{url}"
LOG.debug(url)
html = requests.get(url, timeout=2.0).text
soup = BeautifulSoup(html, 'lxml')
# LOG.debug(html)
# LOG.debug(soup)
# Look through the page and find all anchor tags
for i in soup.find_all("a", href=True):
# LOG.debug(f"DM: found link: {i.text.rstrip()}")
# LOG.debug(f"DM: found href: {i['href']}")
if '://' not in i['href']:
# Assume this is a relative address
href = url + i['href'].lower()
elif url.split('://')[1] in i['href']:
href = i['href'].lower()
else:
href = None
if href:
available_links[unicodedata.normalize('NFKD', i.text.rstrip()
.replace(u'\u2013', '')
.replace(u'\u201d', '')
.replace(u'\u201c', '')
.replace('"', "")
.replace("'", "")
.replace("'", "")
.lower())] = href
LOG.debug("found link: " + unicodedata.normalize("NFKD", i.text.rstrip().replace(u"\u2013", "")
.replace(u"\u201d", "").replace(u"\u201c", "")
.replace('"', "").replace("'", "")
.replace("'", "").replace("\n", "").lower()))
LOG.debug("found href: " + href)
LOG.debug(available_links)
try:
_get_links(url)
except ConnectTimeout:
retry_count += 1
if retry_count < 8:
_get_links(url)
else:
raise ConnectTimeout
except Exception as x:
LOG.error(x)
LOG.debug(available_links)
raise ReferenceError
return available_links
