async def predict_sentiment(request):
payload = await request.json()
x = parse_input(payload)
if not x:
return web.Response(
text=
"No or empty input received. Please post your text body as json of the form {'Text': text body string}.",
status=400)
elif x is KeyError:
return web.Response(
text=
"Wrong input. Please post your text body as json of the form {'Text': text body string}.",
status=400)
elif x is ValueError:
return web.Response(
text=
"Wrong input type. Please post your text body as json of the form {'Text': text body string}.",
status=400)
else:
try:
ft = get_features_from_text(x, tfidf_v)
predicted_spam = get_prediction([ft])
return web.json_response(
{"spam probability": round(predicted_spam[0][0], 3)})
except Exception as e:
LOG.error(f"Application errored: {e.__repr__()}")
return web.Response(text="Something has gone very wrong indeed...",
status=500)
def set_duty_cycle(self, pin, value):
"""
:param pin:
:param value:
:return:
"""
if pin not in self.output_pins:
log.error("pin %s was not registered as an output" % pin)
return
output_pin = self.output_pins[pin]
if not output_pin['pwm']:
log.error("pwm was not registered at pin %d" % pin)
if value > 100:
log.warning("Given dutycycle (%d) is bigger than 100. Has been set to 100" % value)
value = 100
if value < 0:
log.warning("Given dutycycle (%d) is lower than 0. Has been set to 0" % value)
value = 0
if not output_pin['pwm_startet']:
output_pin['pwm'].start(value)
output_pin['pwm_startet'] = True
else:
output_pin['pwm'].ChangeDutyCycle(value)
output_pin['dutycycle'] = value
log.info("Dutycycle of pin %d has been set to %d" % (pin, value))
def size(self):
size = 0
try:
size = os.path.getsize(self.filepath)
except Exception as e:
LOG.error("Get {0}'s size error".format(self.path))
LOG.exception(e)
return size
def load_tfidf_vect(dir: str):
LOG.info("Loading vectorizer...")
filename = "tfidf_vect.joblib"
if filename in listdir(dir):
with open(path.join(dir, filename), 'rb') as f:
vect = joblib.load(f)
LOG.info(f"Successfully loaded vectorizer {filename} from {dir}!")
return vect
else:
LOG.error(
f"Fitted vectorizer {filename} is not in expected {dir} directory! Please regenerate."
)
def load_model(model_dir: str, model_filename: str):
LOG.info("Loading...")
if model_filename in listdir(model_dir):
with open(path.join(model_dir, model_filename), 'rb') as f:
model = joblib.load(f)
LOG.info(
f"Successfully loaded model {model_filename} from {model_dir}!")
return model
else:
LOG.error(
f"Trained model {model_filename} is not in expected {model_dir} directory! Please retrain the model."
)
def crawler():
counter = 1
for url_ref in config.FULL_URLS:
resp = requests.get(url_ref)
if resp.status_code == 200:
_, name = get_name(url_ref)
# Ensure folder exists
folter_path = create_folder([config.LYRICS_FOLDER, name])
# Get all links
parsed_html = BeautifulSoup(resp.content, features='html.parser')
lyrics_links = parsed_html.select('.listalbum-item a')
LOG.info(f"Number of {name.upper()} songs: {len(lyrics_links)}")
lyric_paths = [extract_link(link) for link in lyrics_links]
for lyric_path in lyric_paths:
try:
writer, song_name = get_name(lyric_path)
if name != writer:
alt_folder = create_folder(
[config.LYRICS_FOLDER, writer])
lyrics_file = alt_folder.joinpath(song_name + '.txt')
file_found = lyrics_file.is_file()
else:
writer = name
lyrics_file = folter_path.joinpath(song_name + '.txt')
file_found = lyrics_file.is_file()
if not file_found:
# url = config.BASE_URL + lyric_path
text = get_lyrics(lyric_path).strip()
LOG.info("Downloading (" + str(counter).zfill(3) +
f") [{writer}]: {song_name}")
counter += 1
with open(lyrics_file, "w") as f:
f.write(text)
time.sleep(config.CRAWLER_WAIT +
config.CRAWLER_WAIT * random.random())
except IndexError:
LOG.error(
f"Access denied while scraping: {lyric_path} \n"
f"Try increasing the waiting time.\n"
f"Finishing the scrapping for the moment. Try to access on your browser to unblock access"
)
return
except Exception as err:
print(f"ERROR: {lyric_path}: {err}")
else:
LOG.warning(f"Unable to load: {url_ref}")
def md5sum(self):
ret = setting.UNKNOWN_SYMBOL
try:
with open(self.filepath, 'rb') as fp:
chunk_size = 1024 * hashlib.md5().block_size
chksum = hashlib.md5()
while True:
chunk = fp.read(chunk_size)
if chunk:
chksum.update(chunk)
else:
break
ret = chksum.hexdigest()
except Exception as e:
LOG.error("Get {0}'s md5sum error".format(self.path))
LOG.exception(e)
return ret
def set_output(self, pin, output=1):
"""
:param pin:
:param output:
"""
if pin not in self.output_pins:
log.error("Pin %s was not registered as an output" % pin)
return
output_pin = self.output_pins[pin]
if output_pin['pwm_started']:
output_pin['pwm'].stop()
GPIO.output(pin, output)
log.info("Output of pin %d has been set to %d" % (pin, output))
def character(self):
chunks = list()
size = self.size
ret = setting.UNKNOWN_SYMBOL
try:
with open(self.filepath, 'rb') as f:
for i in range(NUM_CHUNKS):
f.seek(int(size / NUM_CHUNKS) * i)
chunk = f.read(NUM_BLOCKS * hashlib.md5().block_size)
chunks.append(chunk)
character_chunk = bytes()
for chunk in chunks:
character_chunk = character_chunk + chunk
ret = hashlib.md5(character_chunk).hexdigest()
except Exception as e:
LOG.error("Get {0}'s character error".format(self.path))
LOG.exception(e)
return ret
def main():
global PROGRAM_CONFIG
# 加载配置
config = load_config()
if config:
PROGRAM_CONFIG = config
# 获取日志等级
if PROGRAM_CONFIG['log_level'] == 'DEBUG':
config_logging(logging.DEBUG, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'INFO':
config_logging(logging.debug, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'ERROR':
config_logging(logging.ERROR, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'WARN':
config_logging(logging.WARN, PROGRAM_CONFIG['log_name'])
else:
config_logging()
server = ProxyServer(PROGRAM_CONFIG['listen_port'])
# 用于检查客户端在线状态的线程
check_thread = Thread(target=check_alive_thread)
try:
print('启动检查客户端在线状态的线程')
check_thread.start()
server.start()
except Exception as err:
LOG.error(str(err))
finally:
print('写入配置信息')
save_config(PROGRAM_CONFIG)
print('退出程序')
def main():
global PROGRAM_CONFIG
# 加载配置
config = load_config()
if config:
PROGRAM_CONFIG = config
# 获取日志等级
if PROGRAM_CONFIG['log_level'] == 'DEBUG':
config_logging(logging.DEBUG, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'INFO':
config_logging(logging.debug, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'ERROR':
config_logging(logging.ERROR, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'WARN':
config_logging(logging.WARN, PROGRAM_CONFIG['log_name'])
else:
config_logging()
server = ProxyServer(PROGRAM_CONFIG['listen_port'])
# 用于检查客户端在线状态的线程
check_thread = Thread(target=check_alive_thread)
try:
print('start the thread of checking client online status')
check_thread.start()
server.start()
except Exception as err:
LOG.error(str(err))
finally:
print('write config info')
save_config(PROGRAM_CONFIG)
print('to exit progress')
def main():
global PROGRAM_CONFIG
# 加载配置
config = load_config()
if config:
PROGRAM_CONFIG = config
# 获取日志等级
if PROGRAM_CONFIG['log_level'] == 'DEBUG':
config_logging(logging.DEBUG, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'INFO':
config_logging(logging.debug, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'ERROR':
config_logging(logging.ERROR, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'WARN':
config_logging(logging.WARN, PROGRAM_CONFIG['log_name'])
else:
config_logging()
server = ProxyServer(PROGRAM_CONFIG['listen_port'])
# 用于检查客户端在线状态的线程
check_thread = Thread(target=check_alive_thread)
try:
print('启动检查客户端在线状态的线程')
check_thread.start()
server.start()
except Exception as err:
LOG.error(str(err))
finally:
print('写入配置信息')
save_config(PROGRAM_CONFIG)
print('退出程序')
