def do_epoch(model, data, vocab):
before_loss = 0
all_losses = []
current_loss = 0
for epoch in range(1, hparam.steps + 1):
label_name, label_tensor, line, line_tensor = randomTrainingPair(
data, vocab)
output, loss = train(model, label_tensor, line_tensor)
current_loss += loss
# Print epoch number, loss, name and guess
if epoch % hparam.every == 0:
ll = itos(line, vocab)
guess, guess_i = categoryFromOutput(vocab, output)
correct = '✓' if guess == label_name else '✗ (%s)' % label_name
logger.info(
'step:%d %.2f%% loss: %.4f %s / %s %s' %
(epoch, epoch / hparam.steps * 100, loss, ll, guess, correct))
if loss < before_loss or before_loss == 0:
ckpt = "model_%s_%.4f.pt" % (epoch, loss)
utils.save(logger, model.rnn, hparam.save_train, ckpt)
before_loss = loss
all_losses.append(current_loss / hparam.every)
current_loss = 0
f = open(os.path.join(hparam.save_train,
'%s_loss.json' % hparam.loss),
'w',
encoding='utf-8')
json.dump(all_losses, f, indent=0)
def no_integration(self):
logger.info(
f"{self.__class__.__name__} ({inspect.currentframe().f_code.co_name})"
)
return [
f":warning: <@{self.user_id}> You have not set up a zotero integration."
]
def main():
data, vocab = load_data()
model = Model(vocab.size, vocab.label["size"])
logger.info("model: [rnn_size: %s learning_rate: %s loss: %s ]" %
(hparam.n_hidden, hparam.learning_rate, hparam.loss))
do_epoch(model, data, vocab)
def _exportResults(self):
# export results to gui
self.parent.guiTabMoreInfo.v_ind_self_er.set(self.er_self_ind)
self.parent.guiTabMoreInfo.v_ind_self_legs.set(self.leg_self_ind)
self.parent.guiTabMoreInfo.v_ind_eff_er.set(
self.ereff[int(self.nr_of_legs / 4 - 1)] * 1e9)
self.parent.guiTabMoreInfo.v_ind_eff_legs.set(
self.legeff[int(self.nr_of_legs / 4 - 1)] * 1e9)
self.parent.guiTabSettings.v_er_seg_length.set(self.er_segment_length)
if self.coil_shape == self.parent.ELLIPSE and not self.shortaxis: # todo add export multiple C's @ ellipse
self.parent.guiTabResults.v_cap_res.set(
self.cap[int(self.nr_of_legs - 1)])
else:
self.parent.guiTabResults.v_cap_res.set(
self.cap[int(self.nr_of_legs / 4 - 1)])
result = f""" Results:
Result Capacitor: {self.parent.guiTabResults.v_cap_res.get()} pF
Result ER Segment Length: {self.parent.guiTabSettings.v_er_seg_length.get()} nH
Result Self Inductance ER: {self.parent.guiTabMoreInfo.v_ind_self_er.get()} nH
Result Self Inductance Legs: {self.parent.guiTabMoreInfo.v_ind_self_legs.get()} nH
Result Effective Inductance ER: {self.parent.guiTabMoreInfo.v_ind_eff_er.get()} nH
Result Effective Inductance Legs: {self.parent.guiTabMoreInfo.v_ind_eff_legs.get()} mm
"""
# logger.info(result.replace('\t', ''))
logger.info(result)
def make_data(vocab, path, flag):
label_lines = {}
logger.info("process %s data..." % flag)
logger.info("load vocab...size: [%s]" % vocab.size)
for id, filename in enumerate(
utils.findFiles(os.path.join(path, hparam.files))):
logger.info("proess file: %s" % filename)
category = filename.split('/')[-1].split('.')[0]
lines = utils.readLines(filename)
new_lines = []
for id in tqdm(range(len(lines))):
line = lines[id]
if flag == "train":
if len(line) < hparam.max_length:
chars_id = utils.stoi(line, vocab)
new_lines.append(chars_id)
label_lines[category] = new_lines
else:
logger.info("not proess: %s" % len(line))
else:
chars_id = utils.stoi(line, vocab)
new_lines.append(chars_id)
label_lines[category] = new_lines
logger.info("process: %s size: [%s]" % (category, len(new_lines)))
data = utils.data(label_lines)
utils.save(logger, data, hparam.save_data, "%s.pt" % flag)
def test_slack_client_connection(self):
logger.info("Testing slack client connectivity...")
try:
self.bolt.client.api_test().get("ok")
logger.info("Slack client OK.")
except SlackApiError as e:
logger.error("Unable to establish a slack web client connection!")
self.slack_api_error(e)
def startCalculation(self):
inputs_ = self.guiTabSettings.validateInputs()
if not inputs_:
return
logger.info("Calculation started with values:\n\t\t\t" + "\n\t\t\t".join("{}: {}".format(k, v) for k, v in inputs_.items()))
self.calcCapacitance.calculate()
self.guiTabResults.drawCapacitors()
self.guiTabResults.drawGraph()
self.tab_control.select(1) # At end switch tab to results
def who_is_bot(self):
logger.info("Establishing bot presence...")
try:
bot_auth = self.bolt.client.auth_test(token=self.bot_token)
self.bot_user = self.bolt.client.users_info(
user=bot_auth.get("user_id")).get("user")
logger.info("Bot OK")
logger.debug(self.bot_user)
except SlackApiError as e:
self.slack_api_error(e)
def bot_says_bye(self):
logger.info("Announce bot exit from slack...")
if self.bot_channels is not None:
for channel in self.bot_channels:
try:
self.bolt.client.chat_postMessage(
channel=channel.get('id'),
text=
f"Goodbye #{channel.get('name')}! I am shutting down.")
except SlackApiError as e:
self.slack_api_error(e)
def reorg_duplicate():
while True:
deleted = 0
reorgtime = int(time())
for n in list(duplicatemsg):
if duplicatemsg[n] < reorgtime:
duplicatemsg.pop(n)
deleted += 1
logger.info("Reorg duplicate Messages. Deleting {}/{}".format(
deleted, len(duplicatemsg.keys())))
sleep(10)
def bot_was_blocked(connection, botid, chat_id):
cursor = connection.cursor()
logger.info("Bot was blocked. Stopping ChatId {}".format(chat_id))
cursor.execute("delete from user where botid = '%s' and chatid = '%s'" %
(botid, chat_id))
cursor.execute("select count(*) from user where chatid = '%s'" % (chat_id))
result = cursor.fetchone()
if result[0] == 0:
cursor.execute(
"insert ignore into userstop values ('%s', CURRENT_TIMESTAMP)" %
(chat_id))
def get_slack_info(self):
logger.info("Get info about slack team, channels, etc...")
if self.bot_user is not None:
team_id = self.bot_user.get("team_id", None)
try:
self.team = self.bolt.client.team_info(
team=team_id).get("team")
self.channels = self.bolt.client.conversations_list().get(
'channels')
self.users = self.bolt.client.users_list().get('members')
except SlackApiError as e:
self.slack_api_error(e)
def eavl():
for id,filename in enumerate(utils.findFiles(os.path.join(hparam.eval, hparam.files))):
label = filename.split('/')[-1].split('.')[0]
lines=utils.readLines(filename)
logger.info("eval %s"%filename)
err=[]
for l in tqdm(lines):
out=predict(l,label)
if out!=label:
err.append([l,out])
#print(out)
acc=(1-(len(err)/len(lines)))*100
logger.info("accuracy: %s%%"%acc)
def make_vocab():
chars = set()
ntoi, iton = {}, {}
for id, filename in enumerate(
utils.findFiles(os.path.join(hparam.train, hparam.files))):
category = filename.split('/')[-1].split('.')[0]
ntoi[category] = utils.id2tensor(id)
iton[id] = category
logger.info("proess: %s" % filename)
lines = utils.readLines(filename)
label = {"size": len(iton), "ntoi": ntoi, "iton": iton}
for line in tqdm(range(len(lines))):
char = set(list(lines[line]))
chars = chars.union(char)
logger.info("label size: [%s]" % len(iton))
return chars, label
def log_message(bot_instance, message):
logger.info("Message from ID:{}:{}:{}".format(message.from_user.id,
message.from_user.username,
message.text))
bot_instance.userok = False
if (message.content_type == 'text'
and message.text[0] == '/') or message.content_type == 'location':
try:
connection.ping(reconnect=True)
if user_ok(bot, connection, allowmode, tggroup, message.chat.id):
bot_instance.userok = True
else:
sendtelegram(message.chat.id, msg_loc["28"])
except:
sendtelegram(message.chat.id, msg_loc["6"])
def __init__(self):
logger.info("Initializing slack...")
self.bolt = App(
signing_secret=self.signing_secret,
token=self.bot_token,
)
self.test_slack_client_connection()
self.who_is_bot()
self.get_slack_info()
self.bot_says_hi()
self.bolt.use(self.global_middleware_listener)
self.load_plugins()
try:
self.start()
except KeyboardInterrupt:
self.close()
def load_plugins(self):
logger.info("Loading slack plugins...")
plugin_files = glob.glob(self.plugins_dir + os.sep + "**" + os.sep +
"[!__]*.py",
recursive=True)
for plugin_path in plugin_files:
relative_path = os.path.relpath(plugin_path, os.getcwd())
import_path = relative_path.replace(".py", "").replace(os.sep, ".")
for event_type in self.event_types:
plugin = Plugin(event_type,
importlib.import_module(import_path))
if plugin.callback is not None:
if hasattr(self.bolt, plugin.type):
event_handler: callable = getattr(
self.bolt, plugin.type)
event_handler(plugin.keyword)(plugin.callback)
self.plugins.append(plugin)
def bot_says_hi(self):
logger.info("Announce bot presence in slack...")
if self.channels is not None:
for channel in self.channels:
try:
channel_members = self.bolt.client.conversations_members(
channel=channel.get('id')).get('members')
if self.bot_user is not None and self.bot_user.get(
'id') in channel_members:
self.bot_channels.append(channel)
self.bolt.client.chat_postMessage(
channel=channel.get('id'),
text=
f"Hello #{channel.get('name')}! I am waking up.")
except SlackApiError as e:
self.slack_api_error(e)
def load_vocab():
if os.path.exists(hparam.vocab):
vocab = torch.load(hparam.vocab)
else:
logger.info("make vocab")
vocab, label = make_vocab()
if not os.path.exists(hparam.save_data):
os.mkdir(hparam.save_data)
vocab_path = os.path.join(hparam.save_data, "vocab.txt")
with codecs.open(vocab_path, 'w', encoding="utf-8") as f:
f.write(utils.UNK + "\n")
for char in vocab:
f.write(char + "\n")
vocab = utils.readLines(vocab_path)
logger.info("load vocab...size: [%s]" % len(vocab))
vocab = utils.vocabulary(label, vocab)
utils.save(logger, vocab, hparam.save_data, "vocab.pt")
return vocab
def __init__(self, ack: Ack, client: WebClient, context: BoltContext,
logger: logger, request: BoltRequest, view: View):
logger.info("Updating Zotero config...")
super().__init__(context.get('user_id'))
state: dict = view.get('state')
form_fields: dict = state['values']
sql_fields: dict = {'user_id': context.get('user_id')}
for i in form_fields:
form_field: dict = form_fields[i]
for t in form_field:
field_name: str = t
field = form_field[t]
field_type = field.get('type')
field_value: dict = field.get('value')
if field_type == 'static_select':
field_value = field.get('selected_option').get('value')
if field_value is None and hasattr(self, field_name):
field_value = getattr(self, field_name)
if field_name == 'zotero_api_key':
field_value = self.db.cipher.encrypt(
field_value.encode('utf-8'))
sql_fields[field_name] = field_value
placeholders: list = []
for i in range(0, len(sql_fields)):
placeholders.append("?")
placeholder: str = ','.join(placeholders)
field_names: str = ', '.join(sql_fields.keys())
insert_query = f"INSERT OR REPLACE INTO {self.table_name} ({field_names}) VALUES({placeholder})"
self.db.cursor.execute(insert_query, list(sql_fields.values()))
self.db.connection.commit()
ack()
def load_data():
logger.info("load data")
path = os.path.join(hparam.save_data, "vocab.pt")
vocab = torch.load(path)
logger.info("load vocab: %s size: %s" % (path, vocab.size))
path = os.path.join(hparam.save_data, "train.pt")
data = torch.load(path)
logger.info("load vocab: %s " % path)
return data, vocab
def close(self):
logger.info("Disconnecting socket mode...")
self.bot_says_bye()
self.socket_mode.disconnect()
def start(self):
logger.info("Starting slack socket mode...")
self.socket_mode = SocketModeHandler(self.bolt, self.app_token)
self.socket_mode.start()
def __init__(self, ack: Ack, client: WebClient, context: BoltContext,
logger: logger, payload: dict, request: BoltRequest):
logger.info("Init Zotero UI...")
super().__init__(context.get('user_id'))
container = request.body.get('container', None)
view: dict = request.body.get(container.get('type'))
title: dict = view.get('title')
title.update(text="Zotero Integration")
close_button = view.get('close')
close_button.update(text="Go Back")
blocks: list = list()
intro = {
"type": "header",
"text": {
"type":
"plain_text",
"text":
"The area is used to configure your slack / zotero integration."
}
}
link = {
"type": "section",
"text": {
"type":
"mrkdwn",
"text":
"You can create and configure Zotero API settings at <https://www.zotero.org/settings/keys|Zotero API Settings>"
}
}
library_id = {
"type": "input",
"element": {
"type": "plain_text_input",
"action_id": "zotero_library_id",
# "placeholder": {
# "type": "plain_text",
# "emoji": True
# },
},
"label": {
"type": "plain_text",
"text": ":id: Library ID",
"emoji": True
}
}
library_type_options = [{
"text": {
"type": "plain_text",
"text": ":bust_in_silhouette: User",
"emoji": True
},
"value": "user",
}, {
"text": {
"type": "plain_text",
"text": ":busts_in_silhouette: Group",
"emoji": True
},
"value": "group",
}]
library_type = {
"type": "input",
"element": {
"type": "static_select",
"placeholder": {
"type": "plain_text",
"text": "Library Type",
"emoji": True
},
"options": library_type_options,
"action_id": "zotero_library_type"
},
"label": {
"type": "plain_text",
"text": ":books: Library Type",
"emoji": True
}
}
if self.zotero_library_type is not None:
current_library_type = next(
(x for x in library_type_options
if x.get('value') == self.zotero_library_type),
library_type_options[0])
library_type['element']["initial_option"] = current_library_type
if self.zotero_library_id is not None:
library_id['element']["initial_value"] = self.zotero_library_id
api_key = {
"type": "input",
"element": {
"type": "plain_text_input",
"action_id": "zotero_api_key",
"placeholder": {
"type": "plain_text",
"text": "HIDDEN FOR PRIVACY",
"emoji": True
},
},
"label": {
"type": "plain_text",
"text": ":key: Api Key",
"emoji": True
},
"optional": True
}
blocks.extend([intro, link, library_type, library_id, api_key])
client.views_push(trigger_id=request.body.get('trigger_id'),
view={
"type": view.get('type'),
"title": title,
"close": close_button,
"callback_id": self.keyword,
"submit": {
"type": "plain_text",
"text": "Submit",
"emoji": True,
},
"blocks": blocks
})
ack()
##################
# parsing arguments
parser = argparse.ArgumentParser()
parser.add_argument("-d", dest='dry', help="dry run", action='store_true')
parser.add_argument("-c",
dest='configfile',
help="configfile",
type=str,
default="config.ini")
args = parser.parse_args()
dryrun = 0
try:
if args.dry:
dryrun = 1
logger.info("Dryrun")
except:
pass
# read inifile
#
try:
config = ConfigObj(args.configfile)
db = config['dbname']
dbhost = config['dbhost']
dbport = config.get('dbport', '3306')
dbuser = config['dbuser']
dbpassword = config['dbpassword']
reorgdays = int((config.get('reorgdays', '180')))
except:
logger.error("Inifile not given or missing parameter")
scroll_y.pack(side="left", expand=True, fill="y")
text.configure(yscrollcommand=scroll_y.set, width=80, height=21)
text.configure(state='disabled') # make readonly
text.pack()
self._centerWindow(top)
top.deiconify()
if __name__ == "__main__":
root = tk.Tk()
root.withdraw()
default_font = font.nametofont("TkDefaultFont") # set default font
default_font.configure(family='freemono', size=11)
myfont_bold = default_font.copy() # bold font
myfont_bold.configure(weight="bold")
myfont_small_bold = default_font.copy() # small font
myfont_small_bold.configure(weight="bold", size=9)
myfont_small = default_font.copy() # small font
myfont_small.configure(size=8)
mygui = MainApplication(root)
root.deiconify()
logger.info("Program start successfully")
root.mainloop()
logger.info("proess file: %s" % filename)
category = filename.split('/')[-1].split('.')[0]
lines = utils.readLines(filename)
new_lines = []
for id in tqdm(range(len(lines))):
line = lines[id]
if flag == "train":
if len(line) < hparam.max_length:
chars_id = utils.stoi(line, vocab)
new_lines.append(chars_id)
label_lines[category] = new_lines
else:
logger.info("not proess: %s" % len(line))
else:
chars_id = utils.stoi(line, vocab)
new_lines.append(chars_id)
label_lines[category] = new_lines
logger.info("process: %s size: [%s]" % (category, len(new_lines)))
data = utils.data(label_lines)
utils.save(logger, data, hparam.save_data, "%s.pt" % flag)
if __name__ == '__main__':
logger.info("make data start")
vocab = load_vocab()
make_data(vocab, hparam.train, "train")
make_data(vocab, hparam.eval, "test")
logger.info("make data end")
def sendmonster(bot, config, connection, pkmn_loc, geoprovider, geofences,
allowmode):
cursor = connection.cursor()
botid = bot.get_me().id
venuetitle = config['venuetitle']
venuemsg = str(config['venuemsg'])
ivmsg = str(config['ivmsg'])
tggroup = config.get('tggroup', '')
while True:
message = pkmn_queue.get()
pkmn_id = (message['pokemon_id'])
# set monster info
#
try:
pkmn_name = pkmn_loc[str(pkmn_id)]["name"]
except:
pkmn_name = "{} NOTFOUND".format(str(pkmn_id))
logger.warning("{} not found in monster.json".format(str(pkmn_id)))
# check if in geofence
#
if geofences:
inside = False
for geofence in geofences.keys():
mypath = mplPath.Path(geofences[geofence])
if mypath.contains_points(
[tuple((message['latitude'], message['longitude']))]):
inside = True
break
if not inside:
logger.info("{}({}) outside geofence".format(
pkmn_name, pkmn_id))
continue
pkmn_despawn = datetime.datetime.fromtimestamp(
int(message['disappear_time'])).strftime('%H:%M:%S')
logger.info("{}({}) until {} @ {},{}".format(pkmn_name, pkmn_id,
pkmn_despawn,
message['latitude'],
message['longitude']))
# calculate IV if encounting
#
try:
pkmn_iv = float((
(message['individual_attack'] + message['individual_defense'] +
message['individual_stamina']) * 100 / 45)).__round__(2)
logger.info(
"IV:{:.2f} CP:{:4d} ATT:{:2d} DEF:{:2d} STA:{:2d}".format(
pkmn_iv, message['cp'], message['individual_attack'],
message['individual_defense'],
message['individual_stamina']))
except:
pkmn_iv = "None"
message['individual_attack'] = "??"
message['individual_defense'] = "??"
message['individual_stamina'] = "??"
message['cp'] = "??"
message['pokemon_level'] = "??"
if not 'boosted_weather' in message:
message['boosted_weather'] = 0
# add missing data to message
message['iv'] = pkmn_iv
message['name'] = pkmn_name
message['despawn'] = pkmn_despawn
message['geo_ok'] = False
# get all chatids for the monster
# no blocked chat id
#
connection.ping(reconnect=True)
cursor.execute(
"select chatid,iv,level from userassign where pkmnid = '%s' and \
chatid in (select chatid from user where botid = '%s')"
% (pkmn_id, botid))
result_pkmn = cursor.fetchall()
if len(result_pkmn) > 0:
# send monster message to all
#
for chat_id, iv, level in result_pkmn:
if not user_ok(bot, connection, allowmode, tggroup, chat_id):
logger.info(
"ChatID {} not allowed for this Bot but has assignments"
.format(chat_id))
continue
# get the user details
cursor.execute(
"select lat,lon,dist from user where chatid = '%s'" %
(chat_id))
latlon = cursor.fetchone()
lat = float(latlon[0])
lon = float(latlon[1])
dist = float(latlon[2])
# check distance
dist_ok = True
if dist > 0:
pkmn_dist = distance.distance(
(lat, lon),
(message['latitude'], message['longitude'])).kilometers
if pkmn_dist > dist:
dist_ok = False
logger.info(
"{} long distance: dist: {} pkmn_dist: {}".format(
chat_id, dist, pkmn_dist))
# check level
if message['pokemon_level'] == '??':
level_ok = True
elif int(message['pokemon_level']) >= level:
level_ok = True
else:
level_ok = False
if not (dist_ok and level_ok):
continue
if message['iv'] == "None":
if iv == -1:
# get geo only if not set
if not message['geo_ok']:
geo = geo_reverse(geoprovider, message['latitude'],
message['longitude'])
message['road'] = "{} {}".format(geo[0], geo[1])
message['postcode'] = geo[2]
message['town'] = geo[3]
message['geo_ok'] = True
venuetitle1 = textsub(venuetitle, message)
venuemsg1 = textsub(venuemsg, message)
try:
bot.send_venue(chat_id, message['latitude'],
message['longitude'], venuetitle1,
venuemsg1)
logger.info(
"Send Telegram message to {} Monster {}({})".
format(chat_id, pkmn_name, pkmn_id))
except telebot.apihelper.ApiTelegramException as e:
if e.result_json['error_code'] == 403:
bot_was_blocked(connection, botid, chat_id)
except telebot.apihelper.ApiHTTPException as e:
logger.error("Connection Error")
logger.error("HTTP Error Code: {}".format(
e.result))
except:
logger.error(
"ERROR IN SENDING TELEGRAM MESSAGE TO {}".
format(chat_id))
logger.error("Error: {}".format(sys.exc_info()[0]))
else:
logger.info(
"No message send to {}. SearchIV set but Monster {}({}) not encountered"
.format(chat_id, pkmn_name, pkmn_id))
elif message['iv'] >= iv:
try:
# get geo only if not set
if not message['geo_ok']:
geo = geo_reverse(geoprovider, message['latitude'],
message['longitude'])
message['road'] = "{} {}".format(geo[0], geo[1])
message['postcode'] = geo[2]
message['town'] = geo[3]
message['geo_ok'] = True
ivmsg1 = textsub(ivmsg, message)
bot.send_message(chat_id, ivmsg1)
bot.send_location(chat_id, message['latitude'],
message['longitude'])
logger.info(
"Send Telegram IV message to {} Monster {}({})".
format(chat_id, pkmn_name, pkmn_id))
except telebot.apihelper.ApiTelegramException as e:
if e.result_json['error_code'] == 403:
bot_was_blocked(connection, botid, chat_id)
except telebot.apihelper.ApiHTTPException as e:
logger.error("Connection Error")
logger.error("HTTP Error Code: {}".format(e.result))
except:
logger.error(
"ERROR IN SENDING TELEGRAM MESSAGE TO {}".format(
chat_id))
logger.error("Error: {}".format(sys.exc_info()[0]))
else:
logger.info(
"No message send to {}. SearchIV to low for Monster {}({})"
.format(chat_id, pkmn_name, pkmn_id))
try:
cursor.execute("insert into bot (botid, botname) values ('%s','%s')" %
(botid, botname))
except:
pass
except:
logger.error("Error in Telegram. Can not find Botname and ID")
raise
##################
# set geoprovider
if nominatim:
geoprovider = Nominatim(user_agent='Monsterbot',
scheme=nominatim_scheme,
domain=nominatim_url)
logger.info("Using Nominatim for geolocation")
elif gmaps:
geoprovider = GoogleV3(api_key=gmaps_apikey)
logger.info("Using Google for geolocation")
else:
geoprovider = False
logger.info("Geolocation is disabled")
##################
# set geofence
geofence = False
if geofencefile:
try:
with open(geofencefile, "r") as fencefile:
geofence = {}
for i in fencefile:
def start_webhook():
logger.info("MonsterGBot {} serving at port {}".format(botname, whport))
httpd.serve_forever()
