def calculate_score(date_a, date_b):
cluster_a = storage.read(date_a)
cluster_b = storage.read(date_b)
inverse_cluster_a = invert_cluster(cluster_a)
inverse_cluster_b = invert_cluster(cluster_b)
logging.info("calculate score for dates %s - %s" % (date_a, date_b))
for cluster_id in inverse_cluster_a.keys():
systems_a = frozenset(inverse_cluster_a[cluster_id])
systems_b = frozenset(inverse_cluster_b[cluster_id])
# So now if you have 100 system ids in first and 160 in second
# (100 are the same and we have 60 new), you'll count the score as
# 100 / (100+160/2) = 100 / 130 = 0.76.
# While it should be 1, if the 100 systems were clustered the same.
# We can't say anything about the 60 new systems (unless we do other analysis of them),
# so those needs to be ignored.
# TODO: only look at systems present in both runs
number_ids_in_both = len(systems_a.intersection(systems_b))
median_number_ids_in_cluster = (len(systems_a) + len(systems_b)) / 2.0
score = number_ids_in_both / median_number_ids_in_cluster
mlflow.log_metric(("cluster_stability_%s" % cluster_id), score)
logging.info("cluster_id %i - stability score %f" % (cluster_id, score))
def alarm_save():
fields = ['id_', 'name', 'volume', 'stream', 'action']
data = dict([ (k, request.POST.get(k)) for k in fields ])
data['volume'] = int(data['volume'])
data['type'] = 'radio'
try:
date = request.POST.get('date')
hour = request.POST.get('hour')
data['at'] = time.mktime(time.strptime("%s %s" % (date, hour),
"%Y-%m-%d %H:%M:%S"))
dt = datetime.datetime.fromtimestamp(data['at'])
data['date'] = dt.strftime('%Y-%m-%d')
data['hour'] = dt.strftime('%H:%M:%S')
except:
return "Problem with the date... Chek it, please"
if data['id_']:
data['id_'] = int(data['id_'])
alarms_data = storage.replace('alarms', data)
storage.save_table('alarms', alarms_data)
else:
# TODO: All this logic of getting a new ID for the given table should
# be handled by the storage lib
stored = storage.read()
ids = map(lambda x: x['id_'], stored['alarms'])
data['id_'] = max(ids)+1 if ids else 1
stored['alarms'].append(data)
storage.save(stored)
alarms.set_alarms(storage.read('alarms'))
redirect('/alarm/edit/%s' % data['id_'])
def radio(successfully_saved=False):
if not helpers.player.is_installed():
return template("radio-instructions")
radios = sorted(storage.read('radio').items())
helpers.current_tab("radio")
return template("radio", radios=radios, successfully_saved=successfully_saved)
async def logchannel(self, ctx, channel: discord.TextChannel = None):
guildid = str(ctx.guild.id)
setting = self.getmodsetting(guildid, 'logchannel')
if not channel:
channel = ctx.channel
try:
logchannel = self.bot.get_channel(int(setting))
except:
logchannel = None
if logchannel == channel:
await ctx.send(f"The log channel is already {channel.mention}!")
return
if not channel.permissions_for(
ctx.guild.me).read_messages or not channel.permissions_for(
ctx.guild.me).send_messages:
await ctx.send("I can't read and/or send messages in that channel."
)
return
settings = storage.read("mod", key=[guildid])
settings['logchannel'] = channel.id
storage.write("mod", settings, key=guildid)
if channel:
await ctx.send("{} has been made the log channel".format(
channel.name))
else:
await ctx.send("The log channel has been removed")
def getmodsetting(self, guildid, setting):
guildid = str(guildid)
settings = storage.read("mod", key=[guildid], default={})
if setting not in settings:
return None
else:
return settings[setting]
async def on_message(self, message):
if isinstance(message.channel,discord.DMChannel):
return
if message.author.bot:
return
guildid = str(message.guild.id)
guildprefix = storage.read("prefixes",key=[str(message.guild.id)],default=self.bot.default_prefix)
prefixes = [f"{guildprefix} ",guildprefix]
commands = await self.checkGuildDict(guildid)
if not commands:
return
prefix = None
for aPrefix in prefixes:
if message.content.startswith(aPrefix):
prefix = aPrefix
if not prefix:
return
text = message.content[len(prefix)::]
strnumber = 0
cmd = ''
args = ''
text = text.split()
for word in text:
if strnumber == 0:
strnumber = 1
cmd = word.lower()
else:
args += word + " "
args = args[0:len(args)-1]
if cmd not in commands:
return
text = commands[cmd]
text = text.replace("$A",args)
await message.channel.send(text.format(message))
def getuserpersists(self, guildid, userid):
guildid = str(guildid)
userid = str(userid)
persists = storage.read("persists", path="cogs/mod/", key=[guildid])
if userid not in persists:
return None
else:
return persists[userid]
def getMiniMap(self, name):
#self.getMiniMapFromWeb(name)
# the manager likes to use actual filenames, but
# most other stuff will want to simply use the
# name of the map with out the file extension
name = self.getNameByShortName(name)
storkey = 'mapManager.minimap.%s' % name
if storage.exist(storkey):
qimg = QtGui.QImage(storage.read(storkey), 1024, 1024, QtGui.QImage.Format_RGB16)
return qimg
# okay look and see if we have the map file locally
if name not in self.maps:
print('getMiniMap-name-not-found', name)
return None
dstdir = os.path.abspath('.\\7ztmp')
archive = '%s%s' % (self.mapdir, name)
if os.path.exists(dstdir):
self.rmdir(dstdir)
os.mkdir(dstdir)
if archive[-4:] == '.sd7':
subprocess.call(['7zr.exe', '-ir!*.smf', '-y', 'e', '-o%s' % dstdir, archive])
if archive[-4:] == '.sdz':
zf = zipfile.ZipFile(archive)
for node in zf.namelist():
ext = node[-4:]
# skipping tile file... maybe little faster.. (if it goes missing look here for sdz files)
if ext != '.smd' or ext != '.smf':
continue
data = zf.read(node)
node = node.replace('/', '_').replace('\\', '_')
fd = open('%s\\%s' % (dstdir, node), 'wb')
fd.write(data)
fd.close()
nodes = os.listdir(dstdir)
'''
.smd is plain text, lots of interesting information that the user
could be interested in especially hardcore players, also seems
very useful info too about start positions!
.smf is the actual map file with height data and minimap data
.smt is the tile file.. i dont know anything about this one yet
'''
for node in nodes:
ext = node[-4:]
if ext == '.smd':
pass
if ext == '.smf':
qimg, qimgraw = self.readSMF('%s\\%s' % (dstdir, node))
storage.write('mapManager.minimap.%s' % name, qimgraw, useDisk = True)
return qimg
if ext == '.smt':
pass
print('game archive did not contain SMF file!')
self.rmdir('%s' % dstdir)
return None
async def deluserpersist(self, ctx, user, role):
guildid = str(ctx.guild.id)
userid = str(user.id)
roleid = str(role.id)
persists = storage.read("persists", path="cogs/mod/", key=[guildid])
if roleid not in persists[userid]:
return f"{user} doesn't have {role} on their persist list!"
persists[userid].remove(roleid)
storage.write("persists", persists, path="cogs/mod/", key=guildid)
return f"{user} will no longer be given {role} when they rejoin."
async def setprefix(self, ctx, prefix="-"):
guildid = str(ctx.guild.id)
oldPrefix = storage.read("prefixes", key=[guildid])
if prefix == oldPrefix:
await ctx.send("That's already the prefix!")
return
if len(prefix) > 14:
await ctx.send("That's too long! It must be under 15 characters.")
return
storage.write("prefixes", prefix, key=guildid)
await ctx.send(f"Prefix has been set to ``{prefix}``!")
async def adduserpersist(self, ctx, user, role):
guildid = str(ctx.guild.id)
userid = str(user.id)
roleid = str(role.id)
persists = storage.read("persists", path="cogs/mod/", key=[guildid])
if userid not in persists:
persists[userid] = []
if roleid in persists[userid]:
return f"{user} already has {role} on their persist list!"
persists[userid].append(roleid)
storage.write("persists", persists, path="cogs/mod/", key=guildid)
return f"{user} will now be given {role} when they rejoin (until you remove it from them with `{prefix(self,ctx.message)}role remove '{role.name}' '{user.name}''`)"
async def modset_invitecensoring(self, ctx, enabled: bool = True):
guildid = str(ctx.guild.id)
settings = storage.read("mod", key=[guildid])
if "invite" not in settings:
settings["invite"] = True
storage.write("mod", settings, key=guildid)
if enabled == settings["invite"]:
await ctx.send("That's already the setting!")
return
settings["invite"] = enabled
storage.write("mod", settings, key=guildid)
await ctx.send(f"Invite Censoring has been set to {enabled}!")
async def load(self, ctx, *cogs):
"""Loads a cog."""
for cog in cogs:
try:
self.bot.load_extension(f"cogs.{cog}.{cog}")
except:
await ctx.send("Failed.")
raise
else:
extensions = storage.read("cogs")
extensions.append("cogs.{0}.{0}".format(cog))
storage.write("cogs", extensions)
await ctx.send(f"Cog {cog} loaded.")
self.bot.currently_loaded_cogs.append(cog)
def test_top_data(self):
change_storage(test_data)
authors, articles = read()
top_authors, top_articles = get_top_authors_articles(authors, articles)
self.assertEqual(len(top_articles), 5)
self.assertEqual(len(top_authors), 5)
author_names = [author['name'] for author in top_authors]
article_titles = [article['title'] for article in top_articles]
self.assertEqual(author_names, ['author1', 'author2',
'author3', 'author4', 'author5'])
self.assertEqual(article_titles, ['article7', 'article6',
'article5', 'article4', 'article3'])
def execute_command(class_, action, extra_params):
if config.COMMAND_EXECUTION == False:
return "The command execution is NOT available."
command = filter(lambda x: x['class_'] == class_ and x['action'] == action,
storage.read('commands'))
if not command:
return "Command not found"
command = command[0]
for key, value in extra_params.items():
command['command'] = command['command'].replace("$%s" % key, value)
subprocess.call(command['command'], shell=True)
return "Executing: %s" % command
async def unload(self, ctx, *cogs):
"""Unloads a cog."""
for cog in cogs:
if cog == 'owner':
await ctx.send("Cannot unload owner.")
return
try:
self.bot.unload_extension("cogs.{0}.{0}".format(cog))
except:
await ctx.send("Cog not loaded but removing it.")
pass
extensions = storage.read("cogs")
extensions.remove("cogs.{0}.{0}".format(cog))
storage.write("cogs", extensions)
await ctx.send("Cog {} unloaded.".format(cog))
self.bot.currently_loaded_cogs.remove(cog)
def execute_command(class_, action, extra_params):
if config.COMMAND_EXECUTION == False:
return "The command execution is NOT available."
command = filter(lambda x: x['class_'] == class_ and
x['action'] == action,
storage.read('commands'))
if not command:
return "Command not found"
command = command[0]
for key, value in extra_params.items():
command['command'] = command['command'].replace("$%s" % key, value)
subprocess.call(command['command'], shell=True)
return "Executing: %s" % command
def command_save():
id_ = request.POST.get('id')
class_ = request.POST.get('class')
action = request.POST.get('action')
command = request.POST.get('command', '')
if not class_ or not action:
return "Invalid data. CLASS and ACTION are required fields."
if id_:
new_command = {"id_": int(id_), "class_": class_, "action": action, "command": command}
commands = storage.replace('commands', new_command)
storage.save_table('commands', commands)
else:
data = storage.read()
ids = map(lambda x: x['id_'], data['commands'])
id_ = max(ids)+1 if ids else 1
new_command = {"id_": int(id_), "class_": class_, "action": action, "command": command}
data['commands'].append(new_command)
storage.save(data)
redirect("/command/edit/%s" % id_)
def alarm_delete(id_):
storage.delete('alarms', int(id_))
alarms.set_alarms(storage.read('alarms'))
return "ok"
def alarm_edit(id=None):
id = "" if id == "new" else int(id)
alarm = helpers.Dummy(storage.get_by_id('alarms', id))
radios = sorted(storage.read('radio').items())
return template('alarms-radio-edit', radios=radios, alarm=alarm)
def alarm_radio():
alarms = map(helpers.Dummy, storage.read('alarms'))
return template('alarms-radio', alarms=alarms)
def load_config():
configuration = storage.read("config")
for k, v in default_config.items():
# http://oi45.tinypic.com/nnqrl1.jpg
setattr(sys.modules[__name__], k, configuration.get(k, v))
def save_configuration(conf):
configuration = storage.read()
configuration["config"].update(conf)
storage.save(configuration)
# After saving the new configuration, we must load it again
load_config()
def load_config():
configuration = storage.read('config')
for k, v in default_config.items():
# http://oi45.tinypic.com/nnqrl1.jpg
setattr(sys.modules[__name__], k, configuration.get(k, v))
def api_storyline(url):
server.set_code(200)
server.set_header("Content-Type", "application/json")
return storage.read("storyline.json")
# Validate number of arguments
if len(sys.argv) < 5:
print(usage()) # Invalid number of arguments
exit(-1)
# Encryption or decryption
encrypt = True if sys.argv[1][0] == 'e' else False
# Get the input
if '-i' in sys.argv: # Get message from file
if not storage.file_exists(sys.argv[5]): # Check if file exists
print('File not found.')
exit(-1)
message = storage.read(sys.argv[5])
else: # Get message from arguments
message = ' '.join(sys.argv[4:])
# Get the keys
key_caesar = eval(
sys.argv[2]) # In the Caesar cipher the key is an integer number
key_vigenere = sys.argv[3] # In Vigenere cipher the key is a word
# Apply ciphers
caesar = cipher.caesar(message.lower(), key_caesar, encrypt)
vigenere = cipher.vigenere(message.lower(), key_vigenere.lower(), encrypt)
# Output results
results = 'CAESAR CIPHER:\n' + caesar + '\n\nVIGENERE CIPHER:\n' + vigenere
if '-o' in sys.argv: # Write result to file
def getPermissions():
return storage.read("permissions",path="cogs/permissions/")
def getPermissionsAndCheckForGuildAndCommand(ctx,command): # what the f**k
perms = storage.read("permissions",path="cogs/permissions/",key=[str(ctx.guild.id)])
if command not in permissions["commands"]:
permissions["commands"][command] = {"*": ["*"]}
storage.write("permissions",permissions,path="cogs/permissions/",key=)
return permissions
def commands():
helpers.current_tab("commands")
rows = map(helpers.Dummy, storage.read('commands'))
return template('commands', rows=rows)
import logging
logging.basicConfig(level=logging.INFO)
import json
import os
import pickle
from concurrent.futures import ProcessPoolExecutor
from flask import Flask
import clustering
import storage
import sync
app = Flask(__name__)
CLUSTERS = {date: storage.read(date) for date in set(storage.available())}
@app.route("/<date>")
def index(date):
if date in CLUSTERS:
return "<pre>%s</pre>" % CLUSTERS[date].to_string()
else:
return json.dumps(list(storage.available()))
@app.route("/sync")
def sync_endpoint():
with ProcessPoolExecutor(max_workers=1) as executor:
f = executor.submit(sync.sync)
logging.info("Running sync job %s", f)
return "OK"
elif req.params.get('username'):
username = req.params.get('username')
password = req.params.get('password')
if check_login(username, password):
# Login successfull! Create the session and set the cookie
uuid = helpers.session.create()
bottle.response.set_cookie('session', uuid)
# It would be nice to use a redirect here but since I'm
# setting a cookie it is not the best idea ever
return index()
else:
return template('login') # Bad login!
# If the IP is whitelisted, let them in
elif helpers.in_whitelist(config.AUTH_WHITELIST, bottle.request.remote_addr):
return callback(*args, **kwargs)
# You are not authenticated yet or don't have a valid session
return template('login')
return wrapper
if __name__ == '__main__':
import sys
debug = '--debug' in sys.argv
alarms.set_alarms(storage.read('alarms'))
bottle.debug(debug)
bottle.install(authentication_plugin)
bottle.run(host='0.0.0.0', port=config.PORT, reloader=debug)
def save_configuration(conf):
configuration = storage.read()
configuration['config'].update(conf)
storage.save(configuration)
# After saving the new configuration, we must load it again
load_config()
async def checkGuildDict(self,guildid):
if not storage.read("commands",path="cogs/custom/",key=[guildid]):
storage.write("commands",{},path="cogs/custom/",key=guildid)
return storage.read("commands",path="cogs/custom/",key=[guildid])