def __init__(self,
teams_file="settings/team_weights.txt",
words_file="settings/game_words.txt",
full_log=None,
game_log=None):
self.full_log = full_log
self.game_log = game_log
if self.full_log is not None:
self.full_log.info("SpyMaster initialising...")
if self.game_log is not None:
self.game_log.info("SpyMaster initialising...")
# spymaster stuff
self.settings = load_settings(sett_file="settings/spymaster_setts.txt",
default_dict={
"max_top_hints": 10,
"max_levels": 2,
"model_name": "glove-wiki-100",
"use_annoy_indexer": True,
"vocab_limit": 0,
"game_hint_naive_method": False
})
self.strategy = load_settings(
sett_file="settings/spymaster_setts.txt",
default_dict={
"level_{}_limit".format(str(x + 1)): 0
for x in range(self.settings["max_levels"])
})
# game stuff (what game words are available depends on the word model used so it is loaded in load_model)
self.game_words = list()
# teams stuff
self.team_weights = load_settings(sett_file=teams_file,
default_dict={
"t": 30,
"b": -1,
"o": -3,
"k": -10
})
self.team_words = dict(
) # made as an attribute to save passing back and forth while running rounds
# nlp stuff
self.word_model = self.load_word_model(
model_name=self.settings["model_name"], game_words_file=words_file
) # keyed vector model for generating hints
self.indexer = self.load_indexer(
model_name=self.settings["model_name"])
self.ls = LancasterStemmer() # stemmer for checking hint legality
self.spacy_nlp = spacy.load(
"en_core_web_sm") # lemmatiser for checking hint legality
if self.full_log is not None:
self.full_log.info("SpyMaster initialised!")
if self.game_log is not None:
self.game_log.info("SpyMaster initialised!")
def render(view_type=None, login=None, code=None):
""" Renders the login HTML. Exits. """
# init the settings object
settings = utils.load_settings()
template_file_path = "templates/login.html"
if view_type == 'reset':
template_file_path = 'templates/reset_password.html'
# grab up the template
html_raw = file(template_file_path, "rb").read()
html_tmp = Template(html_raw)
# create the output
output = html.meta.basic_http_header
output += html_tmp.safe_substitute(
version=settings.get('application', 'version'),
title=settings.get('application', 'title'),
api_url=api.get_api_url(),
prod_url=settings.get('application', 'tld'),
released=utils.get_latest_update_string(),
login=login,
code=code,
)
logger = utils.get_logger()
# render; don't exit (in case we're metering performance)
print(output)
def smtp_relay_get():
config = utils.load_settings(env)
dkim_rrtxt = ""
rr = config.get("SMTP_RELAY_DKIM_RR", None)
if rr is not None:
if rr.get("p") is None:
raise ValueError("Key doesn't exist!")
for c, d in (("v", "DKIM1"), ("h", None), ("k", "rsa"), ("n", None),
("s", None), ("t", None)):
txt = rr.get(c, d)
if txt is None:
continue
else:
dkim_rrtxt += f"{c}={txt}; "
dkim_rrtxt += f"p={rr.get('p')}"
return {
"enabled": config.get("SMTP_RELAY_ENABLED", False),
"host": config.get("SMTP_RELAY_HOST", ""),
"port": config.get("SMTP_RELAY_PORT", None),
"user": config.get("SMTP_RELAY_USER", ""),
"authorized_servers": config.get("SMTP_RELAY_AUTHORIZED_SERVERS", []),
"dkim_selector": config.get("SMTP_RELAY_DKIM_SELECTOR", None),
"dkim_rr": dkim_rrtxt
}
async def check_group_videos(vk):
plugin.temp_data['s'] = load_settings(plugin)
# Получаем количество видео в группе
values = {'owner_id': -int(plugin.temp_data['s']['public_id']), 'count': 0}
resp = await vk.method('video.get', values)
if resp:
plugin.temp_data['public_video_count'] = resp.get('count')
def __init__(self):
self.settings = utils.load_settings()
self.event = {}
self.read_event()
if not os.environ.has_key('SENSU_TEST'):
self.filter()
self.handle()
def __init__(self):
self.settings = utils.load_settings()
self.event = {}
self.read_event()
if not os.environ.has_key('SENSU_TEST'):
self.filter()
self.handle()
def smtp_relay_get():
config = utils.load_settings(env)
return {
"enabled": config.get("SMTP_RELAY_ENABLED", True),
"host": config.get("SMTP_RELAY_HOST", ""),
"auth_enabled": config.get("SMTP_RELAY_AUTH", False),
"user": config.get("SMTP_RELAY_USER", "")
}
def test(cases):
utils.load_settings(args, "settings/res128.json", cases)
def g_func():
x = Generator(args.base_ch, 32, 10, n_projected_dims=4).cuda()
return x
def z_func():
return torch.FloatTensor(
truncnorm.rvs(-1.5, 1.5, size=(args.batch_size, 128))).cuda()
def y_func():
return torch.eye(10)[torch.randint(0, 10, (args.batch_size, ))].cuda()
inceptions_score_fid_all("cifar_case" + str(cases), g_func, z_func, y_func,
args.use_multi_gpu, 50000 // args.batch_size + 1,
"cifar10_train.pkl")
def start(self):
"""Blocking call to delegate control to frontend."""
self.settings = utils.load_settings()
self.systems = utils.load_systems(self.settings)
# todo : configure joystick if found
pass
self.handlers = [SystemsHandler(self.systems)]
self.handlers[-1].render()
def default_quota_set():
config = utils.load_settings(env)
try:
config["default-quota"] = validate_quota(request.values.get('default_quota'))
utils.write_settings(config, env)
except ValueError as e:
return ("ERROR: %s" % str(e), 400)
return "OK"
def __init__(self):
log.info("Operative initialising...")
self.team_words = dict()
self.board_words = list()
self.hints = dict()
self.settings = load_settings(sett_file="settings/spymaster_setts.txt",
default_dict={"max_top_hints": 10,
"max_levels": 2})
self.evaluation_methods = [self.__concept_net_eval, self.__word_net_path_eval,
self.__word_net_wup_eval, self.__word_net_lch_eval]
log.info("Operative initialised!")
def _load_settings(self):
"""
Load the settings file. The file contains integrity check settings,
ignored entities configuration and other useful settings.
"""
self.lg.debug('Loading settings file from %s', self.args.settings)
try:
self.settings = utils.load_settings(self.args.settings)
except Exception as e:
raise ManagerError('Error loading settings: %s' % e)
self.lg.debug('Settings parsed: %s', self.settings)
def __init__(self, resolution_param='', iterations_param=''):
linear_data, cuircular_data, hyperbolic_data, resolution = load_settings()
self.__linear_data = linear_data
self.__circular_data = cuircular_data
self.__hyperbolic_data = hyperbolic_data
if (resolution_param != ''):
self.__resolution = resolution_param
else:
self.__resolution = resolution
self.__iterations = iterations_param
self.__constants_compute()
def check_miab_version(env, output):
config = load_settings(env)
if config.get("privacy", True):
output.print_warning("Mail-in-a-Box version check disabled by privacy setting.")
else:
this_ver = what_version_is_this(env)
latest_ver = get_latest_miab_version()
if this_ver == latest_ver:
output.print_ok("Mail-in-a-Box is up to date. You are running version %s." % this_ver)
else:
output.print_error("A new version of Mail-in-a-Box is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://mailinabox.email. "
% (this_ver, latest_ver))
def check_miab_version(env, output):
config = load_settings(env)
if config.get("privacy", True):
output.print_warning("Mail-in-a-Box version check disabled by privacy setting.")
else:
this_ver = what_version_is_this(env)
latest_ver = get_latest_miab_version()
if this_ver == latest_ver:
output.print_ok("Mail-in-a-Box is up to date. You are running version %s." % this_ver)
else:
output.print_error("A new version of Mail-in-a-Box is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://mailinabox.email. "
% (this_ver, latest_ver))
async def setup_counter(vk):
plugin.temp_data['s'] = load_settings(plugin)
plugin.temp_data['start_time'] = datetime.datetime.now()
plugin.temp_data['24h'] = datetime.datetime.now()
plugin.temp_data['messages'] = 0
plugin.temp_data['messages_24h'] = 0
plugin.temp_data['commands'] = {}
if plugin.temp_data['s']['set_status']:
schedule_coroutine(update_counters(vk))
def __init__(self, screen: pygame.Surface):
self.screen = screen
self.options = [
KineticButton("<",
15,
15,
40,
40,
"back",
"Calibri",
30,
background_color=(70, 100, 120),
hover_color=(240, 230, 50),
borders=2),
KineticButton("Activé",
200,
200,
150,
30,
"toggle_fullscreen",
"Calibri",
30,
background_color=(70, 100, 120),
hover_color=(240, 230, 50),
borders=2),
KineticButton("Appliquer",
20,
300,
130,
30,
"apply",
'Calibri',
30,
background_color=(70, 100, 120),
hover_color=(240, 230, 50),
borders=2)
]
self.hover = -1
self.config = utils.load_settings()
self._update_button()
self.actual_fullscreen = self.config["fullscreen"]
self.volume_x = self.config["sound"] * 500 + 200
self.click_volume = False
self.offset_click = 0
def __init__(self, master, *args):
"""the settings 'tab' widget of the GUI"""
super().__init__(master)
self.speed_options = ["Fast", "Medium", "Slow"]
self.yes_no_options = ["Yes", "No"]
self.tooltips = {
"Spotify Path":
"The path to Spotify on your local computer. This is where exactly on your PC Spotify.exe is located.",
"Pause When Locked":
"Whether or not to stop the script from restarting Spotify while the Windows account is locked. This helps prevent Spotify restarting when using Spotify on phone and it reaches an ad.",
"Push To Back":
"Whether or not to push Spotify below the active window after restart.",
"Create Shortcut":
"Whether or not to create a desktop shortcut when the GUI launches. Setting this to yes will create a shortcut or overwrite the previous shortcut on GUI launch, so even when you delete it a new one will be generated. Set this to no, and it will no longer generate new ones.",
}
self.settings = load_settings()
self.widget()
def run_network_checks(env, output):
# Also see setup/network-checks.sh.
output.add_heading("Network")
check_ufw(env, output)
# Stop if we cannot make an outbound connection on port 25. Many residential
# networks block outbound port 25 to prevent their network from sending spam.
# See if we can reach one of Google's MTAs with a 5-second timeout.
code, ret = shell("check_call", ["/bin/nc", "-z", "-w5", "aspmx.l.google.com", "25"], trap=True)
if ret == 0:
output.print_ok("Outbound mail (SMTP port 25) is not blocked.")
else:
output.print_warning("""Outbound mail (SMTP port 25) seems to be blocked by your network. You
will not be able to send any mail without a SMTP relay. Many residential networks block port 25 to prevent
hijacked machines from being able to send spam. A quick connection test to Google's mail server on port 25
failed.""")
# Stop if the IPv4 address is listed in the ZEN Spamhaus Block List.
# The user might have ended up on an IP address that was previously in use
# by a spammer, or the user may be deploying on a residential network. We
# will not be able to reliably send mail in these cases.
rev_ip4 = ".".join(reversed(env['PUBLIC_IP'].split('.')))
zen = query_dns(rev_ip4+'.zen.spamhaus.org', 'A', nxdomain=None)
if zen is None:
output.print_ok("IP address is not blacklisted by zen.spamhaus.org.")
else:
output.print_error("""The IP address of this machine %s is listed in the Spamhaus Block List (code %s),
which may prevent recipients from receiving your email. See http://www.spamhaus.org/query/ip/%s."""
% (env['PUBLIC_IP'], zen, env['PUBLIC_IP']))
# Check if a SMTP relay is set up. It's not strictly required, but on some providers
# it might be needed.
config = load_settings(env)
if config.get("SMTP_RELAY_ENABLED"):
if config.get("SMTP_RELAY_AUTH"):
output.print_ok("An authenticated SMTP relay has been set up via port 587.")
else:
output.print_warning("A SMTP relay has been set up, but it is not authenticated.")
elif ret == 0:
output.print_ok("No SMTP relay has been set up (but that's ok since port 25 is not blocked).")
else:
output.print_error("No SMTP relay has been set up. Since port 25 is blocked, you will probably not be able to send any mail.")
def smtp_relay_set():
from editconf import edit_conf
config = utils.load_settings(env)
newconf = request.form
try:
# Write on daemon settings
config["SMTP_RELAY_ENABLED"] = (newconf.get("enabled") == "true")
config["SMTP_RELAY_HOST"] = newconf.get("host")
config["SMTP_RELAY_AUTH"] = (newconf.get("auth_enabled") == "true")
config["SMTP_RELAY_USER"] = newconf.get("user")
utils.write_settings(config, env)
# Write on Postfix configs
edit_conf("/etc/postfix/main.cf", [
"relayhost=" +
(f"[{config['SMTP_RELAY_HOST']}]:587" if
config["SMTP_RELAY_ENABLED"] else ""), "smtp_sasl_auth_enable=" +
("yes" if config["SMTP_RELAY_AUTH"] else "no"),
"smtp_sasl_security_options=" +
("noanonymous" if config["SMTP_RELAY_AUTH"] else "anonymous"),
"smtp_sasl_tls_security_options=" +
("noanonymous" if config["SMTP_RELAY_AUTH"] else "anonymous")
],
delimiter_re=r"\s*=\s*",
delimiter="=",
comment_char="#")
if config["SMTP_RELAY_AUTH"]:
# Edit the sasl password
with open("/etc/postfix/sasl_passwd", "w") as f:
f.write(
f"[{config['SMTP_RELAY_HOST']}]:587 {config['SMTP_RELAY_USER']}:{newconf.get('key')}\n"
)
utils.shell("check_output",
["/usr/bin/chmod", "600", "/etc/postfix/sasl_passwd"],
capture_stderr=True)
utils.shell("check_output",
["/usr/sbin/postmap", "/etc/postfix/sasl_passwd"],
capture_stderr=True)
# Restart Postfix
return utils.shell("check_output",
["/usr/bin/systemctl", "restart", "postfix"],
capture_stderr=True)
except Exception as e:
return (str(e), 500)
def run_random_round(self, out_file=None):
if self.full_log is not None:
self.full_log.info("Running round with random teams...")
self.full_log.debug("Shuffling words")
if self.game_log is not None:
self.game_log.info("Running round with random teams...")
shuffle(self.game_words)
word_gen = cycle(self.game_words)
if self.full_log is not None:
self.full_log.debug("Loading team sizes...")
team_sizes = load_settings(sett_file="settings/team_sizes.txt",
default_dict={
"t": 8,
"o": 8,
"k": 1,
"b": 8
})
if self.full_log is not None:
self.full_log.debug("Team sizes: {0}".format(" - ".join(
[k + ":" + str(team_sizes[k]) for k in ["t", "o", "b", "k"]])))
if self.full_log is not None:
self.full_log.debug("Generating team words...")
self.team_words = {
team: [next(word_gen) for i in range(team_sizes[team])]
for team in ["t", "o", "b", "k"]
}
if self.full_log is not None:
self.full_log.info("Team words:\n{0}".format("\n".join([
team + ": " + ", ".join(self.team_words[team])
for team in ["t", "o", "b", "k"]
])))
if self.game_log is not None:
self.game_log.info("Randomly generated teams are:\n{0}".format(
"\n".join([
team + ": " + ", ".join(self.team_words[team])
for team in ["t", "o", "b", "k"]
])))
return self.__run_round(out_file=out_file)
def check_miab_version(env, output):
config = load_settings(env)
try:
this_ver = what_version_is_this(env)
except:
this_ver = "Unknown"
if config.get("privacy", True):
output.print_warning("You are running version Mail-in-a-Box %s. Mail-in-a-Box version check disabled by privacy setting." % this_ver)
else:
latest_ver = get_latest_miab_version()
if this_ver == latest_ver:
output.print_ok("Mail-in-a-Box is up to date. You are running version %s." % this_ver)
elif latest_ver is None:
output.print_error("Latest Mail-in-a-Box version could not be determined. You are running version %s." % this_ver)
else:
output.print_error("A new version of Mail-in-a-Box is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://mailinabox.email. "
% (this_ver, latest_ver))
def check_miab_version(env, output):
config = load_settings(env)
try:
this_ver = what_version_is_this(env)
except:
this_ver = "Unknown"
if config.get("privacy", True):
output.print_warning("You are running version Mail-in-a-Box %s. Mail-in-a-Box version check disabled by privacy setting." % this_ver)
else:
latest_ver = get_latest_miab_version()
if this_ver == latest_ver:
output.print_ok("Mail-in-a-Box is up to date. You are running version %s." % this_ver)
elif latest_ver is None:
output.print_error("Latest Mail-in-a-Box version could not be determined. You are running version %s." % this_ver)
else:
output.print_error("A new version of Mail-in-a-Box is available. You are running version %s. The latest version is %s. For upgrade instructions, see https://wspecsbox.email. "
% (this_ver, latest_ver))
def create_model():
possible_models = [
"shufflenet_v2_x1_0",
"resnet18",
"mobilenet_v2",
"mnasnet1_0",
"mnasnet1_3",
]
settings = utils.load_settings()
model_name = random.choice(possible_models)
print("Test:: Model: {}".format(model_name))
model = getattr(torchvision.models, model_name)()
script_model = torch.jit.script(model)
script_model.save(os.environ["MODEL"])
settings["model_name"] = model_name
with open(os.environ["SETTINGS"], "w") as file:
yaml.dump(settings, file, default_flow_style=False)
def init_vars(self, prev_sc):
# 1. Initializing variables
# a. loading the settings
print("| Loading settings... |")
self.settings = utils.load_settings()
print("| -- loaded settings... |\n|", self.settings)
# b. starting necessary variables
print("| Loading experiment variable...|")
self.experiment = prev_sc.experiment
# c. intializing round variables
print("| Loading round variables |")
self.clicks = ''
self.points = tkinter.StringVar()
self.points.set(0)
self.repeat = 0
self.reinforcement = []
# d. game variables
print("| Loading game variables... |")
self.combinations = ['EEEE','EEED','EEDE','EDEE',\
'DEEE','EEDD','EDDE','DDEE','DEED','DEDE','EDED','DDDE',\
'DDED','DEDD','EDDD','DDDD']
self.frequency = {'EEEE':1,'EEED':1,'EEDE':1,'EDEE':1,\
'DEEE':1,'EEDD':1,'EDDE':1,'DDEE':1,'DEED':1,'DEDE':1,'EDED':1,'DDDE':1,\
'DDED':1,'DEDD':1,'EDDD':1,'DDDD':1}
self.total_frequency = {'EEEE':1,'EEED':1,'EEDE':1,'EDEE':1,\
'DEEE':1,'EEDD':1,'EDDE':1,'DDEE':1,'DEED':1,'DEDE':1,'EDED':1,'DDDE':1,\
'DDED':1,'DEDD':1,'EDDD':1,'DDDD':1}
self.memo_accuracy = 0
self.memo_reinforced = []
self.saved_order = []
self.stages = []
# e. result variables
print("| Loading result variables... |")
self.results = []
self.result_set = set()
self.blocks = []
def init():
"""Fonction d'initialisation de l'application"""
# D'abord on vérifie l'existence de tous les répertoires utiles
repertories = [
"scenes", "assets", "saves", "assets/backgrounds", "assets/musics"
]
for rep in repertories:
if not os.path.isdir(rep):
os.mkdir(rep)
# Ensuite on construit notre fenêtre pygame
pygame.init()
pygame.font.init()
pygame.mixer.init()
screen = pygame.display.set_mode((1280, 720))
cfg = utils.load_settings()
if cfg["fullscreen"] > 0:
pygame.display.toggle_fullscreen()
return screen
import sys, os, h5py
import numpy as np
import tensorflow as tf
from rich.console import Console
console = Console()
sys.path.append(r"/home/qnl/Git-repositories")
from utils import load_settings, load_repackaged_data, get_data, split_data_same_each_time, dark_mode_compatible
from qutrit_lstm_network import pad_labels
dark_mode_compatible(dark_mode_color=r'#86888A')
settings = load_settings(
r"/home/qnl/Git-repositories/qnl_nonmarkov_ml/qutrit_lstm/settings.yaml")
# NOTE: Note that most of the settings below must be equal to the settings in prep.py
# Path that contains the training/validation dataset.
filepath = settings['voltage_records']['filepath']
filename = settings['voltage_records']['filename']
# last_timestep determines the length of trajectories used for training in units of strong_ro_dt.
# Must be <= the last strong readout point
mask_value = settings['training'][
'mask_value'] # This is the mask value for the data, not the missing labels
num_features = settings['voltage_records']['num_features'] # I and Q
strong_ro_dt = settings['voltage_records'][
'strong_ro_dt'] # Time interval for strong readout in the dataset in seconds
console.print("Loading data...", style="bold red")
weight
))
del imgs[:fps_ratio]
elapsed_time = time.process_time()-time_start
time_list.pop(0)
time_list.append(elapsed_time)
average_time = sum(time_list) / len(time_list)
print('Performance:', '%.3f' % average_time,
'seconds/frame -', '%.3f' % (1/average_time), 'FPS'
)
print('Estimation:', time.strftime('%H:%M:%S', time.gmtime(math.ceil(avg_time*int(output_nframes-i)))))
print(f'Progress: {i}/{output_frames} -', '%.3f' % (100*i/output_frames), '%')
output_video.release()
input_video.release()
utils.add_audio(input_name, output_name)
if settings['cv_colourfix']:
Path(input_name).unlink()
Path('to-fix_{input_name}').rename(input_name)
if __name__ == '__main__':
settings = utils.load_settings()
print(settings)
process_video(settings)
def load_complete_tree(self, filter_enabled=True, complete=True):
'''Creates a folder hash table and another non-folder hash table
stores nodes in the first hash and the id is the key (e.g. /MyDrive/Books/Fiction)
the second one simply stores the file struct and the id is the
key (e.g. Star Wars.pdf).
:param filter_enabled: if whitelist or blacklist is enabled.
:type filter_enabled: bool.
:param complete: If will link files to tree.
:type complete: bool.
'''
whitelist = blacklist = None
if filter_enabled:
settings = load_settings()
if settings['whitelist-enabled']:
whitelist = settings['whitelist-files']
elif settings['blacklist-enabled']:
blacklist = settings['blacklist-files']
# =========== debug code ===========
# just to keep local query to not request files every run
# if not os.path.exists('folders.dat'):
# query = 'trashed = false and mimeType = "%s"' % TYPES['folder']
# fields = 'nextPageToken, files(name, id, parents, mimeType)'
# folders_metadata = []
# pageToken = None
# while True:
# result = self.service.files().list(q=query,\
# fields=fields,\
# pageToken=pageToken,\
# pageSize=1000).execute()
# folders_metadata += result.get('files', [])
# pageToken = result.get('nextPageToken')
# if not pageToken:
# break
# with open('folders.dat', 'wb') as f:
# pickle.dump(folders_metadata, f, pickle.HIGHEST_PROTOCOL)
# else:
# with open('folders.dat', 'rb') as f:
# folders_metadata = pickle.load(f)
# =========== debug code ===========
# =========== real code ===========
query = 'trashed = false and mimeType = "%s"' % TYPES['folder']
folders_metadata = []
pageToken = None
while True:
fields = 'nextPageToken, files(name, id, parents, mimeType)'
result = self.service.files().list(q=query,\
fields=fields,\
pageToken=pageToken,\
pageSize=1000).execute()
folders_metadata += result.get('files', [])
pageToken = result.get('nextPageToken')
if not pageToken:
break
# =========== real code ===========
# just the folders vector, will be converted to hash bellow
folders = [f for f in folders_metadata\
if 'parents' in f]
self.root.children = [] # empty tree
stack = [] # [metadata]
self.folders_hash = {}
i = 0 # used to pin the node that is looking for a parent
j = 0 # used to pin the next node that will look for the parent
while folders or stack:
enqueue = None
j = 0
for folder in folders:
if folders[i]['parents'][0] == folder['id']:
enqueue = folders[i]
break
j += 1
if enqueue:
stack.append(enqueue)
folders.pop(i)
if j < i:
i = j
else:
i = j - 1
elif folders[i]['parents'][0] == self.root.get_id():
title = ('/' + folders[i]['name'] + '/')
if (blacklist and title in blacklist)\
or (whitelist and not any(title in elem for elem in whitelist)):
stack = []
folders.pop(i)
i = 0
continue
child = DriveFile(self.root, folders[i])
self.folders_hash[folders[i]['id']] = child
while stack:
item = stack.pop()
title = title + '/' + item['name'] + '/'
if (blacklist and (title in blacklist)) \
or (whitelist and not \
any(elem in title for elem in whitelist)):
stack = []
break
parent = child
child = DriveFile(parent, item)
self.folders_hash[item['id']] = child
folders.pop(i)
i = 0
else:
parent_id = folders[i]['parents'][0]
if not parent_id in self.folders_hash:
stack = []
folders.pop(i)
i = 0
continue
elif filter_enabled:
title = self.folders_hash[parent_id].get_path(
) + folders[i]['name'] + '/'
if (blacklist and (title in blacklist))\
or (whitelist and not\
any(elem in title for elem in whitelist)):
stack = []
folders.pop(i)
i = 0
continue
child = DriveFile(self.folders_hash[parent_id], folders[i])
self.folders_hash[child.get_id()] = child
while stack:
parent = child
item = stack.pop()
if (blacklist and (title in blacklist))\
or (whitelist and not \
any(elem in title for elem in whitelist)):
stack = []
break
child = DriveFile(parent, item)
self.folders_hash[item['id']] = child
folders.pop(i)
i = 0
if complete:
if self.folders_hash:
parents_query = [
'mimeType != \'%s\' and ("%s" in parents' %
(TYPES['folder'], list(self.folders_hash)[0])
]
i = 0 # counter
j = 0 # index of the list
for item in list(self.folders_hash)[1:]:
adding = '"%s" in parents' % item
# 30000 is the max body size before too complex query
if len(' or ' + parents_query[j]) >= 25000:
parents_query[j] += ')'
j += 1
i = 0
parents_query.append(
'mimeType != \'%s\' and ("%s" in parents' %
(TYPES['folder'], item))
continue
parents_query[j] += ' or ' + adding
i += 1
parents_query[j] += ')'
else:
print('no folders found')
return ()
fields = 'nextPageToken, files(name, id, parents, mimeType)'
pageTokens = [None] * len(parents_query)
files_metadata = []
while True:
for i, query in enumerate(parents_query):
if pageTokens[i] != '0':
result = self.service.files().list(q=query,\
fields=fields,\
pageToken=pageTokens[i],\
pageSize=1000).execute()
files_metadata += result.get('files', [])
pageTokens[i] = result.get('nextPageToken')
if not pageTokens[i]:
pageTokens[i] = '0'
if all(token == '0' for token in pageTokens):
break
for metadata in files_metadata:
if not metadata['parents'][0] == self.root.get_id():
parent = self.folders_hash[metadata['parents'][0]]
else:
if filter_enabled:
continue
parent = self.root
DriveFile(parent, metadata)
def privacy_status_get():
config = utils.load_settings(env)
return json_response(config.get("privacy", True))
def privacy_status_set():
config = utils.load_settings(env)
config["privacy"] = (request.form.get('value') == "private")
utils.write_settings(config, env)
return "OK"
def privacy_status_set():
config = utils.load_settings(env)
config["privacy"] = (request.form.get('value') == "private")
utils.write_settings(config, env)
return "OK"
def privacy_status_get():
config = utils.load_settings(env)
return json_response(config.get("privacy", True))
def _load_settings(self):
for prov in self.provider.values():
prov.load_credentials()
plugin = utils.load_settings("plugin")
self._set_chosen_plugin(plugin)
def get_default_quota(env):
config = utils.load_settings(env)
return config.get("default-quota", '0')
err = "PID dir '%s' does not exist!" % pid_dir
logger.critical(err)
raise Exception(err)
else:
logger.info("PID dir '%s' exists." % pid_dir)
pid_dir_owner = getpwuid(os.stat(pid_dir).st_uid).pw_name
logger.info("PID dir '%s' is owned by '%s'." % (pid_dir, pid_dir_owner))
if pid_dir_owner != os.environ["USER"]:
logger.warn("PID dir owner is not the current user!")
if __name__ == "__main__":
logger = get_logger()
settings = load_settings()
parser = OptionParser()
parser.add_option(
"-i",
dest="interactive",
help="Run the server in 'interactive' mode (print output to STDOUT)",
default=False,
action="store_true")
parser.add_option("-p",
dest="port",
help="Force the server to run on the specified port",
default=None,
metavar="9999")
parser.add_option(
"--action",
def main():
#---Load environment settings from SETTINGS.json in root directory and build filepaths for all base submissions---#
settings = utils.load_settings('SETTINGS.json')
base_filepaths = (settings['file_bryan_submission'],
settings['file_miroslaw_submission'])
segment_weights = settings['ensemble_segment_weights']
segments = segment_weights.keys()
targets = segment_weights[segments[0]].keys()
#---Output the segment weights to be used for ensemble averaging of base submissions---#
log.info('==========ENSEMBLE WEIGHTS (B,M)============')
for segment in segment_weights:
log.info(segment.upper()+':')
for target in segment_weights[segment]:
log.info(' '+target.upper()+' -- ['+segment_weights[segment][target]['0']+','+
segment_weights[segment][target]['1']+']')
#---Load each base submission to a list of dataframes---#
base_subs = []
for file in base_filepaths:
try:
base_subs.append(pd.read_csv(file).set_index(['id'], drop=False).sort())
log.info('Base submission successfully loaded: %s.' % file)
except IOError:
log.info('Base submission file does not exist: %s. Run base model to generate, or update filepath.' %file)
sys.exit('---Exiting---')
utils.line_break()
#---Load id's labeled with segments to a dataframe used for segment based averaging---#
file = settings['file_segment_ids']
try:
segment_ids = pd.read_csv(file)
log.info('Segment IDs successfully loaded from: %s.' % file)
except IOError:
log.info('Segment IDs file does not exist: %s. Update filepath in SETTINGS.json.' % file)
utils.line_break()
#---Transform base predictions to log space prior to averaging, if selected in settings---#
if settings['avg_log_space'] == 'y':
log.info('Transforming base predictions to log space prior to averaging.')
for i in range(len(base_subs)):
for target in targets:
base_subs[i][target] = np.log(base_subs[i][target]+1)
utils.line_break()
#---Apply segment based weights to each base submission then combine them to create ensemble submission---#
log.info('Applying segment weights to base submissions then combining to create ensemble.')
for i in range(len(base_subs)):
#Merge the segment labels from the segment id's file with the base submission dataframe
base_subs[i] = base_subs[i].merge(segment_ids,on='id',how='inner')
for segment in segments:
for target in targets:
base_subs[i][target][base_subs[i]['Segment'] == segment] \
*= float(segment_weights[segment][target][str(i)])
del base_subs[i]['Segment']
ensemble_sub = base_subs[0].ix[:]
for i in range(len(base_subs)-1):
for target in targets:
ensemble_sub[target] += base_subs[i+1][target]
utils.line_break()
#---Transform ensemble predictions back to normal, if use log space averaging was selected in settings---#
if settings['avg_log_space'] == 'y':
log.info('Transforming ensemble predictions back to normal from log space.')
for target in targets:
ensemble_sub[target] = np.exp(ensemble_sub[target])-1
utils.line_break()
#---Apply any final target scalars to ensemble predictions---#
for target in targets:
ensemble_sub[target] *= float(settings['target_scalars'][target])
#---Output ensemble submission to directory set in SETTINGS.json, appending creation date and time---#
timestamp = datetime.now().strftime('%m-%d-%y_%H%M')
filename = settings['dir_ensemble_submissions']+'ensemble_predictions_'+timestamp+'.csv'
ensemble_sub.to_csv(filename, index=False)
log.info('Ensemble submission saved: %s' % filename)
utils.line_break()
#End main
log.info('Program executed successfully without error! Exiting.')
def main():
log.info('********New program instance started********')
#-------------Load Environment----------------------#
#Get program settings and model settings from SETTINGS.json file in root directory
settings, model_settings = utils.load_settings()
#If not using cached data, then load raw data, clean/munge it, create hand-crafted features, slice it for CV
if settings['use_cached_data'] == 'y':
log.info('==========LOADING CACHED FEATURES===========')
dfTrn = data_io.load_cached_object('dfTrn')
dfTest = data_io.load_cached_object('dfTest')
dfCV = data_io.load_flatfile_to_df('Data/CV.csv')
else:
#-------Data Loading/Cleaning/Munging------------#
#Load the data
log.info('===============LOADING DATA=================')
dfTrn = data_io.load_flatfile_to_df(settings['file_data_train'])
dfTest = data_io.load_flatfile_to_df(settings['file_data_test'])
dfCV = data_io.load_flatfile_to_df('Data/CV.csv')
#Clean/Munge the data
log.info('=======CLEANING AND MUNGING DATA============')
dfTrn = munge.clean(dfTrn)
dfTest = munge.clean(dfTest)
#-------Feature creation-------------------------#
#Add all currently used hand crafted features to dataframes
log.info('====CREATING HAND-CRAFTED DATA FEATURES=====')
features.add(dfTrn)
features.add(dfTest)
#---------Data slicing/parsing--------------------------#
#Split data for CV
if settings['generate_cv_score'] == 'y':
log.info('=====SPLITTING DATA FOR CROSS-VALIDATION====')
if settings['cv_method'] == 'april':
dfTrnCV, dfTestCV = munge.temporal_split(dfTrn, (2013, 04, 1))
elif settings['cv_method'] == 'march':
#take an addtional week from February b/c of lack of remote_api source issues in March
dfTrnCV, dfTestCV = munge.temporal_split(dfTrn, (2013, 02, 21))
elif settings['cv_method'] == 'list_split':
#load stored list of data points and use those for CV
dfCVlist = pd.DataFrame({'id': data_io.load_cached_object('Cache/cv_issue_ids.pkl'), 'dummy': 0})
dfTrnCV, dfTestCV = munge.list_split(dfTrn, dfCVlist)
#--------------Modeling-------------------------#
#If cached models exist then load them for reuse into segment_models. Then run through model_settings and for
# each model where 'use_cached_model' is false then clear the cached model and recreate it fresh
log.info('=========LOADING CACHED MODELS==============')
segment_models = data_io.load_cached_object('segment_models')
if segment_models == None:
log.info('=========CACHED MODELS NOT LOADED===========')
for model in model_settings:
model['use_cached_model'] = 'n'
segment_models = []
#Initialize new model for models not set to use cache
log.info('=======INITIALIZING UN-CACHED MODELS========')
index = 0
for model in model_settings:
if model_settings[model]['use_cached_model'] == 'n':
new_model = ensembles.Model(model_name=model,target=model_settings[model]['target'],
segment=model_settings[model]['segment'],
estimator_class=model_settings[model]['estimator_class'],
estimator_params=model_settings[model]['estimator_params'],
features=model_settings[model]['features'],
postprocess_scalar=model_settings[model]['postprocess_scalar'])
#Flag the model as not cached, so that it does not get skipped when running the modeling process
new_model.use_cached_model='n'
#Project specific model attributes not part of base class
new_model.KNN_neighborhood_threshold=model_settings[model]['KNN_neighborhood_threshold']
new_model.sub_zip_neighborhood=model_settings[model]['sub_zip_neighborhood']
segment_models[index] = new_model
log.info('Model %s intialized at index %i' % (model,index))
index += 1
#Cross validate all segment models (optional)
if settings['export_cv_predictions_all_models'] == 'y' or settings['export_cv_predictions_new_models'] == 'y':
log.info('============CROSS VALIDATION================')
for model in segment_models[:]:
#If model has cached CV predictions then skip predicting and just export them (if selected in settings)
if hasattr(model,'dfCVPredictions'):
log.info('Cached CV predictions found. Using cached CV predictions.')
if settings['export_cv_predictions_all_models'] == 'y':
data_io.save_predictions(model.dfCVPredictions,model.target,model_name=model.model_name,
directory=settings['dir_submissions'],
estimator_class=model.estimator_class, note='CV_list')
else:
print_model_header(model)
#Prepare segment model: segment and create feature vectors for the CV data set
dfTrn_Segment, dfTest_Segment = prepare_segment_model(dfTrnCV,dfTestCV,model)
#Generate CV predictions
train.cross_validate(model, settings, dfTrn_Segment, dfTest_Segment)
#Cache the CV predictions as a dataframe stored in each segment model
model.dfCVPredictions = dfTest_Segment.ix[:,['id',model.target]]
if settings['export_cv_predictions_new_models'] == 'y':
data_io.save_predictions(model.dfCVPredictions,model.target,model_name=model.model_name,
directory=settings['dir_submissions'],
estimator_class=model.estimator_class, note='CV_list')
#Generate predictions on test set for all segment models (optional)
if settings['export_predictions_all_models'] == 'y' or settings['export_predictions_new_models'] == 'y'\
or settings['export_predictions_total'] == 'y':
log.info('=======GENERATING TEST PREDICTIONS==========')
for model in segment_models[:]:
#If model has cached test predictions then skip predicting and just export them (if selected in settings)
if hasattr(model,'dfPredictions'):
log.info('Cached test predictions found for model %s. Using cached predictions.' % model.model_name)
if settings['export_predictions_all_models'] == 'y':
data_io.save_predictions(model.dfPredictions,model.target,model_name=model.model_name,
directory=settings['dir_submissions'],
estimator_class=model.estimator_class,note='TESTset')
else:
print_model_header(model)
#Prepare segment model: segment and create feature vectors for the full TEST data set
dfTrn_Segment, dfTest_Segment = prepare_segment_model(dfTrn,dfTest,model)
#Generate TEST set predictions
model.predict(dfTrn_Segment, dfTest_Segment)
if settings['export_predictions_all_models'] == 'y' or settings['export_predictions_new_models'] == 'y':
data_io.save_predictions(model.dfPredictions,model.target,model_name=model.model_name,
directory=settings['dir_submissions'],
estimator_class=model.estimator_class,note='TESTset')
log.info(utils.line_break())
#Cache the trained models and predictions to file (optional)
if settings['export_cached_models'] == 'y':
log.info('==========EXPORTING CACHED MODELS===========')
data_io.save_cached_object(segment_models,'segment_models')
#Merge each segment model's CV predictions into a master dataframe and export it (optional)----#
if settings['export_cv_predictions_total'] == 'y':
log.info('====MERGING CV PREDICTIONS FROM SEGMENTS====')
dfTestPredictionsTotal = merge_segment_predictions(segment_models, dfTestCV, cv=True)
#---Apply post process rules to master dataframe---#
#Set all votes and comments for remote_api segment to 1 and 0
dfTestPredictionsTotal = dfTestPredictionsTotal.merge(dfTest.ix[:][['source','id']], on='id', how='left')
for x in dfTestPredictionsTotal.index:
if dfTestPredictionsTotal.source[x] == 'remote_api_created':
dfTestPredictionsTotal.num_votes[x] = 1
dfTestPredictionsTotal.num_comments[x] = 0
#Export
timestamp = datetime.now().strftime('%m-%d-%y_%H%M')
filename = 'Submits/'+timestamp+'--bryan_CV_predictions.csv'
dfTestPredictionsTotal.to_csv(filename)
#Merge each segment model's TEST predictions into a master dataframe and export it (optional)----#
if settings['export_predictions_total'] == 'y':
log.info('===MERGING TEST PREDICTIONS FROM SEGMENTS===')
dfTestPredictionsTotal = merge_segment_predictions(segment_models, dfTest)
#---Apply post process rules to master dataframe---#
#Set all votes and comments for remote_api segment to 1 and 0
dfTestPredictionsTotal = dfTestPredictionsTotal.merge(dfTest.ix[:][['source','id']], on='id', how='left')
for x in dfTestPredictionsTotal.index:
if dfTestPredictionsTotal.source[x] == 'remote_api_created':
dfTestPredictionsTotal.num_votes[x] = 1
dfTestPredictionsTotal.num_comments[x] = 0
del dfTestPredictionsTotal['source']
#Export
filename = 'bryan_test_predictions.csv'
data_io.save_combined_predictions(dfTestPredictionsTotal, settings['dir_submissions'], filename)
#End main
log.info('********Program ran successfully. Exiting********')
