def generate() -> None:
authorities = get_targz_path(check_exists=False)
if os.path.exists(authorities):
command.fail("authorities.tgz already exists")
# tempfile.TemporaryDirectory() creates the directory with 0o600, which protects the private keys
with tempfile.TemporaryDirectory() as d:
certdir = os.path.join(d, "certdir")
keyserver_yaml = os.path.join(d, "keyserver.yaml")
util.writefile(keyserver_yaml, configuration.get_keyserver_yaml().encode())
os.mkdir(certdir)
print("generating authorities...")
try:
# TODO: avoid having these touch disk
subprocess.check_call(["keygen", keyserver_yaml, certdir, "supervisor-nodes"])
except FileNotFoundError as e:
if e.filename == "keygen":
command.fail("could not find keygen binary. is the homeworld-keyserver dependency installed?")
else:
raise e
print("encrypting authorities...")
cryptdir = os.path.join(d, "cryptdir")
os.mkdir(cryptdir)
for filename in os.listdir(certdir):
if filename.endswith(".pub") or filename.endswith(".pem"):
# public keys; copy over without encryption
util.copy(os.path.join(certdir, filename), os.path.join(cryptdir, filename))
else:
# private keys; encrypt when copying
keycrypt.gpg_encrypt_file(os.path.join(certdir, filename), os.path.join(cryptdir, filename))
subprocess.check_call(["shred", "--"] + os.listdir(certdir), cwd=certdir)
print("packing authorities...")
subprocess.check_call(["tar", "-C", cryptdir, "-czf", authorities, "."])
subprocess.check_call(["shred", "--"] + os.listdir(cryptdir), cwd=cryptdir)
def upload_bytes(node: configuration.Node, source_bytes: bytes, dest_path: str) -> None:
# tempfile.TemporaryDirectory() creates the directory with 0o600, which protects the data if it's sensitive
with tempfile.TemporaryDirectory() as scratchdir:
scratchpath = os.path.join(scratchdir, "scratch")
util.writefile(scratchpath, source_bytes)
check_scp_up(node, scratchpath, dest_path)
os.remove(scratchpath)
def run():
print("Creating timeline/index")
html = templates.get("timeline/index")
html = templates.initial_replace(html, 1)
tablehtml = ""
upcominghtml = ""
for row in t_db:
rowhtml = templates.get("timeline/index_row")
rowhtml = rowhtml.replace("__NUMBER__", row["number"])
rowhtml = rowhtml.replace("__YEAR__", row["year"])
rowhtml = rowhtml.replace("__DATE__", row["date"])
rowhtml = rowhtml.replace("__CODE__", row["code"])
rowhtml = rowhtml.replace("__CITY__", row["city"])
rowhtml = rowhtml.replace("__COUNTRY__", code_to_country[row["code"]])
rowhtml = rowhtml.replace("__P_COUNTRY__", row["p_country"])
rowhtml = rowhtml.replace("__P_STUDENT__", row["p_student"])
if "code2" in row:
rowhtml = rowhtml.replace("__CODE2__", row["code2"])
rowhtml = rowhtml.replace("__COUNTRY2__",
code_to_country[row["code2"]])
rowhtml = rowhtml.replace("__CODE2_STYLE__", "")
else:
rowhtml = rowhtml.replace("__CODE2_STYLE__", "display: none;")
rowhtml = rowhtml.replace("__CODE2__", ".") # Google crawler fix
if int(row["year"]) < int(config.previous_year) + 3:
# Reverse list
tablehtml = rowhtml + tablehtml
else:
upcominghtml = rowhtml + upcominghtml
html = html.replace("__TABLE__", tablehtml)
html = html.replace("__UPCOMING__", upcominghtml)
html = templates.final_replace(html, "..")
util.writefile("../timeline/index.html", html)
def add_password_to_log(password):
passwords = os.path.join(configuration.get_project(), "passwords")
if not os.path.isdir(passwords):
os.mkdir(passwords)
passfile = os.path.join(passwords,
"at-%s.gpg" % datetime.datetime.now().isoformat())
util.writefile(passfile, keycrypt.gpg_encrypt_in_memory(password))
def launch_spec_direct(
path): # TODO: add a flag that enables this instead of launch_spec
with tempfile.TemporaryDirectory() as d:
specfile = os.path.join(d, "spec.yaml")
util.writefile(specfile,
configuration.get_single_kube_spec(path).encode())
access.call_kubectl(["apply", "-f", specfile], return_result=False)
def run():
print("Creating contestants/index")
html = templates.get("contestants/index")
html = templates.initial_replace(html, 2)
def keyfn(item):
return (item[1]["G"], item[1]["S"], item[1]["B"], item[1]["H"],
sum(item[1].values()))
sorted_items = sorted(s_db_h.items(), reverse=True, key=keyfn)
tablehtml = ""
for contestant, history in sorted_items:
rowhtml = templates.get("contestants/index_row") \
.replace("__NAME__", s_db_c[contestant][0]["name"]) \
.replace("__USER_ID__", contestant) \
.replace("__GOLD__", str(history["G"])) \
.replace("__SILVER__", str(history["S"])) \
.replace("__BRONZE__", str(history["B"])) \
.replace("__HONOURABLE_MENTION__", str(history["H"])) \
.replace("__PARTICIPATIONS__", str(sum(history.values())))
tablehtml += rowhtml
html = html.replace("__TABLE__", tablehtml)
html = templates.final_replace(html, "..")
util.writefile("../dest/contestants/index.html", html)
def create_comparison(mtg1,
mtg2,
maxdistance=None,
cacheprefix=None,
cachepath=get_shared_data('comparison')):
set1 = mtg1.vertices(scale=2)
set2 = mtg2.vertices(scale=2)
# bbox properties
bbxprop1 = mtg1.property('bbox')
bbxprop2 = mtg2.property('bbox')
# skeleton
skeletonprop1 = mtg1.property('skeleton')
skeletonprop2 = mtg2.property('skeleton')
# size
sizeprop1 = mtg1.property('size')
sizeprop2 = mtg2.property('size')
# max distance
if maxdistance is None:
# take second element to avoid appended trunk at the begining
firstelem = mtg1.roots_iter(scale=2).next()
secondelem = mtg1.children_iter(firstelem).next()
skel1 = skeletonprop1[secondelem]
skel1length = Polyline(skel1).getLength()
nbsegments = (len(skel1) - 1)
maxdistance = 2 * skel1length / nbsegments
print 'Use max distance :', maxdistance, (skel1length, nbsegments)
# Compute edge
from time import clock
start = clock()
print 'Create edges of the bipartite graph'
# edges = create_edges(bbxprop1,bbxprop2,skeletonprop1,skeletonprop2,maxdistance)
edges = create_edges_with_space_division(bbxprop1, bbxprop2, skeletonprop1,
skeletonprop2, maxdistance)
nbedges = len(edges)
print 'Number of edge and Average degree:', nbedges, ',', nbedges / len(
bbxprop1)
print "Time to compute edges is : ", clock() - start
if cacheprefix:
edge_cache_file = os.path.join(cachepath, cacheprefix + '_edges.pkl')
writefile(edge_cache_file, edges)
use_max_size = False
if use_max_size:
# Compute global bounding box
globalbbox = compute_global_bbx(bbxprop2)
maxsize = norm(globalbbox.getSize())
print 'Use max size as non matching cost :', maxsize
delset1cost = [maxsize for i in set1]
delset2cost = [maxsize for i in set2]
else:
delset1cost = [sizeprop1[i] for i in set1]
delset2cost = [sizeprop2[i] for i in set2]
# return BipartiteMatching
return BipartiteMatching(set1, set2, edges, delset1cost, delset2cost)
def check_user_grant():
"verify that user-grant and its kubeconfigs work"
config = configuration.get_config()
# because we don't yet have load balancing, we have to somehow get *inside the cluster* to test this.
# that means figuring out the IP address for the user-grant service, uploading the local user cert to the master
# node, and then authenticating to user-grant via curl on the master node. bluh.
# TODO: once load balancing is ready, make this whole thing much simpler
# we use a master node so we're confident we aren't connecting to the node where user-grant is hosted. there's
# nothing about this that otherwise requires it; usually we'd choose a worker node to avoid running unnecessary code
# on the master nodes, but this is entirely for testing in non-production clusters, so it doesn't matter.
proxy_node = config.get_any_node("master")
service_ip = get_service_ip("user-grant")
user_key, user_cert = authority.get_local_grant_user_paths()
remote_key, remote_cert = "/etc/homeworld/testing/usergrant.key", "/etc/homeworld/testing/usergrant.pem"
ssh.check_ssh(proxy_node, "rm", "-f", remote_key, remote_cert)
ssh.check_ssh(proxy_node, "mkdir", "-p", "/etc/homeworld/testing")
ssh.check_scp_up(proxy_node, user_key, remote_key)
ssh.check_scp_up(proxy_node, user_cert, remote_cert)
setup.modify_temporary_dns(proxy_node,
{config.user_grant_domain: service_ip})
try:
kubeconfig = ssh.check_ssh_output(
proxy_node, "curl", "--key", remote_key, "--cert", remote_cert,
"https://%s/" % config.user_grant_domain).decode()
finally:
setup.modify_temporary_dns(proxy_node, {})
magic_phrase = "it allows authenticating to the Hyades cluster as you"
if magic_phrase not in kubeconfig:
command.fail(
"invalid kubeconfig: did not see phrase " + repr(magic_phrase),
"kubeconfig received read as follows: " + repr(kubeconfig))
print("successfully retrieved kubeconfig from user-grant!")
# at this point, we have a kubeconfig generated by user-grant, and now we want to confirm that it works.
# we'll confirm that the kubeconfig works by checking that the auto-created rolebinding passes the sniff test.
with tempfile.TemporaryDirectory() as workdir:
kubeconfig_path = os.path.join(workdir, "granted-kubeconfig")
util.writefile(kubeconfig_path, kubeconfig.encode())
rolebinding = json.loads(
subprocess.check_output([
"hyperkube", "kubectl", "--kubeconfig", kubeconfig_path, "-o",
"json", "get", "rolebindings",
"auto-grant-" + authority.UPSTREAM_USER_NAME
]).decode())
if rolebinding.get("roleRef", {}).get("name") != "admin":
command.fail("rolebinding for user was not admin in %s" %
repr(rolebinding))
print("autogenerated rolebinding for user",
repr(authority.UPSTREAM_USER_NAME), "passed basic check!")
def run():
print("Creating search")
util.makedirs("../dest/search")
util.copyfile("templates/search/search.js", "../dest/search/search.js")
html = templates.get("search/index")
html = templates.initial_replace(html, 3)
html = templates.final_replace(html, "..")
util.writefile("../dest/search/index.html", html)
def run():
print("Creating 404")
html = templates.get("404")
html = templates.initial_replace(html, -1)
# This can't both work local and Github :/
# but it only works on Github anyways
html = templates.final_replace(html, "")
util.writefile("../404.html", html)
def run(code):
print("Creating countries/" + code + "/individual")
html = templates.get("countries/code/individual")
html = templates.initial_replace(html, 2)
html = html.replace("__CODE__", code)
html = html.replace("__COUNTRY__", c_db_c[code]["country"])
if code in previous_code:
html = html.replace("__PREVIOUS_CODE__", previous_code[code])
html = html.replace("__PREVIOUS_CODE_STYLE__", "")
else:
html = html.replace("__PREVIOUS_CODE_STYLE__", "display: none;")
html = html.replace("__PREVIOUS_CODE__", ".") # Google crawler fix
if code in next_code:
html = html.replace("__NEXT_CODE__", next_code[code])
html = html.replace("__NEXT_CODE_STYLE__", "")
else:
html = html.replace("__NEXT_CODE_STYLE__", "display: none;")
html = html.replace("__NEXT_CODE__", ".") # Google crawler fix
tablehtml = ""
if code in s_db_c:
yearhtml = ""
lastyear = ""
for studentdata in s_db_c[code]:
rowhtml = templates.get("countries/code/individual_row")
rowhtml = rowhtml.replace("__NAME__", studentdata["name"])
rowhtml = rowhtml.replace("__RANK__", ("≥" if studentdata["rank>="] else "") + studentdata["rank"])
rowhtml = rowhtml.replace("__YEAR__", studentdata["year"])
if studentdata["medal"] == "G":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("countries/code/individual_gold"))
elif studentdata["medal"] == "S":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("countries/code/individual_silver"))
elif studentdata["medal"] == "B":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("countries/code/individual_bronze"))
elif studentdata["medal"] == "H":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("countries/code/individual_honourable"))
else:
rowhtml = rowhtml.replace("__MEDAL__", "")
if lastyear == studentdata["year"]:
rowhtml = rowhtml.replace("__CLASS__", "")
yearhtml += rowhtml
else:
lastyear = studentdata["year"]
# reverse ordered:
tablehtml = yearhtml + tablehtml
rowhtml = rowhtml.replace("__CLASS__", "doubleTopLine")
yearhtml = rowhtml
# Hacky way of removing first double top line:
yearhtml = yearhtml.replace("doubleTopLine", "", 1)
tablehtml = yearhtml + tablehtml
html = html.replace("__TABLE__", tablehtml)
html = templates.final_replace(html, "../..")
util.writefile("../countries/" + code + "/individual.html", html)
def _matcheol(file, origfile):
"Convert EOL markers in a file to match origfile"
tostyle = _eoltype(util.readfile(origfile))
if tostyle:
data = util.readfile(file)
style = _eoltype(data)
if style:
newdata = data.replace(style, tostyle)
if newdata != data:
util.writefile(file, newdata)
def call_keyreq(command, *params, collect=False):
config = configuration.Config.load_from_project()
keyserver_domain = config.keyserver.hostname + "." + config.external_domain + ":20557"
invoke_variant = subprocess.check_output if collect else subprocess.check_call
with tempfile.TemporaryDirectory() as tdir:
https_cert_path = os.path.join(tdir, "server.pem")
util.writefile(https_cert_path, authority.get_pubkey_by_filename("./server.pem"))
return invoke_variant(["keyreq", command, https_cert_path, keyserver_domain] + list(params))
def _matcheol(file, origfile):
"Convert EOL markers in a file to match origfile"
tostyle = _eoltype(util.readfile(origfile))
if tostyle:
data = util.readfile(file)
style = _eoltype(data)
if style:
newdata = data.replace(style, tostyle)
if newdata != data:
util.writefile(file, newdata)
def upload_bytes() -> None:
# tempfile.TemporaryDirectory() creates the directory with 0o600, which protects the data if it's sensitive
with tempfile.TemporaryDirectory() as scratchdir:
scratchpath = os.path.join(scratchdir, "scratch")
util.writefile(scratchpath, source_bytes)
subprocess.check_call([
"scp", "-o", "StrictHostKeyChecking=yes", "-o",
"ConnectTimeout=1", "--", scratchpath, dest_rel
])
os.remove(scratchpath)
def run():
print("Creating search")
util.makedirs("../search")
util.copyfile("database/countries.csv", "../search/countries.csv")
util.copyfile("database/estudiantes.csv", "../search/estudiantes.csv")
util.copyfile("templates/search/search.js", "../search/search.js")
util.copyfile("templates/search/asciify.js", "../search/asciify.js")
html = templates.get("search/index")
html = templates.initial_replace(html, 3)
html = templates.final_replace(html, "..")
util.writefile("../search/index.html", html)
def gen_kube_specs(output_dir: str) -> None:
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
clustered, = keycheck(yaml.safe_load(
resource.get_resource("clustered/list.yaml")),
"clustered",
validator=lambda _, x: type(x) == list)
vars = get_kube_spec_vars()
for configname in clustered:
templated = template.template("clustered/%s" % configname, vars)
util.writefile(os.path.join(output_dir, configname),
templated.encode())
def create_comparison(mtg1,mtg2,maxdistance = None, cacheprefix = None, cachepath = get_shared_data('comparison')):
set1 = mtg1.vertices(scale=2)
set2 = mtg2.vertices(scale=2)
# bbox properties
bbxprop1 = mtg1.property('bbox')
bbxprop2 = mtg2.property('bbox')
# skeleton
skeletonprop1 = mtg1.property('skeleton')
skeletonprop2 = mtg2.property('skeleton')
# size
sizeprop1 = mtg1.property('size')
sizeprop2 = mtg2.property('size')
# max distance
if maxdistance is None:
# take second element to avoid appended trunk at the begining
firstelem = mtg1.roots_iter(scale=2).next()
secondelem = mtg1.children_iter(firstelem).next()
skel1 = skeletonprop1[secondelem]
skel1length = Polyline(skel1).getLength()
nbsegments = (len(skel1)-1)
maxdistance = 2 * skel1length / nbsegments
print 'Use max distance :',maxdistance,(skel1length , nbsegments)
# Compute edge
from time import clock
start = clock()
print 'Create edges of the bipartite graph'
# edges = create_edges(bbxprop1,bbxprop2,skeletonprop1,skeletonprop2,maxdistance)
edges = create_edges_with_space_division(bbxprop1,bbxprop2,skeletonprop1,skeletonprop2,maxdistance)
nbedges = len(edges)
print 'Number of edge and Average degree:',nbedges,',',nbedges/len(bbxprop1)
print "Time to compute edges is : ", clock() - start
if cacheprefix:
edge_cache_file = os.path.join(cachepath,cacheprefix+'_edges.pkl')
writefile(edge_cache_file,edges)
use_max_size = False
if use_max_size:
# Compute global bounding box
globalbbox = compute_global_bbx(bbxprop2)
maxsize = norm(globalbbox.getSize())
print 'Use max size as non matching cost :', maxsize
delset1cost = [maxsize for i in set1]
delset2cost = [maxsize for i in set2]
else:
delset1cost = [sizeprop1[i] for i in set1]
delset2cost = [sizeprop2[i] for i in set2]
# return BipartiteMatching
return BipartiteMatching(set1,set2,edges,delset1cost,delset2cost)
def run(year):
print("Creating timeline/" + year + "/individual")
html = templates.get("timeline/year/individual")
html = templates.initial_replace(html, 1)
yeardata = t_db_y[year]
html = html.replace("__YEAR__", year)
html = html.replace("__NUMBER__", yeardata["number"])
html = html.replace("__ORDINAL__", util.ordinal(yeardata["number"]))
if year in previous_year:
html = html.replace("__PREVIOUS_YEAR__", previous_year[year])
html = html.replace("__PREVIOUS_YEAR_STYLE__", "")
else:
html = html.replace("__PREVIOUS_YEAR_STYLE__", "display: none;")
html = html.replace("__PREVIOUS_YEAR__", ".") # Google crawler fix
if year in next_year:
html = html.replace("__NEXT_YEAR__", next_year[year])
html = html.replace("__NEXT_YEAR_STYLE__", "")
else:
html = html.replace("__NEXT_YEAR_STYLE__", "display: none;")
html = html.replace("__NEXT_YEAR__", ".") # Google crawler fix
tablehtml = ""
if year in s_db_y:
for row in s_db_y[year]:
rowhtml = templates.get("timeline/year/individual_row")
if row["code"] == "":
rowhtml = rowhtml.replace("__CODE__", "TUR") # Yup, this is my hack
rowhtml = rowhtml.replace("__COUNTRY__", "")
else:
rowhtml = rowhtml.replace("__CODE__", row["code"])
rowhtml = rowhtml.replace("__COUNTRY__", code_to_country[row["code"]])
rowhtml = rowhtml.replace("__NAME__", row["name"])
rowhtml = rowhtml.replace("__RANK__", ("≥" if row["rank>="] else "") + row["rank"])
if row["medal"] == "G":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("timeline/year/individual_gold"))
elif row["medal"] == "S":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("timeline/year/individual_silver"))
elif row["medal"] == "B":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("timeline/year/individual_bronze"))
elif row["medal"] == "H":
rowhtml = rowhtml.replace("__MEDAL__", templates.get("timeline/year/individual_honourable"))
else:
rowhtml = rowhtml.replace("__MEDAL__", "")
tablehtml += rowhtml
html = html.replace("__TABLE__", tablehtml)
html = templates.final_replace(html, "../..")
util.writefile("../timeline/" + year + "/individual.html", html)
def launch_spec(path, extra_kvs: dict = None, export=False):
config = configuration.get_config()
spec = configuration.get_single_kube_spec(path, extra_kvs).encode()
assert path[:2] == '//'
yamlname = path[2:].replace(":", "-")
if export:
util.writefile(yamlname, spec)
else:
for node in config.nodes:
if node.kind == "supervisor":
ssh.check_ssh(node, "mkdir", "-p", DEPLOYQUEUE)
ssh.upload_bytes(
node, spec,
"%s/%f.%s" % (DEPLOYQUEUE, time.time(), yamlname))
print("Uploaded spec to deployqueue.")
def call_kubectl(params, return_result: bool):
kubeconfig_data = configuration.get_local_kubeconfig()
key_path, cert_path, ca_path = get_kube_cert_paths()
if needs_rotate(cert_path):
print("rotating kubernetes certs...")
call_keyreq("kube-cert", key_path, cert_path, ca_path)
with tempfile.TemporaryDirectory() as f:
kubeconfig_path = os.path.join(f, "temp-kubeconfig")
util.writefile(kubeconfig_path, kubeconfig_data.encode())
args = ["hyperkube", "kubectl", "--kubeconfig", kubeconfig_path] + list(params)
if return_result:
return subprocess.check_output(args)
else:
subprocess.check_call(args)
def run():
print("Creating countries/index")
html = templates.get("countries/index")
html = templates.initial_replace(html, 2)
tablehtml = ""
for row in c_db:
rowhtml = templates.get("countries/index_row")
rowhtml = rowhtml.replace("__CODE__", row["code"])
rowhtml = rowhtml.replace("__COUNTRY__", code_to_country[row["code"]])
if row["website"] != "":
rowhtml = rowhtml.replace("__NATIONAL_SITE__", row["website"])
if len(row["website"]) < 50:
rowhtml = rowhtml.replace("__NATIONAL_SITE_TEXT__",
row["website"])
else:
rowhtml = rowhtml.replace("__NATIONAL_SITE_TEXT__",
row["website"][0:35] + "...")
rowhtml = rowhtml.replace("__NATIONAL_SITE_STYLE__", "")
else:
rowhtml = rowhtml.replace("__NATIONAL_SITE_STYLE__",
"display: none;")
if row["code"] in t_db_c:
hosts = ""
flag = False
for year in t_db_c[row["code"]]:
if flag:
hosts += ", "
hosts += templates.get("countries/index_hostyear").replace(
"__YEAR__", year["year"])
flag = True
rowhtml = rowhtml.replace("__HOSTS__", hosts)
else:
rowhtml = rowhtml.replace("__HOSTS__", "")
rowhtml = rowhtml.replace("__CLASS__",
"tr-former" if row["former"] else "")
tablehtml += rowhtml
html = html.replace("__TABLE__", tablehtml)
html = templates.final_replace(html, "..")
util.writefile("../countries/index.html", html)
def call_keyreq(keyreq_command, *params):
config = configuration.get_config()
keyserver_domain = config.keyserver.hostname + "." + config.external_domain + ":20557"
with tempfile.TemporaryDirectory() as tdir:
https_cert_path = os.path.join(tdir, "clusterca.pem")
util.writefile(https_cert_path,
authority.get_pubkey_by_filename("./clusterca.pem"))
keyreq_sp = subprocess.Popen(
["keyreq", keyreq_command, https_cert_path, keyserver_domain] +
list(params),
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, err_bytes = keyreq_sp.communicate()
if keyreq_sp.returncode != 0:
err = err_bytes.decode()
raise KeyreqFailed(keyreq_sp.returncode, err)
return output
def run():
print("Creating timeline/index")
html = templates.get("timeline/index")
html = templates.initial_replace(html, 1)
tablehtml = ""
for row in t_db:
rowhtml = templates.get("timeline/index_row")
rowhtml = rowhtml.replace("__NUMBER__", row["number"])
rowhtml = rowhtml.replace("__MONTH__", row["month"])
rowhtml = rowhtml.replace("__CONTEST_NAME__", row["name"])
rowhtml = rowhtml.replace("__DATE__", row["date"])
rowhtml = rowhtml.replace("__P_STUDENT__", row["p_student"])
tablehtml = rowhtml + tablehtml
html = html.replace("__TABLE__", tablehtml)
html = templates.final_replace(html, "..")
util.writefile("../dest/timeline/index.html", html)
def make_predictions(st_line, ed_line, out_file, pname, models):
cat_count, item_count, item_sort, alpha, beta, item_word, bigram_item_word, time_cat_item_dict, cat_word, hot_words = models[
0]
reader = readfile(new_test_file)
writer = writefile(out_file)
line_idx = 0
for (user, category, raw_query, click_time) in reader:
line_idx += 1
if line_idx < st_line:
continue
if line_idx > ed_line:
break
if line_idx % TEST_STEP == 0:
print '%s--%d' % (pname, line_idx / TEST_STEP)
time_block = get_time_feature(click_time)
try:
bound = cat_count[category][PREDICT_HOT_SIZE]
hots = [x[0] for x in item_sort[category][0:bound]]
except:
writer.writerow(["0"])
continue
try:
bigram = get_bigram_word(raw_query, hot_words, category)
words = get_words(raw_query)
query_size = sum([cat_word[category][w] for w in words])
if query_size >= 100 and len(bigram) > 0:
'only queries hot enough and can generate bigram features can be predicted by boosting model'
rank = [[
sku,
boosting_bayes(bigram, words, category, sku, alpha, beta,
item_word, bigram_item_word, item_count,
cat_count, time_cat_item_dict, time_block)
] for sku in hots]
elif query_size >= 100 and len(bigram) == 0:
'if hot enough but can not generate bigram features then use naive bayes with time information'
rank = [[
sku,
time_bayes_query_prediction(words, category, sku, alpha,
beta, item_word, item_count,
cat_count, time_cat_item_dict,
time_block)
] for sku in hots]
else:
'otherwise use plain naive bayes'
rank = [[
sku,
plain_bayes_query_prediction(words, category, sku, alpha,
beta, item_word, item_count,
cat_count)
] for sku in hots]
rank = sorted(rank, key=lambda x: x[1], reverse=True)
guesses = [i[0] for i in rank[0:5]]
guesses = rerank_guess(guesses, user, raw_query)
writer.writerow([" ".join(guesses)])
except (TypeError, KeyError): # a category we haven't seen before
writer.writerow([" ".join(hots[0:5])])
def update_known_hosts():
# uses local copies of machine list and ssh-host pubkey
# TODO: eliminate now-redundant machine.list download from keyserver
machines = configuration.get_machine_list_file().strip()
cert_authority_pubkey = authority.get_pubkey_by_filename("./ssh_host_ca.pub")
homedir = os.getenv("HOME")
if homedir is None:
command.fail("could not determine home directory, so could not find ~/.ssh/known_hosts")
known_hosts_path = os.path.join(homedir, ".ssh", "known_hosts")
known_hosts_old = util.readfile(known_hosts_path).decode().split("\n") if os.path.exists(known_hosts_path) else []
if known_hosts_old and not known_hosts_old[-1]:
known_hosts_old.pop()
known_hosts_new = _replace_cert_authority(known_hosts_old, machines, cert_authority_pubkey)
util.writefile(known_hosts_path, ("\n".join(known_hosts_new) + "\n").encode())
print("~/.ssh/known_hosts updated")
def run():
print("Creating timeline/index")
html = templates.get("timeline/index")
html = templates.initial_replace(html, 1)
tablehtml = ""
upcominghtml = ""
upcoming_row_ctr = 0
for row in t_db:
rowhtml = templates.get("timeline/index_row")
rowhtml = rowhtml.replace("__NUMBER__", row["number"])
rowhtml = rowhtml.replace("__YEAR__", row["year"])
rowhtml = rowhtml.replace("__DATE__", row["date"])
rowhtml = rowhtml.replace("__CODE__", row["code"])
rowhtml = rowhtml.replace("__CITY__", row["city"])
rowhtml = rowhtml.replace("__COUNTRY__", code_to_country[row["code"]])
rowhtml = rowhtml.replace("__P_COUNTRY__", row["p_country"])
rowhtml = rowhtml.replace("__P_STUDENT__", row["p_student"])
if "code2" in row:
rowhtml = rowhtml.replace("__CODE2__", row["code2"])
rowhtml = rowhtml.replace("__COUNTRY2__",
code_to_country[row["code2"]])
rowhtml = rowhtml.replace("__CODE2_STYLE__", "")
else:
rowhtml = rowhtml.replace("__CODE2_STYLE__", "display: none;")
rowhtml = rowhtml.replace("__CODE2__", ".") # Google crawler fix
if int(row["year"]) <= int(config.next_year) + 2:
# Reverse list
tablehtml = rowhtml + tablehtml
else:
upcominghtml = rowhtml + upcominghtml
upcoming_row_ctr += 1
if int(row["year"]) == 2019:
tablehtml = templates.get("timeline/index_row_2020") + tablehtml
# Append an empty row to preserve row parity between tables for styling purposes
if upcoming_row_ctr % 2:
upcominghtml = "<tr style=\"display:none;\"></tr>" + upcominghtml
html = html.replace("__TABLE__", tablehtml)
html = html.replace("__UPCOMING__", upcominghtml)
html = templates.final_replace(html, "..")
util.writefile("../timeline/index.html", html)
def update_known_hosts():
# uses local copies of machine list and ssh-host pubkey
# TODO: eliminate now-redundant machine.list download from keyserver
machines = configuration.get_machine_list_file().strip()
cert_authority_pubkey = authority.get_pubkey_by_filename(
"./ssh_host_ca.pub")
known_hosts_path = get_known_hosts_path()
known_hosts_old = util.readfile(known_hosts_path).decode().split(
"\n") if os.path.exists(known_hosts_path) else []
if known_hosts_old and not known_hosts_old[-1]:
known_hosts_old.pop()
known_hosts_new = _replace_cert_authority(known_hosts_old, machines,
cert_authority_pubkey)
util.writefile(known_hosts_path,
("\n".join(known_hosts_new) + "\n").encode())
print("~/.ssh/known_hosts updated")
def gen_kube_specs(output_dir: str) -> None:
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
config = Config.load_from_project()
vars = {
"NETWORK": config.cidr_pods,
"SERVIP_DNS": config.service_dns,
"INTERNAL_DOMAIN": config.internal_domain
}
clustered, = keycheck(yaml.safe_load(
resource.get_resource("clustered/list.yaml")),
"clustered",
validator=lambda _, x: type(x) == list)
for configname in clustered:
templated = template.template("clustered/%s" % configname, vars)
util.writefile(os.path.join(output_dir, configname),
templated.encode())
def update_known_hosts():
"update ~/.ssh/known_hosts file with @ca-certificates directive"
config = configuration.Config.load_from_project()
machines = ",".join("%s.%s" % (node.hostname, config.external_domain)
for node in config.nodes)
cert_authority_pubkey = authority.get_pubkey_by_filename("./ssh-host.pub")
known_hosts_path = get_known_hosts_path()
known_hosts_old = util.readfile(known_hosts_path).decode().split(
"\n") if os.path.exists(known_hosts_path) else []
if known_hosts_old and not known_hosts_old[-1]:
known_hosts_old.pop()
known_hosts_new = _replace_cert_authority(known_hosts_old, machines,
cert_authority_pubkey)
util.writefile(known_hosts_path,
("\n".join(known_hosts_new) + "\n").encode())
print("~/.ssh/known_hosts updated")
def generate() -> None:
authorities = get_targz_path(check_exists=False)
if os.path.exists(authorities):
command.fail("authorities.tgz already exists")
# tempfile.TemporaryDirectory() creates the directory with 0o600, which protects the private keys
with tempfile.TemporaryDirectory() as d:
certdir = os.path.join(d, "certdir")
keyserver_yaml = os.path.join(d, "keyserver.yaml")
util.writefile(keyserver_yaml, configuration.get_keyserver_yaml().encode())
os.mkdir(certdir)
print("generating authorities...")
try:
subprocess.check_call(["keygen", keyserver_yaml, certdir, "supervisor-nodes"])
except FileNotFoundError as e:
if e.filename == "keygen":
command.fail("could not find keygen binary. is the homeworld-keyserver dependency installed?")
else:
raise e
print("packing authorities...")
subprocess.check_call(["tar", "-C", certdir, "-czf", authorities, "."])
subprocess.check_call(["shred", "--"] + os.listdir(certdir), cwd=certdir)
def call_keyreq(keyreq_command, *params, collect=False):
config = configuration.get_config()
keyserver_domain = config.keyserver.hostname + "." + config.external_domain + ":20557"
invoke_variant = subprocess.check_output if collect else subprocess.check_call
with tempfile.TemporaryDirectory() as tdir:
https_cert_path = os.path.join(tdir, "server.pem")
util.writefile(https_cert_path,
authority.get_pubkey_by_filename("./server.pem"))
try:
return invoke_variant(
["keyreq", keyreq_command, https_cert_path, keyserver_domain] +
list(params))
except subprocess.CalledProcessError as e:
if e.returncode == 1:
fail_hint = "do you have valid kerberos tickets?\n" \
"or, your connection to the server might be faulty.\n" \
"or, the server's keygateway might be broken."
command.fail("keyreq failed: %s" % e, fail_hint)
raise e
def run():
print("Creating countries/index")
html = templates.get("countries/index")
html = templates.initial_replace(html, 2)
tablehtml = ""
for row in c_db:
rowhtml = templates.get("countries/index_row")
rowhtml = rowhtml.replace("__CODE__", row["code"])
rowhtml = rowhtml.replace("__COUNTRY__", code_to_country[row["code"]])
if row["website"] != "":
rowhtml = rowhtml.replace("__NATIONAL_SITE__", row["website"])
if len(row["website"]) < 50:
rowhtml = rowhtml.replace("__NATIONAL_SITE_TEXT__", row["website"])
else:
rowhtml = rowhtml.replace("__NATIONAL_SITE_TEXT__", row["website"][0:35] + "...")
rowhtml = rowhtml.replace("__NATIONAL_SITE_STYLE__", "")
else:
rowhtml = rowhtml.replace("__NATIONAL_SITE_STYLE__", "display: none;")
if row["code"] in t_db_c:
hosts = ""
flag = False
for year in t_db_c[row["code"]]:
if flag:
hosts += ", "
hosts += templates.get("countries/index_hostyear").replace("__YEAR__", year["year"])
flag = True
rowhtml = rowhtml.replace("__HOSTS__", hosts)
else:
rowhtml = rowhtml.replace("__HOSTS__", "")
rowhtml = rowhtml.replace("__CLASS__", "tr-former" if row["former"] else "")
tablehtml += rowhtml
html = html.replace("__TABLE__", tablehtml)
html = templates.final_replace(html, "..")
util.writefile("../countries/index.html", html)
def ctoxml(filePath, cilPath):
logger = log.setup_custom_logger(__name__)
logger.debug("Using CIL executable located at %s" % (cilPath))
if os.path.isdir(filePath):
for root, dirs, files in os.walk(filePath):
#excludes files that don't end in .c
files = [fi for fi in files if fi.endswith(".c")]
for cfile in files:
fileName = os.path.abspath(os.path.join(root,cfile))
xmlName = "%s.xml" % (os.path.splitext(cfile)[0])
#if the generated XML exists, do not re-run CIL on the C file
if not exists(xmlName):
logger.info("Running CIL on %s" % (fileName))
result, data = util.myrun([cilPath] + [fileName] + cilArgs)
if result:
logger.error("CIL processing of %s failed: %s" % (fileName, data))
break
cXmlName= os.path.join(root,xmlName)
logger.debug("Writing results of CIL processing to %s" % (cXmlName))
util.writefile(cXmlName, data)
else:
result, data = util.myrun([cilPath] + [filePath] + cilArgs)
logger.debug("Writing results of CIL proceesing to %s" % (cXmlName))
til.writefile(cXmlName, data)
def make_predictions(st_line, ed_line, out_file, pname, models):
cat_count, item_count, item_sort, alpha, beta, item_word, bigram_item_word, time_cat_item_dict, cat_word, hot_words = models[0]
reader = readfile(new_test_file)
writer = writefile(out_file)
line_idx = 0
for (user, category, raw_query, click_time) in reader:
line_idx += 1
if line_idx < st_line:
continue
if line_idx > ed_line:
break
if line_idx % TEST_STEP == 0:
print '%s--%d' % (pname, line_idx / TEST_STEP)
time_block = get_time_feature(click_time)
try:
bound = cat_count[category][PREDICT_HOT_SIZE]
hots = [x[0] for x in item_sort[category][0:bound]]
except:
writer.writerow(["0"])
continue
try:
bigram = get_bigram_word(raw_query, hot_words, category)
words = get_words(raw_query)
query_size = sum([cat_word[category][w] for w in words])
if query_size >= 100 and len(bigram) > 0:
'only queries hot enough and can generate bigram features can be predicted by boosting model'
rank = [[sku, boosting_bayes(bigram, words, category, sku, alpha, beta, item_word, bigram_item_word, item_count, cat_count, time_cat_item_dict, time_block)] for sku in hots]
elif query_size >= 100 and len(bigram) == 0:
'if hot enough but can not generate bigram features then use naive bayes with time information'
rank = [[sku, time_bayes_query_prediction(words, category, sku, alpha, beta, item_word, item_count, cat_count, time_cat_item_dict, time_block)] for sku in hots]
else:
'otherwise use plain naive bayes'
rank = [[sku, plain_bayes_query_prediction(words, category, sku, alpha, beta, item_word, item_count, cat_count)] for sku in hots]
rank = sorted(rank, key=lambda x:x[1], reverse=True)
guesses = [i[0] for i in rank[0:5]]
guesses = rerank_guess(guesses, user, raw_query)
writer.writerow([" ".join(guesses)])
except (TypeError, KeyError): # a category we haven't seen before
writer.writerow([" ".join(hots[0:5])])
def comparison_process(ref_mtg,tested_mtg,cacheprefix = None,cachepath = get_shared_data('comparison')):
""" Return score of geometrical comparison (nb of common elements) and topological comparison (nb of common edges) """
from time import clock
print 'mtgs sizes :',ref_mtg.nb_vertices(scale=ref_mtg.max_scale()),tested_mtg.nb_vertices(scale=tested_mtg.max_scale())
print 'make mtgs comparable'
ref_mtg = comparable_mtg(ref_mtg)
tested_mtg = comparable_mtg(tested_mtg)
print 'mtgs sizes :',ref_mtg.nb_vertices(scale=2),tested_mtg.nb_vertices(scale=2)
if cacheprefix:
if not os.path.exists(cachepath): os.makedirs(cachepath)
raw_comparison_cache_file = os.path.join(cachepath,cacheprefix+'_raw_comparison.pkl')
if cacheprefix and os.path.exists(raw_comparison_cache_file) :
print 'Read raw matching from',repr(raw_comparison_cache_file)
raw_comparison = readfile(raw_comparison_cache_file)
else:
print 'Compute matching'
start = clock()
matcher = create_comparison(ref_mtg,tested_mtg,None,cacheprefix,cachepath)
if cacheprefix :
raw_comparison_cache_file_intermediate = os.path.join(cachepath,cacheprefix+'_raw_comparison_intermediate.pkl')
else : raw_comparison_cache_file_intermediate = None
raw_comparison = matcher.match(raw_comparison_cache_file_intermediate)
print ">>>>>>>> The time for computing matching is : ", clock() - start , " <<<<<<<<. "
if cacheprefix : writefile(raw_comparison_cache_file,raw_comparison)
distance, mapping, nonmapping1, nonmapping2 = raw_comparison
print 'Nb of mapping :',len(mapping), len(nonmapping1), len(nonmapping2)
print 'Distance :',distance
nbtotvertex = ref_mtg.nb_vertices(scale=2)+tested_mtg.nb_vertices(scale=2)
nbmapped = 2.*len(mapping)
score = nbmapped/float(nbtotvertex)
print 'Mapping score :',score
ref_sizes = ref_mtg.property('size')
tested_sizes = tested_mtg.property('size')
totlength = sum(ref_sizes.itervalues())+sum(tested_sizes.itervalues())
matchedlength = sum([ref_sizes[i] for i,j in mapping])+sum([tested_sizes[j] for i,j in mapping])
cscore = matchedlength/totlength
print 'Weighted mapping score :',cscore
# Extended comparison
with_extension = False
if with_extension:
ext_comparison = extend_comparison(ref_mtg,tested_mtg,mapping, nonmapping1, nonmapping2)
newmapping, newnonmapping1, newnonmapping2 = ext_comparison
assert len(newmapping) == len(mapping)
nbnewmapped = nbtotvertex-len(newnonmapping1)-len(newnonmapping2)
print 'Nb element added to the mapping :',nbnewmapped-nbmapped
nscore = nbnewmapped/float(nbtotvertex)
print 'New mapping score :',nscore
nweunmatchedlength = sum([ref_sizes[i] for i in newnonmapping1])+sum([tested_sizes[i] for i in newnonmapping2])
newcscore = (totlength-nweunmatchedlength)/totlength
print 'New Corrected mapping score :',newcscore
if cacheprefix:
ext_comparison_cache_file = os.path.join(cachepath,cacheprefix+'_extended_comparison.pkl')
writefile(ext_comparison_cache_file,ext_comparison)
# Second visualization
# # scene = comparison_representation(ref_mtg,tested_mtg,newmapping)
# scene = ext_comparison_representation(ref_mtg,tested_mtg,mapping,newmapping)
# Viewer.display(scene)
else:
newmapping, newnonmapping1, newnonmapping2 = mapping, nonmapping1, nonmapping2
newcscore = cscore
# Topological comparison
edgematching ,nonedgematching1,nonedgematching2 = topological_comparison(ref_mtg,tested_mtg,newmapping, newnonmapping1, newnonmapping2)
print map(len,[edgematching,nonedgematching1,nonedgematching2])
tscore = (2*len(edgematching))/float(sum(map(len,[edgematching,edgematching,nonedgematching1,nonedgematching2])))
print 'Topological comparison score',tscore
return newcscore, tscore
def run(year):
print("Creating timeline/" + year + "/country")
html = templates.get("timeline/year/country")
html = templates.initial_replace(html, 1)
yeardata = t_db_y[year]
html = html.replace("__YEAR__", year)
html = html.replace("__NUMBER__", yeardata["number"])
html = html.replace("__ORDINAL__", util.ordinal(yeardata["number"]))
if year in previous_year:
html = html.replace("__PREVIOUS_YEAR__", previous_year[year])
html = html.replace("__PREVIOUS_YEAR_STYLE__", "")
else:
html = html.replace("__PREVIOUS_YEAR_STYLE__", "display: none;")
html = html.replace("__PREVIOUS_YEAR__", ".") # Google crawler fix
if year in next_year:
html = html.replace("__NEXT_YEAR__", next_year[year])
html = html.replace("__NEXT_YEAR_STYLE__", "")
else:
html = html.replace("__NEXT_YEAR_STYLE__", "display: none;")
html = html.replace("__NEXT_YEAR__", ".") # Google crawler fix
medals = {}
if year in s_db_y:
for row in s_db_y[year]:
if row["code"] == "":
# Country unknown
continue
if row["code"] not in medals:
medals[row["code"]] = {
"bestrank": int(row["rank"]),
"bestrank>=": "≥" if row["rank>="] else "",
"gold": 0,
"silver": 0,
"bronze": 0,
"honourable": 0
}
if row["medal"] == "G":
medals[row["code"]]["gold"] += 1
elif row["medal"] == "S":
medals[row["code"]]["silver"] += 1
elif row["medal"] == "B":
medals[row["code"]]["bronze"] += 1
elif row["medal"] == "H":
medals[row["code"]]["honourable"] += 1
def keyfn(code):
m = medals[code]
return (m["gold"], m["silver"], m["bronze"], m["honourable"],
-m["bestrank"])
sortedcodes = reversed(sorted(medals, key = keyfn))
tablehtml = ""
prevcode = ""
prevrank = 0
for i, code in enumerate(sortedcodes):
rowhtml = templates.get("timeline/year/country_row")
rowhtml = rowhtml.replace("__CODE__", code)
rowhtml = rowhtml.replace("__COUNTRY__", code_to_country[code])
if prevcode != "" and keyfn(prevcode) == keyfn(code):
rowhtml = rowhtml.replace("__RANK__", prevrank)
else:
rowhtml = rowhtml.replace("__RANK__", str(i + 1))
prevcode = code
prevrank = str(i + 1)
rowhtml = rowhtml.replace("__GOLD__", str(medals[code]["gold"]))
rowhtml = rowhtml.replace("__SILVER__", str(medals[code]["silver"]))
rowhtml = rowhtml.replace("__BRONZE__", str(medals[code]["bronze"]))
rowhtml = rowhtml.replace("__HONOURABLE__", str(medals[code]["honourable"]))
rowhtml = rowhtml.replace("__BEST_RANK__", medals[code]["bestrank>="] + str(medals[code]["bestrank"]))
tablehtml += rowhtml
html = html.replace("__TABLE__", tablehtml)
html = templates.final_replace(html, "../..")
util.writefile("../timeline/" + year + "/country.html", html)
if os.path.exists(roothg):
raise util.Abort(_('destination already exists'))
if not os.path.isdir(root):
os.mkdir(root)
util.makedir(roothg, notindexed=True)
requirements = ''
try:
requirements = srcrepo.opener.read('requires')
except IOError, inst:
if inst.errno != errno.ENOENT:
raise
requirements += 'shared\n'
util.writefile(os.path.join(roothg, 'requires'), requirements)
util.writefile(os.path.join(roothg, 'sharedpath'), sharedpath)
r = repository(ui, root)
default = srcrepo.ui.config('paths', 'default')
if default:
fp = r.opener("hgrc", "w", text=True)
fp.write("[paths]\n")
fp.write("default = %s\n" % default)
fp.close()
if update:
r.ui.status(_("updating working directory\n"))
if update is not True:
checkout = update
def callfun(self, type, bufstr, md5str):
util.writefile(md5str, bufstr)
def run(year):
print("Creating timeline/" + year + "/index")
html = templates.get("timeline/year/index")
html = templates.initial_replace(html, 1)
yeardata = t_db_y[year]
html = html.replace("__YEAR__", year)
html = html.replace("__NUMBER__", yeardata["number"])
html = html.replace("__ORDINAL__", util.ordinal(yeardata["number"]))
html = html.replace("__DATE__", yeardata["date"])
html = html.replace("__CODE__", yeardata["code"])
html = html.replace("__COUNTRY__", code_to_country[yeardata["code"]])
if "code2" in yeardata:
html = html.replace("__CODE2__", yeardata["code2"])
html = html.replace("__COUNTRY2__", code_to_country[yeardata["code2"]])
html = html.replace("__CODE2_STYLE__", "")
else:
html = html.replace("__CODE2_STYLE__", "display: none;")
html = html.replace("__CODE2__", ".") # Google crawler fix
if yeardata["city"] != "":
html = html.replace("__CITY__", yeardata["city"] + ",")
else:
html = html.replace("__CITY__", "")
if year in previous_year:
html = html.replace("__PREVIOUS_YEAR__", previous_year[year])
html = html.replace("__PREVIOUS_YEAR_STYLE__", "")
else:
html = html.replace("__PREVIOUS_YEAR_STYLE__", "display: none;")
html = html.replace("__PREVIOUS_YEAR__", ".") # Google crawler fix
if year in next_year:
html = html.replace("__NEXT_YEAR__", next_year[year])
html = html.replace("__NEXT_YEAR_STYLE__", "")
else:
html = html.replace("__NEXT_YEAR_STYLE__", "display: none;")
html = html.replace("__NEXT_YEAR__", ".") # Google crawler fix
if yeardata["p_student"] != "":
html = html.replace("__P_STUDENT_STYLE__", "")
html = html.replace("__P_STUDENT__", yeardata["p_student"])
else:
html = html.replace("__P_STUDENT_STYLE__", "display: none;")
if yeardata["p_country"] != "":
html = html.replace("__P_COUNTRY_STYLE__", "")
html = html.replace("__P_COUNTRY__", yeardata["p_country"])
else:
html = html.replace("__P_COUNTRY_STYLE__", "display: none;")
if yeardata["homepage"] != "":
html = html.replace("__HOMEPAGE_STYLE__", "")
html = html.replace("__HOMEPAGE__", yeardata["homepage"])
else:
html = html.replace("__HOMEPAGE_STYLE__", "display: none;")
html = html.replace("__HOMEPAGE__", ".") # Google crawler fix
gold = 0
silver = 0
bronze = 0
honourable = 0
if year in s_db_y:
for studentdata in s_db_y[year]:
if studentdata["medal"] == "G":
gold += 1
elif studentdata["medal"] == "S":
silver += 1
elif studentdata["medal"] == "B":
bronze += 1
elif studentdata["medal"] == "H":
honourable += 1
html = html.replace("__AWARDS_STYLE__", "")
html = html.replace("__GOLD__", str(gold))
html = html.replace("__SILVER__", str(silver))
html = html.replace("__BRONZE__", str(bronze))
html = html.replace("__HONOURABLE__", str(honourable))
else:
html = html.replace("__AWARDS_STYLE__", "display: none;")
html = templates.final_replace(html, "../..")
util.writefile("../timeline/" + year + "/index.html", html)
def run():
print("Creating 404")
html = templates.get("404")
html = templates.initial_replace(html, -1)
html = templates.final_replace(html, ".")
util.writefile("../404.html", html)
def run(code):
print("Creating countries/" + code + "/index")
html = templates.get("countries/code/index")
html = templates.initial_replace(html, 2)
codedata = c_db_c[code]
html = html.replace("__CODE__", code)
html = html.replace("__COUNTRY__", codedata["country"])
if codedata["website"] != "":
html = html.replace("__CONTACT_STYLE__", "")
html = html.replace("__NATIONAL_SITE__", codedata["website"])
if len(codedata["website"]) < 50:
html = html.replace("__NATIONAL_SITE_TEXT__", codedata["website"])
else:
html = html.replace("__NATIONAL_SITE_TEXT__", codedata["website"][0:50] + "...")
else:
html = html.replace("__CONTACT_STYLE__", "display: none;")
html = html.replace("__NATIONAL_SITE__", ".") # Google crawler fix
if code in previous_code:
html = html.replace("__PREVIOUS_CODE__", previous_code[code])
html = html.replace("__PREVIOUS_CODE_STYLE__", "")
else:
html = html.replace("__PREVIOUS_CODE_STYLE__", "display: none;")
html = html.replace("__PREVIOUS_CODE__", ".") # Google crawler fix
if code in next_code:
html = html.replace("__NEXT_CODE__", next_code[code])
html = html.replace("__NEXT_CODE_STYLE__", "")
else:
html = html.replace("__NEXT_CODE_STYLE__", "display: none;")
html = html.replace("__NEXT_CODE__", ".") # Google crawler fix
if code in t_db_c:
hostshtml = ""
for yeardata in t_db_c[code]:
hosthtml = templates.get("countries/code/index_host")
if yeardata["city"] != "":
hosthtml = hosthtml.replace("__CITY__", " - " + yeardata["city"])
else:
hosthtml = hosthtml.replace("__CITY__", "")
if yeardata["homepage"] != "":
homepagehtml = templates.get("countries/code/index_host_homepage")
homepagehtml = homepagehtml.replace("__LINK__", yeardata["homepage"])
hosthtml = hosthtml.replace("__HOMEPAGE__", homepagehtml)
else:
hosthtml = hosthtml.replace("__HOMEPAGE__", "")
hosthtml = hosthtml.replace("__YEAR__", yeardata["year"])
hostshtml += hosthtml
html = html.replace("__HOST__", "<dt>IPhO Host</dt>" + hostshtml)
else:
html = html.replace("__HOST__", "")
gold = 0
silver = 0
bronze = 0
honourable = 0
if code in s_db_c:
for studentdata in s_db_c[code]:
if studentdata["medal"] == "G":
gold += 1
elif studentdata["medal"] == "S":
silver += 1
elif studentdata["medal"] == "B":
bronze += 1
elif studentdata["medal"] == "H":
honourable += 1
html = html.replace("__GOLD__", str(gold))
html = html.replace("__SILVER__", str(silver))
html = html.replace("__BRONZE__", str(bronze))
html = html.replace("__HONOURABLE__", str(honourable))
html = templates.final_replace(html, "../..")
util.writefile("../countries/" + code + "/index.html", html)
def run():
print("Creating index")
html = templates.get("index")
html = templates.initial_replace(html, 0)
html = templates.final_replace(html, ".")
util.writefile("../index.html", html)
def create(name):
util.makedirs("../" + name + ".php")
html = templates.get("backward_compatibility/" + name)
html = templates.final_replace(html, "..")
util.writefile("../" + name + ".php/index.html", html)
