def r_events_files(infiles):
fmt = formatter.formatter()
db = dbinserter.inserter()
tt_dict_type_num = {}
for f in infiles:
#Apr.1-May.2
d_t_n = r_events_file(f, fmt, db, 1427817600, 1430536956)
for etype, num in d_t_n.items():
#print etype, num
if etype not in tt_dict_type_num.keys():
tt_dict_type_num[etype] = 0
tt_dict_type_num[etype] += num
db.close()
return tt_dict_type_num
def build(plugin):
actions = []
attributes = []
for attribute in plugin.text_attributes:
attributes += [PLUGIN_ATTRIBUTE.format(**attribute)]
for action in plugin.actions:
action_attributes = []
for attribute, value in action.text_attributes:
value = formatter(value)
action_attributes += [ACTION_ATTRIBUTE.format(**attribute)]
actions += [
ACTION.format(name=action.name,
description=action.description,
attributes=''.join(action_attributes))
]
def test_formatting3():
assert_equals(formatter("{}, {blah:d}", "k", blah=1), "k, 1")
def test_formatter2():
assert_equals(formatter("{}, {}", "c", "b"), "c, b")
def formatter(self, topic, opts):
return formatter.formatter(self, topic, opts)
import pandas as pd
import time
from ast import literal_eval
import numpy as np
from sklearn import preprocessing
from multiprocessing import pool
from functools import partial
import sys
import time
import pickle
matchID = 'E3G2nI5H'
start = time.time()
sys.path.insert(1, './')
from formatter import formatter
k = formatter(matchID)
k.to_csv('./csv/asdfasdf.csv', index=False)
print(k.shape)
r, l = 'Pandur I.', 'Mishakin V.'
df = pd.read_csv('./csv/working.csv')
df = df[(df['Player_left'] == l) | (df['Player_left'] == r) |
(df['Player_right'] == l) | (df['Player_right'] == r)]
print(df.shape)
print(k.eq(df))
# print(df[['datetime_left', 'datetime_right', 'lastScore_ave_left', 'lastScore_ave_right']])
# df.shape
# df[df['id'] == '6civ8qu0'][['Player_left', 'Player_right']]
# df = df[(df['Player_left'] == 'Mishakin V.') | (df['Player_left'] == 'Belugin 0.') | (df['Player_right'] == 'Mishakin V.') | (df['Player_right'] == 'Belugin 0.') ]
# print(df.shape, k.shape)
from ast import literal_eval
import numpy as np
from sklearn import preprocessing
from multiprocessing import pool
from functools import partial
import sys
import time
import pickle
sys.path.insert(1, './')
from formatter import formatter
start = time.time()
df = pd.read_csv('./matchDF.csv')
Player = 'Mishakin V.'
df = df[(df['lPlayer'] == Player) | (df['rPlayer'] == Player)]
lenargs = len(sys.argv)
if lenargs == 1:
df = formatter(df)
elif lenargs == 2:
if sys.argv[1] == 'test':
df = formatter(df[0:2000], True)
else:
print('not recognized')
elif lenargs == 3:
if sys.argv[1] == 'test':
df = formatter(df[0:int(sys.argv[2])], True)
else:
print('not recognized')
df.to_csv('./csv/working_subsection.csv', index=False)
print(time.time() - start)
def parser(action, text, cmdname, source, fmt, trace, igc, test):
ttid = 0
NEXT = []
SCOPE = []
branch = []
BRANCHES = []
oneliner = -1
chain = []
CCHAINS = []
ubids = []
FLAGS = {}
flag = {}
setting = []
SETTINGS = []
variable = []
VARIABLES = []
USED_VARS = {}
USER_VARS = {}
VARSTABLE = builtins(cmdname)
vindices = {}
(tokens, ttypes, ttids, dtids) = tokenizer(text)
i = 0
l = len(tokens)
S = {
"tid": -1,
"filename": source,
"text": text,
"LINESTARTS": LINESTARTS,
"tokens": tokens,
"ttypes": ttypes,
"ttids": ttids,
"dtids": dtids,
"args": {
"action": action,
"source": source,
"fmt": fmt,
"trace": trace,
"igc": igc,
"test": test,
},
"ubids": ubids,
"excludes": [],
"warnings": {},
"warn_lines": set(),
"warn_lsort": set()
}
def err(tid, message, pos="start", scope=""):
# When token ID points to end-of-parsing token,
# reset the id to the last true token before it.
if tokens[tid]["kind"] == "tkEOP": tid = ttids[-1]
token = tokens[tid]
line = token["line"]
index = token[pos]
# msg = f"{message}"
col = index - LINESTARTS[line]
if message.endswith(":"): message += f" '{tkstr(tid)}'"
# # Add token debug information.
# dbeugmsg = "\n\n\033[1mToken\033[0m: "
# dbeugmsg += "\n - tid: " + str(token["tid"])
# dbeugmsg += "\n - kind: " + token["kind"]
# dbeugmsg += "\n - line: " + str(token["line"])
# dbeugmsg += "\n - start: " + str(token["start"])
# dbeugmsg += "\n - end: " + str(token["end"])
# dbeugmsg += "\n __val__: [" + tkstr(tid) + "]"
# dbeugmsg += "\n\n\033[1mExpected\033[0m: "
# for n in NEXT:
# if not n: n = "\"\""
# dbeugmsg += "\n - " + n
# dbeugmsg += "\n\n\033[1mScopes\033[0m: "
# for s in SCOPE:
# dbeugmsg += "\n - " + s
# decor = "-" * 15
# msg += "\n\n" + decor + " TOKEN_DEBUG_INFO " + decor
# msg += dbeugmsg
# msg += "\n\n" + decor + " TOKEN_DEBUG_INFO " + decor
Issue().error(S["filename"], line, col, message)
def warn(tid, message):
token = tokens[tid]
line = token["line"]
index = token["start"]
col = index - LINESTARTS[line]
if message.endswith(":"): message += f" '{tkstr(tid)}'"
if line not in S["warnings"]: S["warnings"][line] = []
S["warnings"][line].append([S["filename"], line, col, message])
S["warn_lines"].add(line)
def hint(tid, message):
token = tokens[tid]
line = token["line"]
index = token["start"]
col = index - LINESTARTS[line]
if message.endswith(":"): message += f" '{tkstr(tid)}'"
Issue().hint(S["filename"], line, col, message)
def tkstr(tid):
if tid == -1: return ""
if tokens[tid]["kind"] == "tkSTR":
if "$" in tokens[tid]: return tokens[tid]["$"]
return text[tokens[tid]["start"]:tokens[tid]["end"] + 1]
def addtoken(i):
# Interpolate/track interpolation indices for string.
if tokens[i]["kind"] == "tkSTR":
value = tkstr(i)
tokens[i]["$"] = value
if action != "format" and i not in vindices:
end = 0
pointer = 0
tmpstr = ""
vindices[i] = []
# [https://stackoverflow.com/a/3519601]
# [https://docs.python.org/2/library/re.html#re.finditer]
for match in re.finditer(r, value):
start = match.span()[0]
end = len(match.group()) + start
varname = match.group()[2:-1].strip()
if varname not in VARSTABLE:
# Note: Modify token index to point to
# start of the variable position.
tokens[S["tid"]]["start"] += start
err(ttid, "Undefined variable", scope="child")
USED_VARS[varname] = 1
vindices[i].append([start, end])
tmpstr += value[pointer:start]
sub = VARSTABLE.get(varname, "")
# Unquote string if quoted.
tmpstr += sub if sub[0] not in ("\"", "'") else sub[1:-1]
pointer = end
# Get tail-end of string.
tmpstr += value[end:]
tokens[i]["$"] = tmpstr
if not vindices[i]: del vindices[i]
nonlocal BRANCHES
BRANCHES[-1].append(tokens[i])
def expect(*args):
nonlocal NEXT
NEXT.clear()
for a in args:
NEXT.append(a)
def clearscope():
nonlocal SCOPE
SCOPE.clear()
def addscope(s):
nonlocal SCOPE
SCOPE.append(s)
def popscope(pops=1):
nonlocal SCOPE
while pops:
SCOPE.pop()
pops -= 1
def hasscope(s):
return s in SCOPE
def prevscope():
return SCOPE[-1]
def hasnext(s):
return s in NEXT
def nextany():
return NEXT[0] == ""
def addbranch():
nonlocal branch, BRANCHES
BRANCHES.append(branch)
def newbranch():
nonlocal branch
branch = []
def prevtoken():
return tokens[dtids[S["tid"]]]
# Command chain/flag grouping helpers.
# ================================
def newgroup():
nonlocal chain
chain = []
def addtoken_group(i):
nonlocal chain
chain.append(i)
def addgroup(g):
nonlocal CCHAINS
CCHAINS.append([g])
def addtoprevgroup():
nonlocal chain, CCHAINS
newgroup()
CCHAINS[-1].append(chain)
# ============================
def newflag():
nonlocal flag, FLAGS
flag = {
"tid": -1,
"alias": -1,
"boolean": -1,
"assignment": -1,
"multi": -1,
"union": -1,
"values": []
}
index = len(CCHAINS) - 1
if index not in FLAGS:
FLAGS[index] = []
FLAGS[index].append(flag)
setflagprop("tid")
def newvaluegroup(prop):
nonlocal flag
flag[prop].append([-1])
def setflagprop(prop, prev_val_group=False):
index = len(CCHAINS) - 1
if prop != "values":
FLAGS[index][-1][prop] = S["tid"]
else:
if not prev_val_group:
FLAGS[index][-1][prop].append([S["tid"]])
else:
FLAGS[index][-1][prop][-1].append(S["tid"])
# Setting/variable grouping helpers.
# ================================
def newgroup_stn():
nonlocal setting
setting = []
def addtoken_stn_group(i):
nonlocal setting
setting.append(i)
def addgroup_stn(g):
nonlocal SETTINGS
SETTINGS.append(g)
# def addtoprevgroup_stn():
# nonlocal setting, SETTINGS
# newgroup_stn()
# SETTINGS[-1].append(setting)
# ============================
def newgroup_var():
nonlocal variable
variable = []
def addtoken_var_group(i):
nonlocal variable
variable.append(i)
def addgroup_var(g):
nonlocal VARIABLES
VARIABLES.append(g)
# def addtoprevgroup_var():
# nonlocal variable, VARIABLES
# newgroup_var()
# VARIABLES[-1].append(variable)
# ============================
def __stn__asg(kind):
addtoken_stn_group(S["tid"])
expect("tkSTR", "tkAVAL")
def __stn__str(kind):
addtoken_stn_group(S["tid"])
expect("")
vstring(S)
def __stn__aval(kind):
addtoken_stn_group(S["tid"])
expect("")
vsetting_aval(S)
def __var__asg(kind):
addtoken_var_group(S["tid"])
expect("tkSTR")
def __var__str(kind):
addtoken_var_group(S["tid"])
VARSTABLE[tkstr(BRANCHES[-1][-3]["tid"])[1:]] = tkstr(S["tid"])
expect("")
vstring(S)
def __cmd__asg(kind):
# If a universal block, store group id.
if S["tid"] in dtids:
prevtk = prevtoken()
if prevtk["kind"] == "tkCMD" and text[prevtk["start"]] == "*":
ubids.append(len(CCHAINS) - 1)
expect("tkBRC_LB", "tkFLG", "tkKYW")
def __cmd__brc_lb(kind):
addscope(kind)
expect("tkFLG", "tkKYW", "tkBRC_RB")
# # [TODO] Pathway needed?
# def __cmd__brc_rb(kind):
# expect("", "tkCMD")
def __cmd__flg(kind):
newflag()
addscope(kind)
expect("", "tkASG", "tkQMK", "tkDCLN", "tkFVAL", "tkDPPE", "tkBRC_RB")
def __cmd__kyw(kind):
newflag()
addscope(kind)
expect("tkSTR", "tkDLS")
def __cmd__ddot(kind):
expect("tkCMD", "tkBRC_LC")
def __cmd__cmd(kind):
addtoken_group(S["tid"])
expect("", "tkDDOT", "tkASG", "tkDCMA")
def __cmd__brc_lc(kind):
addtoken_group(-1)
addscope(kind)
expect("tkCMD")
def __cmd__dcma(kind):
# If a universal block, store group id.
if S["tid"] in dtids:
prevtk = prevtoken()
if prevtk["kind"] == "tkCMD" and text[prevtk["start"]] == "*":
ubids.append(len(CCHAINS) - 1)
addtoprevgroup()
addscope(kind)
expect("tkCMD")
def __brc_lc__cmd(kind):
addtoken_group(S["tid"])
expect("tkDCMA", "tkBRC_RC")
def __brc_lc__dcma(kind):
expect("tkCMD")
def __brc_lc__brc_rc(kind):
addtoken_group(-1)
popscope()
expect("", "tkDDOT", "tkASG", "tkDCMA")
def __flg__dcln(kind):
if prevtoken()["kind"] != "tkDCLN":
expect("tkDCLN")
else:
expect("tkFLGA")
def __flg__flga(kind):
setflagprop("alias")
expect("", "tkASG", "tkQMK", "tkDPPE")
def __flg__qmk(kind):
setflagprop("boolean")
expect("", "tkDPPE")
def __flg__asg(kind):
setflagprop("assignment")
expect("", "tkDCMA", "tkMTL", "tkDPPE", "tkBRC_LP", "tkFVAL", "tkSTR",
"tkDLS", "tkBRC_RB")
def __flg__dcma(kind):
setflagprop("union")
expect("tkFLG", "tkKYW")
def __flg__mtl(kind):
setflagprop("multi")
expect("", "tkBRC_LP", "tkDPPE")
def __flg__dls(kind):
addscope(kind) # Build cmd-string.
expect("tkBRC_LP")
def __flg__brc_lp(kind):
addscope(kind)
expect("tkFVAL", "tkSTR", "tkFOPT", "tkDLS", "tkBRC_RP")
def __flg__flg(kind):
newflag()
if hasscope("tkBRC_LB") and token["line"] == prevtoken()["line"]:
err(S["tid"], "Flag same line (nth)", scope="child")
expect("", "tkASG", "tkQMK", "tkDCLN", "tkFVAL", "tkDPPE")
def __flg__kyw(kind):
newflag()
# [TODO] Investigate why leaving flag scope doesn't affect
# parsing. For now remove it to keep scopes array clean.
popscope()
if hasscope("tkBRC_LB") and token["line"] == prevtoken()["line"]:
err(S["tid"], "Keyword same line (nth)", scope="child")
addscope(kind)
expect("tkSTR", "tkDLS")
def __flg__str(kind):
setflagprop("values")
expect("", "tkDPPE")
def __flg__fval(kind):
setflagprop("values")
expect("", "tkDPPE")
def __flg__dppe(kind):
expect("tkFLG", "tkKYW")
def __flg__brc_rb(kind):
popscope()
expect("")
def __brc_lp__fopt(kind):
prevtk = prevtoken()
if prevtk["kind"] == "tkBRC_LP":
if prevtk["line"] == line:
err(S["tid"], "Option same line (first)", scope="child")
addscope("tkOPTS")
expect("tkFVAL", "tkSTR", "tkDLS")
def __brc_lp__fval(kind):
setflagprop("values")
expect("tkFVAL", "tkSTR", "tkDLS", "tkBRC_RP", "tkTBD")
# Disable pathway for now.
# def __brc_lp__tbd(kind):
# setflagprop("values")
# expect("tkFVAL", "tkSTR", "tkDLS", "tkBRC_RP", "tkTBD")
def __brc_lp__str(kind):
setflagprop("values")
expect("tkFVAL", "tkSTR", "tkDLS", "tkBRC_RP", "tkTBD")
def __brc_lp__dls(kind):
addscope(kind)
expect("tkBRC_LP")
# # [TODO] Pathway needed?
# def __brc_lp__dcma(kind):
# expect("tkFVAL", "tkSTR")
def __brc_lp__brc_rp(kind):
popscope()
expect("", "tkDPPE")
prevtk = prevtoken()
if prevtk["kind"] == "tkBRC_LP":
warn(prevtk["tid"], "Empty scope (flag)")
# # [TODO] Pathway needed?
# def __brc_lp__brc_rb(kind):
# popscope()
# expect("")
def __dls__brc_lp(kind):
newvaluegroup("values")
setflagprop("values", prev_val_group=True)
expect("tkSTR")
def __dls__dls(kind):
expect("tkSTR")
def __dls__str(kind):
expect("tkDCMA", "tkBRC_RP")
def __dls__dcma(kind):
expect("tkSTR", "tkDLS")
def __dls__brc_rp(kind):
popscope()
setflagprop("values", prev_val_group=True)
# Handle: 'program = --flag=$("cmd")'
# Handle: 'program = default $("cmd")'
if prevscope() in ("tkFLG", "tkKYW"):
if hasscope("tkBRC_LB"):
popscope()
expect("tkFLG", "tkKYW", "tkBRC_RB")
else:
# Handle: oneliner command-string
# 'program = --flag|default $("cmd", $"c", "c")'
# 'program = --flag::f=(1 3)|default $("cmd")|--flag'
# 'program = --flag::f=(1 3)|default $("cmd")|--flag'
# 'program = default $("cmd")|--flag::f=(1 3)'
# 'program = default $("cmd")|--flag::f=(1 3)|default $("cmd")'
expect("", "tkDPPE", "tkFLG", "tkKYW")
# Handle: 'program = --flag=(1 2 3 $("c") 4)'
elif prevscope() in ("tkBRC_LP"):
expect("tkFVAL", "tkSTR", "tkFOPT", "tkDLS", "tkBRC_RP")
# Handle: long-form
# 'program = [
# --flag=(
# - 1
# - $("cmd")
# - true
# )
# ]'
elif prevscope() in ("tkOPTS"):
expect("tkFOPT", "tkBRC_RP")
def __opts__fopt(kind):
if prevtoken()["line"] == line:
err(S["tid"], "Option same line (nth)", scope="child")
expect("tkFVAL", "tkSTR", "tkDLS")
def __opts__dls(kind):
addscope("tkDLS") # Build cmd-string.
expect("tkBRC_LP")
def __opts__fval(kind):
setflagprop("values")
expect("tkFOPT", "tkBRC_RP")
def __opts__str(kind):
setflagprop("values")
expect("tkFOPT", "tkBRC_RP")
def __opts__brc_rp(kind):
popscope(2)
expect("tkFLG", "tkKYW", "tkBRC_RB")
def __brc_lb__flg(kind):
newflag()
if hasscope("tkBRC_LB") and token["line"] == prevtoken()["line"]:
err(S["tid"], "Flag same line (first)", scope="child")
addscope(kind)
expect("tkASG", "tkQMK", "tkDCLN", "tkFVAL", "tkDPPE", "tkBRC_RB")
def __brc_lb__kyw(kind):
newflag()
if hasscope("tkBRC_LB") and token["line"] == prevtoken()["line"]:
err(S["tid"], "Keyword same line (first)", scope="child")
addscope(kind)
expect("tkSTR", "tkDLS", "tkBRC_RB")
def __brc_lb__brc_rb(kind):
popscope()
expect("")
prevtk = prevtoken()
if prevtk["kind"] == "tkBRC_LB":
warn(prevtk["tid"], "Empty scope (command)")
def __kyw__str(kind):
setflagprop("values")
# Collect exclude values for use upstream.
if S["tid"] in dtids:
prevtk = prevtoken()
if (prevtk["kind"] == "tkKYW"
and tkstr(prevtk["tid"]) == "exclude"):
excl_values = tkstr(S["tid"])[1:-1].strip().split(";")
for exclude in excl_values:
S["excludes"].append(exclude)
# [TODO] This pathway re-uses the flag (tkFLG) token
# pathways. If the keyword syntax were to change
# this will need to change as it might no loner work.
popscope()
addscope("tkFLG") # Re-use flag pathways for now.
expect("", "tkDPPE")
def __kyw__dls(kind):
addscope(kind) # Build cmd-string.
expect("tkBRC_LP")
# # [TODO] Pathway needed?
# def __kyw__brc_rb(kind):
# popscope()
# expect("")
# # [TODO] Pathway needed?
# def __kyw__flg(kind):
# expect("tkASG", "tkQMK",
# "tkDCLN", "tkFVAL", "tkDPPE")
# # [TODO] Pathway needed?
# def __kyw__kyw(kind):
# addscope(kind)
# expect("tkSTR", "tkDLS")
def __kyw__dppe(kind):
# [TODO] Because the flag (tkFLG) token pathways are
# reused for the keyword (tkKYW) pathways, the scope
# needs to be removed. This is fine for now but when
# the keyword token pathway change, the keyword
# pathways will need to be fleshed out in the future.
if prevscope() in ("tkKYW"):
popscope()
addscope("tkFLG") # Re-use flag pathways for now.
expect("tkFLG", "tkKYW")
def __dcma__cmd(kind):
addtoken_group(S["tid"])
popscope()
expect("", "tkDDOT", "tkASG", "tkDCMA")
command = tkstr(S["tid"])
if command != "*" and command != cmdname:
warn(S["tid"], "Unexpected command:")
DISPATCH = {
"tkSTN": {
"tkASG": __stn__asg,
"tkSTR": __stn__str,
"tkAVAL": __stn__aval,
},
"tkVAR": {
"tkASG": __var__asg,
"tkSTR": __var__str
},
"tkCMD": {
"tkASG": __cmd__asg,
"tkBRC_LB": __cmd__brc_lb,
# "tkBRC_RB": __cmd__brc_rb,
"tkFLG": __cmd__flg,
"tkKYW": __cmd__kyw,
"tkDDOT": __cmd__ddot,
"tkCMD": __cmd__cmd,
"tkBRC_LC": __cmd__brc_lc,
"tkDCMA": __cmd__dcma
},
"tkBRC_LC": {
"tkCMD": __brc_lc__cmd,
"tkDCMA": __brc_lc__dcma,
"tkBRC_RC": __brc_lc__brc_rc
},
"tkFLG": {
"tkDCLN": __flg__dcln,
"tkFLGA": __flg__flga,
"tkQMK": __flg__qmk,
"tkASG": __flg__asg,
"tkDCMA": __flg__dcma,
"tkMTL": __flg__mtl,
"tkDLS": __flg__dls,
"tkBRC_LP": __flg__brc_lp,
"tkFLG": __flg__flg,
"tkKYW": __flg__kyw,
"tkSTR": __flg__str,
"tkFVAL": __flg__fval,
"tkDPPE": __flg__dppe,
"tkBRC_RB": __flg__brc_rb
},
"tkBRC_LP": {
"tkFOPT": __brc_lp__fopt,
"tkFVAL": __brc_lp__fval,
# "tkTBD": __brc_lp__tbd,
"tkSTR": __brc_lp__str,
"tkDLS": __brc_lp__dls,
# "tkDCMA": __brc_lp__dcma,
"tkBRC_RP": __brc_lp__brc_rp
# "tkBRC_RB": __brc_lp__brc_rb
},
"tkDLS": {
"tkBRC_LP": __dls__brc_lp,
"tkDLS": __dls__dls,
"tkSTR": __dls__str,
"tkDCMA": __dls__dcma,
"tkBRC_RP": __dls__brc_rp
},
"tkOPTS": {
"tkFOPT": __opts__fopt,
"tkDLS": __opts__dls,
"tkFVAL": __opts__fval,
"tkSTR": __opts__str,
"tkBRC_RP": __opts__brc_rp
},
"tkBRC_LB": {
"tkFLG": __brc_lb__flg,
"tkKYW": __brc_lb__kyw,
"tkBRC_RB": __brc_lb__brc_rb,
},
"tkKYW": {
"tkSTR": __kyw__str,
"tkDLS": __kyw__dls,
# "tkFLG": __kyw__flg,
# "tkKYW": __kyw__kyw,
# "tkBRC_RB": __kyw__brc_rb,
"tkDPPE": __kyw__dppe,
},
"tkDCMA": {
"tkCMD": __dcma__cmd
}
}
while i < l:
token = tokens[i]
kind = token["kind"]
line = token["line"]
start = token["start"]
end = token["end"]
S["tid"] = token["tid"]
if kind == "tkNL":
i += 1
continue
if kind != "tkEOP":
ttid = i
if kind == "tkTRM":
if not SCOPE:
addbranch()
addtoken(ttid)
newbranch()
expect("")
else:
addtoken(ttid)
if NEXT and not nextany():
err(ttid, "Improper termination", scope="child")
i += 1
continue
if not SCOPE:
oneliner = -1
if BRANCHES:
ltoken = BRANCHES[-1][-1] # Last branch token.
if line == ltoken["line"] and ltoken["kind"] != "tkTRM":
err(ttid, "Improper termination", scope="parent")
if kind != "tkEOP":
addbranch()
addtoken(ttid)
if kind in ("tkSTN", "tkVAR", "tkCMD"):
addscope(kind)
if kind == "tkSTN":
newgroup_stn()
addgroup_stn(setting)
addtoken_stn_group(S["tid"])
vsetting(S)
expect("", "tkASG")
elif kind == "tkVAR":
newgroup_var()
addgroup_var(variable)
addtoken_var_group(S["tid"])
varname = tkstr(S["tid"])[1:]
VARSTABLE[varname] = ""
if varname not in USER_VARS:
USER_VARS[varname] = []
USER_VARS[varname].append(S["tid"])
vvariable(S)
expect("", "tkASG")
elif kind == "tkCMD":
addtoken_group(S["tid"])
addgroup(chain)
expect("", "tkDDOT", "tkASG", "tkDCMA")
command = tkstr(S["tid"])
if command != "*" and command != cmdname:
warn(S["tid"], "Unexpected command:")
else:
if kind == "tkCMT":
newbranch()
expect("")
else: # Handle unexpected parent tokens.
err(S["tid"], "Unexpected token:", scope="parent")
else:
if kind == "tkCMT":
addtoken(ttid)
i += 1
continue
# Remove/add necessary tokens when parsing long flag form.
if hasscope("tkBRC_LB"):
if hasnext("tkDPPE"):
NEXT.remove("tkDPPE")
NEXT.append("tkFLG")
NEXT.append("tkKYW")
NEXT.append("tkBRC_RB")
if NEXT and not hasnext(kind):
if nextany():
clearscope()
newbranch()
newgroup()
continue
else:
err(S["tid"], "Unexpected token:", scope="child")
addtoken(ttid)
# Oneliners must be declared on oneline, else error.
if BRANCHES[-1][0]["kind"] == "tkCMD" and (
((hasscope("tkFLG") or hasscope("tkKYW"))
or kind in ("tkFLG", "tkKYW")) and not hasscope("tkBRC_LB")):
if oneliner == -1: oneliner = token["line"]
elif token["line"] != oneliner:
err(S["tid"], "Improper oneliner", scope="child")
if kind in DISPATCH[prevscope()]:
DISPATCH[prevscope()][kind](kind)
else:
err(tokens[S["tid"]]["tid"], "Unexpected token:", pos="end")
i += 1
# Check for any unused variables and give warning.
for uservar in USER_VARS:
if uservar not in USED_VARS:
for tid in USER_VARS[uservar]:
warn(tid, f"Unused variable: '{uservar}'")
S["warn_lsort"].add(tokens[tid]["line"])
# Sort warning lines and print issues.
warnlines = list(S["warn_lines"])
warnlines.sort()
for warnline in warnlines:
# Only sort lines where unused variable warning(s) were added.
if warnline in S["warn_lsort"] and len(S["warnings"][warnline]) > 1:
# [https://stackoverflow.com/a/4233482]
S["warnings"][warnline].sort(key=operator.itemgetter(1, 2))
for warning in S["warnings"][warnline]:
Issue().warn(*warning)
if action == "make":
return acdef(BRANCHES, CCHAINS, FLAGS, SETTINGS, S, cmdname)
else:
return formatter(tokens, text, BRANCHES, CCHAINS, FLAGS, SETTINGS, S)
def test_named_args_2():
assert_equals(formatter("{a}, {b}, {a}", a="b", b="a"), "b, a, b")
def test_named_args_1():
assert_equals(formatter("{a}", a="b"), "b")
def test_positional_mix():
assert_equals(formatter("{1}, {}, {2}, {}, {0}", "a", "b", "c"), "b, a, c, b, a")
def test_positional_multi():
assert_equals(formatter("{1}, {0}, {2}, {1}, {0}", "a", "b", "c"), "{1}, {0}, {2}, {1}, {0}".format("a", "b", "c"))
def test_positional():
assert_equals(formatter("{1}, {0}", "a", "b"), "b, a")
def test_formatter4():
formatter("{}, {}, {}")
def test_formatter3():
assert_equals(formatter("{}, {}, {}", "c", "b", "z"), "c, b, z")
def test_whatever():
formatter("{:d}", "blah")
def test_formatter():
assert_equals(formatter("{}, {}", "a", "b"), "a, b")
def test_named_args_3():
assert_equals(formatter("{a}, {0}", "a", a="b"), "b, a")
def parse(self, response):
url = response.request.url
domain = urlparse(url).netloc
if not domain in self.allowed_domains:
return
lastmod = response.request.meta["lastmod"]
sel = Selector(response)
title = sel.xpath('//title/text()').extract_first()
if self.image_xpath is not None:
image_urls = self._get_image(
sel.xpath(self.image_xpath), response, allow_outer_domain=True)
else:
image_urls = self._get_image(
sel.xpath("//article//img"), response, allow_outer_domain=True)
if len(image_urls) == 0:
image_urls = self._get_image(
sel.xpath("//img"), response, allow_outer_domain=True)
user_meta = {}
obj = formatter(sel)
#obj = dict()
#exec(self.json_format, {"sel": sel,
# "re": re, "logger": self.logger}, obj)
user_meta = obj # ["res"]
title = (title.replace('\n', ''))
passage = title + "\n"
if self.document_xpath is not None:
passageXPath = self.document_xpath
else:
passageXPath = "//*[self::h1 or self::h2 or self::h3 or self::h4 or self::h5 or self::h6]/text()|//article//div/text()|//article//p/text()"
passage_word_list = sel.xpath(passageXPath).extract()
regex = r".+"
for p in passage_word_list:
pp = (p.replace('\n', ''))
pp = (pp.replace(' ', ''))
passage += " " + pp
lang = detect(passage)
if lang == 'ja':
words = self.molph_mecab(passage)
else:
words = self.molph_nltk(passage)
json_words = json.dumps(words)
if True:
if len(words) > 0:
objectId = ObjectId()
molphed = self.URL(url=url)
molphed.title = title # one
molphed.lang = lang
molphed.words = words
molphed.words_len = len(words)
molphed.imageUrls = image_urls
molphed.passage = passage
molphed.user_meta = user_meta
molphed.lastmod = lastmod
molphed.requestId = self.request_id
molphed.save()
return
def test_escape():
assert_equals(formatter("{{}}"), "{{}}".format())
import sys from query import query from formatter import formatter # do cli parsing tings word = sys.argv[1] # pass arg parsed from cli into query function # results will be a dictionary w/ google, wikipedia and reddit as keys and links as values results = query(word) # formatter takes in results and returns appropriately formatted text to print message = formatter(**results) # print message to cmd print(message)
def test_int2():
assert_equals(formatter("{0}", 1), "1")
import time
s_serialPort = serial.Serial()
s_recvInterval = 0.05
s_plotInterval = 0.05
s_recvStatusFieldIndex = 0
s_sendStatusFieldIndex = 1
s_infoStatusFieldIndex = 2
s_recvStatusStr = 'Recv: '
s_recvTotalBytes = 0
s_sendStatusStr = 'Send: '
s_sendTotalBytes = 0
s_formatter = formatter.formatter()
s_lastRecvFormat = None
s_lastSendFormat = None
m_plot = ''
ax = []
ay = []
def tcplink(sock, addr):
print('Accept new connection from %s:%s...' % addr)
sock.send(b'Welcome!')
while True:
data = sock.recv(1024)
time.sleep(0.01)
def test_int3():
assert_equals(formatter("{bla}", bla=1), "1")
def formatter(self, topic, opts):
return formatter.formatter(self, topic, opts)
def test_formatting():
assert_equals(formatter("{:d}", 1), "1")
def scrapeBets():
matchDF_filename = './csv/static/matchDF.csv'
test = False
if len(sys.argv) > 1:
if sys.argv[1] == 'test':
test = True
else:
url = sys.argv[1]
search_regex = '.offering-games__table-row'
time_regex = '.offering-games__link'
def processGamesPlaying(driver):
url = 'https://www.flashscore.com/table-tennis/'
def doSomething(driver):
button = driver.find_element_by_xpath(
'//*[@id="live-table"]/div[1]/div/div[2]/div[1]')
button.click()
page_source, driver = doSomethingFetchPageSource(
url, driver=driver, doSomething=[doSomething])
text_file = open("./temp/gamesPlaying.txt", "w")
text_file.write(page_source)
text_file.close()
search_regex = '.event__participant'
s = BeautifulSoup(str(page_source), 'html.parser')
players = s.select(search_regex)
k = []
for (index, player) in enumerate(players):
p = player.text
p = p[:p.find('(')].strip()
k.append(p)
return k
cols = ['time', 'lTeam', 'rTeam', 'lLine', 'rLine', 'link']
def processBetOnline(driver):
url = 'https://beta.betonline.ag/sportsbook/table-tennis/todaygames'
base = 'https://beta.betonline.ag'
page_source = fetchPS(url, test, driver)
text_file = open("./temp/betonline.txt", "w")
text_file.write(page_source)
text_file.close()
def formatTeamNames(teams):
def formatSide(side):
end = side.find(',')
return (side[:end] + ' ' + side[end + 2:end + 3] + '.').strip()
return [formatSide(teams[0].text), formatSide(teams[1].text)]
s = BeautifulSoup(str(page_source), 'html.parser')
df = pd.DataFrame([], columns=cols)
search_regex = '.offering-today-games__table-row'
time_regex = '.offering-today-games__link'
matches = s.select(search_regex)
for (index, match) in enumerate(matches):
teams = match.select('.lines-row__team-name')
lines = match.select('.lines-row__money')
time = match.select(time_regex)
teams = formatTeamNames(teams)
link = base + time[0].get('href')
k = pd.DataFrame([[
time[0].text, teams[0], teams[1], lines[0].text.strip('()'),
lines[1].text.strip('()'), link
]],
columns=cols)
df = df.append(k)
df['platform'] = url
return df.reset_index(0, drop=True)
def processBovada(driver):
url = 'https://www.bovada.lv/sports/table-tennis'
base = 'https://www.bovada.lv'
page_source = fetchPS(url,
test,
driver,
waitFor=['class', 'grouped-events'])
text_file = open("./temp/bovada_ps.txt", "w")
text_file.write(page_source)
text_file.close()
def formatLines(lines):
def formatLine(line):
l = line.strip()
if l == 'EVEN':
return '+100'
return l
if len(lines) > 2:
return [formatLine(lines[2].text), formatLine(lines[3].text)]
return [formatLine(lines[0].text), formatLine(lines[1].text)]
def formatTeamNames(teams):
def formatSide(side):
end = side.find(',')
if end == -1:
end = side.find(' ')
return (side[end:] + ' ' + side[0:1] + '.').strip()
return (side[:end] + ' ' + side[end + 2:end + 3] + '.').strip()
return [formatSide(teams[0].text), formatSide(teams[1].text)]
def formatTime(time):
text = time[0].text
e = text.find(' ', 2)
return text[e + 1:]
s = BeautifulSoup(str(page_source), 'html.parser')
df = pd.DataFrame([], columns=cols)
search_regex = '.coupon-content.more-info'
time_regex = '.period'
s = s.select('.next-events-bucket')
if len(s) == 0:
return df
s = s[0]
matches = s.select(search_regex)
for (index, match) in enumerate(matches):
teams = match.select('.competitor-name')
link = match.select('.game-view-cta')
link = link[0].find_all('a', href=True)[0].get('href')
link = base + link
lines = match.select('.bet-price')
time = match.select(time_regex)
lines = formatLines(lines)
teams = formatTeamNames(teams)
time = formatTime(time)
k = pd.DataFrame([[
time, teams[0], teams[1], lines[0].strip('()'),
lines[1].strip('()'), link
]],
columns=cols)
df = df.append(k)
df['platform'] = url
return df.reset_index(0, drop=True)
def findCorresponding(df, l, r):
# print(l, r)
today = date.today()
d = today.strftime("%d.%m")
k = df.loc[((df['lPlayer'] == r.strip()) &
(df['rPlayer'] == l.strip())) |
((df['lPlayer'] == l.strip()) &
(df['rPlayer'] == r.strip()))]
return k
def getCorrespondingGames(df):
gameDF = pd.read_csv(matchDF_filename)
gameDF = gameDF.loc[gameDF['lScore'] == '-']
d = pd.DataFrame()
o = pd.DataFrame()
for index, i in df.iterrows():
k = findCorresponding(gameDF, i['lTeam'], i['rTeam'])
if k.shape[0] != 0:
d = d.append(k.iloc[0])
i['merge_index'] = k.id.values[0]
o = o.append(i)
return d, o
def fetchPS(url, test, driver, **kwargs):
ps = ''
if test == False:
if 'waitFor' in kwargs:
ps, driver = fetchPageSource(url,
waitFor=kwargs['waitFor'],
driver=driver)
else:
ps, driver = fetchPageSource(url, driver=driver)
text_file = open("./Debug/ps.txt", "w")
text_file.write(ps)
text_file.close()
else:
file = open('./Debug/ps.txt', 'r')
ps = file.read()
file.close()
return ps
driver = createDriver()
gamesPlaying = processGamesPlaying(driver)
k = processBetOnline(driver)
l = processBovada(driver)
k.to_csv('./temp/betOnline.csv')
l.to_csv('./temp/bovada.csv')
bettingSitesDF = [k, l]
df = pd.concat(bettingSitesDF)
def formatLines(df):
df.loc[df['rTeam'].str.strip() > df['lTeam'].str.strip(),
['lTeam', 'rTeam', 'lLine', 'rLine']] = df.loc[
df['rTeam'].str.strip() > df['lTeam'].str.strip()][[
'rTeam', 'lTeam', 'rLine', 'lLine'
]].values
df = df.sort_values('time')
df.to_csv('./temp/combined.csv')
return df
df.to_csv('./temp/bettingSitesDF.csv')
df = formatLines(df).reset_index()
cdf, cbdf = getCorrespondingGames(df)
print(cdf.to_csv('./temp/cdf.csv'))
print(cbdf.to_csv('./temp/cbdf.csv'))
ul = set(list(cdf['lPlayer'].unique()) + list(cdf['rPlayer'].unique()))
mdf = pd.read_csv(matchDF_filename)
udf = mdf[(mdf['lPlayer'].isin(ul)) | (mdf['rPlayer'].isin(ul))]
udf.to_csv('./test_before.csv')
formatted = formatter(udf, True, ignore_ids=cdf['id'])
formatted = formatted[formatted['id'].isin(cdf['id'])]
formatted = formatted.merge(mdf, on='id')
def formatSequencer(df, seq):
df.loc[df[seq].str.contains(' ') == False, seq] = df[seq] + '0000'
df.loc[df[seq].str.contains(' '),
seq] = df[seq].str[0:6] + '2020' + df[seq].str[-5:]
df[seq] = df[seq].str.replace('.', '')
df[seq] = df[seq].str.replace(':', '')
df[seq] = df[seq].str[4:8] + df[seq].str[2:4] + df[seq].str[0:2] + df[
seq].str[8:]
k = df[df[seq].str.contains(' ')][[seq]]
df[seq] = df[seq].astype(int)
return df
formatted = formatSequencer(formatted, 'datetime')
formatted.to_csv('./temp/merged.csv')
predictions = predict(formatted)
formatted['predictions'] = predictions.tolist()
formatted['rWinPred'] = formatted['predictions'].apply(lambda x: x[0])
formatted['lWinPred'] = formatted['predictions'].apply(lambda x: x[1])
formatted = formatted.merge(cbdf, left_on='id', right_on='merge_index')
formatted.to_csv('./temp/formatted.csv')
formatted = formatted[formatted['lLine'] != '']
formatted = formatted[formatted['rLine'] != '']
formatted = swap(formatted, ['lTeam', 'Player_left'],
[['lTeam', 'rTeam'], ['lLine', 'rLine']])
formatted['lOdds'] = formatted['lLine'].astype(int).apply(
americanToImplied)
formatted['rOdds'] = formatted['rLine'].astype(int).apply(
americanToImplied)
formatted['ledge'] = round(formatted['lWinPred'] - formatted['lOdds'], 4)
formatted['redge'] = round(formatted['rWinPred'] - formatted['rOdds'], 4)
formatted['lOdds'] = round(formatted['lOdds'], 4)
formatted['rOdds'] = round(formatted['rOdds'], 4)
formatted['lWinPred'] = round(formatted['lWinPred'], 4)
formatted['rWinPred'] = round(formatted['rWinPred'], 4)
formatted = formatted.sort_values('datetime')
formatted = getLargestInGroup(formatted, ['id'], 'ledge', 'redge')
formatted = formatted.sort_values('datetime')
formatted = filterOnlyNew(formatted, gamesPlaying, 'datetime')
formatted = formatted.sort_values('datetime')
cols = [
'datetime', 'id', 'lTeam', 'rTeam', 'Player_left', 'Player_right',
'lWinPred', 'rWinPred', 'lOdds', 'rOdds', 'lLine', 'rLine', 'ledge',
'redge', 'platform', 'link'
]
formatted[cols].to_csv('./predictions.csv')
print("done")
def test_formatting2():
assert_equals(formatter("{}, {:d}", "k", 1), "k, 1")