def std_out_analyze(self):
pattern_raw_out = '(TP = %d, TN = %d, FP = %d, FN = %d)\n'
pattern_coef_out = '(Re = %.2f%%, Sp = %.2f%%, FPR = %.2f%%, FNR = %.2f%%, PWC = %.2f%%, Precision = %.2f%%, F-Measure = %.2f%%)\n'
cm_str = re.findall(r'\d+', self._output)
cm = []
for item in cm_str:
cm.append(float(item))
recall, specficity, fpr, fnr, pwc, precision, fmeasure = getStats(cm)
self._output = pattern_raw_out % tuple(cm)
self._output += pattern_coef_out % (recall, specficity, fpr, fnr, pwc, precision, fmeasure)
def std_out_analyze(self):
pattern_raw_out = '(TP = %d, TN = %d, FP = %d, FN = %d)\n'
pattern_coef_out = '(Re = %.2f%%, Sp = %.2f%%, FPR = %.2f%%, FNR = %.2f%%, PWC = %.2f%%, Precision = %.2f%%, F-Measure = %.2f%%)\n'
cm_str = re.findall(r'\d+', self._output)
cm = []
for item in cm_str:
cm.append(float(item))
recall, specficity, fpr, fnr, pwc, precision, fmeasure = getStats(cm)
self._output = pattern_raw_out % tuple(cm)
self._output += pattern_coef_out % (recall, specficity, fpr, fnr, pwc,
precision, fmeasure)
def page2(request):
if request.method =="POST":
print(request.POST)
str1="media/documents/"
str2=request.FILES['docfile'].name
if os.path.isfile(str1+str2):
os.remove(str1+str2)
newdoc = Document(docfile = request.FILES['docfile'])
newdoc.save()
request.session['classes']=request.POST['classes']
request.session['remove']=request.POST['remove']
request.session['docfile']=request.FILES['docfile'].name
colonne=range(3)
data = open_dataset(request.session['docfile'])
normalizedData=normalize(data)
stats = getStats(normalizedData)
return render(request,'kmeans/page2.html',{'colonne':colonne,'stats':stats})
else:
return render(request,'kmeans/page1.html',{})
def runNRTTest(r, indexPath, runLogDir):
cmd = '%s -classpath "%s" perf.NRTPerfTest %s "%s" multi "%s" 17 %s %s %s %s %s update 5 yes 0.0' % \
(constants.JAVA_COMMAND,
r.classPathToString(r.getClassPath(NIGHTLY_DIR)),
DIR_IMPL,
indexPath + '/index',
MEDIUM_LINE_FILE,
NRT_DOCS_PER_SECOND,
NRT_RUN_TIME,
NRT_SEARCH_THREADS,
NRT_INDEX_THREADS,
NRT_REOPENS_PER_SEC)
logFile = '%s/nrt.log' % runLogDir
cmd += '> %s 2>&1' % logFile
runCommand(cmd)
times = []
for s in reNRTReopenTime.findall(open(logFile, 'rb').read()):
times.append(float(s))
# Discard first 10 (JVM warmup)
times = times[10:]
# Discard worst 2%
times.sort()
numDrop = len(times)/50
if numDrop > 0:
message('drop: %s' % ' '.join(['%.1f' % x for x in times[-numDrop:]]))
times = times[:-numDrop]
message('times: %s' % ' '.join(['%.1f' % x for x in times]))
min, max, mean, stdDev = stats.getStats(times)
message('NRT reopen time (msec) mean=%.4f stdDev=%.4f' % (mean, stdDev))
checkIndex(r, indexPath, '%s/checkIndex.nrt.log' % runLogDir)
return mean, stdDev
def page2(request):
if request.method == "POST":
print(request.POST)
str1 = "media/documents/"
str2 = request.FILES['docfile'].name
if os.path.isfile(str1 + str2):
os.remove(str1 + str2)
newdoc = Document(docfile=request.FILES['docfile'])
newdoc.save()
request.session['classes'] = request.POST['classes']
request.session['remove'] = request.POST['remove']
request.session['docfile'] = request.FILES['docfile'].name
colonne = range(3)
data = open_dataset(request.session['docfile'])
normalizedData = normalize(data)
stats = getStats(normalizedData)
return render(request, 'kmeans/page2.html', {
'colonne': colonne,
'stats': stats
})
else:
return render(request, 'kmeans/page1.html', {})
def runOne(classpath,
data,
docsPerSec,
reopensPerSec,
fullIndexPath,
mode='update',
dir='MMapDirectory',
seed=17,
runTimeSec=60,
numSearchThreads=1,
numIndexThreads=constants.INDEX_NUM_THREADS,
statsEverySec=1,
commit="no"):
logFileName = '%s/%s_dps%s_reopen%s.txt' % (constants.LOGS_DIR, mode,
docsPerSec, reopensPerSec)
print 'log: %s' % logFileName
command = constants.JAVA_COMMAND
command += ' -cp "%s"' % classpath
command += ' perf.NRTPerfTest'
command += ' %s' % dir
command += ' %s' % fullIndexPath
command += ' multi'
command += ' %s' % data.lineFile
command += ' %s' % seed
command += ' %s' % docsPerSec
command += ' %s' % runTimeSec
command += ' %s' % numSearchThreads
command += ' %s' % numIndexThreads
command += ' %s' % reopensPerSec
command += ' %s' % mode
command += ' %s' % statsEverySec
command += ' %s' % commit
command += ' 0.0'
command += ' %s' % data.tasksFile
command += ' > %s 2>&1' % logFileName
if VERBOSE:
print
print 'run: %s' % command
os.system(command)
result = open(logFileName, 'rb').read()
if VERBOSE:
print result
try:
reopenStats = ReopenStats()
for line in result.split('\n'):
m = reByTime.match(line.rstrip())
if m is not None:
t = int(m.group(1))
# discard first 5 seconds -- warmup
if t >= (5 * float(reopensPerSec)):
reopenStats.totalSearches += int(m.group(2))
reopenStats.totalDocs += int(m.group(3))
reopenStats.totalReopens += int(m.group(4))
reopenStats.totalUpdateTime += int(m.group(5))
reopenStats.qtCount += 1
# Reopen times
times = []
for line in open(logFileName, 'rb').read().split('\n'):
m = reNRTReopenTime.match(line.rstrip())
if m is not None:
times.append(float(m.group(1)))
# Discard first 10% (JVM warmup): minimum of 1 but no more than 10
times = times[min(10, max(1, len(times) / 10)):]
# Discard worst 2%
times.sort()
numDrop = len(times) / 50
if numDrop > 0:
print 'drop: %s' % ' '.join(['%.1f' % x for x in times[-numDrop:]])
times = times[:-numDrop]
if VERBOSE:
print 'times: %s' % ' '.join(['%.1f' % x for x in times])
minVal, maxVal, mean, stdDev = stats.getStats(times)
reopenStats.meanReopenTime = mean
reopenStats.stddevReopenTime = stdDev
if VERBOSE:
print 'reopen stats:'
reopenStats.toString()
print
return reopenStats
except:
print 'FAILED -- output:\n%s' % result
raise
# SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from path_to_constraint import PathToConstraint
from symbolic import instrumentation
from collections import deque
import logging
from stats import getStats
log = logging.getLogger("se.conc")
stats = getStats()
class ConcolicEngine:
def __init__(self, funcinv, reset, debug):
self.invocation = funcinv
self.reset_func = reset
self.constraints_to_solve = deque([])
self.num_processed_constraints = 0
self.path = PathToConstraint(self)
instrumentation.SI = self.path
self.execution_return_values = []
# self.tracer = PythonTracer(debug)
# self.tracer.setInterpreter(self.path)
# self.path.setTracer(self.tracer)
stats.newCounter("explored paths")
self.generated_inputs = []
if len(args) == 0 or not os.path.exists(args[0]):
parser.error("Missing app to execute")
sys.exit(1)
filename = os.path.abspath(args[0])
app_args = args[1:]
options.filename = filename
# Get the object describing the application
app = loaderFactory(filename)
if app == None:
sys.exit(1)
logging.basicConfig(filename=options.logfile, level=logging.DEBUG)
log = logging.getLogger()
stats = getStats()
stats.pushProfile("se total")
print("Running PyExZ3 on " + app.test_name)
stats.pushProfile("engine only")
engine = ConcolicEngine(app.create_invocation(), app.reset_callback, options)
if options.single_step:
return_vals = engine.run(1)
inputs = engine.generateAllInputs()
else:
return_vals = engine.run()
stats.popProfile()
# print statistics
stats.popProfile() # SE total
def update():
updateStats()
return jsonify({"stats": getStats(), "answer": isAveragingTripleDouble()})
def main(stdscr):
# Clear screen
stdscr.clear()
# Curses Settings // DO NOT CHANGE
curses.curs_set(False)
stdscr.nodelay(True)
black = curses.COLOR_BLACK
curses.init_pair(1, curses.COLOR_WHITE, black)
curses.init_pair(2, curses.COLOR_RED, black)
curses.init_pair(3, curses.COLOR_GREEN, black)
curses.init_pair(4, curses.COLOR_BLUE, black)
curses.init_pair(5, curses.COLOR_YELLOW, black)
curses.init_pair(6, curses.COLOR_CYAN, black)
curses.init_pair(7, curses.COLOR_MAGENTA, black)
curses.init_pair(8, curses.COLOR_BLUE, curses.COLOR_WHITE)
curses.init_pair(9, black, curses.COLOR_BLUE)
while True:
try:
stats = getStats()
try:
users = stats['users']
interfaces = stats['network']['interfaces']
connections = stats['network']['connections']
procs = stats['processes']
except TypeError:
pass
stdscr.clear() # Dont Change!
# Define section specific variables
num_connects = len(connections.items())
num_closed = 0
for i in range(0, num_connects):
if connections[i]['status'] == 'CLOSE_WAIT':
num_closed += 1
num_procs = len(procs.items())
# Manipulate row numbers according to user input
try:
row_conn
except NameError:
row_conn = 0
try:
row_proc
except NameError:
row_proc = 0
k = stdscr.getch()
if k == ord('a'):
if num_connects > connect_height:
row_conn = connect_height
curses.flushinp()
elif k == ord('s'):
if num_connects > connect_height * 2:
row_conn = connect_height * 2
curses.flushinp()
elif k == ord('d'):
if num_connects > connect_height * 3:
row_conn = connect_height * 3
curses.flushinp()
elif k == ord('w'):
if num_connects > connect_height * 4:
row_conn = connect_height * 4
curses.flushinp()
elif k == ord('z'):
if num_connects > connect_height * 5:
row_conn = connect_height * 5
curses.flushinp()
elif k == ord('x'):
row_conn = 0
curses.flushinp()
elif k == ord('j'):
if num_procs >= term_size[0] * 2 - 2:
row_proc = term_size[0] * 2 - 4
curses.flushinp()
elif k == ord('k'):
if num_procs >= term_size[0] * 4 - 2:
row_proc = term_size[0] * 4 - 8
curses.flushinp()
elif k == ord('l'):
if num_procs >= term_size[0] * 6 - 2:
row_proc = term_size[0] * 6 - 12
curses.flushinp()
elif k == ord('i'):
if num_procs >= term_size[0] * 8 - 2:
row_proc = term_size[0] * 8 - 16
curses.flushinp()
elif k == ord('m'):
if num_procs >= term_size[0] * 10 - 2:
row_proc = term_size[0] * 10 - 20
curses.flushinp()
elif k == ord('b'):
row_proc = 0
curses.flushinp()
# Define max width and height
try:
term_size
except NameError:
term_size = stdscr.getmaxyx()
resized = curses.is_term_resized(term_size[0], term_size[1])
# Recalculate if terminal has been resized
if resized == True:
term_size = stdscr.getmaxyx()
# Create sub-windows
win_width = term_size[1] / 3
term_1 = stdscr.subwin(term_size[0], floor(win_width) - 2, 0, 0)
term_2 = stdscr.subwin(term_size[0], floor(win_width), 0,
floor(win_width) - 3)
term_3 = stdscr.subwin(term_size[0], floor(win_width), 0,
floor(win_width * 2) - 3)
# Left column of terminal screen (term_1)
curr = term_1.getyx()
col_2 = (curr[0], curr[1] + (floor(win_width / 5)) - 5)
col_3 = (curr[0], curr[1] + (floor((win_width / 5) * 2)) - 4)
col_4 = (curr[0], curr[1] + (floor((win_width / 5) * 3)) - 7)
col_5 = (curr[0], curr[1] + (floor((win_width / 5) * 4)) - 3)
usr_head(term_1, col_2, col_3, col_4, col_5, 4)
for c in range(0, len(users.items())):
user = users[c]
curr = term_1.getyx()
col_2 = (curr[0], curr[1] + (floor(win_width / 5)) - 2)
col_3 = (curr[0], curr[1] + (floor((win_width / 5) * 2)) - 1)
col_4 = (curr[0], curr[1] + (floor((win_width / 5) * 3)) - 5)
col_5 = (curr[0], curr[1] + floor((win_width / 5) * 4))
usr_row(term_1, user, col_2, col_3, col_4, col_5, 3, 1)
# Network Info Display (Packets)
curr = term_1.getyx()
col_2 = (curr[0], curr[1] + floor(win_width / 5))
# Save the current location as coordinates to print the usage later on.
col_instruct = (col_2[0] + 1, col_2[1] + floor(
(win_width / 5)) * 2 - 3)
pck_head(term_1, col_2, 8, 4, win_width)
for i in range(0, len(interfaces.items())):
curr = term_1.getyx()
col_2 = (curr[0], curr[1] + floor(win_width / 5))
pck_row(term_1, interfaces, col_2, i, 3, 2)
net_head(term_1, num_connects, num_closed)
curr = term_1.getyx()
connect_height = term_size[0] - curr[0] - 1
col_2 = (curr[0], curr[1] + (floor(win_width / 5)) - 5)
if row_conn > 0:
row_c = row_conn
else:
row_c = 0
h1 = term_size[0] - 1
for c in range(curr[0], h1):
if row_c == num_connects:
break
cur = term_1.getyx()
col_2 = (cur[0], cur[1] + (floor(win_width / 5)) - 4)
col_3 = (cur[0], cur[1] + (floor((win_width / 5) * 2)) + 3)
col_4 = (cur[0], cur[1] + (floor((win_width / 5) * 3)) - 1)
conn_row(term_1, connections, col_2, col_3, col_4, row_c, 3, 1,
4, 5, 9)
row_c += 1
# Print usage instructions
usage(term_1, col_instruct)
term_1.refresh()
# Middle column of terminal screen (term_2)
#Processes
curr = term_2.getyx()
col_2 = (curr[0], curr[1] + (floor(win_width / 5)) - 5)
col_3 = (curr[0], curr[1] + (floor((win_width / 5) * 3)))
proc_head(term_2, col_2, col_3, 7, 8)
curr = term_2.getyx()
if row_proc > 0:
row_p = row_proc
else:
row_p = 0
h2 = term_size[0] - 1
for c in range(curr[0], h2):
if row_p == num_procs:
break
curr = term_2.getyx()
col_2 = (curr[0], curr[1] + (floor(win_width / 5)) - 6)
col_3 = (curr[0], curr[1] + (floor((win_width / 5) * 3)))
col_4 = (curr[0], curr[1] + (floor((win_width / 5) * 4)) + 3)
if procs[row_p]['user'] == 'root':
proc_row(term_2, procs, col_2, col_3, col_4, row_p, 2)
elif procs[row_p]['user'] == users[0]['name']:
proc_row(term_2, procs, col_2, col_3, col_4, row_p, 6)
else:
proc_row(term_2, procs, col_2, col_3, col_4, row_p, 5)
row_p += 1
term_2.refresh()
# Right column of terminal screen (term_3)
curr = term_3.getyx()
col_2 = (curr[0], curr[1] + (floor(win_width / 5)) - 5)
col_3 = (curr[0], curr[1] + (floor((win_width / 5) * 3)))
proc_head(term_3, col_2, col_3, 7, 8)
curr = term_3.getyx()
h3 = term_size[0] - 1
for a in range(curr[0], h3):
if row_p == num_procs:
break
curr = term_3.getyx()
col_2 = (curr[0], curr[1] + (floor(win_width / 5)) - 6)
col_3 = (curr[0], curr[1] + (floor((win_width / 5) * 3)))
col_4 = (curr[0], curr[1] + (floor((win_width / 5) * 4)) + 3)
if procs[row_p]['user'] == 'root':
proc_row(term_3, procs, col_2, col_3, col_4, row_p, 2)
elif procs[row_p]['user'] == users[0]['name']:
proc_row(term_3, procs, col_2, col_3, col_4, row_p, 6)
else:
proc_row(term_3, procs, col_2, col_3, col_4, row_p, 5)
row_p += 1
term_3.refresh()
time.sleep(0.2)
except KeyboardInterrupt:
return sys.exit('Thank you! Goodbye!')
def route_ratings(): return jsonify(getStats())
def home():
"""Is Westbrook averaging a triple double?"""
return jsonify({"stats": getStats(), "answer": isAveragingTripleDouble()})
def runOne(classpath, data, docsPerSec, reopensPerSec, fullIndexPath,
mode='update',
dir='MMapDirectory',
seed=17,
runTimeSec=60,
numSearchThreads=1,
numIndexThreads=constants.INDEX_NUM_THREADS,
statsEverySec=1,
commit="no"):
logFileName = '%s/%s_dps%s_reopen%s.txt' % (constants.LOGS_DIR, mode, docsPerSec, reopensPerSec)
print 'log: %s' % logFileName
command = constants.JAVA_COMMAND
command += ' -cp "%s"' % classpath
command += ' perf.NRTPerfTest'
command += ' %s' % dir
command += ' %s' % fullIndexPath
command += ' multi'
command += ' %s' % data.lineFile
command += ' %s' % seed
command += ' %s' % docsPerSec
command += ' %s' % runTimeSec
command += ' %s' % numSearchThreads
command += ' %s' % numIndexThreads
command += ' %s' % reopensPerSec
command += ' %s' % mode
command += ' %s' % statsEverySec
command += ' %s' % commit
command += ' 0.0'
command += ' %s' % data.tasksFile
command += ' > %s 2>&1' % logFileName
if VERBOSE:
print
print 'run: %s' % command
os.system(command)
result = open(logFileName, 'rb').read()
if VERBOSE:
print result
try:
reopenStats = ReopenStats()
for line in result.split('\n'):
m = reByTime.match(line.rstrip())
if m is not None:
t = int(m.group(1))
# discard first 5 seconds -- warmup
if t >= (5*float(reopensPerSec)):
reopenStats.totalSearches += int(m.group(2))
reopenStats.totalDocs += int(m.group(3))
reopenStats.totalReopens += int(m.group(4))
reopenStats.totalUpdateTime += int(m.group(5))
reopenStats.qtCount += 1
# Reopen times
times = []
for line in open(logFileName, 'rb').read().split('\n'):
m = reNRTReopenTime.match(line.rstrip())
if m is not None:
times.append(float(m.group(1)))
# Discard first 10% (JVM warmup): minimum of 1 but no more than 10
times = times[min(10, max(1, len(times) / 10)):]
# Discard worst 2%
times.sort()
numDrop = len(times)/50
if numDrop > 0:
print 'drop: %s' % ' '.join(['%.1f' % x for x in times[-numDrop:]])
times = times[:-numDrop]
if VERBOSE:
print 'times: %s' % ' '.join(['%.1f' % x for x in times])
minVal, maxVal, mean, stdDev = stats.getStats(times)
reopenStats.meanReopenTime = mean
reopenStats.stddevReopenTime = stdDev
if VERBOSE:
print 'reopen stats:'
reopenStats.toString()
print
return reopenStats
except:
print 'FAILED -- output:\n%s' % result
raise
def get_data(start_year, end_year):
print("Team Matches")
team_matches = []
id = 1
for year in range(start_year, end_year + 1):
print(year)
for m in utils.loadProcessedMatches(year):
event = m.key.split("_")[0][4:]
match = m.key.split("_")[1]
time = m.time
for i in range(len(m.red)):
start_elo = round(m.red_ratings[i])
end_elo = round(m.red_ratings_end[i])
elo_diff, append = round(end_elo - start_elo), True
for item in blacklist:
if year == item[0] and \
event == item[1] and \
match == item[2]:
append = False
if (append):
team_matches.append([
id, year, event, match, time, m.red[i], start_elo,
end_elo, elo_diff
])
id += 1
for i in range(len(m.blue)):
start_elo = round(m.blue_ratings[i])
end_elo = round(m.blue_ratings_end[i])
elo_diff, append = round(end_elo - start_elo), True
for item in blacklist:
if year == item[0] and \
event == item[1] and \
match == item[2]:
append = False
if (append):
team_matches.append([
id, year, event, match, time, m.blue[i], start_elo,
end_elo, elo_diff
])
id += 1
team_matches = pd.DataFrame(team_matches,
columns=[
"id", "year", "event", "match", "time",
"team", "elo_start", "elo_end", "elo_diff"
])
team_matches = team_matches.sort_values(by=['year', 'event', 'team'])
print("Team Events")
team_events = []
id = 1
for year in range(start_year, end_year + 1):
print(year)
year_data = team_matches[team_matches.year == year]
for event in year_data['event'].unique():
event_data = year_data[year_data.event == event]
time = event_data["time"].iloc[0]
for team in event_data['team'].unique():
team_data = event_data[event_data.team == team]
team_data = team_data.sort_values(by=['time'], ascending=True)
elo_start = team_data["elo_start"].iloc[0]
if (team_data["match"].iloc[0])[:2] != "qm":
elo_pre_playoffs = elo_start # handles only elim events
else:
# handles no playoffs
temp = team_data[team_data.match.str.startswith("qm")]
elo_pre_playoffs = temp["elo_end"].iloc[temp.shape[0] - 1]
elo_end = team_data["elo_end"].iloc[team_data.shape[0] - 1]
elo_mean = round(team_data['elo_end'].mean())
elo_max = max(elo_start, team_data['elo_end'].max())
elo_diff = round(elo_end - elo_start)
team_events.append([
id, year, event, time, team, elo_start, elo_pre_playoffs,
elo_end, elo_mean, elo_max, elo_diff
])
id += 1
team_events = pd.DataFrame(team_events,
columns=[
"id", "year", "event", "time", "team",
"elo_start", "elo_pre_playoffs", "elo_end",
"elo_mean", "elo_max", "elo_diff"
])
team_events = team_events.sort_values(by=['year', 'event', 'team'])
champ_keys = [
'arc', 'cars', 'carv', 'cur', 'dal', 'dar', 'gal', 'hop', 'new', 'roe',
'tes', 'tur'
]
print("Team Years")
team_years = []
all_teams_info = utils.loadAllTeamsInfo()
id = 1
for year in range(start_year, end_year + 1):
print(year)
year_data_events = team_events[team_events.year == year]
year_data_matches = team_matches[team_matches.year == year]
teams_temp = utils.loadTeams(year)
for team in year_data_events['team'].unique():
team_data_events = year_data_events[year_data_events.team == team]
team_data_matches = year_data_matches[year_data_matches.team ==
team]
team_data_events = team_data_events.sort_values(by=['time'],
ascending=True)
team_data_matches = team_data_matches.sort_values(by=['time'],
ascending=True)
elo_start = team_data_events["elo_start"].iloc[0]
elo_end = team_data_events["elo_end"].iloc[
team_data_events.shape[0] - 1]
elo_diff = round(elo_end - elo_start)
if len(
set(champ_keys).intersection(set(
team_data_events["event"]))) == 0:
elo_pre_champs = elo_end
else:
elo_pre_champs = round(
team_data_events[team_data_events.event.isin(
champ_keys)]["elo_start"].iloc[0])
elo_mean = round(team_data_matches['elo_end'].mean())
elo_max = teams_temp[team].get_rating_max()
elo_max = round(elo_max) # after so doesn't mess with index
[name, country, state, district, _] = all_teams_info[team]
team_years.append([
id, year, team, name, country, state, district, elo_start,
elo_pre_champs, elo_end, elo_mean, elo_max, elo_diff
])
id += 1
team_years = pd.DataFrame(team_years,
columns=[
"id", "year", "team", "name", "country",
"state", "district", "elo_start",
"elo_pre_champs", "elo_end", "elo_mean",
"elo_max", "elo_diff"
])
team_years = team_years.sort_values(by=['year', 'team'])
print("Teams")
teams = []
utils.saveAllTeams(team_years['team'].unique())
for team in team_years['team'].unique():
team_data = team_years[team_years.team == team]
elos, elo_sum, count = [-1] * (end_year - start_year + 1), 0, 0
for i in range(team_data.shape[0]):
elos[team_data["year"].iloc[i]-start_year] \
= round(team_data["elo_max"].iloc[i])
elo_sum, count = elo_sum + team_data["elo_max"].iloc[i], count + 1
elo, elo_mean, elo_max = elos[-1], round(elo_sum / count), max(elos)
# takes whatever years exist 2017-Present (two champs)
recent_start = 2017
recent_years = end_year - recent_start + 1
total, years = sum(elos[-recent_years:]), recent_years
for i in range(recent_years):
if (elos[-i - 1] == -1):
total, years = total + 1, years - 1
if (years == 0):
elo_recent = -1
else:
elo_recent = round(total / years)
'''accounts for 2020 season suspension (with mean revision)'''
if (elo == -1):
try:
elo_1yr = elos[-2]
except Exception:
elo_1yr = -1
try:
elo_2yr = elos[-3]
except Exception:
elo_2yr = -1
if (elo_1yr == -1):
elo = -1 # team has not played last two years, inactive
elif (elo_2yr == -1):
elo = int(elo_1yr * 0.56 + 1450 * 0.44) # rookie team
else:
elo = int(elo_1yr * 0.56 + elo_2yr * 0.24 + 1450 * 0.20)
elo_max_year = start_year + elos.index(elo_max)
[name, country, state, district, years] = all_teams_info[team]
active = (elo != -1) # have a current elo
teams.append([
team, name, country, state, district, years, active, elo,
elo_recent, elo_mean, elo_max, elo_max_year
])
teams = pd.DataFrame(teams,
columns=[
"team", "name", "country", "state", "district",
"years_active", "active", "elo", "elo_recent",
"elo_mean", "elo_max", "elo_max_year"
])
teams = teams.sort_values(by=['team'])
print("Events")
events = []
id = 1
for year in range(start_year, end_year + 1):
year_data = team_events[team_events.year == year]
for event in year_data["event"].unique():
elos = []
event_data = year_data[year_data.event == event]
for i in range(event_data.shape[0]):
elos.append(round(event_data["elo_max"].iloc[i]))
elos.sort(reverse=True)
try:
elo_max = elos[0]
except Exception:
elo_max = -1
try:
elo_top8 = elos[7]
except Exception:
elo_top8 = -1
try:
elo_top24 = elos[23]
except Exception:
elo_top24 = -1
elo_mean = round(sum(elos) / len(elos))
elo_sd = round(statistics.pstdev(elos))
events.append([
id, year, event, elo_max, elo_top8, elo_top24, elo_mean, elo_sd
])
id += 1
events = pd.DataFrame(events,
columns=[
"id", "year", "event", "elo_max", "elo_top8",
"elo_top24", "elo_mean", "elo_sd"
])
events = events.sort_values(by=['year', 'event'])
print("Years")
years = []
for year in range(start_year, end_year + 1):
teams_temp = utils.loadTeams(year)
board, elos = sorted(teams_temp.values()), []
for team in board:
elos.append(round(team.get_rating_max()))
elo_max = elos[0]
elo_1p = round(elos[round(0.01 * len(elos))])
elo_5p = round(elos[round(0.05 * len(elos))])
elo_10p = round(elos[round(0.10 * len(elos))])
elo_25p = round(elos[round(0.25 * len(elos))])
elo_median = round(elos[round(0.50 * len(elos))])
elo_mean = round(sum(elos) / len(elos))
elo_sd = round(statistics.pstdev(elos))
mse, acc = stats.getStats(year)
mse, acc = round(mse, 4), round(acc, 4)
years.append([
year, elo_max, elo_1p, elo_5p, elo_10p, elo_25p, elo_median,
elo_mean, elo_sd, acc, mse
])
years = pd.DataFrame(years,
columns=[
"year", "elo_max", "elo_1p", "elo_5p", "elo_10p",
"elo_25p", "elo_median", "elo_mean", "elo_sd",
"acc", "mse"
])
years = years.sort_values(by=['year'])
return team_matches, team_events, team_years, teams, events, years
import stats import warehouse_data #Write 10,000 data objects to a CSV warehouse_data.warehouse_data() #Get statistics from CSV file stats.getStats()
tab_class.append(row.pop())
data.append(row)
data.pop() # enleve la derniere ligne
tab_class.pop()
finaldata = []
for row in data:
r = []
r.append(row[2]) #pour l'instant c'est ici que l'on chsoie les colonne
r.append(row[18])
r.append(row[20])
finaldata.append(r)
return finaldata
if __name__ == "__main__":
tmps1 = time.clock()
data = open_dataset()
dat = normalize.normalize(data)
f = kmeans.kmeans(dat, 2)
b = kmeans.virer(f, 3)
stat = stats.getStats(dat)
stat.sort(cmp=comp)
for row in stat:
print row
print b
for c in b:
print(len(c.observations))
tmps2 = time.clock()
print "Temps d'execution = %d\n" % (tmps2 - tmps1)
def logs(self, data, handler):
return stats.getStats()
def logs(self, data, handler) :
return stats.getStats();
row = row.rstrip().split(",") #on utilise les , comme des delimiteurs
tab_class.append(row.pop())
data.append(row)
data.pop() # enleve la derniere ligne
tab_class.pop()
finaldata=[]
for row in data:
r=[]
r.append(row[2])#pour l'instant c'est ici que l'on chsoie les colonne
r.append(row[18])
r.append(row[20])
finaldata.append(r)
return finaldata
if __name__ == "__main__":
tmps1=time.clock()
data = open_dataset()
dat=normalize.normalize(data)
f=kmeans.kmeans(dat, 2)
b=kmeans.virer(f, 3)
stat = stats.getStats(dat)
stat.sort(cmp=comp)
for row in stat:
print row
print b
for c in b:
print(len(c.observations))
tmps2=time.clock()
print "Temps d'execution = %d\n" %(tmps2-tmps1)
