score = 1 if score == True else \ -1 if score == False else \ score if lesson not in lessons: print("{} not in lessons!".format(lesson)) lessons[lesson] = 0.5 tmp[0] += fr(score) * ft(score) * fp(lessons[lesson])**score * percentage tmp[1] += fr(score) * fp(lessons[lesson])**score d[user] = tmp dict_student_problem = {} with codecs.open(get_file_name_from_dates('logs_player_filtered', dates), 'r', "utf-8-sig") as fin: for line in fin: i += 1 if time.time() - start_time > 10: print("i", i) start_time = time.time() m = re.search( "\(([0-9]+), '([^']+)', ([0-9]+), '([^']+)', '([^']+)', datetime\.datetime\(([^\)]+)\), '([^']+)', ([0-9]+)\)", line) try: userid, name, id, eventName, eventType, datetime, JSONParams, contextualInfoId = m.groups( )
type=str) args = parser.parse_args() IP = args.ip + ":" + str(args.port) username = get_value_from_file('config.txt', 'username') password = get_value_from_file('config.txt', 'password') DBname = args.db file_type = { "collaborative": "download_collaborative", "AR": "download_AR", "competitive": "download_player" } file_name = get_file_name_from_dates(file_type[args.type], args.date) if os.path.isfile(file_name): print("file with name {} already exists".format(file_name)) exit(0) conn = pymssql.connect(server=IP, user=username, password=password, database=DBname) print("successfully connected to server (IP:{}, username:{} DBname:{})".format( IP, username, DBname)) cursor = conn.cursor() def generate_query(dates, type):
score = 1 if score == True else \ -1 if score == False else \ score if lesson not in lessons: print("{} not in lessons!".format(lesson)) lessons[lesson] = 0.8 tmp[0] += fr(score) * ft(score) * fp(lessons[lesson])**score * percentage tmp[1] += fr(score) * fp(lessons[lesson])**score d[user] = tmp with codecs.open(get_file_name_from_dates('logs_collaborative', dates), 'r', "utf-8-sig") as fin: for i, line in enumerate(fin): if time.time() - start_time > 10: print("i", i) start_time = time.time() m = re.search( "\(([0-9]+), '([^']+)', ([0-9]+), '([^']+)', '([^']+)', datetime\.datetime\(([^\)]+)\), '([^']+)', ([0-9]+)\)", line) try: userid, name, id, eventName, eventType, datetime, JSONParams, contextualInfoId = m.groups( )
description="Removes unnesesary quotes that some logs might have.") parser.add_argument('date', help="dates (dd.mm.YYYY)", type=lambda x: datetime.strptime(x, '%d.%m.%Y'), nargs='+') args = parser.parse_args() dates = args.date print("reading started") start_time = time.time() i = 0 j = 0 with codecs.open(get_file_name_from_dates('slashes_player', dates), 'r', "utf-8-sig") as fin: with codecs.open(get_file_name_from_dates('logs_player', dates), 'w', "utf-8-sig") as fout: for line in fin: i += 1 if time.time() - start_time > 10: print("i", i) start_time = time.time() line = line.replace('"[', '[') line = line.replace(']"', ']') line = line.replace('"Provjeri"', r'\"Provjeri\"') fout.write(line)
password = get_value_from_file('config.txt', 'password') type_tag = {"collaborative": "", "AR": "_AR", "competitive": "_player"} users = set() print("copying _real files to _tmp files") for file_name in glob.glob('tmp/users/results/*_real.txt'): new_file_name = "{}_tmp.txt".format(file_name[:-9]) copyfile(file_name, new_file_name) i = 0 print("adding data to _tmp files") for date in dates: file = get_file_name_from_dates("users", [date], prefix="tmp/users/logs/", suffix="{}.txt".format(type_tag[type])) if os.path.isfile(file): with codecs.open(file, "r", "utf-8-sig") as flog: for line in flog: user, grade = line.strip().split(":") with open( "tmp/users/results/{}{}_tmp.txt".format( user, type_tag[type]), "a") as fuser: fuser.write("{}:{}\n".format(date.strftime("%d.%m.%Y"), grade)) users.add(user)
return query for type in args.types: dates = [] query = generate_query(type, args.sd, args.ed) print("executing query ({})".format(query)) cursor.execute(query) print("query executed") i = 0 start_time = time.time() file_name = get_file_name_from_dates('dates_{}'.format(type), [args.sd, args.ed]) print("writing to {}".format('dates_{}.txt'.format(type))) with codecs.open(file_name, 'w', "utf-8-sig") as f: for row in cursor: i += 1 if time.time() - start_time > 10: start_time = time.time() print("i", i) f.write("{}\n".format(row[0])) dates.append(row[0].strip()) print("i", i) print("finished writing to {}".format('dates_{}.txt'.format(type)))
nargs='+') args = parser.parse_args() dates = args.date print("reading started") regex = r'(.*)<img([^>]+)([^\\>])"(.*)' ct = r'\1<img\2\3\"\4' i = 0 i2 = 0 j = 0 start_time = time.time() with codecs.open(get_file_name_from_dates('download_collaborative', dates), 'r', "utf-8-sig") as fin: with codecs.open(get_file_name_from_dates('slashes_collaborative', dates), 'w', "utf-8-sig") as fout: for line in fin: j += 1 if time.time() - start_time > 10: print("i", i) print("j", j) start_time = time.time() if "\\" in line: while "\\\"" in line: line = re.sub(r'\\\"', "\"", line)
import re import time from utils import get_file_name_from_dates import argparse from datetime import datetime parser = argparse.ArgumentParser(description="Removes unnesesary slashes that some logs might have.") parser.add_argument('date', help="dates (dd.mm.YYYY)", type=lambda x: datetime.strptime(x, '%d.%m.%Y'), nargs='+') args = parser.parse_args() dates = args.date from pprint import pprint print("reading started") start_time = time.time() with codecs.open(get_file_name_from_dates('download_AR', dates), 'r', "utf-8-sig") as fin: with codecs.open(get_file_name_from_dates('logs_AR', dates), 'w', "utf-8-sig") as fout: for i, line in enumerate(fin): line = line.replace('":""Lijepu našu" je napisao?"', '":"\\"Lijepu našu\\" je napisao?"') line = line.replace('"Tko je pokrenuo prve novine na hrvatskom jeziku "Novine horvatske" i prilog "Danica"?"', '"Tko je pokrenuo prve novine na hrvatskom jeziku \\"Novine horvatske\\" i prilog \\"Danica\\"?"') line = line.replace('"Koji politicar je pokrenuo "Novine horvatske"', '"Koji politicar je pokrenuo \\"Novine horvatske\\"') line = line.replace('""Ljubav i zloba" je opera koju je napisao?"', '"\\"Ljubav i zloba\\" je opera koju je napisao?"') line = line.replace('"Tko je pokrenuo "Novine horvatske", prve novine na hrvatskom jeziku?"', '"Tko je pokrenuo \\"Novine horvatske\\", prve novine na hrvatskom jeziku?"') fout.write(line) print("reading ended")
description="Removes unnesesary quotes that some logs might have.") parser.add_argument('date', help="dates (dd.mm.YYYY)", type=lambda x: datetime.strptime(x, '%d.%m.%Y'), nargs='+') args = parser.parse_args() dates = args.date print("reading started") start_time = time.time() i = 0 j = 0 with codecs.open(get_file_name_from_dates('slashes_collaborative', dates), 'r', "utf-8-sig") as fin: with codecs.open(get_file_name_from_dates('quotes_collaborative', dates), 'w', "utf-8-sig") as fout: for line in fin: i += 1 if time.time() - start_time > 10: print("i", i) start_time = time.time() line = line.replace('"[', '[') line = line.replace(']"', ']') line = line.replace('"Provjeri"', r'\"Provjeri\"') fout.write(line)
score = 1 if score == True else \ -1 if score == False else \ score if user not in users: if len(str(user)) < 20: print("{} not in users!".format(user)) users[user] = 0.5 tmp[0] += fr(score) * ft(score) * fp(users[user])**score * percentage tmp[1] += fr(score) * fp(users[user])**score d[lesson] = tmp with codecs.open(get_file_name_from_dates('logs_AR', dates), 'r', "utf-8-sig") as fin: for line in fin: i += 1 if time.time() - start_time > 10: print("i", i) start_time = time.time() m = re.search( "\(([0-9]+), '([^']+)', ([0-9]+), '([^']+)', '([^']+)', datetime\.datetime\(([^\)]+)\), '([^']+)', ([0-9]+)\)", line) try: userid, name, id, eventName, eventType, datetime, JSONParams, contextualInfoId = m.groups( )
parser.add_argument('-port', help="PORT of the database", default=1955, type=int) parser.add_argument('-db', help="Database name", default="ExperientialSampling1", type=str) action = parser.add_mutually_exclusive_group(required=True) action.add_argument('-t', help='calculate lessons/users (train system)', type=int) action.add_argument('-e', help='evaluate users\' score', action='store_true') args = parser.parse_args() # get all dates in range arguments = ["-types", args.type, "-sd", args.starting_date.strftime("%d.%m.%Y"), "-ed", args.ending_date.strftime("%d.%m.%Y"), "-ip", args.ip, "-port", args.port, "-db", args.db] if args.f: arguments += ["-f", args.f] execute_python_script("get_dates.py", arguments) dates_file_name = get_file_name_from_dates("dates_{}".format(args.type), [args.starting_date, args.ending_date]) if not os.path.isfile(get_file_name_from_dates("dates_{}".format(args.type), [args.starting_date, args.ending_date])): print("ERROR: get_dates.py didn't create {}".format(dates_file_name)) exit(1) preprocess = { "AR": "quotes_AR.py", "competitive": "slashes_quotes_filter_player.py", "collaborative": "slashes_qoutes_commas.py" } analyse_users = { "AR": "analyseUsers_AR.py", "competitive": "analyseUsers_player.py", "collaborative": "analyseUsers.py", }
score = 1 if score == True else \ -1 if score == False else \ score if user not in users: print("{} not in users!".format(user)) users[user] = 0.5 tmp[0] += fr(score) * ft(score) * fp(users[user])**score * percentage tmp[1] += fr(score) * fp(users[user])**score d[lesson] = tmp dict_student_problem = {} with codecs.open(get_file_name_from_dates('logs_player_filtered', dates), 'r', "utf-8-sig") as fin: for line in fin: i += 1 if time.time() - start_time > 10: print("i", i) start_time = time.time() m = re.search("\(([0-9]+), '([^']+)', ([0-9]+), '([^']+)', '([^']+)', datetime\.datetime\(([^\)]+)\), '([^']+)', ([0-9]+)\)", line) try: userid, name, id, eventName, eventType, datetime, JSONParams, contextualInfoId = m.groups(); name = name.strip() year, month, day, *rest = [int(item) for item in datetime.split(', ')]