def adjust(self, adjust_categories=[], categories_as_strings=[]): # adjust_categories as indexes [5, 0 , 4...] # categories_as_strings as string [coding, future...] # Make cat_string list to indexes: for cat in categories_as_strings: for index, _ in enumerate(config.CATEGORIES): if cat == config.CATEGORIES[index]: adjust_categories.append(index) # Categories valued as two posts. # To get the post amount: cat / data_lenght = average_post_amount # Adjust for index in adjust_categories: self.category[index] += float( self.category[index] / self.data_length) * 2 # Update self.update() # Delete old database.commit_query( "DELETE FROM interesting_posts WHERE username=%s", (self.username, ))
def ping(): ''' AJAX Call for ping: Test the connection and maybe start a profiler ''' analytics.REMOTE_ADDRS.append(request.remote_addr) if "username" in request.json: username = request.json["username"] database.commit_query( "INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);", ("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, "")) return jsonify({"status": "succes"})
def set_zero(self, category="", index=-1): if category != "": for i, cat in enumerate(config.CATEGORIES): if cat == category: index = i break if index > -1: self.category[index] = 0 self.update() # Delete old database.commit_query( "DELETE FROM interesting_posts WHERE username=%s", (self.username, ))
def main(): # Init config.init_server() hive.LatestPostManager() reset_analytics() # Start counter thread post_count_thread = Thread(target=database.latest_post_count_manager) post_count_thread.name = "Get latest_posts count" post_count_thread.daemon = True post_count_thread.start() # Start task manager thread task_manager_thread = Thread(target=task_manager) task_manager_thread.name = "Taskmanager" task_manager_thread.daemon = True task_manager_thread.start() # Start remove Profiler thread remove_old_profiler_thread = Thread(target=Profiler.remove_old_profiler) remove_old_profiler_thread.name = "Remove old Profiler" remove_old_profiler_thread.daemon = True #remove_old_profiler_thread.start() # Wait until something happens while True: try: _input = input() if 'exit' in _input or 'break' in _input or 'quit' in _input: # Stop break if 'tasks' in _input: # Get all tasks print(f"Tasks({len(config.statics.task_list)}) are running: ") for name, timestamp, _, _ in config.statics.task_list: print(f"{name} from {timestamp}") if 'profiler' in _input: # Profile one --> Add task username = _input.split(' ')[1] #profiler christopher2002 database.commit_query("INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);", ("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, "")) print("Created profiling task for " + username) except KeyboardInterrupt: break
def enter_posts_by_block(self, block): for op in block.operations: if op['type'] == 'comment_operation': action = op['value'] if action['parent_author'] == '': # found post --> Categorize _input = network.WordEmbedding.vectorize_text( config.statics.Word2Vec, html=action['body'], text=action['title'] + ". ") if _input is None: # to short or error continue # Categorize _output = config.statics.TextCNN(_input).cpu() # Enter in Mysql str_arr = ' '.join(map(str, _output.data[0].tolist())) result = database.commit_query( self.query, (action['author'], action['permlink'], str_arr, block["timestamp"].strftime("%d.%m.%YT%H:%M:%S"))) if result == -1: print("[WARNING] Can't enter post in database!") time.sleep(5)
def update(self): # Update category, data_length, timestamp in database query = "UPDATE profiler SET category=%s, length=%s, timestamp=%s, finished=%s WHERE username=%s;" str_arr = ' '.join(map(str, self.category)) result = database.commit_query( query, (str_arr, self.data_length, datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), self.finished, self.username))
def run(): con = config.get_connection() while 1: start_time = time.time() request_count = len(REMOTE_ADDRS) connection_count = len(list(dict.fromkeys(REMOTE_ADDRS))) REMOTE_ADDRS.clear() # Update database.commit_query("UPDATE analytics SET value_one=%s WHERE name=%s", (request_count, "requests"), con=con, close_con=False) database.commit_query("UPDATE analytics SET value_one=%s WHERE name=%s", (connection_count, "connections"), con=con, close_con=False) # wait a second (exactly) time.sleep((1000 - (time.time() - start_time)))
def clean_up(self): # TODO: Implement sorting order by timestamp query = "SELECT timestamp FROM latest_posts;" for item in database.read_query(query, None): timestamp = datetime.strptime(item[0], "%d.%m.%YT%H:%M:%S") if timestamp < (datetime.utcnow() - timedelta(days=5)): result = database.commit_query( "DELETE FROM latest_posts WHERE timestamp=%s;", (item[0], ))
def delete_user(): ''' AJAX CALL: delete user''' analytics.REMOTE_ADDRS.append(request.remote_addr) if "username" not in request.json: # Error (No username is given) return jsonify({ "status": "failed", "code": 3, "msg": "No username is given." }) username = request.json["username"] # Enter in tasks database.commit_query( "INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);", ("delete_user", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, "")) return jsonify({"status": "succes"})
def ping(request): username = request.GET.get('username', None) if username is None: # If only the server status is requested return render(request, 'ping.js', context={'cmd': 'true'}, content_type="application/x-javascript") # Start profiler if needed database.commit_query( "INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);", ("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, "")) return render(request, 'ping.js', context={'cmd': 'true'}, content_type="application/x-javascript")
def set_to_zero(): ''' AJAX CALL: Set category for user to zero''' analytics.REMOTE_ADDRS.append(request.remote_addr) if "username" not in request.json or "cat" not in request.json: # Error (No username, cats is given) --> return Error.js return jsonify({ "status": "failed", "code": 7, "msg": "No username/category is given." }) username = request.json["username"] cat = request.json["cat"] # Enter in tasks database.commit_query( "INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);", ("set_to_zero", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, cat)) return jsonify({"status": "succes"})
def reset_analytics(): con = config.get_connection() # Delete all database.commit_query("SET SQL_SAFE_UPDATES = 0;", (), con=con, close_con=False) database.commit_query("DELETE FROM analytics", (), con=con, close_con=False) # tasks_running (currently) database.commit_query("INSERT INTO analytics(name, value_one, value_two, value_three) VALUES (%s, %s, %s, %s)", ("tasks_running", 0, None, None), con=con, close_con=False) # connections (per second) database.commit_query("INSERT INTO analytics(name, value_one, value_two, value_three) VALUES (%s, %s, %s, %s)", ("connections", 0, None, None), con=con, close_con=False) # requests (per second) database.commit_query("INSERT INTO analytics(name, value_one, value_two, value_three) VALUES (%s, %s, %s, %s)", ("requests", 0, None, None), con=con, close_con=False) con.close() print("[INFO] Analysing is running")
def remove_old_profiler(): con = config.get_connection() while 1: # wait time.sleep(30) # Get oldest profilers (10) profilers = database.read_query( "SELECT username, timestamp FROM profiler ORDER BY timestamp ASC LIMIT 10;", (), con=con, close_con=False) for username, timestamp in profilers: timestamp = datetime.strptime(timestamp, "%d.%m.%YT%H:%M:%S") if timestamp < (datetime.utcnow() - timedelta(hours=config.PROFILER_DELETE_TIME)): # To old --> remove database.commit_query("SET SQL_SAFE_UPDATES = 0;", (), con=con, close_con=False) database.commit_query( "DELETE FROM profiler WHERE username=%s;", (username, ), con=con, close_con=False) # Remove interesting Posts database.commit_query( "DELETE FROM interesting_posts WHERE username=%s;", (username, ), con=con, close_con=False)
def __init__(self, username: str, start_get_post_thread=True): self.username = username self.account = Account(username, blockchain_instance=Hive()) mysql_con = config.get_connection() if mysql_con is None: print( "[INFO] Can't start Latest Post Manager because of an mysql database error!" ) return result = database.read_query( "SELECT * FROM profiler WHERE username=%s;", (username, ), con=mysql_con, close_con=False) if len(result) == 0: # No profiler exists, create one self.category = [0 for i in config.CATEGORIES] self.data_length = 0 self.finished = False result = database.commit_query( "INSERT INTO profiler(username, category, length, timestamp, finished) VALUES (%s, %s, %s, %s, %s);", (username, ' '.join(map(str, self.category)), self.data_length, datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), False), con=mysql_con, close_con=False) if result <= 0: # Error print("[WARNING] Can't add Profiler for " + username) else: # Start analyze Thread, if the profiler existed bevor, this thread already run! self.analyze_thread = Thread(target=self.analyze_activity) self.analyze_thread.name = "Analyze Activities from " + username self.analyze_thread.daemon = True self.analyze_thread.start() else: # Load existent Profiler self.update_timestamp() self.category = [float(x) for x in result[0][1].split(' ')] self.data_length = result[0][2] self.finished = "1" in result[0][4] mysql_con.close() # Start finder thread self.find_posts_thread = Thread(target=self.find_interestings) self.find_posts_thread.name = "Find interesting Posts for " + username self.find_posts_thread.daemon = True if start_get_post_thread: self.find_posts_thread.start()
def run(task): name, timestamp, p_one, p_two = task if 'profiler' in name: p = Profiler(p_one, start_get_post_thread=False) p.find_interestings() if 'adjust' in name: p = Profiler(p_one, start_get_post_thread=False) p.adjust(categories_as_strings=p_two.split(',')) if 'set_to_zero' in name: p = Profiler(p_one, start_get_post_thread=False) p.set_zero(category=p_two) if 'delete_user' in name: database.commit_query("DELETE FROM profiler WHERE username=%s;", (p_one, )) database.commit_query("DELETE FROM interesting_posts WHERE username=%s;", (p_one, )) # delete task config.statics.task_list.remove(task)
def get_interesting_posts(request): username = request.GET.get('username', None) if username is None: # Error (No Username is given) --> return Error.js return render( request, 'error.js', context={"info": "Please enter a 'username' and use GET"}, content_type="application/x-javascript") con = config.get_connection() posts = database.read_query( "SELECT * FROM interesting_posts WHERE username=%s;", (username, ), con=con, close_con=False) database.commit_query( "INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);", ("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, ""), con=con, close_con=False) database.commit_query("SET SQL_SAFE_UPDATES = 0;", (), con=con, close_con=False) obj = "" length = 0 for _, author, permlink in posts: obj += f"{author}/{permlink};" database.commit_query( "DELETE FROM interesting_posts WHERE username=%s AND author=%s AND permlink=%s;", (username, author, permlink), con=con, close_con=False) length += 1 if length >= 3: # Return only 3 break return render(request, 'get_interesting_posts.js', context={"posts": obj[:-1]}, content_type="application/x-javascript")
def get_interesting_posts(): ''' Ajax Call: get 3 interesting posts ''' analytics.REMOTE_ADDRS.append(request.remote_addr) if "username" not in request.json: # Return error json, if no username is given return jsonify({ "status": "failed", "code": 1, "message": "No username is given" }) username = request.json["username"] # Get 3 posts LIMIT = 3 con = config.get_connection() posts = database.read_query( "SELECT * FROM interesting_posts WHERE username=%s LIMIT %s;", (username, LIMIT), con=con, close_con=False) # Prepare and Delete them database.commit_query("SET SQL_SAFE_UPDATES = 0;", (), con=con, close_con=False) for index, (_, author, permlink) in enumerate(posts): posts[index] = {"author": author, "permlink": permlink} database.commit_query( "DELETE FROM interesting_posts WHERE username=%s AND author=%s AND permlink=%s;", (username, author, permlink), con=con, close_con=False) # Start profiler database.commit_query( "INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);", ("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, ""), con=con, close_con=False) return jsonify({"status": "succes", "posts": posts})
def task_manager(): con = config.get_connection() while 1: # Update analytics database.commit_query("UPDATE analytics SET value_one=%s WHERE name=%s", (len(config.statics.task_list), "tasks_running"), con=con, close_con=False) while len(config.statics.task_list) >= config.MAX_TASK_THREADS: # Thread limit is reached --> Wait for completing time.sleep(0.5) # Get first available task tasks = database.read_query("SELECT * FROM tasks LIMIT 1;", (), con=con, close_con=False) if len(tasks) == 0: # If nothing is to do time.sleep(0.5) continue # Get first element and test if it is running name, timestamp, p_one, p_two = tasks[0] already_running = False for _name, _, _p_one, _p_two in config.statics.task_list: if name in _name and p_one in _p_one and p_two in _p_two: # already running already_running = True break # Delete element database.commit_query("SET SQL_SAFE_UPDATES = 0;", (), con=con, close_con=False) database.commit_query("DELETE FROM tasks WHERE name=%s AND parameter_one=%s AND parameter_two=%s;", (name, p_one, p_two), con=con, close_con=False) if already_running: # abort continue # Insert in list and Run config.statics.task_list.append(tasks[0]) def run(task): name, timestamp, p_one, p_two = task if 'profiler' in name: p = Profiler(p_one, start_get_post_thread=False) p.find_interestings() if 'adjust' in name: p = Profiler(p_one, start_get_post_thread=False) p.adjust(categories_as_strings=p_two.split(',')) if 'set_to_zero' in name: p = Profiler(p_one, start_get_post_thread=False) p.set_zero(category=p_two) if 'delete_user' in name: database.commit_query("DELETE FROM profiler WHERE username=%s;", (p_one, )) database.commit_query("DELETE FROM interesting_posts WHERE username=%s;", (p_one, )) # delete task config.statics.task_list.remove(task) # Start thread t = Thread(target=run, args=(tasks[0], )) t.name = f"T - {name} ({str(p_one)};{str(p_two)})" t.daemon = True t.start()
def update_timestamp(self): # Update timestamp in database --> No kickoff query = "UPDATE profiler SET timestamp=%s WHERE username=%s;" result = database.commit_query( query, (datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), self.username))
def find_interestings(self): mysql_con = config.get_connection() while self.data_length < config.PROFILER_MIN_DATA: # Wait until enough data is availabel time.sleep(0.2) percentages, top_cats = [], [] last_data_len = -1 while 1: # 0 Step: Check if post limit is reached interesting_posts = database.read_query( "SELECT * FROM interesting_posts WHERE username=%s", (self.username, ), con=mysql_con, close_con=False) if len(interesting_posts) >= config.MAX_INTERSTING_POSTS: break # 1 Step: Calc percentages and top cats if len(percentages) == 0 or last_data_len != self.data_length: # If no percentages are calced or data_length increased last_data_len = self.data_length percentages = helper.calc_percentages(self.category) top_cats = helper.get_top_elements(percentages, 10) # 2 Step: get random Posts offset = random.randint(0, config.statics.LATEST_POSTS_START_LIMIT - 50) # random offset posts = database.read_query("SELECT * FROM latest_posts LIMIT " + str(offset) + ", 50;", (), con=mysql_con, close_con=False) # 3 Step: prepare posts to compare post_percentages = [] # ([top_percentages], author, permlink) for author, permlink, category, timestamp in posts: category = [float(x) for x in category.split(' ')] # Calc percentages and top ones percs = helper.calc_percentages(category) top_pers = helper.get_top_elements(percs, 10) post_percentages.append((top_pers, author, permlink)) # 4 Step: Compare for top_pers, author, permlink in post_percentages: score = 0 diff = 0 for value, index in top_cats: for p_value, p_index in top_pers: diff += abs(value - p_value) if p_index == index: # Same top Category score += 1 #if score >= 7: if diff <= 2: exists = database.read_query( "SELECT author FROM interesting_posts WHERE username=%s AND author=%s AND permlink=%s;", (self.username, author, permlink), con=mysql_con, close_con=False) if len(exists) > 0: # Already listed --> Next one continue result = database.commit_query( "INSERT INTO interesting_posts(username, author, permlink) VALUES (%s, %s, %s);", (self.username, author, permlink), con=mysql_con, close_con=False) if result < 1: # Error print("[WARNING] Can't insert an interesting post!") time.sleep(5) self.update_timestamp()