def adjust(self, adjust_categories=[], categories_as_strings=[]):
# adjust_categories as indexes [5, 0 , 4...]
# categories_as_strings as string [coding, future...]
# Make cat_string list to indexes:
for cat in categories_as_strings:
for index, _ in enumerate(config.CATEGORIES):
if cat == config.CATEGORIES[index]:
adjust_categories.append(index)
# Categories valued as two posts.
# To get the post amount: cat / data_lenght = average_post_amount
# Adjust
for index in adjust_categories:
self.category[index] += float(
self.category[index] / self.data_length) * 2
# Update
self.update()
# Delete old
database.commit_query(
"DELETE FROM interesting_posts WHERE username=%s",
(self.username, ))
def ping():
''' AJAX Call for ping: Test the connection and maybe start a profiler '''
analytics.REMOTE_ADDRS.append(request.remote_addr)
if "username" in request.json:
username = request.json["username"]
database.commit_query(
"INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);",
("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"),
username, ""))
return jsonify({"status": "succes"})
def set_zero(self, category="", index=-1):
if category != "":
for i, cat in enumerate(config.CATEGORIES):
if cat == category:
index = i
break
if index > -1:
self.category[index] = 0
self.update()
# Delete old
database.commit_query(
"DELETE FROM interesting_posts WHERE username=%s",
(self.username, ))
def main():
# Init
config.init_server()
hive.LatestPostManager()
reset_analytics()
# Start counter thread
post_count_thread = Thread(target=database.latest_post_count_manager)
post_count_thread.name = "Get latest_posts count"
post_count_thread.daemon = True
post_count_thread.start()
# Start task manager thread
task_manager_thread = Thread(target=task_manager)
task_manager_thread.name = "Taskmanager"
task_manager_thread.daemon = True
task_manager_thread.start()
# Start remove Profiler thread
remove_old_profiler_thread = Thread(target=Profiler.remove_old_profiler)
remove_old_profiler_thread.name = "Remove old Profiler"
remove_old_profiler_thread.daemon = True
#remove_old_profiler_thread.start()
# Wait until something happens
while True:
try:
_input = input()
if 'exit' in _input or 'break' in _input or 'quit' in _input:
# Stop
break
if 'tasks' in _input:
# Get all tasks
print(f"Tasks({len(config.statics.task_list)}) are running: ")
for name, timestamp, _, _ in config.statics.task_list:
print(f"{name} from {timestamp}")
if 'profiler' in _input:
# Profile one --> Add task
username = _input.split(' ')[1] #profiler christopher2002
database.commit_query("INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);",
("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username, ""))
print("Created profiling task for " + username)
except KeyboardInterrupt:
break
def enter_posts_by_block(self, block):
for op in block.operations:
if op['type'] == 'comment_operation':
action = op['value']
if action['parent_author'] == '':
# found post --> Categorize
_input = network.WordEmbedding.vectorize_text(
config.statics.Word2Vec,
html=action['body'],
text=action['title'] + ". ")
if _input is None:
# to short or error
continue
# Categorize
_output = config.statics.TextCNN(_input).cpu()
# Enter in Mysql
str_arr = ' '.join(map(str, _output.data[0].tolist()))
result = database.commit_query(
self.query,
(action['author'], action['permlink'], str_arr,
block["timestamp"].strftime("%d.%m.%YT%H:%M:%S")))
if result == -1:
print("[WARNING] Can't enter post in database!")
time.sleep(5)
def update(self):
# Update category, data_length, timestamp in database
query = "UPDATE profiler SET category=%s, length=%s, timestamp=%s, finished=%s WHERE username=%s;"
str_arr = ' '.join(map(str, self.category))
result = database.commit_query(
query, (str_arr, self.data_length,
datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"),
self.finished, self.username))
def run():
con = config.get_connection()
while 1:
start_time = time.time()
request_count = len(REMOTE_ADDRS)
connection_count = len(list(dict.fromkeys(REMOTE_ADDRS)))
REMOTE_ADDRS.clear()
# Update
database.commit_query("UPDATE analytics SET value_one=%s WHERE name=%s",
(request_count, "requests"), con=con, close_con=False)
database.commit_query("UPDATE analytics SET value_one=%s WHERE name=%s",
(connection_count, "connections"), con=con, close_con=False)
# wait a second (exactly)
time.sleep((1000 - (time.time() - start_time)))
def clean_up(self):
# TODO: Implement sorting order by timestamp
query = "SELECT timestamp FROM latest_posts;"
for item in database.read_query(query, None):
timestamp = datetime.strptime(item[0], "%d.%m.%YT%H:%M:%S")
if timestamp < (datetime.utcnow() - timedelta(days=5)):
result = database.commit_query(
"DELETE FROM latest_posts WHERE timestamp=%s;",
(item[0], ))
def delete_user():
''' AJAX CALL: delete user'''
analytics.REMOTE_ADDRS.append(request.remote_addr)
if "username" not in request.json:
# Error (No username is given)
return jsonify({
"status": "failed",
"code": 3,
"msg": "No username is given."
})
username = request.json["username"]
# Enter in tasks
database.commit_query(
"INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);",
("delete_user", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"),
username, ""))
return jsonify({"status": "succes"})
def ping(request):
username = request.GET.get('username', None)
if username is None:
# If only the server status is requested
return render(request,
'ping.js',
context={'cmd': 'true'},
content_type="application/x-javascript")
# Start profiler if needed
database.commit_query(
"INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);",
("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username,
""))
return render(request,
'ping.js',
context={'cmd': 'true'},
content_type="application/x-javascript")
def set_to_zero():
''' AJAX CALL: Set category for user to zero'''
analytics.REMOTE_ADDRS.append(request.remote_addr)
if "username" not in request.json or "cat" not in request.json:
# Error (No username, cats is given) --> return Error.js
return jsonify({
"status": "failed",
"code": 7,
"msg": "No username/category is given."
})
username = request.json["username"]
cat = request.json["cat"]
# Enter in tasks
database.commit_query(
"INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);",
("set_to_zero", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"),
username, cat))
return jsonify({"status": "succes"})
def reset_analytics():
con = config.get_connection()
# Delete all
database.commit_query("SET SQL_SAFE_UPDATES = 0;", (), con=con, close_con=False)
database.commit_query("DELETE FROM analytics", (), con=con, close_con=False)
# tasks_running (currently)
database.commit_query("INSERT INTO analytics(name, value_one, value_two, value_three) VALUES (%s, %s, %s, %s)",
("tasks_running", 0, None, None), con=con, close_con=False)
# connections (per second)
database.commit_query("INSERT INTO analytics(name, value_one, value_two, value_three) VALUES (%s, %s, %s, %s)",
("connections", 0, None, None), con=con, close_con=False)
# requests (per second)
database.commit_query("INSERT INTO analytics(name, value_one, value_two, value_three) VALUES (%s, %s, %s, %s)",
("requests", 0, None, None), con=con, close_con=False)
con.close()
print("[INFO] Analysing is running")
def remove_old_profiler():
con = config.get_connection()
while 1:
# wait
time.sleep(30)
# Get oldest profilers (10)
profilers = database.read_query(
"SELECT username, timestamp FROM profiler ORDER BY timestamp ASC LIMIT 10;",
(),
con=con,
close_con=False)
for username, timestamp in profilers:
timestamp = datetime.strptime(timestamp, "%d.%m.%YT%H:%M:%S")
if timestamp < (datetime.utcnow() -
timedelta(hours=config.PROFILER_DELETE_TIME)):
# To old --> remove
database.commit_query("SET SQL_SAFE_UPDATES = 0;", (),
con=con,
close_con=False)
database.commit_query(
"DELETE FROM profiler WHERE username=%s;",
(username, ),
con=con,
close_con=False)
# Remove interesting Posts
database.commit_query(
"DELETE FROM interesting_posts WHERE username=%s;",
(username, ),
con=con,
close_con=False)
def __init__(self, username: str, start_get_post_thread=True):
self.username = username
self.account = Account(username, blockchain_instance=Hive())
mysql_con = config.get_connection()
if mysql_con is None:
print(
"[INFO] Can't start Latest Post Manager because of an mysql database error!"
)
return
result = database.read_query(
"SELECT * FROM profiler WHERE username=%s;", (username, ),
con=mysql_con,
close_con=False)
if len(result) == 0:
# No profiler exists, create one
self.category = [0 for i in config.CATEGORIES]
self.data_length = 0
self.finished = False
result = database.commit_query(
"INSERT INTO profiler(username, category, length, timestamp, finished) VALUES (%s, %s, %s, %s, %s);",
(username, ' '.join(map(str, self.category)), self.data_length,
datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), False),
con=mysql_con,
close_con=False)
if result <= 0:
# Error
print("[WARNING] Can't add Profiler for " + username)
else:
# Start analyze Thread, if the profiler existed bevor, this thread already run!
self.analyze_thread = Thread(target=self.analyze_activity)
self.analyze_thread.name = "Analyze Activities from " + username
self.analyze_thread.daemon = True
self.analyze_thread.start()
else:
# Load existent Profiler
self.update_timestamp()
self.category = [float(x) for x in result[0][1].split(' ')]
self.data_length = result[0][2]
self.finished = "1" in result[0][4]
mysql_con.close()
# Start finder thread
self.find_posts_thread = Thread(target=self.find_interestings)
self.find_posts_thread.name = "Find interesting Posts for " + username
self.find_posts_thread.daemon = True
if start_get_post_thread:
self.find_posts_thread.start()
def run(task):
name, timestamp, p_one, p_two = task
if 'profiler' in name:
p = Profiler(p_one, start_get_post_thread=False)
p.find_interestings()
if 'adjust' in name:
p = Profiler(p_one, start_get_post_thread=False)
p.adjust(categories_as_strings=p_two.split(','))
if 'set_to_zero' in name:
p = Profiler(p_one, start_get_post_thread=False)
p.set_zero(category=p_two)
if 'delete_user' in name:
database.commit_query("DELETE FROM profiler WHERE username=%s;",
(p_one, ))
database.commit_query("DELETE FROM interesting_posts WHERE username=%s;",
(p_one, ))
# delete task
config.statics.task_list.remove(task)
def get_interesting_posts(request):
username = request.GET.get('username', None)
if username is None:
# Error (No Username is given) --> return Error.js
return render(
request,
'error.js',
context={"info": "Please enter a 'username' and use GET"},
content_type="application/x-javascript")
con = config.get_connection()
posts = database.read_query(
"SELECT * FROM interesting_posts WHERE username=%s;", (username, ),
con=con,
close_con=False)
database.commit_query(
"INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);",
("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username,
""),
con=con,
close_con=False)
database.commit_query("SET SQL_SAFE_UPDATES = 0;", (),
con=con,
close_con=False)
obj = ""
length = 0
for _, author, permlink in posts:
obj += f"{author}/{permlink};"
database.commit_query(
"DELETE FROM interesting_posts WHERE username=%s AND author=%s AND permlink=%s;",
(username, author, permlink),
con=con,
close_con=False)
length += 1
if length >= 3:
# Return only 3
break
return render(request,
'get_interesting_posts.js',
context={"posts": obj[:-1]},
content_type="application/x-javascript")
def get_interesting_posts():
''' Ajax Call: get 3 interesting posts '''
analytics.REMOTE_ADDRS.append(request.remote_addr)
if "username" not in request.json:
# Return error json, if no username is given
return jsonify({
"status": "failed",
"code": 1,
"message": "No username is given"
})
username = request.json["username"]
# Get 3 posts
LIMIT = 3
con = config.get_connection()
posts = database.read_query(
"SELECT * FROM interesting_posts WHERE username=%s LIMIT %s;",
(username, LIMIT),
con=con,
close_con=False)
# Prepare and Delete them
database.commit_query("SET SQL_SAFE_UPDATES = 0;", (),
con=con,
close_con=False)
for index, (_, author, permlink) in enumerate(posts):
posts[index] = {"author": author, "permlink": permlink}
database.commit_query(
"DELETE FROM interesting_posts WHERE username=%s AND author=%s AND permlink=%s;",
(username, author, permlink),
con=con,
close_con=False)
# Start profiler
database.commit_query(
"INSERT INTO tasks(name, timestamp, parameter_one, parameter_two) VALUES (%s, %s, %s, %s);",
("profiler", datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), username,
""),
con=con,
close_con=False)
return jsonify({"status": "succes", "posts": posts})
def task_manager():
con = config.get_connection()
while 1:
# Update analytics
database.commit_query("UPDATE analytics SET value_one=%s WHERE name=%s",
(len(config.statics.task_list), "tasks_running"), con=con, close_con=False)
while len(config.statics.task_list) >= config.MAX_TASK_THREADS:
# Thread limit is reached --> Wait for completing
time.sleep(0.5)
# Get first available task
tasks = database.read_query("SELECT * FROM tasks LIMIT 1;", (), con=con, close_con=False)
if len(tasks) == 0:
# If nothing is to do
time.sleep(0.5)
continue
# Get first element and test if it is running
name, timestamp, p_one, p_two = tasks[0]
already_running = False
for _name, _, _p_one, _p_two in config.statics.task_list:
if name in _name and p_one in _p_one and p_two in _p_two:
# already running
already_running = True
break
# Delete element
database.commit_query("SET SQL_SAFE_UPDATES = 0;", (), con=con, close_con=False)
database.commit_query("DELETE FROM tasks WHERE name=%s AND parameter_one=%s AND parameter_two=%s;",
(name, p_one, p_two), con=con, close_con=False)
if already_running:
# abort
continue
# Insert in list and Run
config.statics.task_list.append(tasks[0])
def run(task):
name, timestamp, p_one, p_two = task
if 'profiler' in name:
p = Profiler(p_one, start_get_post_thread=False)
p.find_interestings()
if 'adjust' in name:
p = Profiler(p_one, start_get_post_thread=False)
p.adjust(categories_as_strings=p_two.split(','))
if 'set_to_zero' in name:
p = Profiler(p_one, start_get_post_thread=False)
p.set_zero(category=p_two)
if 'delete_user' in name:
database.commit_query("DELETE FROM profiler WHERE username=%s;",
(p_one, ))
database.commit_query("DELETE FROM interesting_posts WHERE username=%s;",
(p_one, ))
# delete task
config.statics.task_list.remove(task)
# Start thread
t = Thread(target=run, args=(tasks[0], ))
t.name = f"T - {name} ({str(p_one)};{str(p_two)})"
t.daemon = True
t.start()
def update_timestamp(self):
# Update timestamp in database --> No kickoff
query = "UPDATE profiler SET timestamp=%s WHERE username=%s;"
result = database.commit_query(
query,
(datetime.utcnow().strftime("%d.%m.%YT%H:%M:%S"), self.username))
def find_interestings(self):
mysql_con = config.get_connection()
while self.data_length < config.PROFILER_MIN_DATA:
# Wait until enough data is availabel
time.sleep(0.2)
percentages, top_cats = [], []
last_data_len = -1
while 1:
# 0 Step: Check if post limit is reached
interesting_posts = database.read_query(
"SELECT * FROM interesting_posts WHERE username=%s",
(self.username, ),
con=mysql_con,
close_con=False)
if len(interesting_posts) >= config.MAX_INTERSTING_POSTS:
break
# 1 Step: Calc percentages and top cats
if len(percentages) == 0 or last_data_len != self.data_length:
# If no percentages are calced or data_length increased
last_data_len = self.data_length
percentages = helper.calc_percentages(self.category)
top_cats = helper.get_top_elements(percentages, 10)
# 2 Step: get random Posts
offset = random.randint(0,
config.statics.LATEST_POSTS_START_LIMIT -
50) # random offset
posts = database.read_query("SELECT * FROM latest_posts LIMIT " +
str(offset) + ", 50;", (),
con=mysql_con,
close_con=False)
# 3 Step: prepare posts to compare
post_percentages = [] # ([top_percentages], author, permlink)
for author, permlink, category, timestamp in posts:
category = [float(x) for x in category.split(' ')]
# Calc percentages and top ones
percs = helper.calc_percentages(category)
top_pers = helper.get_top_elements(percs, 10)
post_percentages.append((top_pers, author, permlink))
# 4 Step: Compare
for top_pers, author, permlink in post_percentages:
score = 0
diff = 0
for value, index in top_cats:
for p_value, p_index in top_pers:
diff += abs(value - p_value)
if p_index == index:
# Same top Category
score += 1
#if score >= 7:
if diff <= 2:
exists = database.read_query(
"SELECT author FROM interesting_posts WHERE username=%s AND author=%s AND permlink=%s;",
(self.username, author, permlink),
con=mysql_con,
close_con=False)
if len(exists) > 0:
# Already listed --> Next one
continue
result = database.commit_query(
"INSERT INTO interesting_posts(username, author, permlink) VALUES (%s, %s, %s);",
(self.username, author, permlink),
con=mysql_con,
close_con=False)
if result < 1:
# Error
print("[WARNING] Can't insert an interesting post!")
time.sleep(5)
self.update_timestamp()
