def delete_non_existing_remote_posts(node: str, posts_dict_list: list):
try:
existing_posts_ids = [post_dict["id"] for post_dict in posts_dict_list]
Post.objects.filter(origin__icontains=node).exclude(
id__in=existing_posts_ids).delete()
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def update_remote_authors(host: str, auth: str):
try:
url = f"{host}author"
stime = time.time()
response = requests.get(
url,
headers={
"Authorization": f"Basic {auth}",
"Accept": "application/json",
},
)
print("Time used for request user:"******"{response.text}")
else:
raw_author_dict_list = response.json()
author_dict_list = [] # processed valid list
for raw_author_dict in raw_author_dict_list:
valid, author_dict = tidy_user_data(raw_author_dict, host)
if not valid:
continue
author_dict_list.append(author_dict)
create_or_update_remote_users(host, author_dict_list)
delete_non_existing_remote_users(host, author_dict_list)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def tidy_comment_data(data: dict, post_id: str) -> (bool, dict):
"""
Tidy up the comment data received from other servers
"""
new_data = {}
try:
author_dict = data.pop("author", None)
author = None
if author_dict["host"] == REMOTE_HOST1:
author_dict["non_uuid_id"] = author_dict["id"].split("/")[-1]
author = User.objects.filter(
host=author_dict["host"],
non_uuid_id=author_dict["non_uuid_id"]).first()
else:
author_dict["id"] = author_dict["id"].split("/")[-1]
author = User.objects.filter(id=author_dict["id"]).first()
if not author:
return False, new_data
new_data["author"] = author
new_data["post"] = post_id
new_data["id"] = data["id"]
new_data["comment"] = data["comment"]
new_data["published"] = data["published"]
if "contentType" in data.keys():
new_data["contentType"] = data["contentType"]
return True, new_data
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
return False, new_data
def create_or_update_remote_posts(posts_dict_list: list):
try:
for post_dict in posts_dict_list:
obj, created = Post.objects.update_or_create(
id=post_dict["id"],
defaults=post_dict,
)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def send_remote_comments(comment, post, author) -> bool:
try:
author_id = None
if post.origin == REMOTE_HOST1:
author_id = author.id
else:
author_id = str(author.id).replace("-", "")
author_dict = {
"id": f"{author.host}author/{author_id}",
"host": f"{author.host}",
"displayName": f"{author.displayName}",
"url": f"{author.host}author/{author_id}",
"github": f"{author.github}",
}
comment_id = None
if post.origin == REMOTE_HOST1:
comment_id = comment["id"]
else:
comment_id = comment["id"].replace("-", "")
comment_dict = {
"author": author_dict,
"comment": comment["comment"],
"contentType": comment["contentType"],
"published": comment["published"],
"id": comment_id,
}
request_data = {
"query": "addComment",
"post": f"{post.origin}posts/{post.id}",
"comment": comment_dict,
}
url = f"{post.origin}posts/{str(post.id)}/comments"
if post.origin != REMOTE_HOST1:
url += "/"
node = Node.objects.filter(host=post.origin).first()
headers = {
"Authorization": f"Basic {node.auth}",
"Content-Type": "application/json",
"Accept": "application/json",
}
response = requests.post(
url,
data=json.dumps(request_data),
headers=headers,
)
if response.status_code not in range(200, 300):
print(response.status_code)
print(url)
print(headers)
print(json.dumps(request_data))
raise Exception(response.text)
return True
except Exception as e:
utils.print_warning(e)
return False
def post_comments(self, request, *args, **kwargs):
"""
# POST to http://service/posts/{POST_ID}/comments
"""
response_data = {
"query": "addComment",
"success": "",
"message": "",
}
try:
post_id = kwargs["POST_ID"]
post = Post.objects.filter(id=post_id).first()
if not post:
raise Exception("Not Found")
except:
response_data["success"] = "false"
response_data["message"] = "Post does not exist"
return Response(response_data, status=status.HTTP_404_NOT_FOUND)
else:
try:
if is_post_visible_to(post, request.user):
try:
comment = request.data["comment"].copy()
if Comment.objects.filter(id=comment["id"]).exists():
raise Exception("Comment id already exists.")
author_data = comment.pop("author")
author_data["id"] = author_data["id"].split("/")[-1]
author = User.objects.filter(
id=author_data["id"]).first()
if not author:
raise Exception("Author not found")
serializer = CommentSerializer(data=comment)
if serializer.is_valid():
serializer.save(author=author, post=post)
response_data["success"] = "true"
response_data["message"] = "Comment Added"
return Response(response_data,
status=status.HTTP_201_CREATED)
else:
raise Exception("Bad request body")
except Exception as e:
response_data["success"] = "false"
response_data[
"message"] = f"{str(type(e).__name__)}:{str(e)}"
return Response(response_data,
status=status.HTTP_400_BAD_REQUEST)
else:
response_data["success"] = "false"
response_data["message"] = "Comment not allowed"
return Response(response_data,
status=status.HTTP_403_FORBIDDEN)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def send_friend_request(author: User, friend: User) -> bool:
"""
send friend request to remote user
Params:
author: User, friend: User, request_user: User
"""
try:
node = Node.objects.filter(host=friend.host).first()
if not node:
raise Exception("Node does not exist")
url = f"{node.host}friendrequest"
author_dict = {
"id": f"{author.host}author/{author.id}",
"host": author.host,
"displayName": author.displayName,
"url": f"{author.host}author/{author.id}",
}
if friend.host == REMOTE_HOST1:
friend_dict = {
"id": f"{friend.host}author/{friend.non_uuid_id}",
"host": friend.host,
"displayName": friend.displayName,
"url": f"{friend.host}author/{friend.non_uuid_id}",
}
else:
friend_dict = {
"id": f"{friend.host}author/{friend.id}",
"host": friend.host,
"displayName": friend.displayName,
"url": f"{friend.host}author/{friend.id}",
}
request_body = {
"query": "friendrequest",
"author": author_dict,
"friend": friend_dict,
}
response = requests.post(
url,
data=json.dumps(request_body),
headers={
"Authorization": f"Basic {node.auth}",
"Content-Type": "application/json",
"Accept": "application/json",
},
)
if response.status_code not in range(200, 300):
print(response.status_code)
raise Exception(response.text)
return True
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
return False
def delete_non_existing_remote_comments(posts_dict_list: list,
comments_dict_list: list):
try:
existing_comments_ids = [
comment_dict["id"] for comment_dict in comments_dict_list
]
for post_dict in posts_dict_list:
post = Post.objects.filter(id=post_dict["id"]).first()
Comment.objects.filter(post=post).exclude(
id__in=existing_comments_ids).delete()
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def create_or_update_remote_comments(comments_dict_list: list):
try:
for comment_dict in comments_dict_list:
post_id = comment_dict.pop("post", None)
post = Post.objects.filter(id=post_id).first()
obj, created = Comment.objects.update_or_create(
id=comment_dict["id"],
post=post,
defaults=comment_dict,
)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def delete_non_existing_remote_users(host: str, author_dict_list: list):
try:
if host == REMOTE_HOST1:
non_uuid_ids = [
author_dict["non_uuid_id"] for author_dict in author_dict_list
]
User.objects.filter(host=host).exclude(
non_uuid_id__in=non_uuid_ids).delete()
else:
ids = [author_dict["id"] for author_dict in author_dict_list]
User.objects.filter(host=host).exclude(id__in=ids).delete()
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def update_remote_posts(host: str, auth: str):
url = f"{host}author/posts"
stime = time.time()
response = requests.get(url,
headers={
"Authorization": f"Basic {auth}",
"Accept": "application/json",
})
print(f"Time used fot post request:", time.time() - stime)
if response.status_code not in range(200, 300):
utils.print_warning(
f"Warning: {url} GET method failed with status code {response.status_code}"
)
else:
try:
data = response.json()
raw_posts_dict_list = data["posts"]
posts_dict_list = []
all_comments_dict_list = []
for raw_post_dict in raw_posts_dict_list:
author_dict = raw_post_dict.pop("author", None)
author = None
if author_dict["host"] == REMOTE_HOST1:
author_dict["non_uuid_id"] = author_dict["id"].split(
"/")[-1]
author = User.objects.filter(
host=author_dict["host"],
non_uuid_id=author_dict["non_uuid_id"]).first()
else:
author_dict["id"] = author_dict["id"].split("/")[-1]
author = User.objects.filter(id=author_dict["id"]).first()
if not author:
# author not cached, ignore this post
continue
else:
valid, post_dict, comments_dict_list = tidy_post_data(
raw_post_dict, host, author)
all_comments_dict_list += comments_dict_list
if valid:
posts_dict_list.append(post_dict)
create_or_update_remote_posts(posts_dict_list)
delete_non_existing_remote_posts(host, posts_dict_list)
create_or_update_remote_comments(all_comments_dict_list)
delete_non_existing_remote_comments(posts_dict_list,
all_comments_dict_list)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def pull_github_events(user: User):
"""
Pull user's github events into his stream
"""
try:
if not user:
raise Exception("param 'user' is null")
if user.github is None or user.github == "":
Post.objects.filter(author=user).exclude(githubId=None).delete()
return
git_username = user.github.split("/")[-1]
url = f"https://api.github.com/users/{git_username}/events"
response = requests.get(url, headers={"Accept": "application/json"})
if response.status_code not in range(200, 300):
raise Exception(response.text)
events = response.json()
for event in events:
if not Post.objects.filter(githubId=int(event["id"])).exists():
actor = event["actor"]["login"]
action = None
try:
action = event["payload"]["action"]
except:
action = "had"
event_type = event["type"]
repo = event["repo"]["name"]
visibility = "PUBLIC" if event["public"] else "PRIVATE"
create_at = parser.parse(event["created_at"].replace(
"Z", ".326198Z"))
create_at = create_at.replace(
tzinfo=pytz.timezone("MST7MDT")).isoformat()
Post.objects.create(
title="Github Activity",
description="Github Activity",
content=f"{actor} {action} {event_type} at {repo}",
contentType="text/plain",
author=user,
visibility=visibility,
published=create_at,
githubId=int(event["id"]),
)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
return
def update_friendship(self, request, *args, **kwargs):
try:
friend_data = request.data["friend"]
friend_data["id"] = friend_data["id"].split("/")[-1]
friend = User.objects.filter(id=friend_data["id"]).first()
if not friend:
raise Exception("'Friend' does not exist")
author_data = request.data["author"]
author_data["id"] = author_data["id"].split("/")[-1]
author = User.objects.filter(id=author_data["id"]).first()
if not author:
raise Exception("'Author' does not exist")
instance1 = Friend.objects.filter(f1Id=author.id,
f2Id=friend.id).first()
instance2 = Friend.objects.filter(f1Id=friend.id,
f2Id=author.id).first()
if not instance1 or not instance2:
raise Exception("'author' and 'friend' are not friends")
if request.data["status"] == "A":
if request.user != friend:
return Response(status=status.HTTP_403_FORBIDDEN)
data = {"status": "A"}
serializer1 = FriendSerializer(instance=instance1, data=data)
serializer2 = FriendSerializer(instance=instance2, data=data)
if serializer1.is_valid() and serializer2.is_valid():
serializer1.save()
serializer2.save()
return Response(status=status.HTTP_200_OK)
else:
raise Exception("Bad request")
elif request.data["status"] == "R":
if request.user != friend and request.user != author:
return Response(status=status.HTTP_403_FORBIDDEN)
self.perform_destroy(instance1)
self.perform_destroy(instance2)
return Response(status=status.HTTP_204_NO_CONTENT)
else:
raise Exception("Invalid opearation")
except Exception as e:
utils.print_warning(e)
return Response(status=status.HTTP_400_BAD_REQUEST)
def tidy_user_data(data: dict, node: str) -> (bool, dict):
"""
Tidy up the data received from other servers
"""
new_data = {}
try:
host = data.pop("host", None)
id = data.pop("id", None)
displayName = data.pop("displayName", None)
if not host or not id or not displayName or host not in node:
return False, new_data
else:
if host != node:
host = node
new_data["host"] = host
new_data["displayName"] = displayName
id = id.split("/")[-1]
if host == REMOTE_HOST1:
new_data["id"] = str(uuid.uuid4())
new_data["non_uuid_id"] = id
else:
new_data["id"] = id
github = data.pop("github", None)
if github:
new_data["github"] = github
bio = data.pop("bio", None)
if bio:
new_data["bio"] = github
email = data.pop("email", None)
new_data["email"] = (str(new_data["id"]) +
email if email else str(new_data["id"]) +
"@email.com")
new_data["username"] = str(new_data["id"])
return True, new_data
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
return False, new_data
def tidy_post_data(data: dict, node: str, author: User) -> (bool, dict, list):
"""
Tidy up the post data received from other servers
"""
new_data = {}
try:
new_data["author"] = author
new_data["id"] = data["id"]
new_data["title"] = data["title"]
new_data["source"] = node
new_data["content"] = data["content"]
new_data["published"] = data["published"]
new_data["visibility"] = data["visibility"]
new_data["unlisted"] = data["unlisted"]
if "description" in data.keys():
new_data["description"] = data["description"]
if "contentType" in data.keys():
new_data["contentType"] = data["contentType"]
if "categories" in data.keys():
new_data["categoriesStr"] = json.dumps(data["categories"])
if "visibleTo" in data.keys():
new_data["visibleToStr"] = json.dumps(data["visibleTo"])
if "origin" in data.keys():
new_data["origin"] = data["origin"].split("posts/")[0]
else:
new_data["origin"] = node
raw_comments = data["comments"]
valid_comments = []
for raw_comment in raw_comments:
valid, valid_comment = tidy_comment_data(raw_comment,
new_data["id"])
if valid:
valid_comments.append(valid_comment)
return True, new_data, valid_comments
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
return False, new_data, []
def create_or_update_remote_users(host: str, author_dict_list: list):
try:
for author_dict in author_dict_list:
if host == REMOTE_HOST1:
if User.objects.filter(
non_uuid_id=int(author_dict["non_uuid_id"])).exists():
author_dict.pop("id")
author_dict.pop("username")
author_dict.pop("email")
obj, created = User.objects.update_or_create(
non_uuid_id=int(author_dict["non_uuid_id"]),
defaults=author_dict,
)
else:
obj, created = User.objects.update_or_create(
non_uuid_id=int(author_dict["non_uuid_id"]),
defaults=author_dict,
)
else:
obj, created = User.objects.update_or_create(
id=author_dict["id"], defaults=author_dict)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")
def update_friends(user, depth, ignoreuser):
"""
update friendships of user
"""
try:
friends = []
if not user:
utils.print_warning("Parameter 'user' is None!")
return
if user.host == DEFAULT_HOST:
# user from local server
friends += deal_unprocessed_active_requests(user)
tmp1 = Friend.objects.filter(status="A",
f1Id=user.id,
isCopy=False).values_list("f2Id",
flat=True)
friends += deal_current_friends(tmp1, user)
tmp2 = Friend.objects.filter(status="A",
f2Id=user.id,
isCopy=False).values_list("f1Id",
flat=True)
friends += deal_current_friends(tmp2, user)
else:
# user from remote servers
url = f"{user.host}author/{str(user.id)}"
if user.host == REMOTE_HOST1:
url = f"{user.host}author/{user.non_uuid_id}"
auth = Node.objects.filter(host=user.host).first().auth
response = requests.get(
url,
headers={
"Authorization": f"Basic {auth}",
"Accept": "application/json",
},
)
if response.status_code not in range(200, 300):
no_dash_uuid = str(user.id).replace("-", "")
url = f"{user.host}author/{no_dash_uuid}"
response = requests.get(
url,
headers={
"Authorization": f"Basic {auth}",
"Accept": "application/json",
},
)
if response.status_code not in range(200, 300):
raise Exception(response.text)
friend_dicts = response.json()["friends"]
for friend_dict in friend_dicts:
friend_id_str = friend_dict["id"].split("/")[-1]
friend = None
try:
friend_id = uuid.UUID(friend_id_str)
friend = User.objects.filter(id=friend_id).first()
except ValueError:
friend = User.objects.filter(non_uuid_id=id).first()
if friend:
friends.append(friend)
if not Friend.objects.filter(
f1Id=user.id, f2Id=friend.id, status="A").exists():
Friend.objects.create(f1Id=user,
f2Id=friend,
status="A",
isCopy=False)
Friend.objects.create(f1Id=friend,
f2Id=user,
status="A",
isCopy=True)
if ignoreuser:
friends += [ignoreuser]
Friend.objects.filter(
status="A", f1Id=user.id).exclude(f2Id__in=friends).delete()
Friend.objects.filter(
status="A", f2Id=user.id).exclude(f1Id__in=friends).delete()
if depth:
for friend in friends:
update_friends(friend, 0, user)
except Exception as e:
utils.print_warning(f"{type(e).__name__} {str(e)}")