def delete_non_existing_remote_posts(node: str, posts_dict_list: list): try: existing_posts_ids = [post_dict["id"] for post_dict in posts_dict_list] Post.objects.filter(origin__icontains=node).exclude( id__in=existing_posts_ids).delete() except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def update_remote_authors(host: str, auth: str): try: url = f"{host}author" stime = time.time() response = requests.get( url, headers={ "Authorization": f"Basic {auth}", "Accept": "application/json", }, ) print("Time used for request user:"******"{response.text}") else: raw_author_dict_list = response.json() author_dict_list = [] # processed valid list for raw_author_dict in raw_author_dict_list: valid, author_dict = tidy_user_data(raw_author_dict, host) if not valid: continue author_dict_list.append(author_dict) create_or_update_remote_users(host, author_dict_list) delete_non_existing_remote_users(host, author_dict_list) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def tidy_comment_data(data: dict, post_id: str) -> (bool, dict): """ Tidy up the comment data received from other servers """ new_data = {} try: author_dict = data.pop("author", None) author = None if author_dict["host"] == REMOTE_HOST1: author_dict["non_uuid_id"] = author_dict["id"].split("/")[-1] author = User.objects.filter( host=author_dict["host"], non_uuid_id=author_dict["non_uuid_id"]).first() else: author_dict["id"] = author_dict["id"].split("/")[-1] author = User.objects.filter(id=author_dict["id"]).first() if not author: return False, new_data new_data["author"] = author new_data["post"] = post_id new_data["id"] = data["id"] new_data["comment"] = data["comment"] new_data["published"] = data["published"] if "contentType" in data.keys(): new_data["contentType"] = data["contentType"] return True, new_data except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}") return False, new_data
def create_or_update_remote_posts(posts_dict_list: list): try: for post_dict in posts_dict_list: obj, created = Post.objects.update_or_create( id=post_dict["id"], defaults=post_dict, ) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def send_remote_comments(comment, post, author) -> bool: try: author_id = None if post.origin == REMOTE_HOST1: author_id = author.id else: author_id = str(author.id).replace("-", "") author_dict = { "id": f"{author.host}author/{author_id}", "host": f"{author.host}", "displayName": f"{author.displayName}", "url": f"{author.host}author/{author_id}", "github": f"{author.github}", } comment_id = None if post.origin == REMOTE_HOST1: comment_id = comment["id"] else: comment_id = comment["id"].replace("-", "") comment_dict = { "author": author_dict, "comment": comment["comment"], "contentType": comment["contentType"], "published": comment["published"], "id": comment_id, } request_data = { "query": "addComment", "post": f"{post.origin}posts/{post.id}", "comment": comment_dict, } url = f"{post.origin}posts/{str(post.id)}/comments" if post.origin != REMOTE_HOST1: url += "/" node = Node.objects.filter(host=post.origin).first() headers = { "Authorization": f"Basic {node.auth}", "Content-Type": "application/json", "Accept": "application/json", } response = requests.post( url, data=json.dumps(request_data), headers=headers, ) if response.status_code not in range(200, 300): print(response.status_code) print(url) print(headers) print(json.dumps(request_data)) raise Exception(response.text) return True except Exception as e: utils.print_warning(e) return False
def post_comments(self, request, *args, **kwargs): """ # POST to http://service/posts/{POST_ID}/comments """ response_data = { "query": "addComment", "success": "", "message": "", } try: post_id = kwargs["POST_ID"] post = Post.objects.filter(id=post_id).first() if not post: raise Exception("Not Found") except: response_data["success"] = "false" response_data["message"] = "Post does not exist" return Response(response_data, status=status.HTTP_404_NOT_FOUND) else: try: if is_post_visible_to(post, request.user): try: comment = request.data["comment"].copy() if Comment.objects.filter(id=comment["id"]).exists(): raise Exception("Comment id already exists.") author_data = comment.pop("author") author_data["id"] = author_data["id"].split("/")[-1] author = User.objects.filter( id=author_data["id"]).first() if not author: raise Exception("Author not found") serializer = CommentSerializer(data=comment) if serializer.is_valid(): serializer.save(author=author, post=post) response_data["success"] = "true" response_data["message"] = "Comment Added" return Response(response_data, status=status.HTTP_201_CREATED) else: raise Exception("Bad request body") except Exception as e: response_data["success"] = "false" response_data[ "message"] = f"{str(type(e).__name__)}:{str(e)}" return Response(response_data, status=status.HTTP_400_BAD_REQUEST) else: response_data["success"] = "false" response_data["message"] = "Comment not allowed" return Response(response_data, status=status.HTTP_403_FORBIDDEN) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def send_friend_request(author: User, friend: User) -> bool: """ send friend request to remote user Params: author: User, friend: User, request_user: User """ try: node = Node.objects.filter(host=friend.host).first() if not node: raise Exception("Node does not exist") url = f"{node.host}friendrequest" author_dict = { "id": f"{author.host}author/{author.id}", "host": author.host, "displayName": author.displayName, "url": f"{author.host}author/{author.id}", } if friend.host == REMOTE_HOST1: friend_dict = { "id": f"{friend.host}author/{friend.non_uuid_id}", "host": friend.host, "displayName": friend.displayName, "url": f"{friend.host}author/{friend.non_uuid_id}", } else: friend_dict = { "id": f"{friend.host}author/{friend.id}", "host": friend.host, "displayName": friend.displayName, "url": f"{friend.host}author/{friend.id}", } request_body = { "query": "friendrequest", "author": author_dict, "friend": friend_dict, } response = requests.post( url, data=json.dumps(request_body), headers={ "Authorization": f"Basic {node.auth}", "Content-Type": "application/json", "Accept": "application/json", }, ) if response.status_code not in range(200, 300): print(response.status_code) raise Exception(response.text) return True except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}") return False
def delete_non_existing_remote_comments(posts_dict_list: list, comments_dict_list: list): try: existing_comments_ids = [ comment_dict["id"] for comment_dict in comments_dict_list ] for post_dict in posts_dict_list: post = Post.objects.filter(id=post_dict["id"]).first() Comment.objects.filter(post=post).exclude( id__in=existing_comments_ids).delete() except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def create_or_update_remote_comments(comments_dict_list: list): try: for comment_dict in comments_dict_list: post_id = comment_dict.pop("post", None) post = Post.objects.filter(id=post_id).first() obj, created = Comment.objects.update_or_create( id=comment_dict["id"], post=post, defaults=comment_dict, ) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def delete_non_existing_remote_users(host: str, author_dict_list: list): try: if host == REMOTE_HOST1: non_uuid_ids = [ author_dict["non_uuid_id"] for author_dict in author_dict_list ] User.objects.filter(host=host).exclude( non_uuid_id__in=non_uuid_ids).delete() else: ids = [author_dict["id"] for author_dict in author_dict_list] User.objects.filter(host=host).exclude(id__in=ids).delete() except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def update_remote_posts(host: str, auth: str): url = f"{host}author/posts" stime = time.time() response = requests.get(url, headers={ "Authorization": f"Basic {auth}", "Accept": "application/json", }) print(f"Time used fot post request:", time.time() - stime) if response.status_code not in range(200, 300): utils.print_warning( f"Warning: {url} GET method failed with status code {response.status_code}" ) else: try: data = response.json() raw_posts_dict_list = data["posts"] posts_dict_list = [] all_comments_dict_list = [] for raw_post_dict in raw_posts_dict_list: author_dict = raw_post_dict.pop("author", None) author = None if author_dict["host"] == REMOTE_HOST1: author_dict["non_uuid_id"] = author_dict["id"].split( "/")[-1] author = User.objects.filter( host=author_dict["host"], non_uuid_id=author_dict["non_uuid_id"]).first() else: author_dict["id"] = author_dict["id"].split("/")[-1] author = User.objects.filter(id=author_dict["id"]).first() if not author: # author not cached, ignore this post continue else: valid, post_dict, comments_dict_list = tidy_post_data( raw_post_dict, host, author) all_comments_dict_list += comments_dict_list if valid: posts_dict_list.append(post_dict) create_or_update_remote_posts(posts_dict_list) delete_non_existing_remote_posts(host, posts_dict_list) create_or_update_remote_comments(all_comments_dict_list) delete_non_existing_remote_comments(posts_dict_list, all_comments_dict_list) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def pull_github_events(user: User): """ Pull user's github events into his stream """ try: if not user: raise Exception("param 'user' is null") if user.github is None or user.github == "": Post.objects.filter(author=user).exclude(githubId=None).delete() return git_username = user.github.split("/")[-1] url = f"https://api.github.com/users/{git_username}/events" response = requests.get(url, headers={"Accept": "application/json"}) if response.status_code not in range(200, 300): raise Exception(response.text) events = response.json() for event in events: if not Post.objects.filter(githubId=int(event["id"])).exists(): actor = event["actor"]["login"] action = None try: action = event["payload"]["action"] except: action = "had" event_type = event["type"] repo = event["repo"]["name"] visibility = "PUBLIC" if event["public"] else "PRIVATE" create_at = parser.parse(event["created_at"].replace( "Z", ".326198Z")) create_at = create_at.replace( tzinfo=pytz.timezone("MST7MDT")).isoformat() Post.objects.create( title="Github Activity", description="Github Activity", content=f"{actor} {action} {event_type} at {repo}", contentType="text/plain", author=user, visibility=visibility, published=create_at, githubId=int(event["id"]), ) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}") return
def update_friendship(self, request, *args, **kwargs): try: friend_data = request.data["friend"] friend_data["id"] = friend_data["id"].split("/")[-1] friend = User.objects.filter(id=friend_data["id"]).first() if not friend: raise Exception("'Friend' does not exist") author_data = request.data["author"] author_data["id"] = author_data["id"].split("/")[-1] author = User.objects.filter(id=author_data["id"]).first() if not author: raise Exception("'Author' does not exist") instance1 = Friend.objects.filter(f1Id=author.id, f2Id=friend.id).first() instance2 = Friend.objects.filter(f1Id=friend.id, f2Id=author.id).first() if not instance1 or not instance2: raise Exception("'author' and 'friend' are not friends") if request.data["status"] == "A": if request.user != friend: return Response(status=status.HTTP_403_FORBIDDEN) data = {"status": "A"} serializer1 = FriendSerializer(instance=instance1, data=data) serializer2 = FriendSerializer(instance=instance2, data=data) if serializer1.is_valid() and serializer2.is_valid(): serializer1.save() serializer2.save() return Response(status=status.HTTP_200_OK) else: raise Exception("Bad request") elif request.data["status"] == "R": if request.user != friend and request.user != author: return Response(status=status.HTTP_403_FORBIDDEN) self.perform_destroy(instance1) self.perform_destroy(instance2) return Response(status=status.HTTP_204_NO_CONTENT) else: raise Exception("Invalid opearation") except Exception as e: utils.print_warning(e) return Response(status=status.HTTP_400_BAD_REQUEST)
def tidy_user_data(data: dict, node: str) -> (bool, dict): """ Tidy up the data received from other servers """ new_data = {} try: host = data.pop("host", None) id = data.pop("id", None) displayName = data.pop("displayName", None) if not host or not id or not displayName or host not in node: return False, new_data else: if host != node: host = node new_data["host"] = host new_data["displayName"] = displayName id = id.split("/")[-1] if host == REMOTE_HOST1: new_data["id"] = str(uuid.uuid4()) new_data["non_uuid_id"] = id else: new_data["id"] = id github = data.pop("github", None) if github: new_data["github"] = github bio = data.pop("bio", None) if bio: new_data["bio"] = github email = data.pop("email", None) new_data["email"] = (str(new_data["id"]) + email if email else str(new_data["id"]) + "@email.com") new_data["username"] = str(new_data["id"]) return True, new_data except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}") return False, new_data
def tidy_post_data(data: dict, node: str, author: User) -> (bool, dict, list): """ Tidy up the post data received from other servers """ new_data = {} try: new_data["author"] = author new_data["id"] = data["id"] new_data["title"] = data["title"] new_data["source"] = node new_data["content"] = data["content"] new_data["published"] = data["published"] new_data["visibility"] = data["visibility"] new_data["unlisted"] = data["unlisted"] if "description" in data.keys(): new_data["description"] = data["description"] if "contentType" in data.keys(): new_data["contentType"] = data["contentType"] if "categories" in data.keys(): new_data["categoriesStr"] = json.dumps(data["categories"]) if "visibleTo" in data.keys(): new_data["visibleToStr"] = json.dumps(data["visibleTo"]) if "origin" in data.keys(): new_data["origin"] = data["origin"].split("posts/")[0] else: new_data["origin"] = node raw_comments = data["comments"] valid_comments = [] for raw_comment in raw_comments: valid, valid_comment = tidy_comment_data(raw_comment, new_data["id"]) if valid: valid_comments.append(valid_comment) return True, new_data, valid_comments except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}") return False, new_data, []
def create_or_update_remote_users(host: str, author_dict_list: list): try: for author_dict in author_dict_list: if host == REMOTE_HOST1: if User.objects.filter( non_uuid_id=int(author_dict["non_uuid_id"])).exists(): author_dict.pop("id") author_dict.pop("username") author_dict.pop("email") obj, created = User.objects.update_or_create( non_uuid_id=int(author_dict["non_uuid_id"]), defaults=author_dict, ) else: obj, created = User.objects.update_or_create( non_uuid_id=int(author_dict["non_uuid_id"]), defaults=author_dict, ) else: obj, created = User.objects.update_or_create( id=author_dict["id"], defaults=author_dict) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")
def update_friends(user, depth, ignoreuser): """ update friendships of user """ try: friends = [] if not user: utils.print_warning("Parameter 'user' is None!") return if user.host == DEFAULT_HOST: # user from local server friends += deal_unprocessed_active_requests(user) tmp1 = Friend.objects.filter(status="A", f1Id=user.id, isCopy=False).values_list("f2Id", flat=True) friends += deal_current_friends(tmp1, user) tmp2 = Friend.objects.filter(status="A", f2Id=user.id, isCopy=False).values_list("f1Id", flat=True) friends += deal_current_friends(tmp2, user) else: # user from remote servers url = f"{user.host}author/{str(user.id)}" if user.host == REMOTE_HOST1: url = f"{user.host}author/{user.non_uuid_id}" auth = Node.objects.filter(host=user.host).first().auth response = requests.get( url, headers={ "Authorization": f"Basic {auth}", "Accept": "application/json", }, ) if response.status_code not in range(200, 300): no_dash_uuid = str(user.id).replace("-", "") url = f"{user.host}author/{no_dash_uuid}" response = requests.get( url, headers={ "Authorization": f"Basic {auth}", "Accept": "application/json", }, ) if response.status_code not in range(200, 300): raise Exception(response.text) friend_dicts = response.json()["friends"] for friend_dict in friend_dicts: friend_id_str = friend_dict["id"].split("/")[-1] friend = None try: friend_id = uuid.UUID(friend_id_str) friend = User.objects.filter(id=friend_id).first() except ValueError: friend = User.objects.filter(non_uuid_id=id).first() if friend: friends.append(friend) if not Friend.objects.filter( f1Id=user.id, f2Id=friend.id, status="A").exists(): Friend.objects.create(f1Id=user, f2Id=friend, status="A", isCopy=False) Friend.objects.create(f1Id=friend, f2Id=user, status="A", isCopy=True) if ignoreuser: friends += [ignoreuser] Friend.objects.filter( status="A", f1Id=user.id).exclude(f2Id__in=friends).delete() Friend.objects.filter( status="A", f2Id=user.id).exclude(f1Id__in=friends).delete() if depth: for friend in friends: update_friends(friend, 0, user) except Exception as e: utils.print_warning(f"{type(e).__name__} {str(e)}")