def __init__(self, users_util, logger, db_stub):
self._logger = logger
self._db_stub = db_stub
self._users_util = users_util
self._activ_util = ActivitiesUtil(logger, db_stub)
self._recommender_util = RecommendersUtil(
logger, db_stub, self.DEFAULT_RECOMMENDER, self.ENV_VAR, self.RECOMMENDERS)
# self.active_recommenders contains one or more recommender system
# objects (out of the constructors in self.RECOMMENDERS).
self.active_recommenders = self._recommender_util._get_active_recommenders()
def test_build_activity(self):
e = self.servicer._build_activity('FOLLOWER', 'FOLLOWED')
self.assertEqual(e['@context'], ActivitiesUtil.rabble_context())
self.assertEqual(e['type'], 'Follow')
self.assertEqual(e['actor'], 'FOLLOWER')
self.assertEqual(e['object'], 'FOLLOWED')
self.assertEqual(e['to'], ['FOLLOWED'])
def main():
args = get_args()
logger = get_logger('actors_service', args.v)
logger.info('Creating server')
with get_service_channel(logger, "DB_SERVICE_HOST", 1798) as db_chan, \
get_service_channel(logger, "FOLLOWS_SERVICE_HOST", 1641) as follows_chan:
db_stub = database_pb2_grpc.DatabaseStub(db_chan)
follows_stub = follows_pb2_grpc.FollowsStub(follows_chan)
users_util = UsersUtil(logger, db_stub)
activities_util = ActivitiesUtil(logger, db_stub)
host_name = get_host_name(logger)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
servicer = Servicer(logger, users_util, activities_util, db_stub,
host_name, follows_stub)
actors_pb2_grpc.add_ActorsServicer_to_server(servicer, server)
server.add_insecure_port('0.0.0.0:1973')
logger.info("Starting actors service on port 1973")
server.start()
try:
while True:
time.sleep(60 * 60 * 24) # One day
except KeyboardInterrupt:
pass
def setUp(self):
self.activ_util = ActivitiesUtil(Mock(), Mock())
self.activ_util._get_activitypub_actor_url = lambda host, handle: (host + '/ap/@' + handle)
self.activ_util.build_inbox_url = lambda handle, host: (host + '/ap/@' + handle + '/inbox')
self.servicer = SendFollowServicer(Mock(), self.activ_util, Mock())
self.mock_response = Mock()
self.mock_response.text = ''
self.mock_response.status_code = 200
def build_like_activity(like_actor, liked_object):
return {
"@context": ActivitiesUtil.rabble_context(),
'type': 'Like',
'object': liked_object,
'actor': {
'type': 'Person',
'id': like_actor,
},
}
class ActivitiesUtilTest(unittest.TestCase):
def setUp(self):
self.activ_util = ActivitiesUtil(Mock(), Mock())
self.activ_util._host_name = 'b.com'
def test_build_local_actor_url(self):
self.assertEqual(self.activ_util._build_local_actor_url('a', 'b.com'),
'b.com/ap/@a')
def test_build_inbox_url(self):
self.assertEqual(self.activ_util.build_inbox_url('a', 'b.com'),
'https://b.com/ap/@a/inbox')
def test_send_activity(self):
import requests
requests.Session = Mock()
activity = {'@context': 'https://test.com', 'data': 'yes'}
_, e = self.activ_util.send_activity(
activity, 'https://followed.com/ap/@b/inbox')
self.assertIsNone(e)
def setUp(self):
os.environ["HOST_NAME"] = "cianisharrypotter.secret"
self.activ_util = ActivitiesUtil(Mock(), Mock())
self.activ_util._get_activitypub_actor_url = lambda host, handle: (
host + '/ap/@' + handle)
self.activ_util.build_inbox_url = lambda handle, host: (
host + '/ap/@' + handle + '/inbox')
self.servicer = SendFollowServicer(Mock(), self.activ_util, Mock())
self.mock_response = Mock()
self.mock_response.text = ''
self.mock_response.status_code = 200
def setUp(self):
self.db = MockDB()
self.activ_util = ActivitiesUtil(Mock(), self.db)
self.activ_util.build_actor = lambda han, host: f'{host}/ap/@{han}'
self.activ_util.build_inbox_url = lambda han, host: f'{host}/ap/@{han}/inbox'
self.users_util = UsersUtil(Mock(), self.db)
self.servicer = SendLikeServicer(Mock(), self.db, self.users_util,
self.activ_util, "localhost")
self.data = None
self.url = None
self.activ_util.send_activity = self.save_request
def setUp(self):
self.req = upb.LikeUndoDetails(
article_id=3,
liker_handle="cian",
)
self.db = MockDB()
self.activ_util = ActivitiesUtil(Mock(), self.db)
self.users_util = UsersUtil(Mock(), self.db)
self.hostname = "skinny_123"
self.servicer = SendLikeUndoServicer(
Mock(), self.db, self.activ_util, self.users_util, self.hostname)
self.data = None
self.url = None
self.activ_util.send_activity = self.save_request
self.activ_util._get_activitypub_actor_url = lambda host, handle: (host + '/' + handle)
self.activ_util.build_inbox_url = lambda handle, host: (host + '/' + handle + '/inbox')
def main():
logger = get_logger("undo_service")
db_stub = get_db_stub(logger)
activ_util = ActivitiesUtil(logger, db_stub)
users_util = UsersUtil(logger, db_stub)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
undo_pb2_grpc.add_S2SUndoServicer_to_server(
S2SUndoServicer(logger, db_stub, activ_util, users_util), server)
server.add_insecure_port("0.0.0.0:1608")
logger.info("Starting Undo service on port 1608")
server.start()
try:
while True:
time.sleep(60 * 60 * 24) # One day
except KeyboardInterrupt:
pass
def main():
args = get_args()
logger = get_logger("s2s_follow_service", args.v)
users_util = UsersUtil(logger, None)
with get_future_channel(logger, "DB_SERVICE_HOST", 1798) as db_chan, \
get_future_channel(logger, "FOLLOWS_SERVICE_HOST", 1641) as logger_chan:
db_stub = database_pb2_grpc.DatabaseStub(db_chan)
activ_util = ActivitiesUtil(logger, db_stub)
follows_service = follows_pb2_grpc.FollowsStub(logger_chan)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
s2s_follow_pb2_grpc.add_S2SFollowServicer_to_server(
FollowServicer(logger, users_util, activ_util, follows_service,
db_stub), server)
server.add_insecure_port('0.0.0.0:1922')
logger.info("Starting s2s follow server on port 1922")
server.start()
while True:
time.sleep(60 * 60 * 24) # One day
def main():
logger = get_logger("likes_service")
db_stub = get_db_stub(logger)
user_util = UsersUtil(logger, db_stub)
activ_util = ActivitiesUtil(logger, db_stub)
recommender_util = RecommendersUtil(logger, db_stub)
post_recommendation_stub = recommender_util.get_post_recommendation_stub()
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
like_pb2_grpc.add_S2SLikeServicer_to_server(
S2SLikeServicer(logger, db_stub, user_util, activ_util,
post_recommendation_stub), server)
server.add_insecure_port("0.0.0.0:1848")
logger.info("Starting Like service on port 1848")
server.start()
try:
while True:
time.sleep(60 * 60 * 24) # One day
except KeyboardInterrupt:
pass
def main():
logger = get_logger("announce_service")
db_stub = get_db_stub(logger)
article_stub = get_article_stub(logger)
user_util = UsersUtil(logger, db_stub)
activ_util = ActivitiesUtil(logger, db_stub)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
announce_pb2_grpc.add_AnnounceServicer_to_server(
AnnounceServicer(logger, db_stub, user_util, activ_util, article_stub),
server
)
server.add_insecure_port("0.0.0.0:1919")
logger.info("Starting Announce service on port 1919")
server.start()
try:
while True:
time.sleep(60 * 60 * 24) # One day
except KeyboardInterrupt:
pass
def main():
args = get_args()
logger = get_logger("update_service", args.v)
db_stub = get_db_stub(logger)
md_stub = get_md_stub(logger)
activ_util = ActivitiesUtil(logger, db_stub)
users_util = UsersUtil(logger, db_stub)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
update_pb2_grpc.add_S2SUpdateServicer_to_server(
S2SUpdateServicer(logger, db_stub, md_stub, activ_util, users_util),
server)
server.add_insecure_port("0.0.0.0:2029")
logger.info("Starting Update service on port 2029")
server.start()
try:
while True:
time.sleep(60 * 60 * 24) # One day
except KeyboardInterrupt:
pass
def main():
logger = get_logger("create_service")
logger.info("Creating db connection")
with get_service_channel(logger, "DB_SERVICE_HOST", 1798) as db_chan:
db_stub = database_pb2_grpc.DatabaseStub(db_chan)
users_util = UsersUtil(logger, db_stub)
activ_util = ActivitiesUtil(logger, db_stub)
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
approver_pb2_grpc.add_ApproverServicer_to_server(
ApproverServicer(logger, db_stub, activ_util, users_util),
server,
)
server.add_insecure_port('0.0.0.0:2077')
logger.info("Starting approver service on port 2077")
server.start()
while True:
time.sleep(60 * 60 * 24) # One day
def main():
logger = get_logger("create_service")
db_channel = get_service_channel(logger, "DB_SERVICE_HOST", 1798)
db_stub = database_pb2_grpc.DatabaseStub(db_channel)
article_channel = get_future_channel(logger, "ARTICLE_SERVICE_HOST", 1601)
article_stub = article_pb2_grpc.ArticleStub(article_channel)
logger.info("Creating create server")
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
users_util = UsersUtil(logger, db_stub)
activ_util = ActivitiesUtil(logger, db_stub)
create_pb2_grpc.add_CreateServicer_to_server(
CreateServicer(db_stub, article_stub, logger, users_util, activ_util),
server
)
server.add_insecure_port('0.0.0.0:1922')
logger.info("Starting create server on port 1922")
server.start()
try:
while True:
time.sleep(60 * 60 * 24) # One day
except KeyboardInterrupt:
db_channel.close()
pass
def setUp(self):
os.environ["HOST_NAME"] = "b.com"
self.activ_util = ActivitiesUtil(Mock(), Mock())
self.activ_util._hostname = 'b.com'
def setUp(self):
activ_util = ActivitiesUtil(Mock(), Mock())
activ_util.send_activity = Mock(return_value=(None, None))
self.servicer = SendApprovalServicer(Mock(), activ_util)
def __init__(self, logger, db_stub):
self._logger = logger
self._db = db_stub
self._activ_util = ActivitiesUtil(logger, db_stub)
class UsersUtil:
def __init__(self, logger, db_stub):
self._logger = logger
self._db = db_stub
self._activ_util = ActivitiesUtil(logger, db_stub)
def parse_username(self, username):
username = username.lstrip('@')
p = username.split('@')
if len(p) == 1:
# Local user like 'admin'.
return p[0], None
if len(p) == 2:
# Foreign user like '*****@*****.**'
return tuple(p)
# Username is incorrect/malicious/etc.
self._logger.warning('Couldn\'t parse username %s', username)
return None, None
def get_actor_details(self, handle, host):
actor_url = self._activ_util.build_actor(handle, host)
return self.parse_actor_details(actor_url)
def parse_actor(self, actor_uri):
"""
Given an actor URI, return the (host, handle) tuple for this
actor.
"""
actor_doc = self._activ_util.fetch_actor(actor_uri)
if actor_doc is None:
self._logger.warning(f'No actor doc found for user {actor_uri}.')
return None, None
handle = actor_doc['preferredUsername']
host = self._activ_util.normalise_hostname(urlparse(actor_uri).netloc)
return host, handle
def parse_actor_details(self, actor_uri):
"""
Given an actor URI, return the (host, handle, bio) tuple for this
actor.
"""
actor_doc = self._activ_util.fetch_actor(actor_uri)
if actor_doc is None:
self._logger.warning(f'No actor doc found for user {actor_uri}.')
return None, None, None
handle = actor_doc['preferredUsername']
summary = ""
if 'summary' in actor_doc:
summary = actor_doc['summary']
host = self._activ_util.normalise_hostname(urlparse(actor_uri).netloc)
return host, handle, summary
def download_profile_pic(self, host, handle, global_id):
"""
If a user being added is from a foreign host then get their actor
and from that download their profile picture to the static assets
directory.
"""
try:
actor_url = self._activ_util.build_actor(handle, host)
actor = self._activ_util.fetch_actor(actor_url)
if "icon" not in actor:
return # No profile pic
pic_url = actor["icon"]["url"]
self._logger.info("Getting profile pic URL '%s'", pic_url)
image_resp = requests.get(pic_url)
if image_resp.status_code < 200 or image_resp.status_code >= 300:
self._logger.warning(
"Could not get profile pic: error %d",
image_resp.status_code)
image_bytes = image_resp.content
profile_pic_path = "/repo/build_out/chump_dist/user_{}".format(
global_id)
self._logger.info("Writing profile pic to %s", profile_pic_path)
with open(profile_pic_path, "wb") as f:
f.write(image_bytes)
except Exception as e:
self._logger.error(
"Error when downloading profile pic: %s", str(e))
# Just pretend that didn't happen
def _create_user_in_db(self, entry):
self._logger.debug('Creating user %s@%s in database',
entry.handle, entry.host)
insert_req = database_pb2.UsersRequest(
request_type=database_pb2.UsersRequest.INSERT,
entry=entry
)
insert_resp = self._db.Users(insert_req)
if insert_resp.result_type != database_pb2.UsersResponse.OK:
self._logger.error("Error inserting into users db %s",
insert_resp.error)
return None
return insert_resp.global_id
def delete_user_from_db(self, global_id):
self._logger.debug("Deleteing user with global_id %d", global_id)
resp = self._db.Users(database_pb2.UsersRequest(
request_type=database_pb2.UsersRequest.DELETE,
entry=database_pb2.UsersEntry(
global_id=global_id
),
))
if resp.result_type != database_pb2.UsersResponse.OK:
self._logger.error("Error deleting from db: %s",
resp.error)
return False
return True
def get_or_create_user_from_db(self,
handle=None,
host=None,
global_id=None,
host_is_null=False,
attempt_number=0,
bio=None):
if attempt_number > MAX_FIND_RETRIES:
self._logger.error('Retried query too many times.')
return None
host = self._activ_util.normalise_hostname(host) if host else host
user = self.get_user_from_db(handle, host, global_id, host_is_null)
if user is not None:
return user
if global_id is not None or handle is None:
# Should not try to create a user and hope it has this ID.
# Also shouldn't create a user with no handle.
return None
user_entry = database_pb2.UsersEntry(
handle=handle,
host=host,
bio=bio,
host_is_null=host_is_null
)
new_global_id = self._create_user_in_db(user_entry)
if new_global_id is not None and host:
self.download_profile_pic(host, handle, new_global_id)
return self.get_or_create_user_from_db(handle,
host,
attempt_number=attempt_number + 1)
def user_is_local(self, global_id):
user = self.get_user_from_db(global_id=global_id)
if user is None:
self._logger.error(
"Could not get user from DB, "
"assuming they're foreign and continuing"
)
return False
# Host is empty if user is local.
return user.host == "" or user.host is None or user.host_is_null
def get_user_from_db(self, handle=None, host=None, global_id=None, host_is_null=False):
self._logger.debug('User %s@%s (id %s) host_is_null: %s requested from database',
handle, host, global_id, host_is_null)
host = self._activ_util.normalise_hostname(host) if host else host
user_entry = database_pb2.UsersEntry(
handle=handle,
host=host,
host_is_null=host_is_null,
global_id=global_id
)
find_req = database_pb2.UsersRequest(
request_type=database_pb2.UsersRequest.FIND,
match=user_entry
)
find_resp = self._db.Users(find_req)
if len(find_resp.results) == 0:
self._logger.warning('No user %s@%s (id %s) found',
handle, host, global_id)
return None
elif len(find_resp.results) == 1:
self._logger.debug('Found user %s@%s (id %s) from database',
handle, host, global_id)
return find_resp.results[0]
else:
self._logger.error('> 1 user found in database for %s@%s (id %s)'
+ ', returning first one.',
handle, host, global_id)
return find_resp.results[0]
def get_follower_list(self, user_id):
follow_entry = database_pb2.Follow(
followed=user_id
)
follow_req = database_pb2.DbFollowRequest(
request_type=database_pb2.DbFollowRequest.FIND,
match=follow_entry
)
follow_resp = self._db.Follow(follow_req)
if follow_resp.result_type == database_pb2.DbFollowResponse.ERROR:
self._logger.error(
"Find for followers of id: %s returned error: %s",
user_id,
follow_resp.error
)
return []
return follow_resp.results
def remove_local_users(self, followers):
foreign_followers = []
for follow in followers:
follower_entry = self.get_user_from_db(
global_id=follow.follower
)
if follower_entry is None:
self._logger.error(
"Could not find follower in db. Id: %s", follow.follower)
continue
if follower_entry.host:
foreign_followers.append(follower_entry)
return foreign_followers
def setUp(self):
self.activ_util = ActivitiesUtil(Mock(), Mock())
self.activ_util._host_name = 'b.com'
class PostRecommendationsServicer(recommend_posts_pb2_grpc.PostRecommendationsServicer):
RECOMMENDERS = {
'random': RandomRecommender,
'cosine': CosineRecommender,
}
DEFAULT_RECOMMENDER = 'random'
ENV_VAR = 'POSTS_RECOMMENDER_METHOD'
MAX_RECOMMENDATIONS = 50
DEFAULT_IMAGE = "https://upload.wikimedia.org/wikipedia/commons/8/89/Portrait_Placeholder.png"
def __init__(self, users_util, logger, db_stub):
self._logger = logger
self._db_stub = db_stub
self._users_util = users_util
self._activ_util = ActivitiesUtil(logger, db_stub)
self._recommender_util = RecommendersUtil(
logger, db_stub, self.DEFAULT_RECOMMENDER, self.ENV_VAR, self.RECOMMENDERS)
# self.active_recommenders contains one or more recommender system
# objects (out of the constructors in self.RECOMMENDERS).
self.active_recommenders = self._recommender_util._get_active_recommenders()
def Get(self, request, context):
self._logger.debug('Get PostRecommendations, user_id = %s',
request.user_id)
resp = recommend_posts_pb2.PostRecommendationsResponse()
recommended_posts = []
post_ids = set()
max_posts_per_r = self.MAX_RECOMMENDATIONS // len(
self.active_recommenders)
for r in self.active_recommenders:
r_posts, error = r.get_recommendations(
request.user_id, max_posts_per_r)
if error:
resp.result_type = \
recommend_posts_pb2.PostRecommendationsResponse.ERROR
resp.message = error
return resp
recommended_posts.append(r_posts)
# Join recommendations together, with the highest recommended first
posts = []
for i in range(max_posts_per_r + 1):
for r_p in recommended_posts:
# See proto/Feed.Post and proto/database.PostsEntry
if i < len(r_p):
author = self._users_util.get_user_from_db(
global_id=r_p[i].author_id)
if author == None:
resp.result_type = \
recommend_posts_pb2.PostRecommendationsResponse.ERROR
resp.message = "Post Author could not be found"
return resp
post_obj = resp.results.add()
post_obj.global_id = r_p[i].global_id
post_obj.author = author.handle
post_obj.author_host = author.host
post_obj.author_id = r_p[i].author_id
post_obj.title = r_p[i].title
post_obj.body = r_p[i].body
post_obj.published = self._activ_util.timestamp_to_rfc(
r_p[i].creation_datetime)
post_obj.likes_count = r_p[i].likes_count
post_obj.bio = author.bio
post_obj.image = self.DEFAULT_IMAGE
post_obj.is_liked = r_p[i].is_liked
post_obj.is_followed = r_p[i].is_followed
post_obj.shares_count = r_p[i].shares_count
post_obj.summary = r_p[i].summary
tags = self._recommender_util.split_tags(r_p[i].tags)
post_obj.tags.extend(tags)
resp.result_type = \
recommend_posts_pb2.PostRecommendationsResponse.OK
return resp
def UpdateModel(self, request, context):
self._logger.debug('UpdateModel PostRecommendations, user_id = %s',
request.user_id)
resp = recommend_posts_pb2.PostRecommendationsResponse()
for r in self.active_recommenders:
error = r.update_model(request.user_id, request.article_id)
if error:
resp.result_type = \
recommend_posts_pb2.PostRecommendationsResponse.ERROR
resp.message = error
return resp
resp.result_type = \
recommend_posts_pb2.PostRecommendationsResponse.OK
return resp
def AddPost(self, request, context):
self._logger.debug('UpdateModel PostRecommendations, user_id = %s',
request.author_id)
resp = recommend_posts_pb2.PostRecommendationsResponse()
for r in self.active_recommenders:
error = r.add_post(request)
if error:
resp.result_type = \
recommend_posts_pb2.PostRecommendationsResponse.ERROR
resp.message = error
return resp
resp.result_type = \
recommend_posts_pb2.PostRecommendationsResponse.OK
return resp