def test_insert(smv_storage: SMVStorage):
setup_parties_collection(
smv_storage,
[],
)
# Before
assert smv_storage.get_parties() == []
# note:
# - each call to get date-time will tick global time by 15seconds
# - tz_offset - make sure it works with a random timezone
with freeze_time(START_TIME, tz_offset=-10, auto_tick_seconds=15):
# Insert
smv_storage.upsert_verified_party(
pub_key=PUB_KEY,
user_id=TWITTER_ID,
screen_name=TWITTER_HANDLE,
)
# validate
parties = smv_storage.get_parties()
assert len(parties) == 1
party = parties[0]
assert party["twitter_handle"] == TWITTER_HANDLE
assert party["party_id"] == PUB_KEY
assert party["twitter_user_id"] == TWITTER_ID
assert party["created"] == START_TIME_EPOCH
assert party["last_modified"] == START_TIME_EPOCH
def test_get_remove_duplicates(smv_storage: SMVStorage):
setup_todo_tweets_collection(
smv_storage,
list(range(1, 10)) + list(range(12, 20)) + list(range(5, 15)),
)
assert sorted(smv_storage.get_todo_tweets()) == list(range(1, 20))
assert smv_storage.is_todo_tweet(10)
assert not smv_storage.is_todo_tweet(1000)
def test_get_limit_results(smv_storage: SMVStorage):
setup_todo_tweets_collection(
smv_storage,
list(range(300)),
)
assert len(smv_storage.get_todo_tweets()) == 50
assert smv_storage.is_todo_tweet(10)
assert not smv_storage.is_todo_tweet(1000)
def test_properties():
db = mock.MagicMock()
SMVStorage(db).col_identities
assert db.mock_calls == [
mock.call.get_collection("identities"),
]
db = mock.MagicMock()
SMVStorage(db).col_tweets
assert db.mock_calls == [
mock.call.get_collection("tweets"),
]
def test_block_participants(
smv_storage: SMVStorage,
pub_key: str,
twitter_id: int,
twitter_handle: str,
description: str,
):
#
# Prepare
#
setup_parties_collection(
smv_storage,
[],
)
#
# Create three entries
#
smv_storage.upsert_verified_party(
pub_key=A_PUB_KEY,
user_id=A_TWITTER_ID,
screen_name=A_TWITTER_HANDLE,
)
smv_storage.upsert_verified_party(
pub_key=B_PUB_KEY,
user_id=B_TWITTER_ID,
screen_name=B_TWITTER_HANDLE,
)
smv_storage.upsert_verified_party(
pub_key=C_PUB_KEY,
user_id=C_TWITTER_ID,
screen_name=C_TWITTER_HANDLE,
)
#
# New Sign-up
#
with pytest.raises(BlocklistPartyError):
smv_storage.upsert_verified_party(
pub_key=pub_key,
user_id=twitter_id,
screen_name=twitter_handle,
)
#
# Validate
#
parties = smv_storage.get_parties()
assert (
len(parties) == 1
), "Parties A and B should be blocked and not returned by get_parties"
party = parties[0]
assert party["twitter_handle"] == C_TWITTER_HANDLE
assert party["party_id"] == C_PUB_KEY
assert party["twitter_user_id"] == C_TWITTER_ID
def test_add_todo_tweets(smv_storage: SMVStorage, tweet_ids: List[int]):
setup_todo_tweets_collection(
smv_storage,
[],
)
# Before
assert smv_storage.get_todo_tweets() == []
# Insert
for tweet_id in tweet_ids:
smv_storage.add_todo_tweet(tweet_id)
# Verify
assert sorted(tweet_ids) == sorted(smv_storage.get_todo_tweets())
def test_new_user_sign_ups(
smv_storage: SMVStorage,
new_pub_key: str,
new_twitter_id: int,
new_twitter_handle: str,
description: str,
):
#
# Prepare
#
setup_parties_collection(
smv_storage,
[],
)
#
# Create
#
smv_storage.upsert_verified_party(
pub_key=PUB_KEY,
user_id=TWITTER_ID,
screen_name=TWITTER_HANDLE,
)
#
# Create another
#
smv_storage.upsert_verified_party(
pub_key=new_pub_key,
user_id=new_twitter_id,
screen_name=new_twitter_handle,
)
#
# Validate
#
parties = smv_storage.get_parties()
assert len(parties) == 2
old_party = parties[0]
new_party = parties[1]
assert old_party["twitter_handle"] in TWITTER_HANDLE
assert old_party["party_id"] == PUB_KEY
assert old_party["twitter_user_id"] == TWITTER_ID
assert new_party["twitter_handle"] == new_twitter_handle
assert new_party["party_id"] == new_pub_key
assert new_party["twitter_user_id"] == new_twitter_id
def test_get_storage(get_mongodb_connection_mock: mock.MagicMock, ):
storage = SMVStorage.get_storage(gcp_secret_name="TEST_DB_SECRET")
assert storage
assert isinstance(storage, SMVStorage)
get_mongodb_connection_mock.assert_called_once_with(
gcp_secret_name="TEST_DB_SECRET")
def test_allowed_user_info_updates(
smv_storage: SMVStorage,
new_pub_key: str,
new_twitter_id: int,
new_twitter_handle: str,
description: str,
):
#
# Prepare
#
setup_parties_collection(
smv_storage,
[],
)
#
# Create
#
smv_storage.upsert_verified_party(
pub_key=PUB_KEY,
user_id=TWITTER_ID,
screen_name=TWITTER_HANDLE,
)
#
# Update
#
smv_storage.upsert_verified_party(
pub_key=new_pub_key,
user_id=new_twitter_id,
screen_name=new_twitter_handle,
)
#
# Validate
#
parties = smv_storage.get_parties()
assert len(parties) == 1
party = parties[0]
assert party["twitter_handle"] == new_twitter_handle
assert party["party_id"] == new_pub_key
assert party["twitter_user_id"] == new_twitter_id
def test_cleanup(
smv_storage: SMVStorage,
todo_tweet_ids: List[int],
tweet_ids: List[int],
todo_after_cleanup: List[int],
):
setup_todo_tweets_collection(
smv_storage,
todo_tweet_ids,
)
setup_tweets_collection(
smv_storage,
[random_tweet(tweet_id) for tweet_id in tweet_ids],
)
# Before
assert sorted(smv_storage.get_todo_tweets()) == sorted(todo_tweet_ids)
# Cleanup
smv_storage.cleanup_todo_tweets()
# Validate after
assert sorted(smv_storage.get_todo_tweets()) == sorted(todo_after_cleanup)
for tweet_id in todo_after_cleanup:
assert smv_storage.is_todo_tweet(tweet_id)
for tweet_id in tweet_ids:
assert not smv_storage.is_todo_tweet(tweet_id)
def handle_statistics(
storage: SMVStorage, onelog: OneLog = None
) -> flask.Response:
try:
return flask.jsonify(
{
"time": datetime.utcnow()
.replace(tzinfo=timezone.utc)
.isoformat(),
"tweet_status_count": storage.get_tweet_count_by_status(),
"last_tweet_id": storage.get_last_tweet_id(),
"total_tweets": storage.get_tweet_count(),
"status": "success",
}
)
except Exception as err:
return flask.jsonify(
{
"status": "failed",
"error": str(err),
}
)
def test_upsert_tweet_record():
db = mock.MagicMock()
storage = SMVStorage(db)
storage.upsert_tweet_record(tweet_id=123,
user_id="user.1",
text="Test tweet",
status="OK")
assert db.mock_calls == [
mock.call.get_collection("tweets"),
mock.call.get_collection().update_one(
{"tweet_id": 123},
{
"$set": {
"user_id": "user.1",
"text": "Test tweet",
"status": "OK",
"last_modified": mock.ANY,
}
},
upsert=True,
),
]
def test_signup_with_twitter_handle_matching_two_participants(
smv_storage: SMVStorage, ):
#
# Prepare
#
setup_parties_collection(
smv_storage,
[],
)
#
# Create two entries
#
smv_storage.upsert_verified_party(
pub_key=PUB_KEY,
user_id=TWITTER_ID,
screen_name=TWITTER_HANDLE,
)
smv_storage.upsert_verified_party(
pub_key=NEW_PUB_KEY,
user_id=NEW_TWITTER_ID,
screen_name=NEW_TWITTER_HANDLE,
)
#
# New Sign-up
#
smv_storage.upsert_verified_party(
pub_key=PUB_KEY,
user_id=TWITTER_ID,
screen_name=NEW_TWITTER_HANDLE,
)
#
# Validate
#
parties = smv_storage.get_parties()
assert len(parties) == 2
a_party = parties[0]
b_party = parties[1]
assert a_party["party_id"] == PUB_KEY
assert a_party["twitter_user_id"] == TWITTER_ID
assert a_party["twitter_handle"] == NEW_TWITTER_HANDLE
assert b_party["party_id"] == NEW_PUB_KEY
assert b_party["twitter_user_id"] == NEW_TWITTER_ID
assert b_party["twitter_handle"] == NEW_TWITTER_HANDLE
def test_dates_after_update(smv_storage: SMVStorage):
setup_parties_collection(
smv_storage,
[],
)
#
# Create
#
# note:
# - each call to get date-time will tick global time by 15seconds
# - tz_offset - make sure it works with a random timezone
with freeze_time(START_TIME, tz_offset=-10, auto_tick_seconds=15):
smv_storage.upsert_verified_party(
pub_key=PUB_KEY,
user_id=TWITTER_ID,
screen_name=TWITTER_HANDLE,
)
parties = smv_storage.get_parties()
assert len(parties) == 1
party = parties[0]
assert party["created"] == START_TIME_EPOCH
assert party["last_modified"] == START_TIME_EPOCH
#
# Update
#
# note:
# - each call to get date-time will tick global time by 15seconds
# - tz_offset - make sure it works with a random timezone
with freeze_time(
START_TIME + timedelta(seconds=11), tz_offset=-10, auto_tick_seconds=15
):
smv_storage.upsert_verified_party(
pub_key=PUB_KEY,
user_id=TWITTER_ID,
screen_name=TWITTER_HANDLE,
)
parties = smv_storage.get_parties()
assert len(parties) == 1
party = parties[0]
assert party["created"] == START_TIME_EPOCH
assert party["last_modified"] == START_TIME_EPOCH + 11
def smv_storage() -> SMVStorage:
MONGO_DB_USER = os.getenv("MONGO_DB_USER")
MONGO_DB_PASS = os.getenv("MONGO_DB_PASS")
MONGO_DB_HOSTNAME = os.getenv("MONGO_DB_HOSTNAME")
MONGO_DB_NAME = os.getenv("MONGO_DB_NAME")
if (not MONGO_DB_USER or not MONGO_DB_PASS or not MONGO_DB_HOSTNAME
or not MONGO_DB_NAME):
return None
assert "local" in MONGO_DB_NAME
os.environ["MONGO_SECRET"] = f"""{{
"DB_USER": "******",
"DB_PASS": "******",
"DB_HOSTNAME": "{MONGO_DB_HOSTNAME}",
"DB_NAME": "{MONGO_DB_NAME}",
"SCHEME": "mongodb"
}}"""
try:
return SMVStorage.get_storage(gcp_secret_name="MONGO_SECRET")
except Exception:
pass
return None
def handle_parties(storage: SMVStorage,
onelog: OneLog = None) -> flask.Response:
parties = storage.get_parties()
onelog.info(parties_count=len(parties))
return flask.jsonify(parties)
import os
from services.smv_storage import SMVStorage
MONGO_DB_USER = "******"
MONGO_DB_PASS = "******"
MONGO_DB_HOSTNAME = "[DATABASE CONNECTION HOSTNAME]"
MONGO_DB_NAME = "[DATABASE NAME]"
os.environ[
"MONGO_SECRET"
] = f"""{{
"DB_USER": "******",
"DB_PASS": "******",
"DB_HOSTNAME": "{MONGO_DB_HOSTNAME}",
"DB_NAME": "{MONGO_DB_NAME}"
}}"""
storage = SMVStorage.get_storage(gcp_secret_name="MONGO_SECRET")
if __name__ == "__main__":
print(f"parties={storage.get_parties()}")
# storage.upsert_tweet_record(1233, text="gg", screen_name="abc_userrrrrr")
print(f"tweet_record={storage.get_tweet_record(1233)}")
print(f"get_tweet_count_by_status={storage.get_tweet_count_by_status()}")
print(f"get_last_tweet_id={storage.get_last_tweet_id()}")
app.config["JSONIFY_PRETTYPRINT_REGULAR"] = True
CONFIG = SMVConfig(
twitter_search_text=os.environ["TWITTER_SEARCH_TEXT"],
twitter_reply_message_success=os.environ["TWITTER_REPLY_SUCCESS"],
twitter_reply_message_invalid_format=os.environ[
"TWITTER_REPLY_INVALID_FORMAT"
],
twitter_reply_message_invalid_signature=os.environ[
"TWITTER_REPLY_INVALID_SIGNATURE"
],
twitter_reply_delay=float(os.getenv("TWITTER_REPLY_DELAY", "0.25")),
)
STORAGE = SMVStorage.get_storage(
gcp_secret_name=os.environ["MONGO_SECRET_NAME"],
)
TWCLIENT = TwitterClient(
gcp_secret_name=os.environ["TWITTER_SECRET_NAME"],
)
def router(request: flask.Request):
if request.path.endswith("/parties"):
return handle_parties(storage=STORAGE)
elif request.path.endswith("/tweet"):
tweet_id: str = request.args.get("id")
return handle_tweet(
storage=STORAGE,
tweet_id=tweet_id,
def process_tweet(
tweet: Tweet,
storage: SMVStorage,
twclient: TwitterClient,
config: SMVConfig,
tweet_prefix: str,
twitter_handle: str, # starts with @ character
onelog: OneLog = None,
):
onelog.info(
tweet_id=tweet.tweet_id,
tweet_handle=tweet.user_screen_name,
tweet_user_id=tweet.user_id,
tweet_message=tweet.full_text,
twitter_handle=twitter_handle,
)
if storage.get_tweet_record(tweet.tweet_id) is not None:
onelog.info(tweet_processed=True, status="SKIP")
else:
storage.upsert_tweet_record(
tweet_id=tweet.tweet_id,
user_id=tweet.user_id,
screen_name=tweet.user_screen_name,
text=tweet.full_text,
status="PROCESSING",
)
try:
pubkey, signed_message = parse_tweet_message(
tweet.full_text, twitter_handle)
onelog.info(pubkey=pubkey, signed_message=signed_message)
validate_signature(
pubkey,
signed_message,
twitter_handle=tweet.user_screen_name,
)
# Do not reply to user on Twitter
# update DB - verified party
storage.upsert_verified_party(
pubkey,
tweet.user_id,
tweet.user_screen_name,
)
# update DB
storage.upsert_tweet_record(
tweet_id=tweet.tweet_id,
reply="",
status="PASSED",
)
except TweetInvalidFormatError:
onelog.info(error="Invalid Format", status="FAILED")
# reply on twitter
# Do not reply to user on Twitter
# update DB
storage.upsert_tweet_record(
tweet_id=tweet.tweet_id,
reply="",
status="INVALID_FORMAT",
)
except TweetInvalidSignatureError:
onelog.info(error="Invalid Signature", status="FAILED")
# reply on twitter
time.sleep(config.twitter_reply_delay)
twclient.reply(
config.twitter_reply_message_invalid_signature,
tweet,
)
# update DB
storage.upsert_tweet_record(
tweet_id=tweet.tweet_id,
reply=config.twitter_reply_message_invalid_signature,
status="INVALID_SIGNATURE",
)
except BlocklistPartyError as err:
onelog.info(error="Party blocked", status="FAILED")
# reply on twitter
# Do not reply to user on Twitter
# update DB
storage.upsert_tweet_record(
tweet_id=tweet.tweet_id,
reply="",
status="BLOCKLISTED",
description=str(err),
)
def handle_process_tweets(
storage: SMVStorage,
twclient: TwitterClient,
config: SMVConfig,
onelog: OneLog = None,
):
# Fetch all tweets from Twitter API
try:
# twitter_search_text = (
# f"{config.twitter_search_text} @{twclient.account_name}"
# )
twitter_search_text = f"@{twclient.account_name}"
onelog.info(twitter_search_text=twitter_search_text)
since_tweet_id = storage.get_last_tweet_id()
onelog.info(since_tweet_id=since_tweet_id)
tweets = list(
twclient.get_tweets(
twitter_search_text,
since_tweet_id,
))
onelog.info(total_count=len(tweets))
except Exception as err:
onelog.info(error=str(err), status="FAILED")
traceback.print_exc()
print(err)
return (
flask.jsonify({
"status": "failed",
"error": "Failed to fetch tweets."
}),
500,
)
storage.cleanup_todo_tweets()
todo_tweets = storage.get_todo_tweets()
for tweet_id in todo_tweets:
new_tweet = twclient.get_by_id(tweet_id)
if new_tweet:
tweets.append(new_tweet)
# Process tweets one by one from oldest to newest
try:
processed_count = 0
for twt in reversed(tweets):
process_tweet(
twt,
storage,
twclient,
config,
tweet_prefix=twitter_search_text,
twitter_handle=f"@{twclient.account_name}",
)
processed_count += 1
onelog.info(processed_count=processed_count)
except Exception as err:
onelog.info(processed_count=processed_count,
error=str(err),
status="FAILED")
traceback.print_exc()
print(err)
return flask.jsonify({"status": "failed", "error": str(err)}), 500
storage.remove_todo_tweets(todo_tweets)
return flask.jsonify({"status": "success"})
def handle_tweet(
storage: SMVStorage, tweet_id: str, onelog: OneLog = None
) -> flask.Response:
onelog.info(tweet_id=tweet_id)
if not tweet_id:
return flask.jsonify(
{"status": "failed", "error": "Missing id argument in the request"}
)
if isinstance(tweet_id, str):
if not tweet_id.isdigit():
return flask.jsonify(
{
"status": "failed",
"tweet_id": tweet_id,
"error": "id argument must contain digits only",
}
)
tweet_id = int(tweet_id)
if not isinstance(tweet_id, int):
return flask.jsonify(
{
"status": "failed",
"tweet_id": tweet_id,
"error": "id argument must contain digits only",
}
)
processed_tweet = storage.get_tweet_record(tweet_id)
if processed_tweet:
del processed_tweet["_id"]
# add extra info
description = {
"PROCESSING": "The tweet is being processed.",
"PASSED": "Successful sign up",
"INVALID_FORMAT": "The tweet does not look like a sign up tweet.",
"INVALID_SIGNATURE": (
"Signature does not match user's twitter handle: make sure"
" user signed their twitter handle not name, i.e."
" @twitter_handle - it starts with @. Other common mistakes"
" are typo, wrong lower-upper case."
),
"BLOCKLISTED": (
"User tried to do something not allowed, e.g. transfer Wallet"
" to another user"
),
}
processed_tweet["status_description"] = description.get(
processed_tweet["status"]
)
return flask.jsonify(
{
"status": "success",
"tweet_id": tweet_id,
"message": "Tweet has been already processed",
"tweet": processed_tweet,
}
)
if storage.is_todo_tweet(tweet_id):
return flask.jsonify(
{
"status": "success",
"tweet_id": tweet_id,
"message": "Tweet is waiting to be processed",
}
)
storage.add_todo_tweet(tweet_id)
return flask.jsonify(
{
"status": "success",
"tweet_id": tweet_id,
"message": "Tweet added to the queue to be processed",
}
)
def test_constructor():
db = mock.MagicMock()
assert SMVStorage(db)