def twitter_publish_events_import(event, context):
print("Event:", event)
credentials = config["credentials"]
context_parts = context.invoked_function_arn.split(':')
topic_name = "Twitter-{0}".format(config["communityName"])
topic_arn = "arn:aws:sns:{region}:{account_id}:{topic}".format(region=context_parts[3], account_id=context_parts[4],
topic=topic_name)
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
neo4j_user = credentials["readonly"].get('user', "neo4j")
neo4j_password = decrypt_value(credentials["readonly"]['password'])
sns = boto3.client('sns')
twitter_bearer = decrypt_value(credentials["twitterBearer"])
search = config["twitterSearch"]
since_id = twitter.find_last_tweet(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password)
print(f"Most recent tweet: {since_id}")
tweets = twitter.find_tweets_since(since_id=since_id, search=search, bearer_token=twitter_bearer)
for tweet in tweets:
sns.publish(TopicArn=topic_arn, Message=json.dumps(tweet))
def meetup_events_import(event, _):
print("Event:", event)
credentials = config["credentials"]
write_credentials = credentials["write"]
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
neo4j_user = write_credentials.get('user', "neo4j")
neo4j_password = decrypt_value(write_credentials['password'])
meetup_key = decrypt_value(credentials["meetupApiKey"])
meetup.import_events(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password, meetup_key=meetup_key)
def meetup_events_import(event, _):
print("Event:", event)
neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost")
neo4j_user = os.environ.get('NEO4J_USER', "neo4j")
neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD'])
meetup_key = decrypt_value(os.environ["MEETUP_API_KEY"])
meetup.import_events(neo4j_url=neo4j_url,
neo4j_user=neo4j_user,
neo4j_pass=neo4j_password,
meetup_key=meetup_key)
def twitter_import(event, _):
print("Event:", event)
credentials = config["credentials"]
write_credentials = credentials["write"]
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
neo4j_user = write_credentials.get('user', "neo4j")
neo4j_password = decrypt_value(write_credentials['password'])
twitter_bearer = decrypt_value(credentials["twitterBearer"])
search = config["twitterSearch"]
twitter.import_links(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password,
bearer_token=twitter_bearer, search=search)
def github_import(event, _):
print("Event:", event)
neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost")
neo4j_user = os.environ.get('NEO4J_USER', "neo4j")
neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD'])
github_token = decrypt_value(os.environ["GITHUB_TOKEN"])
tag = os.environ.get('TAG')
github.import_github(neo4j_url=neo4j_url,
neo4j_user=neo4j_user,
neo4j_pass=neo4j_password,
tag=tag,
github_token=github_token)
def twitter_topic_import(event, _):
print("Event:", event)
credentials = config["credentials"]
write_credentials = credentials["write"]
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
neo4j_user = write_credentials.get('user', "neo4j")
neo4j_password = decrypt_value(write_credentials['password'])
importer = twitter.TwitterImporter(neo4j_url, neo4j_user, neo4j_password)
for record in event["Records"]:
tweet = json.loads(record["Sns"]["Message"])
print(f"Processing tweet {tweet['id']}")
for url in tweet["entities"]["urls"]:
initial_uri = url["expanded_url"]
expanded_uri = importer.unshorten(initial_uri)
cleaned_uri = importer.clean_uri(expanded_uri)
print(f"Initial: {initial_uri}, Expanded: {expanded_uri}, Cleaned: {cleaned_uri}")
url["expanded_url"] = cleaned_uri
title = importer.hydrate_url(expanded_uri)
url["title"] = title
importer.import_tweet(tweet)
def twitter_import(event, _):
print("Event:", event)
neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost")
neo4j_user = os.environ.get('NEO4J_USER', "neo4j")
neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD'])
twitter_bearer = decrypt_value(os.environ['TWITTER_BEARER'])
search = os.environ.get("TWITTER_SEARCH")
twitter.import_links(neo4j_url=neo4j_url,
neo4j_user=neo4j_user,
neo4j_pass=neo4j_password,
bearer_token=twitter_bearer,
search=search)
def twitter_unshorten_links(event, _):
print("Event:", event)
neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost")
neo4j_user = os.environ.get('NEO4J_USER', "neo4j")
neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD'])
twitter.unshorten_links(neo4j_url=neo4j_url,
neo4j_user=neo4j_user,
neo4j_pass=neo4j_password)
def constraints(event, _):
print("Event:", event)
credentials = config["credentials"]
write_credentials = credentials["write"]
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
neo4j_user = write_credentials.get('user', "neo4j")
neo4j_password = decrypt_value(write_credentials['password'])
schema.configure_constraints(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password)
def twitter_unshorten_links(event, _):
print("Event:", event)
credentials = config["credentials"]
write_credentials = credentials["write"]
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
neo4j_user = write_credentials.get('user', "neo4j")
neo4j_password = decrypt_value(write_credentials['password'])
twitter.unshorten_links(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password)
def so_import(event, _):
print("Event:", event)
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
credentials = config["credentials"]
write_credentials = credentials["write"]
neo4j_user = write_credentials.get('user', "neo4j")
neo4j_password = decrypt_value(write_credentials['password'])
so_key = decrypt_value(credentials["stackOverflowApiKey"])
importer = so.SOImporter(neo4j_url, neo4j_user, neo4j_password, so_key)
for record in event["Records"]:
message = json.loads(record["Sns"]["Message"])
tags = message["tags"]
start_date = as_timestamp(parser.parse(message["startDate"]))
end_date = as_timestamp(parser.parse(message["endDate"]))
importer.process_tag(tags, start_date, end_date)
def github_import(event, _):
print("Event:", event)
credentials = config["credentials"]
write_credentials = credentials["write"]
neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost"))
neo4j_user = write_credentials.get('user', "neo4j")
neo4j_password = decrypt_value(write_credentials['password'])
github_token = decrypt_value(credentials["githubToken"])
importer = github.GitHubImporter(neo4j_url, neo4j_user, neo4j_password, github_token)
importer.update_release_assets()
for record in event["Records"]:
message = json.loads(record["Sns"]["Message"])
tags = message["tags"]
start_date = message["startDate"]
end_date = message["endDate"]
importer.process_tag(tags, start_date, end_date)
def so_import(event, _):
print("Event:", event)
neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost")
neo4j_user = os.environ.get('NEO4J_USER', "neo4j")
neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD'])
tag = os.environ.get('TAG')
so.import_so(neo4j_url=neo4j_url,
neo4j_user=neo4j_user,
neo4j_pass=neo4j_password,
tag=tag)
def generate_page_summary(event, _):
print("Event:", event)
url = config["serverUrl"]
read_only_credentials = config["credentials"]["readonly"]
user = read_only_credentials["user"]
password = decrypt_value(read_only_credentials["password"])
title = config["communityName"]
short_name = config["s3Bucket"]
logo_src = config["logo"]
summary.generate(url, user, password, title, short_name, logo_src)
