def twitter_publish_events_import(event, context): print("Event:", event) credentials = config["credentials"] context_parts = context.invoked_function_arn.split(':') topic_name = "Twitter-{0}".format(config["communityName"]) topic_arn = "arn:aws:sns:{region}:{account_id}:{topic}".format(region=context_parts[3], account_id=context_parts[4], topic=topic_name) neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) neo4j_user = credentials["readonly"].get('user', "neo4j") neo4j_password = decrypt_value(credentials["readonly"]['password']) sns = boto3.client('sns') twitter_bearer = decrypt_value(credentials["twitterBearer"]) search = config["twitterSearch"] since_id = twitter.find_last_tweet(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password) print(f"Most recent tweet: {since_id}") tweets = twitter.find_tweets_since(since_id=since_id, search=search, bearer_token=twitter_bearer) for tweet in tweets: sns.publish(TopicArn=topic_arn, Message=json.dumps(tweet))
def meetup_events_import(event, _): print("Event:", event) credentials = config["credentials"] write_credentials = credentials["write"] neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) neo4j_user = write_credentials.get('user', "neo4j") neo4j_password = decrypt_value(write_credentials['password']) meetup_key = decrypt_value(credentials["meetupApiKey"]) meetup.import_events(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password, meetup_key=meetup_key)
def meetup_events_import(event, _): print("Event:", event) neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost") neo4j_user = os.environ.get('NEO4J_USER', "neo4j") neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD']) meetup_key = decrypt_value(os.environ["MEETUP_API_KEY"]) meetup.import_events(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password, meetup_key=meetup_key)
def twitter_import(event, _): print("Event:", event) credentials = config["credentials"] write_credentials = credentials["write"] neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) neo4j_user = write_credentials.get('user', "neo4j") neo4j_password = decrypt_value(write_credentials['password']) twitter_bearer = decrypt_value(credentials["twitterBearer"]) search = config["twitterSearch"] twitter.import_links(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password, bearer_token=twitter_bearer, search=search)
def github_import(event, _): print("Event:", event) neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost") neo4j_user = os.environ.get('NEO4J_USER', "neo4j") neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD']) github_token = decrypt_value(os.environ["GITHUB_TOKEN"]) tag = os.environ.get('TAG') github.import_github(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password, tag=tag, github_token=github_token)
def twitter_topic_import(event, _): print("Event:", event) credentials = config["credentials"] write_credentials = credentials["write"] neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) neo4j_user = write_credentials.get('user', "neo4j") neo4j_password = decrypt_value(write_credentials['password']) importer = twitter.TwitterImporter(neo4j_url, neo4j_user, neo4j_password) for record in event["Records"]: tweet = json.loads(record["Sns"]["Message"]) print(f"Processing tweet {tweet['id']}") for url in tweet["entities"]["urls"]: initial_uri = url["expanded_url"] expanded_uri = importer.unshorten(initial_uri) cleaned_uri = importer.clean_uri(expanded_uri) print(f"Initial: {initial_uri}, Expanded: {expanded_uri}, Cleaned: {cleaned_uri}") url["expanded_url"] = cleaned_uri title = importer.hydrate_url(expanded_uri) url["title"] = title importer.import_tweet(tweet)
def twitter_import(event, _): print("Event:", event) neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost") neo4j_user = os.environ.get('NEO4J_USER', "neo4j") neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD']) twitter_bearer = decrypt_value(os.environ['TWITTER_BEARER']) search = os.environ.get("TWITTER_SEARCH") twitter.import_links(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password, bearer_token=twitter_bearer, search=search)
def twitter_unshorten_links(event, _): print("Event:", event) neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost") neo4j_user = os.environ.get('NEO4J_USER', "neo4j") neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD']) twitter.unshorten_links(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password)
def constraints(event, _): print("Event:", event) credentials = config["credentials"] write_credentials = credentials["write"] neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) neo4j_user = write_credentials.get('user', "neo4j") neo4j_password = decrypt_value(write_credentials['password']) schema.configure_constraints(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password)
def twitter_unshorten_links(event, _): print("Event:", event) credentials = config["credentials"] write_credentials = credentials["write"] neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) neo4j_user = write_credentials.get('user', "neo4j") neo4j_password = decrypt_value(write_credentials['password']) twitter.unshorten_links(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password)
def so_import(event, _): print("Event:", event) neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) credentials = config["credentials"] write_credentials = credentials["write"] neo4j_user = write_credentials.get('user', "neo4j") neo4j_password = decrypt_value(write_credentials['password']) so_key = decrypt_value(credentials["stackOverflowApiKey"]) importer = so.SOImporter(neo4j_url, neo4j_user, neo4j_password, so_key) for record in event["Records"]: message = json.loads(record["Sns"]["Message"]) tags = message["tags"] start_date = as_timestamp(parser.parse(message["startDate"])) end_date = as_timestamp(parser.parse(message["endDate"])) importer.process_tag(tags, start_date, end_date)
def github_import(event, _): print("Event:", event) credentials = config["credentials"] write_credentials = credentials["write"] neo4j_url = "bolt+routing://{url}".format(url=config.get("serverUrl", "localhost")) neo4j_user = write_credentials.get('user', "neo4j") neo4j_password = decrypt_value(write_credentials['password']) github_token = decrypt_value(credentials["githubToken"]) importer = github.GitHubImporter(neo4j_url, neo4j_user, neo4j_password, github_token) importer.update_release_assets() for record in event["Records"]: message = json.loads(record["Sns"]["Message"]) tags = message["tags"] start_date = message["startDate"] end_date = message["endDate"] importer.process_tag(tags, start_date, end_date)
def so_import(event, _): print("Event:", event) neo4j_url = os.environ.get('NEO4J_URL', "bolt://localhost") neo4j_user = os.environ.get('NEO4J_USER', "neo4j") neo4j_password = decrypt_value(os.environ['NEO4J_PASSWORD']) tag = os.environ.get('TAG') so.import_so(neo4j_url=neo4j_url, neo4j_user=neo4j_user, neo4j_pass=neo4j_password, tag=tag)
def generate_page_summary(event, _): print("Event:", event) url = config["serverUrl"] read_only_credentials = config["credentials"]["readonly"] user = read_only_credentials["user"] password = decrypt_value(read_only_credentials["password"]) title = config["communityName"] short_name = config["s3Bucket"] logo_src = config["logo"] summary.generate(url, user, password, title, short_name, logo_src)