def poll(): client = authenticate() for key in LINKEDIN_ORGS.keys(): data_points = poll_stats_data(client, LINKEDIN_ORGS[key], key) for data_point in data_points: PollerUtil.post_to_ingest_api(data_point, LINKEDIN_STATS_TYPE)
def poll(): """ This method calls the poll_account_data with the defined accounts in TWITTER_ACCOUNTS and is returned new tweets. If new tweets are found, these get put in to dynamoDB and last_inserted_id is updated :return: True """ api = authenticate() week_ago = (datetime.now() - timedelta(days=7)).date() for account in TWITTER_ACCOUNTS: last_inserted_doc = PollerUtil.fetch_last_inserted_doc(TWITTER_ACCOUNTS[account]) last_inserted_id = 1 if last_inserted_doc: last_inserted_id = int(last_inserted_doc) data_points = poll_account_data(api, account, last_inserted_id, week_ago) for data_point in data_points: result = PollerUtil.post_to_ingest_api(data_point, TWITTER_ACCOUNT_TYPE) if result is not None: last_inserted_id = data_point['id'] PollerUtil.upload_last_inserted_doc(last_inserted_id, TWITTER_ACCOUNTS[account]) return True
def poll(): client = authenticate() for org_name in LINKEDIN_ORGS.keys(): for api_type in LINKEDIN_APIS.keys(): doc_name = api_type + org_name.replace(" ", "").upper() last_inserted_timestamp = 0 last_inserted_doc = PollerUtil.fetch_last_inserted_doc(doc_name) if last_inserted_doc: last_inserted_timestamp = int(last_inserted_doc) data_points = poll_daily_stats_data(client, LINKEDIN_ORGS[org_name], org_name, LINKEDIN_APIS[api_type], last_inserted_timestamp) if not data_points: break for data_point in data_points: result = PollerUtil.post_to_ingest_api(data_point, api_type) if result is not None: last_inserted_timestamp = data_point['data']['timeRange'][ 'end'] PollerUtil.upload_last_inserted_doc(last_inserted_timestamp, doc_name) return True
def insert_new_ubw_data(doc): """ :param doc: A ubw document. :return: This method attempts to upload the ubw document into the ingest API and if that was successful it returns the reg_period of this document. (aka the last_inserted_doc) """ if PollerUtil.post_to_ingest_api(doc, UBW_TYPE) == 200: # This method is always updating the last_inserted_doc global after uploading new data. return doc["reg_period"] return None
def poll(): """ This method gets run every day and should fetch data from the website and compare it to a database in order to avoid duplicates. :return: True if everything was successful. """ # Should actually be called most_recent here in blog_poller. last_inserted_doc = PollerUtil.fetch_last_inserted_doc(KNOWITLABS_TYPE) html = get_html_from_blog() medium_data = get_medium_data_dict(html) docs = create_docs(medium_data) most_recent = docs[0]["id"] for doc in docs: if should_upload_ingest(doc, last_inserted_doc): PollerUtil.post_to_ingest_api(doc, KNOWITLABS_TYPE) else: break if last_inserted_doc != most_recent: PollerUtil.upload_last_inserted_doc(most_recent, KNOWITLABS_TYPE) return True
def poll(): date_now = datetime.now().date() last_inserted_doc = PollerUtil.fetch_last_inserted_doc(DOC_TYPE) if last_inserted_doc: last_inserted_date = datetime.strptime(str(last_inserted_doc), DATETIME_FORMAT).date() if not last_inserted_date < date_now: return False client = authenticate() for key in LINKEDIN_ORGS.keys(): data_points = poll_stats_data(client, LINKEDIN_ORGS[key], key) for data_point in data_points: for data_point_key in data_point.keys(): PollerUtil.post_to_ingest_api(data_point[data_point_key], data_point_key) PollerUtil.upload_last_inserted_doc(str(date_now), DOC_TYPE) return True
def post_commits(commits, repo): """ post the given commits, and update last_inserted_doc for the repo :param commits: A list of commits to post into ingest :param repo: the repo which the commits are associated with """ last_inserted_id = None for commit in commits: result = PollerUtil.post_to_ingest_api( type=BITBUCKET_TYPE, data=commit) if result is 200: last_inserted_id = commit["id"] if last_inserted_id is not None: last_inserted_name = BITBUCKET_TYPE + repo["slug"] PollerUtil.upload_last_inserted_doc( last_inserted_doc=last_inserted_id, type=last_inserted_name)
def poll(): last_inserted_doc = PollerUtil.fetch_last_inserted_doc(YR_TYPE) last_inserted_timestamp = 0 if last_inserted_doc: last_inserted_timestamp = int(last_inserted_doc) location = os.getenv("DATAPLATTFORM_YR_LOCATION", "Norway/Oslo/Oslo/Lakkegata") data_points = get_yr_data(location, last_inserted_timestamp) for forecast in data_points: result = PollerUtil.post_to_ingest_api(forecast, YR_TYPE) if result is not None: last_inserted_timestamp = forecast["time_from"] PollerUtil.upload_last_inserted_doc(last_inserted_timestamp, YR_TYPE) return True
def poll(): """ This method calls the poll_search_data method and is returned new tweets. If new tweets are found, these get put in to dynamoDB and last_inserted_id is updated :return: True """ api = authenticate() last_inserted_doc = PollerUtil.fetch_last_inserted_doc(TWITTER_SEARCH_TYPE) last_inserted_id = 1 if last_inserted_doc: last_inserted_id = int(last_inserted_doc) data_points = poll_search_data(api, last_inserted_id) for data_point in data_points: result = PollerUtil.post_to_ingest_api(data_point, TWITTER_SEARCH_TYPE) if result is not None: last_inserted_id = data_point['id'] PollerUtil.upload_last_inserted_doc(last_inserted_id, TWITTER_SEARCH_TYPE) return True