def __init__(self):
Scraper.__init__(self)
api_key = self.config["youtube"]["api_key"]
self.url = "https://www.googleapis.com/youtube/v3/search"
self.params = {
"order": "date",
"maxResults": 10,
"channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
"key": api_key,
"type": "upload",
"part": "snippet"
}
self.details = Bernie2016VideoDetailScraper()
self.video_provider = VideoProvider()
self.push_provider = PushProvider()
def __init__(self):
Scraper.__init__(self)
api_key = self.config["youtube"]["api_key"]
self.url = "https://www.googleapis.com/youtube/v3/search"
self.params = {
"order": "date",
"maxResults": 10,
"channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
"key": api_key,
"type": "upload",
"part": "snippet",
}
self.details = Bernie2016VideoDetailScraper()
self.video_provider = VideoProvider()
self.push_provider = PushProvider()
class Bernie2016VideosScraper(Scraper):
def __init__(self):
Scraper.__init__(self)
api_key = self.config["youtube"]["api_key"]
self.url = "https://www.googleapis.com/youtube/v3/search"
self.params = {
"order": "date",
"maxResults": 10,
"channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
"key": api_key,
"type": "upload",
"part": "snippet"
}
self.details = Bernie2016VideoDetailScraper()
self.video_provider = VideoProvider()
self.push_provider = PushProvider()
def translate(self, json):
idJson = json["id"]
snippetJson = json["snippet"]
record = {
"site": "youtube.com",
"video_id": idJson["videoId"],
"url": "https://www.youtube.com/watch?v=" + idJson["videoId"],
"title": snippetJson["title"],
"snippet": snippetJson["description"],
"thumbnail_url": snippetJson["thumbnails"]["high"]["url"],
"timestamp_publish": snippetJson["publishedAt"]
}
return record
def go(self):
r = self.get(self.url, params=self.params, result_format="json")
for item in r["items"]:
if item["id"]["kind"] != 'youtube#video':
continue
record = self.translate(item)
record["description"] = self.fetch_full_description(
record["video_id"])
record["title"] = record["title"].replace(" | Bernie Sanders", "")
if self.video_provider.exists_by_video_id(record["video_id"]):
print "found"
else:
print "not found"
msg = "Inserting record for '{0}'."
logging.info(msg.format(record["title"].encode("utf8")))
record["timestamp_creation"] = datetime.now()
video = self.video_provider.create(record)
# Add push record for possible notification pushing
push_record = {
"object_type": "video",
"object_uuid": video.uuid,
"title":
video.title + " - new video posted by Bernie Sanders",
"body": "See this new video now",
"url": video.url
}
push = self.push_provider.create(push_record)
def fetch_full_description(self, video_id):
self.details.params = {
"key": self.config["youtube"]["api_key"],
"part": "snippet,contentDetails",
"id": video_id
}
r = self.details.get(self.details.url,
params=self.details.params,
result_format="json")
return r["items"][0]["snippet"]["description"]
@app.route('/issue/<uuid:issue_uuid>', methods=['GET', 'POST'])
@auth.login_required
def issue_detail(issue_uuid):
issue = issue_provider.read(issue_uuid)
updated = False
if request.method == 'POST' and issue_provider.update(issue, request):
updated = True
return render_template('issue.html', issue=issue, updated=updated)
if __name__ == '__main__':
try:
with open('/opt/bernie/config.yml', 'r') as f:
conf = yaml.load(f)['flask']
except IOError:
msg = "Could not open config file: {0}"
logging.info(msg.format(self.configfile))
raise
else:
event_provider = EventProvider()
issue_provider = IssueProvider()
video_provider = VideoProvider()
article_provider = ArticleProvider()
news_provider = NewsProvider()
push_provider = PushProvider()
users = {conf['httpauth_username']: conf['httpauth_password']}
app.run(host=conf['host'], debug=conf['debug'])
register(conf['parse_application_id'], conf['parse_rest_api_key'],
conf['parse_master_key'])
#Push.message("Good morning", channels=["Mike Testing"])
class Bernie2016VideosScraper(Scraper):
def __init__(self):
Scraper.__init__(self)
api_key = self.config["youtube"]["api_key"]
self.url = "https://www.googleapis.com/youtube/v3/search"
self.params = {
"order": "date",
"maxResults": 10,
"channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
"key": api_key,
"type": "upload",
"part": "snippet",
}
self.details = Bernie2016VideoDetailScraper()
self.video_provider = VideoProvider()
self.push_provider = PushProvider()
def translate(self, json):
idJson = json["id"]
snippetJson = json["snippet"]
record = {
"site": "youtube.com",
"video_id": idJson["videoId"],
"url": "https://www.youtube.com/watch?v=" + idJson["videoId"],
"title": snippetJson["title"],
"snippet": snippetJson["description"],
"thumbnail_url": snippetJson["thumbnails"]["high"]["url"],
"timestamp_publish": snippetJson["publishedAt"],
}
return record
def go(self):
r = self.get(self.url, params=self.params, result_format="json")
for item in r["items"]:
if item["id"]["kind"] != "youtube#video":
continue
record = self.translate(item)
record["description"] = self.fetch_full_description(record["video_id"])
record["title"] = record["title"].replace(" | Bernie Sanders", "")
if self.video_provider.exists_by_video_id(record["video_id"]):
print "found"
else:
print "not found"
msg = "Inserting record for '{0}'."
logging.info(msg.format(record["title"].encode("utf8")))
record["timestamp_creation"] = datetime.now()
video = self.video_provider.create(record)
# Add push record for possible notification pushing
push_record = {
"object_type": "video",
"object_uuid": video.uuid,
"title": video.title + " - new video posted by Bernie Sanders",
"body": "See this new video now",
"url": video.url,
}
push = self.push_provider.create(push_record)
def fetch_full_description(self, video_id):
self.details.params = {
"key": self.config["youtube"]["api_key"],
"part": "snippet,contentDetails",
"id": video_id,
}
r = self.details.get(self.details.url, params=self.details.params, result_format="json")
return r["items"][0]["snippet"]["description"]
