Пример #1
0
 def __init__(self):
     Scraper.__init__(self)
     api_key = self.config["youtube"]["api_key"]
     self.url = "https://www.googleapis.com/youtube/v3/search"
     self.params = {
         "order": "date",
         "maxResults": 10,
         "channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
         "key": api_key,
         "type": "upload",
         "part": "snippet"
     }
     self.details = Bernie2016VideoDetailScraper()
     self.video_provider = VideoProvider()
     self.push_provider = PushProvider()
Пример #2
0
 def __init__(self):
     Scraper.__init__(self)
     api_key = self.config["youtube"]["api_key"]
     self.url = "https://www.googleapis.com/youtube/v3/search"
     self.params = {
         "order": "date",
         "maxResults": 10,
         "channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
         "key": api_key,
         "type": "upload",
         "part": "snippet",
     }
     self.details = Bernie2016VideoDetailScraper()
     self.video_provider = VideoProvider()
     self.push_provider = PushProvider()
Пример #3
0
class Bernie2016VideosScraper(Scraper):
    def __init__(self):
        Scraper.__init__(self)
        api_key = self.config["youtube"]["api_key"]
        self.url = "https://www.googleapis.com/youtube/v3/search"
        self.params = {
            "order": "date",
            "maxResults": 10,
            "channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
            "key": api_key,
            "type": "upload",
            "part": "snippet"
        }
        self.details = Bernie2016VideoDetailScraper()
        self.video_provider = VideoProvider()
        self.push_provider = PushProvider()

    def translate(self, json):
        idJson = json["id"]
        snippetJson = json["snippet"]

        record = {
            "site": "youtube.com",
            "video_id": idJson["videoId"],
            "url": "https://www.youtube.com/watch?v=" + idJson["videoId"],
            "title": snippetJson["title"],
            "snippet": snippetJson["description"],
            "thumbnail_url": snippetJson["thumbnails"]["high"]["url"],
            "timestamp_publish": snippetJson["publishedAt"]
        }
        return record

    def go(self):
        r = self.get(self.url, params=self.params, result_format="json")
        for item in r["items"]:
            if item["id"]["kind"] != 'youtube#video':
                continue
            record = self.translate(item)
            record["description"] = self.fetch_full_description(
                record["video_id"])
            record["title"] = record["title"].replace(" | Bernie Sanders", "")

            if self.video_provider.exists_by_video_id(record["video_id"]):
                print "found"
            else:
                print "not found"
                msg = "Inserting record for '{0}'."
                logging.info(msg.format(record["title"].encode("utf8")))
                record["timestamp_creation"] = datetime.now()
                video = self.video_provider.create(record)

                # Add push record for possible notification pushing
                push_record = {
                    "object_type": "video",
                    "object_uuid": video.uuid,
                    "title":
                    video.title + " - new video posted by Bernie Sanders",
                    "body": "See this new video now",
                    "url": video.url
                }
                push = self.push_provider.create(push_record)

    def fetch_full_description(self, video_id):
        self.details.params = {
            "key": self.config["youtube"]["api_key"],
            "part": "snippet,contentDetails",
            "id": video_id
        }
        r = self.details.get(self.details.url,
                             params=self.details.params,
                             result_format="json")
        return r["items"][0]["snippet"]["description"]
Пример #4
0
@app.route('/issue/<uuid:issue_uuid>', methods=['GET', 'POST'])
@auth.login_required
def issue_detail(issue_uuid):
    issue = issue_provider.read(issue_uuid)
    updated = False
    if request.method == 'POST' and issue_provider.update(issue, request):
        updated = True
    return render_template('issue.html', issue=issue, updated=updated)


if __name__ == '__main__':
    try:
        with open('/opt/bernie/config.yml', 'r') as f:
            conf = yaml.load(f)['flask']
    except IOError:
        msg = "Could not open config file: {0}"
        logging.info(msg.format(self.configfile))
        raise
    else:
        event_provider = EventProvider()
        issue_provider = IssueProvider()
        video_provider = VideoProvider()
        article_provider = ArticleProvider()
        news_provider = NewsProvider()
        push_provider = PushProvider()
        users = {conf['httpauth_username']: conf['httpauth_password']}
        app.run(host=conf['host'], debug=conf['debug'])
        register(conf['parse_application_id'], conf['parse_rest_api_key'],
                 conf['parse_master_key'])
        #Push.message("Good morning", channels=["Mike Testing"])
Пример #5
0
class Bernie2016VideosScraper(Scraper):
    def __init__(self):
        Scraper.__init__(self)
        api_key = self.config["youtube"]["api_key"]
        self.url = "https://www.googleapis.com/youtube/v3/search"
        self.params = {
            "order": "date",
            "maxResults": 10,
            "channelId": "UCH1dpzjCEiGAt8CXkryhkZg",
            "key": api_key,
            "type": "upload",
            "part": "snippet",
        }
        self.details = Bernie2016VideoDetailScraper()
        self.video_provider = VideoProvider()
        self.push_provider = PushProvider()

    def translate(self, json):
        idJson = json["id"]
        snippetJson = json["snippet"]

        record = {
            "site": "youtube.com",
            "video_id": idJson["videoId"],
            "url": "https://www.youtube.com/watch?v=" + idJson["videoId"],
            "title": snippetJson["title"],
            "snippet": snippetJson["description"],
            "thumbnail_url": snippetJson["thumbnails"]["high"]["url"],
            "timestamp_publish": snippetJson["publishedAt"],
        }
        return record

    def go(self):
        r = self.get(self.url, params=self.params, result_format="json")
        for item in r["items"]:
            if item["id"]["kind"] != "youtube#video":
                continue
            record = self.translate(item)
            record["description"] = self.fetch_full_description(record["video_id"])
            record["title"] = record["title"].replace(" | Bernie Sanders", "")

            if self.video_provider.exists_by_video_id(record["video_id"]):
                print "found"
            else:
                print "not found"
                msg = "Inserting record for '{0}'."
                logging.info(msg.format(record["title"].encode("utf8")))
                record["timestamp_creation"] = datetime.now()
                video = self.video_provider.create(record)

                # Add push record for possible notification pushing
                push_record = {
                    "object_type": "video",
                    "object_uuid": video.uuid,
                    "title": video.title + " - new video posted by Bernie Sanders",
                    "body": "See this new video now",
                    "url": video.url,
                }
                push = self.push_provider.create(push_record)

    def fetch_full_description(self, video_id):
        self.details.params = {
            "key": self.config["youtube"]["api_key"],
            "part": "snippet,contentDetails",
            "id": video_id,
        }
        r = self.details.get(self.details.url, params=self.details.params, result_format="json")
        return r["items"][0]["snippet"]["description"]