コード例 #1
0
ファイル: celery_worker.py プロジェクト: rudolfce/scrapper
import scrapper.controllers.miner
from scrapper import create_app, make_celery

flask_app = create_app('scrapper.config.testing')
celery = make_celery(flask_app)
コード例 #2
0
ファイル: miner.py プロジェクト: rudolfce/scrapper
#-*- encoding: utf-8 -*-
import json
from datetime import datetime

import requests
from bs4 import BeautifulSoup
from flask import current_app

from scrapper import make_celery
from scrapper.models.user import User

celery = make_celery(current_app)


@celery.task
def scrape_twitter(user_name):
    user = User()
    user.username = user_name
    url_request = 'https://twitter.com/' + str(user_name)
    response = requests.get(url_request)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, "html.parser")

        data = soup.find("input", attrs={"class": "json-data"})
        data = json.loads(data["value"])
        profile_data = data["profile_user"]
        user.name = profile_data["name"]
        user.bio = profile_data["description"]
        user.location = profile_data["location"]
        user.exists = True
        time = datetime.now()
コード例 #3
0
ファイル: miner.py プロジェクト: rudolfce/scrapper
#-*- encoding: utf-8 -*-
import json
from datetime import datetime

import requests
from bs4 import BeautifulSoup
from flask import current_app

from scrapper import make_celery
from scrapper.models.user import User

celery = make_celery(current_app)

@celery.task
def scrape_twitter(user_name):
    user = User()
    user.username = user_name
    url_request = 'https://twitter.com/' + str(user_name)
    response = requests.get(url_request)
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, "html.parser")

        data = soup.find("input", attrs={"class":"json-data"})
        data = json.loads(data["value"])
        profile_data = data["profile_user"]
        user.name = profile_data["name"]
        user.bio = profile_data["description"]
        user.location = profile_data["location"]
        user.exists = True
        time = datetime.now()
        user.query_date = time.strftime('%m/%d/%Y %I:%M%p')