def scrape_instagram_account_by_id(upstream_id, stub=False):
""" Scrape instagram bio data for upstream ID and update a profile. """
db_session = worker.get_session()
proxies = _get_proxies(db_session)
# Instagram API request.
api_url = 'https://api.instagram.com/v1/users/{}'.format(upstream_id)
response = requests.get(
api_url,
proxies=proxies,
verify=False
)
response.raise_for_status()
data = response.json()['data']
# Update the profile.
data = response.json()[0]
profile = Profile('instagram', upstream_id, data['screen_name'])
db_session.add(profile)
try:
db_session.commit()
except IntegrityError:
# Already exists: use the existing profile.
db_session.rollback()
profile = db_session.query(Profile) \
.filter(Profile.site == 'instagram') \
.filter(Profile.upstream_id == upstream_id) \
.one()
# Update profile
profile.last_update = datetime.now()
profile.description = data['bio']
profile.follower_count = int(data['counts']['followed_by'])
profile.friend_count = int(data['counts']['follows'])
profile.homepage = data['website']
profile.name = data['full_name']
profile.post_count = int(data['counts']['media'])
profile.is_stub = stub
db_session.commit()
# Schedule followup jobs.
app.queue.schedule_index_profile(profile) # index all profiles, inc stubs
if not stub:
app.queue.schedule_avatar(profile, data['profile_picture'])
app.queue.schedule_posts(profile, recent=True)
app.queue.schedule_relations(profile)
return profile.as_dict()
def scrape_instagram_account(username, stub=False):
""" Scrape instagram bio data and create (or update) a profile. """
# Getting a user ID is more difficult than it ought to be: you need to
# search for the username and iterate through the search results results to
# find an exact match.
db_session = worker.get_session()
proxies = _get_proxies(db_session)
api_url = 'https://api.instagram.com/v1/users/search'
params = {'q': username}
response = requests.get(
api_url,
params=params,
proxies=proxies,
verify=False
)
response.raise_for_status()
search_results = response.json()
username_lower = username.lower()
user_id = None
for user_result in search_results['data']:
if user_result['username'].lower() == username_lower:
user_id = user_result['id']
break
if user_id is None:
raise ScrapeException('Can\'t find Instagram user named {}.'
.format(username))
# Now make another request to get this user's profile data.
api_url = 'https://api.instagram.com/v1/users/{}'.format(user_id)
response = requests.get(
api_url,
proxies=proxies,
verify=False
)
response.raise_for_status()
data = response.json()['data']
profile = Profile('instagram', user_id, data['username'])
db_session.add(profile)
try:
db_session.commit()
except IntegrityError:
# Already exists: use the existing profile.
db_session.rollback()
profile = db_session.query(Profile) \
.filter(Profile.site == 'instagram') \
.filter(Profile.upstream_id == user_id) \
.one()
profile.last_update = datetime.now()
profile.description = data['bio']
profile.follower_count = int(data['counts']['followed_by'])
profile.friend_count = int(data['counts']['follows'])
profile.homepage = data['website']
profile.name = data['full_name']
profile.post_count = int(data['counts']['media'])
profile.is_stub = stub
db_session.commit()
# Schedule followup jobs.
app.queue.schedule_index_profile(profile) # index all profiles, inc stubs
if not stub:
app.queue.schedule_avatar(profile, data['profile_picture'])
app.queue.schedule_posts(profile, recent=True)
app.queue.schedule_relations(profile)
return profile.as_dict()
def scrape_instagram_account(username):
''' Scrape instagram bio data and create (or update) a profile. '''
# Getting a user ID is more difficult than it ought to be: you need to
# search for the username and iterate through the search results results to
# find an exact match.
db_session = worker.get_session()
proxies = _get_proxies(db_session)
api_url = 'https://api.instagram.com/v1/users/search'
params = {'q': username}
response = requests.get(
api_url,
params=params,
proxies=proxies,
verify=False
)
response.raise_for_status()
search_results = response.json()
username_lower = username.lower()
user_id = None
for user_result in search_results['data']:
if user_result['username'].lower() == username_lower:
user_id = user_result['id']
break
if user_id is None:
raise ScrapeException('Can\'t find Instagram user named {}.'
.format(username))
# Now make another request to get this user's profile data.
api_url = 'https://api.instagram.com/v1/users/{}'.format(user_id)
response = requests.get(
api_url,
proxies=proxies,
verify=False
)
response.raise_for_status()
data = response.json()['data']
profile = Profile('instagram', user_id, data['username'])
db_session.add(profile)
try:
db_session.commit()
except IntegrityError:
# Already exists: use the existing profile.
db_session.rollback()
profile = db_session.query(Profile) \
.filter(Profile.site=='instagram') \
.filter(Profile.upstream_id==user_id) \
.one()
profile.description = data['bio']
profile.follower_count = int(data['counts']['followed_by'])
profile.friend_count = int(data['counts']['follows'])
profile.homepage = data['website']
profile.name = data['full_name']
profile.post_count = int(data['counts']['media'])
profile.is_stub = False
db_session.commit()
# Schedule followup jobs.
app.queue.schedule_avatar(profile, data['profile_picture'])
app.queue.schedule_index_profile(profile)
app.queue.schedule_posts(profile, recent=True)
app.queue.schedule_relations(profile)
return profile.as_dict()
