def _add_following(self, of_user, following):
#local = threading.local()
for user in following:
with db.read_transaction:
found = User.match_username(
user['username']) or Business.match_username(
user['username'])
if not found:
with db.write_transaction:
new_user = User(
user_id=user['pk'],
is_private=user['is_private'],
full_name=user['full_name'],
username=user['username'],
is_verified=user['is_verified'],
has_anonymous_profile_pic=user[
'has_anonymous_profile_picture']).save()
if user['has_anonymous_profile_picture'] is False:
with db.write_transaction:
new_profile_pic = ProfilePicture(
profile_pic_url=user['profile_pic_url']).save()
new_user.profile_pic.connect(new_profile_pic)
with db.write_transaction:
new_user.followers.connect(of_user)
data = {'scrape_type': 'basic', 'username': new_user.username}
self.r.rpush('queue:scrape', json.dumps(data))
else:
with db.write_transaction:
found.followers.connect(of_user)
def fill_users(session):
# add admin user
admin = get_admin(session)
if not admin:
admin = User(name=admin_name, role_id=Role.ADMIN)
session.add(admin)
session.commit()
# add client user
client = get_test_client(session)
if not client:
client = User(name=test_client_name, role_id=Role.CLIENT)
session.add(client)
session.commit()
def __save_like(self, node, media):
with db.read_transaction:
liker = User.match_username(node['username'])
if not liker:
liker = self.__save_user_basic(node)
with db.write_transaction:
media.liked_by.connect(liker)
def setUp(self):
super().setUp()
u1 = User(firstname="Fernando",
lastname="Alonso",
email="*****@*****.**")
q1 = Question(
title="What is the fatest car?",
body="Which team should I chose to win the F1 world championship?",
user=u1)
q2 = Question(
title="What is the fatest car?",
body="Which team should I chose to win the F1 world championship?",
user=u1)
q3 = Question(
title="What is the fatest car?",
body="Which team should I chose to win the F1 world championship?",
user=u1)
q4 = Question(
title="What is the fatest car?",
body="Which team should I chose to win the F1 world championship?",
user=u1)
a1 = Answer(body="Message 1", question=q1, user=u1)
self.db.add(u1)
self.db.add(q1)
self.db.add(q2)
self.db.add(q3)
self.db.add(q4)
self.db.add(a1)
self.db.commit()
def setUp(self):
super().setUp()
self.u1 = User(firstname="Fernando",
lastname="Alonso",
email="*****@*****.**")
self.q1 = Question(
title="What is the fatest car?",
body="Which team should I chose to win the F1 world championship?",
user=self.u1)
self.q2 = Question(
title="What is the fatest car?",
body="Which team should I chose to win the F1 world championship?",
user=self.u1)
self.q3 = Question(
title="What is the fatest car?",
body="Which team should I chose to win the F1 world championship?",
user=self.u1)
self.db.add(self.u1)
self.db.add(self.q1)
self.db.add(self.q2)
self.db.add(self.q3)
self.db.commit()
v = VoteQuestion(user_id=self.request_user.id, question_id=self.q3.id)
self.db.add(v)
self.db.commit()
def setUp(self):
super().setUp()
u1 = User(firstname="Fernando", lastname="Alonso", email="*****@*****.**")
self.q1 = Question(title="What is the fatest car?", body="Which team should I chose to win the F1 world championship?", user=u1)
self.db.add(u1)
self.db.add(self.q1)
self.db.commit()
def _get_stories(self, username=None, user=None):
if not user:
if not username:
return False
with db.read_transaction:
user = User.match_username(username)
if not user:
user = self._scrape_user(username, returns=True)
self.log.info('Fetching stories...')
data = self.scraper.fetch_stories(user_id=user.user_id)
new_stories = []
for item in reversed(data['items']):
if user.stories.search(taken_at_timestamp=datetime.datetime.
fromtimestamp(item['taken_at_timestamp'])):
break
new_stories.append(item)
for story in tqdm(reversed(new_stories),
desc='Saving new stories',
total=len(new_stories),
ascii=False):
self.__save_story(story,
user,
can_reshare=data.get('can_reshare'),
can_reply=data.get('can_reply'))
def _get_followers(self, username):
"""
Get followers list from Instagram and
update database if necessary.
`Required`
:param str username Instagram username
"""
self.log.info("Fetching followers for %s" % username)
with db.read_transaction:
user = User.match_username(username) or Business.match_username(
username)
if user:
user_id = user.user_id
else:
with self.lock(): # IMPORTANT! Pause queue processing thread
user = self._scrape_user(username, returns="user_id")
user_id = user.user_id
self._api_busy = True
followers = []
next_max_id = True
while next_max_id:
# First iteration
if next_max_id is True: next_max_id = ''
# Pagination of followers
_ = self.api.getUserFollowers(user_id, maxid=next_max_id)
result = self.api.LastJson.get('users', [])
followers.extend(result)
self._add_followers(user, result)
next_max_id = self.api.LastJson.get('next_max_id', '')
self._api_busy = False
return followers
def enable_mode(user: User, mode_id: int):
"""Функция активации режима работы программы."""
now = datetime.now()
actual_mode_usage = ModeUsage.objects(user=user, date_end=None).first()
# Закрываем работу прошлого режима
if actual_mode_usage:
actual_mode_usage.date_end = now
actual_mode_usage.save()
# Начинаем работу нового режима
ModeUsage(user=user, mode_id=mode_id, date_start=now).save()
if not user.program_is_active:
user.program_is_active = True
user.last_smoked = now
user.save()
def setUp(self):
super().setUp()
u1 = User(firstname="Fernando", lastname="Alonso", email="*****@*****.**")
self.q1 = Question(title="What is the fatest car?", body="Which team should I chose to win the F1 world championship?", user=u1, creator_id=self.request_user.id)
a1 = Answer(body="Message 1", question=self.q1, user=self.request_user, creator_id=self.request_user.id)
self.db.add(u1)
self.db.add(self.q1)
self.db.add(a1)
self.db.commit()
def setUp(self):
super().setUp()
u1 = User(firstname="Fernando", lastname="Alonso", email="*****@*****.**")
self.q1 = Question(title="What is the fatest car?", body="Which team should I chose to win the F1 world championship?", user=u1, creator_id=self.request_user.id)
tag = Tag(label='JavaScript')
self.db.add(u1)
self.db.add(tag)
self.db.add(self.q1)
self.db.commit()
def _add_followers(self, user, followers):
"""
Adds followers to given user. Saves to database if node doesn't exist.
If node already exists, update relationship between user and follower.
`Required`
:param obj user User object
:param list followers List of JSON from InstagramAPI getUserFollowers()
"""
local = threading.local()
for follower in followers:
with db.read_transaction:
local.found = User.match_username(
follower['username']) or Business.match_username(
follower['username'])
if not local.found:
with db.write_transaction:
local.new_user = User(
user_id=follower['pk'],
is_private=follower['is_private'],
full_name=follower['full_name'],
username=follower['username'],
is_verified=follower['is_verified']).save()
if follower['has_anonymous_profile_picture'] is False:
with db.write_transaction:
local.new_profile_pic = ProfilePicture(
profile_pic_url=follower['profile_pic_url']).save(
)
local.new_user.profile_pic.connect(
local.new_profile_pic)
with db.write_transaction:
local.new_user.following.connect(user)
local.data = {
'scrape_type': 'basic',
'username': local.new_user.username
}
self.r.rpush('queue:scrape', json.dumps(local.data))
local.data = {
'scrape_type': 'following',
'username': local.new_user.username
}
self.r.rpush('queue:scrape', json.dumps(local.data))
else:
with db.transaction:
local.found.following.connect(user)
def register_user(self, user_dict):
user = User(**user_dict)
# Commit in DB
try:
self.application.db.add(user)
self.application.db.commit()
except SQLAlchemyError as error:
self.application.db.rollback()
return user
def _init_user(self):
email = self.USER_DATA['email']
user = self.db.query(User).filter_by(email=email).first()
if not user:
self.request_user = User(**self.USER_DATA)
self.db.add(self.request_user)
self.db.commit()
else:
self.request_user = user
self.db.commit()
def __get_or_save_user(**kwargs):
with db.read_transaction:
if kwargs.get('id'):
user = User.nodes.first_or_none(user_id=kwargs['id'])
elif kwargs.get('username'):
user = User.nodes.first_or_none(username=kwargs['username'])
else:
return False
if not user:
with db.write_transaction:
user = User(**kwargs).save()
return user
def __save_sponsors(self, node, media):
if not node.get('edge_media_to_sponsor_user', {
'edges': []
}).get('edges'):
return
for edge in node['edge_media_to_sponsor_user']['edges']:
with db.read_transaction:
sponsor = User.match_username(
edge['node']['sponsor']['username'])
if not sponsor:
sponsor = self.__save_user_basic(edge['node']['sponsor'])
with db.write_transaction:
media.sponsors.connect(sponsor)
def setUp(self):
super().setUp()
u = User(firstname='Fernando',
lastname='Alonso',
email='*****@*****.**')
self.question = Question(title='Title',
body='Body',
user=u,
creator_id=self.request_user.id)
self.db.add(u)
self.db.add(self.question)
self.db.commit()
async def post(self):
# Create data
data = json.loads(self.request.body.decode('utf-8'))
firstname = check_param(data,
name='firstname',
type_param='string',
required=True)
lastname = check_param(data,
name='lastname',
type_param='string',
required=True)
email = check_param(data,
name='email',
type_param='string',
required=True)
password = check_param(data,
name='password',
type_param='string',
required=True)
hashed_password = hashlib.md5(password.encode('utf-8')).hexdigest()
user = User(firstname=firstname,
lastname=lastname,
email=email,
password=hashed_password)
self.application.db.add(user)
# Commit in DB
try:
self.application.db.commit()
except SQLAlchemyError as error:
self.application.db.rollback()
raise InternalServerError('Unable to create the user.', error)
# Returns response
self.set_status(201)
self.write({'data': user.to_dict()})
self.finish()
def setUp(self):
super().setUp()
self.u1 = User(firstname="Fernando", lastname="Alonso", email="*****@*****.**")
self.q1 = Question(title="What is the fatest car?", body="Which team should I chose to win the F1 world championship?", user=self.u1)
self.a1 = Answer(body="Message 1", question=self.q1, user=self.u1)
self.a2 = Answer(body="Message 1", question=self.q1, user=self.u1)
self.a3 = Answer(body="Message 1", question=self.q1, user=self.u1, vote_counter=1)
self.db.add(self.u1)
self.db.add(self.q1)
self.db.add(self.a1)
self.db.add(self.a2)
self.db.commit()
v = VoteAnswer(user_id=self.request_user.id, answer_id=self.a3.id)
self.db.add(v)
self.db.commit()
def __save_comment(self, node, media):
with db.read_transaction:
if Comment.nodes.first_or_none(comment_id=node['id']): return
created_at = datetime.datetime.fromtimestamp(node['created_at'])
with db.write_transaction:
new_comment = Comment(comment_id=node['id'],
created_at=created_at,
text=node['text']).save()
with db.read_transaction:
comment_owner = User.match_username(node['owner']['username'])
if not comment_owner:
comment_owner = self.__save_user_basic(node['owner'])
with db.write_transaction:
new_comment.owner.connect(comment_owner,
{'created_at': created_at})
media.comments.connect(new_comment)
def _deep_scrape(self,
username,
post_depth=50,
comment_depth=None,
geolocations=False,
tagged_users=False,
post_likes=False):
user = User.match_username(username)
if not user:
user = self._scrape_user(username, returns=True)
new_media = list(
self.scraper.query_media_gen(user, max_number=post_depth))
m_desc = f"Saving {user.username}'s {post_depth} most recent posts"
for media in tqdm(reversed(new_media),
desc=m_desc,
ascii=False,
total=len(new_media)):
post = self.__save_media(media, user)
if post is False: continue
if geolocations is True or tagged_users is True:
node = self.scraper._get_media_details(media['shortcode'])
self.__save_geotag(node, post)
self.__save_user_tags(node, post)
self.__save_top_comments(node, post)
self.__save_sponsors(node, post)
if media['comments_disabled']:
continue
num_comments = media['edge_media_to_comment']['count']
c_desc = f"Saving comments for post https://instagram.com/p/{media['shortcode']}/"
for comment in tqdm(self.scraper.query_comments_gen(
media['shortcode'], max_number=comment_depth),
total=num_comments,
desc=c_desc,
ascii=False):
self.__save_comment(comment, post)
if post_likes is False:
continue
for like in tqdm(self.scraper.query_likes_gen(media['shortcode']),
total=num_likes,
desc=l_desc,
ascii=False):
self.__save_like(like, post)
def __save_user_tags(self, node, media):
tagged_users = node.get('edge_media_to_tagged_user', {
'edges': None
}).get('edges')
if not tagged_users: return
for tagged_user in tagged_users:
with db.read_transaction:
user_tagged = User.match_username(
tagged_user['node']['user']['username'])
if not user_tagged:
user_tagged = self.__save_user_basic(
tagged_user['node']['user'])
with db.write_transaction:
media.tagged_users.connect(user_tagged, {
'x': tagged_user['node']['x'],
'y': tagged_user['node']['y']
})
def get_or_create_user(telegram_user: TelegramUser) -> User:
try:
user = User.objects.get(user_id=telegram_user.id)
except DoesNotExist:
user = User(
user_id=telegram_user.id,
username=telegram_user.username,
first_name=telegram_user.first_name,
last_name=telegram_user.last_name,
).save()
else:
user.username = telegram_user.username
user.first_name = telegram_user.first_name
user.last_name = telegram_user.last_name
user.save()
return user
def __save_top_comments(self, node, media):
if not node.get('edge_media_to_parent_comment', {
'edges': []
}).get('edges'):
return False
edges = node['edge_media_to_parent_comment']['edges']
for edge in edges:
edge_node = edge['node']
with db.read_transaction:
comment_exists = Comment.nodes.first_or_none(
comment_id=edge_node['id'])
if comment_exists:
if comment_exists.edge_liked_by_count is not None:
return
with db.write_transaction:
comment_exists.edge_liked_by = edge_node['edge_liked_by'][
'count']
comment_exists.edge_threaded_comments_count = edge_node[
'edge_threaded_comments']['count']
comment_exists.save()
else:
created_at = datetime.datetime.fromtimestamp(
edge_node['created_at'])
with db.write_transaction:
new_comment = Comment(
comment_id=edge_node['id'],
text=edge_node['text'],
created_at=created_at,
edge_liked_by_count=edge_node['edge_liked_by']
['count'],
edge_threaded_comments_count=edge_node[
'edge_threaded_comments']['count']).save()
with db.read_transaction:
comment_owner = User.match_username(
edge_node['owner']['username'])
if not comment_owner:
comment_owner = self.__save_user_basic(edge_node['owner'])
with db.write_transaction:
new_comment.owner.connect(comment_owner,
{'created_at': created_at})
media.comments.connect(new_comment)
def _get_following(self, username):
with db.read_transaction:
user = User.match_username(username) or Business.match_username(
username)
if user:
user_id = user.user_id
else:
with self.lock():
user = self._scrape_user(username, returns="user_id")
user_id = user.user_id
self.log.info(f"Getting {username}'s following")
self._api_busy = True
following = []
next_max_id = True
while next_max_id:
if next_max_id is True: next_max_id = ''
_ = self.api.getUserFollowings(user_id, maxid=next_max_id)
result = self.api.LastJson.get('users', [])
following.extend(result)
self._add_following(user, result)
next_max_id = self.api.LastJson.get('next_max_id', '')
self._api_busy = False
def __save_user_basic(node):
if node.get('id'):
existing_user = User.nodes.first_or_none(user_id=node['id'])
elif node.get('username'):
existing_user = User.nodes.first_or_none(username=node['username'])
else:
return False
if existing_user: return existing_user
user_kwargs = {
'user_id': node.get('id'),
'username': node.get('username'),
'full_name': node.get('full_name'),
'is_verified': node.get('is_verified'),
'is_private': node.get('is_private')
}
with db.write_transaction:
new_user = User(**user_kwargs).save()
if node.get('profile_pic_url'):
new_pp = ProfilePicture(
profile_pic_url=node['profile_pic_url']).save()
new_user.profile_pic.connect(new_pp)
return new_user
def _scrape_user(self, username, returns=False):
"""
Task must not be added to queue or consumer thread must be
locked to prevent wrong data stored in self.scraper.get result
`Required`
:param str username Instagram username
`Optional`
:param bool returns User graphql attribute to return
"""
self.log.info(f"Scraping user {username}")
user_info = self.__get_user_info(username)
if not user_info:
self.log.info(f"No user data found for @{username}")
return
with db.read_transaction:
user = User.match_username(username) or Business.match_username(
username)
if not user:
# Create new user
user_kwargs = {
'user_id':
user_info['id'],
'full_name':
user_info['full_name'],
'username':
username,
'bio':
user_info['biography'],
'external_url':
user_info['external_url'],
'is_private':
user_info['is_private'],
'connected_fb_page':
user_info['connected_fb_page'],
'last_scraped_timestamp':
datetime.datetime.utcnow(),
'edge_following_count':
user_info['edge_follow']['count'],
'edge_followers_count':
user_info['edge_followed_by']['count'],
'has_channel':
user_info['has_channel'],
'country_block':
user_info['country_block'],
'joined_recently':
user_info['is_joined_recently'],
'edge_timeline_media_count':
user_info['edge_owner_to_timeline_media']['count']
}
if user_info['country_block'] is True:
self.log.warning(f"@{username} has country block!")
if user_info['is_business_account'] is True:
user_kwargs['business_category_name'] = user_info[
'business_category_name']
with db.write_transaction:
user = Business(**user_kwargs).save()
else:
with db.write_transaction:
user = User(**user_kwargs).save()
else:
if user.last_scraped_timestamp:
difference = datetime.datetime.now(
datetime.timezone.utc) - user.last_scraped_timestamp
if difference.days < 7:
self.log.warning(
f"User {username} already scraped in last 7 days... Skipping"
)
return
# Update user records
user.bio = user_info['biography']
user.external_url = user_info['external_url']
user.connected_fb_page = user_info['connected_fb_page']
user.last_scraped_timestamp = datetime.datetime.utcnow()
user.edge_following_count = user_info['edge_follow']['count']
user.edge_followers_count = user_info['edge_followed_by']['count']
user.has_channel = user_info['has_channel']
user.country_block = user_info['country_block']
user.joined_recently = user_info['is_joined_recently']
if user_info['is_business_account']:
user.business_category_name = user_info[
'business_category_name']
with db.write_transaction:
user.save()
# Add profile picture if URL isn't saved
with db.read_transaction:
profile_pic_found = user.profile_pic.search(
profile_pic_url=user_info['profile_pic_url'])
if not profile_pic_found:
with db.write_transaction:
profile_picture = ProfilePicture(
profile_pic_url=user_info['profile_pic_url'],
profile_pic_url_hd=user_info['profile_pic_url_hd']).save()
user.profile_pic.connect(profile_picture)
# Add graph media if not exists
for item in user_info['edge_owner_to_timeline_media']['edges']:
node = item['node']
media = self.__save_media(node, user)
if media is False: continue
self.__save_geotag(node, media)
if returns is True:
return user
def add_data_test(db):
# Add users
u1 = User(firstname="Fernando",
lastname="Alonso",
email="*****@*****.**")
u2 = User(firstname="Kimi",
lastname="Raikkonen",
email="*****@*****.**")
db.add(u1)
db.add(u2)
# Add tags
t1 = Tag(label='JavaScript')
t2 = Tag(label='ReactJS')
t3 = Tag(label='Weekend')
t4 = Tag(label='Setup')
t5 = Tag(label='Docker')
db.add(t1)
db.add(t2)
db.add(t3)
db.add(t4)
db.add(t5)
# Add answers
a1 = Answer(
body=
'Mix Answer has been setup to work with Docker. You just need to install Docker on your machine and follow the instruction, it\'s as easy as that.',
user=u2,
vote_counter=2)
a2 = Answer(body='I confirm, just follow the instructions, it\'s so easy.',
user=u2)
a3 = Answer(
body=
'There are a lot of popular frameworks today so it\'s difficult to choose. I can recommend you ReactJS, this is the tool used for Mix Answer',
user=u1)
db.add(a1)
db.add(a2)
db.add(a3)
db.flush()
# Add questions
q1 = Question(
title='How do you setup Mix Answer?',
body='I\'m trying to install Mix Answer, what tool do I need?',
user_id=u1.id,
answers=[a1, a2],
tags=[t4, t5],
vote_counter=1)
q2 = Question(
title='What is the best front-end technology?',
body=
'I\'d like to create a website but I don\'t know which framework to use',
user_id=u1.id,
answers=[a3],
tags=[t1, t4, t5],
vote_counter=1)
q3 = Question(title='What did you plan for this weekend?',
body='What are you planning to do this weekend?',
user_id=u2.id,
tags=[t3])
db.add(q1)
db.add(q2)
db.add(q3)
db.flush()
# Add votes
v1 = VoteAnswer(answer_id=a1.id, user_id=u1.id)
v2 = VoteAnswer(answer_id=a1.id, user_id=u2.id)
v3 = VoteQuestion(question_id=q1.id, user_id=u1.id)
v4 = VoteQuestion(question_id=q2.id, user_id=u2.id)
db.add(v1)
db.add(v2)
db.add(v3)
db.add(v4)
# Commit data
db.commit()
def get_by_token(session, token):
return User.get_by_token(session=session, token=token)
def is_authenticated(session, token):
user = User.get_by_token(session, token)
return user is not None, user