def load_avatars(filename):
df = pd.read_excel(filename)
avatar_list = list(df['Name'])
for url in avatar_list:
avatar = Avatar(url=url)
db.session.add(avatar)
db.session.commit()
def scrape_avatar(id_, site, url):
"""
Get an twitter avatar from ``url`` and save it to the Profile identified by
``id_``.
"""
worker.start_job()
redis = worker.get_redis()
db_session = worker.get_session()
avatar = None
profile = db_session.query(Profile).filter(Profile.id == id_).first()
if profile is None:
raise ValueError('No profile exists with id={}'.format(id_))
if site == 'twitter':
# Twitter points you to a scaled image by default, but we can
# get the original resolution by removing "_normal" from the URL.
#
# See: https://dev.twitter.com/overview/general/user-profile-images-and-banners
url = url.replace('_normal', '')
# Update Avatar if it's already stored in the db
for profile_avatar in profile.avatars:
if profile_avatar.upstream_url == url:
profile_avatar.end_date = datetime.today()
avatar = profile_avatar
break
# Otherwise, scrape the new Avatar and append to the profile
if avatar is None:
response = requests.get(url)
response.raise_for_status()
if 'content-type' in response.headers:
mime = response.headers['content-type']
else:
mime = 'application/octet-stream'
image = response.content
avatar = Avatar(url, mime, image)
profile.avatars.append(avatar)
profile.current_avatar = avatar
db_session.commit()
worker.finish_job()
redis.publish('avatar', json.dumps({
'id': id_,
'thumb_url': '/api/file/' + str(avatar.thumb_file.id),
'url': '/api/file/' + str(avatar.file.id),
}))
def load_avatars(avatars_filename):
"""Load avatars."""
print "Avatar"
for i, row in enumerate(open(avatars_filename)):
row = row.rstrip()
avatar_src = row
avatar = Avatar(avatar_src=avatar_src)
db.session.add(avatar)
if i % 10 == 0:
print i
db.session.commit()
def _create_sample_profiles(self, config):
''' Create some sample profiles. '''
session = app.database.get_session(self._db)
sample_dir = os.path.join(os.path.dirname(__file__), 'sample-data')
# Maurice Moss
moss_twitter = Profile(site='twitter',
upstream_id='12345',
username=ProfileUsername(
'maurice.moss', start_date='2014-04-01'))
moss_twitter.usernames.append(
ProfileUsername('maurice',
start_date='2013-06-01',
end_date='2014-03-31'))
moss_twitter.usernames.append(
ProfileUsername('maurie',
start_date='2013-02-15',
end_date='2013-05-30'))
Post(author=moss_twitter,
content='Going to the grocery store.',
upstream_id='1234',
upstream_created='2015-02-04 12:34:50')
post = Post(author=moss_twitter,
content='Love this band!.',
upstream_id='2345',
upstream_created='2015-03-01')
post.attachments.append(
File(name='helloworld.txt',
mime='text/plain',
content='Hello world!\n\n'.encode('utf8')))
moss_twitter.posts.append(post)
with open(os.path.join(sample_dir, 'moss.jpg'), 'rb') as moss_jpg:
moss_twitter.avatars.append(
Avatar(url='http://foobar.com/moss-avatar.jpg',
mime='image/jpeg',
image=moss_jpg.read()))
moss_twitter.description = "I do IT at Reynholm Industries."
moss_twitter.post_count = 1205
moss_twitter.friend_count = 1
moss_twitter.follower_count = 3
moss_twitter.join_date = dateutil.parser.parse('2013-06-01')
moss_twitter.join_date_is_exact = False
session.add(moss_twitter)
# Jen Barber
jen_twitter = Profile(site='twitter',
upstream_id='23456',
username=ProfileUsername(
'jen.barber', start_date='2013-11-12'))
jen_twitter.usernames.append(
ProfileUsername('jenb',
start_date='2013-06-14',
end_date='2013-11-12'))
jen_twitter.usernames.append(
ProfileUsername('jenny',
start_date='2013-03-15',
end_date='2013-06-14'))
with open(os.path.join(sample_dir, 'jen.jpg'), 'rb') as jen_jpg:
jen_twitter.avatars.append(
Avatar(url='http://foobar.com/jen-avatar.jpg',
mime='image/jpeg',
image=jen_jpg.read()))
jen_twitter.description = "Relationship Manager for the IT department."
jen_twitter.post_count = 1543
jen_twitter.friend_count = 1
jen_twitter.follower_count = 1
jen_twitter.join_date = dateutil.parser.parse('2013-03-15')
jen_twitter.join_date_is_exact = True
moss_twitter.followers.append(jen_twitter)
session.add(jen_twitter)
# A couple of randos.
moss_twitter.followers.append(
Profile(site='twitter', upstream_id='345678', username='******'))
moss_twitter.followers.append(
Profile(site='twitter', upstream_id='456789', username='******'))
jen_twitter.followers.append(
Profile(site='twitter', upstream_id='567890', username='******'))
session.commit()
