def update_user_video():
helper_firestore.initialize_firebase()
ref = firestore.client().collection('users')
query = ref \
.where('region', '==', 'JP') \
.order_by('follower_count', direction=firestore.Query.DESCENDING)
docs = query.get()
user_list = [doc.to_dict() for doc in docs]
video_list = []
for user in user_list:
if user.get('region') != 'JP':
continue
result = get_user_video(user['uid'])
for aweme in result.get('aweme_list', []):
video = create_video_data(aweme)
if video:
video_list.append(video)
print(video['nickname'])
if video_list:
helper_firestore.batch_update(collection='videos',
data_list=video_list,
unique_key='aweme_id')
print("SUCCESS: tiktok, update_user_video")
def update_users():
filter_time = datetime.now() - timedelta(weeks=1)
helper_firestore.initialize_firebase()
ref = firestore.client().collection('users')
query = ref \
.where('update_at', '<', filter_time) \
.order_by('update_at') \
.order_by('follower_count', direction=firestore.Query.DESCENDING) \
.limit(100)
docs = query.get()
user_list = [doc.to_dict() for doc in docs]
for user in user_list:
uid = user.get('uid')
if not uid:
continue
data = get_user_detail(uid)
if not data or not data.get('uid'):
continue
result = create_user_data(data)
if not result:
continue
user.update(result)
if user_list:
helper_firestore.batch_update(collection='users',
data_list=user_list,
unique_key='uid')
print("SUCCESS: tiktok, update_users")
def trace_hashtag():
helper_firestore.initialize_firebase()
ref = firestore.client().collection('hashtags')
filter_time = datetime.now().replace(
hour=0, minute=0, second=0, microsecond=0) - timedelta(hours=9)
query = ref \
.where('update_at', '<', filter_time) \
.order_by('update_at') \
.order_by('user_count', direction=firestore.Query.DESCENDING) \
.limit(100)
docs = query.get()
hashtag_list = [doc.to_dict() for doc in docs]
results = []
for hashtag in hashtag_list:
params = deepcopy(user_params)
params['ch_id'] = hashtag['cid']
res = requests.get(BASE_URL + "/challenge/detail/",
headers=feed_headers,
params=params)
data = res.json()
challenge = data.get('ch_info')
result = create_hashtag_data(challenge)
if not result:
continue
hashtag.update(result)
results.append(hashtag)
if results:
helper_firestore.batch_update(collection='hashtags',
data_list=results,
unique_key='cid')
print("SUCCESS: tiktok, trace_hashtag")
def tweet_tiktok():
account = 'tiktok'
try:
api = get_twitter_api(account)
except Exception:
return
redis_key = 'tweet_tiktok'
r = redis.from_url(REDIS_URL)
rcache = r.get(redis_key)
start_follower = None
if rcache:
print("cache HIT!! %s" % (redis_key))
start_follower = json.loads(rcache.decode())
helper_firestore.initialize_firebase()
ref = firestore.client().collection('users')
query = ref.order_by('follower_count', direction=firestore.Query.DESCENDING)
if start_follower:
query = query.start_after({'follower_count': start_follower})
docs = query.limit(1).get()
data = list(docs)[0].to_dict()
pprint(data)
content_list = []
if data.get('custom_verify'):
content_list.append('【%s】' % (data['custom_verify']))
content_list.append(data.get('nickname', ''))
_follower = data['follower_count']
follower = round(_follower, -len(str(_follower)) + 2)
follower = "{:,d}".format(follower)
content_list.append('ファン： %s人' % (follower))
if data.get('signature'):
content_list.append('\n' + str(data['signature']) + '\n')
# if data.get('twitter_name'):
# content_list.append('【Twitter】@%s' % (data['twitter_name']))
content_list.append('【TikTok】%s' % (data['share_url'].replace('/?', '')))
image_url_list = []
if data.get('avatar_medium'):
image_url_list.append(data['avatar_medium'])
media_ids = []
for image_url in image_url_list:
try:
media = urllib.request.urlopen(image_url).read()
except Exception as e:
print(image_url)
pprint(e)
continue
response = api.upload_media(media)
if response.get('errors'):
pprint(response)
return
media_ids.append(response['media_id_string'])
if len(media_ids) >= 4:
break
status = '\n'.join(content_list)
print(status)
print(len(status))
# twitter_tool.post_tweet(
# username=api.username,
# password=api.password,
# status=status,
# image_url_list=image_url_list,
# )
response = api.post_tweet(status=status, media_ids=media_ids)
if response.get('errors'):
pprint(response)
start_follower = data['follower_count']
r.set(redis_key, json.dumps(start_follower), ex=60 * 60 * 24 * 30)
print("SUCCESS: tweet_tiktok", account)
def tweet_tiktok_video():
account = 'tiktok'
try:
api = get_twitter_api(account)
except Exception:
return
redis_key = 'tweet_tiktok_video'
r = redis.from_url(REDIS_URL)
rcache = r.get(redis_key)
id_list = []
if rcache:
print("cache HIT!! %s" % (redis_key))
id_list = json.loads(rcache.decode())
# 1週間前
# filter_time = int(time()) - (60 * 60 * 24 * 7)
helper_firestore.initialize_firebase()
ref = firestore.client().collection('videos')
# .where('create_time', '>', filter_time) \
# .order_by('create_time') \
last_data = None
target = None
tmp = None
for _ in range(50):
query = ref \
.order_by('create_time') \
.order_by('digg_count', direction=firestore.Query.DESCENDING)
if last_data:
query = query.start_after({
'create_time': last_data.get('create_time'),
'digg_count': last_data.get('digg_count'),
})
docs = query.limit(10).get()
results = [doc.to_dict() for doc in docs]
for data in results:
if not tmp:
tmp = data
if data.get('aweme_id') in id_list:
continue
target = data
break
if target:
break
last_data = results[-1]
if not target:
target = tmp
id_list = []
media_id = upload_video(api, video_url=target['download_url'])
if not media_id:
return
content_list = []
content_list.append(target.get('nickname'))
content_list.append(target.get('desc', ''))
content_list.append('\n【詳細URL】%s' % (target.get('share_url')))
status = '\n'.join(content_list)
response = api.post_tweet(status=status, media_ids=[media_id])
if response.get('errors'):
pprint(response)
id_list.append(target.get('aweme_id'))
r.set(redis_key, json.dumps(id_list), ex=60 * 60 * 24 * 30)
print("SUCCESS: tweet_tiktok_video", account)
def update_spread_sheet():
helper_firestore.initialize_firebase()
ref = firestore.client().collection('hashtags')
query = ref \
.order_by('view_count', direction=firestore.Query.DESCENDING) \
.limit(20)
docs = query.get()
label_list = []
hashtag_list = []
for doc in docs:
hashtag = doc.to_dict()
hashtag['ハッシュタグ'] = hashtag['cha_name']
for _period, stat in hashtag['stats'].items():
period = _period.replace('_', '/')
if period not in label_list:
label_list.append(period)
hashtag[period] = stat['view_count']
hashtag_list.append(hashtag)
label_list = sorted(label_list)
label_list.insert(0, 'ハッシュタグ')
body = {
'values': gspread.convert_to_sheet_values(label_list, hashtag_list),
'majorDimension': 'COLUMNS',
}
gspread.update_sheet_values(SHEET_ID,
'視聴回数合計',
body,
valueInputOption='RAW')
del label_list[0]
for hashtag in hashtag_list:
prev_count = 0
for period in label_list:
count = hashtag.get(period)
if not count:
continue
if prev_count == 0:
hashtag[period] = ''
else:
hashtag[period] = count - prev_count
prev_count = count
del label_list[0]
label_list.insert(0, 'ハッシュタグ')
body = {
'values': gspread.convert_to_sheet_values(label_list, hashtag_list),
'majorDimension': 'COLUMNS',
}
gspread.update_sheet_values(SHEET_ID,
'視聴回数（日別）',
body,
valueInputOption='RAW')
print('SUCCESS: update_spread_sheet')