def add_youtube_feeds(cls):
channels = Channel.all().fetch(100)
for channel in channels:
for keyword in channel.keywords:
yt_service = gdata.youtube.service.YouTubeService()
gdata.alt.appengine.run_on_appengine(yt_service)
uri = Programming.YOUTUBE_FEED % ('most_popular', keyword)
feed = yt_service.GetRecentlyFeaturedVideoFeed()
medias = Media.add_from_entry(feed.entry)
for media in medias:
Program.add_program(channel, media)
def add_program(self, channel, media, time):
programming = memcache.get('programming') or {}
program = Program.add_program(channel, media, time=time, min_time=time, max_time=(time + datetime.timedelta(media.duration + 50)))
logging.info(program)
if program:
if not programming.get(channel.id, None):
programming[channel.id] = []
programming.get(channel.id).append(program.toJson(fetch_channel=False, fetch_media=True, media_desc=False, pub_desc=False))
memcache.set('programming', programming)
def set_user_channel_programs(self, key, channel, medias, time=None, reset=False):
from model import Program
programs = []
programs_json = []
channel_json = {}
if not reset and memcache.get(channel.id):
channel_json = memcache.get(channel.id) or {}
programs_json = channel_json.get('programs', [])
else:
channel_json['channel'] = channel.toJson()
next_time = time or datetime.datetime.now()
if len(programs_json):
next_time = iso8601.parse_date(programs_json[-1]['time']).replace(tzinfo=None) + \
datetime.timedelta(seconds=programs_json[-1]['media']['duration'])
for media in medias:
program = Program.add_program(channel, media, time=next_time)
if program:
programs.append(program)
programs_json.append(program.toJson(fetch_channel=False, fetch_media=True, media_desc=False, pub_desc=False))
next_time = next_time + datetime.timedelta(seconds=media.duration)
if len(pickle.dumps(programs_json)) > 1000000:
# We can only fit 1mb into memcache
break
deferred.defer(Programming.fetch_related_tweets, medias,
_name='twitter-' + channel.id + '-' + str(uuid.uuid1()),
_queue='twitter')
user_obj = memcache.get(key) or {}
user_channels = (user_obj.get('channels') or []) if user_obj else []
if not channel.id in user_channels:
user_channels.append(channel.id)
user_obj['channels'] = user_channels
memcache.set(key, user_obj)
channel_json['programs'] = programs_json
memcache.set(channel.id, channel_json)
return programs
def set_programming(cls, channel_id, duration=3600, schedule_next=False, fetch_twitter=True,
queue='programming', target=None, kickoff=False):
import broadcast
import constants
from model import Channel
from model import Program
# Stored programming
programming = memcache.get('programming') or {}
onlineUsers = memcache.get('web_channels') or {}
logging.info('programming: ' + channel_id)
next_programs = Programming.next_programs(programming.get(channel_id, []), duration, prelude=300)
gap = Programming.gap(programming.get(channel_id, []), duration)
logging.info('GAP: ' + str(gap))
if programming.get(channel_id) and len(programming[channel_id]) and \
programming[channel_id][0]['media'].get('live') == True:
logging.info('live tweets')
# Update tweets for live events
media = Media.get_by_key_name(programming[channel_id][0]['media']['id'])
deferred.defer(Programming.fetch_related_tweets, [],
_name='twitter-' + channel_id + '-' + str(uuid.uuid1()),
_queue='twitter')
programs = []
if not programming.get(channel_id) or gap > 60:
logging.info('PROGRAOMMING')
channel = Channel.get_by_key_name(channel_id)
#channel.update_next_time()
viewers = (memcache.get('channel_viewers') or {}).get(str(channel_id), [])
cols = channel.get_collections()
all_medias = []
backup_medias = []
limit = 100
for col in cols:
medias = []
filtered_medias = []
offset = 0
while offset <= 400:
medias = col.get_medias(limit=limit, offset=offset)
logging.info('fetched medias: ' + str(len(medias)))
if not len(medias):
break
backup_medias += medias
# Dont reprogram anytihng already scheduled
filtered_medias = Programming.no_reprogram(programming.get(channel_id, []), medias)
# Don't repeat the same program within two hour
cutoff = 7200 if col.lifespan else 43200
filtered_medias = [c for c in filtered_medias if not c.last_programmed or
(datetime.datetime.now() - c.last_programmed).seconds > cutoff]
# At most, 30% of the audience has already "witnessed" this program
# filtered_medias = [m for m in filtered_medias if not len(viewers) or
# float(len(Programming.have_seen(m, viewers)))/len(viewers) < .3]
all_medias += filtered_medias
logging.info('all medias: ' + str(len(all_medias)))
offset += limit
all_medias = backup_medias if not len(all_medias) else all_medias
# Don't repeat already programmed
# all_medias = Programming.no_reprogram(next_programs, all_medias)
# StorySort algorithm
# all_medias = Programming.story_sort(all_medias)
# Only one publisher per story
all_medias = Programming.unique_publishers(all_medias)
# Grab "duration" seconds of programming
all_medias = Programming.timed_subset(all_medias, duration)
if fetch_twitter:
# Find related twitter posts
deferred.defer(Programming.fetch_related_tweets, all_medias,
_name='twitter-' + channel.name.replace(' ', '') + '-' + str(uuid.uuid1()),
_queue='twitter',
_countdown=30)
# Truncate old programs
programming[channel_id] = Programming.cutoff_programs(programming.get(channel_id), 300)
for media in all_medias:
program = Program.add_program(channel, media, min_time=datetime.datetime.now(),
max_time=(datetime.datetime.now() + datetime.timedelta(seconds=duration)))
logging.info(program)
if program:
if not programming.get(channel_id, None):
programming[channel_id] = []
programming.get(channel_id).append(program.toJson(fetch_channel=False, fetch_media=True, media_desc=False, pub_desc=False))
programs.append(program)
logging.info('ADDING: ' + media.name + ' at: ' + program.time.isoformat())
if len(pickle.dumps(programming)) > 1000000:
# We can only fit 1mb into memcache
break
if len(programs):
broadcast.broadcastNewPrograms(channel, programs)
memcache.set('programming', programming)
channels = memcache.get('channels') or []
updated = False
for i,c in enumerate(channels):
if c['id'] == channel_id:
channels[i] = channel.toJson(get_programming=False)
updated = True
if not updated:
channels.append(channel.toJson(get_programming=False))
memcache.set('channels', channels)
# Schedule our next programming selection
if schedule_next and (not constants.SLEEP_PROGRAMMING or
(constants.SLEEP_PROGRAMMING and (kickoff or len(onlineUsers.keys())))):
logging.info('NUMBER OF PROGRAMS: ' + str(len(programs)))
if len(programs) > 1:
next_gen = (programs[-2].time - datetime.datetime.now()).seconds / 2
elif len(programs) == 1:
next_gen = programs[0].media.duration / 2
else:
next_gen = 60
next_gen = min(next_gen,
reduce(lambda x, y: x + y, [p.media.duration for p in programs], 0) \
if len(programs) else 10)
next_gen = min(next_gen, duration / 2)
logging.info('COUNTDOWN FOR ' + channel_id + ': ' + str(next_gen))
deferred.defer(Programming.set_programming, channel_id, fetch_twitter=fetch_twitter,
_name=channel_id + '-' + str(uuid.uuid1()),
_countdown=next_gen,
_queue=queue)
return programs
def schedule_youtube_channel(name, user_id=None, token=None, channel_id=None, yt_channel_id=None, yt_playlist_id=None):
import broadcast
import constants
import datetime
import programming
from model import Media
from model import Channel
from model import User
from model import common
from model import Program
user = None
if user_id:
user = User.get_by_key_name(user_id)
channel = Channel(key_name=(user_id or token) + '-' + (yt_channel_id or yt_playlist_id),
name=name, privacy=constants.Privacy.FRIENDS, online=True, user=user)
medias = []
youtube3 = common.get_youtube3_service()
next_page_token = ''
search_response = {}
if yt_channel_id:
if yt_channel_id.startswith('HC'):
channel_response = youtube3.channels().list(
id=yt_channel_id,
part='topicDetails',
maxResults=1
).execute()
if len(channel_response.get('items', [])):
topic_id = channel_response.get('items')[0]['topicDetails']['topicIds'][0]
search_response = youtube3.search().list(
topicId=topic_id,
order='date',
part='id,snippet',
maxResults=10,
fields='items',
type='video'
).execute()
else:
search_response = youtube3.search().list(
channelId=yt_channel_id,
part='id,snippet',
order='date',
maxResults=10,
type='video'
).execute()
elif yt_playlist_id:
search_response = youtube3.playlistItems().list(
playlistId=yt_playlist_id,
part='id,snippet',
maxResults=10,
fields='items'
).execute()
search_ids = ''
for item in search_response.get('items', [])[1:]:
if item['kind'] == 'youtube#searchResult':
search_ids += item['id']['videoId'] + ','
elif item['kind'] == 'youtube#playlistItem':
search_ids += item['snippet']['resourceId']['videoId'] + ','
videos_response = youtube3.videos().list(
id=search_ids,
part="id,snippet,topicDetails,contentDetails,statistics"
).execute()
for item in videos_response.get("items", []):
medias = Media.add_from_snippet([item], approve=True)
programs = []
if user_id:
programs = programming.Programming.set_user_channel_programs(user_id, channel, medias)
else:
next_time = datetime.datetime.now()
for media in medias:
program = Program.add_program(channel, media, time=next_time)
if program:
programs.append(program)
next_time = next_time + datetime.timedelta(seconds=media.duration)
broadcast.broadcastNewPrograms(channel, programs, token=token)