def __init__(self,
credentials,
hidden=False,
category_id=23,
language="en"):
self.logger = logging.getLogger(type(self).__name__)
self.client = GoogleAPIClient(
credentials['client_id'],
credentials['client_secret'],
credentials['refresh_token'],
)
self.hidden = hidden
self.category_id = category_id
self.language = language
def __init__(self, credentials, hidden=False, category_id=23, language="en", use_yt_recommended_encoding=False, mime_type='video/MP2T'): self.logger = logging.getLogger(type(self).__name__) self.client = GoogleAPIClient( credentials['client_id'], credentials['client_secret'], credentials['refresh_token'], ) self.hidden = hidden self.category_id = category_id self.language = language self.mime_type = mime_type if use_yt_recommended_encoding: self.encoding_settings = self.recommended_settings self.encoding_streamable = False
def main(
dbconnect,
creds_file,
playlists,
upload_location_allowlist="youtube",
interval=600,
metrics_port=8007,
backdoor_port=0,
):
"""
dbconnect should be a postgres connection string
creds_file should contain youtube api creds
upload_location_allowlist is a comma-seperated list of database upload locations to
consider as eligible to being added to playlists. For these locations, the database video id
must be a youtube video id.
interval is how often to check for new videos, default every 10min.
"""
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
upload_locations = upload_location_allowlist.split(
",") if upload_location_allowlist else []
playlists = dict(playlists)
stop = gevent.event.Event()
gevent.signal_handler(signal.SIGTERM, stop.set) # shut down on sigterm
logging.info("Starting up")
with open(creds_file) as f:
creds = json.load(f)
client = GoogleAPIClient(creds['client_id'], creds['client_secret'],
creds['refresh_token'])
dbmanager = DBManager(dsn=dbconnect)
manager = PlaylistManager(dbmanager, client, upload_locations, playlists)
while not stop.is_set():
try:
manager.run_once()
except Exception:
logging.exception("Failed to run playlist manager")
manager.reset()
stop.wait(interval) # wait for interval, or until stopping
logging.info("Stopped")
def main(*targets):
"""Does an action on the given google api targets, preventing issues due to api inactivity.
A target should consist of a comma-seperated list of apis to hit, then a colon, then a creds file.
eg. "sheets,youtube:my_creds.json".
"""
for target in targets:
if ':' not in target:
raise ValueError("Bad target: {!r}".format(target))
apis, credfile = target.split(':', 1)
apis = apis.split(',')
with open(credfile) as f:
creds = json.load(f)
client = GoogleAPIClient(creds['client_id'], creds['client_secret'],
creds['refresh_token'])
for api in apis:
if api not in ACTIONS:
raise ValueError("No such api {!r}".format(api))
ACTIONS[api](client)
class Youtube(UploadBackend):
"""Represents a youtube channel to upload to, and settings for doing so.
Config args besides credentials:
hidden:
If false, video is public. If true, video is unlisted. Default false.
category_id:
The numeric category id to set as the youtube category of all videos.
Default is 23, which is the id for "Comedy". Set to null to not set.
language:
The language code to describe all videos as.
Default is "en", ie. English. Set to null to not set.
use_yt_recommended_encoding:
Default False. If True, use the ffmpeg settings that Youtube recommends for
fast processing once uploaded. We suggest not bothering, as it doesn't appear
to make much difference.
mime_type: You must set this to the correct mime type for the encoded video.
Default is video/MP2T, suitable for fast cuts or -f mpegts.
"""
needs_transcode = True
recommended_settings = [
# Youtube's recommended settings:
'-codec:v',
'libx264', # Make the video codec x264
'-crf',
'21', # Set the video quality, this produces the bitrate range that YT likes
'-bf',
'2', # Have 2 consecutive bframes, as requested
'-flags',
'+cgop', # Use closed GOP, as requested
'-pix_fmt',
'yuv420p', # chroma subsampling 4:2:0, as requrested
'-codec:a',
'aac',
'-strict',
'-2', # audio codec aac, as requested
'-b:a',
'384k' # audio bitrate at 348k for 2 channel, use 512k if 5.1 audio
'-r:a',
'48000', # set audio sample rate at 48000Hz, as requested
'-movflags',
'faststart', # put MOOV atom at the front of the file, as requested
]
def __init__(self,
credentials,
hidden=False,
category_id=23,
language="en",
use_yt_recommended_encoding=False,
mime_type='video/MP2T'):
self.logger = logging.getLogger(type(self).__name__)
self.client = GoogleAPIClient(
credentials['client_id'],
credentials['client_secret'],
credentials['refresh_token'],
)
self.hidden = hidden
self.category_id = category_id
self.language = language
self.mime_type = mime_type
if use_yt_recommended_encoding:
self.encoding_settings = self.recommended_settings
self.encoding_streamable = False
def upload_video(self, title, description, tags, data):
json = {
'snippet': {
'title': title,
'description': description,
'tags': tags,
},
}
if self.category_id is not None:
json['snippet']['categoryId'] = self.category_id
if self.language is not None:
json['snippet']['defaultLanguage'] = self.language
json['snippet']['defaultAudioLanguage'] = self.language
if self.hidden:
json['status'] = {
'privacyStatus': 'unlisted',
}
resp = self.client.request(
'POST',
'https://www.googleapis.com/upload/youtube/v3/videos',
headers={'X-Upload-Content-Type': self.mime_type},
params={
'part': 'snippet,status' if self.hidden else 'snippet',
'uploadType': 'resumable',
},
json=json,
metric_name='create_video',
)
if not resp.ok:
# Don't retry, because failed calls still count against our upload quota.
# The risk of repeated failed attempts blowing through our quota is too high.
raise UploadError(
"Youtube create video call failed with {resp.status_code}: {resp.content}"
.format(resp=resp))
upload_url = resp.headers['Location']
resp = self.client.request(
'POST',
upload_url,
data=data,
metric_name='upload_video',
)
if 400 <= resp.status_code < 500:
# As above, don't retry. But with 4xx's we know the upload didn't go through.
# On a 5xx, we can't be sure (the server is in an unspecified state).
raise UploadError(
"Youtube video data upload failed with {resp.status_code}: {resp.content}"
.format(resp=resp))
resp.raise_for_status()
id = resp.json()['id']
return id, 'https://youtu.be/{}'.format(id)
def check_status(self, ids):
output = []
# Break up into groups of 10 videos. I'm not sure what the limit is so this is reasonable.
for i in range(0, len(ids), 10):
group = ids[i:i + 10]
resp = self.client.request(
'GET',
'https://www.googleapis.com/youtube/v3/videos',
params={
'part': 'id,status',
'id': ','.join(group),
},
metric_name='list_videos',
)
resp.raise_for_status()
for item in resp.json()['items']:
if item['status']['uploadStatus'] == 'processed':
output.append(item['id'])
return output
def __init__(self, client_id, client_secret, refresh_token):
self.logger = logging.getLogger(type(self).__name__)
self.client = GoogleAPIClient(client_id, client_secret, refresh_token)
class Sheets(object):
"""Manages Google Sheets API operations"""
def __init__(self, client_id, client_secret, refresh_token):
self.logger = logging.getLogger(type(self).__name__)
self.client = GoogleAPIClient(client_id, client_secret, refresh_token)
def get_rows(self, spreadsheet_id, sheet_name, range=None):
"""Return a list of rows, where each row is a list of the values of each column.
Range optionally restricts returned rows, and uses A1 format, eg. "A1:B5".
"""
if range:
range = "'{}'!{}".format(sheet_name, range)
else:
range = "'{}'".format(sheet_name)
resp = self.client.request(
'GET',
'https://sheets.googleapis.com/v4/spreadsheets/{}/values/{}'.
format(
spreadsheet_id,
range,
),
metric_name='get_rows',
)
resp.raise_for_status()
data = resp.json()
return data['values']
def write_value(self, spreadsheet_id, sheet_name, row, column, value):
"""Write value to the row and column (0-based) given."""
range = "'{sheet}'!{col}{row}:{col}{row}".format(
sheet=sheet_name,
row=row + 1, # 1-indexed rows in range syntax
col=self.index_to_column(column),
)
resp = self.client.request(
'PUT',
'https://sheets.googleapis.com/v4/spreadsheets/{}/values/{}'.
format(
spreadsheet_id,
range,
),
params={
"valueInputOption": "1", # RAW
},
json={
"range": range,
"values": [[value]],
},
metric_name='write_value',
)
resp.raise_for_status()
def index_to_column(self, index):
"""For a given column index, convert to a column description, eg. 0 -> A, 1 -> B, 26 -> AA."""
# This is equivalent to the 0-based index in base-26 (where A = 0, B = 1, ..., Z = 25)
digits = []
while index:
index, digit = divmod(index, 26)
digits.append(digit)
# We now have the digits, but they're backwards.
digits = digits[::-1]
# Now convert the digits to letters
digits = [chr(ord('A') + digit) for digit in digits]
# Finally, convert to string
return ''.join(digits)