def main(dbconnect,
sheets_creds_file,
edit_url,
bustime_start,
sheet_id,
worksheet_names,
metrics_port=8005,
backdoor_port=0,
allocate_ids=False):
"""
Sheet sync constantly scans a Google Sheets sheet and a database, copying inputs from the sheet
to the DB and outputs from the DB to the sheet.
With the exception of id allocation, all operations are idempotent and multiple sheet syncs
may be run for redundancy.
"""
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
register_uuid()
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
stop = gevent.event.Event()
gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm
logging.info("Starting up")
dbmanager = DBManager(dsn=dbconnect)
while True:
try:
# Get a test connection so we know the database is up,
# this produces a clearer error in cases where there's a connection problem.
conn = dbmanager.get_conn()
except Exception:
delay = common.jitter(10)
logging.info(
'Cannot connect to database. Retrying in {:.0f} s'.format(
delay))
stop.wait(delay)
else:
# put it back so it gets reused on next get_conn()
dbmanager.put_conn(conn)
break
sheets_creds = json.load(open(sheets_creds_file))
sheets = Sheets(
client_id=sheets_creds['client_id'],
client_secret=sheets_creds['client_secret'],
refresh_token=sheets_creds['refresh_token'],
)
SheetSync(stop, dbmanager, sheets, sheet_id, worksheet_names, edit_url,
bustime_start, allocate_ids).run()
logging.info("Gracefully stopped")
def main(channels,
base_dir='.',
qualities='source',
first_hour=None,
last_hour=None,
metrics_port=8006,
backdoor_port=0):
"""Segment coverage service"""
qualities = qualities.split(',') if qualities else []
qualities = [quality.strip() for quality in qualities]
if first_hour is not None:
first_hour = dateutil.parse(first_hour)
if last_hour is not None:
last_hour = dateutil.parse(last_hour)
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
managers = []
workers = []
for channel in channels:
logging.info(
'Starting coverage checks {} with {} as qualities in {}'.format(
channel, ', '.join(qualities), base_dir))
manager = CoverageChecker(channel, qualities, base_dir, first_hour,
last_hour)
managers.append(manager)
workers.append(gevent.spawn(manager.run))
def stop():
for manager in managers:
manager.stop()
gevent.signal(signal.SIGTERM, stop)
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
# Wait for any to die
gevent.wait(workers, count=1)
# If one has stopped, either:
# 1. stop() was called and all are stopping
# 2. one errored and we should stop all remaining and report the error
# Our behaviour in both cases is the same:
# 1. Tell all managers to gracefully stop
stop()
# 2. Wait (with timeout) until they've stopped
gevent.wait(workers)
# 3. Check if any of them failed. If they did, report it. If mulitple
# failed, we report one arbitrarily.
for worker in workers:
worker.get()
logging.info('Gracefully stopped')
def main(
dbconnect,
creds_file,
playlists,
upload_location_allowlist="youtube",
interval=600,
metrics_port=8007,
backdoor_port=0,
):
"""
dbconnect should be a postgres connection string
creds_file should contain youtube api creds
upload_location_allowlist is a comma-seperated list of database upload locations to
consider as eligible to being added to playlists. For these locations, the database video id
must be a youtube video id.
interval is how often to check for new videos, default every 10min.
"""
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
upload_locations = upload_location_allowlist.split(
",") if upload_location_allowlist else []
playlists = dict(playlists)
stop = gevent.event.Event()
gevent.signal_handler(signal.SIGTERM, stop.set) # shut down on sigterm
logging.info("Starting up")
with open(creds_file) as f:
creds = json.load(f)
client = GoogleAPIClient(creds['client_id'], creds['client_secret'],
creds['refresh_token'])
dbmanager = DBManager(dsn=dbconnect)
manager = PlaylistManager(dbmanager, client, upload_locations, playlists)
while not stop.is_set():
try:
manager.run_once()
except Exception:
logging.exception("Failed to run playlist manager")
manager.reset()
stop.wait(interval) # wait for interval, or until stopping
logging.info("Stopped")
def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0):
app.static_folder = base_dir
server = WSGIServer((host, port), cors(app))
PromLogCountsHandler.install()
install_stacksampler()
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
serve_with_graceful_shutdown(server)
def main(channels,
base_dir='.',
qualities='source',
metrics_port=8002,
static_nodes='',
backdoor_port=0,
start=None,
delete_old=False,
run_once=False,
node_file=None,
node_database=None,
localhost=socket.gethostname(),
download_concurrency=5,
recent_cutoff=120):
"""Backfiller service."""
qualities = qualities.split(',') if qualities else []
qualities = [quality.strip() for quality in qualities]
static_nodes = static_nodes.split(',') if static_nodes else []
static_nodes = [static_node.strip() for static_node in static_nodes]
if start is not None:
try:
start = float(start)
logging.info('Backfilling last {} hours'.format(start))
except ValueError:
start = dateutil.parse(start)
logging.info('Backfilling since {}'.format(start))
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
logging.info('Starting backfilling {} with {} as qualities to {}'.format(
', '.join(channels), ', '.join(qualities), base_dir))
manager = BackfillerManager(base_dir, channels, qualities, static_nodes,
start, delete_old, run_once, node_file,
node_database, localhost, download_concurrency,
recent_cutoff)
def stop():
manager.stop()
gevent.signal(signal.SIGTERM, stop)
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
manager.run()
logging.info('Gracefully stopped')
def main(connection_string,
default_channel,
bustime_start,
host='0.0.0.0',
port=8004,
backdoor_port=0,
no_authentication=False,
title_header=None,
description_footer=None,
upload_locations=''):
"""Thrimshim service."""
server = WSGIServer((host, port), cors(app))
app.no_authentication = no_authentication
app.default_channel = default_channel
app.bustime_start = bustime_start
app.title_header = "" if title_header is None else "{} - ".format(
title_header)
app.description_footer = "" if description_footer is None else "\n\n{}".format(
description_footer)
app.upload_locations = upload_locations.split(
',') if upload_locations else []
stopping = gevent.event.Event()
def stop():
logging.info("Shutting down")
stopping.set()
# handle when the server is running
if hasattr(server, 'socket'):
server.stop()
# and when not
else:
sys.exit()
gevent.signal(signal.SIGTERM, stop)
app.db_manager = database.DBManager(dsn=connection_string)
common.PromLogCountsHandler.install()
common.install_stacksampler()
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
logging.info('Starting up')
if app.no_authentication:
logging.warning('Not authenticating POST requests')
server.serve_forever()
logging.info("Gracefully shut down")
def main(channels,
base_dir=".",
qualities="source",
metrics_port=8001,
backdoor_port=0):
qualities = qualities.split(",") if qualities else []
managers = [
StreamsManager(channel.rstrip('!'),
base_dir,
qualities,
important=channel.endswith('!')) for channel in channels
]
def stop():
for manager in managers:
manager.stop()
gevent.signal(signal.SIGTERM, stop) # shut down on sigterm
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
logging.info("Starting up")
workers = [gevent.spawn(manager.run) for manager in managers]
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
# Wait for any to die
gevent.wait(workers, count=1)
# If one has stopped, either:
# 1. stop() was called and all are stopping
# 2. one errored and we should stop all remaining and report the error
# Our behaviour in both cases is the same:
# 1. Tell all managers to gracefully stop
stop()
# 2. Wait (with timeout) until they've stopped
gevent.wait(workers)
# 3. Check if any of them failed. If they did, report it. If mulitple failed, we report
# one arbitrarily.
for worker in workers:
worker.get() # re-raise error if failed
logging.info("Gracefully stopped")
def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0):
app.static_folder = base_dir
server = WSGIServer((host, port), cors(app))
def stop():
logging.info("Shutting down")
server.stop()
gevent.signal(signal.SIGTERM, stop)
PromLogCountsHandler.install()
install_stacksampler()
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start()
logging.info("Starting up")
server.serve_forever()
logging.info("Gracefully shut down")
def main(
connection_string,
default_channel,
bustime_start,
host='0.0.0.0',
port=8004,
backdoor_port=0,
no_authentication=False,
title_header=None,
description_footer=None,
upload_locations='',
):
server = WSGIServer((host, port), cors(app))
app.no_authentication = no_authentication
app.default_channel = default_channel
app.bustime_start = bustime_start
app.title_header = "" if title_header is None else "{} - ".format(
title_header)
app.description_footer = "" if description_footer is None else "\n\n{}".format(
description_footer)
app.upload_locations = upload_locations.split(
',') if upload_locations else []
app.db_manager = database.DBManager(dsn=connection_string)
common.PromLogCountsHandler.install()
common.install_stacksampler()
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
if app.no_authentication:
logging.warning('Not authenticating POST requests')
common.serve_with_graceful_shutdown(server)
def main(
dbconnect,
config,
creds_file,
name=None,
base_dir=".",
tags='',
metrics_port=8003,
backdoor_port=0,
):
"""dbconnect should be a postgres connection string, which is either a space-separated
list of key=value pairs, or a URI like:
postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE
config should be a json blob mapping upload location names to a config object
for that location. This config object should contain the keys:
type:
the name of the upload backend type
no_transcode_check:
bool. If true, won't check for when videos are done transcoding.
This is useful when multiple upload locations actually refer to the
same place just with different settings, and you only want one of them
to actually do the check.
cut_type:
One of 'fast' or 'full'. Default 'fast'. This indicates whether to use
fast_cut_segments() or full_cut_segments() for this location.
along with any additional config options defined for that backend type.
creds_file should contain any required credentials for the upload backends, as JSON.
name defaults to hostname.
tags should be a comma-seperated list of tags to attach to all videos.
"""
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
if name is None:
name = socket.gethostname()
tags = tags.split(',') if tags else []
stop = gevent.event.Event()
gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm
logging.info("Starting up")
# We have two independent jobs to do - to perform cut jobs (cutter),
# and to check the status of transcoding videos to see if they're done (transcode checker).
# We want to error if either errors, and shut down if either exits.
dbmanager = None
stopping = gevent.event.Event()
dbmanager = DBManager(dsn=dbconnect)
while True:
try:
# Get a test connection so we know the database is up,
# this produces a clearer error in cases where there's a connection problem.
conn = dbmanager.get_conn()
except Exception:
delay = common.jitter(10)
logging.warning(
'Cannot connect to database. Retrying in {:.0f} s'.format(
delay),
exc_info=True)
stop.wait(delay)
else:
# put it back so it gets reused on next get_conn()
dbmanager.put_conn(conn)
break
with open(creds_file) as f:
credentials = json.load(f)
config = json.loads(config)
upload_locations = {}
needs_transcode_check = {}
for location, backend_config in config.items():
backend_type = backend_config.pop('type')
no_transcode_check = backend_config.pop('no_transcode_check', False)
cut_type = backend_config.pop('cut_type', 'full')
if backend_type == 'youtube':
backend_type = Youtube
elif backend_type == 'local':
backend_type = Local
else:
raise ValueError(
"Unknown upload backend type: {!r}".format(backend_type))
backend = backend_type(credentials, **backend_config)
if cut_type == 'fast':
# mark for fast cut by clearing encoding settings
backend.encoding_settings = None
elif cut_type != 'full':
raise ValueError("Unknown cut type: {!r}".format(cut_type))
upload_locations[location] = backend
if backend.needs_transcode and not no_transcode_check:
needs_transcode_check[location] = backend
cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir, tags)
transcode_checkers = [
TranscodeChecker(location, backend, dbmanager, stop)
for location, backend in needs_transcode_check.items()
]
jobs = [gevent.spawn(cutter.run)] + [
gevent.spawn(transcode_checker.run)
for transcode_checker in transcode_checkers
]
# Block until any one exits
gevent.wait(jobs, count=1)
# Stop the others if they aren't stopping already
stop.set()
# Block until all have exited
gevent.wait(jobs)
# Call get() for each one to re-raise if any errored
for job in jobs:
job.get()
logging.info("Gracefully stopped")
def main(channels,
base_dir='.',
qualities='source',
metrics_port=8002,
static_nodes='',
backdoor_port=0,
start=None,
delete_old=False,
run_once=False,
node_file=None,
node_database=None,
localhost=socket.gethostname(),
download_concurrency=5,
recent_cutoff=120):
"""Backfiller service."""
qualities = qualities.split(',') if qualities else []
qualities = [quality.strip() for quality in qualities]
static_nodes = static_nodes.split(',') if static_nodes else []
static_nodes = [static_node.strip() for static_node in static_nodes]
if start is not None:
try:
start = float(start)
logging.info('Backfilling last {} hours'.format(start))
except ValueError:
start = dateutil.parse(start)
logging.info('Backfilling since {}'.format(start))
common.PromLogCountsHandler.install()
common.install_stacksampler()
prom.start_http_server(metrics_port)
managers = []
workers = []
for channel in channels:
logging.info(
'Starting backfilling {} with {} as qualities to {}'.format(
channel, ', '.join(qualities), base_dir))
manager = BackfillerManager(base_dir, channel, qualities, static_nodes,
start, delete_old, run_once, node_file,
node_database, localhost,
download_concurrency, recent_cutoff)
managers.append(manager)
workers.append(gevent.spawn(manager.run))
def stop():
for manager in managers:
manager.stop()
gevent.signal(signal.SIGTERM, stop)
if backdoor_port:
gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port),
locals=locals()).start()
# Wait for any to die
gevent.wait(workers, count=1)
# If one has stopped, either:
# 1. stop() was called and all are stopping
# 2. one errored and we should stop all remaining and report the error
# Our behaviour in both cases is the same:
# 1. Tell all managers to gracefully stop
stop()
# 2. Wait (with timeout) until they've stopped
gevent.wait(workers)
# 3. Check if any of them failed. If they did, report it. If mulitple
# failed, we report one arbitrarily.
for worker in workers:
worker.get()
logging.info('Gracefully stopped')
