def main(dbconnect, sheets_creds_file, edit_url, bustime_start, sheet_id, worksheet_names, metrics_port=8005, backdoor_port=0, allocate_ids=False): """ Sheet sync constantly scans a Google Sheets sheet and a database, copying inputs from the sheet to the DB and outputs from the DB to the sheet. With the exception of id allocation, all operations are idempotent and multiple sheet syncs may be run for redundancy. """ common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) register_uuid() if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() stop = gevent.event.Event() gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm logging.info("Starting up") dbmanager = DBManager(dsn=dbconnect) while True: try: # Get a test connection so we know the database is up, # this produces a clearer error in cases where there's a connection problem. conn = dbmanager.get_conn() except Exception: delay = common.jitter(10) logging.info( 'Cannot connect to database. Retrying in {:.0f} s'.format( delay)) stop.wait(delay) else: # put it back so it gets reused on next get_conn() dbmanager.put_conn(conn) break sheets_creds = json.load(open(sheets_creds_file)) sheets = Sheets( client_id=sheets_creds['client_id'], client_secret=sheets_creds['client_secret'], refresh_token=sheets_creds['refresh_token'], ) SheetSync(stop, dbmanager, sheets, sheet_id, worksheet_names, edit_url, bustime_start, allocate_ids).run() logging.info("Gracefully stopped")
def main(channels, base_dir='.', qualities='source', first_hour=None, last_hour=None, metrics_port=8006, backdoor_port=0): """Segment coverage service""" qualities = qualities.split(',') if qualities else [] qualities = [quality.strip() for quality in qualities] if first_hour is not None: first_hour = dateutil.parse(first_hour) if last_hour is not None: last_hour = dateutil.parse(last_hour) common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) managers = [] workers = [] for channel in channels: logging.info( 'Starting coverage checks {} with {} as qualities in {}'.format( channel, ', '.join(qualities), base_dir)) manager = CoverageChecker(channel, qualities, base_dir, first_hour, last_hour) managers.append(manager) workers.append(gevent.spawn(manager.run)) def stop(): for manager in managers: manager.stop() gevent.signal(signal.SIGTERM, stop) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() # Wait for any to die gevent.wait(workers, count=1) # If one has stopped, either: # 1. stop() was called and all are stopping # 2. one errored and we should stop all remaining and report the error # Our behaviour in both cases is the same: # 1. Tell all managers to gracefully stop stop() # 2. Wait (with timeout) until they've stopped gevent.wait(workers) # 3. Check if any of them failed. If they did, report it. If mulitple # failed, we report one arbitrarily. for worker in workers: worker.get() logging.info('Gracefully stopped')
def main( dbconnect, creds_file, playlists, upload_location_allowlist="youtube", interval=600, metrics_port=8007, backdoor_port=0, ): """ dbconnect should be a postgres connection string creds_file should contain youtube api creds upload_location_allowlist is a comma-seperated list of database upload locations to consider as eligible to being added to playlists. For these locations, the database video id must be a youtube video id. interval is how often to check for new videos, default every 10min. """ common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() upload_locations = upload_location_allowlist.split( ",") if upload_location_allowlist else [] playlists = dict(playlists) stop = gevent.event.Event() gevent.signal_handler(signal.SIGTERM, stop.set) # shut down on sigterm logging.info("Starting up") with open(creds_file) as f: creds = json.load(f) client = GoogleAPIClient(creds['client_id'], creds['client_secret'], creds['refresh_token']) dbmanager = DBManager(dsn=dbconnect) manager = PlaylistManager(dbmanager, client, upload_locations, playlists) while not stop.is_set(): try: manager.run_once() except Exception: logging.exception("Failed to run playlist manager") manager.reset() stop.wait(interval) # wait for interval, or until stopping logging.info("Stopped")
def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0): app.static_folder = base_dir server = WSGIServer((host, port), cors(app)) PromLogCountsHandler.install() install_stacksampler() if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() serve_with_graceful_shutdown(server)
def main(channels, base_dir='.', qualities='source', metrics_port=8002, static_nodes='', backdoor_port=0, start=None, delete_old=False, run_once=False, node_file=None, node_database=None, localhost=socket.gethostname(), download_concurrency=5, recent_cutoff=120): """Backfiller service.""" qualities = qualities.split(',') if qualities else [] qualities = [quality.strip() for quality in qualities] static_nodes = static_nodes.split(',') if static_nodes else [] static_nodes = [static_node.strip() for static_node in static_nodes] if start is not None: try: start = float(start) logging.info('Backfilling last {} hours'.format(start)) except ValueError: start = dateutil.parse(start) logging.info('Backfilling since {}'.format(start)) common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) logging.info('Starting backfilling {} with {} as qualities to {}'.format( ', '.join(channels), ', '.join(qualities), base_dir)) manager = BackfillerManager(base_dir, channels, qualities, static_nodes, start, delete_old, run_once, node_file, node_database, localhost, download_concurrency, recent_cutoff) def stop(): manager.stop() gevent.signal(signal.SIGTERM, stop) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() manager.run() logging.info('Gracefully stopped')
def main(connection_string, default_channel, bustime_start, host='0.0.0.0', port=8004, backdoor_port=0, no_authentication=False, title_header=None, description_footer=None, upload_locations=''): """Thrimshim service.""" server = WSGIServer((host, port), cors(app)) app.no_authentication = no_authentication app.default_channel = default_channel app.bustime_start = bustime_start app.title_header = "" if title_header is None else "{} - ".format( title_header) app.description_footer = "" if description_footer is None else "\n\n{}".format( description_footer) app.upload_locations = upload_locations.split( ',') if upload_locations else [] stopping = gevent.event.Event() def stop(): logging.info("Shutting down") stopping.set() # handle when the server is running if hasattr(server, 'socket'): server.stop() # and when not else: sys.exit() gevent.signal(signal.SIGTERM, stop) app.db_manager = database.DBManager(dsn=connection_string) common.PromLogCountsHandler.install() common.install_stacksampler() if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() logging.info('Starting up') if app.no_authentication: logging.warning('Not authenticating POST requests') server.serve_forever() logging.info("Gracefully shut down")
def main(channels, base_dir=".", qualities="source", metrics_port=8001, backdoor_port=0): qualities = qualities.split(",") if qualities else [] managers = [ StreamsManager(channel.rstrip('!'), base_dir, qualities, important=channel.endswith('!')) for channel in channels ] def stop(): for manager in managers: manager.stop() gevent.signal(signal.SIGTERM, stop) # shut down on sigterm common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) logging.info("Starting up") workers = [gevent.spawn(manager.run) for manager in managers] if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() # Wait for any to die gevent.wait(workers, count=1) # If one has stopped, either: # 1. stop() was called and all are stopping # 2. one errored and we should stop all remaining and report the error # Our behaviour in both cases is the same: # 1. Tell all managers to gracefully stop stop() # 2. Wait (with timeout) until they've stopped gevent.wait(workers) # 3. Check if any of them failed. If they did, report it. If mulitple failed, we report # one arbitrarily. for worker in workers: worker.get() # re-raise error if failed logging.info("Gracefully stopped")
def main(host='0.0.0.0', port=8000, base_dir='.', backdoor_port=0): app.static_folder = base_dir server = WSGIServer((host, port), cors(app)) def stop(): logging.info("Shutting down") server.stop() gevent.signal(signal.SIGTERM, stop) PromLogCountsHandler.install() install_stacksampler() if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() logging.info("Starting up") server.serve_forever() logging.info("Gracefully shut down")
def main( connection_string, default_channel, bustime_start, host='0.0.0.0', port=8004, backdoor_port=0, no_authentication=False, title_header=None, description_footer=None, upload_locations='', ): server = WSGIServer((host, port), cors(app)) app.no_authentication = no_authentication app.default_channel = default_channel app.bustime_start = bustime_start app.title_header = "" if title_header is None else "{} - ".format( title_header) app.description_footer = "" if description_footer is None else "\n\n{}".format( description_footer) app.upload_locations = upload_locations.split( ',') if upload_locations else [] app.db_manager = database.DBManager(dsn=connection_string) common.PromLogCountsHandler.install() common.install_stacksampler() if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() if app.no_authentication: logging.warning('Not authenticating POST requests') common.serve_with_graceful_shutdown(server)
def main( dbconnect, config, creds_file, name=None, base_dir=".", tags='', metrics_port=8003, backdoor_port=0, ): """dbconnect should be a postgres connection string, which is either a space-separated list of key=value pairs, or a URI like: postgresql://USER:PASSWORD@HOST/DBNAME?KEY=VALUE config should be a json blob mapping upload location names to a config object for that location. This config object should contain the keys: type: the name of the upload backend type no_transcode_check: bool. If true, won't check for when videos are done transcoding. This is useful when multiple upload locations actually refer to the same place just with different settings, and you only want one of them to actually do the check. cut_type: One of 'fast' or 'full'. Default 'fast'. This indicates whether to use fast_cut_segments() or full_cut_segments() for this location. along with any additional config options defined for that backend type. creds_file should contain any required credentials for the upload backends, as JSON. name defaults to hostname. tags should be a comma-seperated list of tags to attach to all videos. """ common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() if name is None: name = socket.gethostname() tags = tags.split(',') if tags else [] stop = gevent.event.Event() gevent.signal(signal.SIGTERM, stop.set) # shut down on sigterm logging.info("Starting up") # We have two independent jobs to do - to perform cut jobs (cutter), # and to check the status of transcoding videos to see if they're done (transcode checker). # We want to error if either errors, and shut down if either exits. dbmanager = None stopping = gevent.event.Event() dbmanager = DBManager(dsn=dbconnect) while True: try: # Get a test connection so we know the database is up, # this produces a clearer error in cases where there's a connection problem. conn = dbmanager.get_conn() except Exception: delay = common.jitter(10) logging.warning( 'Cannot connect to database. Retrying in {:.0f} s'.format( delay), exc_info=True) stop.wait(delay) else: # put it back so it gets reused on next get_conn() dbmanager.put_conn(conn) break with open(creds_file) as f: credentials = json.load(f) config = json.loads(config) upload_locations = {} needs_transcode_check = {} for location, backend_config in config.items(): backend_type = backend_config.pop('type') no_transcode_check = backend_config.pop('no_transcode_check', False) cut_type = backend_config.pop('cut_type', 'full') if backend_type == 'youtube': backend_type = Youtube elif backend_type == 'local': backend_type = Local else: raise ValueError( "Unknown upload backend type: {!r}".format(backend_type)) backend = backend_type(credentials, **backend_config) if cut_type == 'fast': # mark for fast cut by clearing encoding settings backend.encoding_settings = None elif cut_type != 'full': raise ValueError("Unknown cut type: {!r}".format(cut_type)) upload_locations[location] = backend if backend.needs_transcode and not no_transcode_check: needs_transcode_check[location] = backend cutter = Cutter(upload_locations, dbmanager, stop, name, base_dir, tags) transcode_checkers = [ TranscodeChecker(location, backend, dbmanager, stop) for location, backend in needs_transcode_check.items() ] jobs = [gevent.spawn(cutter.run)] + [ gevent.spawn(transcode_checker.run) for transcode_checker in transcode_checkers ] # Block until any one exits gevent.wait(jobs, count=1) # Stop the others if they aren't stopping already stop.set() # Block until all have exited gevent.wait(jobs) # Call get() for each one to re-raise if any errored for job in jobs: job.get() logging.info("Gracefully stopped")
def main(channels, base_dir='.', qualities='source', metrics_port=8002, static_nodes='', backdoor_port=0, start=None, delete_old=False, run_once=False, node_file=None, node_database=None, localhost=socket.gethostname(), download_concurrency=5, recent_cutoff=120): """Backfiller service.""" qualities = qualities.split(',') if qualities else [] qualities = [quality.strip() for quality in qualities] static_nodes = static_nodes.split(',') if static_nodes else [] static_nodes = [static_node.strip() for static_node in static_nodes] if start is not None: try: start = float(start) logging.info('Backfilling last {} hours'.format(start)) except ValueError: start = dateutil.parse(start) logging.info('Backfilling since {}'.format(start)) common.PromLogCountsHandler.install() common.install_stacksampler() prom.start_http_server(metrics_port) managers = [] workers = [] for channel in channels: logging.info( 'Starting backfilling {} with {} as qualities to {}'.format( channel, ', '.join(qualities), base_dir)) manager = BackfillerManager(base_dir, channel, qualities, static_nodes, start, delete_old, run_once, node_file, node_database, localhost, download_concurrency, recent_cutoff) managers.append(manager) workers.append(gevent.spawn(manager.run)) def stop(): for manager in managers: manager.stop() gevent.signal(signal.SIGTERM, stop) if backdoor_port: gevent.backdoor.BackdoorServer(('127.0.0.1', backdoor_port), locals=locals()).start() # Wait for any to die gevent.wait(workers, count=1) # If one has stopped, either: # 1. stop() was called and all are stopping # 2. one errored and we should stop all remaining and report the error # Our behaviour in both cases is the same: # 1. Tell all managers to gracefully stop stop() # 2. Wait (with timeout) until they've stopped gevent.wait(workers) # 3. Check if any of them failed. If they did, report it. If mulitple # failed, we report one arbitrarily. for worker in workers: worker.get() logging.info('Gracefully stopped')