def register_parser_arguments():
options.get_parser('execute').add_argument(
'--cli-config',
nargs='+',
type=key_value_pair,
metavar='VARIABLE=VALUE',
help='configuration parameters through commandline',
)
def register_parser_arguments():
options.get_parser('execute').add_argument(
'--discover-now',
action='store_true',
dest='discover_now',
default=False,
help='Immediately try to discover everything',
)
def register_parser_arguments():
options.get_parser('execute').add_argument(
'--dl-path',
dest='dl_path',
default=False,
metavar='PATH',
help='override path for download plugin, applies to all executed tasks',
)
def register_parser_arguments():
options.get_parser('execute').add_argument(
'--now',
action='store_true',
dest='interval_ignore',
default=False,
help='run task(s) even if the interval plugin would normally prevent it',
)
def register_parser_arguments():
options.get_parser("execute").add_argument(
"--cli-config",
nargs="+",
type=key_value_pair,
metavar="VARIABLE=VALUE",
help="configuration parameters trough commandline",
)
def register_parser_arguments():
options.get_parser("execute").add_argument(
"--dump",
nargs="*",
choices=["eval", "trace", "accepted", "rejected", "undecided", "title"],
dest="dump_entries",
help="display all entries in task with fields they contain, "
"use `--dump eval` to evaluate all lazy fields. Specify an entry state/states to only dump matching entries.",
)
def register_parser_arguments():
options.get_parser("execute").add_argument(
"--dump",
nargs="?",
choices=["eval", "trace"],
dest="dump_entries",
const=True,
help="display all entries in task with fields they contain, " "use `--dump eval` to evaluate all lazy fields",
)
def register_parser_arguments():
options.get_parser('execute').add_argument(
'--tail-reset',
action='store',
dest='tail_reset',
default=False,
metavar='FILE|TASK',
help='reset tail position for a file',
)
def register_parser_arguments():
options.get_parser("execute").add_argument(
"--tail-reset",
action="store",
dest="tail_reset",
default=False,
metavar="FILE|TASK",
help="reset tail position for a file",
)
def register_parser_arguments():
options.get_parser('execute').add_argument(
'--dump',
nargs='*',
choices=['eval', 'trace', 'accepted', 'rejected', 'undecided', 'title'],
dest='dump_entries',
help=(
'display all entries in task with fields they contain, '
'use `--dump eval` to evaluate all lazy fields. Specify an entry '
'state/states to only dump matching entries.'
),
)
def main(args=None):
"""Main entry point for Command Line Interface"""
logger.initialize()
plugin.load_plugins()
options = get_parser().parse_args(args)
manager = Manager(options)
log_level = logging.getLevelName(options.loglevel.upper())
log_file = os.path.expanduser(manager.options.logfile)
# If an absolute path is not specified, use the config directory.
if not os.path.isabs(log_file):
log_file = os.path.join(manager.config_base, log_file)
logger.start(log_file, log_level)
if options.profile:
try:
import cProfile as profile
except ImportError:
import profile
profile.runctx('manager.start()', globals(), locals(),
os.path.join(manager.config_base, options.profile))
else:
manager.start()
def setup_params(mgr):
parser = get_parser('execute')
for action in parser._optionals._actions:
# Ignore list for irrelevant actions
ignore = ['help', 'verbose', 'silent', 'try-regexp', 'dump-config', 'dump']
name = action.option_strings[-1].strip('--')
if name in ignore or action.help == '==SUPPRESS==':
continue
name = name.replace('-', '_')
property_data = {'description': action.help.capitalize()}
if isinstance(action, argparse._StoreConstAction):
property_data['type'] = 'boolean'
elif isinstance(action, argparse._StoreAction):
if action.nargs in ['+', '*']:
property_data['type'] = 'array'
property_data['items'] = {'type': 'string'}
property_data['minItems'] = 1
else:
property_data['type'] = 'string'
else:
# Unknown actions should not be added to schema
property_data = None
# Some options maybe pre-added to schema with additional options, don't override them
if property_data and name not in ObjectsContainer.task_execution_input['properties']:
ObjectsContainer.task_execution_input['properties'][name] = property_data
ObjectsContainer.task_execution_input['additionalProperties'] = False
def register_parser_arguments():
exec_parser = options.get_parser('execute')
inject_parser = options.register_command(
'inject',
do_cli,
add_help=False,
parents=[exec_parser],
help='inject an entry from command line into tasks',
usage='%(prog)s title/url [url] [--accept] [--force] '
'[--fields NAME=VALUE [NAME=VALUE...]] [<execute arguments>]',
epilog=(
'If only a URL and no title is given, Flexget will attempt to '
'find a title in the URL\'s response headers.'
),
)
inject_group = inject_parser.add_argument_group('inject arguments')
inject_group.add_argument(
'title', metavar='title/url', help='title or url of the entry to inject'
)
inject_group.add_argument('url', nargs='?', help='url of the entry to inject')
inject_group.add_argument(
'--force', action='store_true', help='prevent any plugins from rejecting this entry'
)
inject_group.add_argument(
'--accept', action='store_true', help='accept this entry immediately upon injection'
)
inject_group.add_argument('--fields', metavar='NAME=VALUE', nargs='+', type=key_equals_value)
# Hack the title of the exec options a bit (would be 'optional arguments' otherwise)
inject_parser._action_groups[1].title = 'execute arguments'
# The exec arguments show first... unless we switch them
inject_parser._action_groups.remove(inject_group)
inject_parser._action_groups.insert(0, inject_group)
def setup_once():
global plugins_loaded, test_arguments
if not plugins_loaded:
flexget.logger.initialize(True)
setup_logging_level()
load_plugins()
# store options for MockManager
test_arguments = get_parser().parse_args(['execute'])
plugins_loaded = True
def setup_once():
global plugins_loaded, test_arguments
if not plugins_loaded:
flexget.logger.initialize(True)
setup_logging_level()
warnings.simplefilter('error', DeprecationWarning)
load_plugins()
# store options for MockManager
test_arguments = get_parser().parse_args(['execute'])
plugins_loaded = True
def setup_params(mgr):
parser = get_parser('execute')
for action in parser._optionals._actions:
ignore = ['v', 's', 'try-regexp', 'dump-config']
name = action.option_strings[0].strip('--')
if name in ignore:
continue
if isinstance(action, argparse._StoreConstAction) and action.help != '==SUPPRESS==':
task_execution_input['properties'][name] = {'type': 'boolean'}
TaskExecutionAPI.__apidoc__['description'] += "'{0}': {1}<br>".format(name, action.help)
def setup_params(mgr):
parser = get_parser("execute")
for action in parser._optionals._actions:
ignore = ["v", "s", "try-regexp", "dump-config"]
name = action.option_strings[0].strip("--")
if name in ignore:
continue
if isinstance(action, argparse._StoreConstAction) and action.help != "==SUPPRESS==":
name = name.replace("-", "_")
task_execution_input["properties"][name] = {"type": "boolean"}
TaskExecutionAPI.__apidoc__["description"] += "'{0}': {1}<br>".format(name, action.help)
def register_parser_arguments():
exec_parser = options.get_parser("execute")
exec_parser.add_argument(
"-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose undecided entries"
)
exec_parser.add_argument(
"-s",
"--silent",
action="store_true",
dest="silent",
default=False,
help="don't verbose any actions (accept, reject, fail)",
)
def register_parser_arguments():
archive_parser = options.register_command(
'archive', do_cli, help='Search and manipulate the archive database'
)
archive_parser.add_subparsers(title='Actions', metavar='<action>', dest='archive_action')
# Default usage shows the positional arguments after the optional ones, override usage to fix it
search_parser = archive_parser.add_subparser(
'search',
help='Search from the archive',
usage='%(prog)s [-h] <keyword> [<keyword> ...] [optional arguments]',
parents=[table_parser],
)
search_parser.add_argument(
'keywords', metavar='<keyword>', nargs='+', help='Keyword(s) to search for'
)
search_parser.add_argument(
'--tags', metavar='TAG', nargs='+', default=[], help='Tag(s) to search within'
)
search_parser.add_argument(
'--sources', metavar='SOURCE', nargs='+', default=[], help='Source(s) to search within'
)
inject_parser = archive_parser.add_subparser(
'inject', help='Inject entries from the archive back into tasks'
)
inject_parser.add_argument(
'ids', nargs='+', type=int, metavar='ID', help='Archive ID of an item to inject'
)
inject_parser.add_argument(
'--immortal',
action='store_true',
help='Injected entries will not be able to be ' 'rejected by any plugins',
)
exec_group = inject_parser.add_argument_group('execute arguments')
exec_group.add_argument(
'execute_options', action=ParseExtrasAction, parser=get_parser('execute')
)
tag_parser = archive_parser.add_subparser(
'tag-source', help='Tag all archived entries within a given source'
)
tag_parser.add_argument(
'source', metavar='<source>', help='The source whose entries you would like to tag'
)
tag_parser.add_argument(
'tags',
nargs='+',
metavar='<tag>',
help='The tag(s) you would like to apply to the entries',
)
archive_parser.add_subparser(
'consolidate', help='Migrate old archive data to new model, may take a long time'
)
def setup_params(mgr):
parser = get_parser('execute')
for action in parser._optionals._actions:
ignore = ['v', 's', 'try-regexp', 'dump-config']
name = action.option_strings[0].strip('--')
if name in ignore:
continue
if isinstance(action, argparse._StoreConstAction) and action.help != '==SUPPRESS==':
execute_parser.add_argument(name,
type=flask_restplus.inputs.boolean,
required=False,
help=action.help)
api.doc(parser=execute_parser)(TaskExecutionAPI)
def exposed_handle_cli(self, args):
args = rpyc.utils.classic.obtain(args)
parser = get_parser()
try:
options = parser.parse_args(args, raise_errors=True)
except ParserError as e:
# Recreate the normal error text to the client's console
self.client_console('error: ' + e.message)
e.parser.print_help(self.client_out_stream)
return
if not options.cron:
with capture_output(self.client_out_stream, loglevel=options.loglevel):
self.manager.handle_cli(options)
else:
self.manager.handle_cli(options)
def register_parser_arguments():
exec_parser = options.get_parser('execute')
exec_parser.add_argument(
'--dump-config',
action='store_true',
dest='dump_config',
default=False,
help='display the config of each feed after template merging/config generation occurs',
)
exec_parser.add_argument(
'--dump-config-python',
action='store_true',
dest='dump_config_python',
default=False,
help=SUPPRESS,
)
def post(self, session=None):
""" Execute task(s) """
options = request.json or {}
options_string = options.pop('options_string', '')
if options_string:
try:
options['options'] = get_parser('execute').parse_args(options_string, raise_errors=True)
except ValueError as e:
return {'error': 'invalid options_string specified: %s' % e.message}, 400
tasks = [{'id': task_id, 'name': task_name}
for task_id, task_name, task_event
in self.manager.execute(options=options)]
return {'tasks': tasks}
def post(self, session=None):
""" Execute task
Return a unique execution ID for tracking and log streaming
"""
kwargs = request.json or {}
options_string = kwargs.pop('options_string', '')
if options_string:
try:
kwargs['options'] = get_parser('execute').parse_args(options_string, raise_errors=True)
except ValueError as e:
return {'error': 'invalid options_string specified: %s' % e.message}, 400
tasks = self.manager.execute(**kwargs)
return {"tasks": [_task_info_dict(self.manager.task_queue.tasks_info[task_id]) for task_id, event in tasks]}
def exposed_handle_cli(self, args):
args = rpyc.utils.classic.obtain(args)
log.verbose('Running command `%s` for client.' % ' '.join(args))
parser = get_parser()
try:
options = parser.parse_args(args, file=self.client_out_stream)
except SystemExit as e:
if e.code:
# TODO: Not sure how to properly propagate the exit code back to client
log.debug('Parsing cli args caused system exit with status %s.' % e.code)
return
if not options.cron:
with capture_output(self.client_out_stream, loglevel=options.loglevel):
self.manager.handle_cli(options)
else:
self.manager.handle_cli(options)
def post(self, session=None):
""" Execute task(s) and stream results """
options = request.json or {}
args = stream_parser.parse_args()
options_string = options.pop('options_string', '')
if options_string:
try:
options['options'] = get_parser('execute').parse_args(options_string, raise_errors=True)
except ValueError as e:
return {'error': 'invalid options_string specified: %s' % e.message}, 400
queue = ExecuteQueue()
output = queue if args['log'] else None
tasks_queued = []
for task_id, task_name, task_event in self.manager.execute(options=options, output=output):
tasks_queued.append({'id': task_id, 'name': task_name, 'event': task_event})
_streams[task_id] = {
'queue': queue,
'last_update': datetime.now(),
'args': args
}
def stream_response():
# First return the tasks to execute
yield '{"stream": ['
yield json.dumps({'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]}) + ',\n'
while True:
# If the server is shutting down then end the stream nicely
if cherrypy.engine.state != cherrypy.engine.states.STARTED:
break
try:
yield queue.get(timeout=1) + ',\n'
continue
except Empty:
pass
if queue.empty() and all([task['event'].is_set() for task in tasks_queued]):
for task in tasks_queued:
del _streams[task['id']]
break
yield '{}]}'
return Response(stream_response(), mimetype='text/event-stream')
def setup_once():
global plugins_loaded, test_arguments
if not plugins_loaded:
flexget.logger.initialize(True)
setup_logging_level()
warnings.simplefilter('error')
# VCR.py mocked functions not handle ssl verification well. Older versions of urllib3 don't have this
if VCR_RECORD_MODE != 'off':
try:
from requests.packages.urllib3.exceptions import SecurityWarning
warnings.simplefilter('ignore', SecurityWarning)
except ImportError:
pass
load_plugins()
# store options for MockManager
test_arguments = get_parser().parse_args(['execute'])
plugins_loaded = True
def register_parser_arguments():
exec_parser = options.get_parser('execute')
exec_parser.add_argument(
'-v',
'--verbose',
action='store_true',
dest='verbose',
default=False,
help='verbose undecided entries',
)
exec_parser.add_argument(
'-s',
'--silent',
action='store_true',
dest='silent',
default=False,
help='don\'t verbose any actions (accept, reject, fail)',
)
def main(args=None):
"""Main entry point for Command Line Interface"""
logger.initialize()
plugin.load_plugins()
options = get_parser().parse_args(args)
manager = Manager(options)
log_level = logging.getLevelName(options.loglevel.upper())
log_file = os.path.expanduser(manager.options.logfile)
# If an absolute path is not specified, use the config directory.
if not os.path.isabs(log_file):
log_file = os.path.join(manager.config_base, log_file)
logger.start(log_file, log_level)
manager.run_cli_command()
def initialize(self):
"""
Load plugins, database, and config. Also initializes (but does not start) the task queue and ipc server.
This should only be called after obtaining a lock.
"""
if self.initialized:
raise RuntimeError('Cannot call initialize on an already initialized manager.')
plugin.load_plugins(
extra_plugins=[os.path.join(self.config_base, 'plugins')],
extra_components=[os.path.join(self.config_base, 'components')],
)
# Reparse CLI options now that plugins are loaded
if not self.args:
self.args = ['--help']
self.options = get_parser().parse_args(self.args)
self.task_queue = TaskQueue()
self.ipc_server = IPCServer(self, self.options.ipc_port)
self.setup_yaml()
self.init_sqlalchemy()
fire_event('manager.initialize', self)
try:
self.load_config()
except ValueError as e:
log.critical('Failed to load config file: %s' % e.args[0])
raise
# cannot be imported at module level because of circular references
from flexget.utils.simple_persistence import SimplePersistence
self.persist = SimplePersistence('manager')
if db_schema.upgrade_required():
log.info('Database upgrade is required. Attempting now.')
fire_event('manager.upgrade', self)
if manager.db_upgraded:
fire_event('manager.db_upgraded', self)
fire_event('manager.startup', self)
self.initialized = True
def initialize(self) -> None:
"""
Load plugins, database, and config. Also initializes (but does not start) the task queue and ipc server.
This should only be called after obtaining a lock.
"""
if self.initialized:
raise RuntimeError(
'Cannot call initialize on an already initialized manager.')
plugin.load_plugins(
extra_plugins=[os.path.join(self.config_base, 'plugins')],
extra_components=[os.path.join(self.config_base, 'components')],
)
# Reparse CLI options now that plugins are loaded
self.options = get_parser().parse_args(self.args)
self.task_queue = TaskQueue()
self.ipc_server = IPCServer(self, self.options.ipc_port)
self.setup_yaml()
self.init_sqlalchemy()
fire_event('manager.initialize', self)
try:
self.load_config()
except ValueError as e:
logger.critical('Failed to load config file: {}', e.args[0])
raise
# cannot be imported at module level because of circular references
from flexget.utils.simple_persistence import SimplePersistence
self.persist = SimplePersistence('manager')
if db_schema.upgrade_required():
logger.info('Database upgrade is required. Attempting now.')
fire_event('manager.upgrade', self)
if manager.db_upgraded:
fire_event('manager.db_upgraded', self)
fire_event('manager.startup', self)
self.initialized = True
def post(self, session=None):
""" Execute task(s) """
options = request.json or {}
options_string = options.pop('options_string', '')
if options_string:
try:
options['options'] = get_parser('execute').parse_args(
options_string, raise_errors=True)
except ValueError as e:
return {
'error': 'invalid options_string specified: %s' % e.message
}, 400
tasks = [{
'id': task_id,
'name': task_name
} for task_id, task_name, task_event in self.manager.execute(
options=options)]
return {'tasks': tasks}
def register_parser_arguments():
exec_parser = options.get_parser('execute')
inject_parser = options.register_command(
'inject',
do_cli,
add_help=False,
parents=[exec_parser],
help='inject an entry from command line into tasks',
usage='%(prog)s title/url [url] [--accept] [--force] '
'[--fields NAME=VALUE [NAME=VALUE...]] [<execute arguments>]',
epilog=
'If only a URL and no title is given, Flexget will attempt to find a title in the URL\'s response headers.'
)
inject_group = inject_parser.add_argument_group('inject arguments')
inject_group.add_argument('title',
metavar='title/url',
help='title or url of the entry to inject')
inject_group.add_argument('url',
nargs='?',
help='url of the entry to inject')
inject_group.add_argument(
'--force',
action='store_true',
help='prevent any plugins from rejecting this entry')
inject_group.add_argument(
'--accept',
action='store_true',
help='accept this entry immediately upon injection')
inject_group.add_argument('--fields',
metavar='NAME=VALUE',
nargs='+',
type=key_equals_value)
# Hack the title of the exec options a bit (would be 'optional arguments' otherwise)
inject_parser._action_groups[1].title = 'execute arguments'
# The exec arguments show first... unless we switch them
inject_parser._action_groups.remove(inject_group)
inject_parser._action_groups.insert(0, inject_group)
def exposed_handle_cli(self, args):
args = rpyc.utils.classic.obtain(args)
log.verbose('Running command `%s` for client.' % ' '.join(args))
parser = get_parser()
try:
options = parser.parse_args(args, file=self.client_out_stream)
except SystemExit as e:
if e.code:
# TODO: Not sure how to properly propagate the exit code back to client
log.debug('Parsing cli args caused system exit with status %s.' % e.code)
return
# Saving original terminal size to restore after monkeypatch
original_terminal_info = terminal.terminal_info
# Monkeypatching terminal_size so it'll work using IPC
terminal.terminal_info = self._conn.root.terminal_info
try:
if not options.cron:
with capture_output(self.client_out_stream, loglevel=options.loglevel):
self.manager.handle_cli(options)
else:
self.manager.handle_cli(options)
finally:
# Restoring original terminal_size value
terminal.terminal_info = original_terminal_info
def main(args=None):
"""Main entry point for Command Line Interface"""
logger.initialize()
plugin.load_plugins()
options = get_parser().parse_args(args)
try:
manager = Manager(options)
except (IOError, ValueError) as e:
print('Could not initialize manager: %s' % e, file=sys.stderr)
sys.exit(1)
if options.profile:
try:
import cProfile as profile
except ImportError:
import profile
profile.runctx('manager.start()', globals(), locals(),
os.path.join(manager.config_base, options.profile))
else:
manager.start()
def register_parser_arguments():
options.get_parser('execute').add_argument('--discover-now', action='store_true', dest='discover_now',
default=False, help='Immediately try to discover everything')
def register_options():
options.get_parser().add_argument('--log-start',
action='store_true',
help=SUPPRESS)
def register_parser_arguments():
options.get_parser('execute').add_argument('--explain-sql',
action='store_true',
dest='explain_sql',
default=False,
help=SUPPRESS)
def register_parser_arguments():
options.get_parser('execute').add_argument('--tail-reset', action='store', dest='tail_reset', default=False,
metavar='FILE|TASK', help='reset tail position for a file')
def register_parser_arguments():
options.get_parser().add_argument(
'--debug-db-sessions',
action='store_true',
help='debug session starts and ends, for finding problems with db locks'
)
def register_parser_arguments():
options.get_parser().add_argument('--debug-warnings', action='store_true',
help='elevate warnings to errors for debugging purposes, so a traceback is shown')
def register_parser_arguments():
options.get_parser('execute').add_argument('--debug-perf',
action='store_true',
dest='debug_perf',
default=False,
help=SUPPRESS)
def register_parser_arguments():
options.get_parser('execute').add_argument('--dump', nargs='*', choices=['eval', 'trace', 'accepted', 'rejected',
'undecided', 'title'], dest='dump_entries',
help='display all entries in task with fields they contain, '
'use `--dump eval` to evaluate all lazy fields. Specify an entry state/states to only dump matching entries.')
def register_parser_arguments():
exec_parser = options.get_parser('execute')
exec_parser.add_argument('-v', '--verbose', action='store_true', dest='verbose', default=False,
help='verbose undecided entries')
exec_parser.add_argument('-s', '--silent', action='store_true', dest='silent', default=False,
help='don\'t verbose any actions (accept, reject, fail)')
def register_parser_arguments():
options.get_parser('execute').add_argument('--now', action='store_true', dest='interval_ignore', default=False,
help='run task(s) even if the interval plugin would normally prevent it')
def register_parser_arguments():
options.get_parser('execute').add_argument('--cli-config', nargs='+', type=key_value_pair, metavar='VARIABLE=VALUE',
help='configuration parameters through commandline')
def register_parser_arguments():
options.get_parser('execute').add_argument('--template', metavar='NAME', help='execute tasks using given template')
def register_parser_arguments():
options.get_parser().add_argument('--mem-usage', action='store_true', dest='mem_usage', default=False,
help='display memory usage debug information')
@api.after_request
def attach_schema(response):
# TODO: Check if /schema/ourpath exists
schema_path = '/schema' + request.path
response.headers[b'Content-Type'] += '; profile=%s' % schema_path
return response
# TODO: These endpoints should probably return a header which points to a json schema describing the return data
@api.route('/version')
def version():
return jsonify(flexget_version=flexget.__version__,
api_version=API_VERSION)
exec_parser = RaiseErrorArgumentParser(parents=[get_parser('execute')])
@api.route('/execute', methods=['GET', 'POST'])
def execute():
kwargs = request.json or {}
options_string = kwargs.pop('options_string', '')
if options_string:
try:
kwargs['options'] = exec_parser.parse_args(options_string)
except ValueError as e:
return jsonify(error='invalid options_string specified: %s' %
e.message), 400
# We'll stream the log results as they arrive in the bufferqueue
kwargs['output'] = BufferQueue()
import logging
from Queue import Empty
from flask import render_template, request, flash
from flask import Blueprint, escape, jsonify
from flexget.options import get_parser
from flexget.ui.webui import register_plugin, manager
from flexget.scheduler import BufferQueue
execute = Blueprint('execute', __name__)
log = logging.getLogger('ui.execute')
bufferqueue = BufferQueue()
exec_parser = get_parser('execute')
@execute.route('/', methods=['POST', 'GET'])
def index():
context = {'progress': exec_parser.format_help().split('\n')}
if request.method == 'POST':
try:
options = exec_parser.parse_args(request.form.get('options', ''),
raise_errors=True)
except ValueError as e:
flash(escape(e.message), 'error')
context['options'] = request.form['options']
else:
manager.scheduler.execute(options=options.execute,
output=bufferqueue)
def register_parser_arguments():
options.get_parser('execute').add_argument('--dl-path', dest='dl_path', default=False, metavar='PATH',
help='override path for download plugin, applies to all executed tasks')
