def do_cli(manager, options):
if not options.url:
# Determine if first positional argument is a URL or a title
if '://' in options.title:
options.url = options.title
options.title = None
if options.url and not options.title:
# Attempt to get a title from the URL response's headers
try:
value, params = cgi.parse_header(
requests.head(options.url).headers['Content-Disposition'])
options.title = params['filename']
except KeyError:
console(
'No title given, and couldn\'t get one from the URL\'s HTTP response. Aborting.'
)
return
entry = Entry(title=options.title)
if options.url:
entry['url'] = options.url
else:
entry['url'] = 'http://localhost/inject/%s' % ''.join(
random.sample(string.letters + string.digits, 30))
if options.force:
entry['immortal'] = True
if options.accept:
entry.accept(reason='accepted by CLI inject')
if options.fields:
for key, value in options.fields:
entry[key] = value
options.inject = [entry]
manager.execute_command(options)
def on_task_input(self, task):
if not task.manager.options.inject:
return
options = self.parse_arguments(task.manager.options.inject)
# disable other inputs
log.info('Disabling the rest of the input phase.')
task.disable_phase('input')
# create our injected entry
entry = Entry(options['entry'], injected=True)
if not 'url' in entry:
entry['url'] = 'http://localhost/inject/%s' % ''.join([
random.choice(string.letters + string.digits)
for x in range(1, 30)
])
if entry.get('immortal'):
log.debug('Injected entry is immortal')
task.all_entries.append(entry)
if options.get('accept', False):
log.debug('accepting the injection')
entry.accept('--inject accepted')
def do_cli(manager, options):
if not options.url:
# Determine if first positional argument is a URL or a title
if '://' in options.title:
options.url = options.title
options.title = None
if options.url and not options.title:
# Attempt to get a title from the URL response's headers
try:
value, params = cgi.parse_header(requests.head(options.url).headers['Content-Disposition'])
options.title = params['filename']
except KeyError:
console('No title given, and couldn\'t get one from the URL\'s HTTP response. Aborting.')
return
entry = Entry(title=options.title)
if options.url:
entry['url'] = options.url
else:
entry['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30))
if options.force:
entry['immortal'] = True
if options.accept:
entry.accept(reason='accepted by CLI inject')
if options.fields:
for key, value in options.fields:
entry[key] = value
options.inject = [entry]
manager.execute_command(options)
def __call__(self, parser, namespace, values, option_string=None):
kwargs = {'title': values.pop(0)}
if values:
kwargs['url'] = values.pop(0)
else:
kwargs['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30))
if 'force' in [v.lower() for v in values]:
kwargs['immortal'] = True
entry = Entry(**kwargs)
if 'accept' in [v.lower() for v in values]:
entry.accept(reason='accepted by --inject')
setattr(namespace, self.dest, [entry])
def __call__(self, parser, namespace, values, option_string=None):
kwargs = {"title": values.pop(0)}
if values:
kwargs["url"] = values.pop(0)
else:
kwargs["url"] = "http://localhost/inject/%s" % "".join(random.sample(string.letters + string.digits, 30))
if "force" in [v.lower() for v in values]:
kwargs["immortal"] = True
entry = Entry(**kwargs)
if "accept" in [v.lower() for v in values]:
entry.accept(reason="accepted by --inject")
setattr(namespace, self.dest, [entry])
def __call__(self, parser, namespace, values, option_string=None):
kwargs = {'title': values.pop(0)}
if values:
kwargs['url'] = values.pop(0)
else:
kwargs['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30))
if 'force' in [v.lower() for v in values]:
kwargs['immortal'] = True
entry = Entry(**kwargs)
if 'accept' in [v.lower() for v in values]:
entry.accept(reason='accepted by --inject')
existing = getattr(namespace, self.dest, None) or []
setattr(namespace, self.dest, existing + [entry])
def do_cli(manager, options):
entry = Entry(title=options.title)
if options.url:
entry['url'] = options.url
else:
entry['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30))
if options.force:
entry['immortal'] = True
if options.accept:
entry.accept(reason='accepted by CLI inject')
exec_options = dict(options.execute)
exec_options['inject'] = [entry]
manager.execute_command(exec_options)
def cli_inject(manager, options):
logger.debug('Finding inject content')
inject_entries = defaultdict(list)
with Session() as session:
for id in options.ids:
archive_entry = session.query(
flexget.components.archive.db.ArchiveEntry).get(id)
# not found
if not archive_entry:
logger.critical("There's no archived item with ID `{}`", id)
continue
# find if there is no longer any task within sources
if not any(source.name in manager.tasks
for source in archive_entry.sources):
logger.error(
'None of sources ({}) exists anymore, cannot inject `{}` from archive!',
', '.join([s.name for s in archive_entry.sources]),
archive_entry.title,
)
continue
inject_entry = Entry(archive_entry.title, archive_entry.url)
if archive_entry.description:
inject_entry['description'] = archive_entry.description
if options.immortal:
logger.debug('Injecting as immortal')
inject_entry['immortal'] = True
inject_entry['accepted_by'] = 'archive inject'
inject_entry.accept('injected')
# update list of tasks to be injected
for source in archive_entry.sources:
inject_entries[source.name].append(inject_entry)
for task_name in inject_entries:
for inject_entry in inject_entries[task_name]:
logger.info('Injecting from archive `{}` into `{}`',
inject_entry['title'], task_name)
for index, task_name in enumerate(inject_entries):
options.inject = inject_entries[task_name]
options.tasks = [task_name]
# TODO: This is a bit hacky, consider a better way
if index == len(inject_entries) - 1:
# We use execute_command on the last item, rather than regular execute, to start FlexGet running.
break
manager.execute(options)
manager.execute_command(options)
def cli_inject(manager, options):
log.debug('Finding inject content')
inject_entries = defaultdict(list)
session = Session()
try:
for id in options.ids:
archive_entry = session.query(ArchiveEntry).get(id)
# not found
if not archive_entry:
log.critical('There\'s no archived item with ID `%s`' % id)
continue
# find if there is no longer any task within sources
if not any(source.name in manager.tasks
for source in archive_entry.sources):
log.error(
'None of sources (%s) exists anymore, cannot inject `%s` from archive!'
% (', '.join([s.name for s in archive_entry.sources
]), archive_entry.title))
continue
# update list of tasks to be injected
for source in archive_entry.sources:
inject_entries[source.name].append(archive_entry)
finally:
session.close()
for task_name in inject_entries:
entries = []
for inject_entry in inject_entries[task_name]:
log.info('Injecting from archive `%s`' % inject_entry.title)
entry = Entry(inject_entry.title, inject_entry.url)
if inject_entry.description:
entry['description'] = inject_entry.description
if options.immortal:
log.debug('Injecting as immortal')
entry['immortal'] = True
entry['accepted_by'] = 'archive inject'
entry.accept('injected')
entries.append(entry)
manager.scheduler.execute(options={
'inject': entries,
'tasks': [task_name]
})
with manager.acquire_lock():
manager.scheduler.start(run_schedules=False)
manager.shutdown()
def do_cli(manager, options):
entry = Entry(title=options.title)
if options.url:
entry['url'] = options.url
else:
entry['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30))
if options.force:
entry['immortal'] = True
if options.accept:
entry.accept(reason='accepted by CLI inject')
if options.fields:
for key, value in options.fields:
entry[key] = value
options.inject = [entry]
manager.execute_command(options)
def cli_inject(manager, options):
log.debug("Finding inject content")
inject_entries = defaultdict(list)
session = Session()
try:
for id in options.ids:
archive_entry = session.query(ArchiveEntry).get(id)
# not found
if not archive_entry:
log.critical("There's no archived item with ID `%s`" % id)
continue
# find if there is no longer any task within sources
if not any(source.name in manager.tasks for source in archive_entry.sources):
log.error(
"None of sources (%s) exists anymore, cannot inject `%s` from archive!"
% (", ".join([s.name for s in archive_entry.sources]), archive_entry.title)
)
continue
# update list of tasks to be injected
for source in archive_entry.sources:
inject_entries[source.name].append(archive_entry)
finally:
session.close()
for task_name in inject_entries:
entries = []
for inject_entry in inject_entries[task_name]:
log.info("Injecting from archive `%s`" % inject_entry.title)
entry = Entry(inject_entry.title, inject_entry.url)
if inject_entry.description:
entry["description"] = inject_entry.description
if options.immortal:
log.debug("Injecting as immortal")
entry["immortal"] = True
entry["accepted_by"] = "archive inject"
entry.accept("injected")
entries.append(entry)
manager.execute(options={"inject": entries, "tasks": [task_name]})
with manager.acquire_lock():
manager.shutdown(finish_queue=True)
manager.run()
def cli_inject(manager, options):
log.debug("Finding inject content")
inject_entries = defaultdict(list)
with Session() as session:
for id in options.ids:
archive_entry = session.query(ArchiveEntry).get(id)
# not found
if not archive_entry:
log.critical("There's no archived item with ID `%s`" % id)
continue
# find if there is no longer any task within sources
if not any(source.name in manager.tasks for source in archive_entry.sources):
log.error(
"None of sources (%s) exists anymore, cannot inject `%s` from archive!"
% (", ".join([s.name for s in archive_entry.sources]), archive_entry.title)
)
continue
inject_entry = Entry(archive_entry.title, archive_entry.url)
if archive_entry.description:
inject_entry["description"] = archive_entry.description
if options.immortal:
log.debug("Injecting as immortal")
inject_entry["immortal"] = True
inject_entry["accepted_by"] = "archive inject"
inject_entry.accept("injected")
# update list of tasks to be injected
for source in archive_entry.sources:
inject_entries[source.name].append(inject_entry)
for task_name in inject_entries:
for inject_entry in inject_entries[task_name]:
log.info("Injecting from archive `%s` into `%s`" % (inject_entry["title"], task_name))
for index, task_name in enumerate(inject_entries):
options.inject = inject_entries[task_name]
options.tasks = [task_name]
# TODO: This is a bit hacky, consider a better way
if index == len(inject_entries) - 1:
# We use execute_command on the last item, rather than regular execute, to start FlexGet running.
break
manager.execute(options)
manager.execute_command(options)
def cli_inject(manager, options):
log.debug('Finding inject content')
inject_entries = defaultdict(list)
session = Session()
try:
for id in options.ids:
archive_entry = session.query(ArchiveEntry).get(id)
# not found
if not archive_entry:
log.critical('There\'s no archived item with ID `%s`' % id)
continue
# find if there is no longer any task within sources
if not any(source.name in manager.tasks for source in archive_entry.sources):
log.error('None of sources (%s) exists anymore, cannot inject `%s` from archive!' %
(', '.join([s.name for s in archive_entry.sources]), archive_entry.title))
continue
# update list of tasks to be injected
for source in archive_entry.sources:
inject_entries[source.name].append(archive_entry)
finally:
session.close()
for task_name in inject_entries:
entries = []
for inject_entry in inject_entries[task_name]:
log.info('Injecting from archive `%s`' % inject_entry.title)
entry = Entry(inject_entry.title, inject_entry.url)
if inject_entry.description:
entry['description'] = inject_entry.description
if options.immortal:
log.debug('Injecting as immortal')
entry['immortal'] = True
entry['accepted_by'] = 'archive inject'
entry.accept('injected')
entries.append(entry)
manager.scheduler.execute(options={'inject': entries, 'tasks': [task_name]})
with manager.acquire_lock():
manager.scheduler.start(run_schedules=False)
manager.shutdown()
def on_task_input(self, task):
if not task.manager.options.inject:
return
options = self.parse_arguments(task.manager.options.inject)
# disable other inputs
log.info('Disabling the rest of the input phase.')
task.disable_phase('input')
# create our injected entry
entry = Entry(options['entry'], injected=True)
if not 'url' in entry:
entry['url'] = 'http://localhost/inject/%s' % ''.join([random.choice(string.letters + string.digits) for x in range(1, 30)])
if entry.get('immortal'):
log.debug('Injected entry is immortal')
task.all_entries.append(entry)
if options.get('accept', False):
log.debug('accepting the injection')
entry.accept('--inject accepted')
def post(self, session=None):
""" Execute task and stream results """
data = request.json
for task in data.get("tasks"):
if task.lower() not in [t.lower() for t in self.manager.user_config.get("tasks", {}).keys()]:
return {"error": "task %s does not exist" % task}, 404
queue = ExecuteLog()
output = queue if data.get("loglevel") else None
stream = (
True
if any(arg[0] in ["progress", "summary", "loglevel", "entry_dump"] for arg in data.items() if arg[1])
else False
)
loglevel = data.pop("loglevel", None)
# This emulates the CLI command of using `--now`
options = {"interval_ignore": data.pop("now", None)}
for option, value in data.items():
options[option] = value
if data.get("inject"):
entries = []
for item in data.get("inject"):
entry = Entry()
entry["url"] = item["url"]
if not item.get("title"):
try:
value, params = cgi.parse_header(requests.head(item["url"]).headers["Content-Disposition"])
entry["title"] = params["filename"]
except KeyError:
return (
{
"status": "error",
"message": "No title given, and couldn't get one from the URL's HTTP response",
},
500,
)
else:
entry["title"] = item.get("title")
if item.get("force"):
entry["immortal"] = True
if item.get("accept"):
entry.accept(reason="accepted by API inject")
if item.get("fields"):
for key, value in item.get("fields").items():
entry[key] = value
entries.append(entry)
options["inject"] = entries
executed_tasks = self.manager.execute(options=options, output=output, loglevel=loglevel)
tasks_queued = []
for task_id, task_name, task_event in executed_tasks:
tasks_queued.append({"id": task_id, "name": task_name, "event": task_event})
_streams[task_id] = {"queue": queue, "last_update": datetime.now(), "args": data}
if not stream:
return jsonify({"tasks": [{"id": task["id"], "name": task["name"]} for task in tasks_queued]})
def stream_response():
# First return the tasks to execute
yield '{"stream": ['
yield json.dumps({"tasks": [{"id": task["id"], "name": task["name"]} for task in tasks_queued]}) + ",\n"
while True:
try:
yield queue.get(timeout=1) + ",\n"
continue
except Empty:
pass
if queue.empty() and all([task["event"].is_set() for task in tasks_queued]):
for task in tasks_queued:
del _streams[task["id"]]
break
yield "{}]}"
return Response(stream_response(), mimetype="text/event-stream")
def post(self, task, session=None):
""" Execute task and stream results """
data = request.json
if task.lower() not in [t.lower() for t in self.manager.user_config.get('tasks', {}).keys()]:
return {'error': 'task does not exist'}, 404
queue = ExecuteLog()
output = queue if data.get('log') else None
stream = True if any(
arg[0] in ['progress', 'summary', 'log', 'entry_dump'] for arg in data.items() if arg[1]) else False
loglevel = data.get('loglevel')
options = {'tasks': [task]}
if data.get('inject'):
entries = []
for item in data.get('inject'):
entry = Entry()
entry['url'] = item['url']
if not item.get('title'):
try:
value, params = cgi.parse_header(requests.head(item['url']).headers['Content-Disposition'])
entry['title'] = params['filename']
except KeyError:
return {'status': 'error',
'message': 'No title given, and couldn\'t get one from the URL\'s HTTP response'}, 500
else:
entry['title'] = item.get('title')
if item.get('force'):
entry['immortal'] = True
if item.get('accept'):
entry.accept(reason='accepted by API inject')
if item.get('fields'):
for key, value in item.get('fields').items():
entry[key] = value
entries.append(entry)
options['inject'] = entries
task_id, __, task_event = self.manager.execute(options=options, output=output, loglevel=loglevel)[0]
if not stream:
return {'task': {'id': task_id, 'name': task}}
_streams[task_id] = {
'queue': queue,
'last_update': datetime.now(),
'args': data
}
def stream_response():
yield '{"task": {"id": "%s", "name": "%s", "stream": [' % (task_id, task)
while True:
# If the server is shutting down then end the stream nicely
if cherrypy.engine.state != cherrypy.engine.states.STARTED:
break
try:
yield queue.get(timeout=1) + ',\n'
continue
except Empty:
pass
if queue.empty() and task_event.is_set():
del _streams[task_id]
break
yield '{}]}}'
return Response(stream_response(), mimetype='text/event-stream')
def post(self, session=None):
""" Execute task and stream results """
data = request.json
for task in data.get('tasks'):
if task.lower() not in [
t.lower()
for t in self.manager.user_config.get('tasks', {}).keys()
]:
return {'error': 'task %s does not exist' % task}, 404
queue = ExecuteLog()
output = queue if data.get('loglevel') else None
stream = True if any(
arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump']
for arg in data.items() if arg[1]) else False
loglevel = data.get('loglevel')
options = {'tasks': data.get('tasks')}
if data.get('inject'):
entries = []
for item in data.get('inject'):
entry = Entry()
entry['url'] = item['url']
if not item.get('title'):
try:
value, params = cgi.parse_header(
requests.head(
item['url']).headers['Content-Disposition'])
entry['title'] = params['filename']
except KeyError:
return {
'status':
'error',
'message':
'No title given, and couldn\'t get one from the URL\'s HTTP response'
}, 500
else:
entry['title'] = item.get('title')
if item.get('force'):
entry['immortal'] = True
if item.get('accept'):
entry.accept(reason='accepted by API inject')
if item.get('fields'):
for key, value in item.get('fields').items():
entry[key] = value
entries.append(entry)
options['inject'] = entries
executed_tasks = self.manager.execute(options=options,
output=output,
loglevel=loglevel)
tasks_queued = []
for task_id, task_name, task_event in executed_tasks:
tasks_queued.append({
'id': task_id,
'name': task_name,
'event': task_event
})
_streams[task_id] = {
'queue': queue,
'last_update': datetime.now(),
'args': data
}
if not stream:
return jsonify({
'tasks': [{
'id': task['id'],
'name': task['name']
} for task in tasks_queued]
})
def stream_response():
# First return the tasks to execute
yield '{"stream": ['
yield json.dumps({
'tasks': [{
'id': task['id'],
'name': task['name']
} for task in tasks_queued]
}) + ',\n'
while True:
try:
yield queue.get(timeout=1) + ',\n'
continue
except Empty:
pass
if queue.empty() and all(
[task['event'].is_set() for task in tasks_queued]):
for task in tasks_queued:
del _streams[task['id']]
break
yield '{}]}'
return Response(stream_response(), mimetype='text/event-stream')
def post(self, session: Session = None) -> Response:
""" Execute task and stream results """
data = request.json
for task in data.get('tasks'):
if task.lower() not in [
t.lower()
for t in self.manager.user_config.get('tasks', {}).keys()
]:
raise NotFoundError(f'task {task} does not exist')
queue = ExecuteLog()
output = queue if data.get('loglevel') else None
stream = (True if any(
arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump']
for arg in data.items() if arg[1]) else False)
loglevel = data.pop('loglevel', None)
if loglevel:
loglevel = loglevel.upper()
# This emulates the CLI command of using `--now` and `no-cache`
options = {
'interval_ignore': data.pop('now', None),
'nocache': data.pop('no_cache', None),
'allow_manual': True,
}
for option, value in data.items():
options[option] = value
if data.get('inject'):
entries = []
for item in data.get('inject'):
entry = Entry()
entry['url'] = item['url']
if not item.get('title'):
try:
value, params = cgi.parse_header(
requests.head(
item['url']).headers['Content-Disposition'])
entry['title'] = params['filename']
except KeyError:
raise BadRequest(
'No title given, and couldn\'t get one from the URL\'s HTTP response'
)
else:
entry['title'] = item.get('title')
if item.get('force'):
entry['immortal'] = True
if item.get('accept'):
entry.accept(reason='accepted by API inject')
if item.get('fields'):
for key, value in item.get('fields').items():
entry[key] = value
entries.append(entry)
options['inject'] = entries
if output:
with capture_console(output), capture_logs(output, level=loglevel):
executed_tasks = self.manager.execute(options=options)
else:
executed_tasks = self.manager.execute(options=options)
tasks_queued = []
for task_id, task_name, task_event in executed_tasks:
tasks_queued.append({
'id': task_id,
'name': task_name,
'event': task_event
})
_streams[task_id] = {
'queue': queue,
'last_update': datetime.now(),
'args': data
}
if not stream:
return jsonify({
'tasks': [{
'id': task['id'],
'name': task['name']
} for task in tasks_queued]
})
def stream_response():
# First return the tasks to execute
yield '{"stream": ['
yield json.dumps({
'tasks': [{
'id': task['id'],
'name': task['name']
} for task in tasks_queued]
}) + ',\n'
while True:
try:
yield queue.get(timeout=1) + ',\n'
continue
except Empty:
pass
if queue.empty() and all(
[task['event'].is_set() for task in tasks_queued]):
for task in tasks_queued:
del _streams[task['id']]
break
yield '{}]}'
return Response(stream_response(), mimetype='text/event-stream')
def post(self, session=None):
""" Execute task and stream results """
data = request.json
for task in data.get('tasks'):
if task.lower() not in [t.lower() for t in self.manager.user_config.get('tasks', {}).keys()]:
return {'error': 'task %s does not exist' % task}, 404
queue = ExecuteLog()
output = queue if data.get('loglevel') else None
stream = True if any(
arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump'] for arg in data.items() if arg[1]) else False
loglevel = data.get('loglevel')
options = {'tasks': data.get('tasks')}
if data.get('inject'):
entries = []
for item in data.get('inject'):
entry = Entry()
entry['url'] = item['url']
if not item.get('title'):
try:
value, params = cgi.parse_header(requests.head(item['url']).headers['Content-Disposition'])
entry['title'] = params['filename']
except KeyError:
return {'status': 'error',
'message': 'No title given, and couldn\'t get one from the URL\'s HTTP response'}, 500
else:
entry['title'] = item.get('title')
if item.get('force'):
entry['immortal'] = True
if item.get('accept'):
entry.accept(reason='accepted by API inject')
if item.get('fields'):
for key, value in item.get('fields').items():
entry[key] = value
entries.append(entry)
options['inject'] = entries
executed_tasks = self.manager.execute(options=options, output=output, loglevel=loglevel)
tasks_queued = []
for task_id, task_name, task_event in executed_tasks:
tasks_queued.append({'id': task_id, 'name': task_name, 'event': task_event})
_streams[task_id] = {
'queue': queue,
'last_update': datetime.now(),
'args': data
}
if not stream:
return jsonify({'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]})
def stream_response():
# First return the tasks to execute
yield '{"stream": ['
yield json.dumps({'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]}) + ',\n'
while True:
try:
yield queue.get(timeout=1) + ',\n'
continue
except Empty:
pass
if queue.empty() and all([task['event'].is_set() for task in tasks_queued]):
for task in tasks_queued:
del _streams[task['id']]
break
yield '{}]}'
return Response(stream_response(), mimetype='text/event-stream')
