def do_cli(manager, options): if not options.url: # Determine if first positional argument is a URL or a title if '://' in options.title: options.url = options.title options.title = None if options.url and not options.title: # Attempt to get a title from the URL response's headers try: value, params = cgi.parse_header( requests.head(options.url).headers['Content-Disposition']) options.title = params['filename'] except KeyError: console( 'No title given, and couldn\'t get one from the URL\'s HTTP response. Aborting.' ) return entry = Entry(title=options.title) if options.url: entry['url'] = options.url else: entry['url'] = 'http://localhost/inject/%s' % ''.join( random.sample(string.letters + string.digits, 30)) if options.force: entry['immortal'] = True if options.accept: entry.accept(reason='accepted by CLI inject') if options.fields: for key, value in options.fields: entry[key] = value options.inject = [entry] manager.execute_command(options)
def on_task_input(self, task): if not task.manager.options.inject: return options = self.parse_arguments(task.manager.options.inject) # disable other inputs log.info('Disabling the rest of the input phase.') task.disable_phase('input') # create our injected entry entry = Entry(options['entry'], injected=True) if not 'url' in entry: entry['url'] = 'http://localhost/inject/%s' % ''.join([ random.choice(string.letters + string.digits) for x in range(1, 30) ]) if entry.get('immortal'): log.debug('Injected entry is immortal') task.all_entries.append(entry) if options.get('accept', False): log.debug('accepting the injection') entry.accept('--inject accepted')
def do_cli(manager, options): if not options.url: # Determine if first positional argument is a URL or a title if '://' in options.title: options.url = options.title options.title = None if options.url and not options.title: # Attempt to get a title from the URL response's headers try: value, params = cgi.parse_header(requests.head(options.url).headers['Content-Disposition']) options.title = params['filename'] except KeyError: console('No title given, and couldn\'t get one from the URL\'s HTTP response. Aborting.') return entry = Entry(title=options.title) if options.url: entry['url'] = options.url else: entry['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30)) if options.force: entry['immortal'] = True if options.accept: entry.accept(reason='accepted by CLI inject') if options.fields: for key, value in options.fields: entry[key] = value options.inject = [entry] manager.execute_command(options)
def __call__(self, parser, namespace, values, option_string=None): kwargs = {'title': values.pop(0)} if values: kwargs['url'] = values.pop(0) else: kwargs['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30)) if 'force' in [v.lower() for v in values]: kwargs['immortal'] = True entry = Entry(**kwargs) if 'accept' in [v.lower() for v in values]: entry.accept(reason='accepted by --inject') setattr(namespace, self.dest, [entry])
def __call__(self, parser, namespace, values, option_string=None): kwargs = {"title": values.pop(0)} if values: kwargs["url"] = values.pop(0) else: kwargs["url"] = "http://localhost/inject/%s" % "".join(random.sample(string.letters + string.digits, 30)) if "force" in [v.lower() for v in values]: kwargs["immortal"] = True entry = Entry(**kwargs) if "accept" in [v.lower() for v in values]: entry.accept(reason="accepted by --inject") setattr(namespace, self.dest, [entry])
def __call__(self, parser, namespace, values, option_string=None): kwargs = {'title': values.pop(0)} if values: kwargs['url'] = values.pop(0) else: kwargs['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30)) if 'force' in [v.lower() for v in values]: kwargs['immortal'] = True entry = Entry(**kwargs) if 'accept' in [v.lower() for v in values]: entry.accept(reason='accepted by --inject') existing = getattr(namespace, self.dest, None) or [] setattr(namespace, self.dest, existing + [entry])
def do_cli(manager, options): entry = Entry(title=options.title) if options.url: entry['url'] = options.url else: entry['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30)) if options.force: entry['immortal'] = True if options.accept: entry.accept(reason='accepted by CLI inject') exec_options = dict(options.execute) exec_options['inject'] = [entry] manager.execute_command(exec_options)
def cli_inject(manager, options): logger.debug('Finding inject content') inject_entries = defaultdict(list) with Session() as session: for id in options.ids: archive_entry = session.query( flexget.components.archive.db.ArchiveEntry).get(id) # not found if not archive_entry: logger.critical("There's no archived item with ID `{}`", id) continue # find if there is no longer any task within sources if not any(source.name in manager.tasks for source in archive_entry.sources): logger.error( 'None of sources ({}) exists anymore, cannot inject `{}` from archive!', ', '.join([s.name for s in archive_entry.sources]), archive_entry.title, ) continue inject_entry = Entry(archive_entry.title, archive_entry.url) if archive_entry.description: inject_entry['description'] = archive_entry.description if options.immortal: logger.debug('Injecting as immortal') inject_entry['immortal'] = True inject_entry['accepted_by'] = 'archive inject' inject_entry.accept('injected') # update list of tasks to be injected for source in archive_entry.sources: inject_entries[source.name].append(inject_entry) for task_name in inject_entries: for inject_entry in inject_entries[task_name]: logger.info('Injecting from archive `{}` into `{}`', inject_entry['title'], task_name) for index, task_name in enumerate(inject_entries): options.inject = inject_entries[task_name] options.tasks = [task_name] # TODO: This is a bit hacky, consider a better way if index == len(inject_entries) - 1: # We use execute_command on the last item, rather than regular execute, to start FlexGet running. break manager.execute(options) manager.execute_command(options)
def cli_inject(manager, options): log.debug('Finding inject content') inject_entries = defaultdict(list) session = Session() try: for id in options.ids: archive_entry = session.query(ArchiveEntry).get(id) # not found if not archive_entry: log.critical('There\'s no archived item with ID `%s`' % id) continue # find if there is no longer any task within sources if not any(source.name in manager.tasks for source in archive_entry.sources): log.error( 'None of sources (%s) exists anymore, cannot inject `%s` from archive!' % (', '.join([s.name for s in archive_entry.sources ]), archive_entry.title)) continue # update list of tasks to be injected for source in archive_entry.sources: inject_entries[source.name].append(archive_entry) finally: session.close() for task_name in inject_entries: entries = [] for inject_entry in inject_entries[task_name]: log.info('Injecting from archive `%s`' % inject_entry.title) entry = Entry(inject_entry.title, inject_entry.url) if inject_entry.description: entry['description'] = inject_entry.description if options.immortal: log.debug('Injecting as immortal') entry['immortal'] = True entry['accepted_by'] = 'archive inject' entry.accept('injected') entries.append(entry) manager.scheduler.execute(options={ 'inject': entries, 'tasks': [task_name] }) with manager.acquire_lock(): manager.scheduler.start(run_schedules=False) manager.shutdown()
def do_cli(manager, options): entry = Entry(title=options.title) if options.url: entry['url'] = options.url else: entry['url'] = 'http://localhost/inject/%s' % ''.join(random.sample(string.letters + string.digits, 30)) if options.force: entry['immortal'] = True if options.accept: entry.accept(reason='accepted by CLI inject') if options.fields: for key, value in options.fields: entry[key] = value options.inject = [entry] manager.execute_command(options)
def cli_inject(manager, options): log.debug("Finding inject content") inject_entries = defaultdict(list) session = Session() try: for id in options.ids: archive_entry = session.query(ArchiveEntry).get(id) # not found if not archive_entry: log.critical("There's no archived item with ID `%s`" % id) continue # find if there is no longer any task within sources if not any(source.name in manager.tasks for source in archive_entry.sources): log.error( "None of sources (%s) exists anymore, cannot inject `%s` from archive!" % (", ".join([s.name for s in archive_entry.sources]), archive_entry.title) ) continue # update list of tasks to be injected for source in archive_entry.sources: inject_entries[source.name].append(archive_entry) finally: session.close() for task_name in inject_entries: entries = [] for inject_entry in inject_entries[task_name]: log.info("Injecting from archive `%s`" % inject_entry.title) entry = Entry(inject_entry.title, inject_entry.url) if inject_entry.description: entry["description"] = inject_entry.description if options.immortal: log.debug("Injecting as immortal") entry["immortal"] = True entry["accepted_by"] = "archive inject" entry.accept("injected") entries.append(entry) manager.execute(options={"inject": entries, "tasks": [task_name]}) with manager.acquire_lock(): manager.shutdown(finish_queue=True) manager.run()
def cli_inject(manager, options): log.debug("Finding inject content") inject_entries = defaultdict(list) with Session() as session: for id in options.ids: archive_entry = session.query(ArchiveEntry).get(id) # not found if not archive_entry: log.critical("There's no archived item with ID `%s`" % id) continue # find if there is no longer any task within sources if not any(source.name in manager.tasks for source in archive_entry.sources): log.error( "None of sources (%s) exists anymore, cannot inject `%s` from archive!" % (", ".join([s.name for s in archive_entry.sources]), archive_entry.title) ) continue inject_entry = Entry(archive_entry.title, archive_entry.url) if archive_entry.description: inject_entry["description"] = archive_entry.description if options.immortal: log.debug("Injecting as immortal") inject_entry["immortal"] = True inject_entry["accepted_by"] = "archive inject" inject_entry.accept("injected") # update list of tasks to be injected for source in archive_entry.sources: inject_entries[source.name].append(inject_entry) for task_name in inject_entries: for inject_entry in inject_entries[task_name]: log.info("Injecting from archive `%s` into `%s`" % (inject_entry["title"], task_name)) for index, task_name in enumerate(inject_entries): options.inject = inject_entries[task_name] options.tasks = [task_name] # TODO: This is a bit hacky, consider a better way if index == len(inject_entries) - 1: # We use execute_command on the last item, rather than regular execute, to start FlexGet running. break manager.execute(options) manager.execute_command(options)
def cli_inject(manager, options): log.debug('Finding inject content') inject_entries = defaultdict(list) session = Session() try: for id in options.ids: archive_entry = session.query(ArchiveEntry).get(id) # not found if not archive_entry: log.critical('There\'s no archived item with ID `%s`' % id) continue # find if there is no longer any task within sources if not any(source.name in manager.tasks for source in archive_entry.sources): log.error('None of sources (%s) exists anymore, cannot inject `%s` from archive!' % (', '.join([s.name for s in archive_entry.sources]), archive_entry.title)) continue # update list of tasks to be injected for source in archive_entry.sources: inject_entries[source.name].append(archive_entry) finally: session.close() for task_name in inject_entries: entries = [] for inject_entry in inject_entries[task_name]: log.info('Injecting from archive `%s`' % inject_entry.title) entry = Entry(inject_entry.title, inject_entry.url) if inject_entry.description: entry['description'] = inject_entry.description if options.immortal: log.debug('Injecting as immortal') entry['immortal'] = True entry['accepted_by'] = 'archive inject' entry.accept('injected') entries.append(entry) manager.scheduler.execute(options={'inject': entries, 'tasks': [task_name]}) with manager.acquire_lock(): manager.scheduler.start(run_schedules=False) manager.shutdown()
def on_task_input(self, task): if not task.manager.options.inject: return options = self.parse_arguments(task.manager.options.inject) # disable other inputs log.info('Disabling the rest of the input phase.') task.disable_phase('input') # create our injected entry entry = Entry(options['entry'], injected=True) if not 'url' in entry: entry['url'] = 'http://localhost/inject/%s' % ''.join([random.choice(string.letters + string.digits) for x in range(1, 30)]) if entry.get('immortal'): log.debug('Injected entry is immortal') task.all_entries.append(entry) if options.get('accept', False): log.debug('accepting the injection') entry.accept('--inject accepted')
def post(self, session=None): """ Execute task and stream results """ data = request.json for task in data.get("tasks"): if task.lower() not in [t.lower() for t in self.manager.user_config.get("tasks", {}).keys()]: return {"error": "task %s does not exist" % task}, 404 queue = ExecuteLog() output = queue if data.get("loglevel") else None stream = ( True if any(arg[0] in ["progress", "summary", "loglevel", "entry_dump"] for arg in data.items() if arg[1]) else False ) loglevel = data.pop("loglevel", None) # This emulates the CLI command of using `--now` options = {"interval_ignore": data.pop("now", None)} for option, value in data.items(): options[option] = value if data.get("inject"): entries = [] for item in data.get("inject"): entry = Entry() entry["url"] = item["url"] if not item.get("title"): try: value, params = cgi.parse_header(requests.head(item["url"]).headers["Content-Disposition"]) entry["title"] = params["filename"] except KeyError: return ( { "status": "error", "message": "No title given, and couldn't get one from the URL's HTTP response", }, 500, ) else: entry["title"] = item.get("title") if item.get("force"): entry["immortal"] = True if item.get("accept"): entry.accept(reason="accepted by API inject") if item.get("fields"): for key, value in item.get("fields").items(): entry[key] = value entries.append(entry) options["inject"] = entries executed_tasks = self.manager.execute(options=options, output=output, loglevel=loglevel) tasks_queued = [] for task_id, task_name, task_event in executed_tasks: tasks_queued.append({"id": task_id, "name": task_name, "event": task_event}) _streams[task_id] = {"queue": queue, "last_update": datetime.now(), "args": data} if not stream: return jsonify({"tasks": [{"id": task["id"], "name": task["name"]} for task in tasks_queued]}) def stream_response(): # First return the tasks to execute yield '{"stream": [' yield json.dumps({"tasks": [{"id": task["id"], "name": task["name"]} for task in tasks_queued]}) + ",\n" while True: try: yield queue.get(timeout=1) + ",\n" continue except Empty: pass if queue.empty() and all([task["event"].is_set() for task in tasks_queued]): for task in tasks_queued: del _streams[task["id"]] break yield "{}]}" return Response(stream_response(), mimetype="text/event-stream")
def post(self, task, session=None): """ Execute task and stream results """ data = request.json if task.lower() not in [t.lower() for t in self.manager.user_config.get('tasks', {}).keys()]: return {'error': 'task does not exist'}, 404 queue = ExecuteLog() output = queue if data.get('log') else None stream = True if any( arg[0] in ['progress', 'summary', 'log', 'entry_dump'] for arg in data.items() if arg[1]) else False loglevel = data.get('loglevel') options = {'tasks': [task]} if data.get('inject'): entries = [] for item in data.get('inject'): entry = Entry() entry['url'] = item['url'] if not item.get('title'): try: value, params = cgi.parse_header(requests.head(item['url']).headers['Content-Disposition']) entry['title'] = params['filename'] except KeyError: return {'status': 'error', 'message': 'No title given, and couldn\'t get one from the URL\'s HTTP response'}, 500 else: entry['title'] = item.get('title') if item.get('force'): entry['immortal'] = True if item.get('accept'): entry.accept(reason='accepted by API inject') if item.get('fields'): for key, value in item.get('fields').items(): entry[key] = value entries.append(entry) options['inject'] = entries task_id, __, task_event = self.manager.execute(options=options, output=output, loglevel=loglevel)[0] if not stream: return {'task': {'id': task_id, 'name': task}} _streams[task_id] = { 'queue': queue, 'last_update': datetime.now(), 'args': data } def stream_response(): yield '{"task": {"id": "%s", "name": "%s", "stream": [' % (task_id, task) while True: # If the server is shutting down then end the stream nicely if cherrypy.engine.state != cherrypy.engine.states.STARTED: break try: yield queue.get(timeout=1) + ',\n' continue except Empty: pass if queue.empty() and task_event.is_set(): del _streams[task_id] break yield '{}]}}' return Response(stream_response(), mimetype='text/event-stream')
def post(self, session=None): """ Execute task and stream results """ data = request.json for task in data.get('tasks'): if task.lower() not in [ t.lower() for t in self.manager.user_config.get('tasks', {}).keys() ]: return {'error': 'task %s does not exist' % task}, 404 queue = ExecuteLog() output = queue if data.get('loglevel') else None stream = True if any( arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump'] for arg in data.items() if arg[1]) else False loglevel = data.get('loglevel') options = {'tasks': data.get('tasks')} if data.get('inject'): entries = [] for item in data.get('inject'): entry = Entry() entry['url'] = item['url'] if not item.get('title'): try: value, params = cgi.parse_header( requests.head( item['url']).headers['Content-Disposition']) entry['title'] = params['filename'] except KeyError: return { 'status': 'error', 'message': 'No title given, and couldn\'t get one from the URL\'s HTTP response' }, 500 else: entry['title'] = item.get('title') if item.get('force'): entry['immortal'] = True if item.get('accept'): entry.accept(reason='accepted by API inject') if item.get('fields'): for key, value in item.get('fields').items(): entry[key] = value entries.append(entry) options['inject'] = entries executed_tasks = self.manager.execute(options=options, output=output, loglevel=loglevel) tasks_queued = [] for task_id, task_name, task_event in executed_tasks: tasks_queued.append({ 'id': task_id, 'name': task_name, 'event': task_event }) _streams[task_id] = { 'queue': queue, 'last_update': datetime.now(), 'args': data } if not stream: return jsonify({ 'tasks': [{ 'id': task['id'], 'name': task['name'] } for task in tasks_queued] }) def stream_response(): # First return the tasks to execute yield '{"stream": [' yield json.dumps({ 'tasks': [{ 'id': task['id'], 'name': task['name'] } for task in tasks_queued] }) + ',\n' while True: try: yield queue.get(timeout=1) + ',\n' continue except Empty: pass if queue.empty() and all( [task['event'].is_set() for task in tasks_queued]): for task in tasks_queued: del _streams[task['id']] break yield '{}]}' return Response(stream_response(), mimetype='text/event-stream')
def post(self, session: Session = None) -> Response: """ Execute task and stream results """ data = request.json for task in data.get('tasks'): if task.lower() not in [ t.lower() for t in self.manager.user_config.get('tasks', {}).keys() ]: raise NotFoundError(f'task {task} does not exist') queue = ExecuteLog() output = queue if data.get('loglevel') else None stream = (True if any( arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump'] for arg in data.items() if arg[1]) else False) loglevel = data.pop('loglevel', None) if loglevel: loglevel = loglevel.upper() # This emulates the CLI command of using `--now` and `no-cache` options = { 'interval_ignore': data.pop('now', None), 'nocache': data.pop('no_cache', None), 'allow_manual': True, } for option, value in data.items(): options[option] = value if data.get('inject'): entries = [] for item in data.get('inject'): entry = Entry() entry['url'] = item['url'] if not item.get('title'): try: value, params = cgi.parse_header( requests.head( item['url']).headers['Content-Disposition']) entry['title'] = params['filename'] except KeyError: raise BadRequest( 'No title given, and couldn\'t get one from the URL\'s HTTP response' ) else: entry['title'] = item.get('title') if item.get('force'): entry['immortal'] = True if item.get('accept'): entry.accept(reason='accepted by API inject') if item.get('fields'): for key, value in item.get('fields').items(): entry[key] = value entries.append(entry) options['inject'] = entries if output: with capture_console(output), capture_logs(output, level=loglevel): executed_tasks = self.manager.execute(options=options) else: executed_tasks = self.manager.execute(options=options) tasks_queued = [] for task_id, task_name, task_event in executed_tasks: tasks_queued.append({ 'id': task_id, 'name': task_name, 'event': task_event }) _streams[task_id] = { 'queue': queue, 'last_update': datetime.now(), 'args': data } if not stream: return jsonify({ 'tasks': [{ 'id': task['id'], 'name': task['name'] } for task in tasks_queued] }) def stream_response(): # First return the tasks to execute yield '{"stream": [' yield json.dumps({ 'tasks': [{ 'id': task['id'], 'name': task['name'] } for task in tasks_queued] }) + ',\n' while True: try: yield queue.get(timeout=1) + ',\n' continue except Empty: pass if queue.empty() and all( [task['event'].is_set() for task in tasks_queued]): for task in tasks_queued: del _streams[task['id']] break yield '{}]}' return Response(stream_response(), mimetype='text/event-stream')
def post(self, session=None): """ Execute task and stream results """ data = request.json for task in data.get('tasks'): if task.lower() not in [t.lower() for t in self.manager.user_config.get('tasks', {}).keys()]: return {'error': 'task %s does not exist' % task}, 404 queue = ExecuteLog() output = queue if data.get('loglevel') else None stream = True if any( arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump'] for arg in data.items() if arg[1]) else False loglevel = data.get('loglevel') options = {'tasks': data.get('tasks')} if data.get('inject'): entries = [] for item in data.get('inject'): entry = Entry() entry['url'] = item['url'] if not item.get('title'): try: value, params = cgi.parse_header(requests.head(item['url']).headers['Content-Disposition']) entry['title'] = params['filename'] except KeyError: return {'status': 'error', 'message': 'No title given, and couldn\'t get one from the URL\'s HTTP response'}, 500 else: entry['title'] = item.get('title') if item.get('force'): entry['immortal'] = True if item.get('accept'): entry.accept(reason='accepted by API inject') if item.get('fields'): for key, value in item.get('fields').items(): entry[key] = value entries.append(entry) options['inject'] = entries executed_tasks = self.manager.execute(options=options, output=output, loglevel=loglevel) tasks_queued = [] for task_id, task_name, task_event in executed_tasks: tasks_queued.append({'id': task_id, 'name': task_name, 'event': task_event}) _streams[task_id] = { 'queue': queue, 'last_update': datetime.now(), 'args': data } if not stream: return jsonify({'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]}) def stream_response(): # First return the tasks to execute yield '{"stream": [' yield json.dumps({'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]}) + ',\n' while True: try: yield queue.get(timeout=1) + ',\n' continue except Empty: pass if queue.empty() and all([task['event'].is_set() for task in tasks_queued]): for task in tasks_queued: del _streams[task['id']] break yield '{}]}' return Response(stream_response(), mimetype='text/event-stream')