def handle(self, today, run_only, *args, **options): from ralph_pricing.plugins import collects # noqa if today: today = datetime.datetime.strptime(today, '%Y-%m-%d').date() else: today = datetime.date.today() if options.get('yesterday'): today -= datetime.timedelta(days=1) logger.info('Synchronizing for {0}.'.format(today.isoformat())) if not run_only: done = set() tried = set() while True: to_run = plugin.next('pricing', done) - tried if not to_run: break name = plugin.highest_priority('pricing', to_run) tried.add(name) if (name in settings.PRICING_COLLECT_PLUGINS and self._run_plugin(name, today)): done.add(name) else: self._run_plugin(run_only, today)
def handle(self, today, run_only, *args, **options): from ralph_pricing import plugins # noqa if today: today = datetime.datetime.strptime(today, '%Y-%m-%d').date() else: today = datetime.date.today() print('Synchronizing for {0}.'.format(today.isoformat())) if run_only: print('Running only {0}...'.format(run_only)) success, message, context = plugin.run( 'pricing', run_only, today=today, ) print('{1}: {0}'.format(message, 'Done' if success else 'Failed')) return done = set() tried = set() while True: to_run = plugin.next('pricing', done) - tried if not to_run: break name = plugin.highest_priority('pricing', to_run) tried.add(name) print('Running {0}...'.format(name)) success, message, context = plugin.run( 'pricing', name, today=today, ) print('{1}: {0}'.format(message, 'Done' if success else 'Failed')) if success: done.add(name)
def run_deployment(deployment): if deployment.is_running: return deployment.is_running = True deployment.save() try: done = set(name.strip() for name in deployment.done_plugins.split(',') if name.strip()) tried = set(done) while True: plugins = plugin.next('deployment', done) - tried if not plugins: break name = plugin.highest_priority('deployment', plugins) tried.add(name) try: success = plugin.run( 'deployment', name, deployment_id=deployment.id, ) except Exception: traceback.print_exc() else: if success: done.add(name) # The plugins might have changed the deployment object. deployment = Deployment.objects.get(id=deployment.id) deployment.done_plugins = ', '.join(done) deployment.save() finally: deployment.is_running = False deployment.save()
def run_chain(context, chain_name, requirements=None, interactive=False, clear_down=True, done_requirements=None, outputs=None): if requirements is None: requirements = set() if done_requirements is None: done_requirements = set() to_run = plugin.next(chain_name, requirements) - done_requirements if not to_run: return plugin_name = plugin.highest_priority(chain_name, to_run) _run_plugin(context, chain_name, plugin_name, requirements, interactive, clear_down, done_requirements, outputs) run_chain(context, chain_name, requirements, interactive, clear_down, done_requirements, outputs)
def run_next_plugin(context, requirements=None, interactive=False, clear_down=True, done_requirements=None, outputs=None): discover = discover_single if not interactive: discover = discover.delay to_run = plugin.next('discovery', requirements) - done_requirements if to_run: plugin_name = plugin.highest_priority('discovery', to_run) discover(context, plugin_name, requirements, interactive, clear_down, done_requirements) return to_run = plugin.next('postprocess', requirements) - done_requirements for plugin_name in plugin.prioritize('postprocess', to_run): _run_plugin(context, 'postprocess', plugin_name, requirements, interactive, clear_down, done_requirements, outputs)
def _run_chain(context, chain_name, requirements=None, interactive=False, done_requirements=None, outputs=None): if requirements is None: requirements = set() if done_requirements is None: done_requirements = set() to_run = plugin.next(chain_name, requirements) - done_requirements if not to_run: return plugin_name = plugin.highest_priority(chain_name, to_run) try: _run_plugin(context, chain_name, plugin_name, requirements, interactive, done_requirements, outputs) finally: run_chain(context, chain_name, requirements, interactive, done_requirements, outputs)
def run_next_plugin(context, chains, requirements=None, interactive=False, done_requirements=None, outputs=None, after=None): """Runs the next plugin, asynchronously if interactive=False is given.""" if requirements is None: requirements = set() if done_requirements is None: done_requirements = set() run = _select_run_method(context, interactive, run_plugin, after) for index, chain in enumerate(chains): to_run = plugin.next(chain, requirements) - done_requirements if to_run: plugin_name = plugin.highest_priority(chain, to_run) run(context, chains[index:], plugin_name, requirements, interactive, done_requirements, outputs) return
def handle(self, today, run_only, *args, **options): setup_scrooge_logger() from ralph_pricing.plugins import collects # noqa if today: today = datetime.datetime.strptime(today, '%Y-%m-%d').date() else: today = datetime.date.today() logger.info('Synchronizing for {0}.'.format(today.isoformat())) if not run_only: done = set() tried = set() while True: to_run = plugin.next('pricing', done) - tried if not to_run: break name = plugin.highest_priority('pricing', to_run) tried.add(name) if self._run_plugin(name, today): done.add(name) else: self._run_plugin(run_only, today)
def run_deployment(deployment): if deployment.is_running: return deployment.is_running = True deployment.save() try: done = set(name.strip() for name in deployment.done_plugins.split(',')) tried = set(done) while True: plugins = plugin.next('deployment', done) - tried if not plugins: break name = plugin.highest_priority('deployment', plugins) tried.add(name) if plugin.run('deployment', name, deployment_id=deployment.id): done.add(name) deployment.done_plugins = ', '.join(done) deployment.save() finally: deployment.is_running = False deployment.save()
def discover_single_synchro(ip): requirements = set() done_requirements = set() to_run = ['ping'] context = {'ip': ip} messages = [] def output(*args, **kwargs): messages.append(''.join(args)) messages.append(kwargs.get('end', '\n')) outputs = (output, output, output) while to_run: messages[:] = [] plugin_name = plugin.highest_priority('discovery', to_run) for retry in range(5): try: _run_plugin(context, 'discovery', plugin_name, requirements, True, False, done_requirements, outputs) except plugin.Restart: pass else: break else: messages.append('Plugin failed after %d retries.\n' % retry) to_run = plugin.next('discovery', requirements) - done_requirements yield ''.join(messages) yield '\n' to_run = plugin.next('postprocess', requirements) - done_requirements for plugin_name in plugin.prioritize('postprocess', to_run): messages[:] = [] _run_plugin(context, 'postprocess', plugin_name, requirements, True, False, done_requirements, outputs) yield ''.join(messages) yield '\n' yield 'Finished.\n'
def discover_single_synchro(ip): """Run discovery of a single host in a synchronous way.""" requirements = set() done_requirements = set() to_run = ['ping'] context = {'ip': ip} messages = [] def output(*args, **kwargs): messages.append(''.join(args)) messages.append(kwargs.get('end', '\n')) outputs = (output, output, output) while to_run: messages[:] = [] plugin_name = plugin.highest_priority('discovery', to_run) for retry in range(5): try: _run_plugin(context, 'discovery', plugin_name, requirements, True, False, done_requirements, outputs) except plugin.Restart: pass else: break else: messages.append('Plugin failed after %d retries.\n' % retry) to_run = plugin.next('discovery', requirements) - done_requirements yield ''.join(messages) yield '\n' to_run = plugin.next('postprocess', requirements) - done_requirements for plugin_name in plugin.prioritize('postprocess', to_run): messages[:] = [] _run_plugin(context, 'postprocess', plugin_name, requirements, True, False, done_requirements, outputs) yield ''.join(messages) yield '\n' yield 'Finished.\n'
def run_deployment(deployment): if deployment.is_running: return deployment.is_running = True deployment.save() try: done = set( name.strip() for name in deployment.done_plugins.split(',') if name.strip() ) tried = set(done) while True: plugins = plugin.next('deployment', done) - tried if not plugins: break name = plugin.highest_priority('deployment', plugins) tried.add(name) try: success = plugin.run( 'deployment', name, deployment_id=deployment.id, ) except Exception: traceback.print_exc() else: if success: done.add(name) # The plugins might have changed the deployment object. deployment = Deployment.objects.get(id=deployment.id) deployment.done_plugins = ', '.join(done) deployment.save() finally: deployment.is_running = False deployment.save()
def discover_single_synchro(ip): requirements = set() done_requirements = set() to_run = ["ping"] context = {"ip": ip} messages = [] def output(*args, **kwargs): messages.append("".join(args)) messages.append(kwargs.get("end", "\n")) outputs = (output, output, output) while to_run: messages[:] = [] plugin_name = plugin.highest_priority("discovery", to_run) for retry in range(5): try: _run_plugin(context, "discovery", plugin_name, requirements, True, False, done_requirements, outputs) except plugin.Restart: pass else: break else: messages.append("Plugin failed after %d retries.\n" % retry) to_run = plugin.next("discovery", requirements) - done_requirements yield "".join(messages) yield "\n" to_run = plugin.next("postprocess", requirements) - done_requirements for plugin_name in plugin.prioritize("postprocess", to_run): messages[:] = [] _run_plugin(context, "postprocess", plugin_name, requirements, True, False, done_requirements, outputs) yield "".join(messages) yield "\n" yield "Finished.\n"
def run_plugins(today, plugins, run_only=False): _load_plugins() logger.info('Synchronizing for {0}.'.format(today.isoformat())) done = set() tried = set() if run_only: name = plugins[0] try: _run_plugin(name, today) yield name, True except Exception: yield name, False else: while True: to_run = plugin.next('scrooge', done) - tried if not to_run: break name = plugin.highest_priority('scrooge', to_run) tried.add(name) if name in plugins: try: _run_plugin(name, today) done.add(name) yield name, True except PluginError: yield name, False # save not executed plugins for p in set(plugins) - tried: status = SyncStatus.objects.get_or_create( date=today, plugin=p, )[0] status.success = False status.remarks = 'Not executed' status.save()