def run_next_plugin(context, requirements=None, interactive=False,
clear_down=True, done_requirements=None, outputs=None):
discover = discover_single
if not interactive:
discover = discover.delay
to_run = plugin.next('discovery', requirements) - done_requirements
if to_run:
plugin_name = plugin.highest_priority('discovery', to_run)
discover(context, plugin_name, requirements, interactive, clear_down,
done_requirements)
return
to_run = plugin.next('postprocess', requirements) - done_requirements
for plugin_name in plugin.prioritize('postprocess', to_run):
_run_plugin(context, 'postprocess', plugin_name, requirements,
interactive, clear_down, done_requirements, outputs)
def run_next_plugin(context, requirements=None, interactive=False,
clear_down=True, done_requirements=None, outputs=None):
discover = discover_single
if not interactive:
discover = discover.delay
to_run = plugin.next('discovery', requirements) - done_requirements
if to_run:
plugin_name = plugin.highest_priority('discovery', to_run)
discover(context, plugin_name, requirements, interactive, clear_down,
done_requirements)
return
to_run = plugin.next('postprocess', requirements) - done_requirements
for plugin_name in plugin.prioritize('postprocess', to_run):
_run_plugin(context, 'postprocess', plugin_name, requirements,
interactive, clear_down, done_requirements, outputs)
def run_deployment(deployment):
if deployment.is_running:
return
deployment.is_running = True
deployment.save()
try:
done = set(name.strip() for name in deployment.done_plugins.split(',')
if name.strip())
tried = set(done)
while True:
plugins = plugin.next('deployment', done) - tried
if not plugins:
break
name = plugin.highest_priority('deployment', plugins)
tried.add(name)
try:
success = plugin.run(
'deployment',
name,
deployment_id=deployment.id,
)
except Exception:
traceback.print_exc()
else:
if success:
done.add(name)
# The plugins might have changed the deployment object.
deployment = Deployment.objects.get(id=deployment.id)
deployment.done_plugins = ', '.join(done)
deployment.save()
finally:
deployment.is_running = False
deployment.save()
def handle(self, today, run_only, *args, **options):
from ralph_pricing.plugins import collects # noqa
if today:
today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
else:
today = datetime.date.today()
if options.get('yesterday'):
today -= datetime.timedelta(days=1)
logger.info('Synchronizing for {0}.'.format(today.isoformat()))
if not run_only:
done = set()
tried = set()
while True:
to_run = plugin.next('pricing', done) - tried
if not to_run:
break
name = plugin.highest_priority('pricing', to_run)
tried.add(name)
if (name in settings.PRICING_COLLECT_PLUGINS
and self._run_plugin(name, today)):
done.add(name)
else:
self._run_plugin(run_only, today)
def handle(self, today, run_only, *args, **options):
from ralph_pricing import plugins # noqa
if today:
today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
else:
today = datetime.date.today()
print('Synchronizing for {0}.'.format(today.isoformat()))
if run_only:
print('Running only {0}...'.format(run_only))
success, message, context = plugin.run(
'pricing',
run_only,
today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
return
done = set()
tried = set()
while True:
to_run = plugin.next('pricing', done) - tried
if not to_run:
break
name = plugin.highest_priority('pricing', to_run)
tried.add(name)
print('Running {0}...'.format(name))
success, message, context = plugin.run(
'pricing',
name,
today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
if success:
done.add(name)
def handle(self, today, run_only, *args, **options):
from ralph_pricing.plugins import collects # noqa
if today:
today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
else:
today = datetime.date.today()
if options.get('yesterday'):
today -= datetime.timedelta(days=1)
logger.info('Synchronizing for {0}.'.format(today.isoformat()))
if not run_only:
done = set()
tried = set()
while True:
to_run = plugin.next('pricing', done) - tried
if not to_run:
break
name = plugin.highest_priority('pricing', to_run)
tried.add(name)
if (name in settings.PRICING_COLLECT_PLUGINS and
self._run_plugin(name, today)):
done.add(name)
else:
self._run_plugin(run_only, today)
def handle(self, today, run_only, *args, **options):
from ralph_pricing import plugins # noqa
if today:
today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
else:
today = datetime.date.today()
print('Synchronizing for {0}.'.format(today.isoformat()))
if run_only:
print('Running only {0}...'.format(run_only))
success, message, context = plugin.run(
'pricing',
run_only,
today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
return
done = set()
tried = set()
while True:
to_run = plugin.next('pricing', done) - tried
if not to_run:
break
name = plugin.highest_priority('pricing', to_run)
tried.add(name)
print('Running {0}...'.format(name))
success, message, context = plugin.run(
'pricing', name, today=today,
)
print('{1}: {0}'.format(message, 'Done' if success else 'Failed'))
if success:
done.add(name)
def discover_single_synchro(ip):
requirements = set()
done_requirements = set()
to_run = ['ping']
context = {'ip': ip}
messages = []
def output(*args, **kwargs):
messages.append(''.join(args))
messages.append(kwargs.get('end', '\n'))
outputs = (output, output, output)
while to_run:
messages[:] = []
plugin_name = plugin.highest_priority('discovery', to_run)
for retry in range(5):
try:
_run_plugin(context, 'discovery', plugin_name, requirements,
True, False, done_requirements, outputs)
except plugin.Restart:
pass
else:
break
else:
messages.append('Plugin failed after %d retries.\n' % retry)
to_run = plugin.next('discovery', requirements) - done_requirements
yield ''.join(messages)
yield '\n'
to_run = plugin.next('postprocess', requirements) - done_requirements
for plugin_name in plugin.prioritize('postprocess', to_run):
messages[:] = []
_run_plugin(context, 'postprocess', plugin_name, requirements, True,
False, done_requirements, outputs)
yield ''.join(messages)
yield '\n'
yield 'Finished.\n'
def discover_single_synchro(ip):
"""Run discovery of a single host in a synchronous way."""
requirements = set()
done_requirements = set()
to_run = ['ping']
context = {'ip': ip}
messages = []
def output(*args, **kwargs):
messages.append(''.join(args))
messages.append(kwargs.get('end', '\n'))
outputs = (output, output, output)
while to_run:
messages[:] = []
plugin_name = plugin.highest_priority('discovery', to_run)
for retry in range(5):
try:
_run_plugin(context, 'discovery', plugin_name, requirements,
True, False, done_requirements, outputs)
except plugin.Restart:
pass
else:
break
else:
messages.append('Plugin failed after %d retries.\n' % retry)
to_run = plugin.next('discovery', requirements) - done_requirements
yield ''.join(messages)
yield '\n'
to_run = plugin.next('postprocess', requirements) - done_requirements
for plugin_name in plugin.prioritize('postprocess', to_run):
messages[:] = []
_run_plugin(context, 'postprocess', plugin_name, requirements,
True, False, done_requirements, outputs)
yield ''.join(messages)
yield '\n'
yield 'Finished.\n'
def run_chain(context, chain_name, requirements=None, interactive=False,
clear_down=True, done_requirements=None, outputs=None):
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
to_run = plugin.next(chain_name, requirements) - done_requirements
if not to_run:
return
plugin_name = plugin.highest_priority(chain_name, to_run)
_run_plugin(context, chain_name, plugin_name, requirements, interactive,
clear_down, done_requirements, outputs)
run_chain(context, chain_name, requirements, interactive, clear_down,
done_requirements, outputs)
def run_chain(context, chain_name, requirements=None, interactive=False,
clear_down=True, done_requirements=None, outputs=None):
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
to_run = plugin.next(chain_name, requirements) - done_requirements
if not to_run:
return
plugin_name = plugin.highest_priority(chain_name, to_run)
_run_plugin(context, chain_name, plugin_name, requirements, interactive,
clear_down, done_requirements, outputs)
run_chain(context, chain_name, requirements, interactive, clear_down,
done_requirements, outputs)
def discover_single_synchro(ip):
requirements = set()
done_requirements = set()
to_run = ["ping"]
context = {"ip": ip}
messages = []
def output(*args, **kwargs):
messages.append("".join(args))
messages.append(kwargs.get("end", "\n"))
outputs = (output, output, output)
while to_run:
messages[:] = []
plugin_name = plugin.highest_priority("discovery", to_run)
for retry in range(5):
try:
_run_plugin(context, "discovery", plugin_name, requirements, True, False, done_requirements, outputs)
except plugin.Restart:
pass
else:
break
else:
messages.append("Plugin failed after %d retries.\n" % retry)
to_run = plugin.next("discovery", requirements) - done_requirements
yield "".join(messages)
yield "\n"
to_run = plugin.next("postprocess", requirements) - done_requirements
for plugin_name in plugin.prioritize("postprocess", to_run):
messages[:] = []
_run_plugin(context, "postprocess", plugin_name, requirements, True, False, done_requirements, outputs)
yield "".join(messages)
yield "\n"
yield "Finished.\n"
def run_next_plugin(context, chains, requirements=None, interactive=False,
done_requirements=None, outputs=None, after=None):
"""Runs the next plugin, asynchronously if interactive=False is given."""
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
run = _select_run_method(context, interactive, run_plugin, after)
for index, chain in enumerate(chains):
to_run = plugin.next(chain, requirements) - done_requirements
if to_run:
plugin_name = plugin.highest_priority(chain, to_run)
run(context, chains[index:], plugin_name, requirements,
interactive, done_requirements, outputs)
return
def _run_chain(context, chain_name, requirements=None, interactive=False,
done_requirements=None, outputs=None):
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
to_run = plugin.next(chain_name, requirements) - done_requirements
if not to_run:
return
plugin_name = plugin.highest_priority(chain_name, to_run)
try:
_run_plugin(context, chain_name, plugin_name, requirements,
interactive, done_requirements, outputs)
finally:
run_chain(context, chain_name, requirements, interactive,
done_requirements, outputs)
def _run_chain(context, chain_name, requirements=None, interactive=False,
done_requirements=None, outputs=None):
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
to_run = plugin.next(chain_name, requirements) - done_requirements
if not to_run:
return
plugin_name = plugin.highest_priority(chain_name, to_run)
try:
_run_plugin(context, chain_name, plugin_name, requirements,
interactive, done_requirements, outputs)
finally:
run_chain(context, chain_name, requirements, interactive,
done_requirements, outputs)
def run_next_plugin(context, chains, requirements=None, interactive=False,
done_requirements=None, outputs=None, after=None):
"""Runs the next plugin, asynchronously if interactive=False is given."""
if requirements is None:
requirements = set()
if done_requirements is None:
done_requirements = set()
run = _select_run_method(context, interactive, run_plugin, after)
for index, chain in enumerate(chains):
to_run = plugin.next(chain, requirements) - done_requirements
if to_run:
plugin_name = plugin.highest_priority(chain, to_run)
run(context, chains[index:], plugin_name, requirements,
interactive, done_requirements, outputs)
return
def handle(self, today, run_only, *args, **options):
setup_scrooge_logger()
from ralph_pricing.plugins import collects # noqa
if today:
today = datetime.datetime.strptime(today, '%Y-%m-%d').date()
else:
today = datetime.date.today()
logger.info('Synchronizing for {0}.'.format(today.isoformat()))
if not run_only:
done = set()
tried = set()
while True:
to_run = plugin.next('pricing', done) - tried
if not to_run:
break
name = plugin.highest_priority('pricing', to_run)
tried.add(name)
if self._run_plugin(name, today):
done.add(name)
else:
self._run_plugin(run_only, today)
def run_deployment(deployment):
if deployment.is_running:
return
deployment.is_running = True
deployment.save()
try:
done = set(name.strip() for name in deployment.done_plugins.split(','))
tried = set(done)
while True:
plugins = plugin.next('deployment', done) - tried
if not plugins:
break
name = plugin.highest_priority('deployment', plugins)
tried.add(name)
if plugin.run('deployment', name, deployment_id=deployment.id):
done.add(name)
deployment.done_plugins = ', '.join(done)
deployment.save()
finally:
deployment.is_running = False
deployment.save()
def run_deployment(deployment):
if deployment.is_running:
return
deployment.is_running = True
deployment.save()
try:
done = set(
name.strip() for
name in
deployment.done_plugins.split(',')
if name.strip()
)
tried = set(done)
while True:
plugins = plugin.next('deployment', done) - tried
if not plugins:
break
name = plugin.highest_priority('deployment', plugins)
tried.add(name)
try:
success = plugin.run(
'deployment',
name,
deployment_id=deployment.id,
)
except Exception:
traceback.print_exc()
else:
if success:
done.add(name)
# The plugins might have changed the deployment object.
deployment = Deployment.objects.get(id=deployment.id)
deployment.done_plugins = ', '.join(done)
deployment.save()
finally:
deployment.is_running = False
deployment.save()
def run_plugins(today, plugins, run_only=False):
_load_plugins()
logger.info('Synchronizing for {0}.'.format(today.isoformat()))
done = set()
tried = set()
if run_only:
name = plugins[0]
try:
_run_plugin(name, today)
yield name, True
except Exception:
yield name, False
else:
while True:
to_run = plugin.next('scrooge', done) - tried
if not to_run:
break
name = plugin.highest_priority('scrooge', to_run)
tried.add(name)
if name in plugins:
try:
_run_plugin(name, today)
done.add(name)
yield name, True
except PluginError:
yield name, False
# save not executed plugins
for p in set(plugins) - tried:
status = SyncStatus.objects.get_or_create(
date=today,
plugin=p,
)[0]
status.success = False
status.remarks = 'Not executed'
status.save()
def run_plugins(today, plugins, run_only=False):
_load_plugins()
logger.info('Synchronizing for {0}.'.format(today.isoformat()))
done = set()
tried = set()
if run_only:
name = plugins[0]
try:
_run_plugin(name, today)
yield name, True
except Exception:
yield name, False
else:
while True:
to_run = plugin.next('scrooge', done) - tried
if not to_run:
break
name = plugin.highest_priority('scrooge', to_run)
tried.add(name)
if name in plugins:
try:
_run_plugin(name, today)
done.add(name)
yield name, True
except PluginError:
yield name, False
# save not executed plugins
for p in set(plugins) - tried:
status = SyncStatus.objects.get_or_create(
date=today,
plugin=p,
)[0]
status.success = False
status.remarks = 'Not executed'
status.save()
