def execute(self):
from noc.sa.models import ReduceTask,ManagedObject
from noc.ip.models import Address
# Get a list of managed objects to fetch ARP cache
objects=list(ManagedObject.objects.raw("""
SELECT DISTINCT a.managed_object_id AS id
FROM ip_address a JOIN ip_prefix p ON (a.prefix_id=p.id)
WHERE
a.managed_object_id IS NOT NULL
AND EXISTS (SELECT id FROM ip_address ma WHERE ma.afi='4' AND ma.auto_update_mac AND ma.prefix_id=p.id)
"""))
# Get a list of addresses to syncronize
addresses=list(Address.objects.raw("""
SELECT a.id,a.address,a.mac
FROM ip_address a JOIN ip_prefix p ON (a.prefix_id=p.id)
WHERE
a.auto_update_mac
AND a.afi='4'
AND EXISTS (SELECT id FROM ip_address oa WHERE oa.prefix_id=p.id AND oa.managed_object_id IS NOT NULL)
"""))
# Get ARP cache
task=ReduceTask.create_task(object_selector=objects,
reduce_script=sync_macs_reduce,
reduce_script_params={"addresses":addresses},
map_script="get_arp",
map_script_params={},
timeout=self.TIMEOUT)
# Wait for tasks completion
ReduceTask.wait_for_tasks([task])
return True
def api_run_mrt(self, request, name):
from noc.sa.models import ReduceTask, ManagedObjectSelector
# Check MRT configured
if name not in self.mrt_config:
return self.response_not_found("MRT %s is not found" % name)
# Check MRT access
if not self.check_mrt_access(request, name):
return self.response_forbidden("Forbidden")
#
data = json_decode(request.raw_post_data)
if "selector" not in data:
return self.response_bad_request("'selector' is missed")
# Run MRT
mc = self.mrt_config[name]
map_params = data.get("map_params", {})
map_params = dict((str(k), v) for k, v in map_params.iteritems())
objects = ManagedObjectSelector.resolve_expression(data["selector"])
task = ReduceTask.create_task(objects, "pyrule:mrt_result", {},
mc["map_script"], map_params,
mc.get("timeout", 0))
if mc["map_script"] == "commands" and settings.LOG_MRT_COMMAND:
# Log commands
now = datetime.datetime.now()
safe_append(
os.path.join(settings.LOG_MRT_COMMAND, "commands",
"%04d" % now.year, "%02d" % now.month,
"%02d.log" % now.day),
"%s\nDate: %s\nObjects: %s\nUser: %s\nCommands:\n%s\n" %
("-" * 72, now.isoformat(), ",".join(
str(o) for o in objects), request.user.username, " " +
"\n".join(map_params["commands"]).replace("\n", "\n ")))
return task.id
def execute(self):
from noc.peer.models import PrefixListCache
from noc.sa.models import ManagedObject, ReduceTask
#
prefix_lists = {} # PeeringPoint -> [prefix_lists]
# For all out-of-dated prefix lists
for pc in PrefixListCache.objects.filter():
if not pc.peering_point.enable_prefix_list_provisioning:
continue
if pc.pushed is not None and pc.pushed > pc.changed:
continue
if not pc.data:
continue
if pc.peering_point not in prefix_lists:
prefix_lists[pc.peering_point] = [pc]
else:
prefix_lists[pc.peering_point] += [pc]
# Run Map/Reduce tasks
tasks = []
for peering_point in prefix_lists:
# Try to find managed object
try:
pp = ManagedObject.objects.get(name=peering_point.hostname)
except ManagedObject.DoesNotExist:
logging.warning(
"No managed object for peering point '%s' found" %
peering_point.name)
continue
task = ReduceTask.create_task(
object_selector=[pp],
reduce_script="pyrule:prefix_list_provisioning",
reduce_script_params={"peering_point": peering_point},
map_script="sync_prefix_lists",
map_script_params={
"changed_prefix_lists": [{
"name": pl.name,
"strict": pl.strict,
"prefix_list": pl.data
} for pl in prefix_lists[peering_point]]
},
timeout=TIMEOUT)
tasks += [task]
# Wait for tasks completion
ReduceTask.wait_for_tasks(tasks)
return True
def execute(self):
from noc.vc.models import VCDomain
from noc.sa.models import ReduceTask
tasks = []
# Get config
for vc_domain in VCDomain.objects.filter(enable_provisioning=True):
# Get VCDomain vcs
vcs = [{
"vlan_id": vc.l1,
"name": vc.name
} for vc in vc_domain.vc_set.all()]
# Run Map/Reduce task
for c in vc_domain.vcdomainprovisioningconfig_set.filter(
is_enabled=True):
if c.vc_filter:
vc_list = [
v for v in vcs if c.vc_filter.check(v["vlan_id"])
]
else:
vc_list = vcs[:]
if not vc_list:
continue # Refuse to drop all vlans on switches
task = ReduceTask.create_task(
object_selector=c.selector,
reduce_script="pyrule:vc_provisioning",
reduce_script_params={"config": c},
map_script="sync_vlans",
map_script_params={
"vlans": vc_list,
"tagged_ports": c.tagged_ports_list
},
timeout=TIMEOUT)
tasks += [task]
# Wait for tasks completion
ReduceTask.wait_for_tasks(tasks)
return True
def view_run(self, request):
if request.POST:
form = self.MRTaskForm(request.POST)
if form.is_valid():
t = ReduceTask.create_task(
object_selector=form.cleaned_data["selector"],
reduce_script="pyrule:" +
form.cleaned_data["reduce_script"].name,
reduce_script_params=form.
cleaned_data["reduce_script_params"],
map_script=form.cleaned_data["map_script"],
map_script_params=form.cleaned_data["map_script_params"],
timeout=form.cleaned_data["timeout"])
return self.response_redirect_to_object(t)
else:
form = self.MRTaskForm(initial={"timeout": 180})
return self.render(request, "mr_task.html", {"form": form})
def run_task(self, snippet, objects, params):
def get_map_script_params(snippet, data):
def inner(obj):
v = data.copy()
v["object"] = obj
return {
"commands": snippet.expand(v).splitlines(),
"ignore_cli_errors": snippet.ignore_cli_errors
}
return inner
map_task = self.get_map_script(snippet)
task = ReduceTask.create_task(
object_selector=objects,
reduce_script=reduce_task,
reduce_script_params={"snippet": snippet},
map_script=map_task,
map_script_params=get_map_script_params(snippet, params),
timeout=snippet.timeout)
return task.id
objects = ManagedObjectSelector.resolve_expression(r["selector"])
except ManagedObjectSelector.DoesNotExist, why:
return self.response_not_found(str(why))
except ManagedObject.DoesNotExist, why:
return self.response_not_found(str(why))
# Check all objects fall within MRTConfig selector
unauthorized = set(objects).difference(
set(config.selector.managed_objects))
if unauthorized:
return self.response_forbidden(
"Unauthorized objects: %s" %
(", ".join([o.name for o in unauthorized])))
# Run MRT
timeout = r.get("timeout", None) or config.timeout
t = ReduceTask.create_task(objects,
"pyrule:%s" % config.reduce_pyrule.name, {},
config.map_script, r.get("map_args",
{}), timeout)
return self.response_accepted(location="/sa/mrt/%s/%d/" % (task, t.id))
@view(url="^(?P<task>[0-9a-zA-Z_\-]+)/(?P<task_id>\d+)/$",
method=["GET"],
access="launch",
api=True)
def api_result(self, request, task, task_id):
# Get task
config = MRTConfig.objects.filter(name=task, is_active=True).first()
if not config:
return self.response_not_found("Task not found")
# Check permissions
pn = "sa:mrt:%s" % config.permission_name
if not Permission.has_perm(request.user, pn):
def test_sae_task_multiple2(self):
task = ReduceTask.create_task(["SAE", "SAE"], None, {},
["ping_test"] * 2,
[{"activator_name": "default", "addresses": ["127.0.0.1"]}] * 2, 1)
self.assertEquals(task.maptask_set.count(), 4)
def test_sae_task_single(self):
task = ReduceTask.create_task("SAE", None, {}, "ping_test",
{"activator_name": "default", "addresses": ["127.0.0.1"]}, 1)
self.assertEquals(task.maptask_set.count(), 1)
mt = task.maptask_set.all()[0]
self.assertEquals(mt.map_script, "NOC.SAE.ping_test")