def execute(self): from noc.sa.models import ReduceTask,ManagedObject from noc.ip.models import Address # Get a list of managed objects to fetch ARP cache objects=list(ManagedObject.objects.raw(""" SELECT DISTINCT a.managed_object_id AS id FROM ip_address a JOIN ip_prefix p ON (a.prefix_id=p.id) WHERE a.managed_object_id IS NOT NULL AND EXISTS (SELECT id FROM ip_address ma WHERE ma.afi='4' AND ma.auto_update_mac AND ma.prefix_id=p.id) """)) # Get a list of addresses to syncronize addresses=list(Address.objects.raw(""" SELECT a.id,a.address,a.mac FROM ip_address a JOIN ip_prefix p ON (a.prefix_id=p.id) WHERE a.auto_update_mac AND a.afi='4' AND EXISTS (SELECT id FROM ip_address oa WHERE oa.prefix_id=p.id AND oa.managed_object_id IS NOT NULL) """)) # Get ARP cache task=ReduceTask.create_task(object_selector=objects, reduce_script=sync_macs_reduce, reduce_script_params={"addresses":addresses}, map_script="get_arp", map_script_params={}, timeout=self.TIMEOUT) # Wait for tasks completion ReduceTask.wait_for_tasks([task]) return True
def api_run_mrt(self, request, name): from noc.sa.models import ReduceTask, ManagedObjectSelector # Check MRT configured if name not in self.mrt_config: return self.response_not_found("MRT %s is not found" % name) # Check MRT access if not self.check_mrt_access(request, name): return self.response_forbidden("Forbidden") # data = json_decode(request.raw_post_data) if "selector" not in data: return self.response_bad_request("'selector' is missed") # Run MRT mc = self.mrt_config[name] map_params = data.get("map_params", {}) map_params = dict((str(k), v) for k, v in map_params.iteritems()) objects = ManagedObjectSelector.resolve_expression(data["selector"]) task = ReduceTask.create_task(objects, "pyrule:mrt_result", {}, mc["map_script"], map_params, mc.get("timeout", 0)) if mc["map_script"] == "commands" and settings.LOG_MRT_COMMAND: # Log commands now = datetime.datetime.now() safe_append( os.path.join(settings.LOG_MRT_COMMAND, "commands", "%04d" % now.year, "%02d" % now.month, "%02d.log" % now.day), "%s\nDate: %s\nObjects: %s\nUser: %s\nCommands:\n%s\n" % ("-" * 72, now.isoformat(), ",".join( str(o) for o in objects), request.user.username, " " + "\n".join(map_params["commands"]).replace("\n", "\n "))) return task.id
def execute(self): from noc.peer.models import PrefixListCache from noc.sa.models import ManagedObject, ReduceTask # prefix_lists = {} # PeeringPoint -> [prefix_lists] # For all out-of-dated prefix lists for pc in PrefixListCache.objects.filter(): if not pc.peering_point.enable_prefix_list_provisioning: continue if pc.pushed is not None and pc.pushed > pc.changed: continue if not pc.data: continue if pc.peering_point not in prefix_lists: prefix_lists[pc.peering_point] = [pc] else: prefix_lists[pc.peering_point] += [pc] # Run Map/Reduce tasks tasks = [] for peering_point in prefix_lists: # Try to find managed object try: pp = ManagedObject.objects.get(name=peering_point.hostname) except ManagedObject.DoesNotExist: logging.warning( "No managed object for peering point '%s' found" % peering_point.name) continue task = ReduceTask.create_task( object_selector=[pp], reduce_script="pyrule:prefix_list_provisioning", reduce_script_params={"peering_point": peering_point}, map_script="sync_prefix_lists", map_script_params={ "changed_prefix_lists": [{ "name": pl.name, "strict": pl.strict, "prefix_list": pl.data } for pl in prefix_lists[peering_point]] }, timeout=TIMEOUT) tasks += [task] # Wait for tasks completion ReduceTask.wait_for_tasks(tasks) return True
def execute(self): from noc.vc.models import VCDomain from noc.sa.models import ReduceTask tasks = [] # Get config for vc_domain in VCDomain.objects.filter(enable_provisioning=True): # Get VCDomain vcs vcs = [{ "vlan_id": vc.l1, "name": vc.name } for vc in vc_domain.vc_set.all()] # Run Map/Reduce task for c in vc_domain.vcdomainprovisioningconfig_set.filter( is_enabled=True): if c.vc_filter: vc_list = [ v for v in vcs if c.vc_filter.check(v["vlan_id"]) ] else: vc_list = vcs[:] if not vc_list: continue # Refuse to drop all vlans on switches task = ReduceTask.create_task( object_selector=c.selector, reduce_script="pyrule:vc_provisioning", reduce_script_params={"config": c}, map_script="sync_vlans", map_script_params={ "vlans": vc_list, "tagged_ports": c.tagged_ports_list }, timeout=TIMEOUT) tasks += [task] # Wait for tasks completion ReduceTask.wait_for_tasks(tasks) return True
def view_run(self, request): if request.POST: form = self.MRTaskForm(request.POST) if form.is_valid(): t = ReduceTask.create_task( object_selector=form.cleaned_data["selector"], reduce_script="pyrule:" + form.cleaned_data["reduce_script"].name, reduce_script_params=form. cleaned_data["reduce_script_params"], map_script=form.cleaned_data["map_script"], map_script_params=form.cleaned_data["map_script_params"], timeout=form.cleaned_data["timeout"]) return self.response_redirect_to_object(t) else: form = self.MRTaskForm(initial={"timeout": 180}) return self.render(request, "mr_task.html", {"form": form})
def run_task(self, snippet, objects, params): def get_map_script_params(snippet, data): def inner(obj): v = data.copy() v["object"] = obj return { "commands": snippet.expand(v).splitlines(), "ignore_cli_errors": snippet.ignore_cli_errors } return inner map_task = self.get_map_script(snippet) task = ReduceTask.create_task( object_selector=objects, reduce_script=reduce_task, reduce_script_params={"snippet": snippet}, map_script=map_task, map_script_params=get_map_script_params(snippet, params), timeout=snippet.timeout) return task.id
objects = ManagedObjectSelector.resolve_expression(r["selector"]) except ManagedObjectSelector.DoesNotExist, why: return self.response_not_found(str(why)) except ManagedObject.DoesNotExist, why: return self.response_not_found(str(why)) # Check all objects fall within MRTConfig selector unauthorized = set(objects).difference( set(config.selector.managed_objects)) if unauthorized: return self.response_forbidden( "Unauthorized objects: %s" % (", ".join([o.name for o in unauthorized]))) # Run MRT timeout = r.get("timeout", None) or config.timeout t = ReduceTask.create_task(objects, "pyrule:%s" % config.reduce_pyrule.name, {}, config.map_script, r.get("map_args", {}), timeout) return self.response_accepted(location="/sa/mrt/%s/%d/" % (task, t.id)) @view(url="^(?P<task>[0-9a-zA-Z_\-]+)/(?P<task_id>\d+)/$", method=["GET"], access="launch", api=True) def api_result(self, request, task, task_id): # Get task config = MRTConfig.objects.filter(name=task, is_active=True).first() if not config: return self.response_not_found("Task not found") # Check permissions pn = "sa:mrt:%s" % config.permission_name if not Permission.has_perm(request.user, pn):
def test_sae_task_multiple2(self): task = ReduceTask.create_task(["SAE", "SAE"], None, {}, ["ping_test"] * 2, [{"activator_name": "default", "addresses": ["127.0.0.1"]}] * 2, 1) self.assertEquals(task.maptask_set.count(), 4)
def test_sae_task_single(self): task = ReduceTask.create_task("SAE", None, {}, "ping_test", {"activator_name": "default", "addresses": ["127.0.0.1"]}, 1) self.assertEquals(task.maptask_set.count(), 1) mt = task.maptask_set.all()[0] self.assertEquals(mt.map_script, "NOC.SAE.ping_test")