def save_chart(self, request, chart_id):
chart = self.get_object_or_404(NetworkChart, id=int(chart_id))
for cmd in json_decode(request.raw_post_data):
if cmd["type"] == "mo":
if cmd["cmd"] == "move":
chart.update_state(
"mo", cmd["id"], {
"x": max(cmd["x"], 0),
"y": max(cmd["y"], 0),
"w": cmd["w"],
"h": cmd["h"]
})
elif cmd["cmd"] == "label_position":
chart.update_state(
"mo", cmd["id"],
{"label_position": cmd["label_position"]})
elif cmd["cmd"] == "collapsed":
chart.update_state("mo", cmd["id"],
{"collapsed": cmd["collapsed"]})
elif cmd["cmd"] == "shape":
try:
o = ManagedObject.objects.get(id=int(cmd["id"]))
except ManagedObject.DoesNotExist:
continue
if o.shape != cmd["shape"]:
o.shape = cmd["shape"]
o.save()
elif cmd["type"] == "link":
if cmd["cmd"] == "edge_style":
chart.update_state("link", cmd["id"],
{"edge_style": cmd["edge_style"]})
return True
def api_link_approve(self, request):
d = json_decode(request.raw_post_data)
plc = self.get_object_or_404(PendingLinkCheck, id=d.get("link"))
li = Interface.objects.filter(
managed_object=plc.local_object.id,
name=plc.local_interface
).first()
if not li:
return {
"success": False,
"error": "Interface not found: %s:%s" % (
plc.local_object.name, plc.local_interface)
}
ri = Interface.objects.filter(
managed_object=plc.remote_object.id,
name=plc.remote_interface
).first()
if not ri:
return {
"success": False,
"error": "Interface not found: %s:%s" % (
plc.remote_object.name, plc.remote_interface)
}
li.link_ptp(ri, method=plc.method + "+manual")
plc.delete()
return {
"success": True
}
def handle_upgrade(self, collections):
for c in collections:
self.log("Upgrading %s" % c)
dc = Collection(c)
cr = "%s/collections/%s" % (dc.module, dc.name)
if os.path.exists(os.path.join(cr, "manifest.csv")):
raise CommandError("Collection %s is already upgraded" % c)
fl = []
for root, dirs, files in os.walk(cr):
for f in files:
if f.endswith(".json"):
fl += [os.path.join(root, f)]
for p in fl:
s = os.stat(p)
data = json_decode(read_file(p))
if isinstance(data, list):
for o in data:
if "__aliases" in o:
del o["__aliases"]
dc.install_item(o)
elif isinstance(data, dict):
if "__aliases" in data:
del data["__aliases"]
dc.install_item(data)
if os.stat(p)[stat.ST_MTIME] == s[stat.ST_MTIME]:
# Remove stale file
self.log(" ... removing %s" % p)
os.unlink(p)
self.log(" ... saving manifest.csv")
dc.save()
self.log(" ... done")
def api_run_mrt(self, request, name):
from noc.sa.models import ReduceTask, ManagedObjectSelector
# Check MRT configured
if name not in self.mrt_config:
return self.response_not_found("MRT %s is not found" % name)
# Check MRT access
if not self.check_mrt_access(request, name):
return self.response_forbidden("Forbidden")
#
data = json_decode(request.raw_post_data)
if "selector" not in data:
return self.response_bad_request("'selector' is missed")
# Run MRT
mc = self.mrt_config[name]
map_params = data.get("map_params", {})
map_params = dict((str(k), v) for k, v in map_params.iteritems())
objects = ManagedObjectSelector.resolve_expression(data["selector"])
task = ReduceTask.create_task(objects, "pyrule:mrt_result", {},
mc["map_script"], map_params,
mc.get("timeout", 0))
if mc["map_script"] == "commands" and settings.LOG_MRT_COMMAND:
# Log commands
now = datetime.datetime.now()
safe_append(
os.path.join(settings.LOG_MRT_COMMAND, "commands",
"%04d" % now.year, "%02d" % now.month,
"%02d.log" % now.day),
"%s\nDate: %s\nObjects: %s\nUser: %s\nCommands:\n%s\n" %
("-" * 72, now.isoformat(), ",".join(
str(o) for o in objects), request.user.username, " " +
"\n".join(map_params["commands"]).replace("\n", "\n ")))
return task.id
def load_beef(self, path):
"""
Load beef from JSON file
:param path:
:return:
"""
def q(s):
ts = type(s)
if ts == dict:
return dict((k, q(s[k])) for k in s)
elif ts == list:
return [q(x) for x in s]
elif isinstance(s, basestring):
return s.decode("string_escape")
else:
return s
data = read_file(path)
if not data:
raise OSError("Cannot read file: %s" % path)
beef = json_decode(data)
if beef.get("type") != self.type_signature:
raise ValueError("Invalid beef '%s'. Signature mismatch" % path)
for n in self.beef_args:
if n in beef:
setattr(self, n, q(beef[n]))
self._testMethodDoc = "%s [%s]" % (beef["script"], beef["guid"])
def scan(cls):
"""
Scan cls.NEW_ROOT for new crashinfos
"""
for f in os.listdir(cls.NEW_ROOT):
if not f.endswith(".json"):
continue
try:
u = uuid.UUID(f[:-5])
except ValueError:
continue # Badly formed UUID
if Crashinfo.objects.filter(uuid=u).count():
continue # Known
logger.info("New crashinfo found: %s", u)
try:
with open(os.path.join(cls.NEW_ROOT, f)) as f:
ci = json_decode(f.read())
except Exception, why:
logger.error("Unable to load and decode crashinfo %s: %s", u,
why)
continue
c = Crashinfo(uuid=u,
timestamp=dateutil.parser.parse(ci["ts"]),
process=ci["process"],
branch=ci.get("branch"),
tip=ci.get("tip"),
status="N")
c.save()
def get_beef_result(cls, uuid):
fn = "%s.json" % uuid
for dirpath, dirs, files in os.walk("local/repos/sa/"):
if fn in files:
with open(os.path.join(dirpath, fn)) as f:
return json_decode(f.read())["result"]
raise OSError("File not found: %s" % fn)
def api_set_interface(self, request, id):
def get_or_none(c, v):
if not v:
return None
return c.objects.get(id=v)
o = self.get_object_or_404(ManagedObject, id=int(id))
if not o.has_access(request.user):
return self.response_forbidden("Access denied")
d = json_decode(request.raw_post_data)
if "id" in d:
i = self.get_object_or_404(Interface, id=d["id"])
if i.managed_object.id != o.id:
return self.response_not_found()
# Set profile
if "profile" in d:
p = get_or_none(InterfaceProfile, d["profile"])
i.profile = p
if p:
i.profile_locked = True
# Project
if "project" in d:
i.project = get_or_none(Project, d["project"])
# State
if "state" in d:
i.state = get_or_none(ResourceState, d["state"])
# VC Domain
if "vc_domain" in d:
i.vc_domain = get_or_none(VCDomain, d["vc_domain"])
#
i.save()
return {
"success": True
}
def api_link_reject(self, request):
d = json_decode(request.raw_post_data)
plc = self.get_object_or_404(PendingLinkCheck, id=d.get("link"))
plc.delete()
return {
"success": True
}
def json(self):
path = self.json_path
if os.path.exists(path):
try:
with open(path) as f:
return json_decode(f.read())
except Exception, why:
logger.error("Unable to load and decode crashinfo %s: %s",
self.uuid, why)
def handle_install(self, files):
def iter_files(names):
for f in names:
if os.path.isdir(f):
for dirpath, dirnames, filenames in os.walk(f):
for ff in filenames:
if ff.endswith(".json"):
yield os.path.join(dirpath, ff)
else:
yield f
dcs = {} # name -> collection
files = list(files)
if files[0] in Collection.COLLECTIONS:
# Get collection name hints
current_name = files.pop(0)
else:
# Inplace update
parts = files[0].split(os.path.sep)
if (len(parts) > 3 and parts[1] == "collections" and "%s.%s" %
(parts[0], parts[2]) in Collection.COLLECTIONS):
current_name = "%s.%s" % (parts[0], parts[2])
else:
current_name = None
self.log("Installing files")
for f in iter_files(files):
self.log(" ... read %s" % f)
fd = read_file(f)
if not fd:
raise CommandError("Cannot read file: %s" % f)
try:
data = json_decode(fd)
except ValueError:
raise CommandError("Unable to parse JSON file: %s" % f)
if "$collection" in data:
current_name = data["$collection"]
if not current_name:
raise CommandError("Cannot detect collection for file: %s" % f)
if current_name in dcs:
dc = dcs[current_name]
else:
self.log(" ... open collection %s" % current_name)
dc = Collection(current_name)
dc.load()
dcs[current_name] = dc
if "uuid" in data:
ci = dc.get_item(data["uuid"])
if ci and dc.get_hash(fd) == ci.hash:
self.log(" ... not changed. Skipping")
continue # Not changed
dc.install_item(data, load=True)
# Save all modified manifests
for n, dc in dcs.iteritems():
self.log(" ... saving manifest.csv for %s" % n)
dc.save()
def load_item(self, mi):
p = self.get_item_path(mi)
if not os.path.exists(p):
self.die("File not found: %s" % p)
with open(p) as f:
fdata = f.read()
try:
data = json_decode(fdata)
except ValueError, why:
self.die("Failed to read JSON file '%s': %s" % (p, why))
def api_save_nodes(self, request, wf_id):
def get_by_id(nid):
if not nid:
return None
return Node.objects.filter(id=id_map.get(nid, nid)).first()
wf = self.get_object_or_404(Workflow, id=wf_id)
data = json_decode(request.raw_post_data)
id_map = {} # JSON -> DB id
# Process changed nodes
for r in data:
if r["type"] == "node":
if r["id"] and r.get("deleted"):
# Delete node
n = self.get_object_or_404(Node, id=r["id"])
n.delete()
continue
if not r.get("handler"):
continue
# Create/change node
if r["id"] and not r["id"].startswith("id:"):
n = self.get_object_or_404(Node, id=r["id"])
else:
n = Node(workflow=wf)
n.name = r["name"]
n.label = r.get("label")
n.description = r["description"]
n.handler = r["handler"]
n.params = r["params"]
n.x = r["x"]
n.y = r["y"]
if not r.get("next_node"):
n.next_node = None
if not r.get("next_true_node"):
n.next_true_node = None
if not r.get("next_false_node"):
n.next_false_node = None
n.save()
id_map[r["id"]] = n.id
if r.get("start"):
# Move start node
wf.start_node = str(n.id)
wf.save()
# Process changed edges
for r in data:
if r["type"] == "node":
if r["id"] and r.get("deleted"):
continue
n = get_by_id(r["id"])
n.next_node = get_by_id(r.get("next_node"))
n.next_true_node = get_by_id(r.get("next_true_node"))
n.next_false_node = get_by_id(r.get("next_false_node"))
n.save() # @todo: Eliminate redundant save
return True
def _api_install_json(self, request, id):
"""
Expose JSON collection item when available
"""
from noc.lib.collection import Collection
o = self.get_object_or_404(self.model, id=id)
data = json_decode(o.to_json())
dc = Collection(self.json_collection)
dc.load()
dc.install_item(data)
dc.save()
return True
def create_mib(self, data):
# Deserialze
d = json_decode(data)
# Create MIB
mib = MIB(name=d["name"],
description=d["description"],
last_updated=self.decode_date(d["last_updated"]),
version=d.get("version", 0),
depends_on=d["depends_on"])
mib.save()
# Upload
if d["data"]:
mib.load_data(d["data"])
def update_mib(self, mib, data, version=None):
# Deserealize
d = json_decode(data)
# Update timestamp
mib.last_updated = self.decode_date(d["last_updated"])
# Update version
if version is not None:
mib.version = version
mib.save()
# Upload
mib.clean()
if d["data"]:
mib.load_data(d["data"])
def api_stop_discovery(self, request, id):
o = self.get_object_or_404(ManagedObject, id=id)
if not o.has_access(request.user):
return self.response_forbidden("Access denied")
r = json_decode(request.raw_post_data).get("names", [])
d = 0
for name in get_active_discovery_methods():
cfg = "enable_%s" % name
if getattr(o.object_profile, cfg) and name in r:
stop_schedule("inv.discovery", name, o.id)
d += 1
return {
"success": True
}
def api_action(self, request, id, action):
def execute(o, a, args):
return a.execute(o, **args)
o = self.get_object_or_404(ManagedObject, id=id)
if not o.has_access(request.user):
return self.response_forbidden("Access denied")
a = self.get_object_or_404(Action, name=action)
# @todo: Check access
body = request.raw_post_data
if body:
args = json_decode(body)
else:
args = {}
return self.submit_slow_op(request, execute, o, a, args)
def request(self, method, path, user=None, **kwargs):
if user is None:
kwargs["credentials"] = {}
else:
kwargs["credentials"] = {"user": user, "password": user}
# Convert input to JSON
if "data" in kwargs:
kwargs["data"] = json_encode(kwargs["data"])
if method in ("post", "put"):
kwargs["content_type"] = "text/json"
r = getattr(self.client, method)(path, **kwargs)
if r.has_header("Content-Type") and r["Content-Type"].startswith(
"text/json"):
return (r.status_code, json_decode(r.content))
else:
return r.status_code, r.content
def api_run(self, request, task):
"""
Run new MRT
:param request:
:param task:
:return:
"""
# Get task
config = MRTConfig.objects.filter(name=task, is_active=True).first()
if not config:
return self.response_not_found("Task not found")
# Check permissions
pn = "sa:mrt:%s" % config.permission_name
if not Permission.has_perm(request.user, pn):
return self.response_forbidden(
"Permission denied: '%s' permission required" % pn)
# Parse request
try:
r = json_decode(request.raw_post_data)
except Exception, why:
return self.response_bad_request(str(why))
def import_files(self, paths):
for p in paths:
if not os.path.exists(p):
raise ValueError("File does not exists: %s" % p)
with open(p) as f:
try:
data = json_decode(f.read())
except ValueError, why:
self.die("Failed to read JSON file '%s': %s" % (p, why))
if not isinstance(data, dict):
self.die("Invalid JSON file: %s" % p)
doc = self.doc(**self.dereference(self.doc, data))
mi = CollectionItem(name=doc.name,
uuid=doc.uuid,
path=doc.get_json_path(),
hash=self.get_hash(data))
self.items[mi.uuid] = mi
self.logger.info("Importing %s", doc.name)
safe_rewrite(os.path.join(self.module, "collections", self.name,
doc.get_json_path()),
data,
mode=0644)
def api_save_config(self, request, id):
"""
Get Config data
:param request:
:param id:
:return:
"""
data = json_decode(request.raw_post_data)
if id not in self.configs:
return self.response_not_found()
# Read config
config = ConfigParser.SafeConfigParser()
config.read(self.configs[id])
# Apply updates
for d in data:
if not config.has_section(d["section"]):
config.add_section(d["section"])
config.set(d["section"], d["key"], d["value"].replace("%", "%%"))
# Save
with open(self.configs[id], "w") as f:
config.write(f)
return True
def api_import(self, request, json):
try:
jdata = json_decode(json)
except Exception, why:
return {"status": False, "error": "Invalid JSON: %s" % why}
logger.error("Cannot get config from %s: %s",
self.url, why)
return None
except socket.timeout:
logger.error("Cannot get config from %s: Timed out",
self.url)
return None
except socket.error, why:
logger.error("Cannot get config from %s: Socket error: %s",
self.url, why)
return None
except Exception, why:
logger.error("Cannot get config from %s: %s", self.url, why)
return None
try:
data = json_decode(resp.read())
except:
logger.error("Failed to parse config")
return None
logger.debug("Config retrieved")
return data
def apply_config(self, config):
n = 0
n_created = 0
n_changed = 0
n_errors = 0
n_deleted = 0
for cfg in config["config"]:
n += 1
try:
def cmd_MESSAGE(self, headers, body):
if "subscription" in headers and "destination" in headers:
if headers.get("content-type") == "text/json":
body = json_decode(body)
self.client.on_message(headers["destination"],
headers["subscription"], body)
def inner(request, *args, **kwargs):
def nq(s):
"""
Convert var[]=xxxx to var=xxxx
"""
if s.endswith("[]"):
return s[:-2]
else:
return s
try:
v = view_map[request.method]
except KeyError:
logger.info("No handler for '%s' method", request.method)
return HttpResponseNotFound("No handler for '%s' method" %
request.method)
if not request.user or not v.access.check(app, request.user):
return HttpResponseForbidden()
to_log_api_call = (self.log_api_calls and hasattr(v, "api")
and v.api)
app_logger = v.im_self.logger
try:
# Validate requests
if (hasattr(v, "validate") and v.validate):
# Additional validation
errors = None
if isinstance(v.validate, DictParameter):
# Validate via NOC interfaces
if request.method == "GET":
g = dict((nq(k), v[0] if len(v) == 1 else v)
for k, v in request.GET.lists()
if k != "_dc")
else:
ct = request.META.get("CONTENT_TYPE")
if ct and ("text/json" in ct
or "application/json" in ct):
g = json_decode(request.raw_post_data)
else:
g = dict((k, v[0] if len(v) == 1 else v)
for k, v in request.POST.lists())
try:
kwargs.update(v.validate.clean(g))
except InterfaceTypeError, why:
errors = str(why)
elif issubclass(v.validate, Form):
# Validate via django forms
f = v.validate(request.GET) # @todo: Post
if f.is_valid():
kwargs.update(f.cleaned_data)
else:
errors = dict([(f, "; ".join(e))
for f, e in f.errors.items()])
if errors:
#
if to_log_api_call:
app_logger.error("ERROR: %s", errors)
# Return error response
ext_format = (
"__format=ext"
in request.META["QUERY_STRING"].split("&"))
r = JSONEncoder(ensure_ascii=False).encode({
"status":
False,
"errors":
errors
})
status = 200 if ext_format else 400 # OK or BAD_REQUEST
return HttpResponse(
r,
status=status,
mimetype="text/json; charset=utf-8")
# Log API call
if to_log_api_call:
a = {}
if request.method in ("POST", "PUT"):
ct = request.META.get("CONTENT_TYPE")
if ct and ("text/json" in ct
or "application/json" in ct):
a = json_decode(request.raw_post_data)
else:
a = dict((k, v[0] if len(v) == 1 else v)
for k, v in request.POST.lists())
elif request.method == "GET":
a = dict((k, v[0] if len(v) == 1 else v)
for k, v in request.GET.lists())
app_logger.debug("API %s %s %s", request.method,
request.path, a)
# Call handler
v.__dict__["hits_metric"] += 1
with v.__dict__["time_metric"].timer():
r = v(request, *args, **kwargs)
# Dump SQL statements
if self.log_sql_statements:
from django.db import connections
tsc = 0
sc = defaultdict(int)
for conn in connections.all():
for q in conn.queries:
stmt = q["sql"].strip().split(" ", 1)[0].upper()
sc[stmt] += 1
tsc += 1
app_logger.debug("SQL %(sql)s %(time)ss" % q)
x = ", ".join(
["%s: %d" % (k, v) for k, v in sc.iteritems()])
if x:
x = " (%s)" % x
app_logger.debug("SQL statements: %d%s" % (tsc, x))
def deserialize(self, data):
return json_decode(data)
