def _do_upsert(self, c, relations, force_create): affected, cache_affected = 0, 0 node_cache = dict() debug("# of relations ready to process: %d" % len(relations)) for node_id, relation in relations: if force_create: s = (self.SQL_CREATE_NODE_WITH_ID, self.SQL_CREATE_NODE)[node_id is None] elif node_id is None: s = self.SQL_CREATE_NODE else: s = self.SQL_UPDATE_NODE c = yield c.execute(s, relation) affected += c._cursor.rowcount if not node_id: result = yield c.execute(self.SQL_GET_LAST_ID) id = result.fetchall() node_id = id[0][0] cache = yield self._build_cache(c, node_id, node_cache) if cache["success"]: cache_affected += cache["affected"] depends = yield self._select_depends(c, node_id) for depend in depends: cache = yield self._build_cache(c, depend) if cache["success"]: affected = affected + 1 cache = dict(success=True, affacted=affected) node_cache.clear() defer.returnValue((affected, cache_affected))
def _search(self, c, where_clause, start, num, order_by, order, return_total): def _reduce(result, row): if not isinstance(result, list): id, key, value = result result = [{".id": id, key: value}] id, key, value = row last_id = result[-1][".id"] if last_id == id: result[-1][key] = value else: result.append({".id": id, key: value}) return result # count total if return_total: startTime = time.time() s = self.SQL_COUNT_CACHE_EX % dict(where_clause=where_clause) debug('SQL_SELECT_CACHE_EX: %s' % s) c = yield c.execute(s) total = c.fetchall()[0][0] debug("count duration: %.3fms" % \ (1000 * (time.time() - startTime))) else: total = 0 # fetch result startTime = time.time() order = ("DESC", "ASC")[order.upper() == "ASC"] if order_by not in ("id", "manifest", "cn"): order_by = "value->'%s'" % (self._quote(order_by)) s = self.SQL_SELECT_CACHE_EX % dict(where_clause=where_clause, offset=start, limit=(num, 'ALL')[num == 0], order_by=order_by, order=order) debug('SQL_SELECT_CACHE_EX: %s' % s) c = yield c.execute(s) result = c.fetchall() debug("fetch duration: %.3fms" % (1000 * (time.time() - startTime))) # data processing nodes = defaultdict(dict) startTime = time.time() if result: nodes = reduce(_reduce, result) else: nodes = list() debug("value parsing duration: %.3fms" \ % (1000 * (time.time() - startTime))) defer.returnValue(dict(start=start, num=len(nodes), total=total, result=nodes))
def finish(self, value, request): request.setHeader('Content-Type', 'application/json; charset=UTF-8') if isinstance(value, Failure): err = value.value if self.debug: print "-" * 30, "TRACEKBACK", "-" * 30 value.printTraceback() print "^" * 30, "TRACEKBACK", "^" * 30 request.setResponseCode(500) if isinstance(err, backend.ValidationError): request.setResponseCode(400) elif isinstance(err, backend.NodeNotFound): request.setResponseCode(404) elif isinstance(err, backend.NodeInUseError): request.setResponseCode(400) elif isinstance(err, backend.EmptyInputData): request.setResponseCode(400) elif isinstance(err, backend.BatchOperationError): request.setResponseCode(400) elif (isinstance(err, Exception) and not isinstance(err, backend.GenericError)): err = dict(error="UnknownError", message=err.message) debug("original error: %s" % err) request.write(json_encode(dict(err)) + "\n") else: request.setResponseCode(200) request.write(json_encode(value) + "\n") log.msg("respone time: %.3fms" % ( (time.time() - self.startTime) * 1000)) request.finish()
def upsert(self, input, force_create=False, check_only=False): relations = list() errors = list() if not input: raise backend.EmptyInputData() debug("upsert: get %d items" % len(input)) for item in input: if not isinstance(item, dict): raise backend.GenericError("Input data is invalid") try: node_id = item.get("id", None) if node_id: node_id = int(node_id) except ValueError: errors.append((node_id, backend.DataError("id", node_id, "node id must be integer"))) continue manifest, value = item["manifest"], item["value"] if manifest not in self.manifest: errors.append((node_id, backend.ManifestNotFound(manifest=manifest))) continue defers = list() for field in self.manifest[manifest]["field"]: if field in self.field: d = self.validate_value( node_id, manifest, field, value, create=force_create or node_id is None) defers.append(d) try: verified = yield DeferredList(defers, consumeErrors=True) relation = yield self._map_relation(node_id, manifest, verified, create=force_create) relation["value"] = self._serialize_hstore(relation["value"]) relations.append((node_id, relation)) except backend.ValidationError as e: errors.append((node_id, e)) if errors: raise backend.BatchOperationError(errors=errors) if check_only: defer.returnValue({"success": True}) (affected, cache_affected) = yield self.pool.runInteraction( self._do_upsert, relations, force_create) defer.returnValue(dict(success=True, affected=affected, cache=dict(success=True, affected=cache_affected)))
def prepare(self, request): request.content.seek(0, 0) content = request.content.read() debug("content size = %d" % len(content)) if content: return defer.succeed(json_decode(content)) else: return defer.succeed(None)
def cancel(self, err, call): debug("Request cancelling.") call.cancel()