def test_ro(self):
s = {"readonly_user": self.U, "readonly_password": self.P}
normalize_auth(s)
self._check(s)
s = {"readonly_user": self.U, "readonly_password": self.P}
normalize_auth(s, readonly=False)
self._check_absent(s)
def test_ro(self):
s = {"readonly_user": self.U, "readonly_password": self.P}
normalize_auth(s)
self._check(s)
s = {"readonly_user": self.U, "readonly_password": self.P}
normalize_auth(s, readonly=False)
self._check_absent(s)
def test_admin(self):
s = {"admin_user": self.U, "admin_password": self.P}
normalize_auth(s)
self._check(s)
s = {"admin_user": self.U, "admin_password": self.P}
normalize_auth(s, admin=False)
self._check_absent(s)
def test_admin(self):
s = {"admin_user": self.U, "admin_password": self.P}
normalize_auth(s)
self._check(s)
s = {"admin_user": self.U, "admin_password": self.P}
normalize_auth(s, admin=False)
self._check_absent(s)
def test_order(self):
s = {"readonly_user": self.U + "_1", "readonly_password": self.P,
"admin_user": self.U + "_2", "admin_password": self.P}
normalize_auth(s, readonly_first=False)
self.assertEqual(s["user"], self.U + "_2")
s = {"readonly_user": self.U + "_1", "readonly_password": self.P,
"admin_user": self.U + "_2", "admin_password": self.P}
normalize_auth(s, readonly_first=True)
self.assertEqual(s["user"], self.U + "_1")
def test_order(self):
s = {
"readonly_user": self.U + "_1",
"readonly_password": self.P,
"admin_user": self.U + "_2",
"admin_password": self.P
}
normalize_auth(s, readonly_first=False)
self.assertEqual(s["user"], self.U + "_2")
s = {
"readonly_user": self.U + "_1",
"readonly_password": self.P,
"admin_user": self.U + "_2",
"admin_password": self.P
}
normalize_auth(s, readonly_first=True)
self.assertEqual(s["user"], self.U + "_1")
def test_plain(self):
s = {"user": self.U, "password": self.P}
normalize_auth(s)
self._check(s)
def diff(self, c1, c2, only_missing=False, only_values=False, allow_dup=False):
"""Perform a difference between the 2 collections.
The first collection is treated as the previous one, and the second
is treated as the new one.
Note: this is not 'big data'-ready; we assume all the records can fit in memory.
:param c1: Collection (1) config file, or QueryEngine object
:type c1: str or QueryEngine
:param c2: Collection (2) config file, or QueryEngine object
:type c2: str or QueryEngine
:param only_missing: Only find and return self.MISSING; ignore 'new' keys
:param only_values: Only find and return self.CHANGED; ignore new or missing keys
:param allow_dup: Allow duplicate keys, otherwise fail with ValueError
:return: dict with keys self.MISSING, self.NEW (unless only_missing is True), & self.CHANGED,
each a list of records with the key and
any other fields given to the constructor 'info' argument.
The meaning is: 'missing' are keys that are in c1 not found in c2
'new' is keys found in c2 that are not found in c1, and 'changed' are records
with the same key that have different 'props' values.
"""
# Connect.
_log.info("connect.start")
if isinstance(c1, QueryEngine):
engines = [c1, c2]
else:
engines = []
for cfg in c1, c2:
settings = util.get_settings(cfg)
if not normalize_auth(settings):
_log.warn("Config file {} does not have a username/password".format(cfg))
settings["aliases_config"] = {"aliases": {}, "defaults": {}}
engine = QueryEngine(**settings)
engines.append(engine)
_log.info("connect.end")
# Query DB.
keys = [set(), set()]
eqprops = [{}, {}]
numprops = [{}, {}]
# Build query fields.
fields = dict.fromkeys(self._info + self._all_props + [self._key_field], True)
if not '_id' in fields: # explicitly remove _id if not given
fields['_id'] = False
# Initialize for query loop.
info = {} # per-key information
has_info, has_props = bool(self._info), bool(self._all_props)
has_numprops, has_eqprops = bool(self._prop_deltas), bool(self._props)
_log.info("query.start query={} fields={}".format(self._filter, fields))
t0 = time.time()
# Main query loop.
for i, coll in enumerate(engines):
_log.debug("collection {:d}".format(i))
count, missing_props = 0, 0
for rec in coll.query(criteria=self._filter, properties=fields):
count += 1
# Extract key from record.
try:
key = rec[self._key_field]
except KeyError:
_log.critical("Key '{}' not found in record: {}. Abort.".format(
self._key_field, rec))
return {}
if not allow_dup and key in keys[i]:
raise ValueError("Duplicate key: {}".format(key))
keys[i].add(key)
# Extract numeric properties.
if has_numprops:
pvals = {}
for pkey in self._prop_deltas.keys():
try:
pvals[pkey] = float(rec[pkey])
except KeyError:
#print("@@ missing {} on {}".format(pkey, rec))
missing_props += 1
continue
except (TypeError, ValueError):
raise ValueError("Not a number: collection={c} key={k} {p}='{v}'"
.format(k=key, c=("old", "new")[i], p=pkey, v=rec[pkey]))
numprops[i][key] = pvals
# Extract properties for exact match.
if has_eqprops:
try:
propval = tuple([(p, str(rec[p])) for p in self._props])
except KeyError:
missing_props += 1
#print("@@ missing {} on {}".format(pkey, rec))
continue
eqprops[i][key] = propval
# Extract informational fields.
if has_info:
if key not in info:
info[key] = {}
for k in self._info:
info[key][k] = rec[k]
# Stop if we don't have properties on any record at all
if 0 < count == missing_props:
_log.critical("Missing one or more properties on all {:d} records"
.format(count))
return {}
# ..but only issue a warning for partially missing properties.
elif missing_props > 0:
_log.warn("Missing one or more properties for {:d}/{:d} records"
.format(missing_props, count))
t1 = time.time()
_log.info("query.end sec={:f}".format(t1 - t0))
# Compute missing and new keys.
if only_values:
missing, new = [], []
else:
_log.debug("compute_difference.start")
missing, new = keys[0] - keys[1], []
if not only_missing:
new = keys[1] - keys[0]
_log.debug("compute_difference.end")
# Compute mis-matched properties.
if has_props:
changed = self._changed_props(keys, eqprops, numprops, info,
has_eqprops=has_eqprops, has_numprops=has_numprops)
else:
changed = []
# Build result.
_log.debug("build_result.begin")
result = {}
if not only_values:
result[self.MISSING] = []
for key in missing:
rec = {self._key_field: key}
if has_info:
rec.update(info.get(key, {}))
result[self.MISSING].append(rec)
if not only_missing:
result[self.NEW] = []
for key in new:
rec = {self._key_field: key}
if has_info:
rec.update(info.get(key, {}))
result[self.NEW].append(rec)
result[self.CHANGED] = changed
_log.debug("build_result.end")
return result
from matgendb.query_engine import QueryEngine
from matgendb import dbconfig
import bson
import datetime
from django.utils.encoding import force_unicode
from django.core.serializers.json import DjangoJSONEncoder
qe = None
mgdb_config = os.environ.get("MGDB_CONFIG", "")
if mgdb_config:
config = json.loads(mgdb_config)
if not dbconfig.normalize_auth(config, readonly_first=True):
config["user"] = config["password"] = None
qe = QueryEngine(host=config["host"],
port=config["port"],
database=config["database"],
user=config["user"],
password=config["password"],
collection=config["collection"],
aliases_config=config.get("aliases_config", None))
def index(request, rest_query):
if request.method == "GET":
if qe is None:
return HttpResponseBadRequest(json.dumps(
{"error": "no database configured"}),
def diff(self,
c1,
c2,
only_missing=False,
only_values=False,
allow_dup=False):
"""Perform a difference between the 2 collections.
The first collection is treated as the previous one, and the second
is treated as the new one.
Note: this is not 'big data'-ready; we assume all the records can fit in memory.
:param c1: Collection (1) config file, or QueryEngine object
:type c1: str or QueryEngine
:param c2: Collection (2) config file, or QueryEngine object
:type c2: str or QueryEngine
:param only_missing: Only find and return self.MISSING; ignore 'new' keys
:param only_values: Only find and return self.CHANGED; ignore new or missing keys
:param allow_dup: Allow duplicate keys, otherwise fail with ValueError
:return: dict with keys self.MISSING, self.NEW (unless only_missing is True), & self.CHANGED,
each a list of records with the key and
any other fields given to the constructor 'info' argument.
The meaning is: 'missing' are keys that are in c1 not found in c2
'new' is keys found in c2 that are not found in c1, and 'changed' are records
with the same key that have different 'props' values.
"""
# Connect.
_log.info("connect.start")
if isinstance(c1, QueryEngine):
engines = [c1, c2]
else:
engines = []
for cfg in c1, c2:
settings = util.get_settings(cfg)
if not normalize_auth(settings):
_log.warn(
"Config file {} does not have a username/password".
format(cfg))
settings["aliases_config"] = {"aliases": {}, "defaults": {}}
engine = QueryEngine(**settings)
engines.append(engine)
_log.info("connect.end")
# Query DB.
keys = [set(), set()]
eqprops = [{}, {}]
numprops = [{}, {}]
# Build query fields.
fields = dict.fromkeys(
self._info + self._all_props + [self._key_field], True)
if not '_id' in fields: # explicitly remove _id if not given
fields['_id'] = False
# Initialize for query loop.
info = {} # per-key information
has_info, has_props = bool(self._info), bool(self._all_props)
has_numprops, has_eqprops = bool(self._prop_deltas), bool(self._props)
_log.info("query.start query={} fields={}".format(
self._filter, fields))
t0 = time.time()
# Main query loop.
for i, coll in enumerate(engines):
_log.debug("collection {:d}".format(i))
count, missing_props = 0, 0
for rec in coll.query(criteria=self._filter, properties=fields):
count += 1
# Extract key from record.
try:
key = rec[self._key_field]
except KeyError:
_log.critical(
"Key '{}' not found in record: {}. Abort.".format(
self._key_field, rec))
return {}
if not allow_dup and key in keys[i]:
raise ValueError("Duplicate key: {}".format(key))
keys[i].add(key)
# Extract numeric properties.
if has_numprops:
pvals = {}
for pkey in self._prop_deltas.keys():
try:
pvals[pkey] = float(rec[pkey])
except KeyError:
#print("@@ missing {} on {}".format(pkey, rec))
missing_props += 1
continue
except (TypeError, ValueError):
raise ValueError(
"Not a number: collection={c} key={k} {p}='{v}'"
.format(k=key,
c=("old", "new")[i],
p=pkey,
v=rec[pkey]))
numprops[i][key] = pvals
# Extract properties for exact match.
if has_eqprops:
try:
propval = tuple([(p, str(rec[p]))
for p in self._props])
except KeyError:
missing_props += 1
#print("@@ missing {} on {}".format(pkey, rec))
continue
eqprops[i][key] = propval
# Extract informational fields.
if has_info:
if key not in info:
info[key] = {}
for k in self._info:
info[key][k] = rec[k]
# Stop if we don't have properties on any record at all
if 0 < count == missing_props:
_log.critical(
"Missing one or more properties on all {:d} records".
format(count))
return {}
# ..but only issue a warning for partially missing properties.
elif missing_props > 0:
_log.warn(
"Missing one or more properties for {:d}/{:d} records".
format(missing_props, count))
t1 = time.time()
_log.info("query.end sec={:f}".format(t1 - t0))
# Compute missing and new keys.
if only_values:
missing, new = [], []
else:
_log.debug("compute_difference.start")
missing, new = keys[0] - keys[1], []
if not only_missing:
new = keys[1] - keys[0]
_log.debug("compute_difference.end")
# Compute mis-matched properties.
if has_props:
changed = self._changed_props(keys,
eqprops,
numprops,
info,
has_eqprops=has_eqprops,
has_numprops=has_numprops)
else:
changed = []
# Build result.
_log.debug("build_result.begin")
result = {}
if not only_values:
result[self.MISSING] = []
for key in missing:
rec = {self._key_field: key}
if has_info:
rec.update(info.get(key, {}))
result[self.MISSING].append(rec)
if not only_missing:
result[self.NEW] = []
for key in new:
rec = {self._key_field: key}
if has_info:
rec.update(info.get(key, {}))
result[self.NEW].append(rec)
result[self.CHANGED] = changed
_log.debug("build_result.end")
return result
from matgendb.query_engine import QueryEngine
from matgendb import dbconfig
import bson
import datetime
from django.utils.encoding import force_unicode
from django.core.serializers.json import DjangoJSONEncoder
qe = None
mgdb_config = os.environ.get("MGDB_CONFIG", "")
if mgdb_config:
config = json.loads(mgdb_config)
if not dbconfig.normalize_auth(config, readonly_first=True):
config["user"] = config["password"] = None
qe = QueryEngine(host=config["host"], port=config["port"],
database=config["database"], user=config["user"],
password=config["password"],
collection=config["collection"],
aliases_config=config.get("aliases_config", None))
def index(request, rest_query):
if request.method == "GET":
if qe is None:
return HttpResponseBadRequest(
json.dumps({"error": "no database configured"}),
mimetype="application/json")
try:
def test_plain(self):
s = {"user": self.U, "password": self.P}
normalize_auth(s)
self._check(s)