def configure(self, conf):
self.snapshot_config = conf
for name, envdict in conf.get("env", {}).items():
# Merge Indexer Config
# ----------------------------------------
dx = envdict["indexer"]
if isinstance(dx, str): # {"indexer": "prod"}
dx = dict(name=dx) # . ↓
if not isinstance(dx, dict): # {"indexer": {"name": "prod"}}
raise TypeError(dx)
# compatibility with previous hubs.
dx.setdefault("name", dx.pop("env", None))
x = self.index_manager[dx["name"]]
x = dict(x) # merge into a copy
merge(x, dx) # <-
envdict["indexer"] = x
# ------------------------------------------
envdict["name"] = name
self.register[name] = SnapshotEnv(self.job_manager, **envdict)
def _snapshot(snapshot):
x = CumulativeResult()
build_doc = self._doc(index)
cfg = self.repcfg.format(build_doc)
for step in ("pre", "snapshot", "post"):
state = registrar.dispatch(step) # _TaskState Class
state = state(get_src_build(), build_doc.get("_id"))
logging.info(state)
state.started()
job = yield from self.job_manager.defer_to_thread(
self.pinfo.get_pinfo(step, snapshot),
partial(getattr(self, state.func), cfg, index, snapshot))
try:
dx = yield from job
dx = StepResult(dx)
except Exception as exc:
logging.exception(exc)
state.failed({}, exc)
raise exc
else:
merge(x.data, dx.data)
logging.info(dx)
logging.info(x)
state.succeed({snapshot: x.data}, res=dx.data)
return x
def _finished(self, _doc, _job):
doc = self._col.find_one({'_id': self._id})
job = doc["jobs"][-1]
t0 = job["step_started_at"].timestamp()
job["time_in_s"] = round(time() - t0, 0)
job["time"] = timesofar(t0)
if self.regx:
merge(doc, _doc)
merge(job, _job)
self._col.replace_one({"_id": self._id}, doc)
def _update_one(self, doc, update, *args, **kwargs):
if args or kwargs:
raise NotImplementedError()
if not len(update) == 1:
raise ValueError("Invalid operator.")
if next(iter(update)) not in ("$set", "$unset", "$push", "$addToSet",
"$pull"):
raise NotImplementedError(next(iter(update)))
# https://docs.mongodb.com/manual/reference/operator/update/set/
# https://docs.mongodb.com/manual/reference/operator/update/unset/
# https://docs.mongodb.com/manual/reference/operator/update/push/
# https://docs.mongodb.com/manual/reference/operator/update/addToSet/
# https://docs.mongodb.com/manual/reference/operator/update/pull/
if "$set" in update:
_update = json.loads(to_json(update["$set"]))
_update = parse_dot_fields(_update)
doc = update_dict_recur(doc, _update)
elif "$unset" in update:
for dotk, v in traverse(doc):
if dotk in update["$unset"]:
v["__REMOVE__"] = True
doc = merge({}, doc)
elif "$push" in update:
for key, val in update["$push"].items():
if "." in key: # not all mongo operators are fully implemented
raise NotImplementedError("nested key in $push: %s" % key)
doc.setdefault(key, []).append(val)
elif "$addToSet" in update:
for key, val in update["$addToSet"].items():
if "." in key: # not all mongo operators are fully implemented
raise NotImplementedError("nested key in $addToSet: %s" %
key)
field = doc.setdefault(key, [])
if val not in field:
field.append(val)
else: # "$pull" in update:
for key, val in update["$pull"].items():
if "." in key: # not all mongo operators are fully implemented
raise NotImplementedError("nested key in $pull: %s" % key)
if not isinstance(val, (str, int)):
raise NotImplementedError(
"value or condition in $pull: %s" % val)
if isinstance(doc.get(key), list):
doc[key][:] = [x for x in doc[key] if x != val]
self._write_one(doc)
def test_merge_5():
x = {"index": {"X": {"env": "local"}, "Y": {"env": "local"}}}
y = {"index": {"X": {"__REMOVE__": True}}}
merge(x, y)
print(x)
def test_merge_4():
x = {"a": {"__REPLACE__": True, "b": "c"}, "__REPLACE__": True}
y = {"a": {"b": "d"}}
merge(x, y)
print(x)
def test_merge_0():
x = {}
y = {}
merge(x, y)
print(x)
def test_merge_3():
x = {"a": "b"}
y = {"a": {"b": "c"}}
merge(x, y)
print(x)
def test_merge_2():
x = {"a": {"b": "c"}}
y = {"a": {"__REPLACE__": True, "B": {"__REPLACE__": False, "c": "d"}}}
merge(x, y)
print(x)
def test_merge_1():
x = {"index": {"name1": {"doc_type": "news", "happy": False}}}
y = {"index": {"name1": {"happy": True, "count": 100}}}
merge(x, y)
print(x)