def store_modified_at(trendstore, datapackage, modified):
def set_modified(state):
state["modified"] = modified
partition = trendstore.partition(datapackage.timestamp)
set_modified_action = UpdateState(set_modified)
copy_from = CopyFrom(k(partition), k(datapackage))
return DbTransaction(set_modified_action, copy_from)
def create_dialect(conf):
if conf["type"] == "custom":
return k(create_custom(conf["config"]))
elif conf["type"] == "auto":
return sniff_dialect
elif conf["type"] == "prime":
return k(Prime)
else:
raise Exception("Unsupported dialect type")
def create_dialect(conf):
if conf["type"] == "custom":
return k(create_custom(conf["config"]))
elif conf["type"] == "auto":
return sniff_dialect
elif conf["type"] == "prime":
return k(Prime)
else:
raise Exception("Unsupported dialect type")
def store(partition, datapackage):
if len(datapackage.rows) <= LARGE_BATCH_THRESHOLD:
insert_action = BatchInsert
else:
insert_action = CopyFrom
transaction = DbTransaction(
GetTimestamp(),
insert_action(k(partition), k(datapackage)),
MarkModified(k(partition), k(datapackage.timestamp)))
return transaction
def store_raw(self, raw_datapackage):
if raw_datapackage.is_empty():
return DbTransaction()
return DbTransaction(
RefineRawDataPackage(k(raw_datapackage)),
StoreBatch(self, read("datapackage"))
)
def none_or(if_none=k(None), if_value=identity):
def fn(value):
if value is None:
return if_none()
else:
return if_value(value)
return fn
def test_k():
"""
The result of the created function should always be the same
"""
r = k(5)
assert_equal(r(), 5)
assert_equal(r(2), 5)
assert_equal(r("hello", 3), 5)
def test_retry_while():
state = {
"loop": 1,
"val": 10}
exception_handlers = {
Exception: no_op}
def fn():
curr_loop = state["loop"]
state["loop"] = curr_loop + 1
if curr_loop == 1:
raise Exception()
state["val"] = 42
retry_while(fn, exception_handlers, condition=k(True), timeout=k(1.0))
assert_equal(state["val"], 42)
def render(self):
columns_part = ", ".join(c.render() for c in self.columns)
args_part = ", ".join(map(k("%s"), self.columns))
query = (
"INSERT INTO {}({}) "
"VALUES ({})").format(self.into.render(), columns_part, args_part)
if self._returning:
query += " RETURNING {}".format(self._returning)
return query
def store_raw(self, raw_datapackage):
if raw_datapackage.is_empty():
return DbTransaction()
if len(raw_datapackage.rows) <= LARGE_BATCH_THRESHOLD:
insert_action = BatchInsert
else:
insert_action = CopyFrom
return DbTransaction(
RefineRawDataPackage(k(raw_datapackage)),
SetTimestamp(read("datapackage")),
SetPartition(self),
GetTimestamp(),
insert_action(read("partition"), read("datapackage")),
MarkModified(read("partition"), read("timestamp"))
)
def store_txn(self, datapackage):
"""Return transaction to store the data in the attributestore."""
return DbTransaction(StoreBatch(self, k(datapackage)))