def process(self, ctx, m):
keys = self._resolve_lookup_keys(ctx, m)
cache_key = tuple(sorted(keys.items()))
result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED)
if result != CachedTableLookup.NOT_CACHED:
self.cache_hits = self.cache_hits + 1
else:
self.cache_misses = self.cache_misses + 1
result = self._do_lookup(ctx, m, keys)
self._cache.put(cache_key, result)
if result:
Eval.process_evals(ctx, m, self.mappings, result)
else:
print(self.table._rows)
raise Exception("No rows found when looking up in %s: %s" %
(self, keys))
m.update(
{k: ctx.interpolate(v, m)
for k, v in self.default.items()})
if (ctx.debug2):
logger.debug("Cache table lookup (lookup=%s): %s" % (keys, result))
yield m
def process(self, ctx, mo):
logger.debug ("MultiEval (%s mappings)" % len(self.instances))
for instance in self.instances:
m = ctx.copy_message(mo)
Eval.process_evals(ctx, m, instance)
yield m
def process(self, ctx, m):
keys = self._resolve_lookup_keys(ctx, m)
result = self._do_lookup(ctx, m, keys)
if (result):
Eval.process_mappings(ctx, m, self.mappings, result)
else:
m.update({ k: ctx.interpolate(m, v) for k,v in self.default.items() })
yield m
def process(self, ctx, m):
keys = self._resolve_lookup_keys(ctx, m)
result = self._do_lookup(ctx, m, keys)
if (result):
Eval.process_evals(ctx, m, self.mappings, result)
else:
m.update(
{k: ctx.interpolate(m, v)
for k, v in self.default.items()})
yield m
def process(self, ctx, m):
keys = self._resolve_lookup_keys(ctx, m)
result = self._do_lookup(ctx, m, keys)
if (result):
Eval.process_evals(ctx, m, self.mappings, result)
else:
raise Exception("No rows found when looking up in %s: %s" %
(self, keys))
m.update(
{k: ctx.interpolate(v, m)
for k, v in self.default.items()})
yield m
def process(self, ctx, m):
keys = self._resolve_lookup_keys(ctx, m)
cache_key = tuple(sorted(keys.items()))
result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED)
if (result != CachedTableLookup.NOT_CACHED):
self.cache_hits = self.cache_hits + 1
if (ctx.debug2):
logger.debug("Query cache hit: %s" % (result))
else:
self.cache_misses = self.cache_misses + 1
result = self._do_lookup(ctx, m, keys)
self._cache.put(cache_key, result)
if (result):
Eval.process_evals(ctx, m, self.mappings, result)
else:
m.update({ k: ctx.interpolate(m, v) for k, v in self.default.items() })
yield m
def process(self, ctx, m):
keys = self._resolve_lookup_keys(ctx, m)
cache_key = tuple(sorted(keys.items()))
result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED)
if (result != CachedTableLookup.NOT_CACHED):
self.cache_hits = self.cache_hits + 1
if (ctx.debug2):
logger.debug("Query cache hit: %s" % (result))
else:
self.cache_misses = self.cache_misses + 1
result = self._do_lookup(ctx, m, keys)
self._cache.put(cache_key, result)
if (result):
Eval.process_evals(ctx, m, self.mappings, result)
else:
m.update(
{k: ctx.interpolate(m, v)
for k, v in self.default.items()})
yield m
def process(self, ctx, m):
keys = self._resolve_lookup_keys(ctx, m)
cache_key = tuple(sorted(keys.items()))
result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED)
if result != CachedTableLookup.NOT_CACHED:
self.cache_hits = self.cache_hits + 1
else:
self.cache_misses = self.cache_misses + 1
result = self._do_lookup(ctx, m, keys)
self._cache.put(cache_key, result)
if result:
Eval.process_evals(ctx, m, self.mappings, result)
else:
print(self.table._rows)
raise Exception("No rows found when looking up in %s: %s" % (self, keys))
m.update({ k: ctx.interpolate(v, m) for k, v in self.default.items() })
if (ctx.debug2):
logger.debug("Cache table lookup (lookup=%s): %s" % (keys, result))
yield m
def store(self, ctx, m):
# Evaluate evals, but don't store anything
Eval.process_evals(ctx, m, self.eval)
def store(self, ctx, m):
# Resolve evals
Eval.process_evals(ctx, m, self.eval)
# Store automatically or include dimensions
if (self.auto_store != None):
logger.debug("Storing automatically: %s" % (self.auto_store))
for ast in self.auto_store:
did = self.olapmapper.entity_mapper(ast).store(ctx, m)
# TODO: Review and use PK properly
if (did != None):
m[ast.name + "_id"] = did
elif (isinstance(self.entity, cubetl.olap.Fact)):
logger.debug("Storing automatically: %s" % (self.entity.dimensions))
for dim in self.entity.dimensions:
did = self.olapmapper.entity_mapper(dim).store(ctx, m)
# TODO: review this too, or use rarer prefix
if (did != None):
m[dim.name + "_id"] = did
logger.debug("Storing entity in %s (mode: %s, lookup: %s)" % (self, self.store_mode, self.lookup_cols))
data = {}
mappings = self._mappings(ctx)
# First try to look it up
for mapping in mappings:
if (mapping["column"] in self.lookup_cols):
if (mapping["type"] != "AutoIncrement"):
if (mapping["value"] == None):
data[mapping["column"]] = m[mapping["name"]]
else:
data[mapping["column"]] = ctx.interpolate(m, mapping["value"])
row = None
if (self.store_mode == TableMapper.STORE_MODE_LOOKUP):
row = self._sqltable.lookup(ctx, data)
for mapping in mappings:
if (mapping["type"] != "AutoIncrement"):
if (mapping["value"] == None):
if (not mapping["name"] in m):
raise Exception("Field '%s' does not exist in message when assigning data for column %s in %s" % (mapping["name"], mapping["column"], self))
data[mapping["column"]] = m[mapping["name"]]
else:
data[mapping["column"]] = ctx.interpolate(m, mapping["value"])
if (not row):
if (ctx.debug2):
logger.debug("Storing data in %s (data: %s)" % (self, data))
if (self.store_mode in [TableMapper.STORE_MODE_LOOKUP, TableMapper.STORE_MODE_INSERT]):
row = self._sqltable.insert(ctx, data)
else:
raise Exception("Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')" % (self, self.store_mode))
else:
# Check row is identical
for mapping in self._mappings(ctx):
if (mapping["type"] != "AutoIncrement"):
v1 = row[mapping['column']]
v2 = data[mapping['column']]
if (isinstance(v1, basestring) or isinstance(v2, basestring)):
if (not isinstance(v1, basestring)):
v1 = str(v1)
if (not isinstance(v2, basestring)):
v2 = str(v2)
if (v1 != v2):
if (mapping["column"] not in self._lookup_changed_fields):
logger.warn("%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)" % (self, mapping["column"], v1, v2))
self._lookup_changed_fields.append(mapping["column"])
return row[self.pk(ctx)["column"]]
def store(self, ctx, m):
olapmapper = ctx.find(OlapMapper)[0]
# Resolve evals
Eval.process_evals(ctx, m, self.eval)
# Store automatically or include dimensions
if self.auto_store is not None:
logger.debug("Storing automatically: %s" % (self.auto_store))
for ast in self.auto_store:
did = olapmapper.entity_mapper(ast).store(ctx, m)
# FIXME: Review and use PK
m[ast.name + "_id"] = did
else:
dimensions = self.entity.get_dimensions()
if dimensions:
logger.debug("Storing automatically: %s" %
([da.name
for da in self.entity.get_dimensions()]))
for dim_attr in self.entity.get_dimensions():
dim = dim_attr.dimension
mapper = olapmapper.entity_mapper(dim, False)
if mapper:
did = mapper.store(ctx, m)
# FIXME: shall use the correct foreign key column according to mappings
m[dim_attr.name + "_id"] = did
logger.debug("Storing entity in %s (mode: %s, lookup: %s)" %
(self, self.store_mode, self.lookup_cols))
data = {}
mappings = self.sql_mappings(ctx)
# First try to look it up
for mapping in mappings:
if (mapping.sqlcolumn.name in self.lookup_cols):
if (mapping.sqlcolumn.type != "AutoIncrement"):
try:
data[mapping.sqlcolumn.name] = m[
mapping.sqlcolumn.name]
except KeyError as e:
raise ETLException(
"Could not find key '%s' on message when storing data in %s (fields: %s)."
% (mapping.sqlcolumn.name, self.entity,
sorted([str(k) for k in m.keys()])))
row = None
if (self.store_mode == TableMapper.STORE_MODE_LOOKUP):
row = self._sqltable.lookup(ctx, data)
for mapping in mappings:
#print(mapping.sqlcolumn.name + "->" + mapping.field.name)
if (mapping.sqlcolumn.type != "AutoIncrement"):
if mapping.sqlcolumn.name not in m:
raise Exception(
"Key '%s' does not exist in message when assigning data for column %s in %s (fields: %s)"
% (mapping.field.name, mapping.sqlcolumn.name, self,
[f for f in m.keys()]))
data[mapping.sqlcolumn.name] = m[mapping.sqlcolumn.name]
if (not row):
if (ctx.debug2):
logger.debug("Storing data in %s (data: %s)" % (self, data))
if (self.store_mode in [
TableMapper.STORE_MODE_LOOKUP,
TableMapper.STORE_MODE_INSERT
]):
row = self._sqltable.insert(ctx, data)
else:
raise Exception(
"Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')"
% (self, self.store_mode))
else:
# Check row is identical to issue a warning
# TODO: this shall be optional, check is expensive (no check, warning, fail)
for mapping in mappings:
if mapping.sqlcolumn.sqltable != self.sqltable:
continue
if mapping.sqlcolumn.type != "AutoIncrement":
v1 = row[mapping.sqlcolumn.name]
v2 = data[mapping.sqlcolumn.name]
if (isinstance(v1, str) or isinstance(v2, str)):
if not isinstance(v1, str):
v1 = str(v1)
if not isinstance(v2, str):
v2 = str(v2)
if v1 != v2:
# Give warning just one time for each field
if (mapping.sqlcolumn
not in self._lookup_changed_fields):
logger.warn(
"%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)"
% (self, mapping.sqlcolumn, v1, v2))
self._lookup_changed_fields.append(
mapping.sqlcolumn)
pk = self.pk(ctx)
return row[pk.sqlcolumn.name] if pk else None
def store(self, ctx, m):
olapmapper = ctx.find(OlapMapper)[0]
# Resolve evals
Eval.process_evals(ctx, m, self.eval)
# Store automatically or include dimensions
if self.auto_store is not None:
logger.debug("Storing automatically: %s" % (self.auto_store))
for ast in self.auto_store:
did = olapmapper.entity_mapper(ast).store(ctx, m)
# FIXME: Review and use PK
m[ast.name + "_id"] = did
else:
dimensions = self.entity.get_dimensions()
if dimensions:
logger.debug("Storing automatically: %s" % ([da.name for da in self.entity.get_dimensions()]))
for dim_attr in self.entity.get_dimensions():
dim = dim_attr.dimension
mapper = olapmapper.entity_mapper(dim, False)
if mapper:
did = mapper.store(ctx, m)
# FIXME: shall use the correct foreign key column according to mappings
m[dim_attr.name + "_id"] = did
logger.debug("Storing entity in %s (mode: %s, lookup: %s)" % (self, self.store_mode, self.lookup_cols))
data = {}
mappings = self.sql_mappings(ctx)
# First try to look it up
for mapping in mappings:
if (mapping.sqlcolumn.name in self.lookup_cols):
if (mapping.sqlcolumn.type != "AutoIncrement"):
try:
data[mapping.sqlcolumn.name] = m[mapping.sqlcolumn.name]
except KeyError as e:
raise ETLException("Could not find key '%s' on message when storing data in %s (fields: %s)." % (mapping.sqlcolumn.name, self.entity, sorted([str(k) for k in m.keys()])))
row = None
if (self.store_mode == TableMapper.STORE_MODE_LOOKUP):
row = self._sqltable.lookup(ctx, data)
for mapping in mappings:
#print(mapping.sqlcolumn.name + "->" + mapping.field.name)
if (mapping.sqlcolumn.type != "AutoIncrement"):
if mapping.sqlcolumn.name not in m:
raise Exception("Key '%s' does not exist in message when assigning data for column %s in %s (fields: %s)" % (mapping.field.name, mapping.sqlcolumn.name, self, [f for f in m.keys()]))
data[mapping.sqlcolumn.name] = m[mapping.sqlcolumn.name]
if (not row):
if (ctx.debug2):
logger.debug("Storing data in %s (data: %s)" % (self, data))
if (self.store_mode in [TableMapper.STORE_MODE_LOOKUP, TableMapper.STORE_MODE_INSERT]):
row = self._sqltable.insert(ctx, data)
else:
raise Exception("Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')" % (self, self.store_mode))
else:
# Check row is identical to issue a warning
# TODO: this shall be optional, check is expensive (no check, warning, fail)
for mapping in mappings:
if mapping.sqlcolumn.sqltable != self.sqltable:
continue
if mapping.sqlcolumn.type != "AutoIncrement":
v1 = row[mapping.sqlcolumn.name]
v2 = data[mapping.sqlcolumn.name]
if (isinstance(v1, str) or isinstance(v2, str)):
if not isinstance(v1, str):
v1 = str(v1)
if not isinstance(v2, str):
v2 = str(v2)
if v1 != v2:
# Give warning just one time for each field
if (mapping.sqlcolumn not in self._lookup_changed_fields):
logger.warn("%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)" % (self, mapping.sqlcolumn, v1, v2))
self._lookup_changed_fields.append(mapping.sqlcolumn)
pk = self.pk(ctx)
return row[pk.sqlcolumn.name] if pk else None
def store(self, ctx, m):
# Evaluate evals, but don't store anything
Eval.process_evals(ctx, m, self.eval)
def store(self, ctx, m):
# Resolve evals
Eval.process_evals(ctx, m, self.eval)
# Store automatically or include dimensions
if (self.auto_store != None):
logger.debug("Storing automatically: %s" % (self.auto_store))
for ast in self.auto_store:
did = self.olapmapper.entity_mapper(ast).store(ctx, m)
# TODO: Review and use PK properly
if (did != None):
m[ast.name + "_id"] = did
elif (isinstance(self.entity, cubetl.olap.Fact)):
logger.debug("Storing automatically: %s" %
(self.entity.dimensions))
for dim in self.entity.dimensions:
did = self.olapmapper.entity_mapper(dim).store(ctx, m)
# TODO: review this too, or use rarer prefix
if (did != None):
m[dim.name + "_id"] = did
logger.debug("Storing entity in %s (mode: %s, lookup: %s)" %
(self, self.store_mode, self.lookup_cols))
data = {}
mappings = self._mappings(ctx)
# First try to look it up
for mapping in mappings:
if (mapping["column"] in self.lookup_cols):
if (mapping["type"] != "AutoIncrement"):
if (mapping["value"] == None):
data[mapping["column"]] = m[mapping["name"]]
else:
data[mapping["column"]] = ctx.interpolate(
m, mapping["value"])
row = None
if (self.store_mode == TableMapper.STORE_MODE_LOOKUP):
row = self._sqltable.lookup(ctx, data)
for mapping in mappings:
if (mapping["type"] != "AutoIncrement"):
if (mapping["value"] == None):
if (not mapping["name"] in m):
raise Exception(
"Field '%s' does not exist in message when assigning data for column %s in %s"
% (mapping["name"], mapping["column"], self))
data[mapping["column"]] = m[mapping["name"]]
else:
data[mapping["column"]] = ctx.interpolate(
m, mapping["value"])
if (not row):
if (ctx.debug2):
logger.debug("Storing data in %s (data: %s)" % (self, data))
if (self.store_mode in [
TableMapper.STORE_MODE_LOOKUP,
TableMapper.STORE_MODE_INSERT
]):
row = self._sqltable.insert(ctx, data)
else:
raise Exception(
"Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')"
% (self, self.store_mode))
else:
# Check row is identical
for mapping in self._mappings(ctx):
if (mapping["type"] != "AutoIncrement"):
v1 = row[mapping['column']]
v2 = data[mapping['column']]
if (isinstance(v1, basestring)
or isinstance(v2, basestring)):
if (not isinstance(v1, basestring)):
v1 = str(v1)
if (not isinstance(v2, basestring)):
v2 = str(v2)
if (v1 != v2):
if (mapping["column"]
not in self._lookup_changed_fields):
logger.warn(
"%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)"
% (self, mapping["column"], v1, v2))
self._lookup_changed_fields.append(
mapping["column"])
return row[self.pk(ctx)["column"]]
