def process(self, ctx, m): keys = self._resolve_lookup_keys(ctx, m) cache_key = tuple(sorted(keys.items())) result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED) if result != CachedTableLookup.NOT_CACHED: self.cache_hits = self.cache_hits + 1 else: self.cache_misses = self.cache_misses + 1 result = self._do_lookup(ctx, m, keys) self._cache.put(cache_key, result) if result: Eval.process_evals(ctx, m, self.mappings, result) else: print(self.table._rows) raise Exception("No rows found when looking up in %s: %s" % (self, keys)) m.update( {k: ctx.interpolate(v, m) for k, v in self.default.items()}) if (ctx.debug2): logger.debug("Cache table lookup (lookup=%s): %s" % (keys, result)) yield m
def process(self, ctx, mo): logger.debug ("MultiEval (%s mappings)" % len(self.instances)) for instance in self.instances: m = ctx.copy_message(mo) Eval.process_evals(ctx, m, instance) yield m
def process(self, ctx, m): keys = self._resolve_lookup_keys(ctx, m) result = self._do_lookup(ctx, m, keys) if (result): Eval.process_mappings(ctx, m, self.mappings, result) else: m.update({ k: ctx.interpolate(m, v) for k,v in self.default.items() }) yield m
def process(self, ctx, m): keys = self._resolve_lookup_keys(ctx, m) result = self._do_lookup(ctx, m, keys) if (result): Eval.process_evals(ctx, m, self.mappings, result) else: m.update( {k: ctx.interpolate(m, v) for k, v in self.default.items()}) yield m
def process(self, ctx, m): keys = self._resolve_lookup_keys(ctx, m) result = self._do_lookup(ctx, m, keys) if (result): Eval.process_evals(ctx, m, self.mappings, result) else: raise Exception("No rows found when looking up in %s: %s" % (self, keys)) m.update( {k: ctx.interpolate(v, m) for k, v in self.default.items()}) yield m
def process(self, ctx, m): keys = self._resolve_lookup_keys(ctx, m) cache_key = tuple(sorted(keys.items())) result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED) if (result != CachedTableLookup.NOT_CACHED): self.cache_hits = self.cache_hits + 1 if (ctx.debug2): logger.debug("Query cache hit: %s" % (result)) else: self.cache_misses = self.cache_misses + 1 result = self._do_lookup(ctx, m, keys) self._cache.put(cache_key, result) if (result): Eval.process_evals(ctx, m, self.mappings, result) else: m.update({ k: ctx.interpolate(m, v) for k, v in self.default.items() }) yield m
def process(self, ctx, m): keys = self._resolve_lookup_keys(ctx, m) cache_key = tuple(sorted(keys.items())) result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED) if (result != CachedTableLookup.NOT_CACHED): self.cache_hits = self.cache_hits + 1 if (ctx.debug2): logger.debug("Query cache hit: %s" % (result)) else: self.cache_misses = self.cache_misses + 1 result = self._do_lookup(ctx, m, keys) self._cache.put(cache_key, result) if (result): Eval.process_evals(ctx, m, self.mappings, result) else: m.update( {k: ctx.interpolate(m, v) for k, v in self.default.items()}) yield m
def process(self, ctx, m): keys = self._resolve_lookup_keys(ctx, m) cache_key = tuple(sorted(keys.items())) result = self._cache.get(cache_key, CachedTableLookup.NOT_CACHED) if result != CachedTableLookup.NOT_CACHED: self.cache_hits = self.cache_hits + 1 else: self.cache_misses = self.cache_misses + 1 result = self._do_lookup(ctx, m, keys) self._cache.put(cache_key, result) if result: Eval.process_evals(ctx, m, self.mappings, result) else: print(self.table._rows) raise Exception("No rows found when looking up in %s: %s" % (self, keys)) m.update({ k: ctx.interpolate(v, m) for k, v in self.default.items() }) if (ctx.debug2): logger.debug("Cache table lookup (lookup=%s): %s" % (keys, result)) yield m
def store(self, ctx, m): # Evaluate evals, but don't store anything Eval.process_evals(ctx, m, self.eval)
def store(self, ctx, m): # Resolve evals Eval.process_evals(ctx, m, self.eval) # Store automatically or include dimensions if (self.auto_store != None): logger.debug("Storing automatically: %s" % (self.auto_store)) for ast in self.auto_store: did = self.olapmapper.entity_mapper(ast).store(ctx, m) # TODO: Review and use PK properly if (did != None): m[ast.name + "_id"] = did elif (isinstance(self.entity, cubetl.olap.Fact)): logger.debug("Storing automatically: %s" % (self.entity.dimensions)) for dim in self.entity.dimensions: did = self.olapmapper.entity_mapper(dim).store(ctx, m) # TODO: review this too, or use rarer prefix if (did != None): m[dim.name + "_id"] = did logger.debug("Storing entity in %s (mode: %s, lookup: %s)" % (self, self.store_mode, self.lookup_cols)) data = {} mappings = self._mappings(ctx) # First try to look it up for mapping in mappings: if (mapping["column"] in self.lookup_cols): if (mapping["type"] != "AutoIncrement"): if (mapping["value"] == None): data[mapping["column"]] = m[mapping["name"]] else: data[mapping["column"]] = ctx.interpolate(m, mapping["value"]) row = None if (self.store_mode == TableMapper.STORE_MODE_LOOKUP): row = self._sqltable.lookup(ctx, data) for mapping in mappings: if (mapping["type"] != "AutoIncrement"): if (mapping["value"] == None): if (not mapping["name"] in m): raise Exception("Field '%s' does not exist in message when assigning data for column %s in %s" % (mapping["name"], mapping["column"], self)) data[mapping["column"]] = m[mapping["name"]] else: data[mapping["column"]] = ctx.interpolate(m, mapping["value"]) if (not row): if (ctx.debug2): logger.debug("Storing data in %s (data: %s)" % (self, data)) if (self.store_mode in [TableMapper.STORE_MODE_LOOKUP, TableMapper.STORE_MODE_INSERT]): row = self._sqltable.insert(ctx, data) else: raise Exception("Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')" % (self, self.store_mode)) else: # Check row is identical for mapping in self._mappings(ctx): if (mapping["type"] != "AutoIncrement"): v1 = row[mapping['column']] v2 = data[mapping['column']] if (isinstance(v1, basestring) or isinstance(v2, basestring)): if (not isinstance(v1, basestring)): v1 = str(v1) if (not isinstance(v2, basestring)): v2 = str(v2) if (v1 != v2): if (mapping["column"] not in self._lookup_changed_fields): logger.warn("%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)" % (self, mapping["column"], v1, v2)) self._lookup_changed_fields.append(mapping["column"]) return row[self.pk(ctx)["column"]]
def store(self, ctx, m): olapmapper = ctx.find(OlapMapper)[0] # Resolve evals Eval.process_evals(ctx, m, self.eval) # Store automatically or include dimensions if self.auto_store is not None: logger.debug("Storing automatically: %s" % (self.auto_store)) for ast in self.auto_store: did = olapmapper.entity_mapper(ast).store(ctx, m) # FIXME: Review and use PK m[ast.name + "_id"] = did else: dimensions = self.entity.get_dimensions() if dimensions: logger.debug("Storing automatically: %s" % ([da.name for da in self.entity.get_dimensions()])) for dim_attr in self.entity.get_dimensions(): dim = dim_attr.dimension mapper = olapmapper.entity_mapper(dim, False) if mapper: did = mapper.store(ctx, m) # FIXME: shall use the correct foreign key column according to mappings m[dim_attr.name + "_id"] = did logger.debug("Storing entity in %s (mode: %s, lookup: %s)" % (self, self.store_mode, self.lookup_cols)) data = {} mappings = self.sql_mappings(ctx) # First try to look it up for mapping in mappings: if (mapping.sqlcolumn.name in self.lookup_cols): if (mapping.sqlcolumn.type != "AutoIncrement"): try: data[mapping.sqlcolumn.name] = m[ mapping.sqlcolumn.name] except KeyError as e: raise ETLException( "Could not find key '%s' on message when storing data in %s (fields: %s)." % (mapping.sqlcolumn.name, self.entity, sorted([str(k) for k in m.keys()]))) row = None if (self.store_mode == TableMapper.STORE_MODE_LOOKUP): row = self._sqltable.lookup(ctx, data) for mapping in mappings: #print(mapping.sqlcolumn.name + "->" + mapping.field.name) if (mapping.sqlcolumn.type != "AutoIncrement"): if mapping.sqlcolumn.name not in m: raise Exception( "Key '%s' does not exist in message when assigning data for column %s in %s (fields: %s)" % (mapping.field.name, mapping.sqlcolumn.name, self, [f for f in m.keys()])) data[mapping.sqlcolumn.name] = m[mapping.sqlcolumn.name] if (not row): if (ctx.debug2): logger.debug("Storing data in %s (data: %s)" % (self, data)) if (self.store_mode in [ TableMapper.STORE_MODE_LOOKUP, TableMapper.STORE_MODE_INSERT ]): row = self._sqltable.insert(ctx, data) else: raise Exception( "Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')" % (self, self.store_mode)) else: # Check row is identical to issue a warning # TODO: this shall be optional, check is expensive (no check, warning, fail) for mapping in mappings: if mapping.sqlcolumn.sqltable != self.sqltable: continue if mapping.sqlcolumn.type != "AutoIncrement": v1 = row[mapping.sqlcolumn.name] v2 = data[mapping.sqlcolumn.name] if (isinstance(v1, str) or isinstance(v2, str)): if not isinstance(v1, str): v1 = str(v1) if not isinstance(v2, str): v2 = str(v2) if v1 != v2: # Give warning just one time for each field if (mapping.sqlcolumn not in self._lookup_changed_fields): logger.warn( "%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)" % (self, mapping.sqlcolumn, v1, v2)) self._lookup_changed_fields.append( mapping.sqlcolumn) pk = self.pk(ctx) return row[pk.sqlcolumn.name] if pk else None
def store(self, ctx, m): olapmapper = ctx.find(OlapMapper)[0] # Resolve evals Eval.process_evals(ctx, m, self.eval) # Store automatically or include dimensions if self.auto_store is not None: logger.debug("Storing automatically: %s" % (self.auto_store)) for ast in self.auto_store: did = olapmapper.entity_mapper(ast).store(ctx, m) # FIXME: Review and use PK m[ast.name + "_id"] = did else: dimensions = self.entity.get_dimensions() if dimensions: logger.debug("Storing automatically: %s" % ([da.name for da in self.entity.get_dimensions()])) for dim_attr in self.entity.get_dimensions(): dim = dim_attr.dimension mapper = olapmapper.entity_mapper(dim, False) if mapper: did = mapper.store(ctx, m) # FIXME: shall use the correct foreign key column according to mappings m[dim_attr.name + "_id"] = did logger.debug("Storing entity in %s (mode: %s, lookup: %s)" % (self, self.store_mode, self.lookup_cols)) data = {} mappings = self.sql_mappings(ctx) # First try to look it up for mapping in mappings: if (mapping.sqlcolumn.name in self.lookup_cols): if (mapping.sqlcolumn.type != "AutoIncrement"): try: data[mapping.sqlcolumn.name] = m[mapping.sqlcolumn.name] except KeyError as e: raise ETLException("Could not find key '%s' on message when storing data in %s (fields: %s)." % (mapping.sqlcolumn.name, self.entity, sorted([str(k) for k in m.keys()]))) row = None if (self.store_mode == TableMapper.STORE_MODE_LOOKUP): row = self._sqltable.lookup(ctx, data) for mapping in mappings: #print(mapping.sqlcolumn.name + "->" + mapping.field.name) if (mapping.sqlcolumn.type != "AutoIncrement"): if mapping.sqlcolumn.name not in m: raise Exception("Key '%s' does not exist in message when assigning data for column %s in %s (fields: %s)" % (mapping.field.name, mapping.sqlcolumn.name, self, [f for f in m.keys()])) data[mapping.sqlcolumn.name] = m[mapping.sqlcolumn.name] if (not row): if (ctx.debug2): logger.debug("Storing data in %s (data: %s)" % (self, data)) if (self.store_mode in [TableMapper.STORE_MODE_LOOKUP, TableMapper.STORE_MODE_INSERT]): row = self._sqltable.insert(ctx, data) else: raise Exception("Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')" % (self, self.store_mode)) else: # Check row is identical to issue a warning # TODO: this shall be optional, check is expensive (no check, warning, fail) for mapping in mappings: if mapping.sqlcolumn.sqltable != self.sqltable: continue if mapping.sqlcolumn.type != "AutoIncrement": v1 = row[mapping.sqlcolumn.name] v2 = data[mapping.sqlcolumn.name] if (isinstance(v1, str) or isinstance(v2, str)): if not isinstance(v1, str): v1 = str(v1) if not isinstance(v2, str): v2 = str(v2) if v1 != v2: # Give warning just one time for each field if (mapping.sqlcolumn not in self._lookup_changed_fields): logger.warn("%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)" % (self, mapping.sqlcolumn, v1, v2)) self._lookup_changed_fields.append(mapping.sqlcolumn) pk = self.pk(ctx) return row[pk.sqlcolumn.name] if pk else None
def store(self, ctx, m): # Resolve evals Eval.process_evals(ctx, m, self.eval) # Store automatically or include dimensions if (self.auto_store != None): logger.debug("Storing automatically: %s" % (self.auto_store)) for ast in self.auto_store: did = self.olapmapper.entity_mapper(ast).store(ctx, m) # TODO: Review and use PK properly if (did != None): m[ast.name + "_id"] = did elif (isinstance(self.entity, cubetl.olap.Fact)): logger.debug("Storing automatically: %s" % (self.entity.dimensions)) for dim in self.entity.dimensions: did = self.olapmapper.entity_mapper(dim).store(ctx, m) # TODO: review this too, or use rarer prefix if (did != None): m[dim.name + "_id"] = did logger.debug("Storing entity in %s (mode: %s, lookup: %s)" % (self, self.store_mode, self.lookup_cols)) data = {} mappings = self._mappings(ctx) # First try to look it up for mapping in mappings: if (mapping["column"] in self.lookup_cols): if (mapping["type"] != "AutoIncrement"): if (mapping["value"] == None): data[mapping["column"]] = m[mapping["name"]] else: data[mapping["column"]] = ctx.interpolate( m, mapping["value"]) row = None if (self.store_mode == TableMapper.STORE_MODE_LOOKUP): row = self._sqltable.lookup(ctx, data) for mapping in mappings: if (mapping["type"] != "AutoIncrement"): if (mapping["value"] == None): if (not mapping["name"] in m): raise Exception( "Field '%s' does not exist in message when assigning data for column %s in %s" % (mapping["name"], mapping["column"], self)) data[mapping["column"]] = m[mapping["name"]] else: data[mapping["column"]] = ctx.interpolate( m, mapping["value"]) if (not row): if (ctx.debug2): logger.debug("Storing data in %s (data: %s)" % (self, data)) if (self.store_mode in [ TableMapper.STORE_MODE_LOOKUP, TableMapper.STORE_MODE_INSERT ]): row = self._sqltable.insert(ctx, data) else: raise Exception( "Update store mode used at %s (%s) not implemented (available 'lookup', 'insert')" % (self, self.store_mode)) else: # Check row is identical for mapping in self._mappings(ctx): if (mapping["type"] != "AutoIncrement"): v1 = row[mapping['column']] v2 = data[mapping['column']] if (isinstance(v1, basestring) or isinstance(v2, basestring)): if (not isinstance(v1, basestring)): v1 = str(v1) if (not isinstance(v2, basestring)): v2 = str(v2) if (v1 != v2): if (mapping["column"] not in self._lookup_changed_fields): logger.warn( "%s looked up an entity which exists with different attributes (field=%s, existing_value=%r, tried_value=%r) (reported only once per field)" % (self, mapping["column"], v1, v2)) self._lookup_changed_fields.append( mapping["column"]) return row[self.pk(ctx)["column"]]