def initialize(self, ctx):
super().initialize(ctx)
if self.path is None:
raise ETLConfigurationException("Missing path attribute for %s" %
self)
def initialize(self, ctx):
super().initialize(ctx)
if len(self.steps) <= 0:
raise ETLConfigurationException("Union with no steps.")
for p in self.steps:
ctx.comp.initialize(p)
def initialize(self, comp):
if (not self.is_initialized(comp)):
logger.debug("Initializing %s" % (comp, ))
self.component_desc(comp).initialized = True
comp.initialize(self.ctx)
try:
pass
except AttributeError as e:
raise ETLConfigurationException("Tried to initialize invalid component (%s): %s" % (comp, e))
def include(self, configfile):
configfile = self.interpolate(configfile)
# Import only once
abspath = os.path.abspath(configfile)
if abspath in self.included_files:
return
self.included_files.append(abspath)
logger.info("Including config file: %s", configfile)
spec = importlib.util.spec_from_file_location("configmodule",
configfile)
configmodule = importlib.util.module_from_spec(spec)
try:
spec.loader.exec_module(configmodule)
except FileNotFoundError as e:
raise ETLConfigurationException(
"Config include file not found: %s" % (configfile))
except Exception as e:
raise ETLConfigurationException(
"An error ocurred while loading '%s' config file: %s" %
(configfile, e))
configmodule.cubetl_config(self)
def initialize(self, ctx):
super(FactDimension, self).initialize(ctx)
ctx.comp.initialize(self.fact)
if (len(self.attributes) > 0):
raise Exception(
"Cannot define attributes for a FactDimension (it's defined by the linked fact)"
)
self.dimensions = self.fact.dimensions
self.measures = self.fact.measures
self.attributes = self.fact.attributes
if self.name != self.fact.name:
raise ETLConfigurationException(
"FactDimension %s name is '%s' but it should match name of fact %s ('%s')"
% (self, self.name, self.fact, self.fact.name))
def _mappings(self, ctx):
mappings = [mapping.copy() for mapping in self.mappings]
for mapping in mappings:
if (not "entity" in mapping):
mapping["entity"] = self.entity
for dimension in self.entity.dimensions:
#if (not dimension.name in [mapping["name"] for mapping in self.mappings]):
dimension_mapper = self.olapmapper.entity_mapper(dimension)
dimension_mappings = dimension_mapper._mappings_join(ctx)
# TODO: Check if entity/attribute is already mapped?
self._extend_mappings(ctx, mappings, dimension_mappings)
for measure in self.entity.measures:
self._extend_mappings(ctx, mappings, [{
"name":
measure["name"],
"type":
measure["type"] if
("type" in measure and measure["type"] != None) else "Float",
"entity":
self.entity
}])
for attribute in self.entity.attributes:
try:
self._extend_mappings(ctx, mappings, [{
"name": attribute["name"],
"type": attribute["type"],
"entity": self.entity
}])
except KeyError as e:
raise ETLConfigurationException(
"Definition of attribute %s of %s is missing %s" %
(attribute, self.entity, e))
self._ensure_mappings(ctx, mappings)
return mappings
def initialize(self, ctx):
super().initialize(ctx)
for p in self.steps:
if p is None:
raise ETLConfigurationException("Component %s steps contain a None reference." % self)
ctx.comp.initialize(p)
def initialize(self, ctx):
if hasattr(self, '_initialized'):
raise ETLConfigurationException(
"Component already initialized: %s" % self)
self._initialized = True
self.ctx = ctx
def initialize(self, ctx):
super(SQLTable, self).initialize(ctx)
if self._lookup_changed_fields == None:
self._lookup_changed_fields = []
ctx.comp.initialize(self.connection)
logger.debug("Loading table %s on %s" % (self.name, self))
self.sa_metadata = MetaData()
self.sa_table = Table(self.name, self.sa_metadata)
self._selects = 0
self._inserts = 0
self._updates = 0
self._unicode_errors = 0
# Drop?
columns_ex = []
for column in self.columns:
logger.debug("Adding column to %s: %s" % (self, column))
column.sqltable = self
# Check for duplicate names
if (column.name in columns_ex):
raise ETLConfigurationException(
"Duplicate column name '%s' in %s" % (column.name, self))
columns_ex.append(column.name)
# Configure column
if isinstance(column, SQLColumnFK):
if column.fk_sqlcolumn.sqltable.sa_table is None:
logger.warning(
"Column %s foreign key %s table (%s) has not been defined in backend (ignoring).",
column, column.fk_sqlcolumn,
column.fk_sqlcolumn.sqltable)
continue
self.sa_table.append_column(
Column(column.name,
self._get_sa_type(column),
ForeignKey(
column.fk_sqlcolumn.sqltable.sa_table.columns[
column.fk_sqlcolumn.name]),
primary_key=column.pk,
nullable=column.nullable,
autoincrement=(True if column.type
== "AutoIncrement" else False)))
else:
self.sa_table.append_column(
Column(column.name,
self._get_sa_type(column),
primary_key=column.pk,
nullable=column.nullable,
autoincrement=(True if column.type
== "AutoIncrement" else False)))
# Check schema:
# Create if doesn't exist
if (not self.connection.engine().has_table(self.name)):
logger.info("Creating table %s" % self.name)
self.sa_table.create(self.connection.connection())
def interpolate(self, value, m=None, data=None):
"""
Resolves expressions `${ ... }`, lambdas and functions in a value,
with respect to the current context and the current message.
Expressions are CubETL custom syntax for string interpolation.
"""
# FIXME: TO BE REMOVED after migrating all interpolate calls
if isinstance(value, dict):
raise ETLException("Possibly invalid interpolate call!")
if value is None:
return None
# If the value is a callable (function or lambda), inspect
# its parameters. Acceptable signatures are:
# (ctx), (m), (ctx, m)
if callable(value):
sig = inspect.signature(value)
paramnames = list(sig.parameters.keys())
if len(sig.parameters) == 1 and paramnames[0] == 'ctx':
value = value(self)
elif len(sig.parameters) == 1 and paramnames[0] == 'm':
value = value(m)
elif len(
sig.parameters
) == 2 and paramnames[0] == 'ctx' and paramnames[1] == 'm':
value = value(self, m)
else:
raise ETLConfigurationException(
"Invalid lambda expression signature: %s" % sig)
# If the value is not a string, it is immediately returned
if not isinstance(value, str):
return value
# Process string values
value = value.strip()
pos = -1
result = str(value)
for dstart, dend in (('${|', '|}'), ('${', '}')):
if (pos >= -1):
pos = result.find(dstart)
while (pos >= 0):
pos_end = result.find(dend)
expr = result[pos + len(dstart):pos_end].strip()
compiled = self._compiled.get(expr)
try:
if (not compiled):
compiled = compile(expr, '', 'eval')
self._compiled.put(expr, compiled)
c_locals = {
"m": m,
"ctx": self,
"f": self.f,
"props": self.props,
"var": self.var,
"cubetl": cubetl
}
if data:
c_locals.update(data)
res = eval(compiled, self._globals, c_locals)
if (self.debug2):
if (isinstance(res, str)):
logger.debug(
'Evaluated: %s = %r' %
(expr, res if
(len(res) < 100) else res[:100] + ".."))
else:
logger.debug('Evaluated: %s = %r' % (expr, res))
except (Exception) as e:
exc_type, exc_value, exc_traceback = sys.exc_info()
caller_component = None
frame = inspect.currentframe()
for caller in inspect.getouterframes(frame):
fc = Context._class_from_frame(caller[0])
if (isclass(fc) and issubclass(fc, Component)):
caller_component = caller[0].f_locals['self']
break
#logger.error("Error evaluating expression %s on data: %s" % (expr, m))
self._eval_error_message = m
logger.error(
'Error evaluating expression "%s" called from %s:\n%s'
% (expr, caller_component, ("".join(
traceback.format_exception_only(
exc_type, exc_value)))))
raise
if (pos > 0) or (pos_end < len(result) - (len(dend))):
result = result[0:pos] + str(res) + result[pos_end +
(len(dend)):]
pos = result.find(dstart)
else:
# Keep type of non-string types
result = res
pos = -2
return result
def _get_mappings(self, ctx, mapper, base_mapper = None, parent_mapper = None):
"""
Calculates Cubes mappings, which map Cubes entity attributes to database
tables. This is used in the 'mappings' section of Cubes configuration.
Returns a dictionary where keys are the mapped entity.attribute and
values are the alias.column.
"""
logger.debug("Exporting mappings: %s" % mapper)
c_mappings = {}
if (base_mapper == None):
base_mapper = mapper
if (isinstance(mapper, FactMapper)):
for dim in mapper.entity.dimensions:
dim_mapper = mapper.olapmapper.entity_mapper(dim)
sub_mappings = self._get_mappings(ctx, dim_mapper, base_mapper, mapper)
if isinstance(mapper.entity, AliasDimension):
# For AliasDimensions, the table is aliased
for sm in sub_mappings.items():
# Account for "extracted" fields, that return a dictionary:
if isinstance(sm[1], dict):
c_mappings[sm[0]] = { "column": mapper.entity.name + "." + sm[1]["column"].split(".")[1], "extract": sm[1]["extract"] }
else:
c_mappings[sm[0]] = mapper.entity.name + "." + sm[1].split(".")[1]
else:
# XXX mapper.olapmapper.entity_mapper(join["detail_entity"]).entity.name
c_mappings.update(sub_mappings)
elif (isinstance(mapper, MultiTableHierarchyDimensionMapper)):
for dim in mapper.entity.levels:
dim_mapper = self.olapmapper.entity_mapper(dim, False)
sub_mappings = self._get_mappings(ctx, dim_mapper, base_mapper, mapper)
for k,v in sub_mappings.items():
mapping_entityattribute = mapper.entity.name + "." + k.split(".")[1]
if mapping_entityattribute in c_mappings:
raise ETLConfigurationException("Attribute '%s' in %s is being mapped more than once. This can happen if the same column name is used by more than one levels in the hierarchy (hint: this happens often due to the same 'id' or 'names' attribute name being used by several dimensions in the hierarchy: use different names)." % (mapping_entityattribute, mapper))
c_mappings[mapping_entityattribute] = v
elif (isinstance(mapper, EmbeddedDimensionMapper)):
mappings = mapper._mappings_join(ctx)
for mapping in mappings:
if "extract" in mapping:
c_mappings[mapper.entity.name + "." + mapping["name"]] = {
"column": parent_mapper.table + "." + mapping["column"],
"extract": mapping["extract"]
}
else:
c_mappings[mapper.entity.name + "." + mapping["name"]] = parent_mapper.table + "." + mapping["column"]
elif (isinstance(mapper, DimensionMapper) or (isinstance(mapper, CompoundHierarchyDimensionMapper))):
mappings = mapper._mappings(ctx)
for mapping in mappings:
c_mappings[mapper.entity.name + "." + mapping["name"]] = mapper.entity.name + "." + mapping["column"]
else:
raise Exception ("Unknown mapper type for cubes export: %s" % mapper)
"""
joins = mapper._joins(ctx, None)
mappings = mapper._mappings(ctx)
#logger.info( mapper)
#logger.info( mappings)
for mapping in mappings:
#if (mapping["entity"] == mapper.entity):
# c_mappings[mapping["entity"].name + "." + mapping["name"]] = mapper.table + "." + mapping["column"]
if (mapping["entity"] not in [join["detail_entity"] for join in joins]):
c_mappings[mapping["entity"].name + "." + mapping["name"]] = mapper.table + "." + mapping["column"]
else:
dim_mapper = mapper.olapmapper.entity_mapper(mapping["entity"])
sub_mappings = self._get_mappings(ctx, dim_mapper)
if (isinstance(dim_mapper, MultiTableHierarchyDimensionMapper)):
# Enforce current entity (for hierarchies, this causes that all levels are
# mapped for the current dimension, forcing them to have different attribute names)
for k,v in sub_mappings.items():
c_mappings[mapping["entity"].name + "." + k.split(".")[1]] = v
else:
c_mappings.update(sub_mappings)
"""
return c_mappings