def _verify_connection_definitions(self):
"""
Verifies if connection configuration is complete
"""
if self._conf_defs['connection_type'] in [DSN, CONNECTION_STRING]:
if not self._conf_defs['connection_string']:
raise HydroException('Connection dsn is Null')
else:
for att in ('db_user', 'db_password', 'connection_string'):
if not self._conf_defs.get(att):
raise HydroException('Connection {0} is Null'.format(att))
def _connect(self):
if self._conf_defs['connection_type'] == DSN:
conn_string = 'DSN=%s' % self._conf_defs['connection_string']
self.logger.debug('Connect to {0}'.format(conn_string))
self._conn = pyodbc.connect(conn_string, unicode_results=True)
elif self._conf_defs['connection_type'] == CONNECTION_STRING:
self._conn = pyodbc.connect(self._conf_defs['connection_string'], unicode_results=True)
else:
#TODO need to be implemented based on connection string
raise HydroException("Vertica connection string connection is not implemented")
def submit(self, name, params=None):
#Todo get topology from cache
topology = self._topologies.get(name, None)
if not topology:
raise HydroException("Topology doesn't exist")
#setting 2 call back functions
#set_topology_lookup_callback to let the query engine to lookup topologies in order to use them as a stream
topology.query_engine.set_topology_lookup_callback(
self.return_topology_callback_if_exist)
#set topology cache ttl for not being bigger than the minimum of the query engine streams
topology.query_engine.set_topology_cache_ttl_callback(
topology.topology_cache_ttl_callback)
data = topology.submit(deepcopy(params))
execution_plan = topology.get_execution_plan()
return ResultSet(execution_plan, data)
def get_connector_class(self, conn_conf):
"""
dynamically get connector class and instantiate
"""
# TODO: allow loading more connectors provided by the user and not just the built in ones
path = "hydro.connectors.%s" % conn_conf.get('source_type')
module = import_module(path)
classes = [
getattr(module, x) for x in dir(module)
if isinstance(getattr(module, x), type)
]
connector_classes = filter(
lambda x: issubclass(x, ConnectorBase) and x.__name__ not in
('ConnectorBase', 'DBBaseConnector'), classes)
if len(classes) < 1 or len(connector_classes) < 1:
raise HydroException("Connector class doesn't exist")
con_cls = connector_classes[0]
return con_cls
def _connect(self):
if self._conf_defs['connection_type'] == DSN:
raise HydroException("Mysql dsn connection is not implemented")
else:
if connector_type == 'MySQLdb':
db_var = 'db'
password_var = 'passwd'
elif connector_type == 'mysql.connector':
db_var = 'database'
password_var = 'password'
#TODO need to be implemented based on connection string
# conn_string = 'DSN=%s' % self._conf_defs['connection_string']
# self.logger.debug('Connect to {0}'.format(conn_string))
db_params = {'user': self._conf_defs['db_user'],
password_var: self._conf_defs['db_password'],
'host': self._conf_defs['connection_string']}
if self._conf_defs['db_name']:
db_params[db_var] = self._conf_defs['db_name']
self._conn = mysql_connect(**db_params)
def execute(self):
raise HydroException('Not implemented')
def _close(self):
raise HydroException('Not implemented')
def _connect(self):
raise HydroException("Not implemented")
def _verify_connection_definitions(self):
raise HydroException("Not implemented")
def parse(self, value, kwargs):
raise HydroException("Not implemented")
def register(self, name, obj):
if not isinstance(obj, Topology):
raise HydroException("Not a Topology instance")
self._topologies[name] = obj
def _submit(self, params):
raise HydroException('Not implemented')
def parse(self, value, **kwargs):
if isinstance(value, DataFrame):
return value
else:
raise HydroException("Expected a Data Frame")
def parse(self, value, **kwargs):
if isinstance(value, list):
return value
else:
raise HydroException("Expected a list")
def get(self, key):
"""
get a serializable key, return serializable value
"""
raise HydroException('Not implemented')
def put(self, key, value, ttl):
"""
put serializable key and value with ttl(time to live in seconds, 0/-1 - no cache, None-Max tll)
"""
raise HydroException('Not implemented')
def __init__(self, value, **kwargs):
self._value = self.parse(value, **kwargs)
if self._value is None:
raise HydroException("Value is not set")