def dhub(self, procs_file_name):
"""
Return a configured ``DataHub`` using the given SQL procs file.
"""
data_source = {
self.key: {
# @@@ this should depend on self.type
# @@@ shouldn't have to specify this here and below
"hub": "MySQL",
"master_host": {
"host": self.host,
"user": settings.DATAZILLA_DATABASE_USER,
"passwd": settings.DATAZILLA_DATABASE_PASSWORD,
},
"default_db": self.name,
"procs": [
os.path.join(SQL_PATH, procs_file_name),
os.path.join(SQL_PATH, "generic.json"),
],
}
}
if self.read_only_host:
data_source[self.key]['read_host'] = {
"host": self.read_only_host,
"user": settings.DATAZILLA_RO_DATABASE_USER,
"passwd": settings.DATAZILLA_RO_DATABASE_PASSWORD,
}
BaseHub.add_data_source(data_source)
# @@@ the datahub class should depend on self.type
return MySQL(self.key)
def dhub(self, procs_file_name):
"""
Return a configured ``DataHub`` using the given SQL procs file.
"""
data_source = {
self.key: {
# @@@ this should depend on self.type
# @@@ shouldn't have to specify this here and below
"hub":
"MySQL",
"master_host": {
"host": self.host,
"user": settings.TREEHERDER_DATABASE_USER,
"passwd": settings.TREEHERDER_DATABASE_PASSWORD,
},
"read_host": {
"host": self.read_only_host,
"user": settings.TREEHERDER_RO_DATABASE_USER,
"passwd": settings.TREEHERDER_RO_DATABASE_PASSWORD,
},
"require_host_type":
True,
"default_db":
self.name,
"procs": [
os.path.join(SQL_PATH, procs_file_name),
os.path.join(SQL_PATH, "generic.json"),
],
}
}
BaseHub.add_data_source(data_source)
# @@@ the datahub class should depend on self.type
return MySQL(self.key)
def dhub(self, procs_file_name):
"""
Return a configured ``DataHub`` using the given SQL procs file.
"""
master_host_config = {
"host": settings.DATABASES["default"]["HOST"],
"user": settings.DATABASES["default"]["USER"],
"passwd": settings.DATABASES["default"].get("PASSWORD") or "",
}
if "OPTIONS" in settings.DATABASES["default"]:
master_host_config.update(settings.DATABASES["default"]["OPTIONS"])
read_host_config = {
"host": settings.DATABASES["read_only"]["HOST"],
"user": settings.DATABASES["read_only"]["USER"],
"passwd": settings.DATABASES["read_only"].get("PASSWORD") or "",
}
if "OPTIONS" in settings.DATABASES["read_only"]:
read_host_config.update(settings.DATABASES["read_only"]["OPTIONS"])
data_source = {
self.key: {
"hub": "MySQL",
"master_host": master_host_config,
"read_host": read_host_config,
"require_host_type": True,
"default_db": self.name,
"procs": [os.path.join(SQL_PATH, procs_file_name), os.path.join(SQL_PATH, "generic.json")],
}
}
BaseHub.add_data_source(data_source)
return MySQL(self.key)
def __init__(self, sqlFileName):
self.DATAZILLA_DATABASE_NAME = os.environ["DATAZILLA_DATABASE_NAME"]
self.DATAZILLA_DATABASE_USER = os.environ["DATAZILLA_DATABASE_USER"]
self.DATAZILLA_DATABASE_PASSWORD = os.environ["DATAZILLA_DATABASE_PASSWORD"]
self.DATAZILLA_DATABASE_HOST = os.environ["DATAZILLA_DATABASE_HOST"]
self.DATAZILLA_DATABASE_PORT = os.environ["DATAZILLA_DATABASE_PORT"]
self.sqlFileName = sqlFileName
try:
self.DEBUG = os.environ["DATAZILLA_DEBUG"] is not None
except KeyError:
self.DEBUG = False
self.rootPath = os.path.dirname(os.path.abspath(__file__))
####
#Configuration of datasource hub:
# 1 Build the datasource struct
# 2 Add it to the BaseHub
# 3 Instantiate a MySQL hub for all derived classes
####
dataSource = { self.DATAZILLA_DATABASE_NAME : { "hub":"MySQL",
"master_host":{"host":self.DATAZILLA_DATABASE_HOST,
"user":self.DATAZILLA_DATABASE_USER,
"passwd":self.DATAZILLA_DATABASE_PASSWORD},
"default_db":self.DATAZILLA_DATABASE_NAME,
"procs": ["%s%s%s" % (self.rootPath, "/sql/", sqlFileName)]
} }
BaseHub.addDataSource(dataSource)
self.dhub = MySQL(self.DATAZILLA_DATABASE_NAME)
def __init__(self):
procs_path = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
master_host_config = {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default'].get('PASSWORD') or '',
}
if 'OPTIONS' in settings.DATABASES['default']:
master_host_config.update(settings.DATABASES['default']['OPTIONS'])
read_host_config = {
"host": settings.DATABASES['read_only']['HOST'],
"user": settings.DATABASES['read_only']['USER'],
"passwd": settings.DATABASES['read_only'].get('PASSWORD') or '',
}
if 'OPTIONS' in settings.DATABASES['read_only']:
read_host_config.update(settings.DATABASES['read_only']['OPTIONS'])
data_source = {
'reference': {
"hub": "MySQL",
"master_host": master_host_config,
"read_host": read_host_config,
"require_host_type": True,
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
# Support structure for reference data signatures
self.build_signature_placeholders = []
# Support structures for building option collection data structures
self.oc_hash_lookup = dict()
self.oc_where_in_list = []
self.oc_placeholders = []
self.oc_unique_collections = []
# Support structures for building option data structures
self.o_lookup = set()
self.o_placeholders = []
self.o_unique_options = []
self.o_where_in_list = []
# reference id lookup structure
self.id_lookup = {}
def test_parse_data_sources(self):
# Instantiating base hub triggers data_sources.json parsing
BaseHub()
if self.data_source not in BaseHub.data_sources:
msg = "The required data source, %s, was not found in %s" % (self.data_source, BaseHub.source_list_file)
self.fail(msg)
def loadvars():
#####
#Only load the database sources once when the module
#is imported
#####
if not Model.projectHub:
Model.DATAZILLA_DATABASE_NAME = settings.DATAZILLA_DATABASE_NAME
Model.DATAZILLA_DATABASE_USER = settings.DATAZILLA_DATABASE_USER
Model.DATAZILLA_DATABASE_PASSWORD = settings.DATAZILLA_DATABASE_PASSWORD
Model.DATAZILLA_DATABASE_HOST = settings.DATAZILLA_DATABASE_HOST
Model.DATAZILLA_DATABASE_PORT = settings.DATAZILLA_DATABASE_PORT
####
#Configuration of datasource hub:
# 1 Build the datasource struct
# 2 Add it to the BaseHub
# 3 Instantiate a MySQL hub for all derived classes
####
Model.rootPath = os.path.dirname(os.path.abspath(__file__))
dataSource = { Model.DATAZILLA_DATABASE_NAME :
{ "hub":"MySQL",
"master_host":
{ "host":Model.DATAZILLA_DATABASE_HOST,
"user":Model.DATAZILLA_DATABASE_USER,
"passwd":Model.DATAZILLA_DATABASE_PASSWORD
},
"default_db":Model.DATAZILLA_DATABASE_NAME,
"procs": ["%s/%s" % (Model.rootPath,
'sources.json')]
} }
BaseHub.addDataSource(dataSource)
dzHub = MySQL(Model.DATAZILLA_DATABASE_NAME)
Model.databaseSources = dzHub.execute(proc='sources.get_datasources',
key_column='project',
return_type='dict')
Model.loadProjectHub(Model.databaseSources)
def loadProjectHub(databaseSources):
for s in databaseSources:
project = databaseSources[s]['project']
dataSource = { project :
{ "hub":"MySQL",
"master_host":{"host":databaseSources[s]['host'],
"user":Model.DATAZILLA_DATABASE_USER,
"passwd":Model.DATAZILLA_DATABASE_PASSWORD},
"default_db":databaseSources[s]['name'],
"procs": ["%s/%s" % (Model.rootPath, 'graphs.json')]
} }
BaseHub.addDataSource(dataSource)
hub = MySQL( project )
Model.projectHub[ project ] = hub
def _get_dhub(self):
dataSource = {
self.project : {
"hub":"MySQL",
"master_host":{
"host": settings.CLOUDSQL_INSTANCE,
# FIXME: CloudSQL has no users, but datasource requires it
"user": "******",
},
"default_db": settings.CLOUDSQL_DATABASE,
"procs": [os.path.join(SQL_PATH, self.procs_file_name)]
}
}
BaseHub.addDataSource(dataSource)
try:
return CloudSQL(self.project)
except KeyError:
raise KeyError("Failed to create CloudSQL")
def handle(self, *args, **options):
##Load data views##
views_file_obj = open("%s%s" %
(settings.ROOT, "/templates/data/views.json"))
try:
data_view_file = views_file_obj.read()
finally:
views_file_obj.close()
##Strip out comments and newlines##
t = BaseHub.strip_python_comments(data_view_file)
data_views = BaseHub.deserialize_json(data_view_file)
Command.build_nav(data_views)
#Uncomment to see datastructure for debugging
#pp = pprint.PrettyPrinter(indent=3)
#self.stdout.write( pp.pformat(data_views) )
menu_file_obj = open(
"%s%s" % (settings.ROOT, "/html/nav/nav_menu.html"), 'w+')
try:
menu_file_obj.write('<ul class="bh-viewtext">\n%s\n</ul>' %
(bh_unorderedlist(data_views)))
finally:
menu_file_obj.close()
##Write out json for the nav_lookup_hash##
jstring = json.dumps(Command.nav_lookup_hash, ensure_ascii=False)
html = """<input id="bh_nav_json" type="hidden" value="{{ json_data }}" />"""
t = Template(html)
c = Context({'json_data': jstring})
templateString = t.render(c)
nav_lookup_file_obj = open(
"%s%s" % (settings.ROOT, "/templates/bughunter.navlookup.html"),
'w+')
try:
nav_lookup_file_obj.write(templateString)
finally:
nav_lookup_file_obj.close()
def dhub(self, procs_file_name):
"""
Return a configured ``DataHub`` using the given SQL procs file.
"""
master_host_config = {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default'].get('PASSWORD') or '',
}
if 'OPTIONS' in settings.DATABASES['default']:
master_host_config.update(settings.DATABASES['default']['OPTIONS'])
read_host_config = {
"host": settings.DATABASES['read_only']['HOST'],
"user": settings.DATABASES['read_only']['USER'],
"passwd": settings.DATABASES['read_only'].get('PASSWORD') or '',
}
if 'OPTIONS' in settings.DATABASES['read_only']:
read_host_config.update(settings.DATABASES['read_only']['OPTIONS'])
data_source = {
self.key: {
"hub":
"MySQL",
"master_host":
master_host_config,
"read_host":
read_host_config,
"require_host_type":
True,
"default_db":
self.name,
"procs": [
os.path.join(SQL_PATH, procs_file_name),
os.path.join(SQL_PATH, "generic.json"),
],
}
}
BaseHub.add_data_source(data_source)
return MySQL(self.key)
def dhub(self, procs_file_name):
"""
Return a configured ``DataHub`` using the given SQL procs file.
"""
data_source = {
self.key: {
"hub": "MySQL",
"master_host": {
"host": self.host,
"user": settings.DATAZILLA_DATABASE_USER,
"passwd": settings.DATAZILLA_DATABASE_PASSWORD,
},
"default_db": self.name,
"procs": [os.path.join(SQL_PATH, procs_file_name)],
}
}
BaseHub.addDataSource(data_source)
# @@@ the datahub class should depend on self.type
return MySQL(self.key)
def get_proc(self, data_source, proc):
"""
Pass through to the BaseHub.get_proc() method.
Parameters:
data_source - data source to retrive proc from
proc - full proc path ex: mysql.selects.get_stuff
Returns:
proc datastructure from the data source
"""
return BaseHub.get_proc(data_source, proc)
def get_proc(self, data_source, proc):
"""
Pass through to the BaseHub.get_proc() method.
Parameters:
data_source - data source to retrive proc from
proc - full proc path ex: mysql.selects.get_stuff
Returns:
proc datastructure from the data source
"""
return BaseHub.get_proc(data_source, proc)
def validate_data_source(self, data_source_name):
"""
Iterates through data_source_req_keys and confirms required
key/value pairs. Probably a better way of doing this but
not thinking of anything more elegent at the moment. Attempting
to provide the caller with clear messaging regarding missing fields
in the data source file.
Parameters:
data_source_name - name of the datasource to test
Returns:
None
"""
for key in self.data_source_req_keys:
if key is 'req':
msg = 'the %s source object in %s' % (data_source_name, BaseHub.source_list_file)
# Confirm required keys
BaseHub.check_keys(self.data_source_req_keys[key], BaseHub.data_sources[data_source_name], True, msg)
elif key is 'databases':
if key in BaseHub.data_sources[data_source_name]:
for i in range(len(BaseHub.data_sources[data_source_name][key])):
db = BaseHub.data_sources[data_source_name][key][i]
msg = 'the %s.%s index position %i in %s' % (data_source_name, key, i, BaseHub.source_list_file)
BaseHub.check_keys(self.data_source_req_keys[key], db, True, msg)
else:
msg = 'the %s.%s in %s' % (data_source_name, key, BaseHub.source_list_file)
if key in BaseHub.data_sources[data_source_name]:
BaseHub.check_keys(self.data_source_req_keys[key], BaseHub.data_sources[data_source_name][key], True, msg)
def dhub(self, procs_file_name):
"""
Return a configured ``DataHub`` using the given SQL procs file.
"""
master_host_config = {
"host": self.host,
"user": settings.TREEHERDER_DATABASE_USER,
"passwd": settings.TREEHERDER_DATABASE_PASSWORD,
}
if 'OPTIONS' in settings.DATABASES['default']:
master_host_config.update(settings.DATABASES['default']['OPTIONS'])
read_host_config = {
"host": self.read_only_host,
"user": settings.TREEHERDER_RO_DATABASE_USER,
"passwd": settings.TREEHERDER_RO_DATABASE_PASSWORD,
}
if 'OPTIONS' in settings.DATABASES['read_only']:
read_host_config.update(settings.DATABASES['read_only']['OPTIONS'])
data_source = {
self.key: {
# @@@ this should depend on self.type
# @@@ shouldn't have to specify this here and below
"hub": "MySQL",
"master_host": master_host_config,
"read_host": read_host_config,
"require_host_type": True,
"default_db": self.name,
"procs": [
os.path.join(SQL_PATH, procs_file_name),
os.path.join(SQL_PATH, "generic.json"),
],
}
}
BaseHub.add_data_source(data_source)
# @@@ the datahub class should depend on self.type
return MySQL(self.key)
def handle(self, *args, **options):
##Load data views##
views_file_obj = open("%s%s" % (settings.ROOT, "/templates/data/views.json"))
try:
data_view_file = views_file_obj.read()
finally:
views_file_obj.close()
##Strip out comments and newlines##
t = BaseHub.stripPythonComments(data_view_file)
data_views = BaseHub.deserializeJson(data_view_file)
Command.build_nav(data_views)
#Uncomment to see datastructure for debugging
#pp = pprint.PrettyPrinter(indent=3)
#self.stdout.write( pp.pformat(data_views) )
menu_file_obj = open("%s%s" % (settings.ROOT, "/media/html/nav_menu.html"), 'w+')
try:
menu_file_obj.write( '<ul class="dv-viewtext">\n%s\n</ul>' % (dv_unorderedlist(data_views)) )
finally:
menu_file_obj.close()
##Write out json for the nav_lookup_hash##
jstring = json.dumps( Command.nav_lookup_hash, ensure_ascii=False )
html = """<input id="dv_nav_json" type="hidden" value="{{ json_data }}" />"""
t = Template(html)
c = Context({ 'json_data':jstring })
templateString = t.render(c)
nav_lookup_file_obj = open("%s%s" % (settings.ROOT, "/templates/graphs.navlookup.html"), 'w+')
try:
nav_lookup_file_obj.write(templateString)
finally:
nav_lookup_file_obj.close()
def dhub(self, procs_file_name):
"""
Return a configured ``DataHub`` using the given SQL procs file.
"""
master_host_config = {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default']['PASSWORD'],
}
if 'OPTIONS' in settings.DATABASES['default']:
master_host_config.update(settings.DATABASES['default']['OPTIONS'])
read_host_config = {
"host": settings.DATABASES['read_only']['HOST'],
"user": settings.DATABASES['read_only']['USER'],
"passwd": settings.DATABASES['read_only']['PASSWORD'],
}
if 'OPTIONS' in settings.DATABASES['read_only']:
read_host_config.update(settings.DATABASES['read_only']['OPTIONS'])
data_source = {
self.key: {
"hub": "MySQL",
"master_host": master_host_config,
"read_host": read_host_config,
"require_host_type": True,
"default_db": self.name,
"procs": [
os.path.join(SQL_PATH, procs_file_name),
os.path.join(SQL_PATH, "generic.json"),
],
}
}
BaseHub.add_data_source(data_source)
return MySQL(self.key)
def validate_data_source(self, data_source_name):
"""
Iterates through data_source_req_keys and confirms required
key/value pairs. Probably a better way of doing this but
not thinking of anything more elegent at the moment. Attempting
to provide the caller with clear messaging regarding missing fields
in the data source file.
Parameters:
data_source_name - name of the datasource to test
Returns:
None
"""
for key in self.data_source_req_keys:
if key is 'req':
msg = 'the %s source object in %s' % (data_source_name,
BaseHub.source_list_file)
# Confirm required keys
BaseHub.check_keys(self.data_source_req_keys[key],
BaseHub.data_sources[data_source_name],
True, msg)
elif key is 'databases':
if key in BaseHub.data_sources[data_source_name]:
for i in range(
len(BaseHub.data_sources[data_source_name][key])):
db = BaseHub.data_sources[data_source_name][key][i]
msg = 'the %s.%s index position %i in %s' % (
data_source_name, key, i, BaseHub.source_list_file)
BaseHub.check_keys(self.data_source_req_keys[key], db,
True, msg)
else:
msg = 'the %s.%s in %s' % (data_source_name, key,
BaseHub.source_list_file)
if key in BaseHub.data_sources[data_source_name]:
BaseHub.check_keys(
self.data_source_req_keys[key],
BaseHub.data_sources[data_source_name][key], True, msg)
####
#Configuration of datasource hub:
# 1 Build the datasource struct
# 2 Add it to the BaseHub
# 3 Instantiate a MySQL hub
####
dataSource = { SISYPHUS_DATABASE : { "hub":"MySQL",
"master_host":{"host":SISYPHUS_DATABASE_HOST,
"user":SISYPHUS_DATABASE_USER,
"passwd":SISYPHUS_DATABASE_PASSWORD},
"default_db":SISYPHUS_DATABASE,
"procs": ["%s%s" % (ROOT, "/procs/bughunter.json")]
}
}
BaseHub.add_data_source(dataSource)
DHUB = MySQL(SISYPHUS_DATABASE)
LOGGING = {
'version': 1,
'disable_existing_loggers': False,
'handlers': {
'file': {
'level': 'DEBUG',
'class': 'logging.FileHandler',
'filename': 'debug.log',
},
},
'loggers': {
'django': {
'handlers': ['file'],
def __init__(self):
procs_path = os.path.join(os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
data_source = {
'reference': {
"hub": "MySQL",
"master_host": {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default']['PASSWORD']
},
"read_host": {
"host": settings.DATABASES['read_only']['HOST'],
"user": settings.DATABASES['read_only']['USER'],
"passwd": settings.DATABASES['read_only']['PASSWORD']
},
"require_host_type": True,
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
# Support structure for reference data signatures
self.reference_data_signature_lookup = {}
self.build_signature_placeholders = []
# Support structures for building build platform SQL
self.build_platform_lookup = {}
self.build_where_filters = []
self.build_platform_placeholders = []
self.build_unique_platforms = []
# Support structures for building machine platform SQL
self.machine_platform_lookup = {}
self.machine_where_filters = []
self.machine_platform_placeholders = []
self.machine_unique_platforms = []
# Support structures for building job group SQL
self.job_group_lookup = {}
self.job_group_where_filters = []
self.job_group_placeholders = []
self.job_group_names_and_symbols = []
# Support structures for building job types SQL
self.job_type_lookup = {}
self.job_type_where_filters = []
self.job_type_placeholders = []
self.job_type_names_and_symbols = []
# Use this structure to map the job to the group id
self.job_type_to_group_lookup = {}
# Support structures for building product SQL
self.product_lookup = set()
self.product_where_in_list = []
self.product_placeholders = []
self.unique_products = []
# Support structures for building device SQL
self.device_lookup = set()
self.device_where_in_list = []
self.device_placeholders = []
self.unique_devices = []
# Support structures for building machine SQL
self.machine_name_lookup = set()
self.machine_where_in_list = []
self.machine_name_placeholders = []
self.machine_unique_names = []
self.machine_timestamp_update_placeholders = []
# Support structures for building option collection data structures
self.oc_hash_lookup = dict()
self.oc_where_in_list = []
self.oc_placeholders = []
self.oc_unique_collections = []
# Support structures for building option data structures
self.o_lookup = set()
self.o_placeholders = []
self.o_unique_options = []
self.o_where_in_list = []
# reference id lookup structure
self.id_lookup = {}
def __init__(self):
procs_path = os.path.join(
os.path.dirname(os.path.dirname(__file__)),
'sql', 'reference.json')
data_source = {
'reference': {
"hub": "MySQL",
"master_host": {
"host": settings.DATABASES['default']['HOST'],
"user": settings.DATABASES['default']['USER'],
"passwd": settings.DATABASES['default']['PASSWORD']
},
"default_db": settings.DATABASES['default']['NAME'],
"procs": [procs_path]
}
}
BaseHub.add_data_source(data_source)
self.dhub = DataHub.get("reference")
self.DEBUG = settings.DEBUG
# Support structures for building build platform SQL
self.build_platform_lookup = {}
self.build_where_filters = []
self.build_platform_placeholders = []
self.build_unique_platforms = []
# Support structures for building machine platform SQL
self.machine_platform_lookup = {}
self.machine_where_filters = []
self.machine_platform_placeholders = []
self.machine_unique_platforms = []
# Support structures for building job group SQL
self.job_group_lookup = {}
self.job_group_where_filters = []
self.job_group_placeholders = []
self.job_group_names_and_symbols = []
# Support structures for building job types SQL
self.job_type_lookup = {}
self.job_type_where_filters = []
self.job_type_placeholders = []
self.job_type_names_and_symbols = []
#Use this structure to map the job to the group id
self.job_type_to_group_lookup = {}
# Support structures for building product SQL
self.product_lookup = set()
self.product_where_in_list = []
self.product_placeholders = []
self.unique_products = []
# Support structures for building machine SQL
self.machine_name_lookup = set()
self.machine_where_in_list = []
self.machine_name_placeholders = []
self.machine_unique_names = []
self.machine_timestamp_update_placeholders = []
# Support structures for building option collection data structures
self.oc_hash_lookup = dict()
self.oc_where_in_list = []
self.oc_placeholders = []
self.oc_unique_collections = []
# Support structures for building option data structures
self.o_lookup = set()
self.o_placeholders = []
self.o_unique_options = []
self.o_where_in_list = []
# reference id lookup structure
self.id_lookup = {}
def __init__(self, data_source_name):
"""
A derived class of BaseHub, serves as a base class for any Relational
Database hubs.
"""
BaseHub.__init__(self)
# allowed keys in execute
self.execute_keys = set([
'db', 'proc', 'nocommit', 'sql', 'host_type', 'placeholders',
'replace', 'replace_quote', 'limit', 'offset', 'chunk_size',
'chunk_source', 'chunk_min', 'chunk_total', 'executemany',
'return_type', 'key_column', 'callback', 'debug_show', 'debug_noex'
])
# Default values for execute kwargs
self.default_host_type = 'master_host'
self.default_return_type = 'tuple'
# replace string base for replace functionality in execute
self.replace_string = 'REP'
# set of return types that require a key_column
self.return_type_key_columns = set(
['dict', 'dict_json', 'set', 'set_json'])
# One of these keys must be provided to execute
self.execute_required_keys = set(['proc', 'sql'])
# This data structure is used to map the return_type provided to
# execute() to the derived hub method. Derived hub's have to map
# their methods by setting the appropriate function reference to
# its associated key in valid_return_types.
self.valid_return_types = {
'iter': None,
'dict': None,
'dict_json': None,
'tuple': None,
'tuple_json': None,
'set': None,
'set_json': None,
'table': None,
'table_json': None,
'rowcount': None,
'callback': None
}
# Dictionary of required keys for RDBS datasources
self.data_source_req_keys = dict(
# required keys
req=set(['hub', 'master_host']),
# optional keys but if present have additional key requirements
databases=set(['name', 'procs']),
master_host=set(['host', 'user']),
read_host=set(['host', 'user']),
dev_host=set(['host', 'user']))
# List of SQL tokens that must follow a WHERE statement
self.post_where_tokens = [
'GROUP BY', 'HAVING', 'ORDER BY', 'LIMIT', 'OFFSET', 'PROCEDURE',
'INTO', 'FOR UPDATE'
]
# Validate the information in data_sources is complete
# so we can provide the caller with useful messaging
# regarding what is missing when a class is instantiated.
self.validate_data_source(data_source_name)
self.pretty_sql_regex = re.compile('\s+', re.DOTALL)
self.default_placeholder = '?'
__all__ = [
'load_procs', # noqa
'get_proc',
'get_data',
'validate_data_source',
'set_execute_rules',
'get_execute_data'
]
def load_procs(self, data_source):
BaseHub.load_procs(data_source)
def __init__(self, data_source_name):
"""
A derived class of BaseHub, serves as a base class for any Relational
Database hubs.
"""
BaseHub.__init__(self)
# allowed keys in execute
self.execute_keys = set(['db',
'proc',
'nocommit',
'sql',
'host_type',
'placeholders',
'replace',
'replace_quote',
'limit',
'offset',
'chunk_size',
'chunk_source',
'chunk_min',
'chunk_total',
'executemany',
'return_type',
'key_column',
'callback',
'debug_show',
'debug_noex'])
# Default values for execute kwargs
self.default_host_type = 'master_host'
self.default_return_type = 'tuple'
# replace string base for replace functionality in execute
self.replace_string = 'REP'
# set of return types that require a key_column
self.return_type_key_columns = set(['dict', 'dict_json', 'set', 'set_json'])
# One of these keys must be provided to execute
self.execute_required_keys = set(['proc', 'sql'])
# This data structure is used to map the return_type provided to
# execute() to the derived hub method. Derived hub's have to map
# their methods by setting the appropriate function reference to
# its associated key in valid_return_types.
self.valid_return_types = {'iter': None,
'dict': None,
'dict_json': None,
'tuple': None,
'tuple_json': None,
'set': None,
'set_json': None,
'table': None,
'table_json': None,
'rowcount': None,
'callback': None}
# Dictionary of required keys for RDBS datasources
self.data_source_req_keys = dict(
# required keys
req=set(['hub', 'master_host']),
# optional keys but if present have additional key requirements
databases=set(['name', 'procs']),
master_host=set(['host', 'user']),
read_host=set(['host', 'user']),
dev_host=set(['host', 'user']))
# List of SQL tokens that must follow a WHERE statement
self.post_where_tokens = ['GROUP BY', 'HAVING', 'ORDER BY', 'LIMIT', 'OFFSET', 'PROCEDURE', 'INTO', 'FOR UPDATE']
# Validate the information in data_sources is complete
# so we can provide the caller with useful messaging
# regarding what is missing when a class is instantiated.
self.validate_data_source(data_source_name)
self.pretty_sql_regex = re.compile('\s+', re.DOTALL)
self.default_placeholder = '?'
__all__ = ['load_procs', # noqa
'get_proc',
'get_data',
'validate_data_source',
'set_execute_rules',
'get_execute_data']
def loadProcs(self, dataSource):
BaseHub.loadProcs(dataSource)
def load_procs(self, data_source):
BaseHub.load_procs(data_source)
