def test_force_dict_to_strings(self):
unicode_dict = {u'test': u'test'}
string_dict = {'test': 'test'}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
unicode_dict = {u'test': {u'test': u'test'}}
string_dict = {'test': {'test': 'test'}}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
def test_force_dict_to_strings(self):
unicode_dict = {u'test': u'test'}
string_dict = {'test': 'test'}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
unicode_dict = {u'test': {u'test': u'test'}}
string_dict = {'test': {'test': 'test'}}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
def get_query_server_config(server=None):
if not server or server not in DATABASES:
keys = DATABASES.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
'server_name': DATABASES[name].ENGINE.get().split('.')[-1],
'server_host': DATABASES[name].HOST.get(),
'server_port': DATABASES[name].PORT.get(),
'username': DATABASES[name].USER.get(),
'password': get_database_password(name),
'options': force_dict_to_strings(DATABASES[name].OPTIONS.get()),
'alias': name
}
if DATABASES[name].NAME.get():
query_server['name'] = DATABASES[name].NAME.get()
else:
query_server = {}
LOG.debug("Query Server: %s" % query_server)
return query_server
def get_query_server_config(server=None):
if not server or server not in DATABASES:
keys = DATABASES.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
"server_name": DATABASES[name].ENGINE.get().split(".")[-1],
"server_host": DATABASES[name].HOST.get(),
"server_port": DATABASES[name].PORT.get(),
"username": DATABASES[name].USER.get(),
"password": get_database_password(name),
"options": force_dict_to_strings(DATABASES[name].OPTIONS.get()),
"alias": name,
}
if DATABASES[name].NAME.get():
query_server["name"] = DATABASES[name].NAME.get()
else:
query_server = {}
LOG.debug("Query Server: %s" % query_server)
return query_server
def from_dict(job_dict):
job_dict.setdefault("connector", [])
job_dict["connector"] = [Form.from_dict(con_form_dict) for con_form_dict in job_dict["connector"]]
job_dict.setdefault("framework", [])
job_dict["framework"] = [Form.from_dict(framework_form_dict) for framework_form_dict in job_dict["framework"]]
if not "connection_id" in job_dict:
job_dict["connection_id"] = job_dict["connection-id"]
if not "connector_id" in job_dict:
job_dict["connector_id"] = job_dict["connector-id"]
if not "creation_user" in job_dict:
job_dict["creation_user"] = job_dict.setdefault("creation-user", "hue")
if not "creation_date" in job_dict:
job_dict["creation_date"] = job_dict.setdefault("creation-date", 0)
if not "update_user" in job_dict:
job_dict["update_user"] = job_dict.setdefault("update-user", "hue")
if not "update_date" in job_dict:
job_dict["update_date"] = job_dict.setdefault("update-date", 0)
return Job(**force_dict_to_strings(job_dict))
def from_dict(link_dict):
link_dict.setdefault('link_config_values', [])
link_dict['link_config_values'] = [
Config.from_dict(link_config_value_dict)
for link_config_value_dict in link_dict['link-config-values']
]
if not 'connector_id' in link_dict:
link_dict['connector_id'] = link_dict.setdefault(
'connector-id', -1)
if not 'creation_user' in link_dict:
link_dict['creation_user'] = link_dict.setdefault(
'creation-user', 'hue')
if not 'creation_date' in link_dict:
link_dict['creation_date'] = link_dict.setdefault(
'creation-date', 0)
if not 'update_user' in link_dict:
link_dict['update_user'] = link_dict.setdefault(
'update-user', 'hue')
if not 'update_date' in link_dict:
link_dict['update_date'] = link_dict.setdefault('update-date', 0)
return Link(**force_dict_to_strings(link_dict))
def from_dict(job_dict):
job_dict.setdefault('connector', [])
job_dict['connector'] = [
Form.from_dict(con_form_dict)
for con_form_dict in job_dict['connector']
]
job_dict.setdefault('framework', [])
job_dict['framework'] = [
Form.from_dict(framework_form_dict)
for framework_form_dict in job_dict['framework']
]
if not 'connection_id' in job_dict:
job_dict['connection_id'] = job_dict['connection-id']
if not 'connector_id' in job_dict:
job_dict['connector_id'] = job_dict['connector-id']
if not 'creation_user' in job_dict:
job_dict['creation_user'] = job_dict.setdefault(
'creation-user', 'hue')
if not 'creation_date' in job_dict:
job_dict['creation_date'] = job_dict.setdefault('creation-date', 0)
if not 'update_user' in job_dict:
job_dict['update_user'] = job_dict.setdefault('update-user', 'hue')
if not 'update_date' in job_dict:
job_dict['update_date'] = job_dict.setdefault('update-date', 0)
return Job(**force_dict_to_strings(job_dict))
def get_query_server_config(server=None):
if not server or server not in DATABASES:
keys = DATABASES.keys()
name = keys and keys[0] or None
else:
name = server
if name:
query_server = {
'server_name': DATABASES[name].ENGINE.get().split('.')[-1],
'server_host': DATABASES[name].HOST.get(),
'server_port': DATABASES[name].PORT.get(),
'username': DATABASES[name].USER.get(),
'password': get_database_password(name),
'options': force_dict_to_strings(DATABASES[name].OPTIONS.get()),
'alias': name
}
if DATABASES[name].NAME.get():
query_server['name'] = DATABASES[name].NAME.get()
else:
query_server = {}
LOG.debug("Query Server: %s" % query_server)
return query_server
def from_dict(job_dict):
job_dict.setdefault('from_config_values', [])
job_dict['from_config_values'] = [ Config.from_dict(from_config_value_dict) for from_config_value_dict in job_dict['from-config-values'] ]
job_dict.setdefault('to_config_values', [])
job_dict['to_config_values'] = [ Config.from_dict(to_config_value_dict) for to_config_value_dict in job_dict['to-config-values'] ]
job_dict.setdefault('driver_config_values', [])
job_dict['driver_config_values'] = [ Config.from_dict(driver_config_value_dict) for driver_config_value_dict in job_dict['driver-config-values'] ]
if not 'from_link_id' in job_dict:
job_dict['from_link_id'] = job_dict['from-link-id']
if not 'to_link_id' in job_dict:
job_dict['to_link_id'] = job_dict['to-link-id']
if not 'from_connector_id' in job_dict:
job_dict['from_connector_id'] = job_dict['from-connector-id']
if not 'to_connector_id' in job_dict:
job_dict['to_connector_id'] = job_dict['to-connector-id']
if not 'creation_user' in job_dict:
job_dict['creation_user'] = job_dict.setdefault('creation-user', 'hue')
if not 'creation_date' in job_dict:
job_dict['creation_date'] = job_dict.setdefault('creation-date', 0)
if not 'update_user' in job_dict:
job_dict['update_user'] = job_dict.setdefault('update-user', 'hue')
if not 'update_date' in job_dict:
job_dict['update_date'] = job_dict.setdefault('update-date', 0)
return Job(**force_dict_to_strings(job_dict))
def from_dict(connector_dict):
connector_dict.setdefault('link_config', [])
connector_dict['link_config'] = [
Config.from_dict(link_config_dict)
for link_config_dict in connector_dict['link-config']
]
connector_dict.setdefault('job_config', {})
connector_dict['job_config'] = {}
if 'FROM' in connector_dict['job-config']:
connector_dict['job_config']['FROM'] = [
Config.from_dict(from_config_dict)
for from_config_dict in connector_dict['job-config']['FROM']
]
if 'TO' in connector_dict['job-config']:
connector_dict['job_config']['TO'] = [
Config.from_dict(to_config_dict)
for to_config_dict in connector_dict['job-config']['TO']
]
connector_dict['config_resources'] = connector_dict[
'all-config-resources']
return Connector(**force_dict_to_strings(connector_dict))
def from_dict(submission_dict):
submission_dict['job_id'] = submission_dict['job']
submission_dict['created'] = submission_dict['creation-date']
submission_dict['updated'] = submission_dict['last-update-date']
submission_dict['external_id'] = submission_dict.get('external-id', None)
submission_dict['external_link'] = submission_dict.get('external-link', None)
return Submission(**force_dict_to_strings(submission_dict))
def test_force_dict_to_strings(self):
unicode_dict = {u'test': u'test'}
string_dict = {'test': 'test'}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
# Embedded
unicode_dict = {u'test': {u'test': u'test'}}
string_dict = {'test': {'test': 'test'}}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
# Embedded list
unicode_dict = {u'test': [{u'test': u'test'}]}
string_dict = {'test': [{'test': 'test'}]}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
def from_dict(submission_dict):
submission_dict['job_id'] = submission_dict['job']
submission_dict['created'] = submission_dict['creation-date']
submission_dict['updated'] = submission_dict['last-update-date']
submission_dict['exception'] = submission_dict.get('exception', None)
submission_dict['exception_trace'] = submission_dict.get('exception-trace', None)
submission = SqoopSubmissionException(**force_dict_to_strings(submission_dict))
return submission
def test_force_dict_to_strings(self):
unicode_dict = {u'test': u'test'}
string_dict = {'test': 'test'}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
# Embedded
unicode_dict = {u'test': {u'test': u'test'}}
string_dict = {'test': {'test': 'test'}}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
# Embedded list
unicode_dict = {u'test': [{u'test': u'test'}]}
string_dict = {'test': [{'test': 'test'}]}
transformed_dict = force_dict_to_strings(unicode_dict)
assert_equal(string_dict, transformed_dict)
def from_dict(submission_dict):
submission_dict['job_id'] = submission_dict['job']
submission_dict['created'] = submission_dict['creation-date']
submission_dict['updated'] = submission_dict['last-update-date']
submission_dict['external_id'] = submission_dict.get(
'external-id', None)
submission_dict['external_link'] = submission_dict.get(
'external-link', None)
return Submission(**force_dict_to_strings(submission_dict))
def from_dict(submission_dict):
submission_dict['job_id'] = submission_dict['job']
submission_dict['created'] = submission_dict['creation-date']
submission_dict['updated'] = submission_dict['last-update-date']
submission_dict['exception'] = submission_dict.get('exception', None)
submission_dict['exception_trace'] = submission_dict.get(
'exception-trace', None)
submission = SqoopSubmissionException(
**force_dict_to_strings(submission_dict))
return submission
def from_dict(framework_dict):
framework_dict.setdefault('job-forms', {})
framework_dict['job_forms'] = {}
if 'IMPORT' in framework_dict['job-forms']:
framework_dict['job_forms']['IMPORT'] = [ Form.from_dict(job_form_dict) for job_form_dict in framework_dict['job-forms']['IMPORT'] ]
if 'EXPORT' in framework_dict['job-forms']:
framework_dict['job_forms']['EXPORT'] = [ Form.from_dict(job_form_dict) for job_form_dict in framework_dict['job-forms']['EXPORT'] ]
framework_dict.setdefault('con-forms', [])
framework_dict['con_forms'] = [ Form.from_dict(con_form_dict) for con_form_dict in framework_dict['con-forms'] ]
return Framework(**force_dict_to_strings(framework_dict))
def from_dict(connector_dict, resources_dict={}):
connector_dict.setdefault('job-forms', {})
connector_dict['job_forms'] = {}
if 'IMPORT' in connector_dict['job-forms']:
connector_dict['job_forms']['IMPORT'] = [ Form.from_dict(job_form_dict) for job_form_dict in connector_dict['job-forms']['IMPORT'] ]
if 'EXPORT' in connector_dict['job-forms']:
connector_dict['job_forms']['EXPORT'] = [ Form.from_dict(job_form_dict) for job_form_dict in connector_dict['job-forms']['EXPORT'] ]
connector_dict.setdefault('con-forms', [])
connector_dict['con_forms'] = [ Form.from_dict(con_form_dict) for con_form_dict in connector_dict['con-forms'] ]
connector_dict['resources'] = resources_dict.setdefault(unicode(connector_dict['id']), {})
return Connector(**force_dict_to_strings(connector_dict))
def from_dict(connector_dict):
connector_dict.setdefault('link_config', [])
connector_dict['link_config'] = [ Config.from_dict(link_config_dict) for link_config_dict in connector_dict['link-config'] ]
connector_dict.setdefault('job_config', {})
connector_dict['job_config'] = {}
if 'FROM' in connector_dict['job-config']:
connector_dict['job_config']['FROM'] = [ Config.from_dict(from_config_dict) for from_config_dict in connector_dict['job-config']['FROM'] ]
if 'TO' in connector_dict['job-config']:
connector_dict['job_config']['TO'] = [ Config.from_dict(to_config_dict) for to_config_dict in connector_dict['job-config']['TO'] ]
connector_dict['config_resources'] = connector_dict['all-config-resources']
return Connector(**force_dict_to_strings(connector_dict))
def from_dict(job_dict):
job_dict.setdefault('from_config_values', [])
job_dict['from_config_values'] = [
Config.from_dict(from_config_value_dict)
for from_config_value_dict in job_dict['from-config-values']
]
job_dict.setdefault('to_config_values', [])
job_dict['to_config_values'] = [
Config.from_dict(to_config_value_dict)
for to_config_value_dict in job_dict['to-config-values']
]
job_dict.setdefault('driver_config_values', [])
job_dict['driver_config_values'] = [
Config.from_dict(driver_config_value_dict)
for driver_config_value_dict in job_dict['driver-config-values']
]
if not 'from_link_id' in job_dict:
job_dict['from_link_id'] = job_dict['from-link-id']
if not 'to_link_id' in job_dict:
job_dict['to_link_id'] = job_dict['to-link-id']
if not 'from_connector_id' in job_dict:
job_dict['from_connector_id'] = job_dict['from-connector-id']
if not 'to_connector_id' in job_dict:
job_dict['to_connector_id'] = job_dict['to-connector-id']
if not 'creation_user' in job_dict:
job_dict['creation_user'] = job_dict.setdefault(
'creation-user', 'hue')
if not 'creation_date' in job_dict:
job_dict['creation_date'] = job_dict.setdefault('creation-date', 0)
if not 'update_user' in job_dict:
job_dict['update_user'] = job_dict.setdefault('update-user', 'hue')
if not 'update_date' in job_dict:
job_dict['update_date'] = job_dict.setdefault('update-date', 0)
return Job(**force_dict_to_strings(job_dict))
def from_dict(link_dict):
link_dict.setdefault('link_config_values', [])
link_dict['link_config_values'] = [ Config.from_dict(link_config_value_dict) for link_config_value_dict in link_dict['link-config-values'] ]
if not 'connector_id' in link_dict:
link_dict['connector_id'] = link_dict.setdefault('connector-id', -1)
if not 'creation_user' in link_dict:
link_dict['creation_user'] = link_dict.setdefault('creation-user', 'hue')
if not 'creation_date' in link_dict:
link_dict['creation_date'] = link_dict.setdefault('creation-date', 0)
if not 'update_user' in link_dict:
link_dict['update_user'] = link_dict.setdefault('update-user', 'hue')
if not 'update_date' in link_dict:
link_dict['update_date'] = link_dict.setdefault('update-date', 0)
return Link(**force_dict_to_strings(link_dict))
def from_dict(framework_dict):
framework_dict.setdefault('job-forms', {})
framework_dict['job_forms'] = {}
if 'IMPORT' in framework_dict['job-forms']:
framework_dict['job_forms']['IMPORT'] = [
Form.from_dict(job_form_dict)
for job_form_dict in framework_dict['job-forms']['IMPORT']
]
if 'EXPORT' in framework_dict['job-forms']:
framework_dict['job_forms']['EXPORT'] = [
Form.from_dict(job_form_dict)
for job_form_dict in framework_dict['job-forms']['EXPORT']
]
framework_dict.setdefault('con-forms', [])
framework_dict['con_forms'] = [
Form.from_dict(con_form_dict)
for con_form_dict in framework_dict['con-forms']
]
return Framework(**force_dict_to_strings(framework_dict))
def from_dict(connection_dict):
connection_dict.setdefault('connector', [])
connection_dict['connector'] = [ Form.from_dict(con_form_dict) for con_form_dict in connection_dict['connector'] ]
connection_dict.setdefault('framework', [])
connection_dict['framework'] = [ Form.from_dict(framework_form_dict) for framework_form_dict in connection_dict['framework'] ]
if not 'connector_id' in connection_dict:
connection_dict['connector_id'] = connection_dict.setdefault('connector-id', -1)
if not 'creation_user' in connection_dict:
connection_dict['creation_user'] = connection_dict.setdefault('creation-user', 'hue')
if not 'creation_date' in connection_dict:
connection_dict['creation_date'] = connection_dict.setdefault('creation-date', 0)
if not 'update_user' in connection_dict:
connection_dict['update_user'] = connection_dict.setdefault('update-user', 'hue')
if not 'update_date' in connection_dict:
connection_dict['update_date'] = connection_dict.setdefault('update-date', 0)
return Connection(**force_dict_to_strings(connection_dict))
def from_dict(connector_dict, resources_dict={}):
connector_dict.setdefault('job-forms', {})
connector_dict['job_forms'] = {}
if 'IMPORT' in connector_dict['job-forms']:
connector_dict['job_forms']['IMPORT'] = [
Form.from_dict(job_form_dict)
for job_form_dict in connector_dict['job-forms']['IMPORT']
]
if 'EXPORT' in connector_dict['job-forms']:
connector_dict['job_forms']['EXPORT'] = [
Form.from_dict(job_form_dict)
for job_form_dict in connector_dict['job-forms']['EXPORT']
]
connector_dict.setdefault('con-forms', [])
connector_dict['con_forms'] = [
Form.from_dict(con_form_dict)
for con_form_dict in connector_dict['con-forms']
]
connector_dict['resources'] = resources_dict.setdefault(
unicode(connector_dict['id']), {})
return Connector(**force_dict_to_strings(connector_dict))
def from_dict(config_dict):
config_dict['inputs'] = [Input.from_dict(input_dict) for input_dict in config_dict.setdefault('inputs', [])]
return Config(**force_dict_to_strings(config_dict))
def from_dicts(cls, error_dicts): return SqoopException([force_dict_to_strings(d) for d in error_dicts])
default_db['NAME'] = default_db['NAME'].replace('#', ':') # For is_db_alive command
else:
test_name = os.environ.get('DESKTOP_DB_TEST_NAME', get_desktop_root('desktop-test.db'))
logging.debug("DESKTOP_DB_TEST_NAME SET: %s" % test_name)
test_user = os.environ.get('DESKTOP_DB_TEST_USER', 'hue_test')
logging.debug("DESKTOP_DB_TEST_USER SET: %s" % test_user)
default_db = {
"ENGINE" : desktop.conf.DATABASE.ENGINE.get(),
"NAME" : desktop.conf.DATABASE.NAME.get(),
"USER" : desktop.conf.DATABASE.USER.get(),
"PASSWORD" : desktop.conf.get_database_password(),
"HOST" : desktop.conf.DATABASE.HOST.get(),
"PORT" : str(desktop.conf.DATABASE.PORT.get()),
"OPTIONS": force_dict_to_strings(desktop.conf.DATABASE.OPTIONS.get()),
# DB used for tests
"TEST_NAME" : test_name,
"TEST_USER" : test_user,
# Wrap each request in a transaction.
"ATOMIC_REQUESTS" : True,
}
DATABASES = {
'default': default_db
}
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'unique-hue'
def from_dicts(cls, error_dicts):
return SqoopException([force_dict_to_strings(d) for d in error_dicts])
def from_dict(driver_dict):
driver_dict.setdefault('job_config', {})
driver_dict['job_config'] = [ Config.from_dict(job_config_dict) for job_config_dict in driver_dict['job-config']]
driver_dict['config_resources'] = driver_dict['all-config-resources']
return Driver(**force_dict_to_strings(driver_dict))
def from_dict(cls, error_dict): return SqoopConnectionException(**force_dict_to_strings(error_dict))
def from_dict(input_dict):
if 'values' in input_dict and isinstance(input_dict['values'], basestring):
input_dict['values'] = input_dict['values'].split(',')
return Input(**force_dict_to_strings(input_dict))
def from_dict(form_dict):
form_dict['inputs'] = [Input.from_dict(input_dict) for input_dict in form_dict.setdefault('inputs', [])]
return Form(**force_dict_to_strings(form_dict))
def from_dict(config_dict):
config_dict['inputs'] = [Input.from_dict(input_dict) for input_dict in config_dict.setdefault('inputs', [])]
return Config(**force_dict_to_strings(config_dict))
test_name = os.environ.get('DESKTOP_DB_TEST_NAME',
get_desktop_root('desktop-test.db'))
logging.debug("DESKTOP_DB_TEST_NAME SET: %s" % test_name)
test_user = os.environ.get('DESKTOP_DB_TEST_USER', 'hue_test')
logging.debug("DESKTOP_DB_TEST_USER SET: %s" % test_user)
default_db = {
"ENGINE": desktop.conf.DATABASE.ENGINE.get(),
"NAME": desktop.conf.DATABASE.NAME.get(),
"USER": desktop.conf.DATABASE.USER.get(),
"SCHEMA": desktop.conf.DATABASE.SCHEMA.get(),
"PASSWORD": desktop.conf.get_database_password(),
"HOST": desktop.conf.DATABASE.HOST.get(),
"PORT": str(desktop.conf.DATABASE.PORT.get()),
"OPTIONS": force_dict_to_strings(desktop.conf.DATABASE.OPTIONS.get()),
# DB used for tests
"TEST_NAME": test_name,
"TEST_USER": test_user,
# Wrap each request in a transaction.
"ATOMIC_REQUESTS": True,
"CONN_MAX_AGE": desktop.conf.DATABASE.CONN_MAX_AGE.get(),
}
DATABASES = {'default': default_db}
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'unique-hue'
}
def from_dict(cls, error_dict):
return SqoopJobException(**force_dict_to_strings(error_dict))
