def setUpClass(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
self.conf = conf_tup[0]
def setUpClass(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
self.conf = conf_tup[0]
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
udl2_conf = conf_tup[0]
initialize_db_udl(udl2_conf)
self.ref_schema = udl2_conf['udl2_db_conn']['db_schema']
self.ref_table_name = Constants.UDL2_REF_MAPPING_TABLE(Constants.LOAD_TYPE_ASSESSMENT)
# Testable Rules
self.rule_names = transform_rules.keys()
self.rule_conf = transform_rules
self.rule_list = transformation_code_generator.generate_transformations(self.rule_names, rule_conf=self.rule_conf)
self.testable_rules = []
for rule in self.rule_list:
self.testable_rules.append(rule[0])
test_rows = []
for rule in self.testable_rules:
ins_dict = {
'phase': -999,
'source_table': 'ftest_table',
'source_column': 'ftest_column',
'target_table': 'ftest_table1',
'target_column': 'ftest_column1',
'transformation_rule': rule,
}
test_rows.append(ins_dict)
with get_udl_connection() as conn:
self.ref_table = conn.get_table(self.ref_table_name)
conn.execute(self.ref_table.insert(test_rows))
def main():
try:
config_path_file = os.environ['UDL2_CONF']
except Exception:
config_path_file = UDL2_DEFAULT_CONFIG_PATH_FILE
udl2_conf, udl2_flat_conf = read_ini_file(config_path_file)
initialize_db_prod(udl2_conf)
load_data('cat')
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
self.conf = conf_tup[0]
self.output_dir = tempfile.mkdtemp()
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
self.conf = conf_tup[0]
self.output_dir = tempfile.mkdtemp()
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
self.conf = conf_tup[0]
initialize_all_db(udl2_conf, udl2_flat_conf)
self.base_data_dir = os.path.join(os.path.dirname(__file__), "..", "data")
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
self.conf = conf_tup[0]
initialize_all_db(udl2_conf, udl2_flat_conf)
self.base_data_dir = os.path.join(os.path.dirname(__file__), "..",
"data")
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
udl2_conf = read_ini_file(config_path)
if isinstance(udl2_conf, tuple):
self.conf = udl2_conf[0]
else:
self.conf = udl2_conf
self.test_expanded_dir = tempfile.mkdtemp()
def setUp(self, ):
# TODO: don't rely on env. var
# TODO: mock the data instead of using ini file
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
udl2_conf = conf_tup[0]
self.conf = udl2_conf
self.maxDiff = None
def setUp(self,):
# TODO: don't rely on env. var
# TODO: mock the data instead of using ini file
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
udl2_conf = conf_tup[0]
self.conf = udl2_conf
self.maxDiff = None
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
udl2_conf = read_ini_file(config_path)
if isinstance(udl2_conf, tuple):
self.conf = udl2_conf[0]
else:
self.conf = udl2_conf
self.test_expanded_dir = tempfile.mkdtemp()
def setUpClass(cls, data_dir):
try:
config_path_file = os.environ['UDL2_CONF']
except Exception:
config_path_file = UDL2_DEFAULT_CONFIG_PATH_FILE
cls.udl2_conf, cls.settings = read_ini_file(config_path_file)
cls.encrypt_helper = EncryptHelper(cls.settings)
cls.data_dir = data_dir
cls.gpg_home = cls.settings.get('gpg_home', None)
# prepare gpg keys for tests
if cls.gpg_home and not os.path.exists(cls.gpg_home):
config_gpg = os.path.join(os.path.dirname(__file__), "../../../../config/gpg")
shutil.copytree(config_gpg, cls.gpg_home)
def setUpClass(cls, data_dir):
try:
config_path_file = os.environ['UDL2_CONF']
except Exception:
config_path_file = UDL2_DEFAULT_CONFIG_PATH_FILE
cls.udl2_conf, cls.settings = read_ini_file(config_path_file)
cls.encrypt_helper = EncryptHelper(cls.settings)
cls.data_dir = data_dir
cls.gpg_home = cls.settings.get('gpg_home', None)
# prepare gpg keys for tests
if cls.gpg_home and not os.path.exists(cls.gpg_home):
config_gpg = os.path.join(os.path.dirname(__file__),
"../../../../config/gpg")
shutil.copytree(config_gpg, cls.gpg_home)
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
udl2_conf = conf_tup[0]
initialize_db_udl(udl2_conf)
self.rule_names = transform_rules.keys()
self.rule_conf = transform_rules
self.rule_list = transformation_code_generator.generate_transformations(self.rule_names, rule_conf=self.rule_conf)
with get_udl_connection() as conn:
trans = conn.get_transaction()
for rule in self.rule_list:
try:
conn.execute(rule[2])
except ProgrammingError as e:
raise AssertionError('UNABLE TO CREATE FUNCTION: %s, Error: "%s"' % (rule[0], e))
trans.rollback()
trans.commit()
def main():
'''
create or drop udl2 database objects according to command line.
The purpose for this script is to enable clean up whole database artifacts or create
whole database artifacts without problem. Since UDL uses databases to clean data. Database object
in UDL can be dropped or recreated at will for changes. So we can verifiy system
'''
(parser, args) = _parse_args()
if args.config_file is None:
config_path_file = UDL2_DEFAULT_CONFIG_PATH_FILE
else:
config_path_file = args.config_file
conf_tup = read_ini_file(config_path_file)
udl2_conf = conf_tup[0]
if args.action is None:
parser.print_help()
return
if args.action == 'setup':
setup_udl2_schema(udl2_conf)
elif args.action == 'teardown':
teardown_udl2_schema(udl2_conf)
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
udl2_conf = conf_tup[0]
initialize_db_udl(udl2_conf)
self.rule_names = transform_rules.keys()
self.rule_conf = transform_rules
self.rule_list = transformation_code_generator.generate_transformations(
self.rule_names, rule_conf=self.rule_conf)
with get_udl_connection() as conn:
trans = conn.get_transaction()
for rule in self.rule_list:
try:
conn.execute(rule[2])
except ProgrammingError as e:
raise AssertionError(
'UNABLE TO CREATE FUNCTION: %s, Error: "%s"' %
(rule[0], e))
trans.rollback()
trans.commit()
def setUp(self):
try:
config_path = dict(os.environ)['UDL2_CONF']
except Exception:
config_path = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path)
udl2_conf = conf_tup[0]
initialize_db_udl(udl2_conf)
self.ref_schema = udl2_conf['udl2_db_conn']['db_schema']
self.ref_table_name = Constants.UDL2_REF_MAPPING_TABLE(
Constants.LOAD_TYPE_ASSESSMENT)
# Testable Rules
self.rule_names = transform_rules.keys()
self.rule_conf = transform_rules
self.rule_list = transformation_code_generator.generate_transformations(
self.rule_names, rule_conf=self.rule_conf)
self.testable_rules = []
for rule in self.rule_list:
self.testable_rules.append(rule[0])
test_rows = []
for rule in self.testable_rules:
ins_dict = {
'phase': -999,
'source_table': 'ftest_table',
'source_column': 'ftest_column',
'target_table': 'ftest_table1',
'target_column': 'ftest_column1',
'transformation_rule': rule,
}
test_rows.append(ins_dict)
with get_udl_connection() as conn:
self.ref_table = conn.get_table(self.ref_table_name)
conn.execute(self.ref_table.insert(test_rows))
FILES = ['BENCHMARK_RECORDS_10K.tar.gz.asc', 'BENCHMARK_RECORDS_50K.tar.gz.asc', 'BENCHMARK_RECORDS_100K.tar.gz.asc', 'BENCHMARK_RECORDS_500K.tar.gz.asc', 'BENCHMARK_RECORDS_2M.tar.gz.asc', 'BENCHMARK_RECORDS_5M.tar.gz.asc']
#CSV_FILES = ['REALDATA_RECORDS_10K.csv', 'REALDATA_RECORDS_50K.csv', 'REALDATA_RECORDS_100K.csv',
# 'REALDATA_RECORDS_500K.csv', 'REALDATA_RECORDS_2M.csv', 'REALDATA_RECORDS_5M.csv']
#JSON_FILES = ['METADATA_RECORDS_10K.json', 'METADATA_RECORDS_50K.json', 'METADATA_RECORDS_100K.json',
# 'METADATA_RECORDS_500K.json', 'METADATA_RECORDS_2M.json', 'METADATA_RECORDS_5M.json']
HISTORY_TABLE = 'HISTORY_TABLE'
BATCH_TABLE = 'UDL_BATCH'
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
logger = get_task_logger(__name__)
config_path_file = UDL2_DEFAULT_CONFIG_PATH_FILE
conf_tup = read_ini_file(config_path_file)
udl2_conf = conf_tup[0]
def start_test(directory_path, memory, cpu, hist_db, hist_schema, port, user, passwd, host):
'''
kick off the tests by scheduling the task with celery
'''
run_pipeline.apply_async(({'directory': directory_path, 'memory': memory, 'cpu': cpu, 'hist_db': hist_db,
'hist_schema': hist_schema, 'port': port, 'user': user, 'passwd': passwd, 'host': host}, ))
@celery.task(name='benchmarking.run_benchmarking.run_pipeline')
def run_pipeline(msg):
'''
Run the pipeline for the current file and use this task as a callback for future tasks
def main():
config_path_file = UDL2_DEFAULT_CONFIG_PATH_FILE
udl2_conf, udl2_flat_conf = read_ini_file(config_path_file)
initialize_db_target(udl2_conf)
clean_up_unused_schemas()
parser.add_argument('-r', dest='report_hour', help='report hour, e.g. yesterday 10am to today 10am', default='10')
parser.add_argument('-p', dest='pidfile', default='/opt/edware/run/edudl2-report.pid',
help="pid file for edudl2 trigger daemon")
parser.add_argument('-d', dest='daemon', action='store_true', default=False,
help="daemon mode for udl report")
args = parser.parse_args()
config_path_file = args.ini_file
hour = args.hour
report_hour = args.report_hour
daemon_mode = args.daemon
pid_file = args.pidfile
if daemon_mode:
create_daemon(pid_file)
# get udl2 configuration as nested and flat dictionary
udl2_conf, udl2_flat_conf = read_ini_file(config_path_file)
initialize_all_db(udl2_conf, udl2_flat_conf)
email_to = udl2_flat_conf.get(UDL_REPORT_MAIL_TO)
subject = udl2_flat_conf.get(UDL_REPORT_SUBJECT)
email_from = udl2_flat_conf.get(UDL_REPORT_MAIL_FROM)
enabled = udl2_flat_conf.get(UDL_REPORT_ENABLED)
start_date = today = datetime.datetime.today().strftime('%Y-%m-%d %H:00:00')
if enabled is not None and enabled.lower() == 'true':
generate_report_settings = {'report.enable': 'True',
'report.schedule.cron.hour': hour,
'report.schedule.cron.minute': '0',
'report.schedule.cron.second': '0',
'hour': report_hour,
'mail_to': email_to,
'subject': subject,
parser.add_argument('-d',
dest='daemon',
action='store_true',
default=False,
help="daemon mode for udl report")
args = parser.parse_args()
config_path_file = args.ini_file
hour = args.hour
report_hour = args.report_hour
daemon_mode = args.daemon
pid_file = args.pidfile
if daemon_mode:
create_daemon(pid_file)
# get udl2 configuration as nested and flat dictionary
udl2_conf, udl2_flat_conf = read_ini_file(config_path_file)
initialize_all_db(udl2_conf, udl2_flat_conf)
email_to = udl2_flat_conf.get(UDL_REPORT_MAIL_TO)
subject = udl2_flat_conf.get(UDL_REPORT_SUBJECT)
email_from = udl2_flat_conf.get(UDL_REPORT_MAIL_FROM)
enabled = udl2_flat_conf.get(UDL_REPORT_ENABLED)
start_date = today = datetime.datetime.today().strftime(
'%Y-%m-%d %H:00:00')
if enabled is not None and enabled.lower() == 'true':
generate_report_settings = {
'report.enable': 'True',
'report.schedule.cron.hour': hour,
'report.schedule.cron.minute': '0',
'report.schedule.cron.second': '0',
'hour': report_hour,