def test_timestamp_casting_view_from_parquet_with_cast():
conf_string = '''---
staging_database:
name: test_db
tables:
- destination:
name: destination
check_column: col1
columns:
- name: col1
datatype: string
comment: column one comment
- name: col2
datatype: datetime
comment: column two comment'''
expected = test_util.read_file(
'timestamp-casting-view-from-parquet-with-cast.sql')
actual = test_util.merge_single(
conf_string, type_mappings,
'../timestamp-casting-view-from-parquet.sql')
print('actual: \n{}'.format(actual))
print('expected: \n{}'.format(expected))
assert actual == expected
def test_sqoop_creat_sh():
conf_string = '''---
source_database:
name: sourcedb
sqoop_job_name_suffix: unittest
staging_database:
name: test_db
path: /data/
source_database.connection_string: jdbc://mydb
user_name: myuser
sqoop_password_file: mypasswordfile
tables:
- destination:
name: destination
source:
name: source_table
split_by_column: col1
check_column: col1
columns:
- name: col1
datatype: string
comment: column one comment
- name: col2
datatype: int
comment: column two comment'''
expected = test_util.read_file('sqoop-create.sh')
actual = test_util.merge_single(conf_string, None, '../sqoop-create.sh')
assert actual == expected
def test_kudu_table_count_sql():
conf_string = '''---
staging_database:
name: test_db
tables:
- destination:
name: destination'''
expected = test_util.read_file('kudu-table-count.sql')
actual = test_util.merge_single(conf_string, None,
'../kudu-table-count.sql')
assert actual == expected
def test_avro_avsc_with_decimal():
type_mappings = {
'decimal': {
'avro': 'decimal'
},
'string': {
'avro': 'string'
}
}
conf_string = '''---
source_database:
name: sourcedb
sqoop_job_name_suffix: unittest
staging_database:
name: test_db
path: /data/
source_database.connection_string: jdbc://mydb
user_name: myuser
sqoop_password_file: mypasswordfile
tables:
- id: mytable
destination:
name: destination
source:
name: source_table
split_by_column: col1
check_column: col1
columns:
- name: col1
datatype: string
comment: column one comment
- name: col2
datatype: decimal
precision: 4
scale: 2
comment: column two comment'''
expected = test_util.read_file('avro.avsc')
actual = test_util.merge_single(conf_string, type_mappings, '../avro.avsc')
print(actual)
print(expected)
actual_json = json.dumps(json.loads(actual), sort_keys=True)
expected_json = json.dumps(json.loads(expected), sort_keys=True)
assert actual_json == expected_json