def test_substitutions_overwrite(self):
config1 = ConfigFactory.parse_string(
"""
a = 123
a = ${?test}
a = 5
"""
)
assert config1['a'] == 5
config2 = ConfigFactory.parse_string(
"""
{
database {
host = "localhost"
port = 8000
url = ${database.host}":"${database.port}
}
database {
host = ${?DB_HOST}
}
database {
host = "other.host.net"
port = 433
}
}
"""
)
assert config2['database']['host'] == 'other.host.net'
assert config2['database']['port'] == 433
assert config2['database']['url'] == 'other.host.net:433'
def test_substitutions_overwrite(self):
config1 = ConfigFactory.parse_string(
"""
a = 123
a = ${?test}
a = 5
"""
)
assert config1["a"] == 5
config2 = ConfigFactory.parse_string(
"""
{
database {
host = "localhost"
port = 8000
url = ${database.host}":"${database.port}
}
database {
host = ${?DB_HOST}
}
database {
host = "other.host.net"
port = 433
}
}
"""
)
assert config2["database"]["host"] == "other.host.net"
assert config2["database"]["port"] == 433
assert config2["database"]["url"] == "other.host.net:433"
def convert(input_file=None, output_file=None, format='json'):
"""Convert to json, properties or yaml
:param format: json, properties or yaml
:type format: basestring
:return: json, properties or yaml string representation
"""
if input_file is None:
content = sys.stdin.read()
config = ConfigFactory.parse_string(content)
else:
config = ConfigFactory.parse_file(input_file)
if format.lower() == 'json':
res = HOCONConverter.to_json(config)
elif format.lower() == 'properties':
res = HOCONConverter.to_properties(config)
elif format.lower() == 'yaml':
res = HOCONConverter.to_yaml(config)
else:
raise Exception("Format must be 'json', 'properties' or 'yaml'")
if output_file is None:
print(res)
else:
with open(output_file, "w") as fd:
fd.write(res)
def test_substitution_cycle(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
a = ${b}
b = ${c}
c = ${a}
""")
def test_string_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = ${a.b.c}
f = ${a.b.e}
}
"""
)
assert config1.get("a.b.c") == "str"
assert config1.get("d") == "str"
assert config1.get("f") == "str "
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c}
f = test ${a.b.e}
}
"""
)
assert config2.get("a.b.c") == "str"
assert config2.get("d") == "test str"
assert config2.get("f") == "test str "
config3 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c} me
f = test ${a.b.e} me
}
"""
)
assert config3.get("a.b.c") == "str"
assert config3.get("d") == "test str me"
assert config3.get("f") == "test str me"
def test_string_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = ${a.b.c}
f = ${a.b.e}
}
"""
)
assert config1.get('a.b.c') == 'str'
assert config1.get('d') == 'str'
assert config1.get('f') == 'str '
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c}
f = test ${a.b.e}
}
"""
)
assert config2.get('a.b.c') == 'str'
assert config2.get('d') == 'test str'
assert config2.get('f') == 'test str '
config3 = ConfigFactory.parse_string(
u"""
{
a: {
b: {
c = str
e = "str "
}
}
d = test ${a.b.c} me
f = test ${a.b.e} me
}
"""
)
assert config3.get('a.b.c') == 'str'
assert config3.get('d') == 'test str me'
assert config3.get('f') == 'test str me'
def test_invalid_dict(self):
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string(
"""
a = {
f: 5
g
}
""")
with pytest.raises(ParseSyntaxException):
ConfigFactory.parse_string('a = {g}')
def test_fallback_self_ref_substitutions_merge(self):
config1 = ConfigFactory.parse_string(
"""
dict = { x: 1 }
"""
)
config2 = ConfigFactory.parse_string(
"""
dict = ${dict} { y: 2 }
""",
resolve=False
)
config2 = config2.with_fallback(config1)
assert config2.get("dict") == {'x': 1, 'y': 2}
def test_fallback_self_ref_substitutions_concat_string(self):
config1 = ConfigFactory.parse_string(
"""
string = abc
"""
)
config2 = ConfigFactory.parse_string(
"""
string = ${string}def
""",
resolve=False
)
config2 = config2.with_fallback(config1)
assert config2.get("string") == 'abcdef'
def test_non_existent_substitution(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = ${non_existent}
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = abc ${non_existent}
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = ${non_existent} abc
"""
)
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
common_modules = abc ${non_existent} def
"""
)
def test_fallback_self_ref_substitutions_append_plus_equals(self):
config1 = ConfigFactory.parse_string(
"""
list = [ 1, 2, 3 ]
"""
)
config2 = ConfigFactory.parse_string(
"""
list += [ 4, 5, 6 ]
""",
resolve=False
)
config2 = config2.with_fallback(config1)
assert config2.get("list") == [1, 2, 3, 4, 5, 6]
def test_bad_concat(self):
ConfigFactory.parse_string('a = 45\n')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = [4] "4"')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = "4" [5]')
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string('a = {b: 5} "4"')
def test_int_substitutions(self):
config1 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = ${a.b.c}
}
"""
)
assert config1.get('a.b.c') == 5
assert config1.get('d') == 5
config2 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = test ${a.b.c}
}
"""
)
assert config2.get('a.b.c') == 5
assert config2.get('d') == 'test 5'
config3 = ConfigFactory.parse_string(
"""
{
a: {
b: {
c = 5
}
}
d = test ${a.b.c} me
}
"""
)
assert config3.get('a.b.c') == 5
assert config3.get('d') == 'test 5 me'
def test_self_append_nonexistent_object(self):
config = ConfigFactory.parse_string(
"""
x += {a: 1}
"""
)
assert config.get("x") == {'a': 1}
def test_parse_with_comments(self):
config = ConfigFactory.parse_string(
"""
// comment 1
# comment 2
{
c = test // comment 0
g = 6 test # comment 0
# comment 3
a: { # comment 4
b: test, # comment 5
} # comment 6
t = [1, # comment 7
2, # comment 8
3, # comment 9
]
} # comment 10
// comment 11
// comment 12
"""
)
assert config.get('c') == 'test'
assert config.get('g') == '6 test'
assert config.get('a.b') == 'test'
assert config.get_string('a.b') == 'test'
assert config.get('t') == [1, 2, 3]
def test_parse_simple_value(self):
config = ConfigFactory.parse_string(
"""t = {
c = 5
"d" = true
e.y = {
f: 7
g: "hey dude!"
h: hey man!
i = \"\"\"
"first line"
"second" line
\"\"\"
}
j = [1, 2, 3]
u = 192.168.1.3/32
}
"""
)
assert config.get_string("t.c") == "5"
assert config.get_int("t.c") == 5
assert config.get("t.e.y.f") == 7
assert config.get("t.e.y.g") == "hey dude!"
assert config.get("t.e.y.h") == "hey man!"
assert [l.strip() for l in config.get("t.e.y.i").split("\n")] == ["", '"first line"', '"second" line', ""]
assert config.get_bool("t.d") is True
assert config.get_int("t.e.y.f") == 7
assert config.get("t.j") == [1, 2, 3]
assert config.get("t.u") == "192.168.1.3/32"
def test_parse_null(self):
config = ConfigFactory.parse_string(
"""
a = null
"""
)
assert config.get('a') is None
def test_assign_list_strings_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
"a",
"b",
]
b = # test
# test2
[
"c",
"d",]
c =
[
"e",
"f"
]
"""
)
assert config["a"] == ["a", "b"]
assert config["b"] == ["c", "d"]
assert config["c"] == ["e", "f"]
def test_self_append_nonexistent_array(self):
config = ConfigFactory.parse_string(
"""
x += [1,2]
"""
)
assert config.get("x") == [1, 2]
def get_custom_settings(args):
custom_settings_file = vars(args).get('custom_settings_file')
if custom_settings_file and os.path.exists(custom_settings_file):
print('Loading custom settings {}'.format(custom_settings_file))
return ConfigFactory.parse_file(custom_settings_file)
else:
return None
def test_assign_dict_strings_no_equal_sign_with_eol(self):
config = ConfigFactory.parse_string(
"""
a
{
a: 1,
b: 2,
}
b # test
# test2
{
c: 3,
d: 4,}
c
{
e: 5,
f: 6
}
"""
)
assert config["a"] == {"a": 1, "b": 2}
assert config["b"] == {"c": 3, "d": 4}
assert config["c"] == {"e": 5, "f": 6}
def test_assign_list_numbers_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
1,
2,
]
b = # test
# test2
[
3,
4,]
c =
[
5,
6
]
"""
)
assert config['a'] == [1, 2]
assert config['b'] == [3, 4]
assert config['c'] == [5, 6]
def test_validation_success(self):
job = WordCountSparkJob()
result = job.validate(
self.sc,
None,
ConfigFactory.parse_string('input.strings = ["a", "a", "b"]'))
self.assertEqual(result, ['a', 'a', 'b'])
def test_assign_list_strings_with_eol(self):
config = ConfigFactory.parse_string(
"""
a =
[
"a",
"b",
]
b = # test
# test2
[
"c",
"d",]
c =
[
"e",
"f"
]
"""
)
assert config['a'] == ['a', 'b']
assert config['b'] == ['c', 'd']
assert config['c'] == ['e', 'f']
def test_assign_dict_strings_no_equal_sign_with_eol(self):
config = ConfigFactory.parse_string(
"""
a
{
a: 1,
b: 2,
}
b # test
# test2
{
c: 3,
d: 4,}
c
{
e: 5,
f: 6
}
"""
)
assert config['a'] == {'a': 1, 'b': 2}
assert config['b'] == {'c': 3, 'd': 4}
assert config['c'] == {'e': 5, 'f': 6}
def test_parse_simple_value(self):
config = ConfigFactory.parse_string(
"""t = {
c = 5
"d" = true
e.y = {
f: 7
g: "hey dude!"
h: hey man!
i = \"\"\"
"first line"
"second" line
\"\"\"
}
j = [1, 2, 3]
u = 192.168.1.3/32
}
"""
)
assert config.get_string('t.c') == '5'
assert config.get_int('t.c') == 5
assert config.get('t.e.y.f') == 7
assert config.get('t.e.y.g') == 'hey dude!'
assert config.get('t.e.y.h') == 'hey man!'
assert [l.strip() for l in config.get('t.e.y.i').split('\n')] == ['', '"first line"', '"second" line', '']
assert config.get_bool('t.d') is True
assert config.get_int('t.e.y.f') == 7
assert config.get('t.j') == [1, 2, 3]
assert config.get('t.u') == '192.168.1.3/32'
def test_parse_with_comments(self):
config = ConfigFactory.parse_string(
"""
// comment 1
# comment 2
{
c = test // comment 0
g = 6 test # comment 0
# comment 3
a: { # comment 4
b: test, # comment 5
} # comment 6
t = [1, # comment 7
2, # comment 8
3, # comment 9
]
} # comment 10
// comment 11
// comment 12
"""
)
assert config.get("c") == "test"
assert config.get("g") == "6 test"
assert config.get("a.b") == "test"
assert config.get_string("a.b") == "test"
assert config.get("t") == [1, 2, 3]
def test_dict_merge(self):
config = ConfigFactory.parse_string(
"""
a {
d {
g.h.j.u: 5
g {
h.d: 4
}
g.h.k: f d
}
h.i.m = 7
h.i {
d: 5
}
h.i {
e:65
}
}
"""
)
expected_result = {
"a": {"d": {"g": {"h": {"j": {"u": 5}, "d": 4, "k": "f d"}}}, "h": {"i": {"m": 7, "d": 5, "e": 65}}}
}
assert expected_result == config
def test_from_dict_with_ordered_dict(self):
d = OrderedDict()
d['banana'] = 3
d['apple'] = 4
d['pear'] = 1
d['orange'] = 2
config = ConfigFactory.from_dict(d)
assert config == d
class GlueExtractor(Extractor):
"""
Extracts metadata from AWS glue. Adapted from Amundsen's glue extractor.
"""
CONNECTION_NAME_KEY = "connection_name"
FILTER_KEY = "filters"
IS_LOCATION_PARSING_ENABLED_KEY = "is_location_parsing_enabled"
DEFAULT_CONFIG = ConfigFactory.from_dict({
FILTER_KEY: None,
IS_LOCATION_PARSING_ENABLED_KEY: False,
CONNECTION_NAME_KEY: None,
})
def init(self, conf: ConfigTree) -> None:
conf = conf.with_fallback(GlueExtractor.DEFAULT_CONFIG)
self._filters = conf.get(GlueExtractor.FILTER_KEY)
self._connection_name = conf.get(
GlueExtractor.CONNECTION_NAME_KEY) or ""
self._is_location_parsing_enabled = conf.get(
GlueExtractor.IS_LOCATION_PARSING_ENABLED_KEY)
self._glue = boto3.client("glue")
self._extract_iter: Union[None, Iterator] = None
def extract(self) -> Union[TableMetadata, None]:
if not self._extract_iter:
self._extract_iter = self._get_extract_iter()
try:
return next(self._extract_iter)
except StopIteration:
return None
def get_scope(self) -> str:
return "extractor.glue"
def _get_extract_iter(self) -> Iterator[TableMetadata]:
for row in self._get_raw_extract_iter():
columns, i = [], 0
for column in row["StorageDescriptor"]["Columns"] + row.get(
"PartitionKeys", []):
columns.append(
ColumnMetadata(
column["Name"],
column["Comment"] if "Comment" in column else None,
column["Type"],
i,
))
i += 1
if self._is_location_parsing_enabled:
catalog, schema, table = self._parse_location(
location=row["StorageDescriptor"]["Location"],
name=row["Name"])
else:
catalog = None
schema = None
table = row["Name"]
if self._connection_name:
database = self._connection_name + "/" + row["DatabaseName"]
else:
database = row["DatabaseName"]
yield TableMetadata(
database,
catalog,
schema,
table,
row.get("Description")
or row.get("Parameters", {}).get("comment"),
columns,
row.get("TableType") == "VIRTUAL_VIEW",
)
def _parse_location(self, location, name):
"""
Location is formatted in glue as `catalog.schema.table`, while name
is formatted as `catalog_schema_table`. To determine what the catalog,
schema, and table are, then, (particularly in the case where catalogs,
schemas, and tables can have underscores and/or periods), we need to
find points where location has a `.`, while name has a `_`."""
start_index = 0
splits = []
for end_index, (location_character,
name_character) in enumerate(zip(location, name)):
if location_character == "." and name_character == "_":
splits.append(location[start_index:end_index])
start_index = end_index + 1
elif end_index == len(location) - 1:
splits.append(location[start_index:])
table = splits[-1]
schema = splits[-2]
if len(splits) == 3:
catalog = splits[-3]
else:
catalog = None
return catalog, schema, table
def _get_raw_extract_iter(self) -> Iterator[Dict[str, Any]]:
tables = self._search_tables()
return iter(tables)
def _search_tables(self) -> List[Dict[str, Any]]:
tables = []
kwargs = {}
if self._filters is not None:
kwargs["Filters"] = self._filters
data = self._glue.search_tables(**kwargs)
tables += data["TableList"]
while "NextToken" in data:
token = data["NextToken"]
kwargs["NextToken"] = token
data = self._glue.search_tables(**kwargs)
tables += data["TableList"]
return tables
def test_parse_with_enclosing_square_bracket(self):
config = ConfigFactory.parse_string("[1, 2, 3]")
assert config == [1, 2, 3]
class AtlasSearchDataExtractor(Extractor):
ATLAS_URL_CONFIG_KEY = 'atlas_url'
ATLAS_PORT_CONFIG_KEY = 'atlas_port'
ATLAS_PROTOCOL_CONFIG_KEY = 'atlas_protocol'
ATLAS_VALIDATE_SSL_CONFIG_KEY = 'atlas_validate_ssl'
ATLAS_USERNAME_CONFIG_KEY = 'atlas_auth_user'
ATLAS_PASSWORD_CONFIG_KEY = 'atlas_auth_pw'
ATLAS_SEARCH_CHUNK_SIZE_KEY = 'atlas_search_chunk_size'
ATLAS_DETAILS_CHUNK_SIZE_KEY = 'atlas_details_chunk_size'
ATLAS_TIMEOUT_SECONDS_KEY = 'atlas_timeout_seconds'
ATLAS_MAX_RETRIES_KEY = 'atlas_max_retries'
PROCESS_POOL_SIZE_KEY = 'process_pool_size'
ENTITY_TYPE_KEY = 'entity_type'
DEFAULT_CONFIG = ConfigFactory.from_dict({
ATLAS_URL_CONFIG_KEY: "localhost",
ATLAS_PORT_CONFIG_KEY: 21000,
ATLAS_PROTOCOL_CONFIG_KEY: 'http',
ATLAS_VALIDATE_SSL_CONFIG_KEY: False,
ATLAS_SEARCH_CHUNK_SIZE_KEY: 250,
ATLAS_DETAILS_CHUNK_SIZE_KEY: 25,
ATLAS_TIMEOUT_SECONDS_KEY: 120,
ATLAS_MAX_RETRIES_KEY: 2,
PROCESS_POOL_SIZE_KEY: 10
})
# @todo fill out below fields for TableESDocument
# tags: List[str],
# es_document field, atlas field path, modification function, default_value
FIELDS_MAPPING_SPEC: type_fields_mapping_spec = {
'Table':
[('database', 'typeName', None, None),
('cluster', 'attributes.qualifiedName',
lambda x: x.split('@')[-1], None),
('schema', 'relationshipAttributes.db.displayText', None, None),
('name', 'attributes.name', None, None),
('key', 'attributes.qualifiedName', None, None),
('description', 'attributes.description', None, None),
('last_updated_timestamp', 'updateTime', lambda x: int(x) / 1000, 0),
('total_usage', 'attributes.popularityScore', lambda x: int(x), 0),
('unique_usage', 'attributes.uniqueUsage', lambda x: int(x), 1),
('column_names', 'relationshipAttributes.columns',
lambda x: AtlasSearchDataExtractorHelpers.get_column_names(x), []),
('column_descriptions', 'relationshipAttributes.columns',
lambda x: AtlasSearchDataExtractorHelpers.get_column_descriptions(x),
[]),
('tags', 'relationshipAttributes.meanings', lambda x:
AtlasSearchDataExtractorHelpers.get_tags_from_glossary_terms(x), []),
('badges', 'classifications', lambda x:
AtlasSearchDataExtractorHelpers.get_badges_from_classifications(x),
[]),
('display_name', 'attributes.qualifiedName',
lambda x: x.split('@')[0], None),
('schema_description', 'attributes.parameters.sourceDescription',
None, None),
('programmatic_descriptions', 'attributes.parameters',
lambda x: [str(s) for s in list(x.values())], {})]
}
ENTITY_MODEL_BY_TYPE = {
'Table':
'databuilder.models.table_elasticsearch_document.TableESDocument'
}
REQUIRED_RELATIONSHIPS_BY_TYPE = {'Table': ['columns']}
def init(self, conf: ConfigTree) -> None:
self.conf = conf.with_fallback(AtlasSearchDataExtractor.DEFAULT_CONFIG)
self.driver = self._get_driver()
self._extract_iter: Optional[Iterator[Any]] = None
@property
def entity_type(self) -> str:
return self.conf.get(AtlasSearchDataExtractor.ENTITY_TYPE_KEY)
@property
def basic_search_query(self) -> Dict:
query = {
'typeName': self.entity_type,
'excludeDeletedEntities': True,
'query': '*'
}
LOGGER.debug(f'Basic Search Query: {query}')
return query
@property
def dsl_search_query(self) -> Dict:
query = {'query': f'{self.entity_type} where __state = "ACTIVE"'}
LOGGER.debug(f'DSL Search Query: {query}')
return query
@property
def model_class(self) -> Any:
model_class = AtlasSearchDataExtractor.ENTITY_MODEL_BY_TYPE.get(
self.entity_type)
if model_class:
module_name, class_name = model_class.rsplit(".", 1)
mod = importlib.import_module(module_name)
return getattr(mod, class_name)
@property
def field_mappings(self) -> type_fields_mapping:
return AtlasSearchDataExtractor.FIELDS_MAPPING_SPEC.get(
self.entity_type) or []
@property
def search_chunk_size(self) -> int:
return self.conf.get_int(
AtlasSearchDataExtractor.ATLAS_SEARCH_CHUNK_SIZE_KEY)
@property
def relationships(self) -> Optional[List[str]]:
return AtlasSearchDataExtractor.REQUIRED_RELATIONSHIPS_BY_TYPE.get(
self.entity_type)
def extract(self) -> Any:
if not self._extract_iter:
self._extract_iter = self._get_extract_iter()
try:
return next(self._extract_iter)
except StopIteration:
return None
def get_scope(self) -> str:
return 'extractor.atlas_search_data'
def _get_driver(self) -> Any:
return Atlas(
host=self.conf.get_string(
AtlasSearchDataExtractor.ATLAS_URL_CONFIG_KEY),
port=self.conf.get_string(
AtlasSearchDataExtractor.ATLAS_PORT_CONFIG_KEY),
username=self.conf.get_string(
AtlasSearchDataExtractor.ATLAS_USERNAME_CONFIG_KEY),
password=self.conf.get_string(
AtlasSearchDataExtractor.ATLAS_PASSWORD_CONFIG_KEY),
protocol=self.conf.get_string(
AtlasSearchDataExtractor.ATLAS_PROTOCOL_CONFIG_KEY),
validate_ssl=self.conf.get_bool(
AtlasSearchDataExtractor.ATLAS_VALIDATE_SSL_CONFIG_KEY),
timeout=self.conf.get_int(
AtlasSearchDataExtractor.ATLAS_TIMEOUT_SECONDS_KEY),
max_retries=self.conf.get_int(
AtlasSearchDataExtractor.ATLAS_MAX_RETRIES_KEY))
def _get_latest_entity_metrics(self) -> Optional[dict]:
admin_metrics = list(self.driver.admin_metrics)
try:
return admin_metrics[-1].entity
except Exception as e:
return None
def _get_count_of_active_entities(self) -> int:
entity_metrics = self._get_latest_entity_metrics()
if entity_metrics:
count = entity_metrics.get('entityActive-typeAndSubTypes',
dict()).get(self.entity_type, 0)
return int(count)
else:
return 0
def _get_entity_guids(self, start_offset: int) -> List[str]:
result = []
batch_start = start_offset
batch_end = start_offset + self.search_chunk_size
LOGGER.info(f'Collecting guids for batch: {batch_start}-{batch_end}')
_params = {
'offset': str(batch_start),
'limit': str(self.search_chunk_size)
}
full_params = deepcopy(self.dsl_search_query)
full_params.update(**_params)
try:
results = self.driver.search_dsl(**full_params)
for hit in results:
for entity in hit.entities:
result.append(entity.guid)
return result
except Exception:
LOGGER.warning(
f'Error processing batch: {batch_start}-{batch_end}',
exc_info=True)
return []
def _get_entity_details(self, guid_list: List[str]) -> List:
result = []
LOGGER.info(f'Processing guids chunk of size: {len(guid_list)}')
try:
bulk_collection = self.driver.entity_bulk(guid=guid_list)
for collection in bulk_collection:
search_chunk = list(
collection.entities_with_relationships(
attributes=self.relationships))
result += search_chunk
return result
except Exception:
LOGGER.warning(f'Error processing guids. {len(guid_list)}',
exc_info=True)
return []
@staticmethod
def split_list_to_chunks(input_list: List[Any], n: int) -> Generator:
"""Yield successive n-sized chunks from lst."""
for i in range(0, len(input_list), n):
yield input_list[i:i + n]
def _execute_query(self) -> Any:
details_chunk_size = self.conf.get_int(
AtlasSearchDataExtractor.ATLAS_DETAILS_CHUNK_SIZE_KEY)
process_pool_size = self.conf.get_int(
AtlasSearchDataExtractor.PROCESS_POOL_SIZE_KEY)
guids = []
entity_count = self._get_count_of_active_entities()
LOGGER.info(f'Received count: {entity_count}')
if entity_count > 0:
offsets = [
i * self.search_chunk_size
for i in range(int(entity_count / self.search_chunk_size) + 1)
]
else:
offsets = []
with multiprocessing.pool.ThreadPool(
processes=process_pool_size) as pool:
guid_list = pool.map(self._get_entity_guids, offsets, chunksize=1)
for sub_list in guid_list:
guids += sub_list
LOGGER.info(f'Received guids: {len(guids)}')
if guids:
guids_chunks = AtlasSearchDataExtractor.split_list_to_chunks(
guids, details_chunk_size)
with multiprocessing.pool.ThreadPool(
processes=process_pool_size) as pool:
return_list = pool.map(self._get_entity_details, guids_chunks)
for sub_list in return_list:
for entry in sub_list:
yield entry
def _get_extract_iter(self) -> Iterator[Any]:
for atlas_entity in self._execute_query():
model_dict = dict()
try:
data = atlas_entity.__dict__['_data']
for spec in self.field_mappings:
model_field, atlas_field_path, _transform_spec, default_value = spec
atlas_value = reduce(lambda x, y: x.get(y, dict()),
atlas_field_path.split('.'),
data) or default_value
transform_spec = _transform_spec or (lambda x: x)
es_entity_value = transform_spec(atlas_value)
model_dict[model_field] = es_entity_value
yield self.model_class(**model_dict)
except Exception:
LOGGER.warning(f'Error building model object.', exc_info=True)
from flask import Flask, request, render_template, jsonify
from pyhocon import ConfigFactory
import os, psycopg2, json
conf = ConfigFactory.parse_file('db.conf')
host = conf['databases.postgres.host']
user = conf['databases.postgres.user']
database = conf['databases.postgres.database']
password = conf['databases.postgres.password']
app = Flask(__name__)
@app.route('/')
@app.route('/<path:path>', methods=['GET'])
def root(path=None):
curr_path = request.path
print(curr_path)
print(path)
count_path(curr_path)
return render_template('index.html')
@app.route('/counts', methods=['GET'])
def count():
data = query()
json_data = to_json(data)
print(json_data)
return jsonify(json_data)
def updates():
conf = ConfigFactory.parse_file(session['path'])
session['confist'] = dict(conf)
def create_es_publisher_sample_job(
elasticsearch_index_alias='table_search_index',
elasticsearch_doc_type_key='table',
model_name='databuilder.models.table_elasticsearch_document.TableESDocument',
entity_type='table',
elasticsearch_mapping=None):
"""
:param elasticsearch_index_alias: alias for Elasticsearch used in
amundsensearchlibrary/search_service/config.py as an index
:param elasticsearch_doc_type_key: name the ElasticSearch index is prepended with. Defaults to `table` resulting in
`table_{uuid}`
:param model_name: the Databuilder model class used in transporting between Extractor and Loader
:param entity_type: Entity type handed to the `Neo4jSearchDataExtractor` class, used to determine
Cypher query to extract data from Neo4j. Defaults to `table`.
:param elasticsearch_mapping: Elasticsearch field mapping "DDL" handed to the `ElasticsearchPublisher` class,
if None is given (default) it uses the `Table` query baked into the Publisher
"""
# loader saves data to this location and publisher reads it from here
extracted_search_data_path = '/var/tmp/amundsen/search_data.json'
task = DefaultTask(loader=FSElasticsearchJSONLoader(),
extractor=Neo4jSearchDataExtractor(),
transformer=NoopTransformer())
# elastic search client instance
elasticsearch_client = es
# unique name of new index in Elasticsearch
elasticsearch_new_index_key = '{}_'.format(
elasticsearch_doc_type_key) + str(uuid.uuid4())
job_config = ConfigFactory.from_dict({
'extractor.search_data.entity_type':
entity_type,
'extractor.search_data.extractor.neo4j.graph_url':
neo4j_endpoint,
'extractor.search_data.extractor.neo4j.model_class':
model_name,
'extractor.search_data.extractor.neo4j.neo4j_auth_user':
neo4j_user,
'extractor.search_data.extractor.neo4j.neo4j_auth_pw':
neo4j_password,
'extractor.search_data.extractor.neo4j.neo4j_encrypted':
False,
'loader.filesystem.elasticsearch.file_path':
extracted_search_data_path,
'loader.filesystem.elasticsearch.mode':
'w',
'publisher.elasticsearch.file_path':
extracted_search_data_path,
'publisher.elasticsearch.mode':
'r',
'publisher.elasticsearch.client':
elasticsearch_client,
'publisher.elasticsearch.new_index':
elasticsearch_new_index_key,
'publisher.elasticsearch.doc_type':
elasticsearch_doc_type_key,
'publisher.elasticsearch.alias':
elasticsearch_index_alias,
})
# only optionally add these keys, so need to dynamically `put` them
if elasticsearch_mapping:
job_config.put(
'publisher.elasticsearch.{}'.format(
ElasticsearchPublisher.ELASTICSEARCH_MAPPING_CONFIG_KEY),
elasticsearch_mapping)
job = DefaultJob(conf=job_config,
task=task,
publisher=ElasticsearchPublisher())
return job
def format_hocon(value):
""""
This convert a dict to hocon(STR) type.
"""
config = ConfigFactory().from_dict(value)
return HOCONConverter.convert(config, 'hocon')
def create_es_publisher_sample_job(
elasticsearch_index_alias="table_search_index",
elasticsearch_doc_type_key="table",
model_name="databuilder.models.table_elasticsearch_document.TableESDocument",
cypher_query=None,
elasticsearch_mapping=None,
):
"""
:param elasticsearch_index_alias: alias for Elasticsearch used in
amundsensearchlibrary/search_service/config.py as an index
:param elasticsearch_doc_type_key: name the ElasticSearch index is prepended with. Defaults to `table` resulting in
`table_search_index`
:param model_name: the Databuilder model class used in transporting between Extractor and Loader
:param cypher_query: Query handed to the `Neo4jSearchDataExtractor` class, if None is given (default)
it uses the `Table` query baked into the Extractor
:param elasticsearch_mapping: Elasticsearch field mapping "DDL" handed to the `ElasticsearchPublisher` class,
if None is given (default) it uses the `Table` query baked into the Publisher
"""
# loader saves data to this location and publisher reads it from here
extracted_search_data_path = "/var/tmp/amundsen/search_data.json"
task = DefaultTask(
loader=FSElasticsearchJSONLoader(),
extractor=Neo4jSearchDataExtractor(),
transformer=NoopTransformer(),
)
# elastic search client instance
elasticsearch_client = es
# unique name of new index in Elasticsearch
elasticsearch_new_index_key = "tables" + str(uuid.uuid4())
job_config = ConfigFactory.from_dict({
f"extractor.search_data.extractor.neo4j.{Neo4jExtractor.GRAPH_URL_CONFIG_KEY}":
neo4j_endpoint,
f"extractor.search_data.extractor.neo4j.{Neo4jExtractor.MODEL_CLASS_CONFIG_KEY}":
model_name,
f"extractor.search_data.extractor.neo4j.{Neo4jExtractor.NEO4J_AUTH_USER}":
neo4j_user,
f"extractor.search_data.extractor.neo4j.{Neo4jExtractor.NEO4J_AUTH_PW}":
neo4j_password,
f"loader.filesystem.elasticsearch.{FSElasticsearchJSONLoader.FILE_PATH_CONFIG_KEY}":
extracted_search_data_path,
f"loader.filesystem.elasticsearch.{FSElasticsearchJSONLoader.FILE_MODE_CONFIG_KEY}":
"w",
f"publisher.elasticsearch.{ElasticsearchPublisher.FILE_PATH_CONFIG_KEY}":
extracted_search_data_path,
f"publisher.elasticsearch.{ElasticsearchPublisher.FILE_MODE_CONFIG_KEY}":
"r",
f"publisher.elasticsearch.{ElasticsearchPublisher.ELASTICSEARCH_CLIENT_CONFIG_KEY}":
elasticsearch_client,
f"publisher.elasticsearch.{ElasticsearchPublisher.ELASTICSEARCH_NEW_INDEX_CONFIG_KEY}":
elasticsearch_new_index_key,
f"publisher.elasticsearch.{ElasticsearchPublisher.ELASTICSEARCH_DOC_TYPE_CONFIG_KEY}":
elasticsearch_doc_type_key,
f"publisher.elasticsearch.{ElasticsearchPublisher.ELASTICSEARCH_ALIAS_CONFIG_KEY}":
elasticsearch_index_alias,
})
# only optionally add these keys, so need to dynamically `put` them
if cypher_query:
job_config.put(
f"extractor.search_data.{Neo4jSearchDataExtractor.CYPHER_QUERY_CONFIG_KEY}",
cypher_query,
)
if elasticsearch_mapping:
job_config.put(
f"publisher.elasticsearch.{ElasticsearchPublisher.ELASTICSEARCH_MAPPING_CONFIG_KEY}",
elasticsearch_mapping,
)
job = DefaultJob(conf=job_config,
task=task,
publisher=ElasticsearchPublisher())
return job
def test_non_compatible_substitution(self):
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string(
"""
common_modules = [perl]
host_modules = 55 ${common_modules}
"""
)
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string(
"""
common_modules = [perl]
host_modules = ${common_modules} 55
"""
)
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string(
"""
common_modules = [perl]
host_modules = aa ${common_modules} bb
"""
)
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string(
"""
common_modules = [perl]
host_modules = aa ${common_modules}
"""
)
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string(
"""
common_modules = [perl]
host_modules = ${common_modules} aa
"""
)
with pytest.raises(ConfigWrongTypeException):
ConfigFactory.parse_string(
"""
common_modules = [perl]
host_modules = aa ${common_modules} bb
"""
)
def test_validation_failure(self):
job = WordCountSparkJob()
result = job.validate(self.sc, None, ConfigFactory.parse_string(""))
self.assertTrue(isinstance(result, list))
self.assertEqual(1, len(result))
self.assertTrue(isinstance(result[0], ValidationProblem))
class PrestoViewMetadataExtractor(Extractor):
"""
Extracts Presto View and column metadata from underlying meta store database using SQLAlchemyExtractor
PrestoViewMetadataExtractor does not require a separate table model but just reuse the existing TableMetadata
"""
# SQL statement to extract View metadata
# {where_clause_suffix} could be used to filter schemas
SQL_STATEMENT = """
SELECT t.TBL_ID, d.NAME as `schema`, t.TBL_NAME name, t.TBL_TYPE, t.VIEW_ORIGINAL_TEXT as view_original_text
FROM TBLS t
JOIN DBS d ON t.DB_ID = d.DB_ID
WHERE t.VIEW_EXPANDED_TEXT = '/* Presto View */'
{where_clause_suffix}
ORDER BY t.TBL_ID desc;
"""
# Presto View data prefix and suffix definition:
# https://github.com/prestodb/presto/blob/43bd519052ba4c56ff1f4fc807075637ab5f4f10/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java#L153-L154
PRESTO_VIEW_PREFIX = '/* Presto View: '
PRESTO_VIEW_SUFFIX = ' */'
# CONFIG KEYS
WHERE_CLAUSE_SUFFIX_KEY = 'where_clause_suffix'
CLUSTER_KEY = 'cluster'
DEFAULT_CONFIG = ConfigFactory.from_dict({
WHERE_CLAUSE_SUFFIX_KEY: ' ',
CLUSTER_KEY: 'gold'
})
def init(self, conf):
# type: (ConfigTree) -> None
conf = conf.with_fallback(PrestoViewMetadataExtractor.DEFAULT_CONFIG)
self._cluster = '{}'.format(
conf.get_string(PrestoViewMetadataExtractor.CLUSTER_KEY))
self.sql_stmt = PrestoViewMetadataExtractor.SQL_STATEMENT.format(
where_clause_suffix=conf.get_string(
PrestoViewMetadataExtractor.WHERE_CLAUSE_SUFFIX_KEY))
LOGGER.info('SQL for hive metastore: {}'.format(self.sql_stmt))
self._alchemy_extractor = SQLAlchemyExtractor()
sql_alch_conf = Scoped.get_scoped_conf(conf, self._alchemy_extractor.get_scope())\
.with_fallback(ConfigFactory.from_dict({SQLAlchemyExtractor.EXTRACT_SQL: self.sql_stmt}))
self._alchemy_extractor.init(sql_alch_conf)
self._extract_iter = None # type: Union[None, Iterator]
def extract(self):
# type: () -> Union[TableMetadata, None]
if not self._extract_iter:
self._extract_iter = self._get_extract_iter()
try:
return next(self._extract_iter)
except StopIteration:
return None
def get_scope(self):
# type: () -> str
return 'extractor.presto_view_metadata'
def _get_extract_iter(self):
# type: () -> Iterator[TableMetadata]
"""
Using itertools.groupby and raw level iterator, it groups to table and yields TableMetadata
:return:
"""
row = self._alchemy_extractor.extract()
while row:
columns = self._get_column_metadata(row['view_original_text'])
yield TableMetadata(database='presto',
cluster=self._cluster,
schema=row['schema'],
name=row['name'],
description=None,
columns=columns,
is_view=True)
row = self._alchemy_extractor.extract()
def _get_column_metadata(self, view_original_text):
# type: (str) -> List[ColumnMetadata]
"""
Get Column Metadata from VIEW_ORIGINAL_TEXT from TBLS table for Presto Views.
Columns are sorted the same way as they appear in Presto Create View SQL.
:param view_original_text:
:return:
"""
# remove encoded Presto View data prefix and suffix
encoded_view_info = (view_original_text.split(
PrestoViewMetadataExtractor.PRESTO_VIEW_PREFIX,
1)[-1].rsplit(PrestoViewMetadataExtractor.PRESTO_VIEW_SUFFIX,
1)[0])
# view_original_text is b64 encoded:
# https://github.com/prestodb/presto/blob/43bd519052ba4c56ff1f4fc807075637ab5f4f10/presto-hive/src/main/java/com/facebook/presto/hive/HiveUtil.java#L602-L605
decoded_view_info = base64.b64decode(encoded_view_info)
columns = json.loads(decoded_view_info).get('columns')
return [
ColumnMetadata(name=column['name'],
description=None,
col_type=column['type'],
sort_order=i) for i, column in enumerate(columns)
]
import os, sys, json
from pyhocon import ConfigFactory
conf = None
def get(key, default=None):
try:
global conf
return conf.get(key, default)
except AttributeError as aex:
return default
mandatory_confs = [
"elasticsearch.hostname",
"data.filedir",
"data.version",
]
conf_dir = os.path.abspath(os.path.dirname(__file__))
conf_path = os.path.join(conf_dir, 'config.json')
conf = ConfigFactory.parse_file(conf_path)
for key in mandatory_confs:
assert key in conf, "Key %s down not exists" % key
def test_include_dict_from_samples(self):
config = ConfigFactory.parse_file("samples/animals.conf")
assert config.get('cat.garfield.say') == 'meow'
assert config.get('dog.mutt.hates.garfield.say') == 'meow'
class PandasProfilingColumnStatsExtractor(Extractor):
FILE_PATH = 'file_path'
DATABASE_NAME = 'database_name'
TABLE_NAME = 'table_name'
SCHEMA_NAME = 'schema_name'
CLUSTER_NAME = 'cluster_name'
# if you wish to collect only selected set of metrics configure stat_mappings option of the extractor providing
# similar dictionary but containing only keys of metrics you wish to collect.
# For example - if you want only min and max value of a column, provide extractor with configuration option:
# PandasProfilingColumnStatsExtractor.STAT_MAPPINGS = {'max': ('Maximum', float), 'min': ('Minimum', float)}
STAT_MAPPINGS = 'stat_mappings'
# - key - raw name of the stat in pandas-profiling. Value - tuple of stat spec.
# - first value of the tuple - full name of the stat
# - second value of the tuple - function modifying the stat (by default we just do type casting)
DEFAULT_STAT_MAPPINGS = {
'25%': ('Quantile 25%', float),
'5%': ('Quantile 5%', float),
'50%': ('Quantile 50%', float),
'75%': ('Quantile 75%', float),
'95%': ('Quantile 95%', float),
'chi_squared': ('Chi squared', lambda x: float(x.get('statistic'))),
'count': ('Count', int),
'is_unique': ('Unique', bool),
'kurtosis': ('Kurtosis', float),
'max': ('Maximum', str),
'max_length': ('Maximum length', int),
'mean': ('Mean', float),
'mean_length': ('Mean length', int),
'median_length': ('Median length', int),
'min': ('Minimum', str),
'min_length': ('Minimum length', int),
'monotonic': ('Monotonic', bool),
'n_characters': ('Characters', int),
'n_characters_distinct': ('Distinct characters', int),
'n_distinct': ('Distinct values', int),
'n_infinite': ('Infinite values', int),
'n_missing': ('Missing values', int),
'n_negative': ('Negative values', int),
'n_unique': ('Unique values', int),
'n_zeros': ('Zeros', int),
'p_distinct': ('Distinct values %', lambda x: float(x * 100)),
'p_infinite': ('Infinite values %', lambda x: float(x * 100)),
'p_missing': ('Missing values %', lambda x: float(x * 100)),
'p_negative': ('Negative values %', lambda x: float(x * 100)),
'p_unique': ('Unique values %', lambda x: float(x * 100)),
'p_zeros': ('Zeros %', lambda x: float(x * 100)),
'range': ('Range', str),
'skewness': ('Skewness', float),
'std': ('Std. deviation', float),
'sum': ('Sum', float),
'variance': ('Variance', float)
# Stats available in pandas-profiling but are not collected by default and require custom, conscious config..
# 'block_alias_char_counts': ('',),
# 'block_alias_counts': ('',),
# 'block_alias_values': ('',),
# 'category_alias_char_counts': ('',),
# 'category_alias_counts': ('',),
# 'category_alias_values': ('',),
# 'character_counts': ('',),
# 'cv': ('',),
# 'first_rows': ('',),
# 'hashable': ('',),
# 'histogram': ('',),
# 'histogram_frequencies': ('',),
# 'histogram_length': ('',),
# 'iqr': ('',),
# 'length': ('',),
# 'mad': ('',),
# 'memory_size': ('',),
# 'monotonic_decrease': ('Monotonic decrease', bool),
# 'monotonic_decrease_strict': ('Strict monotonic decrease', bool),
# 'monotonic_increase': ('Monotonic increase', bool),
# 'monotonic_increase_strict': ('Strict monotonic increase', bool),
# 'n': ('',),
# 'n_block_alias': ('',),
# 'n_category': ('Categories', int),
# 'n_scripts': ('',),
# 'ordering': ('',),
# 'script_char_counts': ('',),
# 'script_counts': ('',),
# 'value_counts_index_sorted': ('',),
# 'value_counts_without_nan': ('',),
# 'word_counts': ('',),
# 'type': ('Type', str)
}
PRECISION = 'precision'
DEFAULT_CONFIG = ConfigFactory.from_dict({
STAT_MAPPINGS: DEFAULT_STAT_MAPPINGS,
PRECISION: 3
})
def get_scope(self) -> str:
return 'extractor.pandas_profiling'
def init(self, conf: ConfigTree) -> None:
self.conf = conf.with_fallback(
PandasProfilingColumnStatsExtractor.DEFAULT_CONFIG)
self._extract_iter = self._get_extract_iter()
def extract(self) -> Any:
try:
result = next(self._extract_iter)
return result
except StopIteration:
return None
def _get_extract_iter(self) -> Any:
report = self._load_report()
variables = report.get('variables', dict())
report_time = self.parse_date(
report.get('analysis', dict()).get('date_start'))
for column_name, column_stats in variables.items():
for _stat_name, stat_value in column_stats.items():
stat_spec = self.stat_mappings.get(_stat_name)
if stat_spec:
stat_name, stat_modifier = stat_spec
if isinstance(stat_value, float):
stat_value = self.round_value(stat_value)
stat = TableColumnStats(table_name=self.table_name,
col_name=column_name,
stat_name=stat_name,
stat_val=stat_modifier(stat_value),
start_epoch=report_time,
end_epoch='0',
db=self.database_name,
cluster=self.cluster_name,
schema=self.schema_name)
yield stat
def _load_report(self) -> Dict[str, Any]:
path = self.conf.get(PandasProfilingColumnStatsExtractor.FILE_PATH)
try:
with open(path, 'r') as f:
_data = f.read()
data = json.loads(_data)
return data
except Exception:
return {}
@staticmethod
def parse_date(string_date: str) -> str:
try:
date_parsed = dateutil.parser.parse(string_date)
# date from pandas-profiling doesn't contain timezone so to be timezone safe we need to assume it's utc
if not date_parsed.tzname():
return PandasProfilingColumnStatsExtractor.parse_date(
f'{string_date}+0000')
return str(int(date_parsed.timestamp()))
except Exception:
return '0'
def round_value(self, value: float) -> float:
return round(
value,
self.conf.get(PandasProfilingColumnStatsExtractor.PRECISION))
@property
def stat_mappings(self) -> Dict[str, Tuple[str, Any]]:
return dict(
self.conf.get(PandasProfilingColumnStatsExtractor.STAT_MAPPINGS))
@property
def cluster_name(self) -> str:
return self.conf.get(PandasProfilingColumnStatsExtractor.CLUSTER_NAME)
@property
def database_name(self) -> str:
return self.conf.get(PandasProfilingColumnStatsExtractor.DATABASE_NAME)
@property
def schema_name(self) -> str:
return self.conf.get(PandasProfilingColumnStatsExtractor.SCHEMA_NAME)
@property
def table_name(self) -> str:
return self.conf.get(PandasProfilingColumnStatsExtractor.TABLE_NAME)
def test_parse_URL_from_samples(self):
config = ConfigFactory.parse_URL("file:samples/aws.conf")
assert config.get('data-center-generic.cluster-size') == 6
assert config.get('large-jvm-opts') == ['-XX:+UseParNewGC', '-Xm16g']
if NETWORK == 'MAINNET':
copyfile('/lto-node/lto-mainnet.conf', '/lto/configs/lto-config.conf')
elif NETWORK == 'TESTNET':
copyfile('/lto-node/lto-testnet.conf', '/lto/configs/lto-config.conf')
api_key = os.environ.get('LTO_API_KEY', 'lt1secretapikey!')
api_key_hash = secureHash(api_key)
env_dict = parse_env_variables()
lto_data = get_wallet_data()
confFilePath = '/lto/configs/local.conf'
if os.path.isfile(confFilePath):
conf = ConfigFactory.parse_file(confFilePath)
if conf.get('lto.wallet.seed') != lto_data[0] or conf.get('lto.wallet.password') != lto_data[1]:
print('The wallet seed or password has changed. You will need create a new container to change the configuration.')
sys.exit(0)
nested_set(env_dict, ['lto', 'directory'], '/lto')
nested_set(env_dict, ['lto', 'data-directory'], '/lto/data')
nested_set(env_dict, ['lto', 'wallet', 'seed'], lto_data[0])
nested_set(env_dict, ['lto', 'wallet', 'password'], lto_data[1])
nested_set(env_dict, ['lto', 'rest-api', 'api-key-hash'], api_key_hash)
ENABLE_REST_API = os.environ.get('ENABLE_REST_API', os.environ.get('LTO_ENABLE_REST_API', 'false'))
if ENABLE_REST_API.lower() in ['yes', 'true', 't', '1', 'on']:
nested_set(env_dict, ['lto', 'rest-api', 'enable'], 'yes')
nested_set(env_dict, ['lto', 'rest-api', 'bind-address'], '0.0.0.0')
def test_validation_success(self):
job = WordCountSparkJob()
result = job.validate(
self.sc, None,
ConfigFactory.parse_string('input.strings = ["a", "a", "b"]'))
self.assertEqual(result, ['a', 'a', 'b'])
sys.exit(0)
datafile_path = Path(sys.argv[1])
mode = sys.argv[2] if len(sys.argv) >= 3 else 'board'
if not datafile_path.is_file():
print_err('Provided Data File not found')
sys.exit(1)
config_path = get_config_path()
width = None
palette = None
default_color_idx = 0
if config_path and not is_no(input('Use found pxls.conf? [Y/n]: ')):
from pyhocon import ConfigFactory as HoconConfigFactory
config = HoconConfigFactory.parse_file(config_path.absolute())
width = config.get('board.width')
palette = config.get('board.palette')
default_color_idx = config.get('board.defaultColor')
else:
while width is None:
try:
width = int(input('Board width: '))
except ValueError:
print('Width is not an integer')
import re
from json import loads
color_regex = re.compile(r'^#[a-f0-9]{6}$', re.IGNORECASE)
import logging
from datetime import datetime
from pyhocon import ConfigFactory # noqa: F401
from pyhocon import ConfigTree # noqa: F401
from typing import Any, Dict # noqa: F401
from databuilder.transformer.base_transformer import Transformer
TIMESTAMP_FORMAT = 'timestamp_format'
FIELD_NAME = 'field_name'
LOGGER = logging.getLogger(__name__)
DEFAULT_CONFIG = ConfigFactory.from_dict({TIMESTAMP_FORMAT: '%Y-%m-%dT%H:%M:%S.%fZ'})
class TimestampStringToEpoch(Transformer):
"""
Transforms string timestamp into epoch
"""
def init(self, conf):
# type: (ConfigTree) -> None
self._conf = conf.with_fallback(DEFAULT_CONFIG)
self._timestamp_format = self._conf.get_string(TIMESTAMP_FORMAT)
self._field_name = self._conf.get_string(FIELD_NAME)
def transform(self, record):
# type: (Dict[str, Any]) -> Dict[str, Any]
print('{tm} ------------------- {nm} started'.format(
tm=time.strftime("%Y-%m-%d %H:%M:%S"), nm=os.path.basename(__file__)))
module_path = os.path.realpath(__file__)
root_dir = dirname(dirname(module_path))
sys.path.append(path_join(root_dir, 'dstools'))
import spark.core as spark_utils
from spark.metrics import lift_splitted
parser = argparse.ArgumentParser()
parser.add_argument('--conf', required=True)
args, overrides = parser.parse_known_args()
file_conf = ConfigFactory.parse_file(args.conf, resolve=False)
overrides = ','.join(overrides)
over_conf = ConfigFactory.parse_string(overrides)
conf = over_conf.with_fallback(file_conf)
sqc = spark_utils.init_session(conf['spark'], app=os.path.basename(args.conf))
lift_cov = lift_splitted(sqc,
query=conf['source.query'],
target=conf['columns.target'],
proba=conf['columns.proba'],
split_by=conf['columns.split-by'],
cost=conf.get('columns.cost', None),
n_buckets=int(conf['n_buckets']))
lift_cov.to_csv(conf['report-path'], sep='\t')
def setUp(self):
logging.basicConfig(level=logging.INFO)
self._conf = ConfigFactory.from_dict({
'base_directory':
'./.test_artifacts',
})
def test_self_ref_substitution_dict_recurse(self):
with pytest.raises(ConfigSubstitutionException):
ConfigFactory.parse_string(
"""
x = ${x}
"""
)
