def test_not_including_empty_attribute(self) -> None:
test_user = User(email='*****@*****.**', foo='bar')
self.assertDictEqual(
neo4_serializer.serialize_node(test_user.create_next_node()), {
'KEY': '*****@*****.**',
'LABEL': 'User',
'email': '*****@*****.**',
'is_active:UNQUOTED': True,
'profile_url': '',
'first_name': '',
'last_name': '',
'full_name': '',
'github_username': '',
'team_name': '',
'employee_type': '',
'slack_id': '',
'role_name': '',
'updated_at:UNQUOTED': 0,
'foo': 'bar'
})
test_user2 = User(email='*****@*****.**',
foo='bar',
is_active=False,
do_not_update_empty_attribute=True)
self.assertDictEqual(
neo4_serializer.serialize_node(test_user2.create_next_node()), {
'KEY': '*****@*****.**',
'LABEL': 'User',
'email': '*****@*****.**',
'foo': 'bar'
})
def _create_node_iterator(self) -> Iterator[GraphNode]:
"""
Create an user node
:return:
"""
user_node = User(email=self.user_email).get_user_node()
yield user_node
def setUp(self) -> None:
self.maxDiff = None
super(TestQuery, self).setUp()
self.user = User(first_name='test_first',
last_name='test_last',
full_name='test_first test_last',
email='*****@*****.**',
github_username='******',
team_name='test_team',
employee_type='FTE',
manager_email='*****@*****.**',
slack_id='slack',
is_active=True,
profile_url='https://profile',
updated_at=1,
role_name='swe')
self.table_metadata = TableMetadata(
'hive', 'gold', 'test_schema1', 'test_table1', 'test_table1', [
ColumnMetadata('test_id1', 'description of test_table1',
'bigint', 0),
ColumnMetadata('test_id2', 'description of test_id2', 'bigint',
1),
ColumnMetadata('is_active', None, 'boolean', 2),
ColumnMetadata('source', 'description of source', 'varchar',
3),
ColumnMetadata('etl_created_at',
'description of etl_created_at', 'timestamp',
4),
ColumnMetadata('ds', None, 'varchar', 5)
])
self.sql = "select * from table"
self.query_metadata = QueryMetadata(sql=self.sql,
tables=[self.table_metadata],
user=self.user)
self._query_hash = 'da44ff72560e593a8eca9ffcee6a2696'
def test_parse_testdata(self) -> None:
bhr = BamboohrUserExtractor()
bhr.init(
ConfigFactory.from_dict({
'api_key': 'api_key',
'subdomain': 'amundsen'
}))
testdata_xml = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'../../../resources/extractor/user/bamboohr/testdata.xml')
with io.open(testdata_xml) as testdata:
responses.add(responses.GET,
bhr._employee_directory_uri(),
body=testdata.read())
expected = User(
email='*****@*****.**',
first_name='Roald',
last_name='Amundsen',
name='Roald Amundsen',
team_name='508 Corporate Marketing',
role_name='Antarctic Explorer',
)
actual_users = list(bhr._get_extract_iter())
self.assertEqual(1, len(actual_users))
self.assertEqual(repr(expected), repr(actual_users[0]))
def create_nodes(self) -> List[GraphNode]:
"""
Create a list of Neo4j node records
:return:
"""
return User(email=self.user_email).create_nodes()
def __init__(self,
dashboard_group_id: Optional[str],
dashboard_id: Optional[str],
email: str,
view_count: int,
should_create_user_node: Optional[bool] = False,
product: Optional[str] = '',
cluster: Optional[str] = 'gold',
**kwargs: Any) -> None:
"""
:param dashboard_group_id:
:param dashboard_id:
:param email:
:param view_count:
:param should_create_user_node: Enable this if it is fine to create/update User node with only with email
address. Please be advised that other fields will be emptied. Current use case is to create anonymous user.
For example, Mode dashboard does not provide which user viewed the dashboard and anonymous user can be used
to show the usage.
:param product:
:param cluster:
:param kwargs:
"""
self._dashboard_group_id = dashboard_group_id
self._dashboard_id = dashboard_id
self._email = email
self._view_count = int(view_count)
self._product = product
self._cluster = cluster
self._user_model = User(email=email)
self._should_create_user_node = bool(should_create_user_node)
self._relation_iterator = self._create_relation_iterator()
self._record_iterator = self._create_record_iterator()
def create_nodes(self):
# type: () -> List[Dict[str, Any]]
"""
Create a list of Neo4j node records
:return:
"""
return User(email=self.user_email).create_nodes()
def _create_record_iterator(self) -> Iterator[RDSModel]:
user_record = User(email=self.user_email).get_user_record()
yield user_record
table_usage_record = RDSTableUsage(
user_rk=self._get_user_key(self.user_email),
table_rk=self._get_table_key(),
read_count=self.read_count
)
yield table_usage_record
def _create_record_iterator(self) -> Iterator[RDSModel]:
for col_reader in self.col_readers:
if col_reader.column == '*':
user_record = User(
email=col_reader.user_email).get_user_record()
yield user_record
table_usage_record = RDSTableUsage(
user_rk=self._get_user_key(col_reader.user_email),
table_rk=self._get_table_key(col_reader),
read_count=col_reader.read_count)
yield table_usage_record
def setUp(self) -> None:
super(TestUser, self).setUp()
self.user = User(first_name='test_first',
last_name='test_last',
name='test_first test_last',
email='*****@*****.**',
github_username='******',
team_name='test_team',
employee_type='FTE',
manager_email='*****@*****.**',
slack_id='slack',
is_active=True,
updated_at=1,
role_name='swe')
def test_create_node_additional_attr(self):
test_user = User(first_name='test_first',
last_name='test_last',
name='test_first test_last',
email='*****@*****.**',
github_username='******',
team_name='test_team',
employee_type='FTE',
manager_email='*****@*****.**',
slack_id='slack',
is_active=True,
updated_at=1,
role_name='swe',
enable_notify=True)
nodes = test_user.create_nodes()
self.assertEqual(nodes[0]['email'], '*****@*****.**')
self.assertEqual(nodes[0]['role_name'], 'swe')
self.assertTrue(nodes[0]['enable_notify'])
def test_create_record_additional_attr_mysql(self) -> None:
test_user = User(first_name='test_first',
last_name='test_last',
name='test_first test_last',
email='*****@*****.**',
github_username='******',
team_name='test_team',
employee_type='FTE',
manager_email='*****@*****.**',
slack_id='slack',
is_active=True,
updated_at=1,
role_name='swe',
enable_notify=True)
record = test_user.create_next_record()
serialized_record = mysql_serializer.serialize_record(record)
self.assertEqual(serialized_record['email'], '*****@*****.**')
self.assertEqual(serialized_record['role_name'], 'swe')
def test_create_node_additional_attr(self) -> None:
test_user = User(first_name='test_first',
last_name='test_last',
full_name='test_first test_last',
email='*****@*****.**',
github_username='******',
team_name='test_team',
employee_type='FTE',
manager_email='*****@*****.**',
slack_id='slack',
is_active=True,
updated_at=1,
role_name='swe',
enable_notify=True)
node = test_user.create_next_node()
serialized_node = neo4_serializer.serialize_node(node)
self.assertEqual(serialized_node['email'], '*****@*****.**')
self.assertEqual(serialized_node['role_name'], 'swe')
self.assertTrue(serialized_node['enable_notify:UNQUOTED'])
def test_create_node_additional_attr_neptune(self) -> None:
test_user = User(first_name='test_first',
last_name='test_last',
name='test_first test_last',
email='*****@*****.**',
github_username='******',
team_name='test_team',
employee_type='FTE',
manager_email='*****@*****.**',
slack_id='slack',
is_active=True,
updated_at=1,
role_name='swe',
enable_notify=True)
nodes = test_user.create_nodes()
serialized_node = neptune_serializer.convert_node(nodes[0])
self.assertEqual(serialized_node['email:String(single)'], '*****@*****.**')
self.assertEqual(serialized_node['role_name:String(single)'], 'swe')
self.assertTrue(serialized_node['enable_notify:Bool(single)'])
def _get_extract_iter(self) -> Iterator[User]:
response = requests.get(self._employee_directory_uri(),
auth=HTTPBasicAuth(self._api_key, 'x'))
root = ElementTree.fromstring(response.content)
for user in root.findall('./employees/employee'):
def get_field(name: str) -> str:
field = user.find('./field[@id=\'{name}\']'.format(name=name))
if field is not None and field.text is not None:
return field.text
else:
return ''
yield User(
email=get_field('workEmail'),
first_name=get_field('firstName'),
last_name=get_field('lastName'),
name=get_field('displayName'),
team_name=get_field('department'),
role_name=get_field('jobTitle'),
)
def _create_node_iterator(self) -> Iterator[Any]:
for col_reader in self.col_readers:
if col_reader.column == '*':
# using yield for better memory efficiency
yield User(email=col_reader.user_email).create_nodes()[0]
def _create_node_iterator(self) -> Iterator[GraphNode]:
for col_reader in self.col_readers:
if col_reader.column == '*':
# using yield for better memory efficiency
user_node = User(email=col_reader.user_email).get_user_node()
yield user_node