def test_marker(self) -> None:
with patch.object(GraphDatabase, 'driver'):
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo'
})
task.init(job_config)
self.assertIsNone(task.ms_to_expire)
self.assertEqual(task.marker, 'foo')
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.MS_TO_EXPIRE}': 86400000,
})
task.init(job_config)
self.assertIsNotNone(task.ms_to_expire)
self.assertEqual(task.marker, 86400000)
def test_ms_to_expire_too_small(self) -> None:
with patch.object(GraphDatabase, 'driver'):
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}': ['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}': ['BAR'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.MS_TO_EXPIRE}': 24 * 60 * 60 * 100 - 10
})
try:
task.init(job_config)
self.assertTrue(False, 'Should have failed with small TTL ')
except Exception:
pass
with patch.object(GraphDatabase, 'driver'):
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}': ['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}': ['BAR'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.MS_TO_EXPIRE}': 24 * 60 * 60 * 1000,
})
task.init(job_config)
def test_validation_receives_correct_counts(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}': ['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}': ['BAR'],
neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo',
})
task.init(job_config)
with patch.object(Neo4jStalenessRemovalTask, '_validate_staleness_pct') as mock_validate:
mock_execute.side_effect = [[{'count': 100}], [{'count': 50}]]
task._validate_node_staleness_pct()
mock_validate.assert_called_with(total_record_count=100,
stale_record_count=50,
target_type='Foo')
mock_execute.side_effect = [[{'count': 100}], [{'count': 50}]]
task._validate_relation_staleness_pct()
mock_validate.assert_called_with(total_record_count=100,
stale_record_count=50,
target_type='BAR')
def test_validation_failure(self) -> None:
with patch.object(GraphDatabase, 'driver'):
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
'job.identifier':
'remove_stale_data_job',
'{}.{}'.format(
task.get_scope(), neo4j_staleness_removal_task.NEO4J_END_POINT_KEY):
'foobar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_USER):
'foo',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_PASSWORD):
'bar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_MAX_PCT):
90,
neo4j_csv_publisher.JOB_PUBLISH_TAG:
'foo'
})
task.init(job_config)
total_records = [{'type': 'foo', 'count': 100}]
stale_records = [{'type': 'foo', 'count': 50}]
targets = {'foo'}
task._validate_staleness_pct(total_records=total_records,
stale_records=stale_records,
types=targets)
def test_validation_statement_ms_to_expire_retain_data_with_no_publisher_metadata(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}': ['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}': ['BAR'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.MS_TO_EXPIRE}': 9876543210,
f'{task.get_scope()}.{neo4j_staleness_removal_task.RETAIN_DATA_WITH_NO_PUBLISHER_METADATA}': True
})
task.init(job_config)
task._validate_node_staleness_pct()
mock_execute.assert_any_call(param_dict={'marker': 9876543210},
statement=textwrap.dedent("""
MATCH (target:Foo)
WHERE (target.publisher_last_updated_epoch_ms < (timestamp() - $marker))
RETURN count(*) as count
"""))
task._validate_relation_staleness_pct()
mock_execute.assert_any_call(param_dict={'marker': 9876543210},
statement=textwrap.dedent("""
MATCH (start_node)-[target:BAR]-(end_node)
WHERE (target.publisher_last_updated_epoch_ms < (timestamp() - $marker))
RETURN count(*) as count
"""))
def test_delete_dry_run(self) -> None:
with patch.object(GraphDatabase, 'driver') as mock_driver:
session_mock = mock_driver.return_value.session
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier':
'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}':
'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}':
'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}':
'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}':
5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}':
['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}':
['BAR'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.DRY_RUN}':
True,
neo4j_csv_publisher.JOB_PUBLISH_TAG:
'foo',
})
task.init(job_config)
task._delete_stale_nodes()
task._delete_stale_relations()
session_mock.assert_not_called()
def test_delete_statement_ms_to_expire(self):
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
mock_execute.return_value.single.return_value = {'count': 0}
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
'job.identifier':
'remove_stale_data_job',
'{}.{}'.format(
task.get_scope(), neo4j_staleness_removal_task.NEO4J_END_POINT_KEY):
'foobar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_USER):
'foo',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_PASSWORD):
'bar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_MAX_PCT):
5,
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.TARGET_NODES):
['Foo'],
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.TARGET_RELATIONS):
['BAR'],
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.MS_TO_EXPIRE):
9876543210
})
task.init(job_config)
task._delete_stale_nodes()
task._delete_stale_relations()
mock_execute.assert_any_call(dry_run=False,
param_dict={
'marker':
'(timestamp() - 9876543210)',
'batch_size': 100
},
statement=textwrap.dedent("""
MATCH (n:Foo)
WHERE
n.publisher_last_updated_epoch_ms < $marker
OR NOT EXISTS(n.publisher_last_updated_epoch_ms)
WITH n LIMIT $batch_size
DETACH DELETE (n)
RETURN COUNT(*) as count;
"""))
mock_execute.assert_any_call(dry_run=False,
param_dict={
'marker':
'(timestamp() - 9876543210)',
'batch_size': 100
},
statement=textwrap.dedent("""
MATCH ()-[n:BAR]-()
WHERE
n.publisher_last_updated_epoch_ms < $marker
OR NOT EXISTS(n.publisher_last_updated_epoch_ms)
WITH n LIMIT $batch_size
DELETE n
RETURN count(*) as count;
"""))
def test_validation(self) -> None:
with patch.object(GraphDatabase, 'driver'):
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier':
'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}':
'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}':
'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}':
'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}':
5,
neo4j_csv_publisher.JOB_PUBLISH_TAG:
'foo'
})
task.init(job_config)
total_records = [{'type': 'foo', 'count': 100}]
stale_records = [{'type': 'foo', 'count': 50}]
targets = {'foo'}
self.assertRaises(Exception, task._validate_staleness_pct,
total_records, stale_records, targets)
def test_delete_statement_publish_tag(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
mock_execute.return_value.single.return_value = {'count': 0}
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier':
'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}':
'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}':
'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}':
'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}':
5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}':
['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}':
['BAR'],
neo4j_csv_publisher.JOB_PUBLISH_TAG:
'foo',
})
task.init(job_config)
task._delete_stale_nodes()
task._delete_stale_relations()
mock_execute.assert_any_call(dry_run=False,
param_dict={
'marker': u'foo',
'batch_size': 100
},
statement=textwrap.dedent("""
MATCH (n:Foo)
WHERE{}
n.published_tag <> $marker
OR NOT EXISTS(n.published_tag)
WITH n LIMIT $batch_size
DETACH DELETE (n)
RETURN COUNT(*) as count;
""".format(' ')))
mock_execute.assert_any_call(dry_run=False,
param_dict={
'marker': u'foo',
'batch_size': 100
},
statement=textwrap.dedent("""
MATCH ()-[n:BAR]-()
WHERE{}
n.published_tag <> $marker
OR NOT EXISTS(n.published_tag)
WITH n LIMIT $batch_size
DELETE n
RETURN count(*) as count;
""".format(' ')))
def test_validation_statement_publish_tag(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
'job.identifier':
'remove_stale_data_job',
'{}.{}'.format(
task.get_scope(), neo4j_staleness_removal_task.NEO4J_END_POINT_KEY):
'foobar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_USER):
'foo',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_PASSWORD):
'bar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_MAX_PCT):
5,
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.TARGET_NODES):
['Foo'],
neo4j_csv_publisher.JOB_PUBLISH_TAG:
'foo',
})
task.init(job_config)
task._validate_node_staleness_pct()
mock_execute.assert_called()
mock_execute.assert_any_call(statement=textwrap.dedent("""
MATCH (n)
WITH DISTINCT labels(n) as node, count(*) as count
RETURN head(node) as type, count
"""))
mock_execute.assert_any_call(param_dict={'marker': u'foo'},
statement=textwrap.dedent("""
MATCH (n)
WHERE{}
n.published_tag <> $marker
OR NOT EXISTS(n.published_tag)
WITH DISTINCT labels(n) as node, count(*) as count
RETURN head(node) as type, count
""".format(' ')))
task._validate_relation_staleness_pct()
mock_execute.assert_any_call(param_dict={'marker': u'foo'},
statement=textwrap.dedent("""
MATCH ()-[n]-()
WHERE{}
n.published_tag <> $marker
OR NOT EXISTS(n.published_tag)
RETURN type(n) as type, count(*) as count
""".format(' ')))
def test_validation_statement_ms_to_expire(self):
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
'job.identifier':
'remove_stale_data_job',
'{}.{}'.format(
task.get_scope(), neo4j_staleness_removal_task.NEO4J_END_POINT_KEY):
'foobar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_USER):
'foo',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.NEO4J_PASSWORD):
'bar',
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.STALENESS_MAX_PCT):
5,
'{}.{}'.format(task.get_scope(), neo4j_staleness_removal_task.MS_TO_EXPIRE):
9876543210
})
task.init(job_config)
task._validate_node_staleness_pct()
mock_execute.assert_called()
mock_execute.assert_any_call(statement=textwrap.dedent("""
MATCH (n)
WITH DISTINCT labels(n) as node, count(*) as count
RETURN head(node) as type, count
"""))
mock_execute.assert_any_call(
param_dict={'marker': '(timestamp() - 9876543210)'},
statement=textwrap.dedent("""
MATCH (n)
WHERE
n.publisher_last_updated_epoch_ms < $marker
OR NOT EXISTS(n.publisher_last_updated_epoch_ms)
WITH DISTINCT labels(n) as node, count(*) as count
RETURN head(node) as type, count
"""))
task._validate_relation_staleness_pct()
mock_execute.assert_any_call(
param_dict={'marker': '(timestamp() - 9876543210)'},
statement=textwrap.dedent("""
MATCH ()-[n]-()
WHERE
n.publisher_last_updated_epoch_ms < $marker
OR NOT EXISTS(n.publisher_last_updated_epoch_ms)
RETURN type(n) as type, count(*) as count
"""))
def remove_stale_data():
task = Neo4jStalenessRemovalTask()
job_config_dict = {
'job.identifier': 'remove_stale_data_job',
'task.remove_stale_data.neo4j_endpoint': neo4j_endpoint,
'task.remove_stale_data.neo4j_user': neo4j_user,
'task.remove_stale_data.neo4j_password': neo4j_password,
'task.remove_stale_data.staleness_max_pct': 110,
'task.remove_stale_data.target_nodes': ['Table', 'Column'],
'task.remove_stale_data.job_publish_tag': 'idk'
}
job_config = ConfigFactory.from_dict(job_config_dict)
job = DefaultJob(conf=job_config, task=task)
return job
def test_validation_threshold_override(self):
# type: () -> None
with patch.object(GraphDatabase, 'driver'):
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
'job.identifier':
'remove_stale_data_job',
neo4j_staleness_removal_task.NEO4J_END_POINT_KEY:
'foobar',
neo4j_staleness_removal_task.NEO4J_USER:
'******',
neo4j_staleness_removal_task.NEO4J_PASSWORD:
'******',
neo4j_staleness_removal_task.STALENESS_MAX_PCT:
5,
neo4j_staleness_removal_task.STALENESS_PCT_MAX_DICT: {
'foo': 51
},
neo4j_csv_publisher.JOB_PUBLISH_TAG:
'foo'
})
task.init(job_config)
total_records = [{
'type': 'foo',
'count': 100
}, {
'type': 'bar',
'count': 100
}]
stale_records = [{
'type': 'foo',
'count': 50
}, {
'type': 'bar',
'count': 3
}]
targets = {'foo', 'bar'}
task._validate_staleness_pct(total_records=total_records,
stale_records=stale_records,
types=targets)
def test_validation_failure(self) -> None:
with patch.object(GraphDatabase, 'driver'):
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 90,
neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo'
})
task.init(job_config)
total_record_count = 100
stale_record_count = 50
target_type = 'foo'
task._validate_staleness_pct(total_record_count=total_record_count,
stale_record_count=stale_record_count,
target_type=target_type)
def test_delete_statement_ms_to_expire(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
mock_execute.return_value.single.return_value = {'count': 0}
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}': ['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}': ['BAR'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.MS_TO_EXPIRE}': 9876543210
})
task.init(job_config)
task._delete_stale_nodes()
task._delete_stale_relations()
mock_execute.assert_any_call(dry_run=False,
param_dict={'marker': 9876543210, 'batch_size': 100},
statement=textwrap.dedent("""
MATCH (target:Foo)
WHERE (target.publisher_last_updated_epoch_ms < (timestamp() - $marker)
OR NOT EXISTS(target.publisher_last_updated_epoch_ms))
WITH target LIMIT $batch_size
DETACH DELETE (target)
RETURN count(*) as count
"""))
mock_execute.assert_any_call(dry_run=False,
param_dict={'marker': 9876543210, 'batch_size': 100},
statement=textwrap.dedent("""
MATCH (start_node)-[target:BAR]-(end_node)
WHERE (target.publisher_last_updated_epoch_ms < (timestamp() - $marker)
OR NOT EXISTS(target.publisher_last_updated_epoch_ms))
WITH target LIMIT $batch_size
DELETE target
RETURN count(*) as count
"""))
def test_validation_statement_publish_tag(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}': ['Foo'],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}': ['BAR'],
neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo',
})
task.init(job_config)
task._validate_node_staleness_pct()
mock_execute.assert_called()
mock_execute.assert_any_call(statement=textwrap.dedent("""
MATCH (target:Foo)
WHERE true
RETURN count(*) as count
"""))
mock_execute.assert_any_call(param_dict={'marker': u'foo'},
statement=textwrap.dedent("""
MATCH (target:Foo)
WHERE (target.published_tag < $marker
OR NOT EXISTS(target.published_tag))
RETURN count(*) as count
"""))
task._validate_relation_staleness_pct()
mock_execute.assert_any_call(param_dict={'marker': u'foo'},
statement=textwrap.dedent("""
MATCH (start_node)-[target:BAR]-(end_node)
WHERE (target.published_tag < $marker
OR NOT EXISTS(target.published_tag))
RETURN count(*) as count
"""))
def test_delete_statement_with_target_condition(self) -> None:
with patch.object(GraphDatabase, 'driver'), patch.object(Neo4jStalenessRemovalTask, '_execute_cypher_query') \
as mock_execute:
mock_execute.return_value.single.return_value = {'count': 0}
task = Neo4jStalenessRemovalTask()
job_config = ConfigFactory.from_dict({
f'job.identifier': 'remove_stale_data_job',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_END_POINT_KEY}': 'foobar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_USER}': 'foo',
f'{task.get_scope()}.{neo4j_staleness_removal_task.NEO4J_PASSWORD}': 'bar',
f'{task.get_scope()}.{neo4j_staleness_removal_task.STALENESS_MAX_PCT}': 5,
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_NODES}': [TargetWithCondition('Foo', '(target)-[:BAR]->(:Foo) AND target.name=\'foo_name\'')],
f'{task.get_scope()}.{neo4j_staleness_removal_task.TARGET_RELATIONS}': [TargetWithCondition('BAR', '(start_node:Foo)-[target]->(end_node:Foo)')],
neo4j_csv_publisher.JOB_PUBLISH_TAG: 'foo',
})
task.init(job_config)
task._delete_stale_nodes()
task._delete_stale_relations()
mock_execute.assert_any_call(dry_run=False,
param_dict={'marker': u'foo', 'batch_size': 100},
statement=textwrap.dedent("""
MATCH (target:Foo)
WHERE (target.published_tag < $marker
OR NOT EXISTS(target.published_tag)) AND (target)-[:BAR]->(:Foo) AND target.name=\'foo_name\'
WITH target LIMIT $batch_size
DETACH DELETE (target)
RETURN count(*) as count
"""))
mock_execute.assert_any_call(dry_run=False,
param_dict={'marker': u'foo', 'batch_size': 100},
statement=textwrap.dedent("""
MATCH (start_node)-[target:BAR]-(end_node)
WHERE (target.published_tag < $marker
OR NOT EXISTS(target.published_tag)) AND (start_node:Foo)-[target]->(end_node:Foo)
WITH target LIMIT $batch_size
DELETE target
RETURN count(*) as count
"""))