def test_set_transform_results(self, test_transform_results, dynamo_test_environment):
function_name = 'set_transform_results'
dynamo_driver = LeechDriver(table_name=blank_table_name)
test_source_vertex = test_transform_results[0]
test_potentials = test_transform_results[1]
test_id_value = test_source_vertex.id_value
test_identifier_stem = test_source_vertex.identifier_stem
test_internal_id = test_source_vertex.internal_id
dynamo_driver.set_transform_results(
test_source_vertex, test_potentials,
identifier_stem=test_identifier_stem,
id_value=test_id_value)
disposition = 'working'
if not test_potentials:
disposition = 'graphing'
test_args = (function_name, test_id_value, test_identifier_stem, dynamo_test_environment)
test_kwargs = {
'stage_name': 'transformation',
'disposition': disposition,
'internal_id': test_internal_id
}
self._assert_dynamo_call(*test_args, **test_kwargs)
attribute_values = dynamo_test_environment.call_args_list[0][0][1]['ExpressionAttributeValues']
self._assert_object_properties_creation(test_source_vertex.object_type, attribute_values[':v'])
self._assert_potentials_creation(attribute_values[':ps'])
def __init__(self, metal_order, **kwargs):
self._assimilate_order = metal_order
self._source_vertex = metal_order.source_vertex
self._potential_vertex = metal_order.potential_vertex
self._rule_entry = metal_order.rule_entry
self._extracted_data = metal_order.extracted_data
self._dynamo_driver = LeechDriver()
def __init__(self, metal_order, **kwargs):
self._extraction_order = metal_order
self._extraction_function_name = metal_order.extraction_function_name
self._extraction_properties = metal_order.extraction_properties
self._schema_entry = metal_order.schema_entry
self._dynamo_driver = LeechDriver()
self._transform_queue = kwargs.get(
'transform_queue', ForgeQueue.get_for_transform_queue(**kwargs))
def test_mark_object_as_graphed(self, test_id, dynamo_test_environment):
function_name = 'mark_object_as_graphed'
test_identifier_stem = test_id[0]
test_id_value = test_id[1]
dynamo_driver = LeechDriver(table_name=blank_table_name)
dynamo_driver.mark_object_as_graphed(identifier_stem=test_identifier_stem, id_value=test_id_value)
self._assert_dynamo_call(function_name, test_id_value, test_identifier_stem, dynamo_test_environment,
stage_name='graphing')
def __init__(self, metal_order, **kwargs):
self._transform_order = metal_order
self._assimilation_queue = kwargs.get(
'assimilate_queue',
ForgeQueue.get_for_assimilation_queue(**kwargs))
self._extracted_data = metal_order.extracted_data
self._schema_entry = metal_order.schema_entry
self._source_vertex_data = metal_order.extracted_data['source']
self._dynamo_driver = LeechDriver()
def __init__(self, **kwargs):
self._scanner = DynamoScanner(kwargs.get('index_name', 'stalled'))
self._driver = LeechDriver()
self._load_graph_orders = []
self._graph_counter = 0
self._load_queue_url = os.getenv(
'LOAD_URL',
'https://sqs.us-east-1.amazonaws.com/803040539655/load')
self._extract_queue_url = os.getenv(
'EXTRACT_URL',
'https://sqs.us-east-1.amazonaws.com/803040539655/extract')
self._load_queue = boto3.resource('sqs').Queue(self._load_queue_url)
self._extraction_queue = ForgeQueue.get_for_extraction_queue()
def test_set_stub_assimilated_vertex(self, stub_potential_vertex, dynamo_test_environment):
function_name = 'set_assimilated_vertex'
dynamo_driver = LeechDriver(table_name=blank_table_name)
results = dynamo_driver.set_assimilated_vertex(stub_potential_vertex, True)
internal_id = None
id_value = None
identifier_stem = IdentifierStem.for_stub(stub_potential_vertex)
if stub_potential_vertex.is_internal_id_set:
internal_id = stub_potential_vertex.internal_id
if stub_potential_vertex.is_id_value_set:
id_value = stub_potential_vertex.id_value
self._assert_dynamo_call(
function_name, id_value, identifier_stem, dynamo_test_environment, stage_name='assimilation',
internal_id=internal_id, id_value=id_value, object_type=stub_potential_vertex.object_type
)
def set_assimilation_result(self, test_assimilation_results, dynamo_test_environment, counter):
function_name = 'set_assimilation_results'
dynamo_driver = LeechDriver(table_name=blank_table_name)
test_assimilation_result = test_assimilation_results[2]
test_edge_type = test_assimilation_results[1]
test_source_vertex = test_assimilation_results[0]
test_identifier_stem = test_source_vertex.identifier_stem
test_id_value = test_source_vertex.id_value
dynamo_driver.set_assimilation_results(
test_edge_type, test_assimilation_result,
identifier_stem=test_identifier_stem,
id_value=test_id_value
)
self._assert_dynamo_call(function_name, test_id_value, test_identifier_stem, dynamo_test_environment,
stage_name='assimilation', edge_type=test_edge_type, counter=counter)
identified_vertexes = dynamo_test_environment.call_args[0][1]['ExpressionAttributeValues'][':iv']
self._assert_identified_vertexes_creation(identified_vertexes, test_assimilation_results)
def load(*args, **kwargs):
logging.info('starting a load task with args/kwargs: %s/%s' % (args, kwargs))
task_args = kwargs['task_args']
dynamo_driver = LeechDriver(**task_args)
key_fields = task_args['keys']
keys = {
'identifier_stem': key_fields['identifier_stem'],
'id_value': key_fields['sid_value']
}
potential_object = dynamo_driver.get_object(**keys)
ogm = Ogm(**task_args)
graph_results = ogm.graph_object(potential_object)
try:
dynamo_driver.mark_object_as_graphed(
identifier_stem=potential_object['source'].identifier_stem,
id_value=potential_object['source'].id_value)
except ClientError as e:
if e.response['Error']['Code'] != 'ConditionalCheckFailedException':
raise e
logging.warning(
'attempted to mark a vertex as graphing, '
'but it appears this step has already happened, no changes to be made' % potential_object)
return graph_results
def test_mark_ids_as_working(self, test_working_ids, dynamo_test_environment):
function_name = 'mark_ids_as_working'
dynamo_driver = LeechDriver(table_name=blank_table_name)
test_id_range = test_working_ids[1]
test_identifier_stem = test_working_ids[0]
results = dynamo_driver.mark_ids_as_working(test_id_range, identifier_stem=test_identifier_stem)
assert results == ([], list(test_id_range))
assert dynamo_test_environment.called is True
assert dynamo_test_environment.call_count == len(test_id_range)
for boto_call in dynamo_test_environment.call_args_list:
dynamo_commands = boto_call[0]
dynamo_args = dynamo_commands[0]
dynamo_kwargs = dynamo_commands[1]
assert dynamo_args == 'UpdateItem'
assert dynamo_kwargs['Key']['identifier_stem'] == str(test_identifier_stem)
assert int(dynamo_kwargs['Key']['sid_value']) in test_id_range
update_expression = dynamo_kwargs['UpdateExpression']
update_names = dynamo_kwargs['ExpressionAttributeNames']
update_values = dynamo_kwargs['ExpressionAttributeValues']
self._assert_update_expression_creation(function_name, update_expression)
self._assert_attribute_names_creation(function_name, update_names)
self._assert_attribute_values_creation(function_name, update_values, id_value_range=test_id_range,
object_type=test_identifier_stem.object_type,
stage_name='assimilation')