def test_singleton():
handler_a = QueuesHandler()
handler_b = QueuesHandler()
assert id(handler_a) != id(handler_b)
assert id(handler_a._QueuesHandler__instance) == id(
handler_b._QueuesHandler__instance)
assert handler_a.default_queue_name == handler_b.default_queue_name
def test_parallel_2():
selected_columns_a = ["A", "B"]
selected_columns_b = ["C", "D"]
@synchronized(queue_name='TEST_3')
@managing(queue_name='TEST_3')
@adding(queue_items_creation_function=create_queue_item,
other_args={"selected_columns": selected_columns_a},
queue_name='TEST_3')
def parallel_add_row_a(dataframe: DataFrame, index: str,
columns_dict: dict) -> Tuple[str, Dict]:
return add_row(dataframe, index, columns_dict)
@synchronized(queue_name='TEST_4')
@managing(queue_name='TEST_4')
@adding(queue_items_creation_function=create_queue_item,
other_args={"selected_columns": selected_columns_b},
queue_name='TEST_4')
def parallel_add_row_b(dataframe: DataFrame, index: str,
columns_dict: dict) -> Tuple[str, Dict]:
return add_row(dataframe, index, columns_dict)
def thread_adding(operation_number: int, dataframe: DataFrame,
adding_function: Callable):
for _ in range(operation_number):
adding_function(
dataframe, str(uuid4()), {
'A': str(uuid4()),
'B': str(uuid4()),
'C': str(uuid4()),
'D': str(uuid4())
})
dataframe = DataFrame(columns=['A', 'B', 'C', 'D'])
assign_dataframe(dataframe, 1000, selected_columns_a, 'TEST_3')
assign_dataframe(dataframe, 500, selected_columns_b, 'TEST_4')
assert dataframe.empty
# noinspection PyProtectedMember
queue_handler_instance = QueuesHandler._QueuesHandler__instance
assert id(QueuesHandler._QueuesHandler__instance.get_assigned_lock('TEST_3')) != \
id(queue_handler_instance.get_assigned_lock(QueuesHandler().default_queue_name))
assert id(QueuesHandler._QueuesHandler__instance.get_assigned_lock('TEST_4')) != \
id(queue_handler_instance.get_assigned_lock(QueuesHandler().default_queue_name))
assert id(QueuesHandler._QueuesHandler__instance.get_assigned_lock('TEST_3')) == \
id(queue_handler_instance.get_assigned_lock('TEST_4'))
with ThreadPoolExecutor(max_workers=2) as executor:
future_a = executor.submit(thread_adding, 4000, dataframe,
parallel_add_row_a)
future_b = executor.submit(thread_adding, 4000, dataframe,
parallel_add_row_b)
future_a.result()
future_b.result()
# We can't predict if dataframe's size will be 500 or 1000
assert len(dataframe) in [500, 1000]
def test_assign_none(queue_name):
real_queue_name = QueuesHandler(
).default_queue_name if queue_name is None else queue_name
assign_dataframe(DataFrame(columns=['A', 'B', 'C', 'D']),
2, ['B'],
queue_name=real_queue_name)
assign_dataframe(None, 1, [], queue_name=real_queue_name)
queue_data = QueuesHandler()[real_queue_name]
assert queue_data[QueueHandlerItem.MAX_SIZE] == 1
assert queue_data[QueueHandlerItem.DATAFRAME] is None
assert queue_data[QueueHandlerItem.QUEUE] == deque()
def test_parallel_1(queue_name):
selected_columns = ["A", "C"]
queue_name = queue_name if queue_name is not None else QueuesHandler(
).default_queue_name
@synchronized(queue_name=queue_name)
@managing(queue_name=queue_name)
@adding(queue_items_creation_function=create_queue_item,
other_args={"selected_columns": selected_columns},
queue_name=queue_name)
def parallel_add_row(dataframe: DataFrame, index: str,
columns_dict: dict) -> Tuple[str, Dict]:
return add_row(dataframe, index, columns_dict)
@synchronized(queue_name=queue_name)
@adding(queue_items_creation_function=create_queue_item,
other_args={"selected_columns": selected_columns},
queue_name=queue_name)
def parallel_change_row_value(dataframe: DataFrame, index: str,
new_columns_dict: dict) -> Tuple[str, Dict]:
return change_row_value(dataframe, index, new_columns_dict)
def thread_adding(operation_number: int, dataframe: DataFrame):
for _ in range(operation_number):
parallel_add_row(
dataframe, str(uuid4()), {
'A': str(uuid4()),
'B': str(uuid4()),
'C': str(uuid4()),
'D': str(uuid4())
})
def thread_change(operation_number: int, dataframe: DataFrame):
for _ in range(operation_number):
parallel_change_row_value(
dataframe,
dataframe.index.values[randint(0,
len(dataframe) - 1)], {
'A': str(uuid4()),
'B': str(uuid4()),
'C': str(uuid4()),
'D': str(uuid4())
})
dataframe = DataFrame(columns=['A', 'B', 'C', 'D'])
assign_dataframe(dataframe, 1000, selected_columns, queue_name)
assert dataframe.empty
with ThreadPoolExecutor(max_workers=2) as executor:
future_a = executor.submit(thread_adding, 4000, dataframe)
future_b = executor.submit(thread_adding, 4000, dataframe)
future_c = executor.submit(thread_change, 1000, dataframe)
future_a.result()
future_b.result()
future_c.result()
assert len(dataframe) == 1000
def test_assign_dataframe(queue_name, dataframe, max_size, selected_columns):
real_queue_name = QueuesHandler(
).default_queue_name if queue_name is None else queue_name
assign_dataframe(dataframe,
max_size,
selected_columns,
queue_name=real_queue_name)
queue_data = QueuesHandler()[real_queue_name]
assert queue_data[QueueHandlerItem.MAX_SIZE] == max_size
assert id(queue_data[QueueHandlerItem.DATAFRAME]) == id(dataframe)
assert len(queue_data[QueueHandlerItem.QUEUE]) == len(dataframe)
for queue_item in queue_data[QueueHandlerItem.QUEUE]:
index = queue_item[0]
values = queue_item[1]
assert len(values) == len(selected_columns)
for column_name, column_value in values.items():
assert dataframe[column_name][index] == column_value
def test_valid_get_item():
handler = QueuesHandler()
default_queue_name = handler.default_queue_name
queue_data = handler[default_queue_name]
assert isinstance(queue_data, dict)
assert len(queue_data) == len(QueueHandlerItem)
assert all([item in queue_data for item in QueueHandlerItem])
assert isinstance(queue_data[QueueHandlerItem.QUEUE], deque)
assert queue_data[QueueHandlerItem.DATAFRAME] is None
assert isinstance(queue_data[QueueHandlerItem.MAX_SIZE], int)
def test_valid_set_item(queue_iterable, dataframe, max_size, counter,
behaviour):
handler = QueuesHandler()
default_queue_name = handler.default_queue_name
handler[default_queue_name] = {
QueueHandlerItem.QUEUE: queue_iterable,
QueueHandlerItem.COUNTER: counter,
QueueHandlerItem.DATAFRAME: dataframe,
QueueHandlerItem.MAX_SIZE: max_size,
QueueHandlerItem.BEHAVIOUR: behaviour
}
queue_data = handler[default_queue_name]
assert queue_data[QueueHandlerItem.QUEUE] == deque(queue_iterable)
assert queue_data[QueueHandlerItem.COUNTER] == counter
assert id(queue_data[QueueHandlerItem.DATAFRAME]) == id(dataframe)
assert queue_data[QueueHandlerItem.MAX_SIZE] == max_size
assert queue_data[QueueHandlerItem.BEHAVIOUR] == behaviour
def test_invalid_set_item():
handler = QueuesHandler()
default_queue_name = handler.default_queue_name
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.QUEUE: deque(),
QueueHandlerItem.DATAFRAME: DataFrame()
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.MAX_SIZE: 1,
QueueHandlerItem.DATAFRAME: DataFrame()
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.MAX_SIZE: 1,
QueueHandlerItem.QUEUE: deque()
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.QUEUE: deque(),
QueueHandlerItem.COUNTER: dict()
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.MAX_SIZE: 1,
QueueHandlerItem.COUNTER: dict()
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.DATAFRAME: DataFrame(),
QueueHandlerItem.COUNTER: dict()
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.QUEUE: deque(),
QueueHandlerItem.DATAFRAME: DataFrame(),
QueueHandlerItem.MAX_SIZE: None
}
with pytest.raises(TypeError):
handler[default_queue_name] = {
QueueHandlerItem.QUEUE: None,
QueueHandlerItem.COUNTER: dict(),
QueueHandlerItem.DATAFRAME: DataFrame(),
QueueHandlerItem.MAX_SIZE: 1,
QueueHandlerItem.BEHAVIOUR: QueueBehaviour.LAST_ITEM
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.QUEUE: deque(),
QueueHandlerItem.COUNTER: dict(),
QueueHandlerItem.DATAFRAME: "UNKNOWN",
QueueHandlerItem.MAX_SIZE: 1,
QueueHandlerItem.BEHAVIOUR: QueueBehaviour.LAST_ITEM
}
with pytest.raises(AssertionError):
handler[default_queue_name] = {
QueueHandlerItem.QUEUE: deque(),
QueueHandlerItem.COUNTER: dict(),
QueueHandlerItem.DATAFRAME: DataFrame(),
QueueHandlerItem.MAX_SIZE: 1,
QueueHandlerItem.BEHAVIOUR: "UNKNOWN"
}
def test_invalid_get_item():
handler = QueuesHandler()
invalid_queue_name = "UNKNOWN"
with pytest.raises(AssertionError):
handler[invalid_queue_name]
def test_sequential_1(queue_name, columns, selected_columns):
queue_name = queue_name if queue_name is not None else QueuesHandler(
).default_queue_name
@managing(queue_name=queue_name)
@adding(queue_items_creation_function=create_queue_item,
other_args={"selected_columns": selected_columns},
queue_name=queue_name)
def sequential_add_row(dataframe: DataFrame, index: str,
columns_dict: dict) -> Tuple[str, Dict]:
return add_row(dataframe, index, columns_dict)
@adding(queue_items_creation_function=create_queue_item,
other_args={"selected_columns": selected_columns},
queue_name=queue_name)
def sequential_change_row_value(
dataframe: DataFrame, index: str,
new_columns_dict: dict) -> Tuple[str, Dict]:
return change_row_value(dataframe, index, new_columns_dict)
dataframe = DataFrame(columns=columns)
assign_dataframe(dataframe, 2, selected_columns, queue_name)
assert dataframe.empty
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque()
columns_dict_row_1 = {column: str(uuid4()) for column in columns}
sequential_add_row(dataframe, "1", columns_dict_row_1)
assert len(dataframe) == 1
result_row_1 = ("1", {
selected_column: columns_dict_row_1[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_1])
columns_dict_row_2 = {column: str(uuid4()) for column in columns}
sequential_add_row(dataframe, "2", columns_dict_row_2)
assert len(dataframe) == 2
result_row_2 = ("2", {
selected_column: columns_dict_row_2[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_1, result_row_2])
columns_dict_row_3 = {column: str(uuid4()) for column in columns}
sequential_add_row(dataframe, "3", columns_dict_row_3)
assert len(dataframe) == 2
result_row_3 = ("3", {
selected_column: columns_dict_row_3[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_2, result_row_3])
remove_row(dataframe, "3")
assert len(dataframe) == 1
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_2, result_row_3])
remove_row(dataframe, "2")
assert dataframe.empty
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_2, result_row_3])
columns_dict_row_4 = {column: str(uuid4()) for column in columns}
sequential_add_row(dataframe, "4", columns_dict_row_4)
columns_dict_row_5 = {column: str(uuid4()) for column in columns}
sequential_add_row(dataframe, "5", columns_dict_row_5)
assert len(dataframe) == 2
result_row_4 = ("4", {
selected_column: columns_dict_row_4[selected_column]
for selected_column in selected_columns
})
result_row_5 = ("5", {
selected_column: columns_dict_row_5[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_2, result_row_3, result_row_4, result_row_5])
columns_dict_row_6 = {column: str(uuid4()) for column in columns}
sequential_add_row(dataframe, "6", columns_dict_row_6)
assert len(dataframe) == 2
result_row_6 = ("6", {
selected_column: columns_dict_row_6[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_5, result_row_6])
new_columns_dict_row_5 = {column: str(uuid4()) for column in columns}
sequential_change_row_value(dataframe, "5", new_columns_dict_row_5)
assert len(dataframe) == 2
new_result_row_5 = ("5", {
selected_column: new_columns_dict_row_5[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_5, result_row_6, new_result_row_5])
columns_dict_row_7 = {column: str(uuid4()) for column in columns}
sequential_add_row(dataframe, "7", columns_dict_row_7)
assert len(dataframe) == 2
result_row_7 = ("7", {
selected_column: columns_dict_row_7[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[new_result_row_5, result_row_7])
################################################################################################
for behaviour in [QueueBehaviour.ALL_ITEMS, QueueBehaviour.LAST_ITEM]:
values_row_1 = [str(uuid4()) for _ in range(len(columns))]
values_row_2 = [str(uuid4()) for _ in range(len(columns))]
dataframe_2 = DataFrame(data=[values_row_1, values_row_2],
index=['1', '2'],
columns=columns)
assign_dataframe(dataframe_2,
2,
selected_columns,
queue_name,
queue_behaviour=behaviour)
assert len(dataframe_2) == 2
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque([
('1', {
column: values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('2', {
column: values_row_2[list(columns).index(column)]
for column in selected_columns
})
])
new_values_row_1 = [str(uuid4()) for _ in range(len(columns))]
sequential_change_row_value(dataframe_2, '1', {
column: new_values_row_1[i]
for i, column in enumerate(list(columns))
})
assert len(dataframe_2) == 2
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque([
('1', {
column: values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('2', {
column: values_row_2[list(columns).index(column)]
for column in selected_columns
}),
('1', {
column: new_values_row_1[list(columns).index(column)]
for column in selected_columns
})
])
sequential_change_row_value(dataframe_2, '1', {
column: values_row_1[i]
for i, column in enumerate(list(columns))
})
assert len(dataframe_2) == 2
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque([
('1', {
column: values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('2', {
column: values_row_2[list(columns).index(column)]
for column in selected_columns
}),
('1', {
column: new_values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('1', {
column: values_row_1[list(columns).index(column)]
for column in selected_columns
})
])
values_row_3 = [str(uuid4()) for _ in range(len(columns))]
sequential_add_row(
dataframe_2, '3',
{column: values_row_3[i]
for i, column in enumerate(columns)})
assert len(dataframe_2) == 2
if behaviour == QueueBehaviour.ALL_ITEMS:
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[('2', {
column: values_row_2[list(columns).index(column)]
for column in selected_columns
}),
('1', {
column: new_values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('1', {
column: values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('3', {
column: values_row_3[list(columns).index(column)]
for column in selected_columns
})])
assert all(label in dataframe_2.index for label in ['2', '3'])
elif behaviour == QueueBehaviour.LAST_ITEM:
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[('1', {
column: new_values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('1', {
column: values_row_1[list(columns).index(column)]
for column in selected_columns
}),
('3', {
column: values_row_3[list(columns).index(column)]
for column in selected_columns
})])
assert all(label in dataframe_2.index for label in ['1', '3'])
else:
assert False, "Missing test for the behaviour {}".format(behaviour)
def test_sequential_2(queue_name, columns, selected_columns):
@managing(queue_name=queue_name)
@adding(queue_items_creation_function=create_queue_items,
other_args={"selected_columns": selected_columns},
queue_name=queue_name)
def sequential_add_rows(
dataframe: DataFrame, indexes: List[str],
columns_dicts: List[dict]) -> List[Tuple[str, Dict]]:
assert len(indexes) == len(columns_dicts)
result = list()
for index, columns_dict in zip(indexes, columns_dicts):
result.append(add_row(dataframe, index, columns_dict))
return result
dataframe = DataFrame(columns=columns)
assign_dataframe(dataframe, 5, selected_columns, queue_name)
assert dataframe.empty
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque()
columns_dict_row_1 = {column: str(uuid4()) for column in columns}
sequential_add_rows(dataframe, ["1"], [columns_dict_row_1])
assert len(dataframe) == 1
result_row_1 = ("1", {
selected_column: columns_dict_row_1[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_1])
columns_dict_row_2 = {column: str(uuid4()) for column in columns}
columns_dict_row_3 = {column: str(uuid4()) for column in columns}
columns_dict_row_4 = {column: str(uuid4()) for column in columns}
sequential_add_rows(
dataframe, ["2", "3", "4"],
[columns_dict_row_2, columns_dict_row_3, columns_dict_row_4])
assert len(dataframe) == 4
result_row_2 = ("2", {
selected_column: columns_dict_row_2[selected_column]
for selected_column in selected_columns
})
result_row_3 = ("3", {
selected_column: columns_dict_row_3[selected_column]
for selected_column in selected_columns
})
result_row_4 = ("4", {
selected_column: columns_dict_row_4[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_1, result_row_2, result_row_3, result_row_4])
columns_dict_row_5 = {column: str(uuid4()) for column in columns}
columns_dict_row_6 = {column: str(uuid4()) for column in columns}
columns_dict_row_7 = {column: str(uuid4()) for column in columns}
sequential_add_rows(
dataframe, ["5", "6", "7"],
[columns_dict_row_5, columns_dict_row_6, columns_dict_row_7])
assert len(dataframe) == 5
result_row_5 = ("5", {
selected_column: columns_dict_row_5[selected_column]
for selected_column in selected_columns
})
result_row_6 = ("6", {
selected_column: columns_dict_row_6[selected_column]
for selected_column in selected_columns
})
result_row_7 = ("7", {
selected_column: columns_dict_row_7[selected_column]
for selected_column in selected_columns
})
assert QueuesHandler()._QueuesHandler__queues[queue_name] == deque(
[result_row_3, result_row_4, result_row_5, result_row_6, result_row_7])