def test_put_depmap_entity_after_all_ok(use_testdb): pd = PonyDump(testdb) pd.put_depmap_entity_after('many1', after_entity=None) assert pd.depmap == [ ('many2', ({}, {}, { 'many1': 'many1' })), ('pdtest1', ({}, { 'test2': 'pdtest2', 'test4': 'pdtest4' }, { 'test3': 'pdtest3' })), ('pdtest3', ({}, { 'test1': 'pdtest1' }, {})), ('pdtest4', ({}, { 'test1': 'pdtest1' }, {})), ('pdtest2', ({ 'test1': 'pdtest1' }, {}, {})), ('many1', ({}, {}, { 'many2': 'many2' })), ]
def test_json2obj_fail_invalid_entity(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: pd.json2obj({'_entity': 'wtfisthat', 'id': 1}) assert str(excinfo.value) == 'Unknown entity "wtfisthat"'
def test_dump_to_directory_one_entity(use_testdb, dumpdir): pd = PonyDump(testdb) _fill_testdb() dump_statuses = [ {'current': 1, 'count': 2, 'pk': (1,), 'entity': 'many1', 'chunk_size': 250}, {'current': 2, 'count': 2, 'pk': (2,), 'entity': 'many1', 'chunk_size': 250}, {'current': 2, 'count': 2, 'pk': None, 'entity': 'many1', 'chunk_size': 250}, {'entity': None, 'path': None, 'current': 0, 'count': 0, 'pk': None}, # Всегда последний ] filenames = ['many1_dump.jsonl'] for status in pd.dump_to_directory(dumpdir, entities=['many1']): path = status.pop('path') if path is not None: assert os.path.split(path)[1] in filenames assert os.path.isfile(path) else: status['path'] = path # Для проверки последнего элемента assert status in dump_statuses dump_statuses.remove(status) if path is None: assert not dump_statuses assert not dump_statuses
def test_put_depmap_entity_after_required_fail(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: pd.put_depmap_entity_after('pdtest1', after_entity='pdtest2') assert str( excinfo.value) == 'Required dependencies does not allow this moving' assert pd.depmap == [ ('many1', ({}, {}, { 'many2': 'many2' })), ('many2', ({}, {}, { 'many1': 'many1' })), ('pdtest1', ({}, { 'test2': 'pdtest2', 'test4': 'pdtest4' }, { 'test3': 'pdtest3' })), ('pdtest3', ({}, { 'test1': 'pdtest1' }, {})), ('pdtest4', ({}, { 'test1': 'pdtest1' }, {})), ('pdtest2', ({ 'test1': 'pdtest1' }, {}, {})), ]
def test_load_entities_fail_invalid_json_syntax(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: for _ in pd.load_entities(io.StringIO('}')): pass assert str(excinfo.value).startswith('Invalid JSON on line 1: ')
def test_load_entities_fail_invalid_json_unknown_entity(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: for _ in pd.load_entities(io.StringIO('{"_entity": "wat"}')): pass assert str(excinfo.value) == 'Unknown entity "wat" on line 1'
def test_load_entities_fail_invalid_json_nonstr_entity(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: for _ in pd.load_entities(io.StringIO('{"_entity": 7}')): pass assert str(excinfo.value) =='Invalid dump format on line 1'
def test_load_entities_fail_invalid_json_unknown_entity(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: for _ in pd.load_entities(io.StringIO('{"_entity": "wat"}')): pass assert str(excinfo.value) =='Unknown entity "wat" on line 1'
def test_load_entities_fail_invalid_json_nonstr_entity(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: for _ in pd.load_entities(io.StringIO('{"_entity": 7}')): pass assert str(excinfo.value) == 'Invalid dump format on line 1'
def test_json2obj_fail_unknown_attr(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: # test4 ссылается не на тот PDTest4, который прописан в дампе самого PDTest4 pd.json2obj({'_entity': 'pdtest4', 'id': 1, 'wtfisthat': 'that', 'andthat': 'that'}) x = 'Unknown attributes in dump of model "pdtest4": ' assert str(excinfo.value) in (x + 'wtfisthat, andthat', x + 'andthat, wtfisthat')
def test_json2obj_fail_inconsistent_optional_value(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'pdtest4', 'id': 1, 'test1': (1, 2, 3)}) with pytest.raises(ValueError) as excinfo: # test4 ссылается не на тот PDTest4, который прописан в дампе самого PDTest4 pd.json2obj({'_entity': 'pdtest1', 'k1': 1, 'k2': 2, 'k3': 3, 'test4': 7}) assert str(excinfo.value) == 'Inconsistent dump: attribute "test4" of "pdtest1(1, 2, 3)" conflicts with reverse data'
def test_json2obj_fail_inconsistent_set(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'many1', 'id': 1, 'many2': [2]}) with pytest.raises(ValueError) as excinfo: # many1 не должен быть пуст, так как есть связь pd.json2obj({'_entity': 'many2', 'id': 2, 'many1': []}) assert str(excinfo.value) == 'Inconsistent dump: attribute "many1" of "many2(2,)" conflicts with reverse data'
def test_json2obj_fail_inconsistent_optional_count_2(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'pdtest4', 'id': 1, 'test1': (1, 2, 3)}) pd.json2obj({'_entity': 'pdtest4', 'id': 2, 'test1': [1, 2, 3]}) with pytest.raises(ValueError) as excinfo: # То же самое, что и в предыдущем тесте, но тут какой-нибудь test4 прописан pd.json2obj({'_entity': 'pdtest1', 'k1': 1, 'k2': 2, 'k3': 3, 'test4': 1}) assert str(excinfo.value) == 'Inconsistent dump: attribute "test4" of "pdtest1(1, 2, 3)" has multiple values in reverse data'
def test_json2obj_fail_inconsistent_optional_count(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'pdtest4', 'id': 1, 'test1': (1, 2, 3)}) pd.json2obj({'_entity': 'pdtest4', 'id': 2, 'test1': [1, 2, 3]}) with pytest.raises(ValueError) as excinfo: # test4 — Optional, а на один и тот же PDTest1 один ссылаются аж два PDTest2, так не бывает pd.json2obj({'_entity': 'pdtest1', 'k1': 1, 'k2': 2, 'k3': 3}) assert str(excinfo.value) == 'Inconsistent dump: attribute "test4" of "pdtest1(1, 2, 3)" has multiple values in reverse data'
def test_put_depmap_entity_after_all_ok(use_testdb): pd = PonyDump(testdb) pd.put_depmap_entity_after('many1', after_entity=None) assert pd.depmap == [ ('many2', ({}, {}, {'many1': 'many1'})), ('pdtest1', ({}, {'test2': 'pdtest2', 'test4': 'pdtest4'}, {'test3': 'pdtest3'})), ('pdtest3', ({}, {'test1': 'pdtest1'}, {})), ('pdtest4', ({}, {'test1': 'pdtest1'}, {})), ('pdtest2', ({'test1': 'pdtest1'}, {}, {})), ('many1', ({}, {}, {'many2': 'many2'})), ]
def test_dump_to_directory_one_entity(use_testdb, dumpdir): pd = PonyDump(testdb) _fill_testdb() dump_statuses = [ { 'current': 1, 'count': 2, 'pk': (1, ), 'entity': 'many1', 'chunk_size': 250 }, { 'current': 2, 'count': 2, 'pk': (2, ), 'entity': 'many1', 'chunk_size': 250 }, { 'current': 2, 'count': 2, 'pk': None, 'entity': 'many1', 'chunk_size': 250 }, { 'entity': None, 'path': None, 'current': 0, 'count': 0, 'pk': None }, # Всегда последний ] filenames = ['many1_dump.jsonl'] for status in pd.dump_to_directory(dumpdir, entities=['many1']): path = status.pop('path') if path is not None: assert os.path.split(path)[1] in filenames assert os.path.isfile(path) else: status['path'] = path # Для проверки последнего элемента assert status in dump_statuses dump_statuses.remove(status) if path is None: assert not dump_statuses assert not dump_statuses
def test_load_from_files_partial(use_testdb, dumpdir): os.makedirs(dumpdir) with open(os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test3": [17]}\n') with gzip.open(os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'wt', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest2", "id": 1, "test1": [1, 2, 3]}\n') with open(os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest3", "foo_bool": true, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 17}\n') with open(os.path.join(dumpdir, 'many1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many1", "id": 1, "many2": [1, 2]}\n') fp.write('{"_entity": "many1", "id": 2}\n') with open(os.path.join(dumpdir, 'many2_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many2", "id": 1, "many1": [1, 2]}\n') fp.write('{"_entity": "many2", "id": 2, "many1": [1, 2]}\n') pd = PonyDump(testdb) for _ in pd.load_from_files([dumpdir]): pass assert not pd.get_depcache_dict() assert PDTest1.select().count() == 1 test1 = PDTest1.select().first() assert test1.get_pk() == (1, 2, 3) assert test1.test2.id == 1 assert test1.test3.select().count() == 1 assert test1.test3.select().first().id == 17 assert PDTest2.select().count() == 1 test2 = PDTest2.select().first() assert test2.get_pk() == 1 assert test2.test1 is test1 # Pony ORM кэширует всё подряд, поэтому можно is assert PDTest3.select().count() == 1 test3 = PDTest3.select().first() assert test3.get_pk() == 17 assert test3.test1 is test1 many1_1, many1_2 = Many1.select().order_by(Many1.id)[:] many2_1, many2_2 = Many2.select().order_by(Many2.id)[:] assert many1_1.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many1_2.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many2_1.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2] assert many2_2.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2]
def test_dump_to_directory_exclude_attrs(use_testdb, dumpdir): pd = PonyDump(testdb, dict_params={ 'many1': {'exclude': 'many2'}, }) _fill_testdb() for _ in pd.dump_to_directory(dumpdir): pass _check_dump_content(dumpdir, 'many1_dump.jsonl', [ '{"_entity": "many1", "id": 1}\n', '{"_entity": "many1", "id": 2}\n', ])
def test_dump_to_directory_exclude_attrs(use_testdb, dumpdir): pd = PonyDump(testdb, dict_params={ 'many1': { 'exclude': 'many2' }, }) _fill_testdb() for _ in pd.dump_to_directory(dumpdir): pass _check_dump_content(dumpdir, 'many1_dump.jsonl', [ '{"_entity": "many1", "id": 1}\n', '{"_entity": "many1", "id": 2}\n', ])
def test_json2obj_fail_unknown_attr(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: # test4 ссылается не на тот PDTest4, который прописан в дампе самого PDTest4 pd.json2obj({ '_entity': 'pdtest4', 'id': 1, 'wtfisthat': 'that', 'andthat': 'that' }) x = 'Unknown attributes in dump of model "pdtest4": ' assert str(excinfo.value) in (x + 'wtfisthat, andthat', x + 'andthat, wtfisthat')
def test_put_depmap_entity_after_required_fail(use_testdb): pd = PonyDump(testdb) with pytest.raises(ValueError) as excinfo: pd.put_depmap_entity_after('pdtest1', after_entity='pdtest2') assert str(excinfo.value) == 'Required dependencies does not allow this moving' assert pd.depmap == [ ('many1', ({}, {}, {'many2': 'many2'})), ('many2', ({}, {}, {'many1': 'many1'})), ('pdtest1', ({}, {'test2': 'pdtest2', 'test4': 'pdtest4'}, {'test3': 'pdtest3'})), ('pdtest3', ({}, {'test1': 'pdtest1'}, {})), ('pdtest4', ({}, {'test1': 'pdtest1'}, {})), ('pdtest2', ({'test1': 'pdtest1'}, {}, {})), ]
def test_depmap_ok(use_testdb): pd = PonyDump(testdb) # Строго говоря, порядок не абсолютно определён, и некоторые не-required # зависимости позволяют менять модели местами, но у обязательных # зависимостей строгий порядок: нельзя зависеть от моделей ниже по списку assert pd.depmap == [ # (entity_name, ({required}, {optional}, {set})) ('many1', ({}, {}, { 'many2': 'many2' })), ('many2', ({}, {}, { 'many1': 'many1' })), ('pdtest1', ({}, { 'test2': 'pdtest2', 'test4': 'pdtest4' }, { 'test3': 'pdtest3' })), ('pdtest3', ({}, { 'test1': 'pdtest1' }, {})), ('pdtest4', ({}, { 'test1': 'pdtest1' }, {})), ('pdtest2', ({ 'test1': 'pdtest1' }, {}, {})), ]
def test_load_from_files_notfull(use_testdb, dumpdir): os.makedirs(dumpdir) with open(os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write( '{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test2": 12, "test3": [17, 19]}\n' ) pd = PonyDump(testdb) for _ in pd.load_from_files([dumpdir]): pass depcache = pd.get_depcache_dict() assert depcache assert PDTest1.select().count() == 1 test1 = PDTest1.select().first() assert test1.get_pk() == (1, 2, 3) assert not test1.test2 assert test1.test3.select().count() == 0 assert PDTest2.select().count() == 0 assert PDTest3.select().count() == 0 key = (('pdtest1', 'test2'), ('pdtest2', 'test1')) assert key in depcache assert depcache[key] == [ { (1, 2, 3): {(12, )}, }, { (12, ): {(1, 2, 3)}, }, ] key = (('pdtest1', 'test3'), ('pdtest3', 'test1')) assert key in depcache assert depcache[key] == [{ (1, 2, 3): {(19, ), (17, )} }, { (19, ): {(1, 2, 3)}, (17, ): {(1, 2, 3)}, }]
def test_json2obj_ok_datetime_obj(use_testdb): pd = PonyDump(testdb) result = pd.json2obj({ "_entity": "pdtest3", "foo_bool": True, "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 1, "test1": [1, 2, 3], "foo_datetime": datetime(1970, 1, 1, 2, 0, 3, 999987), }) assert result['obj'].foo_datetime == datetime(1970, 1, 1, 2, 0, 3, 999987)
def test_load_from_files_notfull(use_testdb, dumpdir): os.makedirs(dumpdir) with open(os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test2": 12, "test3": [17, 19]}\n') pd = PonyDump(testdb) for _ in pd.load_from_files([dumpdir]): pass depcache = pd.get_depcache_dict() assert depcache assert PDTest1.select().count() == 1 test1 = PDTest1.select().first() assert test1.get_pk() == (1, 2, 3) assert not test1.test2 assert test1.test3.select().count() == 0 assert PDTest2.select().count() == 0 assert PDTest3.select().count() == 0 key = (('pdtest1', 'test2'), ('pdtest2', 'test1')) assert key in depcache assert depcache[key] == [ { (1, 2, 3): {(12,)}, }, { (12,): {(1, 2, 3)}, }, ] key = (('pdtest1', 'test3'), ('pdtest3', 'test1')) assert key in depcache assert depcache[key] == [ { (1, 2, 3): {(19,), (17,)} }, { (19,): {(1, 2, 3)}, (17,): {(1, 2, 3)}, } ]
def test_load_from_files_relations_all_orders(use_testdb, dumpdir, lines): os.makedirs(dumpdir) with open(os.path.join(dumpdir, 'some_dump.jsonl'), 'w', encoding='utf-8') as fp: for line in lines: fp.write(line + '\n') pd = PonyDump(testdb) for _ in pd.load_from_files([dumpdir]): pass assert not pd.get_depcache_dict() many1 = Many1.select().first() many2 = Many2.select().first() assert many1.many2.select()[:] == [many2] assert many2.many1.select()[:] == [many1]
def test_json2obj_ok_uuid_obj(use_testdb): pd = PonyDump(testdb) u = uuid.uuid4() result = pd.json2obj({ "_entity": "pdtest3", "foo_bool": True, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "id": 1, "test1": [1, 2, 3], "foo_uuid": u, }) assert result['obj'].foo_uuid == u
def test_json2obj_fail_inconsistent_set(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'many1', 'id': 1, 'many2': [2]}) with pytest.raises(ValueError) as excinfo: # many1 не должен быть пуст, так как есть связь pd.json2obj({'_entity': 'many2', 'id': 2, 'many1': []}) assert str( excinfo.value ) == 'Inconsistent dump: attribute "many1" of "many2(2,)" conflicts with reverse data'
def test_json2obj_fail_inconsistent_optional_count(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'pdtest4', 'id': 1, 'test1': (1, 2, 3)}) pd.json2obj({'_entity': 'pdtest4', 'id': 2, 'test1': [1, 2, 3]}) with pytest.raises(ValueError) as excinfo: # test4 — Optional, а на один и тот же PDTest1 один ссылаются аж два PDTest2, так не бывает pd.json2obj({'_entity': 'pdtest1', 'k1': 1, 'k2': 2, 'k3': 3}) assert str( excinfo.value ) == 'Inconsistent dump: attribute "test4" of "pdtest1(1, 2, 3)" has multiple values in reverse data'
def test_json2obj_fail_inconsistent_optional_value(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'pdtest4', 'id': 1, 'test1': (1, 2, 3)}) with pytest.raises(ValueError) as excinfo: # test4 ссылается не на тот PDTest4, который прописан в дампе самого PDTest4 pd.json2obj({ '_entity': 'pdtest1', 'k1': 1, 'k2': 2, 'k3': 3, 'test4': 7 }) assert str( excinfo.value ) == 'Inconsistent dump: attribute "test4" of "pdtest1(1, 2, 3)" conflicts with reverse data'
def test_json2obj_fail_inconsistent_optional_count_2(use_testdb): pd = PonyDump(testdb) pd.json2obj({'_entity': 'pdtest4', 'id': 1, 'test1': (1, 2, 3)}) pd.json2obj({'_entity': 'pdtest4', 'id': 2, 'test1': [1, 2, 3]}) with pytest.raises(ValueError) as excinfo: # То же самое, что и в предыдущем тесте, но тут какой-нибудь test4 прописан pd.json2obj({ '_entity': 'pdtest1', 'k1': 1, 'k2': 2, 'k3': 3, 'test4': 1 }) assert str( excinfo.value ) == 'Inconsistent dump: attribute "test4" of "pdtest1(1, 2, 3)" has multiple values in reverse data'
def test_load_from_files_partial(use_testdb, dumpdir): os.makedirs(dumpdir) with open(os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write( '{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test3": [17]}\n' ) with gzip.open(os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'wt', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest2", "id": 1, "test1": [1, 2, 3]}\n') with open(os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write( '{"_entity": "pdtest3", "foo_bool": true, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 17}\n' ) with open(os.path.join(dumpdir, 'many1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many1", "id": 1, "many2": [1, 2]}\n') fp.write('{"_entity": "many1", "id": 2}\n') with open(os.path.join(dumpdir, 'many2_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many2", "id": 1, "many1": [1, 2]}\n') fp.write('{"_entity": "many2", "id": 2, "many1": [1, 2]}\n') pd = PonyDump(testdb) for _ in pd.load_from_files([dumpdir]): pass assert not pd.get_depcache_dict() assert PDTest1.select().count() == 1 test1 = PDTest1.select().first() assert test1.get_pk() == (1, 2, 3) assert test1.test2.id == 1 assert test1.test3.select().count() == 1 assert test1.test3.select().first().id == 17 assert PDTest2.select().count() == 1 test2 = PDTest2.select().first() assert test2.get_pk() == 1 assert test2.test1 is test1 # Pony ORM кэширует всё подряд, поэтому можно is assert PDTest3.select().count() == 1 test3 = PDTest3.select().first() assert test3.get_pk() == 17 assert test3.test1 is test1 many1_1, many1_2 = Many1.select().order_by(Many1.id)[:] many2_1, many2_2 = Many2.select().order_by(Many2.id)[:] assert many1_1.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many1_2.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many2_1.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2] assert many2_2.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2]
def test_load_from_files_full(use_testdb, dumpdir): os.makedirs(dumpdir) with open(os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write( '{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test2": 1, "test3": [17]}\n' ) with gzip.open(os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'wt', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest2", "id": 1, "test1": [1, 2, 3]}\n') with open(os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write( '{"_entity": "pdtest3", "foo_bool": true, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 17, "test1": [1, 2, 3]}\n' ) with open(os.path.join(dumpdir, 'many1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many1", "id": 1, "many2": [1, 2]}\n') fp.write('{"_entity": "many1", "id": 2, "many2": [1, 2]}\n') with open(os.path.join(dumpdir, 'many2_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many2", "id": 1, "many1": [1, 2]}\n') fp.write('{"_entity": "many2", "id": 2, "many1": [1, 2]}\n') pd = PonyDump(testdb) # create test load_statuses = [ { 'created': True, 'updated': True, 'entity': 'many1', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'created': True, 'updated': True, 'entity': 'many1', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'created': True, 'updated': True, 'entity': 'many2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'created': True, 'updated': True, 'entity': 'many2', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'created': True, 'updated': True, 'entity': 'pdtest1', 'pk': (1, 2, 3), 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7 }, { 'created': True, 'updated': True, 'entity': 'pdtest3', 'pk': 17, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7 }, { 'created': True, 'updated': True, 'entity': 'pdtest2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7 }, { 'path': None, 'entity': None, 'pk': None, 'updated': None, 'created': None, 'count': 7, 'all_count': 7 }, ] for statuses in pd.load_from_files([dumpdir]): for status in statuses: assert status in load_statuses load_statuses.remove(status) assert not load_statuses assert not pd.get_depcache_dict() # update test PDTest3.select().first().foo_longstr = 'changed string' Many2.get(id=2).delete() load_statuses = [ # updated=False, потому что зависимость Many2(2) ещё отсутствует в базе { 'created': False, 'updated': False, 'entity': 'many1', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'created': False, 'updated': False, 'entity': 'many1', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'created': False, 'updated': False, 'entity': 'many2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7 }, # ...и создаётся только сейчас { 'created': True, 'updated': True, 'entity': 'many2', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7 }, { 'created': False, 'updated': False, 'entity': 'pdtest1', 'pk': (1, 2, 3), 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7 }, # Ещё вот тут мы меняли атрибут { 'created': False, 'updated': True, 'entity': 'pdtest3', 'pk': 17, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7 }, { 'created': False, 'updated': False, 'entity': 'pdtest2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7 }, { 'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7 }, { 'path': None, 'entity': None, 'pk': None, 'updated': None, 'created': None, 'count': 7, 'all_count': 7 }, ] for statuses in pd.load_from_files([dumpdir]): for status in statuses: load_statuses.remove(status) if status['entity'] == 'many1': # Проверяем, что отсутствующая зависимость Many2 положена в depcache deps = pd.get_depcache_dict()[(('many1', 'many2'), ('many2', 'many1'))] assert (2, ) in deps[0].get((status['pk'], )) assert (status['pk'], ) in deps[1].get((2, )) assert not load_statuses assert not pd.get_depcache_dict() assert PDTest1.select().count() == 1 test1 = PDTest1.select().first() assert test1.get_pk() == (1, 2, 3) assert test1.test2.id == 1 assert test1.test3.select().count() == 1 assert test1.test3.select().first().id == 17 assert PDTest2.select().count() == 1 test2 = PDTest2.select().first() assert test2.get_pk() == 1 assert test2.test1 is test1 # Pony ORM кэширует всё подряд, поэтому можно is assert PDTest3.select().count() == 1 test3 = PDTest3.select().first() assert test3.get_pk() == 17 assert test3.test1 is test1 assert test3.foo_bool is True assert test3.foo_int == -4 assert 0.1 + 0.2 == 0.30000000000000004 assert test3.foo_float == 0.30000000000000004 assert test3.foo_string == 'foo\u0000😊bar' assert test3.foo_longstr == 'longstr' assert test3.foo_datetime == datetime(2017, 6, 1, 1, 2, 3, 999987) assert test3.foo_uuid == uuid.UUID('8e8cdc11-0785-43a8-8203-66c148c3f57c') many1_1, many1_2 = Many1.select().order_by(Many1.id)[:] many2_1, many2_2 = Many2.select().order_by(Many2.id)[:] assert many1_1.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many1_2.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many2_1.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2] assert many2_2.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2]
def test_load_from_files_full(use_testdb, dumpdir): os.makedirs(dumpdir) with open(os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test2": 1, "test3": [17]}\n') with gzip.open(os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'wt', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest2", "id": 1, "test1": [1, 2, 3]}\n') with open(os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "pdtest3", "foo_bool": true, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 17, "test1": [1, 2, 3]}\n') with open(os.path.join(dumpdir, 'many1_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many1", "id": 1, "many2": [1, 2]}\n') fp.write('{"_entity": "many1", "id": 2, "many2": [1, 2]}\n') with open(os.path.join(dumpdir, 'many2_dump.jsonl'), 'w', encoding='utf-8') as fp: fp.write('{"_entity": "many2", "id": 1, "many1": [1, 2]}\n') fp.write('{"_entity": "many2", "id": 2, "many1": [1, 2]}\n') pd = PonyDump(testdb) # create test load_statuses = [ {'created': True, 'updated': True, 'entity': 'many1', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7}, {'created': True, 'updated': True, 'entity': 'many1', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7}, {'created': True, 'updated': True, 'entity': 'many2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7}, {'created': True, 'updated': True, 'entity': 'many2', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7}, {'created': True, 'updated': True, 'entity': 'pdtest1', 'pk': (1, 2, 3), 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7}, {'created': True, 'updated': True, 'entity': 'pdtest3', 'pk': 17, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7}, {'created': True, 'updated': True, 'entity': 'pdtest2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7}, {'path': None, 'entity': None, 'pk': None, 'updated': None, 'created': None, 'count': 7, 'all_count': 7}, ] for statuses in pd.load_from_files([dumpdir]): for status in statuses: assert status in load_statuses load_statuses.remove(status) assert not load_statuses assert not pd.get_depcache_dict() # update test PDTest3.select().first().foo_longstr = 'changed string' Many2.get(id=2).delete() load_statuses = [ # updated=False, потому что зависимость Many2(2) ещё отсутствует в базе {'created': False, 'updated': False, 'entity': 'many1', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7}, {'created': False, 'updated': False, 'entity': 'many1', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many1_dump.jsonl'), 'count': 2, 'all_count': 7}, {'created': False, 'updated': False, 'entity': 'many2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7}, # ...и создаётся только сейчас {'created': True, 'updated': True, 'entity': 'many2', 'pk': 2, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 2, 'current': 2, 'path': os.path.join(dumpdir, 'many2_dump.jsonl'), 'count': 2, 'all_count': 7}, {'created': False, 'updated': False, 'entity': 'pdtest1', 'pk': (1, 2, 3), 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest1_dump.jsonl'), 'count': 1, 'all_count': 7}, # Ещё вот тут мы меняли атрибут {'created': False, 'updated': True, 'entity': 'pdtest3', 'pk': 17, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest3_dump.jsonl'), 'count': 1, 'all_count': 7}, {'created': False, 'updated': False, 'entity': 'pdtest2', 'pk': 1, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7}, {'entity': None, 'updated': None, 'created': None, 'pk': None, 'lineno': 1, 'current': 1, 'path': os.path.join(dumpdir, 'pdtest2_dump.jsonl.gz'), 'count': 1, 'all_count': 7}, {'path': None, 'entity': None, 'pk': None, 'updated': None, 'created': None, 'count': 7, 'all_count': 7}, ] for statuses in pd.load_from_files([dumpdir]): for status in statuses: load_statuses.remove(status) if status['entity'] == 'many1': # Проверяем, что отсутствующая зависимость Many2 положена в depcache deps = pd.get_depcache_dict()[(('many1', 'many2'), ('many2', 'many1'))] assert (2,) in deps[0].get((status['pk'],)) assert (status['pk'],) in deps[1].get((2,)) assert not load_statuses assert not pd.get_depcache_dict() assert PDTest1.select().count() == 1 test1 = PDTest1.select().first() assert test1.get_pk() == (1, 2, 3) assert test1.test2.id == 1 assert test1.test3.select().count() == 1 assert test1.test3.select().first().id == 17 assert PDTest2.select().count() == 1 test2 = PDTest2.select().first() assert test2.get_pk() == 1 assert test2.test1 is test1 # Pony ORM кэширует всё подряд, поэтому можно is assert PDTest3.select().count() == 1 test3 = PDTest3.select().first() assert test3.get_pk() == 17 assert test3.test1 is test1 assert test3.foo_bool is True assert test3.foo_int == -4 assert 0.1 + 0.2 == 0.30000000000000004 assert test3.foo_float == 0.30000000000000004 assert test3.foo_string == 'foo\u0000😊bar' assert test3.foo_longstr == 'longstr' assert test3.foo_datetime == datetime(2017, 6, 1, 1, 2, 3, 999987) assert test3.foo_uuid == uuid.UUID('8e8cdc11-0785-43a8-8203-66c148c3f57c') many1_1, many1_2 = Many1.select().order_by(Many1.id)[:] many2_1, many2_2 = Many2.select().order_by(Many2.id)[:] assert many1_1.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many1_2.many2.select().order_by(Many2.id)[:] == [many2_1, many2_2] assert many2_1.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2] assert many2_2.many1.select().order_by(Many1.id)[:] == [many1_1, many1_2]
def test_dump_to_directory_full(use_testdb, dumpdir, gzip_compression): pd = PonyDump(testdb) _fill_testdb() # Порядок дампа строго не определён dump_statuses = [ { 'current': 1, 'count': 2, 'pk': (1, ), 'entity': 'many1', 'chunk_size': 250 }, { 'current': 2, 'count': 2, 'pk': (2, ), 'entity': 'many1', 'chunk_size': 250 }, { 'current': 2, 'count': 2, 'pk': None, 'entity': 'many1', 'chunk_size': 250 }, { 'current': 1, 'count': 2, 'pk': (1, ), 'entity': 'many2', 'chunk_size': 250 }, { 'current': 2, 'count': 2, 'pk': (2, ), 'entity': 'many2', 'chunk_size': 250 }, { 'current': 2, 'count': 2, 'pk': None, 'entity': 'many2', 'chunk_size': 250 }, { 'current': 1, 'count': 3, 'pk': (1, 2, 3), 'entity': 'pdtest1', 'chunk_size': 250 }, { 'current': 1, 'count': 3, 'pk': (1, 2, 3), 'entity': 'pdtest1', 'chunk_size': 250 }, { 'current': 3, 'count': 3, 'pk': (4, 5, 5), 'entity': 'pdtest1', 'chunk_size': 250 }, { 'current': 3, 'count': 3, 'pk': None, 'entity': 'pdtest1', 'chunk_size': 250 }, { 'current': 1, 'count': 3, 'pk': (1, ), 'entity': 'pdtest3', 'chunk_size': 250 }, { 'current': 3, 'count': 3, 'pk': (3, ), 'entity': 'pdtest3', 'chunk_size': 250 }, { 'current': 3, 'count': 3, 'pk': None, 'entity': 'pdtest3', 'chunk_size': 250 }, { 'current': 1, 'count': 1, 'pk': (1, ), 'entity': 'pdtest2', 'chunk_size': 250 }, { 'current': 1, 'count': 1, 'pk': (1, ), 'entity': 'pdtest2', 'chunk_size': 250 }, { 'current': 1, 'count': 1, 'pk': None, 'entity': 'pdtest2', 'chunk_size': 250 }, { 'current': 0, 'count': 0, 'pk': None, 'entity': 'pdtest4', 'chunk_size': 250 }, { 'entity': None, 'path': None, 'current': 0, 'count': 0, 'pk': None }, # Всегда последний ] filenames = [ 'pdtest1_dump.jsonl', 'pdtest2_dump.jsonl', 'pdtest3_dump.jsonl', 'pdtest4_dump.jsonl', 'many1_dump.jsonl', 'many2_dump.jsonl', ] if gzip_compression: filenames = [x + '.gz' for x in filenames] filenames = set(filenames) # Проверка информацирования о процессе дампа for status in pd.dump_to_directory(dumpdir, gzip_compression=gzip_compression): path = status.pop('path') if path is not None: assert os.path.split(path)[1] in filenames assert os.path.isfile(path) else: status['path'] = path # Для проверки последнего элемента assert status in dump_statuses dump_statuses.remove(status) if path is None: assert not dump_statuses assert not dump_statuses assert set(os.listdir(dumpdir)) == filenames # Проверка содержимого дампов: # - ключи у JSON отсортированы # - после запятых и двоеточий пробелы # - Файлы в UTF-8 без BOM # - перенос строк с помощью \n # - объекты внутри дампа отсортированы по первичным ключам _check_dump_content(dumpdir, 'pdtest1_dump.jsonl', [ '{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test2": 1, "test3": [1, 2], "test4": null}\n', '{"_entity": "pdtest1", "k1": 3, "k2": 5, "k3": 6, "test2": null, "test3": [], "test4": null}\n', '{"_entity": "pdtest1", "k1": 4, "k2": 5, "k3": 5, "test2": null, "test3": [], "test4": null}\n', ]) _check_dump_content(dumpdir, 'pdtest2_dump.jsonl', [ '{"_entity": "pdtest2", "id": 1, "test1": [1, 2, 3]}\n', ]) _check_dump_content(dumpdir, 'pdtest3_dump.jsonl', [ '{"_entity": "pdtest3", "foo_bool": true, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 1, "test1": [1, 2, 3]}\n', '{"_entity": "pdtest3", "foo_bool": false, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": 4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 2, "test1": [1, 2, 3]}\n', '{"_entity": "pdtest3", "foo_bool": null, "foo_datetime": "1970-01-01T00:00:00.000000Z", "foo_float": 0.0, "foo_int": 2147483647, "foo_longstr": "string", "foo_string": "string", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 3, "test1": null}\n', ]) _check_dump_content(dumpdir, 'pdtest4_dump.jsonl', ['']) _check_dump_content(dumpdir, 'many1_dump.jsonl', [ '{"_entity": "many1", "id": 1, "many2": [1, 2]}\n', '{"_entity": "many1", "id": 2, "many2": [1, 2]}\n', ]) _check_dump_content(dumpdir, 'many2_dump.jsonl', [ '{"_entity": "many2", "id": 1, "many1": [1, 2]}\n', '{"_entity": "many2", "id": 2, "many1": [1, 2]}\n', ])
def test_json2obj_ok_entity_case_insensitive(use_testdb): pd = PonyDump(testdb) result = pd.json2obj({'_entity': 'mAnY1', 'id': 7, "many2": []}) assert isinstance(result['obj'], Many1) assert result['obj'].id == 7
def test_dump_to_directory_full(use_testdb, dumpdir, gzip_compression): pd = PonyDump(testdb) _fill_testdb() # Порядок дампа строго не определён dump_statuses = [ {'current': 1, 'count': 2, 'pk': (1,), 'entity': 'many1', 'chunk_size': 250}, {'current': 2, 'count': 2, 'pk': (2,), 'entity': 'many1', 'chunk_size': 250}, {'current': 2, 'count': 2, 'pk': None, 'entity': 'many1', 'chunk_size': 250}, {'current': 1, 'count': 2, 'pk': (1,), 'entity': 'many2', 'chunk_size': 250}, {'current': 2, 'count': 2, 'pk': (2,), 'entity': 'many2', 'chunk_size': 250}, {'current': 2, 'count': 2, 'pk': None, 'entity': 'many2', 'chunk_size': 250}, {'current': 1, 'count': 3, 'pk': (1, 2, 3), 'entity': 'pdtest1', 'chunk_size': 250}, {'current': 1, 'count': 3, 'pk': (1, 2, 3), 'entity': 'pdtest1', 'chunk_size': 250}, {'current': 3, 'count': 3, 'pk': (4, 5, 5), 'entity': 'pdtest1', 'chunk_size': 250}, {'current': 3, 'count': 3, 'pk': None, 'entity': 'pdtest1', 'chunk_size': 250}, {'current': 1, 'count': 3, 'pk': (1,), 'entity': 'pdtest3', 'chunk_size': 250}, {'current': 3, 'count': 3, 'pk': (3,), 'entity': 'pdtest3', 'chunk_size': 250}, {'current': 3, 'count': 3, 'pk': None, 'entity': 'pdtest3', 'chunk_size': 250}, {'current': 1, 'count': 1, 'pk': (1,), 'entity': 'pdtest2', 'chunk_size': 250}, {'current': 1, 'count': 1, 'pk': (1,), 'entity': 'pdtest2', 'chunk_size': 250}, {'current': 1, 'count': 1, 'pk': None, 'entity': 'pdtest2', 'chunk_size': 250}, {'current': 0, 'count': 0, 'pk': None, 'entity': 'pdtest4', 'chunk_size': 250}, {'entity': None, 'path': None, 'current': 0, 'count': 0, 'pk': None}, # Всегда последний ] filenames = [ 'pdtest1_dump.jsonl', 'pdtest2_dump.jsonl', 'pdtest3_dump.jsonl', 'pdtest4_dump.jsonl', 'many1_dump.jsonl', 'many2_dump.jsonl', ] if gzip_compression: filenames = [x + '.gz' for x in filenames] filenames = set(filenames) # Проверка информацирования о процессе дампа for status in pd.dump_to_directory(dumpdir, gzip_compression=gzip_compression): path = status.pop('path') if path is not None: assert os.path.split(path)[1] in filenames assert os.path.isfile(path) else: status['path'] = path # Для проверки последнего элемента assert status in dump_statuses dump_statuses.remove(status) if path is None: assert not dump_statuses assert not dump_statuses assert set(os.listdir(dumpdir)) == filenames # Проверка содержимого дампов: # - ключи у JSON отсортированы # - после запятых и двоеточий пробелы # - Файлы в UTF-8 без BOM # - перенос строк с помощью \n # - объекты внутри дампа отсортированы по первичным ключам _check_dump_content(dumpdir, 'pdtest1_dump.jsonl', [ '{"_entity": "pdtest1", "k1": 1, "k2": 2, "k3": 3, "test2": 1, "test3": [1, 2], "test4": null}\n', '{"_entity": "pdtest1", "k1": 3, "k2": 5, "k3": 6, "test2": null, "test3": [], "test4": null}\n', '{"_entity": "pdtest1", "k1": 4, "k2": 5, "k3": 5, "test2": null, "test3": [], "test4": null}\n', ]) _check_dump_content(dumpdir, 'pdtest2_dump.jsonl', [ '{"_entity": "pdtest2", "id": 1, "test1": [1, 2, 3]}\n', ]) _check_dump_content(dumpdir, 'pdtest3_dump.jsonl', [ '{"_entity": "pdtest3", "foo_bool": true, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": -4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 1, "test1": [1, 2, 3]}\n', '{"_entity": "pdtest3", "foo_bool": false, "foo_datetime": "2017-06-01T01:02:03.999987Z", "foo_float": 0.30000000000000004, "foo_int": 4, "foo_longstr": "longstr", "foo_string": "foo\\u0000😊bar", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 2, "test1": [1, 2, 3]}\n', '{"_entity": "pdtest3", "foo_bool": null, "foo_datetime": "1970-01-01T00:00:00.000000Z", "foo_float": 0.0, "foo_int": 2147483647, "foo_longstr": "string", "foo_string": "string", "foo_uuid": "8e8cdc11-0785-43a8-8203-66c148c3f57c", "id": 3, "test1": null}\n', ]) _check_dump_content(dumpdir, 'pdtest4_dump.jsonl', ['']) _check_dump_content(dumpdir, 'many1_dump.jsonl', [ '{"_entity": "many1", "id": 1, "many2": [1, 2]}\n', '{"_entity": "many1", "id": 2, "many2": [1, 2]}\n', ]) _check_dump_content(dumpdir, 'many2_dump.jsonl', [ '{"_entity": "many2", "id": 1, "many1": [1, 2]}\n', '{"_entity": "many2", "id": 2, "many1": [1, 2]}\n', ])