class TestHistoryDataRepository(TestHistoryDataRepositoryMixin,
unittest.TestCase):
storage = LocalStorage(settings.LOCAL_STORAGE_ROOT_FOLDER)
resource_base_path = "historydatapository"
prop_f_earliest_id = "_f_earliest_resource_id"
def create_resource_repository(self):
return HistoryDataRepository(
self.storage,
self.resource_name,
resource_base_path=self.resource_base_path,
metaname="metadata",
cache=self.cache,
f_earliest_resource_id=self.f_earliest_id)
def get_test_data_keys(self):
return [("2018_01_10_test1.txt", ), ("2018_01_20_test2.txt", ),
("2019_01_10_test3.txt", ), ("2019_01_20_test4.txt", ),
("2020_01_10_test5.txt", ), ("2020_01_20_test6.txt", )]
def set_f_earliest_id(self, resource_id):
if resource_id in ("2019_01_10_test3.txt", "2020_01_10_test5.txt"):
self.f_earliest_id = lambda res_id: resource_id
return True
return False
class TestGroupResourceRepositoryClient(TestResourceRepositoryClientMixin,unittest.TestCase):
storage = LocalStorage(settings.LOCAL_STORAGE_ROOT_FOLDER)
resource_base_path = "groupresourcerepository"
def create_resource_repository(self):
return GroupResourceRepository(
self.storage,
self.resource_name,
resource_base_path=self.resource_base_path,
archive=self.archive,
metaname="metadata",
cache=self.cache,
logical_delete=self.logical_delete
)
def get_test_data_keys(self):
return [
("2019_06","2019_06_01_test3.txt"),
("2019_06","2019_06_02_test4.txt"),
("2018_05","2018_05_01_test1.txt"),
("2018_05","2018_05_02_test2.txt"),
("2020_07","2020_07_01_test5.txt"),
("2020_07","2020_07_02_test6.txt")
]
def get_test_data_keys2(self):
return [
("2021_07","2021_07_01_test7.txt"),
("2021_07","2021_07_02_test8.txt")
]
def create_resource_repository(self):
return IndexedGroupHistoryDataRepository(
LocalStorage(settings.LOCAL_STORAGE_DIR),
settings.RESOURCE_NAME,
get_metaname_code,
index_metaname=self.index_metaname,
f_earliest_metaname=None if settings.ARCHIVE_LIFESPAN is None
or settings.ARCHIVE_LIFESPAN <= 0 else get_earliest_metaname)
def get_consume_client():
"""
Return the blob resource client
"""
global _consume_client
if _consume_client is None:
_consume_client = HistoryDataConsumeClient(
LocalStorage(settings.NGINXLOG_REPOSITORY_DIR),
settings.NGINXLOG_RESOURCE_NAME,
settings.RESOURCE_CLIENTID,
max_saved_consumed_resources=settings.
NGINXLOG_MAX_SAVED_CONSUMED_RESOURCES)
return _consume_client
class TestIndexedGroupHistoryDataRepository(BaseTesterMixin,unittest.TestCase):
storage = LocalStorage(settings.LOCAL_STORAGE_ROOT_FOLDER)
resource_base_path = "indexedgrouphistorydatarepository"
prop_f_earliest_id = "_f_earliest_metaname"
def create_resource_repository(self):
return IndexedGroupHistoryDataRepository(
self.storage,
self.resource_name,
f_metaname_code,
resource_base_path=self.resource_base_path,
cache=self.cache,
f_earliest_metaname=None
)
def get_test_data_keys(self):
return [
("2018_01","2018_01_10_test1.txt",),
("2018_01","2018_01_20_test2.txt",),
("2018_02","2018_02_10_test3.txt",),
("2018_02","2018_02_20_test4.txt",),
("2019_01","2019_01_10_test5.txt",),
("2019_01","2019_01_20_test6.txt",),
("2019_02","2019_02_10_test7.txt",),
("2019_02","2019_02_20_test8.txt",),
("2020_01","2020_01_10_test9.txt",),
("2020_01","2020_01_20_test10.txt",),
("2020_02","2020_02_10_test11.txt",),
("2020_02","2020_02_20_test12.txt",)
]
def test_change_metaindex(self):
self.clean_resources()
self.archive=False
self.logical_delete=False
repository = self.resource_repository
logger.info("{}:Test change_metaindex for indexed resource repository".format(self.prefix))
metadatas = self.prepare_test_datas()
res_metadatas = [res_metadata for res_metadata in repository.metadata_client.resource_metadatas(throw_exception=False,resource_status=ResourceConstant.ALL_RESOURCE,resource_file=None)]
new_repository_metadata = transform.change_metaindex(repository.metadata_client,new_f_metaname_code)
new_res_metadatas = [res_metadata for res_metadata in new_repository_metadata.resource_metadatas(throw_exception=False,resource_status=ResourceConstant.ALL_RESOURCE,resource_file=None)]
self.assertEqual(res_metadatas,new_res_metadatas,"{}The migrated metadatas({}) is not equal with original metadatas({})".format(self.prefix,res_metadatas,new_res_metadatas))
def get_client(cache=True):
"""
Return the blob resource client
"""
global _containerstatus_client
if _containerstatus_client is None or not cache:
client = HistoryDataConsumeClient(
LocalStorage(settings.CONTAINERSTATUS_REPOSITORY_DIR),
settings.CONTAINERSTATUS_RESOURCE_NAME,
settings.RESOURCE_CLIENTID,
max_saved_consumed_resources=settings.CONTAINERSTATUS_MAX_SAVED_CONSUMED_RESOURCES
)
if cache:
_containerstatus_client = client
else:
return client
return _containerstatus_client
class TestGroupHistoryDataRepositoryClient(
TestHistoryDataRepositoryClientMixin, unittest.TestCase):
storage = LocalStorage(settings.LOCAL_STORAGE_ROOT_FOLDER)
resource_base_path = "grouphistorydatarepository"
def create_resource_repository(self):
return GroupHistoryDataRepository(
self.storage,
self.resource_name,
resource_base_path=self.resource_base_path,
metaname="metadata",
cache=self.cache)
def get_test_data_keys(self):
return [(
"2018_01",
"2018_01_10_test1.txt",
), (
"2018_01",
"2018_01_20_test2.txt",
), (
"2019_01",
"2019_01_10_test3.txt",
), (
"2019_01",
"2019_01_20_test4.txt",
), (
"2020_01",
"2020_01_10_test5.txt",
), (
"2020_01",
"2020_01_20_test6.txt",
)]
def get_test_data_keys2(self):
return [(
"2021_01",
"2021_01_10_test7.txt",
), (
"2021_01",
"2021_01_20_test8.txt",
)]