def setUp(self):
# Owner of the project
self.test_index_name = reindex_test_dataset()
self.secondary_index = reindex_test_dataset()
self.index, is_created = Index.objects.get_or_create(
name=self.secondary_index)
self.user = create_test_user('annotator', '*****@*****.**', 'pw')
self.user2 = create_test_user('annotator2', '*****@*****.**', 'pw2')
self.project = project_creation("entityTestProject",
self.test_index_name, self.user)
self.project.indices.add(self.index)
self.project.users.add(self.user)
self.project.users.add(self.user2)
self.client.login(username='******', password='******')
self.ec = ElasticCore()
self.list_view_url = reverse("v2:annotator-list",
kwargs={"project_pk": self.project.pk})
self.annotator = self._create_annotator()
self.pull_document_url = reverse("v2:annotator-pull-document",
kwargs={
"project_pk": self.project.pk,
"pk": self.annotator["id"]
})
def setUp(self) -> None:
self.test_index_name = reindex_test_dataset()
self.user = create_test_user('langDetectUser', '*****@*****.**', 'pw')
self.non_project_user = create_test_user('langDetectUserThatIsNotInProject', '*****@*****.**', 'pw')
self.project = project_creation("langDetectProject", self.test_index_name, self.user)
self.project.users.add(self.user)
self.client.login(username='******', password='******')
self.url = reverse("v2:lang_index-list", kwargs={"project_pk": self.project.pk})
def setUpTestData(cls):
# Owner of the project
cls.admin_user = create_test_user("admin",
"*****@*****.**",
"pw",
superuser=True)
cls.non_admin_user = create_test_user("tester", "*****@*****.**", "pw")
cls.url = f"{TEST_VERSION_PREFIX}/core_variables/"
cls.health_url = f"{TEST_VERSION_PREFIX}/health/"
def setUp(self):
self.client = APIClient()
# Create admin user with pk 1
self.admin = create_test_user(name='admin', password='******')
self.admin.is_superuser = True
self.admin.save()
# Create a new User
self.user = create_test_user(name='user', password='******')
self.user_url = f'{TEST_VERSION_PREFIX}/users/{self.user.profile.id}/'
self.admin_url = f'{TEST_VERSION_PREFIX}/users/{self.admin.profile.id}/'
def setUp(self):
self.normal_user = create_test_user('normal_user', '*****@*****.**',
'pw')
self.admin_user = create_test_user('admin_user',
'*****@*****.**',
'pw',
superuser=True)
self.project = project_creation("AnonymizerTestProject", TEST_INDEX,
self.normal_user)
self.project.users.add(self.normal_user, self.admin_user)
self.client.login(username='******', password='******')
def setUpTestData(cls):
""" user needs to be admin, because of changed indices permissions """
cls.default_password = '******'
cls.default_username = '******'
cls.user = create_test_user(cls.default_username, '*****@*****.**',
cls.default_password)
# create admin to test indices removal from project
cls.admin = create_test_user(name='admin', password="******")
cls.admin.is_superuser = True
cls.admin.save()
cls.project = project_creation("ElasticTestProject", TEST_INDEX,
cls.user)
cls.project.users.add(cls.user)
def setUp(self) -> None:
self.test_index_name = reindex_test_dataset()
self.user = create_test_user('Owner', '*****@*****.**', 'pw')
self.unauthorized_user = create_test_user('unauthorized', '*****@*****.**', 'pw')
self.file_name = "d41d8cd98f00b204e9800998ecf8427e.txt"
self.project = project_creation("test_doc_parser", index_title=None, author=self.user)
self.project.users.add(self.user)
self.unauth_project = project_creation("unauth_project", index_title=None, author=self.user)
self.file = SimpleUploadedFile("text.txt", b"file_content", content_type="text/html")
self.client.login(username='******', password='******')
self._basic_pipeline_functionality()
self.file_path = self._get_file_path()
self.ec = ElasticCore()
def setUp(self):
self.test_index_name = reindex_test_dataset()
self.user = create_test_user('first_user', '*****@*****.**', 'pw')
self.project = project_creation("DocumentImporterAPI",
self.test_index_name, self.user)
self.validation_project = project_creation("validation_project",
"random_index_name",
self.user)
self.document_id = random.randint(10000000, 90000000)
self.uuid = uuid.uuid1()
self.source = {"hello": "world", "uuid": self.uuid}
self.document = {
"_index": self.test_index_name,
"_id": self.document_id,
"_source": self.source
}
self.target_field_random_key = uuid.uuid1()
self.target_field = f"{self.target_field_random_key}_court_case"
self.ec = ElasticCore()
self.client.login(username='******', password='******')
self._check_inserting_documents()
def setUp(self):
self.default_password = '******'
self.admin = create_test_user(name='admin', password='******')
self.admin.is_superuser = True
# self.admin.is_staff = True
self.admin.save()
self.project_user = create_test_user(name='project_user',
password=self.default_password)
self.user = create_test_user(name='user',
password=self.default_password)
self.project = project_creation("proj", TEST_INDEX, self.admin)
self.project.users.add(self.project_user)
self.client = APIClient()
self.project_instance_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/'
def setUp(self):
# Owner of the project
self.test_index = reindex_test_dataset(from_index=TEST_INDEX_ENTITY_EVALUATOR)
self.user = create_test_user("EvaluatorOwner", "*****@*****.**", "pw")
self.project = project_creation("EvaluatorTestProject", self.test_index, self.user)
self.project.users.add(self.user)
self.url = f"{TEST_VERSION_PREFIX}/projects/{self.project.id}/evaluators/"
self.project_url = f"{TEST_VERSION_PREFIX}/projects/{self.project.id}"
self.true_fact_name = "PER"
self.pred_fact_name = "PER_CRF_30"
self.true_fact_name_sent_index = "PER_SENT"
self.pred_fact_name_sent_index = "PER_CRF_31_SENT"
self.fact_name_no_spans = "PER_FN_REGEX_NO_SPANS"
self.fact_name_different_doc_paths = "PER_DOUBLE"
self.core_variables_url = f"{TEST_VERSION_PREFIX}/core_variables/5/"
# TODO! Construct a test query
self.fact_names_to_filter = [self.true_fact_name, self.pred_fact_name]
self.test_query = Query()
self.test_query.add_facts_filter(self.fact_names_to_filter, [], operator="must")
self.test_query = self.test_query.__dict__()
self.client.login(username="******", password="******")
self.token_based_evaluator_id = None
self.value_based_evaluator_id = None
self.token_based_sent_index_evaluator_id = None
self.value_based_sent_index_evaluator_id = None
def setUp(self):
self.test_index_name = reindex_test_dataset()
self.normal_user = create_test_user('user', '*****@*****.**', 'pw')
self.project = project_creation("applyAnalyzers", self.test_index_name,
self.normal_user)
self.project.users.add(self.normal_user)
self.admin_user = create_test_user('admin', '*****@*****.**', 'pw')
self.project.users.add(self.admin_user)
self.unauthorized_user = create_test_user('unauthorized',
'*****@*****.**', 'pw')
self.list_url = reverse(f"{VERSION_NAMESPACE}:apply_analyzers-list",
kwargs={"project_pk": self.project.pk})
self.client.login(username='******', password='******')
def setUp(self):
# Create a new User
self.user = create_test_user(name='user', password='******')
self.project = project_creation("testproj", TEST_INDEX, self.user)
self.project.users.add(self.user)
self.client = APIClient()
self.client.login(username='******', password='******')
def setUp(self):
self.test_index_name = reindex_test_dataset()
self.user = create_test_user('first_user', '*****@*****.**', 'pw')
self.project = project_creation("FactManagementApplicationTests",
self.test_index_name, self.user)
self.uuid = uuid.uuid1().hex
self.content = "miks sa oled loll!?"
self.source = {
TEST_FIELD:
self.content,
TEXTA_TAGS_KEY: [
{
"str_val": "politsei",
"fact": "ORG",
"spans": json.dumps([[0, 0]]),
"doc_path": "hello"
},
]
}
self.ec = ElasticCore()
self.ec.es.index(index=self.test_index_name,
id=self.uuid,
body=self.source,
refresh="wait_for")
self.kwargs = {"project_pk": self.project.pk}
self.client.login(username='******', password='******')
def setUpTestData(cls):
# Owner of the project
cls.user = create_test_user('taggerOwner', '*****@*****.**', 'pw')
cls.user.is_superuser = True
cls.user.save()
cls.project = project_creation("taggerTestProject", TEST_INDEX_LARGE,
cls.user)
cls.url = f'{TEST_VERSION_PREFIX}/projects/{cls.project.id}/taggers/'
def setUp(self):
# Owner of the project
self.test_index_name = reindex_test_dataset()
self.user = create_test_user('taggerOwner', '*****@*****.**', 'pw')
self.project = project_creation("taggerTestProject",
self.test_index_name, self.user)
self.project.users.add(self.user)
self.client.login(username='******', password='******')
def setUp(self):
self.user = create_test_user('torchtaggerOwner', '*****@*****.**', 'pw')
self.project = project_creation("torchtaggerTestProject",
TEST_INDEX_LARGE, self.user)
self.project.users.add(self.user)
self.url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/torchtaggers/'
self.test_embedding_id = None
self.client.login(username='******', password='******')
self.torch_models = list(TORCH_MODELS.keys())
def setUpTestData(cls):
cls.test_index_name = reindex_test_dataset()
cls.user = create_test_user('user', '*****@*****.**', 'pw')
cls.project = project_creation("AnonymizerTestProject",
cls.test_index_name, cls.user)
cls.project.users.add(cls.user)
cls.url = f'{TEST_VERSION_PREFIX}/projects/{cls.project.id}/anonymizers/'
cls.anonymizer_id = None
def setUp(self):
# Create a new project_user, all project extra actions need to be permissible for this project_user.
self.user = create_test_user(name='user', password='******')
self.project_user = create_test_user(name='project_user',
password='******')
self.admin = create_test_user(name='admin', password='******')
self.admin.is_superuser = True
self.admin.save()
self.project_name = "testproj"
self.project = project_creation(self.project_name, TEST_INDEX,
self.user)
self.project.users.add(self.project_user)
self.project.save()
self.project_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}'
self.export_url = reverse(f"{VERSION_NAMESPACE}:project-export-search",
kwargs={"project_pk": self.project.pk})
self.client = APIClient()
self.client.login(username='******', password='******')
def setUp(self):
self.test_index_name = reindex_test_dataset()
self.ec = ElasticCore()
self.user = create_test_user('mlpUser', '*****@*****.**', 'pw')
self.project = project_creation("mlpTestProject", self.test_index_name,
self.user)
self.project.users.add(self.user)
self.client.login(username='******', password='******')
self.url = reverse(f"{VERSION_NAMESPACE}:mlp_index-list",
kwargs={"project_pk": self.project.pk})
def setUp(self):
# Owner of the project
self.user = create_test_user('Owner', '*****@*****.**', 'pw')
self.project = project_creation("testImportDatasetProject", None,
self.user)
self.project.users.add(self.user)
self.url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/elastic/dataset_imports/'
self.project_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}'
self.created_indices = []
self.client.login(username='******', password='******')
def setUp(self):
""" user needs to be admin, because of changed indices permissions """
self.test_index_name = reindex_test_dataset()
self.default_password = '******'
self.default_username = '******'
self.user = create_test_user(self.default_username, '*****@*****.**',
self.default_password)
# create admin to test indices removal from project
self.admin = create_test_user(name='admin', password='******')
self.admin.is_superuser = True
self.admin.save()
self.project = project_creation("ReindexerTestProject",
self.test_index_name, self.user)
self.project.users.add(self.user)
self.ec = ElasticCore()
self.client.login(username=self.default_username,
password=self.default_password)
self.new_index_name = f"{TEST_FIELD}_2"
def setUp(self):
self.test_index_name = reindex_test_dataset()
self.user = create_test_user('embeddingOwner', '*****@*****.**', 'pw')
self.project = project_creation("embeddingTestProject",
self.test_index_name, self.user)
self.project.users.add(self.user)
self.url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/embeddings/'
self.project_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}'
self.test_embedding_id = None
self.client.login(username='******', password='******')
def setUp(self):
self.url = f'{TEST_VERSION_PREFIX}/uaa'
self.test1_user = "******" # Used for test1 user id
self.test2_user = "******" # Used for test2 user id
self.test3_user = "******" # Used for test3 user id
# Create a normal User
self.user = create_test_user(name='normaluser', password='******')
self.create_users()
self.create_groups()
self.create_project1()
self.create_project2()
self.create_project3()
def setUp(self):
""" User needs to be admin, because of changed indices permissions. """
self.test_index_name = reindex_test_dataset()
self.default_password = '******'
self.default_username = '******'
self.user = create_test_user(self.default_username, '*****@*****.**',
self.default_password)
self.admin = create_test_user(name='admin', password='******')
self.admin.is_superuser = True
self.admin.save()
self.project = project_creation("IndexSplittingTestProject",
self.test_index_name, self.user)
self.project.users.add(self.user)
self.url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/elastic/index_splitter/'
self.client.login(username=self.default_username,
password=self.default_password)
self.ec = ElasticCore()
self.FACT = "TEEMA"
def setUp(self):
# Create a new project_user, all project extra actions need to be permissible for this project_user.
self.user = create_test_user(name='user', password='******')
self.unowned_user = create_test_user(name='unowned_user',
password='******')
self.admin = create_test_user(name='admin', password='******')
self.admin.is_superuser = True
self.admin.save()
self.project_user = create_test_user(name='project_user',
password='******')
self.project = project_creation("testproj", TEST_INDEX, self.user)
self.project.users.add(self.project_user)
self.project.users.add(self.user)
self.client = APIClient()
self.client.login(username='******', password='******')
self.scroll_url = reverse(f"{VERSION_NAMESPACE}:project-scroll",
kwargs={"project_pk": self.project.id})
def setUp(self):
# Owner of the project
self.test_index = reindex_test_dataset(from_index=TEST_INDEX_EVALUATOR)
self.user = create_test_user("EvaluatorOwner", "*****@*****.**", "pw")
self.project = project_creation("EvaluatorTestProject",
self.test_index, self.user)
self.project.users.add(self.user)
self.url = f"{TEST_VERSION_PREFIX}/projects/{self.project.id}/evaluators/"
self.project_url = f"{TEST_VERSION_PREFIX}/projects/{self.project.id}"
self.multilabel_avg_functions = choices.MULTILABEL_AVG_FUNCTIONS
self.binary_avg_functions = choices.BINARY_AVG_FUNCTIONS
self.multilabel_evaluators = {
avg: None
for avg in self.multilabel_avg_functions
}
self.binary_evaluators = {
avg: None
for avg in self.binary_avg_functions
}
self.memory_optimized_multilabel_evaluators = {
avg: None
for avg in self.multilabel_avg_functions
}
self.memory_optimized_binary_evaluators = {
avg: None
for avg in self.binary_avg_functions
}
self.true_fact_name = "TRUE_TAG"
self.pred_fact_name = "PREDICTED_TAG"
self.true_fact_value = "650 kapital"
self.pred_fact_value = "650 kuvand"
self.core_variables_url = f"{TEST_VERSION_PREFIX}/core_variables/5/"
# Construct a test query
self.fact_names_to_filter = [self.true_fact_name, self.pred_fact_name]
self.fact_values_to_filter = [
"650 bioeetika", "650 rahvusbibliograafiad"
]
self.test_query = Query()
self.test_query.add_facts_filter(self.fact_names_to_filter,
self.fact_values_to_filter,
operator="must")
self.test_query = self.test_query.__dict__()
self.client.login(username="******", password="******")
def setUp(self):
self.test_incorrect_index_name = TEST_INDEX
self.test_index_name = CRF_TEST_INDEX
self.test_index_copy = reindex_test_dataset(from_index=CRF_TEST_INDEX)
self.user = create_test_user('crfOwner', '*****@*****.**', 'pw')
self.project = project_creation("crfTestProject", self.test_index_name,
self.user)
self.project.users.add(self.user)
self.url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/crf_extractors/'
self.project_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}'
self.embedding_ids = [None]
self.test_crf_ids = []
self.client.login(username='******', password='******')
def setUp(self):
self.user = create_test_user('mlpUser', '*****@*****.**', 'pw')
self.project = project_creation("mlpTestProject", TEST_INDEX,
self.user)
self.project.users.add(self.user)
self.client.login(username='******', password='******')
self.url = reverse(f"{VERSION_NAMESPACE}:mlp_texts")
self.payload = {
"texts": [
"Õnnetus leidis aset eile kella 17.25 ajal Raplamaal Märjamaa alevis Koluvere maantee 2 juures, kus alkoholijoobes 66-aastane mees sõitis mopeedautoga Bellier 503 ringristmikul teelt välja vastu liiklusmärki.",
"Ex- US President Barack Obama and his successor recently exchanged verbal barbs, with the former slamming the administration’s handling of the COVID-19 pandemic",
"Приложение №1 для заботы о здоровье бесплатно по полису ОМС."
]
}
def setUp(self):
self.user = create_test_user('user', '*****@*****.**', 'pw')
self.new_test_index_name = f"ttk_test_query_tagger_{uuid.uuid4().hex[:5]}"
self.project = project_creation("SearchQueryTaggerTestProject", self.new_test_index_name, self.user)
self.project.users.add(self.user)
self.url = reverse("v2:search_query_tagger-list", kwargs={"project_pk": self.project.pk})
self.uuid = "adasda-5874856a-das4das98f5"
self.document = {
"Field_1": "This is sentence1. This is sentence2. This is sentence3. This is sentence4. This is sentence5.",
"Field_2": "This is a different sentence.",
"Field_3": "This is test data.",
"uuid": self.uuid}
self.ed = ElasticDocument(index=self.new_test_index_name)
self.ed.add(self.document)
self.client.login(username='******', password='******')
def setUp(self):
# Owner of the project
self.test_index_name = reindex_test_dataset()
self.user = create_test_user('taggerOwner', '*****@*****.**', 'pw')
self.project = project_creation("taggerTestProject",
self.test_index_name, self.user)
self.project.users.add(self.user)
self.url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/taggers/'
self.project_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}'
self.multitag_text_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/taggers/multitag_text/'
# set vectorizer & classifier options
self.vectorizer_opts = ('TfIdf Vectorizer', )
self.classifier_opts = ('LinearSVC', )
# list tagger_ids for testing. is populated during training test
self.test_tagger_ids = []
self.client.login(username='******', password='******')
# new fact name and value used when applying tagger to index
self.new_fact_name = "TEST_TAGGER_NAME"
self.new_fact_value = "TEST_TAGGER_VALUE"
# Create copy of test index
self.reindex_url = f'{TEST_VERSION_PREFIX}/projects/{self.project.id}/elastic/reindexer/'
# Generate name for new index containing random id to make sure it doesn't already exist
self.test_index_copy = f"test_apply_tagger_{uuid.uuid4().hex}"
self.reindex_payload = {
"description": "test index for applying taggers",
"indices": [self.test_index_name],
"query": json.dumps(TEST_QUERY),
"new_index": self.test_index_copy,
"fields": [TEST_FIELD]
}
resp = self.client.post(self.reindex_url,
self.reindex_payload,
format='json')
print_output("reindex test index for applying tagger:response.data:",
resp.json())
self.reindexer_object = Reindexer.objects.get(pk=resp.json()["id"])
self.test_imported_binary_tagger_id = self.import_test_model(
TEST_TAGGER_BINARY)
