def setUp(self) -> None:
self.settings.IMPORT_REPO_PATH = self.data_dir
self.study = Study(name="some-study")
self.study.save()
self.study_import_manager = StudyImportManager(study=self.study,
redis=False)
return super().setUp()
def update_single_study( # pylint: disable=R0913
study: Study,
local: bool,
entity: tuple = None,
filename: str = None,
clean_import=False,
manager: StudyImportManager = None,
) -> None:
"""Update a single study"""
backup_file = Path()
if clean_import:
backup_file = Basket.backup()
study.delete()
study.save()
if not manager:
StudyImportManager(study)
if not local:
manager.update_repo()
if not entity:
manager.import_all_entities()
elif filename:
manager.import_single_entity(entity[0], filename)
else:
update_study_partial(manager, entity)
if backup_file.is_file():
call_command("loaddata", backup_file)
BasketVariable.clean_basket_variables(study.name)
def test_variable_import_without_concept_csv(self):
csv_path = Study().import_path()
concept_path = csv_path.joinpath("concepts.csv")
os.remove(concept_path)
some_dataset = DatasetFactory(name="some-dataset")
some_dataset.save()
TEST_CASE.assertIsNone(
StudyImportManager(study=some_dataset.study).fix_concepts_csv())
TEST_CASE.assertFalse(concept_path.exists())
def test_update_single_study_entity_filename_without_redis(study):
filename = Study().import_path().joinpath(
"instruments/some-instrument.json")
with unittest.mock.patch("django_rq.enqueue") as redis_enqueue:
with TEST_CASE.assertRaises(SystemExit) as error:
call_command("update", study.name, "instruments.json", "-f",
filename, "-l", "-r")
TEST_CASE.assertEqual(0, error.exception.code)
TEST_CASE.assertFalse(redis_enqueue.called)
def test_variable_import_with_orphaned_concept(self):
csv_path = Study().import_path()
concept_path = csv_path.joinpath("concepts.csv")
some_dataset = DatasetFactory(name="some-dataset")
some_dataset.save()
StudyImportManager(study=some_dataset.study).fix_concepts_csv()
ConceptFactory(name="some-concept").save()
variable_path = csv_path.joinpath("variables.csv")
variable_path = variable_path.absolute()
ConceptImport(concept_path).run_import(filename=concept_path)
VariableImport.run_import(variable_path, study=some_dataset.study)
with open(variable_path, "r", encoding="utf8") as csv_file:
variable_names = {row["name"] for row in csv.DictReader(csv_file)}
result = Variable.objects.filter(name__in=list(variable_names))
TEST_CASE.assertNotEqual(0, len(result))
TEST_CASE.assertEqual(len(variable_names), len(result))
def test_update_single_study_entity_nonexistent_filename(study):
filename = Study().import_path().joinpath("nonexistent-file.json")
logger = logging.getLogger("ddionrails.imports.manager")
with unittest.mock.patch.object(logger, "error") as log:
with TEST_CASE.assertRaises(SystemExit) as error:
call_command("update", study.name, "instruments.json", "-f",
filename, "-l")
logging.getLogger("ddionrails.imports.manager")
TEST_CASE.assertEqual(1, error.exception.code)
log.assert_called_once_with('Study "%s" has no file: "%s"',
"some-study", "nonexistent-file.json")
def test_study_specific_backup(self, basket, variable):
"""Can we limit the backup to a specific study?"""
# A whole lot of boilerplate to set up another study basket.
basket.save()
basket.variables.add(variable)
basket.save()
other_study = Study(name="other-sturdy")
other_study.save()
other_dataset = Dataset(name="other-dataset")
other_dataset.study = other_study
other_dataset.save()
other_variable = Variable(name="other-variable")
other_variable.dataset = other_dataset
other_variable.save()
other_basket = Basket(name="other-basket",
study=other_study,
user=basket.user)
other_basket.save()
other_basket.variables.add(other_variable)
other_basket.save()
TEST_CASE.assertTrue(
BasketVariable.objects.get(variable=other_variable))
TEST_CASE.assertTrue(BasketVariable.objects.get(variable=variable))
# Actual testing
backup_file = Basket.backup(study=basket.study)
BasketVariable.objects.all().delete()
call_command("loaddata", backup_file)
with TEST_CASE.assertRaises(BasketVariable.DoesNotExist):
BasketVariable.objects.get(variable=other_variable)
TEST_CASE.assertTrue(BasketVariable.objects.get(variable=variable))
def test_import_variables_empty_concept(self, study):
"""Do not create concept with empty, "", name."""
variables_csv = Study().import_path().joinpath("variables.csv")
with open(variables_csv, "a", encoding="utf8") as file:
file.write("some-study,some-dataset,a-variable,,")
TEST_CASE.assertEqual(1, Variable.objects.count())
with TEST_CASE.assertRaises(SystemExit) as _exit:
call_command("update", study.name, "concepts", "-l")
TEST_CASE.assertEqual(0, _exit.exception.code)
with TEST_CASE.assertRaises(SystemExit) as _exit:
call_command("update", study.name, "variables", "-l")
TEST_CASE.assertEqual(0, _exit.exception.code)
TEST_CASE.assertEqual(2, Concept.objects.count())
def test_update_command_with_valid_study_name_and_valid_entity_and_filename(
study, option):
file_path = Study().import_path().joinpath(
"instruments/some-instrument.json")
with TEST_CASE.assertRaises(Instrument.DoesNotExist):
Instrument.objects.get(name="some-instrument")
with unittest.mock.patch(
"ddionrails.imports.manager.Repository") as git_api:
with TEST_CASE.assertRaises(SystemExit) as error:
call_command("update", study.name, "instruments.json", option,
file_path)
git_api: unittest.mock.MagicMock
git_api.pull_or_clone.assert_called()
git_api.set_commit_id.assert_called()
TEST_CASE.assertEqual(0, error.exception.code)
Instrument.objects.get(name="some-instrument")
def _import_single_variable(variable: Dict[str, str], study: Study) -> None:
"""Import statistics data for a single defined value."""
try:
variable_object = Variable.objects.get(
name=variable["name"],
dataset__name=variable["dataset"],
dataset__study=study)
except Exception as error:
raise ValueError(f"{variable}") from error
import_base_path: Path = study.import_path().parent
statistics_base_path = import_base_path.joinpath("statistics")
if variable["type"] in ["numerical", "categorical"]:
_import_single_type(variable_object, statistics_base_path,
variable["type"])
if variable["type"] == "ordinal":
_import_single_type(variable_object, statistics_base_path, "numerical")
_import_single_type(variable_object, statistics_base_path,
"categorical")
def remove_from_database(self, study: Study) -> None:
""" Remove the study and all related objects from the database """
study.delete()
self.log_success(
f'Study "{study.name}" succesfully removed from database.')
def __init__(self, study: Study, redis: bool = True):
self.study = study
self.repo = Repository(study)
self.base_dir = study.import_path()
self._concepts_fixed = False
self.redis = redis
self.import_order = OrderedDict(
{
"topics.csv": (TopicImport, self.base_dir / "topics.csv"),
"topics.json": (TopicJsonImport, self.base_dir / "topics.json"),
"concepts": (ConceptImport, self.base_dir / "concepts.csv"),
"analysis_units": (
AnalysisUnitImport,
self.base_dir / "analysis_units.csv",
),
"periods": (PeriodImport, self.base_dir / "periods.csv"),
"conceptual_datasets": (
ConceptualDatasetImport,
self.base_dir / "conceptual_datasets.csv",
),
"instruments.json": (
instrument_import.InstrumentImport,
Path(self.base_dir / "instruments/").glob("*.json"),
),
"instruments": (
instrument_import.instrument_import,
Path(self.base_dir / "instruments.csv"),
),
"questions": (
question_import.question_import,
self.base_dir / "questions.csv",
),
"answers": (question_import.answer_import, self.base_dir / "answers.csv"),
"answers_relations": (
question_import.answer_relation_import,
self.base_dir / "answers.csv",
),
"datasets.csv": (DatasetImport, self.base_dir / "datasets.csv"),
"datasets.json": (
DatasetJsonImport,
Path(self.base_dir / "datasets/").glob("*.json"),
),
"variables": (VariableImport, self.base_dir / "variables.csv"),
"questions_variables": (
question_variable_import.QuestionVariableImport,
self.base_dir / "questions_variables.csv",
),
"concepts_questions": (
concept_question_import.ConceptQuestionImport,
self.base_dir / "questions.csv",
),
"transformations": (
TransformationImport,
self.base_dir / "transformations.csv",
),
"attachments": (AttachmentImport, self.base_dir / "attachments.csv"),
"publications": (PublicationImport, self.base_dir / "publications.csv"),
"study": (StudyDescriptionImport, self.base_dir / "study.md"),
"statistics": (statistics_import, self.base_dir / "variables.csv"),
"questions_images": (
question_image_import.questions_images_import,
self.base_dir.joinpath("questions_images.csv"),
),
"variables_images": (
variables_images_import,
self.base_dir.joinpath("variables_images.csv"),
),
}
)
class TestStudyImportManagerUnittest(unittest.TestCase):
data_dir: Path
settings: Any
study: Study
def setUp(self) -> None:
self.settings.IMPORT_REPO_PATH = self.data_dir
self.study = Study(name="some-study")
self.study.save()
self.study_import_manager = StudyImportManager(study=self.study,
redis=False)
return super().setUp()
def test_import_csv_topics_exception(self):
import_path: Path = self.study_import_manager.study.import_path()
faulty_row = {
"study": "some-nonexistent-study",
"name": "some-topic",
"label": "some-label",
"label_de": "some-german-label",
"description": "Some description",
"description_de": "Eine Beschreibung",
"parent": "some-other-topic",
}
with open(import_path.joinpath("topics.csv"), "a",
encoding="utf8") as topic_file:
writer = csv.DictWriter(topic_file,
fieldnames=list(faulty_row.keys()))
writer.writerow(faulty_row)
with self.assertRaises(Study.DoesNotExist):
self.study_import_manager.import_single_entity("topics.csv")
def test_import_attachments_exception(self):
TEST_CASE.assertEqual(0, Attachment.objects.count())
import_path = self.study_import_manager.study.import_path().joinpath(
"attachments.csv")
header = (
"type",
"study",
"dataset",
"variable",
"instrument",
"question",
"url",
"url_text",
)
row = dict(type="dataset", dataset="Nonexistent-dataset")
with open(import_path, "w", encoding="utf8") as attachements_file:
writer = csv.DictWriter(attachements_file, fieldnames=header)
writer.writeheader()
writer.writerow(row)
with TEST_CASE.assertRaises(Dataset.DoesNotExist) as error:
self.study_import_manager.import_single_entity("attachments")
error_dict = json.loads(error.exception.args[0])
TEST_CASE.assertDictContainsSubset(row, error_dict)
def test_import_attachments(self):
TEST_CASE.assertEqual(0, Attachment.objects.count())
self.study_import_manager.import_single_entity("attachments")
TEST_CASE.assertEqual(1, Attachment.objects.count())
attachment = Attachment.objects.first()
TEST_CASE.assertEqual(self.study, attachment.context_study)
TEST_CASE.assertEqual("https://some-study.de", attachment.url)
TEST_CASE.assertEqual("some-study", attachment.url_text)