def test_resume_migration(self):
        with tempdir() as tmp:
            filename = join(tmp, "file.txt")
            migrator = MIGRATIONS[self.slug]()
            migrated1, skipped = migrator.migrate(filename)
            self.assertGreaterEqual(migrated1, self.test_size)
            self.assertFalse(skipped)

            # discard state to simulate interrupted migration
            for mig in migrator.iter_migrators():
                mig.get_document_provider().get_document_iterator(
                    1).discard_state()

            # resumed migration: all docs already migrated, so BlobMeta records
            # exist, but should not cause errors on attempting to insert them
            migrated2, skipped = MIGRATIONS[self.slug]().migrate(filename)
            self.assertEqual(migrated1, migrated2)
            self.assertFalse(skipped)

            mod.BlobMigrationState.objects.get(slug=self.slug)

        parent_ids = chain(
            (doc.id for doc in self.couch_docs),
            (doc.parent_id for doc in self.sql_docs),
        )

        # should have one blob per parent
        for parent_id in parent_ids:
            metas = list(
                BlobMeta.objects.partitioned_query(parent_id).filter(
                    parent_id=parent_id))
            self.assertEqual(len(metas), 1, metas)
Example #2
0
    def test_migrate_backend(self):
        # verify: migration not complete
        with maybe_not_found():
            self.assertEqual(os.listdir(self.db.new_db.rootdir), [])

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug]().migrate(filename, num_workers=2)
            self.assertGreaterEqual(migrated, self.test_size)

            verify_migration(self, self.slug, filename, self.not_founds)

        # verify: blobs were copied to new blob db
        not_found = set(t[0] for t in self.not_founds)
        for meta in self.blob_metas:
            if meta.id in not_found:
                with self.assertRaises(mod.NotFound):
                    meta.open(self.db.new_db)
                continue
            content = meta.open(self.db.new_db)
            data = content.read()
            self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
            self.assertEqual(len(data), meta.content_length)
Example #3
0
 def __enter__(self):
     from editxt.application import Application
     self.tempdir = tempdir()
     self.tmp = os.path.realpath(self.tempdir.__enter__())
     profile_path = os.path.join(self.tmp, ".profile")
     app = Application(profile_path)
     app.syntax_factory = {}
     if self.config is not None:
         def update(data, key, value):
             if "." in key:
                 base, sub = key.split(".", 1)
                 if base not in data:
                     data[base] = {}
                 update(data[base], sub, value)
             else:
                 data[key] = value
         for key, value in self.config.items():
             assert isinstance(key, str), key
             update(app.config.data, key, value)
     self.items = {}
     self.news = 0
     self.app = app
     self._setup(app)
     app.__test_app = self
     return app
Example #4
0
def test_ag_completions():
    with tempdir() as tmp:
        (Path(tmp) / "dir").mkdir()

        @gentest
        @async_test
        async def test(cmd, description, label="", project_path=None, items=()):
            editor = FakeEditor(join(tmp, "dir/file"), project_path or tmp)
            editor.selection = "b "
            result = await get_completions(cmd, editor)
            items = list(items)
            if description is not None:
                items.insert(0, {
                    "label": label or cmd,
                    "description": description,
                    "offset": 0,
                })
            eq(result["items"], items)
            eq(result["value"], cmd)

        yield test("ag ", "'b ' /dir options ...", "ag", project_path="/dir")
        yield test("ag 'x ", "/dir options ...", "ag 'x '", project_path="/dir")
        yield test("ag x dir/", "options ...")
        yield test("ag x dir/ ", "options ...", "ag x dir/")
        yield test("ag x dir/  ", "options ...", "ag x dir/")
        yield test("ag x ../", "options ...", items=[
            {"label": "dir/", "is_completion": True, "offset": 8},
        ])
    def test_non_swaps(self):
        # Test a form with multiple dups
        form_with_multi_dups_id = self._submit_form()._id
        self._submit_form(form_with_multi_dups_id)
        self._submit_form(form_with_multi_dups_id)
        # Test a form with no dups
        another_form_id = self._submit_form()._id

        with tempdir() as tmp:
            ids_file_path = os.path.join(tmp, 'ids')

            with open(ids_file_path, "w") as ids_file:
                for id_ in (form_with_multi_dups_id, another_form_id):
                    ids_file.write("{} {}\n".format(DOMAIN, id_))

            call_command('swap_duplicate_xforms',
                         ids_file_path,
                         '/dev/null',
                         no_input=True)

            form_with_multi_dups = XFormInstance.get(form_with_multi_dups_id)
            self.assertEqual(form_with_multi_dups.doc_type, "XFormInstance")

            another_form = XFormInstance.get(another_form_id)
            self.assertEqual(another_form.doc_type, "XFormInstance")
Example #6
0
    def test_simple_swap(self):
        # Test a form with a single dup
        bad_form_id = self._submit_form()._id
        good_dup_id = self._submit_form(bad_form_id)._id

        with tempdir() as tmp:
            ids_file_path = os.path.join(tmp, 'ids')

            with open(ids_file_path, "w") as ids_file:
                ids_file.write("{} {}\n".format(DOMAIN, bad_form_id))

            call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)
            # Throw in a second call to the script to test idempotence as well
            call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)

            bad_form = XFormInstance.get(bad_form_id)
            self.assertEqual(bad_form.doc_type, "XFormDuplicate")
            self.assertRegexpMatches(
                bad_form.problem, BAD_FORM_PROBLEM_TEMPLATE.format(good_dup_id, "")
            )

            good_dup_form = XFormInstance.get(good_dup_id)
            self.assertEqual(good_dup_form.doc_type, "XFormInstance")
            self.assertRegexpMatches(
                good_dup_form.problem, FIXED_FORM_PROBLEM_TEMPLATE.format(
                    id_=bad_form_id, datetime_=""
                )
            )
    def test_resume_migration(self):

        class IncompleteDPC(migrate.DocumentProcessorController):

            def _processing_complete(self):
                self.document_iterator.discard_state()

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")
            with replattr(migrate, "DocumentProcessorController", IncompleteDPC):
                # interrupted migration
                migrated1, skipped = MIGRATIONS[self.slug].migrate(filename)
                self.assertGreaterEqual(migrated1, self.test_size)
                self.assertFalse(skipped)

            # resumed migration: all docs already migrated, so BlobMeta records
            # exist, but should not cause errors on attempting to insert them
            migrated2, skipped = MIGRATIONS[self.slug].migrate(filename)
            self.assertEqual(migrated1, migrated2)
            self.assertFalse(skipped)

            mod.BlobMigrationState.objects.get(slug=self.slug)

        parent_ids = chain(
            (doc.id for doc in self.couch_docs),
            (doc.parent_id for doc in self.sql_docs),
        )

        # should have one blob per parent
        for parent_id in parent_ids:
            db = get_db_alias_for_partitioned_doc(parent_id)
            metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id))
            self.assertEqual(len(metas), 1, metas)
    def test_resume_migration(self):
        with tempdir() as tmp:
            filename = join(tmp, "file.txt")
            migrator = MIGRATIONS[self.slug]
            migrated1, skipped = migrator.migrate(filename)
            self.assertGreaterEqual(migrated1, self.test_size)
            self.assertFalse(skipped)

            # discard state to simulate interrupted migration
            for mig in migrator.iter_migrators():
                mig.get_document_provider().get_document_iterator(1).discard_state()

            # resumed migration: all docs already migrated, so BlobMeta records
            # exist, but should not cause errors on attempting to insert them
            migrated2, skipped = MIGRATIONS[self.slug].migrate(filename)
            self.assertEqual(migrated1, migrated2)
            self.assertFalse(skipped)

            mod.BlobMigrationState.objects.get(slug=self.slug)

        parent_ids = chain(
            (doc.id for doc in self.couch_docs),
            (doc.parent_id for doc in self.sql_docs),
        )

        # should have one blob per parent
        for parent_id in parent_ids:
            db = get_db_alias_for_partitioned_doc(parent_id)
            metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id))
            self.assertEqual(len(metas), 1, metas)
Example #9
0
    def test_migrate_backend(self):
        # verify: attachment is in couch and migration not complete
        with maybe_not_found():
            s3_blobs = sum(1 for b in self.db.new_db._s3_bucket().objects.all())
            self.assertEqual(s3_blobs, 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            ids = {d._id for d in self.migrate_docs}
            migrated = {d["_id"] for d in (json.loads(x) for x in lines)}
            self.assertEqual(len(ids.intersection(migrated)), self.test_size)

        # verify: attachment was copied to new blob db
        for doc in self.migrate_docs:
            exp = SavedBasicExport.get(doc._id)
            self.assertEqual(exp._rev, doc._rev)  # rev should not change
            self.assertTrue(doc.blobs)
            bucket = doc._blobdb_bucket()
            for meta in doc.blobs.values():
                content = self.db.new_db.get(meta.id, bucket)
                self.assertEqual(len(content.read()), meta.content_length)
Example #10
0
    def test_fix_xforms_with_missing_xmlns_task_fixed(self):
        """Tests the ability to fix xforms with the periodic cron task
        """

        good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form(
        )
        # Fix bad builds
        for bad_xform in bad_xforms:
            app = Application.get(bad_xform.build_id)
            for form in app.get_forms():
                if form.xmlns == 'undefined':
                    form.xmlns = 'my-fixed-xmlns'
            app.save()

        self._refresh_pillow()

        with tempdir() as tmp:
            with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR',
                       tmp):
                with patch('corehq.apps.cleanup.tasks.mail_admins_async'
                           ) as mocked_mail:
                    stats, log_file_path = fix_xforms_with_missing_xmlns()
                    self.assertTrue(mocked_mail.delay.called)

        self.assertTrue(stats['fixed'][DOMAIN], len(bad_xforms))
Example #11
0
    def test_migrate_backend(self):
        # verify: attachment is in couch and migration not complete
        with maybe_not_found():
            s3_blobs = sum(1
                           for b in self.db.new_db._s3_bucket().objects.all())
            self.assertEqual(s3_blobs, 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            ids = {d._id for d in self.migrate_docs}
            migrated = {d["_id"] for d in (json.loads(x) for x in lines)}
            self.assertEqual(len(ids.intersection(migrated)), self.test_size)

        # verify: attachment was copied to new blob db
        for doc in self.migrate_docs:
            exp = SavedBasicExport.get(doc._id)
            self.assertEqual(exp._rev, doc._rev)  # rev should not change
            self.assertTrue(doc.blobs)
            bucket = doc._blobdb_bucket()
            for meta in doc.blobs.values():
                content = self.db.new_db.get(meta.id, bucket)
                self.assertEqual(len(content.read()), meta.content_length)
    def _test_compress(self, expected_count, domain=None):
        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug]().migrate(
                filename, num_workers=2, domain=domain)
            self.assertGreaterEqual(migrated, expected_count)

            not_founds = {
                nf
                for nf in self.not_founds if not domain or nf[1] == domain
            }
            verify_migration(self, self.slug, filename, not_founds)

        # verify: blobs were compressed
        not_found = set(t[0] for t in self.not_founds)
        for meta in self.blob_metas:
            if domain and meta.domain != domain:
                continue
            meta.refresh_from_db()
            if meta.id in not_found:
                with self.assertRaises(mod.NotFound):
                    meta.open(self.db)
                continue
            content = meta.open(self.db)
            data = content.read()
            self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
            self.assertEqual(len(data), meta.content_length)
            if meta.type_code == CODES.form_xml:
                self.assertGreaterEqual(meta.compressed_length, 1)
            else:
                self.assertIsNone(meta.compressed_length)
Example #13
0
async def test_ag_help():
    with tempdir() as tmp:
        command = "ag x . --help"
        editor = FakeEditor(join(tmp, "file"))
        result = await do_command(command, editor)
        assert len(result["items"]) > 1, result
        result.pop("items")
        eq(result["value"], None)
Example #14
0
async def test_ag_not_installed():
    with tempdir() as tmp:
        command = "ag x xxxx"
        editor = FakeEditor(join(tmp, "file"))
        editor.ag_path = ag_path = join(tmp, "ag")
        result = await do_command(command, editor)
        eq(result["type"], "error")
        eq(result["message"], Regex(f"{ag_path} not found. "))
Example #15
0
    def test(relpath):
        with tempdir() as base:
            filepath = Path(base) / "file.txt"
            filepath.touch()
            assert filepath.exists(), filepath

            path = join(base, relpath)
            mod.create_new_file(path)
            assert exists(path)
Example #16
0
async def test_too_many_results():
    with tempdir() as tmp:
        os.mkdir(join(tmp, "dir"))
        with open(join(tmp, "dir/long_lines.txt"), "w", encoding="utf-8") as fh:
            fh.write("file.txt\n" * (mod.MAX_RESULT_ITEMS + 1))
        editor = FakeEditor(join(tmp, "dir/file"), tmp)
        result = await do_command("ag txt", editor)
        items = result["items"]
        eq(len(items), mod.MAX_RESULT_ITEMS, result)
Example #17
0
async def test_ag_error():
    with tempdir() as tmp:
        command = "ag x xxxx"
        editor = FakeEditor(join(tmp, "file"))
        result = await do_command(command, editor)
        item, = result["items"]
        eq(item["description"], Regex("No such file or directory:"))
        eq(result["value"], command)
        assert "filter_results" not in result, result
Example #18
0
def make_file(name="file.txt", content="text"):
    if os.path.isabs(name):
        name = name.lstrip(os.path.sep).lstrip("/")
        assert name and not os.path.isabs(name), name
    with tempdir() as tmp:
        path = os.path.join(tmp, name)
        if content is not None:
            with open(path, "w", encoding="utf-8") as file:
                file.write(content)
        yield path
Example #19
0
async def test_huge_results():
    with tempdir() as tmp:
        os.mkdir(join(tmp, "dir"))
        with open(join(tmp, "dir/long_lines.txt"), "w", encoding="utf-8") as fh:
            fh.write("xyz " + "file.txt " * 100)
        editor = FakeEditor(join(tmp, "dir/file"), tmp)
        result = await do_command("ag xyz", editor)
        items = result["items"]
        eq(len(items[0]["label"]), mod.MAX_LINE_LENGTH - 3, result)
        eq(len(items), 1, result)
Example #20
0
    def test_write_export_from_all_log_events(self):
        def unpack(row):
            return {k: row[k] for k in ["Date", "Type", "User", "Description"]}

        start = datetime(2021, 2, 1)
        end = datetime(2021, 2, 1)
        with tempdir() as tmp:
            filename = os.path.join(tmp, "events.csv")
            with open(filename, 'w', encoding="utf-8") as csvfile:
                write_export_from_all_log_events(csvfile, start=start, end=end)
            with open(filename, encoding="utf-8") as fh:

                def key(row):
                    return row["Date"], row["Type"]

                rows = DictReader(fh)
                items = sorted([unpack(r) for r in rows], key=key)
                self.assertEqual(items, [
                    {
                        'Date': '2021-02-01 02:00:00',
                        'Type': 'AccessAudit',
                        'User': '******',
                        'Description': 'Login Success',
                    },
                    {
                        'Date': '2021-02-01 02:00:00',
                        'Type': 'NavigationEventAudit',
                        'User': '******',
                        'Description': 'User Name',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'AccessAudit',
                        'User': '******',
                        'Description': 'Login: [email protected]',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'AccessAudit',
                        'User': '******',
                        'Description': 'Logout: [email protected]',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'NavigationEventAudit',
                        'User': '******',
                        'Description': '*****@*****.**',
                    },
                    {
                        'Date': '2021-02-01 03:00:00',
                        'Type': 'NavigationEventAudit',
                        'User': '******',
                        'Description': '*****@*****.**',
                    },
                ])
Example #21
0
def fake_editor(folders=()):
    with tempdir() as tmp:
        base = Path(tmp) / "base"
        base.mkdir()
        (base / "file.txt").touch()
        (base / "dir").mkdir()
        (base / "dir/file.txt").touch()
        for i, folder in enumerate(folders):
            (base / folder).mkdir()
            (base / folder / f"file{i}.txt").touch()
        yield FakeEditor(str(base / "file.txt"), str(base))
Example #22
0
    def test_normal_app(self):
        form, xform = self.build_normal_app()
        self._refresh_pillow()

        with tempdir() as tmp:
            log_file = os.path.join(tmp, 'log.txt')

            call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)

            with open(log_file) as log_file:
                log = log_file.read()
                self.assertTrue(form.unique_id not in log)
                self.assertTrue(xform._id not in log)
    def test_normal_app(self):
        form, xform = self.build_normal_app()
        self._refresh_pillow()

        with tempdir() as tmp:
            log_file = os.path.join(tmp, 'log.txt')

            call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)

            with open(log_file) as log_file:
                log = log_file.read()
                self.assertTrue(form.unique_id not in log)
                self.assertTrue(xform._id not in log)
    def test_app_with_recently_fixed_form(self):
        form, good_build, bad_build, good_xform, bad_xform = self.build_app_with_recently_fixed_form()
        self._refresh_pillow()

        with tempdir() as tmp:
            log_file = os.path.join(tmp, 'log.txt')

            call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)

            with open(log_file) as log_file:
                log = log_file.read()
                self.assertTrue(good_build._id not in log)
                self.assertTrue(bad_build._id in log)
                self.assertTrue(good_xform._id not in log)
                self.assertTrue(bad_xform._id in log)
            self.assertNoMissingXmlnss()
Example #25
0
def test_get_blobdb(self, msg, root=True, blob_dir=None):
    with tempdir() as tmp:
        if root == "file":
            tmp = join(tmp, "file")
            with open(tmp, "w") as fh:
                fh.write("x")
        conf = SharedDriveConfiguration(
            shared_drive_path=tmp if root else root,
            restore_dir=None,
            transfer_dir=None,
            temp_dir=None,
            blob_dir=blob_dir,
        )
        with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None):
            with assert_raises(mod.Error, msg=re.compile(msg)):
                mod.get_blob_db()
Example #26
0
    def test_app_with_recently_fixed_form(self):
        form, good_build, bad_build, good_xform, bad_xform = self.build_app_with_recently_fixed_form()
        self._refresh_pillow()

        with tempdir() as tmp:
            log_file = os.path.join(tmp, 'log.txt')

            call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)

            with open(log_file) as log_file:
                log = log_file.read()
                self.assertTrue(good_build._id not in log)
                self.assertTrue(bad_build._id in log)
                self.assertTrue(good_xform._id not in log)
                self.assertTrue(bad_xform._id in log)
            self.assertNoMissingXmlnss()
Example #27
0
def setup_files():
    def do_setup(tmp):
        os.mkdir(join(tmp, "dir"))
        for path in [
            "dir/a.txt",
            "dir/b.txt",
            "dir/B file",
            "e.txt",
        ]:
            assert not isabs(path), path
            with open(join(tmp, path), "w") as fh:
                fh.write("name: {}\nsize: {}".format(path, len(path)))
        assert " " not in tmp, tmp

    with tempdir() as tmp:
        do_setup(tmp)
        yield tmp
Example #28
0
def test_get_blobdb(self, msg, root=True, blob_dir=None):
    with tempdir() as tmp:
        if root == "file":
            tmp = join(tmp, "file")
            with open(tmp, "w", encoding='utf-8') as fh:
                fh.write("x")
        conf = SharedDriveConfiguration(
            shared_drive_path=tmp if root else root,
            restore_dir=None,
            transfer_dir=None,
            temp_dir=None,
            blob_dir=blob_dir,
        )
        with patch("corehq.blobs._db", new=[]):
            with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None):
                with assert_raises(mod.Error, msg=re.compile(msg)):
                    mod.get_blob_db()
Example #29
0
def test_get_blobdb(self, msg, root=True, blob_dir=None):
    with tempdir() as tmp:
        if (root == "file" and six.PY3) or (root == b"file" and six.PY2):
            tmp = join(tmp, "file" if six.PY3 else b"file")
            with open(tmp, "w", encoding='utf-8') as fh:
                fh.write("x")
        conf = SharedDriveConfiguration(
            shared_drive_path=tmp if root else root,
            restore_dir=None,
            transfer_dir=None,
            temp_dir=None,
            blob_dir=blob_dir,
        )
        with patch("corehq.blobs._db", new=[]):
            with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None):
                with assert_raises(mod.Error, msg=re.compile(msg)):
                    mod.get_blob_db()
    def test_fix_xforms_with_missing_xmlns_task(self):
        """Tests the ability to find anomalies that violate our asssumptions
        about all applications and builds being fixes
        """

        good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form()
        self._refresh_pillow()

        with tempdir() as tmp:
            with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp):
                with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail:
                    stats, log_file_path = fix_xforms_with_missing_xmlns()
                    self.assertTrue(mocked_mail.delay.called)

            self.assertTrue(
                (DOMAIN, bad_xforms[0].build_id) in stats['builds_with_undefined_xmlns']
            )
            self.assertEqual(stats['not_fixed_undefined_xmlns'][DOMAIN], len(bad_xforms))
Example #31
0
    def test_fix_xforms_with_missing_xmlns_task(self):
        """Tests the ability to find anomalies that violate our asssumptions
        about all applications and builds being fixes
        """

        good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form()
        self._refresh_pillow()

        with tempdir() as tmp:
            with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp):
                with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail:
                    stats, log_file_path = fix_xforms_with_missing_xmlns()
                    self.assertTrue(mocked_mail.delay.called)

            self.assertTrue(
                (DOMAIN, bad_xforms[0].build_id) in stats['builds_with_undefined_xmlns']
            )
            self.assertEqual(stats['not_fixed_undefined_xmlns'][DOMAIN], len(bad_xforms))
    def test_migrate_backend(self):
        with tempdir() as tmp:
            filename = join(tmp, "file.txt")
            # do migration
            migrated, skipped = MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

        # verify: metadata was saved in BlobMeta table

        for doc in self.couch_docs:
            obj = doc.class_.get(doc.id)
            self.assertEqual(obj._rev,
                             doc.data["_rev"])  # rev should not change
            self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"]))
            db = get_db_alias_for_partitioned_doc(obj._id)
            metas = {
                meta.name: meta
                for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id)
            }
            for name, meta in obj.blobs.items():
                blobmeta = metas[name]
                dbname = doc.class_.get_db().dbname
                key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower())
                self.assertEqual(blobmeta.key, key, doc)
                self.assertEqual(blobmeta.domain, doc.domain, doc)
                self.assertEqual(blobmeta.content_type, meta.content_type, doc)
                self.assertEqual(blobmeta.content_length, meta.content_length,
                                 doc)

        for doc in self.sql_docs:
            db = get_db_alias_for_partitioned_doc(doc.parent_id)
            blobmeta = BlobMeta.objects.using(db).get(
                parent_id=doc.parent_id,
                type_code=doc.type_code,
                name="",
            )
            self.assertEqual(blobmeta.domain, doc.domain, doc)
            self.assertEqual(blobmeta.content_type, doc.content_type, doc)
            self.assertEqual(blobmeta.content_length, doc.content_length, doc)
Example #33
0
    def test_migrate_backend(self):
        # verify: migration not complete
        with maybe_not_found():
            self.assertEqual(os.listdir(self.db.new_db.rootdir), [])

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(
                filename, num_workers=2)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: missing blobs written to log files
            missing_log = set()
            fields = [
                "blobmeta_id",
                "domain",
                "type_code",
                "parent_id",
                "blob_key",
            ]
            with open(filename, encoding='utf-8') as fh:
                for line in fh:
                    doc = json.loads(line)
                    missing_log.add(tuple(doc[x] for x in fields))
            self.assertEqual(self.not_founds, missing_log)

        # verify: blobs were copied to new blob db
        not_found = set(t[0] for t in self.not_founds)
        for meta in self.blob_metas:
            if meta.id in not_found:
                with self.assertRaises(mod.NotFound):
                    self.db.new_db.get(key=meta.key)
                continue
            content = self.db.new_db.get(key=meta.key)
            data = content.read()
            self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
            self.assertEqual(len(data), meta.content_length)
Example #34
0
    def do_migration(self, docs, num_attachments=1):
        self.docs_to_delete.extend(docs)
        test_types = {d.doc_type for d in docs}
        if test_types != self.doc_types:
            raise Exception(
                "bad test: must have at least one document per doc "
                "type (got: {})".format(test_types))
        if not num_attachments:
            raise Exception("bad test: must have at least one attachment")

        for doc in docs:
            # verify: attachment is in couch and migration not complete
            self.assertEqual(len(doc._attachments), num_attachments)
            self.assertEqual(len(doc.external_blobs), 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, len(docs))

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)}
            for doc in docs:
                self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev)
            self.assertEqual(len(lines), migrated, lines)

        for doc in docs:
            # verify: attachments were moved to blob db
            exp = type(doc).get(doc._id)
            self.assertEqual(exp.doc_type, doc.doc_type)
            self.assertNotEqual(exp._rev, doc._rev)
            self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs))
            self.assertFalse(exp._attachments, exp._attachments)
            self.assertEqual(len(exp.external_blobs), num_attachments)
Example #35
0
    def test_migrate_backend(self):
        # verify: migration not complete
        with maybe_not_found():
            self.assertEqual(os.listdir(self.db.new_db.rootdir), [])

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: missing blobs written to log files
            missing_log = set()
            fields = ["doc_type", "doc_id", "blob_identifier", "blob_bucket"]
            for n, ignore in enumerate(mod.MIGRATIONS[self.slug].migrators):
                with open("{}.{}".format(filename, n)) as fh:
                    for line in fh:
                        doc = json.loads(line)
                        missing_log.add(tuple(doc[x] for x in fields))
            self.assertEqual(
                len(self.not_founds.intersection(missing_log)),
                len(self.not_founds)
            )

        # verify: couch attachments were copied to new blob db
        for doc in self.couch_docs:
            exp = type(doc).get(doc._id)
            self.assertEqual(exp._rev, doc._rev)  # rev should not change
            self.assertTrue(doc.blobs)
            bucket = doc._blobdb_bucket()
            for name, meta in doc.blobs.items():
                if name == "found.not":
                    continue
                content = self.db.new_db.get(meta.id, bucket)
                data = content.read()
                self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
                self.assertEqual(len(data), meta.content_length)
Example #36
0
    def test_migrate_backend(self):
        # verify: migration not complete
        with maybe_not_found():
            self.assertEqual(os.listdir(self.db.new_db.rootdir), [])

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename, num_workers=2)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: missing blobs written to log files
            missing_log = set()
            fields = [
                "blobmeta_id",
                "domain",
                "type_code",
                "parent_id",
                "blob_key",
            ]
            with open(filename, encoding='utf-8') as fh:
                for line in fh:
                    doc = json.loads(line)
                    missing_log.add(tuple(doc[x] for x in fields))
            self.assertEqual(self.not_founds, missing_log)

        # verify: blobs were copied to new blob db
        not_found = set(t[0] for t in self.not_founds)
        for meta in self.blob_metas:
            if meta.id in not_found:
                with self.assertRaises(mod.NotFound):
                    self.db.new_db.get(key=meta.key)
                continue
            content = self.db.new_db.get(key=meta.key)
            data = content.read()
            self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
            self.assertEqual(len(data), meta.content_length)
Example #37
0
    def do_migration(self, docs, num_attachments=1):
        self.docs_to_delete.extend(docs)
        test_types = {d.doc_type for d in docs}
        if test_types != self.doc_types:
            raise Exception("bad test: must have at least one document per doc "
                            "type (got: {})".format(test_types))
        if not num_attachments:
            raise Exception("bad test: must have at least one attachment")

        for doc in docs:
            # verify: attachment is in couch and migration not complete
            self.assertEqual(len(doc._attachments), num_attachments)
            self.assertEqual(len(doc.external_blobs), 0)

        with tempdir() as tmp:
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, len(docs))

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)}
            for doc in docs:
                self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev)
            self.assertEqual(len(lines), migrated, lines)

        for doc in docs:
            # verify: attachments were moved to blob db
            exp = type(doc).get(doc._id)
            self.assertEqual(exp.doc_type, doc.doc_type)
            self.assertNotEqual(exp._rev, doc._rev)
            self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs))
            self.assertFalse(exp._attachments, exp._attachments)
            self.assertEqual(len(exp.external_blobs), num_attachments)
    def test_fix_xforms_with_missing_xmlns_task_fixed(self):
        """Tests the ability to fix xforms with the periodic cron task
        """

        good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form()
        # Fix bad builds
        for bad_xform in bad_xforms:
            app = Application.get(bad_xform.build_id)
            for form in app.get_forms():
                if form.xmlns == 'undefined':
                    form.xmlns = 'my-fixed-xmlns'
            app.save()

        self._refresh_pillow()

        with tempdir() as tmp:
            with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp):
                with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail:
                    stats, log_file_path = fix_xforms_with_missing_xmlns()
                    self.assertTrue(mocked_mail.delay.called)

        self.assertTrue(stats['fixed'][DOMAIN], len(bad_xforms))
Example #39
0
    def test_non_swaps(self):
        # Test a form with multiple dups
        form_with_multi_dups_id = self._submit_form()._id
        self._submit_form(form_with_multi_dups_id)
        self._submit_form(form_with_multi_dups_id)
        # Test a form with no dups
        another_form_id = self._submit_form()._id

        with tempdir() as tmp:
            ids_file_path = os.path.join(tmp, 'ids')

            with open(ids_file_path, "w") as ids_file:
                for id_ in (form_with_multi_dups_id, another_form_id):
                    ids_file.write("{} {}\n".format(DOMAIN, id_))

            call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)

            form_with_multi_dups = XFormInstance.get(form_with_multi_dups_id)
            self.assertEqual(form_with_multi_dups.doc_type, "XFormInstance")

            another_form = XFormInstance.get(another_form_id)
            self.assertEqual(another_form.doc_type, "XFormInstance")
    def test_migrate_backend(self):
        with tempdir() as tmp:
            filename = join(tmp, "file.txt")
            # do migration
            migrated, skipped = MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, self.test_size)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

        # verify: metadata was saved in BlobMeta table

        for doc in self.couch_docs:
            obj = doc.class_.get(doc.id)
            self.assertEqual(obj._rev, doc.data["_rev"])  # rev should not change
            self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"]))
            db = get_db_alias_for_partitioned_doc(obj._id)
            metas = {meta.name: meta
                for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id)}
            for name, meta in obj.blobs.items():
                blobmeta = metas[name]
                dbname = doc.class_.get_db().dbname
                key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower())
                self.assertEqual(blobmeta.key, key, doc)
                self.assertEqual(blobmeta.domain, doc.domain, doc)
                self.assertEqual(blobmeta.content_type, meta.content_type, doc)
                self.assertEqual(blobmeta.content_length, meta.content_length, doc)

        for doc in self.sql_docs:
            db = get_db_alias_for_partitioned_doc(doc.parent_id)
            blobmeta = BlobMeta.objects.using(db).get(
                parent_id=doc.parent_id,
                type_code=doc.type_code,
                name="",
            )
            self.assertEqual(blobmeta.domain, doc.domain, doc)
            self.assertEqual(blobmeta.content_type, doc.content_type, doc)
            self.assertEqual(blobmeta.content_length, doc.content_length, doc)
Example #41
0
def test_default_package():
    with tempdir() as tmp:
        os.mkdir(join(tmp, "dir"))
        os.mkdir(join(tmp, "dir/package"))
        os.mkdir(join(tmp, "dir/other"))
        for path in [
            "dir/package/__init__.py",
            "dir/other/b.txt",
        ]:
            assert not isabs(path), path
            with open(join(tmp, path), "w"):
                pass

        @async_test
        async def test(path, result=""):
            filepath = 'file' if path is None else join(tmp, path)
            editor = FakeEditor(filepath)
            eq(await mod.default_package(editor), result)

        yield test, None
        yield test, "dir/mod.py"
        yield test, "dir/other/mod.py"
        yield test, "dir/package/mod.py", "package"
Example #42
0
    def test_migrate_saved_exports(self):
        # setup data
        saved = SavedBasicExport(configuration=_mk_config())
        saved.save()
        payload = 'something small and simple'
        name = saved.get_attachment_name()
        super(BlobMixin, saved).put_attachment(payload, name)
        saved.save()

        # verify: attachment is in couch and migration not complete
        self.assertEqual(len(saved._attachments), 1)
        self.assertEqual(len(saved.external_blobs), 0)

        with tempdir() as tmp, replattr(SavedBasicExport, "migrating_blobs_from_couch", True):
            filename = join(tmp, "file.txt")

            # do migration
            migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
            self.assertGreaterEqual(migrated, 1)

            # verify: migration state recorded
            mod.BlobMigrationState.objects.get(slug=self.slug)

            # verify: migrated data was written to the file
            with open(filename) as fh:
                lines = list(fh)
            doc = {d["_id"]: d for d in (json.loads(x) for x in lines)}[saved._id]
            self.assertEqual(doc["_rev"], saved._rev)
            self.assertEqual(len(lines), migrated, lines)

        # verify: attachment was moved to blob db
        exp = SavedBasicExport.get(saved._id)
        self.assertNotEqual(exp._rev, saved._rev)
        self.assertEqual(len(exp.blobs), 1, repr(exp.blobs))
        self.assertFalse(exp._attachments, exp._attachments)
        self.assertEqual(len(exp.external_blobs), 1)
        self.assertEqual(exp.get_payload(), payload)
Example #43
0
 def setUpClass(cls):
     cls.tmp_context = tempdir()
     tmp = cls.tmp_context.__enter__()
     cls.filepath = join(tmp, "file.txt")
     with open(cls.filepath, "wb") as fh:
         fh.write(b"data")
Example #44
0
 def setUpClass(cls):
     super(TestCaseDiffProcess, cls).setUpClass()
     cls.tmp = tempdir()
     cls.state_dir = cls.tmp.__enter__()
Example #45
0
def test_File():
    field = File('path')
    eq_(str(field), 'path')
    eq_(repr(field), "File('path')")

    with tempdir() as tmp:
        os.mkdir(join(tmp, "dir"))
        os.mkdir(join(tmp, "space dir"))
        for path in [
                "dir/a.txt",
                "dir/b.txt",
                "dir/B file",
                ".hidden",
                "file.txt",
                "file.doc",
                "space dir/file",
        ]:
            assert not isabs(path), path
            with open(join(tmp, path), "w"):
                pass

        test = make_consume_checker(field)
        yield test, "relative.txt", 0, ("relative.txt", 13)

        test = make_completions_checker(field)
        yield test, "", []

        project_path = join(tmp, "dir")
        editor = FakeEditor(join(tmp, "dir/file.txt"), project_path)
        field = await_coroutine(field.with_context(editor))

        test = make_completions_checker(field)
        yield test, ".../", ["a.txt", "B file", "b.txt"], 4
        with replattr(editor, "_project_path", project_path + "/"):
            yield test, ".../", ["a.txt", "B file", "b.txt"], 4
            yield test, "...//", ["a.txt", "B file", "b.txt"], 5
        with replattr((editor, "_project_path", None),
                      (editor, "_file_path", join(tmp, "space dir/file")),
                      sigcheck=False):
            yield test, "", ["file"], 0
            yield test, "../", ["dir/", "file.doc", "file.txt",
                                "space dir/"], 3
            # yield test, "..//", ["dir", "file.doc", "file.txt", "space dir"], 4
            yield test, "../f", ["file.doc", "file.txt"], 3
            yield test, "../dir/", ["a.txt", "B file", "b.txt"], 7

        test = make_arg_string_checker(field)
        yield test, "/str", "/str"
        yield test, "/a b", '"/a b"'
        yield test, os.path.expanduser("~/a b"), '"~/a b"'
        yield test, join(tmp, "dir/file"), "file"
        yield test, join(tmp, "dir/a b"), '"a b"'
        yield test, join(tmp, "file"), join(tmp, "file")
        yield test, "arg/", Error("not a file: path='arg/'")

        test = make_consume_checker(field)
        yield test, '', 0, (None, 1)
        yield test, 'a', 0, (join(tmp, 'dir/a'), 2)
        yield test, 'abc', 0, (join(tmp, 'dir/abc'), 4)
        yield test, 'abc ', 0, (join(tmp, 'dir/abc'), 4)
        yield test, 'file.txt', 0, (join(tmp, 'dir/file.txt'), 9)
        yield test, '../file.txt', 0, (join(tmp, 'dir/../file.txt'), 12)
        yield test, '/file.txt', 0, ('/file.txt', 10)
        yield test, '~/file.txt', 0, (os.path.expanduser('~/file.txt'), 11)
        yield test, '...', 0, (join(tmp, 'dir'), 4)
        yield test, '.../file.txt', 0, (join(tmp, 'dir/file.txt'), 13)
        yield test, '"ab c"', 0, (join(tmp, 'dir/ab c'), 6)
        yield test, "'ab c'", 0, (join(tmp, 'dir/ab c'), 6)
        yield test, "'ab c/'", 0, (join(tmp, 'dir/ab c/'), 7)

        # completions
        def expanduser(path):
            if path.startswith("~"):
                if len(path) == 1:
                    return tmp
                assert path.startswith("~/"), path
                return tmp + path[1:]
            return path

        @async_test
        async def test(input, output):
            if input.startswith("/"):
                input = tmp + "/"
            with replattr(os.path, "expanduser", expanduser):
                arg = await mod.Arg(field, input, 0, None)
                eq_(await field.get_completions(arg), output)

        yield test, "", ["a.txt", "B file", "b.txt"]
        yield test, "a", ["a.txt"]
        yield test, "a.txt", ["a.txt"]
        yield test, "b", ["B file", "b.txt"]
        yield test, "B", ["B file"]
        yield test, "..", ["../"]
        yield test, "../", ["dir/", "file.doc", "file.txt", "space dir/"]
        yield test, "../.", [".hidden"]
        yield test, "...", [".../"]
        yield test, ".../", ["a.txt", "B file", "b.txt"]
        yield test, "../dir", ["dir/"]
        yield test, "../dir/", ["a.txt", "B file", "b.txt"]
        yield test, "../sp", ["space dir/"]
        yield test, "../space\\ d", ["space dir/"]
        yield test, "../space\\ dir", ["space dir/"]
        yield test, "../space\\ dir/", ["file"]
        yield test, "val", []
        yield test, "/", ["dir/", "file.doc", "file.txt", "space dir/"]
        yield test, "~", ["~/"]
        yield test, "~/", ["dir/", "file.doc", "file.txt", "space dir/"]

        # delimiter completion
        @async_test
        async def test(input, output, start=0):
            arg = await mod.Arg(field, input, 0, None)
            words = await field.get_completions(arg)
            assert all(isinstance(w, CompleteWord) for w in words), \
                repr([w for w in words if not isinstance(w, CompleteWord)])
            eq_([w.complete() for w in words], output)
            eq_([w.start for w in words], [start] * len(words), words)

        yield test, "", ["a.txt ", "B\\ file ", "b.txt "]
        yield test, "x", []
        yield test, "..", ["../"]
        yield test, "../", ["dir/", "file.doc ", "file.txt ",
                            "space\\ dir/"], 3
        yield test, "../dir", ["dir/"], 3
        yield test, "../di", ["dir/"], 3
        yield test, "../sp", ["space\\ dir/"], 3
        yield test, "../space\\ d", ["space\\ dir/"], 3
        yield test, "../space\\ dir", ["space\\ dir/"], 3
        yield test, ".../", ["a.txt ", "B\\ file ", "b.txt "], 4
        yield test, "../space\\ dir/", ["file "], 14
        yield test, "~", ["~/"], None

        field = File('dir', directory=True)
        eq_(str(field), 'dir')
        eq_(repr(field), "File('dir', directory=True)")
        field = await_coroutine(field.with_context(editor))

        test = make_consume_checker(field)
        yield test, '', 0, (None, 1)
        yield test, 'a', 0, (join(tmp, 'dir/a'), 2)
        yield test, 'abc', 0, (join(tmp, 'dir/abc'), 4)
        yield test, 'abc ', 0, (join(tmp, 'dir/abc'), 4)
        yield test, 'abc/', 0, (join(tmp, 'dir/abc/'), 5)
        yield test, '...', 0, (join(tmp, 'dir'), 4)
        yield test, '.../abc/', 0, (join(tmp, 'dir/abc/'), 9)

        test = make_completions_checker(field)
        yield test, "", [], 0
        yield test, "a", [], 0
        yield test, "..", ["../"], 0
        yield test, "../", ["dir/", "space dir/"], 3

        test = make_arg_string_checker(field)
        yield test, "/a", "/a"
        yield test, "/a/", "/a/"
        yield test, "/dir/a", "/dir/a"
        yield test, "/dir/a/", "/dir/a/"

        field = File('dir', default="~/dir")
        check = make_completions_checker(field)

        def test(input, output, *args):
            if input.startswith("/"):
                input = tmp + "/"
            with replattr(os.path, "expanduser", expanduser):
                check(input, output, *args)

        yield test, "", [], 0

        test = make_placeholder_checker(field)
        yield test, "", 0, ("", "~/dir")
        yield test, " ", 0, ("~/dir", "")
Example #46
0
def setup_module():
    global _tmp, state_dir
    _tmp = tempdir()
    state_dir = _tmp.__enter__()