def test_resume_migration(self): with tempdir() as tmp: filename = join(tmp, "file.txt") migrator = MIGRATIONS[self.slug]() migrated1, skipped = migrator.migrate(filename) self.assertGreaterEqual(migrated1, self.test_size) self.assertFalse(skipped) # discard state to simulate interrupted migration for mig in migrator.iter_migrators(): mig.get_document_provider().get_document_iterator( 1).discard_state() # resumed migration: all docs already migrated, so BlobMeta records # exist, but should not cause errors on attempting to insert them migrated2, skipped = MIGRATIONS[self.slug]().migrate(filename) self.assertEqual(migrated1, migrated2) self.assertFalse(skipped) mod.BlobMigrationState.objects.get(slug=self.slug) parent_ids = chain( (doc.id for doc in self.couch_docs), (doc.parent_id for doc in self.sql_docs), ) # should have one blob per parent for parent_id in parent_ids: metas = list( BlobMeta.objects.partitioned_query(parent_id).filter( parent_id=parent_id)) self.assertEqual(len(metas), 1, metas)
def test_migrate_backend(self): # verify: migration not complete with maybe_not_found(): self.assertEqual(os.listdir(self.db.new_db.rootdir), []) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug]().migrate(filename, num_workers=2) self.assertGreaterEqual(migrated, self.test_size) verify_migration(self, self.slug, filename, self.not_founds) # verify: blobs were copied to new blob db not_found = set(t[0] for t in self.not_founds) for meta in self.blob_metas: if meta.id in not_found: with self.assertRaises(mod.NotFound): meta.open(self.db.new_db) continue content = meta.open(self.db.new_db) data = content.read() self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94') self.assertEqual(len(data), meta.content_length)
def __enter__(self): from editxt.application import Application self.tempdir = tempdir() self.tmp = os.path.realpath(self.tempdir.__enter__()) profile_path = os.path.join(self.tmp, ".profile") app = Application(profile_path) app.syntax_factory = {} if self.config is not None: def update(data, key, value): if "." in key: base, sub = key.split(".", 1) if base not in data: data[base] = {} update(data[base], sub, value) else: data[key] = value for key, value in self.config.items(): assert isinstance(key, str), key update(app.config.data, key, value) self.items = {} self.news = 0 self.app = app self._setup(app) app.__test_app = self return app
def test_ag_completions(): with tempdir() as tmp: (Path(tmp) / "dir").mkdir() @gentest @async_test async def test(cmd, description, label="", project_path=None, items=()): editor = FakeEditor(join(tmp, "dir/file"), project_path or tmp) editor.selection = "b " result = await get_completions(cmd, editor) items = list(items) if description is not None: items.insert(0, { "label": label or cmd, "description": description, "offset": 0, }) eq(result["items"], items) eq(result["value"], cmd) yield test("ag ", "'b ' /dir options ...", "ag", project_path="/dir") yield test("ag 'x ", "/dir options ...", "ag 'x '", project_path="/dir") yield test("ag x dir/", "options ...") yield test("ag x dir/ ", "options ...", "ag x dir/") yield test("ag x dir/ ", "options ...", "ag x dir/") yield test("ag x ../", "options ...", items=[ {"label": "dir/", "is_completion": True, "offset": 8}, ])
def test_non_swaps(self): # Test a form with multiple dups form_with_multi_dups_id = self._submit_form()._id self._submit_form(form_with_multi_dups_id) self._submit_form(form_with_multi_dups_id) # Test a form with no dups another_form_id = self._submit_form()._id with tempdir() as tmp: ids_file_path = os.path.join(tmp, 'ids') with open(ids_file_path, "w") as ids_file: for id_ in (form_with_multi_dups_id, another_form_id): ids_file.write("{} {}\n".format(DOMAIN, id_)) call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True) form_with_multi_dups = XFormInstance.get(form_with_multi_dups_id) self.assertEqual(form_with_multi_dups.doc_type, "XFormInstance") another_form = XFormInstance.get(another_form_id) self.assertEqual(another_form.doc_type, "XFormInstance")
def test_simple_swap(self): # Test a form with a single dup bad_form_id = self._submit_form()._id good_dup_id = self._submit_form(bad_form_id)._id with tempdir() as tmp: ids_file_path = os.path.join(tmp, 'ids') with open(ids_file_path, "w") as ids_file: ids_file.write("{} {}\n".format(DOMAIN, bad_form_id)) call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True) # Throw in a second call to the script to test idempotence as well call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True) bad_form = XFormInstance.get(bad_form_id) self.assertEqual(bad_form.doc_type, "XFormDuplicate") self.assertRegexpMatches( bad_form.problem, BAD_FORM_PROBLEM_TEMPLATE.format(good_dup_id, "") ) good_dup_form = XFormInstance.get(good_dup_id) self.assertEqual(good_dup_form.doc_type, "XFormInstance") self.assertRegexpMatches( good_dup_form.problem, FIXED_FORM_PROBLEM_TEMPLATE.format( id_=bad_form_id, datetime_="" ) )
def test_resume_migration(self): class IncompleteDPC(migrate.DocumentProcessorController): def _processing_complete(self): self.document_iterator.discard_state() with tempdir() as tmp: filename = join(tmp, "file.txt") with replattr(migrate, "DocumentProcessorController", IncompleteDPC): # interrupted migration migrated1, skipped = MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated1, self.test_size) self.assertFalse(skipped) # resumed migration: all docs already migrated, so BlobMeta records # exist, but should not cause errors on attempting to insert them migrated2, skipped = MIGRATIONS[self.slug].migrate(filename) self.assertEqual(migrated1, migrated2) self.assertFalse(skipped) mod.BlobMigrationState.objects.get(slug=self.slug) parent_ids = chain( (doc.id for doc in self.couch_docs), (doc.parent_id for doc in self.sql_docs), ) # should have one blob per parent for parent_id in parent_ids: db = get_db_alias_for_partitioned_doc(parent_id) metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id)) self.assertEqual(len(metas), 1, metas)
def test_resume_migration(self): with tempdir() as tmp: filename = join(tmp, "file.txt") migrator = MIGRATIONS[self.slug] migrated1, skipped = migrator.migrate(filename) self.assertGreaterEqual(migrated1, self.test_size) self.assertFalse(skipped) # discard state to simulate interrupted migration for mig in migrator.iter_migrators(): mig.get_document_provider().get_document_iterator(1).discard_state() # resumed migration: all docs already migrated, so BlobMeta records # exist, but should not cause errors on attempting to insert them migrated2, skipped = MIGRATIONS[self.slug].migrate(filename) self.assertEqual(migrated1, migrated2) self.assertFalse(skipped) mod.BlobMigrationState.objects.get(slug=self.slug) parent_ids = chain( (doc.id for doc in self.couch_docs), (doc.parent_id for doc in self.sql_docs), ) # should have one blob per parent for parent_id in parent_ids: db = get_db_alias_for_partitioned_doc(parent_id) metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id)) self.assertEqual(len(metas), 1, metas)
def test_migrate_backend(self): # verify: attachment is in couch and migration not complete with maybe_not_found(): s3_blobs = sum(1 for b in self.db.new_db._s3_bucket().objects.all()) self.assertEqual(s3_blobs, 0) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, self.test_size) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: migrated data was written to the file with open(filename) as fh: lines = list(fh) ids = {d._id for d in self.migrate_docs} migrated = {d["_id"] for d in (json.loads(x) for x in lines)} self.assertEqual(len(ids.intersection(migrated)), self.test_size) # verify: attachment was copied to new blob db for doc in self.migrate_docs: exp = SavedBasicExport.get(doc._id) self.assertEqual(exp._rev, doc._rev) # rev should not change self.assertTrue(doc.blobs) bucket = doc._blobdb_bucket() for meta in doc.blobs.values(): content = self.db.new_db.get(meta.id, bucket) self.assertEqual(len(content.read()), meta.content_length)
def test_fix_xforms_with_missing_xmlns_task_fixed(self): """Tests the ability to fix xforms with the periodic cron task """ good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form( ) # Fix bad builds for bad_xform in bad_xforms: app = Application.get(bad_xform.build_id) for form in app.get_forms(): if form.xmlns == 'undefined': form.xmlns = 'my-fixed-xmlns' app.save() self._refresh_pillow() with tempdir() as tmp: with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp): with patch('corehq.apps.cleanup.tasks.mail_admins_async' ) as mocked_mail: stats, log_file_path = fix_xforms_with_missing_xmlns() self.assertTrue(mocked_mail.delay.called) self.assertTrue(stats['fixed'][DOMAIN], len(bad_xforms))
def _test_compress(self, expected_count, domain=None): with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug]().migrate( filename, num_workers=2, domain=domain) self.assertGreaterEqual(migrated, expected_count) not_founds = { nf for nf in self.not_founds if not domain or nf[1] == domain } verify_migration(self, self.slug, filename, not_founds) # verify: blobs were compressed not_found = set(t[0] for t in self.not_founds) for meta in self.blob_metas: if domain and meta.domain != domain: continue meta.refresh_from_db() if meta.id in not_found: with self.assertRaises(mod.NotFound): meta.open(self.db) continue content = meta.open(self.db) data = content.read() self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94') self.assertEqual(len(data), meta.content_length) if meta.type_code == CODES.form_xml: self.assertGreaterEqual(meta.compressed_length, 1) else: self.assertIsNone(meta.compressed_length)
async def test_ag_help(): with tempdir() as tmp: command = "ag x . --help" editor = FakeEditor(join(tmp, "file")) result = await do_command(command, editor) assert len(result["items"]) > 1, result result.pop("items") eq(result["value"], None)
async def test_ag_not_installed(): with tempdir() as tmp: command = "ag x xxxx" editor = FakeEditor(join(tmp, "file")) editor.ag_path = ag_path = join(tmp, "ag") result = await do_command(command, editor) eq(result["type"], "error") eq(result["message"], Regex(f"{ag_path} not found. "))
def test(relpath): with tempdir() as base: filepath = Path(base) / "file.txt" filepath.touch() assert filepath.exists(), filepath path = join(base, relpath) mod.create_new_file(path) assert exists(path)
async def test_too_many_results(): with tempdir() as tmp: os.mkdir(join(tmp, "dir")) with open(join(tmp, "dir/long_lines.txt"), "w", encoding="utf-8") as fh: fh.write("file.txt\n" * (mod.MAX_RESULT_ITEMS + 1)) editor = FakeEditor(join(tmp, "dir/file"), tmp) result = await do_command("ag txt", editor) items = result["items"] eq(len(items), mod.MAX_RESULT_ITEMS, result)
async def test_ag_error(): with tempdir() as tmp: command = "ag x xxxx" editor = FakeEditor(join(tmp, "file")) result = await do_command(command, editor) item, = result["items"] eq(item["description"], Regex("No such file or directory:")) eq(result["value"], command) assert "filter_results" not in result, result
def make_file(name="file.txt", content="text"): if os.path.isabs(name): name = name.lstrip(os.path.sep).lstrip("/") assert name and not os.path.isabs(name), name with tempdir() as tmp: path = os.path.join(tmp, name) if content is not None: with open(path, "w", encoding="utf-8") as file: file.write(content) yield path
async def test_huge_results(): with tempdir() as tmp: os.mkdir(join(tmp, "dir")) with open(join(tmp, "dir/long_lines.txt"), "w", encoding="utf-8") as fh: fh.write("xyz " + "file.txt " * 100) editor = FakeEditor(join(tmp, "dir/file"), tmp) result = await do_command("ag xyz", editor) items = result["items"] eq(len(items[0]["label"]), mod.MAX_LINE_LENGTH - 3, result) eq(len(items), 1, result)
def test_write_export_from_all_log_events(self): def unpack(row): return {k: row[k] for k in ["Date", "Type", "User", "Description"]} start = datetime(2021, 2, 1) end = datetime(2021, 2, 1) with tempdir() as tmp: filename = os.path.join(tmp, "events.csv") with open(filename, 'w', encoding="utf-8") as csvfile: write_export_from_all_log_events(csvfile, start=start, end=end) with open(filename, encoding="utf-8") as fh: def key(row): return row["Date"], row["Type"] rows = DictReader(fh) items = sorted([unpack(r) for r in rows], key=key) self.assertEqual(items, [ { 'Date': '2021-02-01 02:00:00', 'Type': 'AccessAudit', 'User': '******', 'Description': 'Login Success', }, { 'Date': '2021-02-01 02:00:00', 'Type': 'NavigationEventAudit', 'User': '******', 'Description': 'User Name', }, { 'Date': '2021-02-01 03:00:00', 'Type': 'AccessAudit', 'User': '******', 'Description': 'Login: [email protected]', }, { 'Date': '2021-02-01 03:00:00', 'Type': 'AccessAudit', 'User': '******', 'Description': 'Logout: [email protected]', }, { 'Date': '2021-02-01 03:00:00', 'Type': 'NavigationEventAudit', 'User': '******', 'Description': '*****@*****.**', }, { 'Date': '2021-02-01 03:00:00', 'Type': 'NavigationEventAudit', 'User': '******', 'Description': '*****@*****.**', }, ])
def fake_editor(folders=()): with tempdir() as tmp: base = Path(tmp) / "base" base.mkdir() (base / "file.txt").touch() (base / "dir").mkdir() (base / "dir/file.txt").touch() for i, folder in enumerate(folders): (base / folder).mkdir() (base / folder / f"file{i}.txt").touch() yield FakeEditor(str(base / "file.txt"), str(base))
def test_normal_app(self): form, xform = self.build_normal_app() self._refresh_pillow() with tempdir() as tmp: log_file = os.path.join(tmp, 'log.txt') call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file) with open(log_file) as log_file: log = log_file.read() self.assertTrue(form.unique_id not in log) self.assertTrue(xform._id not in log)
def test_app_with_recently_fixed_form(self): form, good_build, bad_build, good_xform, bad_xform = self.build_app_with_recently_fixed_form() self._refresh_pillow() with tempdir() as tmp: log_file = os.path.join(tmp, 'log.txt') call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file) with open(log_file) as log_file: log = log_file.read() self.assertTrue(good_build._id not in log) self.assertTrue(bad_build._id in log) self.assertTrue(good_xform._id not in log) self.assertTrue(bad_xform._id in log) self.assertNoMissingXmlnss()
def test_get_blobdb(self, msg, root=True, blob_dir=None): with tempdir() as tmp: if root == "file": tmp = join(tmp, "file") with open(tmp, "w") as fh: fh.write("x") conf = SharedDriveConfiguration( shared_drive_path=tmp if root else root, restore_dir=None, transfer_dir=None, temp_dir=None, blob_dir=blob_dir, ) with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None): with assert_raises(mod.Error, msg=re.compile(msg)): mod.get_blob_db()
def setup_files(): def do_setup(tmp): os.mkdir(join(tmp, "dir")) for path in [ "dir/a.txt", "dir/b.txt", "dir/B file", "e.txt", ]: assert not isabs(path), path with open(join(tmp, path), "w") as fh: fh.write("name: {}\nsize: {}".format(path, len(path))) assert " " not in tmp, tmp with tempdir() as tmp: do_setup(tmp) yield tmp
def test_get_blobdb(self, msg, root=True, blob_dir=None): with tempdir() as tmp: if root == "file": tmp = join(tmp, "file") with open(tmp, "w", encoding='utf-8') as fh: fh.write("x") conf = SharedDriveConfiguration( shared_drive_path=tmp if root else root, restore_dir=None, transfer_dir=None, temp_dir=None, blob_dir=blob_dir, ) with patch("corehq.blobs._db", new=[]): with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None): with assert_raises(mod.Error, msg=re.compile(msg)): mod.get_blob_db()
def test_get_blobdb(self, msg, root=True, blob_dir=None): with tempdir() as tmp: if (root == "file" and six.PY3) or (root == b"file" and six.PY2): tmp = join(tmp, "file" if six.PY3 else b"file") with open(tmp, "w", encoding='utf-8') as fh: fh.write("x") conf = SharedDriveConfiguration( shared_drive_path=tmp if root else root, restore_dir=None, transfer_dir=None, temp_dir=None, blob_dir=blob_dir, ) with patch("corehq.blobs._db", new=[]): with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None): with assert_raises(mod.Error, msg=re.compile(msg)): mod.get_blob_db()
def test_fix_xforms_with_missing_xmlns_task(self): """Tests the ability to find anomalies that violate our asssumptions about all applications and builds being fixes """ good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form() self._refresh_pillow() with tempdir() as tmp: with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp): with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail: stats, log_file_path = fix_xforms_with_missing_xmlns() self.assertTrue(mocked_mail.delay.called) self.assertTrue( (DOMAIN, bad_xforms[0].build_id) in stats['builds_with_undefined_xmlns'] ) self.assertEqual(stats['not_fixed_undefined_xmlns'][DOMAIN], len(bad_xforms))
def test_migrate_backend(self): with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, self.test_size) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: metadata was saved in BlobMeta table for doc in self.couch_docs: obj = doc.class_.get(doc.id) self.assertEqual(obj._rev, doc.data["_rev"]) # rev should not change self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"])) db = get_db_alias_for_partitioned_doc(obj._id) metas = { meta.name: meta for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id) } for name, meta in obj.blobs.items(): blobmeta = metas[name] dbname = doc.class_.get_db().dbname key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower()) self.assertEqual(blobmeta.key, key, doc) self.assertEqual(blobmeta.domain, doc.domain, doc) self.assertEqual(blobmeta.content_type, meta.content_type, doc) self.assertEqual(blobmeta.content_length, meta.content_length, doc) for doc in self.sql_docs: db = get_db_alias_for_partitioned_doc(doc.parent_id) blobmeta = BlobMeta.objects.using(db).get( parent_id=doc.parent_id, type_code=doc.type_code, name="", ) self.assertEqual(blobmeta.domain, doc.domain, doc) self.assertEqual(blobmeta.content_type, doc.content_type, doc) self.assertEqual(blobmeta.content_length, doc.content_length, doc)
def test_migrate_backend(self): # verify: migration not complete with maybe_not_found(): self.assertEqual(os.listdir(self.db.new_db.rootdir), []) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate( filename, num_workers=2) self.assertGreaterEqual(migrated, self.test_size) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: missing blobs written to log files missing_log = set() fields = [ "blobmeta_id", "domain", "type_code", "parent_id", "blob_key", ] with open(filename, encoding='utf-8') as fh: for line in fh: doc = json.loads(line) missing_log.add(tuple(doc[x] for x in fields)) self.assertEqual(self.not_founds, missing_log) # verify: blobs were copied to new blob db not_found = set(t[0] for t in self.not_founds) for meta in self.blob_metas: if meta.id in not_found: with self.assertRaises(mod.NotFound): self.db.new_db.get(key=meta.key) continue content = self.db.new_db.get(key=meta.key) data = content.read() self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94') self.assertEqual(len(data), meta.content_length)
def do_migration(self, docs, num_attachments=1): self.docs_to_delete.extend(docs) test_types = {d.doc_type for d in docs} if test_types != self.doc_types: raise Exception( "bad test: must have at least one document per doc " "type (got: {})".format(test_types)) if not num_attachments: raise Exception("bad test: must have at least one attachment") for doc in docs: # verify: attachment is in couch and migration not complete self.assertEqual(len(doc._attachments), num_attachments) self.assertEqual(len(doc.external_blobs), 0) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, len(docs)) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: migrated data was written to the file with open(filename) as fh: lines = list(fh) lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)} for doc in docs: self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev) self.assertEqual(len(lines), migrated, lines) for doc in docs: # verify: attachments were moved to blob db exp = type(doc).get(doc._id) self.assertEqual(exp.doc_type, doc.doc_type) self.assertNotEqual(exp._rev, doc._rev) self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs)) self.assertFalse(exp._attachments, exp._attachments) self.assertEqual(len(exp.external_blobs), num_attachments)
def test_migrate_backend(self): # verify: migration not complete with maybe_not_found(): self.assertEqual(os.listdir(self.db.new_db.rootdir), []) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, self.test_size) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: missing blobs written to log files missing_log = set() fields = ["doc_type", "doc_id", "blob_identifier", "blob_bucket"] for n, ignore in enumerate(mod.MIGRATIONS[self.slug].migrators): with open("{}.{}".format(filename, n)) as fh: for line in fh: doc = json.loads(line) missing_log.add(tuple(doc[x] for x in fields)) self.assertEqual( len(self.not_founds.intersection(missing_log)), len(self.not_founds) ) # verify: couch attachments were copied to new blob db for doc in self.couch_docs: exp = type(doc).get(doc._id) self.assertEqual(exp._rev, doc._rev) # rev should not change self.assertTrue(doc.blobs) bucket = doc._blobdb_bucket() for name, meta in doc.blobs.items(): if name == "found.not": continue content = self.db.new_db.get(meta.id, bucket) data = content.read() self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94') self.assertEqual(len(data), meta.content_length)
def test_migrate_backend(self): # verify: migration not complete with maybe_not_found(): self.assertEqual(os.listdir(self.db.new_db.rootdir), []) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename, num_workers=2) self.assertGreaterEqual(migrated, self.test_size) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: missing blobs written to log files missing_log = set() fields = [ "blobmeta_id", "domain", "type_code", "parent_id", "blob_key", ] with open(filename, encoding='utf-8') as fh: for line in fh: doc = json.loads(line) missing_log.add(tuple(doc[x] for x in fields)) self.assertEqual(self.not_founds, missing_log) # verify: blobs were copied to new blob db not_found = set(t[0] for t in self.not_founds) for meta in self.blob_metas: if meta.id in not_found: with self.assertRaises(mod.NotFound): self.db.new_db.get(key=meta.key) continue content = self.db.new_db.get(key=meta.key) data = content.read() self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94') self.assertEqual(len(data), meta.content_length)
def do_migration(self, docs, num_attachments=1): self.docs_to_delete.extend(docs) test_types = {d.doc_type for d in docs} if test_types != self.doc_types: raise Exception("bad test: must have at least one document per doc " "type (got: {})".format(test_types)) if not num_attachments: raise Exception("bad test: must have at least one attachment") for doc in docs: # verify: attachment is in couch and migration not complete self.assertEqual(len(doc._attachments), num_attachments) self.assertEqual(len(doc.external_blobs), 0) with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, len(docs)) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: migrated data was written to the file with open(filename) as fh: lines = list(fh) lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)} for doc in docs: self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev) self.assertEqual(len(lines), migrated, lines) for doc in docs: # verify: attachments were moved to blob db exp = type(doc).get(doc._id) self.assertEqual(exp.doc_type, doc.doc_type) self.assertNotEqual(exp._rev, doc._rev) self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs)) self.assertFalse(exp._attachments, exp._attachments) self.assertEqual(len(exp.external_blobs), num_attachments)
def test_fix_xforms_with_missing_xmlns_task_fixed(self): """Tests the ability to fix xforms with the periodic cron task """ good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form() # Fix bad builds for bad_xform in bad_xforms: app = Application.get(bad_xform.build_id) for form in app.get_forms(): if form.xmlns == 'undefined': form.xmlns = 'my-fixed-xmlns' app.save() self._refresh_pillow() with tempdir() as tmp: with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp): with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail: stats, log_file_path = fix_xforms_with_missing_xmlns() self.assertTrue(mocked_mail.delay.called) self.assertTrue(stats['fixed'][DOMAIN], len(bad_xforms))
def test_migrate_backend(self): with tempdir() as tmp: filename = join(tmp, "file.txt") # do migration migrated, skipped = MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, self.test_size) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: metadata was saved in BlobMeta table for doc in self.couch_docs: obj = doc.class_.get(doc.id) self.assertEqual(obj._rev, doc.data["_rev"]) # rev should not change self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"])) db = get_db_alias_for_partitioned_doc(obj._id) metas = {meta.name: meta for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id)} for name, meta in obj.blobs.items(): blobmeta = metas[name] dbname = doc.class_.get_db().dbname key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower()) self.assertEqual(blobmeta.key, key, doc) self.assertEqual(blobmeta.domain, doc.domain, doc) self.assertEqual(blobmeta.content_type, meta.content_type, doc) self.assertEqual(blobmeta.content_length, meta.content_length, doc) for doc in self.sql_docs: db = get_db_alias_for_partitioned_doc(doc.parent_id) blobmeta = BlobMeta.objects.using(db).get( parent_id=doc.parent_id, type_code=doc.type_code, name="", ) self.assertEqual(blobmeta.domain, doc.domain, doc) self.assertEqual(blobmeta.content_type, doc.content_type, doc) self.assertEqual(blobmeta.content_length, doc.content_length, doc)
def test_default_package(): with tempdir() as tmp: os.mkdir(join(tmp, "dir")) os.mkdir(join(tmp, "dir/package")) os.mkdir(join(tmp, "dir/other")) for path in [ "dir/package/__init__.py", "dir/other/b.txt", ]: assert not isabs(path), path with open(join(tmp, path), "w"): pass @async_test async def test(path, result=""): filepath = 'file' if path is None else join(tmp, path) editor = FakeEditor(filepath) eq(await mod.default_package(editor), result) yield test, None yield test, "dir/mod.py" yield test, "dir/other/mod.py" yield test, "dir/package/mod.py", "package"
def test_migrate_saved_exports(self): # setup data saved = SavedBasicExport(configuration=_mk_config()) saved.save() payload = 'something small and simple' name = saved.get_attachment_name() super(BlobMixin, saved).put_attachment(payload, name) saved.save() # verify: attachment is in couch and migration not complete self.assertEqual(len(saved._attachments), 1) self.assertEqual(len(saved.external_blobs), 0) with tempdir() as tmp, replattr(SavedBasicExport, "migrating_blobs_from_couch", True): filename = join(tmp, "file.txt") # do migration migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename) self.assertGreaterEqual(migrated, 1) # verify: migration state recorded mod.BlobMigrationState.objects.get(slug=self.slug) # verify: migrated data was written to the file with open(filename) as fh: lines = list(fh) doc = {d["_id"]: d for d in (json.loads(x) for x in lines)}[saved._id] self.assertEqual(doc["_rev"], saved._rev) self.assertEqual(len(lines), migrated, lines) # verify: attachment was moved to blob db exp = SavedBasicExport.get(saved._id) self.assertNotEqual(exp._rev, saved._rev) self.assertEqual(len(exp.blobs), 1, repr(exp.blobs)) self.assertFalse(exp._attachments, exp._attachments) self.assertEqual(len(exp.external_blobs), 1) self.assertEqual(exp.get_payload(), payload)
def setUpClass(cls): cls.tmp_context = tempdir() tmp = cls.tmp_context.__enter__() cls.filepath = join(tmp, "file.txt") with open(cls.filepath, "wb") as fh: fh.write(b"data")
def setUpClass(cls): super(TestCaseDiffProcess, cls).setUpClass() cls.tmp = tempdir() cls.state_dir = cls.tmp.__enter__()
def test_File(): field = File('path') eq_(str(field), 'path') eq_(repr(field), "File('path')") with tempdir() as tmp: os.mkdir(join(tmp, "dir")) os.mkdir(join(tmp, "space dir")) for path in [ "dir/a.txt", "dir/b.txt", "dir/B file", ".hidden", "file.txt", "file.doc", "space dir/file", ]: assert not isabs(path), path with open(join(tmp, path), "w"): pass test = make_consume_checker(field) yield test, "relative.txt", 0, ("relative.txt", 13) test = make_completions_checker(field) yield test, "", [] project_path = join(tmp, "dir") editor = FakeEditor(join(tmp, "dir/file.txt"), project_path) field = await_coroutine(field.with_context(editor)) test = make_completions_checker(field) yield test, ".../", ["a.txt", "B file", "b.txt"], 4 with replattr(editor, "_project_path", project_path + "/"): yield test, ".../", ["a.txt", "B file", "b.txt"], 4 yield test, "...//", ["a.txt", "B file", "b.txt"], 5 with replattr((editor, "_project_path", None), (editor, "_file_path", join(tmp, "space dir/file")), sigcheck=False): yield test, "", ["file"], 0 yield test, "../", ["dir/", "file.doc", "file.txt", "space dir/"], 3 # yield test, "..//", ["dir", "file.doc", "file.txt", "space dir"], 4 yield test, "../f", ["file.doc", "file.txt"], 3 yield test, "../dir/", ["a.txt", "B file", "b.txt"], 7 test = make_arg_string_checker(field) yield test, "/str", "/str" yield test, "/a b", '"/a b"' yield test, os.path.expanduser("~/a b"), '"~/a b"' yield test, join(tmp, "dir/file"), "file" yield test, join(tmp, "dir/a b"), '"a b"' yield test, join(tmp, "file"), join(tmp, "file") yield test, "arg/", Error("not a file: path='arg/'") test = make_consume_checker(field) yield test, '', 0, (None, 1) yield test, 'a', 0, (join(tmp, 'dir/a'), 2) yield test, 'abc', 0, (join(tmp, 'dir/abc'), 4) yield test, 'abc ', 0, (join(tmp, 'dir/abc'), 4) yield test, 'file.txt', 0, (join(tmp, 'dir/file.txt'), 9) yield test, '../file.txt', 0, (join(tmp, 'dir/../file.txt'), 12) yield test, '/file.txt', 0, ('/file.txt', 10) yield test, '~/file.txt', 0, (os.path.expanduser('~/file.txt'), 11) yield test, '...', 0, (join(tmp, 'dir'), 4) yield test, '.../file.txt', 0, (join(tmp, 'dir/file.txt'), 13) yield test, '"ab c"', 0, (join(tmp, 'dir/ab c'), 6) yield test, "'ab c'", 0, (join(tmp, 'dir/ab c'), 6) yield test, "'ab c/'", 0, (join(tmp, 'dir/ab c/'), 7) # completions def expanduser(path): if path.startswith("~"): if len(path) == 1: return tmp assert path.startswith("~/"), path return tmp + path[1:] return path @async_test async def test(input, output): if input.startswith("/"): input = tmp + "/" with replattr(os.path, "expanduser", expanduser): arg = await mod.Arg(field, input, 0, None) eq_(await field.get_completions(arg), output) yield test, "", ["a.txt", "B file", "b.txt"] yield test, "a", ["a.txt"] yield test, "a.txt", ["a.txt"] yield test, "b", ["B file", "b.txt"] yield test, "B", ["B file"] yield test, "..", ["../"] yield test, "../", ["dir/", "file.doc", "file.txt", "space dir/"] yield test, "../.", [".hidden"] yield test, "...", [".../"] yield test, ".../", ["a.txt", "B file", "b.txt"] yield test, "../dir", ["dir/"] yield test, "../dir/", ["a.txt", "B file", "b.txt"] yield test, "../sp", ["space dir/"] yield test, "../space\\ d", ["space dir/"] yield test, "../space\\ dir", ["space dir/"] yield test, "../space\\ dir/", ["file"] yield test, "val", [] yield test, "/", ["dir/", "file.doc", "file.txt", "space dir/"] yield test, "~", ["~/"] yield test, "~/", ["dir/", "file.doc", "file.txt", "space dir/"] # delimiter completion @async_test async def test(input, output, start=0): arg = await mod.Arg(field, input, 0, None) words = await field.get_completions(arg) assert all(isinstance(w, CompleteWord) for w in words), \ repr([w for w in words if not isinstance(w, CompleteWord)]) eq_([w.complete() for w in words], output) eq_([w.start for w in words], [start] * len(words), words) yield test, "", ["a.txt ", "B\\ file ", "b.txt "] yield test, "x", [] yield test, "..", ["../"] yield test, "../", ["dir/", "file.doc ", "file.txt ", "space\\ dir/"], 3 yield test, "../dir", ["dir/"], 3 yield test, "../di", ["dir/"], 3 yield test, "../sp", ["space\\ dir/"], 3 yield test, "../space\\ d", ["space\\ dir/"], 3 yield test, "../space\\ dir", ["space\\ dir/"], 3 yield test, ".../", ["a.txt ", "B\\ file ", "b.txt "], 4 yield test, "../space\\ dir/", ["file "], 14 yield test, "~", ["~/"], None field = File('dir', directory=True) eq_(str(field), 'dir') eq_(repr(field), "File('dir', directory=True)") field = await_coroutine(field.with_context(editor)) test = make_consume_checker(field) yield test, '', 0, (None, 1) yield test, 'a', 0, (join(tmp, 'dir/a'), 2) yield test, 'abc', 0, (join(tmp, 'dir/abc'), 4) yield test, 'abc ', 0, (join(tmp, 'dir/abc'), 4) yield test, 'abc/', 0, (join(tmp, 'dir/abc/'), 5) yield test, '...', 0, (join(tmp, 'dir'), 4) yield test, '.../abc/', 0, (join(tmp, 'dir/abc/'), 9) test = make_completions_checker(field) yield test, "", [], 0 yield test, "a", [], 0 yield test, "..", ["../"], 0 yield test, "../", ["dir/", "space dir/"], 3 test = make_arg_string_checker(field) yield test, "/a", "/a" yield test, "/a/", "/a/" yield test, "/dir/a", "/dir/a" yield test, "/dir/a/", "/dir/a/" field = File('dir', default="~/dir") check = make_completions_checker(field) def test(input, output, *args): if input.startswith("/"): input = tmp + "/" with replattr(os.path, "expanduser", expanduser): check(input, output, *args) yield test, "", [], 0 test = make_placeholder_checker(field) yield test, "", 0, ("", "~/dir") yield test, " ", 0, ("~/dir", "")
def setup_module(): global _tmp, state_dir _tmp = tempdir() state_dir = _tmp.__enter__()