def test_resume_migration(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
migrator = MIGRATIONS[self.slug]()
migrated1, skipped = migrator.migrate(filename)
self.assertGreaterEqual(migrated1, self.test_size)
self.assertFalse(skipped)
# discard state to simulate interrupted migration
for mig in migrator.iter_migrators():
mig.get_document_provider().get_document_iterator(
1).discard_state()
# resumed migration: all docs already migrated, so BlobMeta records
# exist, but should not cause errors on attempting to insert them
migrated2, skipped = MIGRATIONS[self.slug]().migrate(filename)
self.assertEqual(migrated1, migrated2)
self.assertFalse(skipped)
mod.BlobMigrationState.objects.get(slug=self.slug)
parent_ids = chain(
(doc.id for doc in self.couch_docs),
(doc.parent_id for doc in self.sql_docs),
)
# should have one blob per parent
for parent_id in parent_ids:
metas = list(
BlobMeta.objects.partitioned_query(parent_id).filter(
parent_id=parent_id))
self.assertEqual(len(metas), 1, metas)
def test_migrate_backend(self):
# verify: migration not complete
with maybe_not_found():
self.assertEqual(os.listdir(self.db.new_db.rootdir), [])
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug]().migrate(filename, num_workers=2)
self.assertGreaterEqual(migrated, self.test_size)
verify_migration(self, self.slug, filename, self.not_founds)
# verify: blobs were copied to new blob db
not_found = set(t[0] for t in self.not_founds)
for meta in self.blob_metas:
if meta.id in not_found:
with self.assertRaises(mod.NotFound):
meta.open(self.db.new_db)
continue
content = meta.open(self.db.new_db)
data = content.read()
self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
self.assertEqual(len(data), meta.content_length)
def __enter__(self):
from editxt.application import Application
self.tempdir = tempdir()
self.tmp = os.path.realpath(self.tempdir.__enter__())
profile_path = os.path.join(self.tmp, ".profile")
app = Application(profile_path)
app.syntax_factory = {}
if self.config is not None:
def update(data, key, value):
if "." in key:
base, sub = key.split(".", 1)
if base not in data:
data[base] = {}
update(data[base], sub, value)
else:
data[key] = value
for key, value in self.config.items():
assert isinstance(key, str), key
update(app.config.data, key, value)
self.items = {}
self.news = 0
self.app = app
self._setup(app)
app.__test_app = self
return app
def test_ag_completions():
with tempdir() as tmp:
(Path(tmp) / "dir").mkdir()
@gentest
@async_test
async def test(cmd, description, label="", project_path=None, items=()):
editor = FakeEditor(join(tmp, "dir/file"), project_path or tmp)
editor.selection = "b "
result = await get_completions(cmd, editor)
items = list(items)
if description is not None:
items.insert(0, {
"label": label or cmd,
"description": description,
"offset": 0,
})
eq(result["items"], items)
eq(result["value"], cmd)
yield test("ag ", "'b ' /dir options ...", "ag", project_path="/dir")
yield test("ag 'x ", "/dir options ...", "ag 'x '", project_path="/dir")
yield test("ag x dir/", "options ...")
yield test("ag x dir/ ", "options ...", "ag x dir/")
yield test("ag x dir/ ", "options ...", "ag x dir/")
yield test("ag x ../", "options ...", items=[
{"label": "dir/", "is_completion": True, "offset": 8},
])
def test_non_swaps(self):
# Test a form with multiple dups
form_with_multi_dups_id = self._submit_form()._id
self._submit_form(form_with_multi_dups_id)
self._submit_form(form_with_multi_dups_id)
# Test a form with no dups
another_form_id = self._submit_form()._id
with tempdir() as tmp:
ids_file_path = os.path.join(tmp, 'ids')
with open(ids_file_path, "w") as ids_file:
for id_ in (form_with_multi_dups_id, another_form_id):
ids_file.write("{} {}\n".format(DOMAIN, id_))
call_command('swap_duplicate_xforms',
ids_file_path,
'/dev/null',
no_input=True)
form_with_multi_dups = XFormInstance.get(form_with_multi_dups_id)
self.assertEqual(form_with_multi_dups.doc_type, "XFormInstance")
another_form = XFormInstance.get(another_form_id)
self.assertEqual(another_form.doc_type, "XFormInstance")
def test_simple_swap(self):
# Test a form with a single dup
bad_form_id = self._submit_form()._id
good_dup_id = self._submit_form(bad_form_id)._id
with tempdir() as tmp:
ids_file_path = os.path.join(tmp, 'ids')
with open(ids_file_path, "w") as ids_file:
ids_file.write("{} {}\n".format(DOMAIN, bad_form_id))
call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)
# Throw in a second call to the script to test idempotence as well
call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)
bad_form = XFormInstance.get(bad_form_id)
self.assertEqual(bad_form.doc_type, "XFormDuplicate")
self.assertRegexpMatches(
bad_form.problem, BAD_FORM_PROBLEM_TEMPLATE.format(good_dup_id, "")
)
good_dup_form = XFormInstance.get(good_dup_id)
self.assertEqual(good_dup_form.doc_type, "XFormInstance")
self.assertRegexpMatches(
good_dup_form.problem, FIXED_FORM_PROBLEM_TEMPLATE.format(
id_=bad_form_id, datetime_=""
)
)
def test_resume_migration(self):
class IncompleteDPC(migrate.DocumentProcessorController):
def _processing_complete(self):
self.document_iterator.discard_state()
with tempdir() as tmp:
filename = join(tmp, "file.txt")
with replattr(migrate, "DocumentProcessorController", IncompleteDPC):
# interrupted migration
migrated1, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated1, self.test_size)
self.assertFalse(skipped)
# resumed migration: all docs already migrated, so BlobMeta records
# exist, but should not cause errors on attempting to insert them
migrated2, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertEqual(migrated1, migrated2)
self.assertFalse(skipped)
mod.BlobMigrationState.objects.get(slug=self.slug)
parent_ids = chain(
(doc.id for doc in self.couch_docs),
(doc.parent_id for doc in self.sql_docs),
)
# should have one blob per parent
for parent_id in parent_ids:
db = get_db_alias_for_partitioned_doc(parent_id)
metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id))
self.assertEqual(len(metas), 1, metas)
def test_resume_migration(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
migrator = MIGRATIONS[self.slug]
migrated1, skipped = migrator.migrate(filename)
self.assertGreaterEqual(migrated1, self.test_size)
self.assertFalse(skipped)
# discard state to simulate interrupted migration
for mig in migrator.iter_migrators():
mig.get_document_provider().get_document_iterator(1).discard_state()
# resumed migration: all docs already migrated, so BlobMeta records
# exist, but should not cause errors on attempting to insert them
migrated2, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertEqual(migrated1, migrated2)
self.assertFalse(skipped)
mod.BlobMigrationState.objects.get(slug=self.slug)
parent_ids = chain(
(doc.id for doc in self.couch_docs),
(doc.parent_id for doc in self.sql_docs),
)
# should have one blob per parent
for parent_id in parent_ids:
db = get_db_alias_for_partitioned_doc(parent_id)
metas = list(BlobMeta.objects.using(db).filter(parent_id=parent_id))
self.assertEqual(len(metas), 1, metas)
def test_migrate_backend(self):
# verify: attachment is in couch and migration not complete
with maybe_not_found():
s3_blobs = sum(1 for b in self.db.new_db._s3_bucket().objects.all())
self.assertEqual(s3_blobs, 0)
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: migrated data was written to the file
with open(filename) as fh:
lines = list(fh)
ids = {d._id for d in self.migrate_docs}
migrated = {d["_id"] for d in (json.loads(x) for x in lines)}
self.assertEqual(len(ids.intersection(migrated)), self.test_size)
# verify: attachment was copied to new blob db
for doc in self.migrate_docs:
exp = SavedBasicExport.get(doc._id)
self.assertEqual(exp._rev, doc._rev) # rev should not change
self.assertTrue(doc.blobs)
bucket = doc._blobdb_bucket()
for meta in doc.blobs.values():
content = self.db.new_db.get(meta.id, bucket)
self.assertEqual(len(content.read()), meta.content_length)
def test_fix_xforms_with_missing_xmlns_task_fixed(self):
"""Tests the ability to fix xforms with the periodic cron task
"""
good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form(
)
# Fix bad builds
for bad_xform in bad_xforms:
app = Application.get(bad_xform.build_id)
for form in app.get_forms():
if form.xmlns == 'undefined':
form.xmlns = 'my-fixed-xmlns'
app.save()
self._refresh_pillow()
with tempdir() as tmp:
with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR',
tmp):
with patch('corehq.apps.cleanup.tasks.mail_admins_async'
) as mocked_mail:
stats, log_file_path = fix_xforms_with_missing_xmlns()
self.assertTrue(mocked_mail.delay.called)
self.assertTrue(stats['fixed'][DOMAIN], len(bad_xforms))
def test_migrate_backend(self):
# verify: attachment is in couch and migration not complete
with maybe_not_found():
s3_blobs = sum(1
for b in self.db.new_db._s3_bucket().objects.all())
self.assertEqual(s3_blobs, 0)
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: migrated data was written to the file
with open(filename) as fh:
lines = list(fh)
ids = {d._id for d in self.migrate_docs}
migrated = {d["_id"] for d in (json.loads(x) for x in lines)}
self.assertEqual(len(ids.intersection(migrated)), self.test_size)
# verify: attachment was copied to new blob db
for doc in self.migrate_docs:
exp = SavedBasicExport.get(doc._id)
self.assertEqual(exp._rev, doc._rev) # rev should not change
self.assertTrue(doc.blobs)
bucket = doc._blobdb_bucket()
for meta in doc.blobs.values():
content = self.db.new_db.get(meta.id, bucket)
self.assertEqual(len(content.read()), meta.content_length)
def _test_compress(self, expected_count, domain=None):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug]().migrate(
filename, num_workers=2, domain=domain)
self.assertGreaterEqual(migrated, expected_count)
not_founds = {
nf
for nf in self.not_founds if not domain or nf[1] == domain
}
verify_migration(self, self.slug, filename, not_founds)
# verify: blobs were compressed
not_found = set(t[0] for t in self.not_founds)
for meta in self.blob_metas:
if domain and meta.domain != domain:
continue
meta.refresh_from_db()
if meta.id in not_found:
with self.assertRaises(mod.NotFound):
meta.open(self.db)
continue
content = meta.open(self.db)
data = content.read()
self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
self.assertEqual(len(data), meta.content_length)
if meta.type_code == CODES.form_xml:
self.assertGreaterEqual(meta.compressed_length, 1)
else:
self.assertIsNone(meta.compressed_length)
async def test_ag_help():
with tempdir() as tmp:
command = "ag x . --help"
editor = FakeEditor(join(tmp, "file"))
result = await do_command(command, editor)
assert len(result["items"]) > 1, result
result.pop("items")
eq(result["value"], None)
async def test_ag_not_installed():
with tempdir() as tmp:
command = "ag x xxxx"
editor = FakeEditor(join(tmp, "file"))
editor.ag_path = ag_path = join(tmp, "ag")
result = await do_command(command, editor)
eq(result["type"], "error")
eq(result["message"], Regex(f"{ag_path} not found. "))
def test(relpath):
with tempdir() as base:
filepath = Path(base) / "file.txt"
filepath.touch()
assert filepath.exists(), filepath
path = join(base, relpath)
mod.create_new_file(path)
assert exists(path)
async def test_too_many_results():
with tempdir() as tmp:
os.mkdir(join(tmp, "dir"))
with open(join(tmp, "dir/long_lines.txt"), "w", encoding="utf-8") as fh:
fh.write("file.txt\n" * (mod.MAX_RESULT_ITEMS + 1))
editor = FakeEditor(join(tmp, "dir/file"), tmp)
result = await do_command("ag txt", editor)
items = result["items"]
eq(len(items), mod.MAX_RESULT_ITEMS, result)
async def test_ag_error():
with tempdir() as tmp:
command = "ag x xxxx"
editor = FakeEditor(join(tmp, "file"))
result = await do_command(command, editor)
item, = result["items"]
eq(item["description"], Regex("No such file or directory:"))
eq(result["value"], command)
assert "filter_results" not in result, result
def make_file(name="file.txt", content="text"):
if os.path.isabs(name):
name = name.lstrip(os.path.sep).lstrip("/")
assert name and not os.path.isabs(name), name
with tempdir() as tmp:
path = os.path.join(tmp, name)
if content is not None:
with open(path, "w", encoding="utf-8") as file:
file.write(content)
yield path
async def test_huge_results():
with tempdir() as tmp:
os.mkdir(join(tmp, "dir"))
with open(join(tmp, "dir/long_lines.txt"), "w", encoding="utf-8") as fh:
fh.write("xyz " + "file.txt " * 100)
editor = FakeEditor(join(tmp, "dir/file"), tmp)
result = await do_command("ag xyz", editor)
items = result["items"]
eq(len(items[0]["label"]), mod.MAX_LINE_LENGTH - 3, result)
eq(len(items), 1, result)
def test_write_export_from_all_log_events(self):
def unpack(row):
return {k: row[k] for k in ["Date", "Type", "User", "Description"]}
start = datetime(2021, 2, 1)
end = datetime(2021, 2, 1)
with tempdir() as tmp:
filename = os.path.join(tmp, "events.csv")
with open(filename, 'w', encoding="utf-8") as csvfile:
write_export_from_all_log_events(csvfile, start=start, end=end)
with open(filename, encoding="utf-8") as fh:
def key(row):
return row["Date"], row["Type"]
rows = DictReader(fh)
items = sorted([unpack(r) for r in rows], key=key)
self.assertEqual(items, [
{
'Date': '2021-02-01 02:00:00',
'Type': 'AccessAudit',
'User': '******',
'Description': 'Login Success',
},
{
'Date': '2021-02-01 02:00:00',
'Type': 'NavigationEventAudit',
'User': '******',
'Description': 'User Name',
},
{
'Date': '2021-02-01 03:00:00',
'Type': 'AccessAudit',
'User': '******',
'Description': 'Login: [email protected]',
},
{
'Date': '2021-02-01 03:00:00',
'Type': 'AccessAudit',
'User': '******',
'Description': 'Logout: [email protected]',
},
{
'Date': '2021-02-01 03:00:00',
'Type': 'NavigationEventAudit',
'User': '******',
'Description': '*****@*****.**',
},
{
'Date': '2021-02-01 03:00:00',
'Type': 'NavigationEventAudit',
'User': '******',
'Description': '*****@*****.**',
},
])
def fake_editor(folders=()):
with tempdir() as tmp:
base = Path(tmp) / "base"
base.mkdir()
(base / "file.txt").touch()
(base / "dir").mkdir()
(base / "dir/file.txt").touch()
for i, folder in enumerate(folders):
(base / folder).mkdir()
(base / folder / f"file{i}.txt").touch()
yield FakeEditor(str(base / "file.txt"), str(base))
def test_normal_app(self):
form, xform = self.build_normal_app()
self._refresh_pillow()
with tempdir() as tmp:
log_file = os.path.join(tmp, 'log.txt')
call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)
with open(log_file) as log_file:
log = log_file.read()
self.assertTrue(form.unique_id not in log)
self.assertTrue(xform._id not in log)
def test_normal_app(self):
form, xform = self.build_normal_app()
self._refresh_pillow()
with tempdir() as tmp:
log_file = os.path.join(tmp, 'log.txt')
call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)
with open(log_file) as log_file:
log = log_file.read()
self.assertTrue(form.unique_id not in log)
self.assertTrue(xform._id not in log)
def test_app_with_recently_fixed_form(self):
form, good_build, bad_build, good_xform, bad_xform = self.build_app_with_recently_fixed_form()
self._refresh_pillow()
with tempdir() as tmp:
log_file = os.path.join(tmp, 'log.txt')
call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)
with open(log_file) as log_file:
log = log_file.read()
self.assertTrue(good_build._id not in log)
self.assertTrue(bad_build._id in log)
self.assertTrue(good_xform._id not in log)
self.assertTrue(bad_xform._id in log)
self.assertNoMissingXmlnss()
def test_get_blobdb(self, msg, root=True, blob_dir=None):
with tempdir() as tmp:
if root == "file":
tmp = join(tmp, "file")
with open(tmp, "w") as fh:
fh.write("x")
conf = SharedDriveConfiguration(
shared_drive_path=tmp if root else root,
restore_dir=None,
transfer_dir=None,
temp_dir=None,
blob_dir=blob_dir,
)
with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None):
with assert_raises(mod.Error, msg=re.compile(msg)):
mod.get_blob_db()
def test_app_with_recently_fixed_form(self):
form, good_build, bad_build, good_xform, bad_xform = self.build_app_with_recently_fixed_form()
self._refresh_pillow()
with tempdir() as tmp:
log_file = os.path.join(tmp, 'log.txt')
call_command('fix_forms_and_apps_with_missing_xmlns', '/dev/null', log_file)
with open(log_file) as log_file:
log = log_file.read()
self.assertTrue(good_build._id not in log)
self.assertTrue(bad_build._id in log)
self.assertTrue(good_xform._id not in log)
self.assertTrue(bad_xform._id in log)
self.assertNoMissingXmlnss()
def setup_files():
def do_setup(tmp):
os.mkdir(join(tmp, "dir"))
for path in [
"dir/a.txt",
"dir/b.txt",
"dir/B file",
"e.txt",
]:
assert not isabs(path), path
with open(join(tmp, path), "w") as fh:
fh.write("name: {}\nsize: {}".format(path, len(path)))
assert " " not in tmp, tmp
with tempdir() as tmp:
do_setup(tmp)
yield tmp
def test_get_blobdb(self, msg, root=True, blob_dir=None):
with tempdir() as tmp:
if root == "file":
tmp = join(tmp, "file")
with open(tmp, "w", encoding='utf-8') as fh:
fh.write("x")
conf = SharedDriveConfiguration(
shared_drive_path=tmp if root else root,
restore_dir=None,
transfer_dir=None,
temp_dir=None,
blob_dir=blob_dir,
)
with patch("corehq.blobs._db", new=[]):
with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None):
with assert_raises(mod.Error, msg=re.compile(msg)):
mod.get_blob_db()
def test_get_blobdb(self, msg, root=True, blob_dir=None):
with tempdir() as tmp:
if (root == "file" and six.PY3) or (root == b"file" and six.PY2):
tmp = join(tmp, "file" if six.PY3 else b"file")
with open(tmp, "w", encoding='utf-8') as fh:
fh.write("x")
conf = SharedDriveConfiguration(
shared_drive_path=tmp if root else root,
restore_dir=None,
transfer_dir=None,
temp_dir=None,
blob_dir=blob_dir,
)
with patch("corehq.blobs._db", new=[]):
with override_settings(SHARED_DRIVE_CONF=conf, S3_BLOB_DB_SETTINGS=None):
with assert_raises(mod.Error, msg=re.compile(msg)):
mod.get_blob_db()
def test_fix_xforms_with_missing_xmlns_task(self):
"""Tests the ability to find anomalies that violate our asssumptions
about all applications and builds being fixes
"""
good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form()
self._refresh_pillow()
with tempdir() as tmp:
with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp):
with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail:
stats, log_file_path = fix_xforms_with_missing_xmlns()
self.assertTrue(mocked_mail.delay.called)
self.assertTrue(
(DOMAIN, bad_xforms[0].build_id) in stats['builds_with_undefined_xmlns']
)
self.assertEqual(stats['not_fixed_undefined_xmlns'][DOMAIN], len(bad_xforms))
def test_fix_xforms_with_missing_xmlns_task(self):
"""Tests the ability to find anomalies that violate our asssumptions
about all applications and builds being fixes
"""
good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form()
self._refresh_pillow()
with tempdir() as tmp:
with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp):
with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail:
stats, log_file_path = fix_xforms_with_missing_xmlns()
self.assertTrue(mocked_mail.delay.called)
self.assertTrue(
(DOMAIN, bad_xforms[0].build_id) in stats['builds_with_undefined_xmlns']
)
self.assertEqual(stats['not_fixed_undefined_xmlns'][DOMAIN], len(bad_xforms))
def test_migrate_backend(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: metadata was saved in BlobMeta table
for doc in self.couch_docs:
obj = doc.class_.get(doc.id)
self.assertEqual(obj._rev,
doc.data["_rev"]) # rev should not change
self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"]))
db = get_db_alias_for_partitioned_doc(obj._id)
metas = {
meta.name: meta
for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id)
}
for name, meta in obj.blobs.items():
blobmeta = metas[name]
dbname = doc.class_.get_db().dbname
key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower())
self.assertEqual(blobmeta.key, key, doc)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, meta.content_type, doc)
self.assertEqual(blobmeta.content_length, meta.content_length,
doc)
for doc in self.sql_docs:
db = get_db_alias_for_partitioned_doc(doc.parent_id)
blobmeta = BlobMeta.objects.using(db).get(
parent_id=doc.parent_id,
type_code=doc.type_code,
name="",
)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, doc.content_type, doc)
self.assertEqual(blobmeta.content_length, doc.content_length, doc)
def test_migrate_backend(self):
# verify: migration not complete
with maybe_not_found():
self.assertEqual(os.listdir(self.db.new_db.rootdir), [])
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(
filename, num_workers=2)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: missing blobs written to log files
missing_log = set()
fields = [
"blobmeta_id",
"domain",
"type_code",
"parent_id",
"blob_key",
]
with open(filename, encoding='utf-8') as fh:
for line in fh:
doc = json.loads(line)
missing_log.add(tuple(doc[x] for x in fields))
self.assertEqual(self.not_founds, missing_log)
# verify: blobs were copied to new blob db
not_found = set(t[0] for t in self.not_founds)
for meta in self.blob_metas:
if meta.id in not_found:
with self.assertRaises(mod.NotFound):
self.db.new_db.get(key=meta.key)
continue
content = self.db.new_db.get(key=meta.key)
data = content.read()
self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
self.assertEqual(len(data), meta.content_length)
def do_migration(self, docs, num_attachments=1):
self.docs_to_delete.extend(docs)
test_types = {d.doc_type for d in docs}
if test_types != self.doc_types:
raise Exception(
"bad test: must have at least one document per doc "
"type (got: {})".format(test_types))
if not num_attachments:
raise Exception("bad test: must have at least one attachment")
for doc in docs:
# verify: attachment is in couch and migration not complete
self.assertEqual(len(doc._attachments), num_attachments)
self.assertEqual(len(doc.external_blobs), 0)
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, len(docs))
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: migrated data was written to the file
with open(filename) as fh:
lines = list(fh)
lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)}
for doc in docs:
self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev)
self.assertEqual(len(lines), migrated, lines)
for doc in docs:
# verify: attachments were moved to blob db
exp = type(doc).get(doc._id)
self.assertEqual(exp.doc_type, doc.doc_type)
self.assertNotEqual(exp._rev, doc._rev)
self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs))
self.assertFalse(exp._attachments, exp._attachments)
self.assertEqual(len(exp.external_blobs), num_attachments)
def test_migrate_backend(self):
# verify: migration not complete
with maybe_not_found():
self.assertEqual(os.listdir(self.db.new_db.rootdir), [])
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: missing blobs written to log files
missing_log = set()
fields = ["doc_type", "doc_id", "blob_identifier", "blob_bucket"]
for n, ignore in enumerate(mod.MIGRATIONS[self.slug].migrators):
with open("{}.{}".format(filename, n)) as fh:
for line in fh:
doc = json.loads(line)
missing_log.add(tuple(doc[x] for x in fields))
self.assertEqual(
len(self.not_founds.intersection(missing_log)),
len(self.not_founds)
)
# verify: couch attachments were copied to new blob db
for doc in self.couch_docs:
exp = type(doc).get(doc._id)
self.assertEqual(exp._rev, doc._rev) # rev should not change
self.assertTrue(doc.blobs)
bucket = doc._blobdb_bucket()
for name, meta in doc.blobs.items():
if name == "found.not":
continue
content = self.db.new_db.get(meta.id, bucket)
data = content.read()
self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
self.assertEqual(len(data), meta.content_length)
def test_migrate_backend(self):
# verify: migration not complete
with maybe_not_found():
self.assertEqual(os.listdir(self.db.new_db.rootdir), [])
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename, num_workers=2)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: missing blobs written to log files
missing_log = set()
fields = [
"blobmeta_id",
"domain",
"type_code",
"parent_id",
"blob_key",
]
with open(filename, encoding='utf-8') as fh:
for line in fh:
doc = json.loads(line)
missing_log.add(tuple(doc[x] for x in fields))
self.assertEqual(self.not_founds, missing_log)
# verify: blobs were copied to new blob db
not_found = set(t[0] for t in self.not_founds)
for meta in self.blob_metas:
if meta.id in not_found:
with self.assertRaises(mod.NotFound):
self.db.new_db.get(key=meta.key)
continue
content = self.db.new_db.get(key=meta.key)
data = content.read()
self.assertEqual(data, b'binary data not valid utf-8 \xe4\x94')
self.assertEqual(len(data), meta.content_length)
def do_migration(self, docs, num_attachments=1):
self.docs_to_delete.extend(docs)
test_types = {d.doc_type for d in docs}
if test_types != self.doc_types:
raise Exception("bad test: must have at least one document per doc "
"type (got: {})".format(test_types))
if not num_attachments:
raise Exception("bad test: must have at least one attachment")
for doc in docs:
# verify: attachment is in couch and migration not complete
self.assertEqual(len(doc._attachments), num_attachments)
self.assertEqual(len(doc.external_blobs), 0)
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, len(docs))
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: migrated data was written to the file
with open(filename) as fh:
lines = list(fh)
lines_by_id = {d["_id"]: d for d in (json.loads(x) for x in lines)}
for doc in docs:
self.assertEqual(lines_by_id[doc._id]["_rev"], doc._rev)
self.assertEqual(len(lines), migrated, lines)
for doc in docs:
# verify: attachments were moved to blob db
exp = type(doc).get(doc._id)
self.assertEqual(exp.doc_type, doc.doc_type)
self.assertNotEqual(exp._rev, doc._rev)
self.assertEqual(len(exp.blobs), num_attachments, repr(exp.blobs))
self.assertFalse(exp._attachments, exp._attachments)
self.assertEqual(len(exp.external_blobs), num_attachments)
def test_fix_xforms_with_missing_xmlns_task_fixed(self):
"""Tests the ability to fix xforms with the periodic cron task
"""
good_form, bad_form, good_xform, bad_xforms = self.build_app_with_bad_form()
# Fix bad builds
for bad_xform in bad_xforms:
app = Application.get(bad_xform.build_id)
for form in app.get_forms():
if form.xmlns == 'undefined':
form.xmlns = 'my-fixed-xmlns'
app.save()
self._refresh_pillow()
with tempdir() as tmp:
with patch('corehq.apps.cleanup.tasks.UNDEFINED_XMLNS_LOG_DIR', tmp):
with patch('corehq.apps.cleanup.tasks.mail_admins_async') as mocked_mail:
stats, log_file_path = fix_xforms_with_missing_xmlns()
self.assertTrue(mocked_mail.delay.called)
self.assertTrue(stats['fixed'][DOMAIN], len(bad_xforms))
def test_non_swaps(self):
# Test a form with multiple dups
form_with_multi_dups_id = self._submit_form()._id
self._submit_form(form_with_multi_dups_id)
self._submit_form(form_with_multi_dups_id)
# Test a form with no dups
another_form_id = self._submit_form()._id
with tempdir() as tmp:
ids_file_path = os.path.join(tmp, 'ids')
with open(ids_file_path, "w") as ids_file:
for id_ in (form_with_multi_dups_id, another_form_id):
ids_file.write("{} {}\n".format(DOMAIN, id_))
call_command('swap_duplicate_xforms', ids_file_path, '/dev/null', no_input=True)
form_with_multi_dups = XFormInstance.get(form_with_multi_dups_id)
self.assertEqual(form_with_multi_dups.doc_type, "XFormInstance")
another_form = XFormInstance.get(another_form_id)
self.assertEqual(another_form.doc_type, "XFormInstance")
def test_migrate_backend(self):
with tempdir() as tmp:
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, self.test_size)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: metadata was saved in BlobMeta table
for doc in self.couch_docs:
obj = doc.class_.get(doc.id)
self.assertEqual(obj._rev, doc.data["_rev"]) # rev should not change
self.assertEqual(set(obj.blobs), set(doc.data["external_blobs"]))
db = get_db_alias_for_partitioned_doc(obj._id)
metas = {meta.name: meta
for meta in BlobMeta.objects.using(db).filter(parent_id=doc.id)}
for name, meta in obj.blobs.items():
blobmeta = metas[name]
dbname = doc.class_.get_db().dbname
key = "%s/%s/%s" % (dbname, obj._id, doc.doc_type.lower())
self.assertEqual(blobmeta.key, key, doc)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, meta.content_type, doc)
self.assertEqual(blobmeta.content_length, meta.content_length, doc)
for doc in self.sql_docs:
db = get_db_alias_for_partitioned_doc(doc.parent_id)
blobmeta = BlobMeta.objects.using(db).get(
parent_id=doc.parent_id,
type_code=doc.type_code,
name="",
)
self.assertEqual(blobmeta.domain, doc.domain, doc)
self.assertEqual(blobmeta.content_type, doc.content_type, doc)
self.assertEqual(blobmeta.content_length, doc.content_length, doc)
def test_default_package():
with tempdir() as tmp:
os.mkdir(join(tmp, "dir"))
os.mkdir(join(tmp, "dir/package"))
os.mkdir(join(tmp, "dir/other"))
for path in [
"dir/package/__init__.py",
"dir/other/b.txt",
]:
assert not isabs(path), path
with open(join(tmp, path), "w"):
pass
@async_test
async def test(path, result=""):
filepath = 'file' if path is None else join(tmp, path)
editor = FakeEditor(filepath)
eq(await mod.default_package(editor), result)
yield test, None
yield test, "dir/mod.py"
yield test, "dir/other/mod.py"
yield test, "dir/package/mod.py", "package"
def test_migrate_saved_exports(self):
# setup data
saved = SavedBasicExport(configuration=_mk_config())
saved.save()
payload = 'something small and simple'
name = saved.get_attachment_name()
super(BlobMixin, saved).put_attachment(payload, name)
saved.save()
# verify: attachment is in couch and migration not complete
self.assertEqual(len(saved._attachments), 1)
self.assertEqual(len(saved.external_blobs), 0)
with tempdir() as tmp, replattr(SavedBasicExport, "migrating_blobs_from_couch", True):
filename = join(tmp, "file.txt")
# do migration
migrated, skipped = mod.MIGRATIONS[self.slug].migrate(filename)
self.assertGreaterEqual(migrated, 1)
# verify: migration state recorded
mod.BlobMigrationState.objects.get(slug=self.slug)
# verify: migrated data was written to the file
with open(filename) as fh:
lines = list(fh)
doc = {d["_id"]: d for d in (json.loads(x) for x in lines)}[saved._id]
self.assertEqual(doc["_rev"], saved._rev)
self.assertEqual(len(lines), migrated, lines)
# verify: attachment was moved to blob db
exp = SavedBasicExport.get(saved._id)
self.assertNotEqual(exp._rev, saved._rev)
self.assertEqual(len(exp.blobs), 1, repr(exp.blobs))
self.assertFalse(exp._attachments, exp._attachments)
self.assertEqual(len(exp.external_blobs), 1)
self.assertEqual(exp.get_payload(), payload)
def setUpClass(cls):
cls.tmp_context = tempdir()
tmp = cls.tmp_context.__enter__()
cls.filepath = join(tmp, "file.txt")
with open(cls.filepath, "wb") as fh:
fh.write(b"data")
def setUpClass(cls):
super(TestCaseDiffProcess, cls).setUpClass()
cls.tmp = tempdir()
cls.state_dir = cls.tmp.__enter__()
def test_File():
field = File('path')
eq_(str(field), 'path')
eq_(repr(field), "File('path')")
with tempdir() as tmp:
os.mkdir(join(tmp, "dir"))
os.mkdir(join(tmp, "space dir"))
for path in [
"dir/a.txt",
"dir/b.txt",
"dir/B file",
".hidden",
"file.txt",
"file.doc",
"space dir/file",
]:
assert not isabs(path), path
with open(join(tmp, path), "w"):
pass
test = make_consume_checker(field)
yield test, "relative.txt", 0, ("relative.txt", 13)
test = make_completions_checker(field)
yield test, "", []
project_path = join(tmp, "dir")
editor = FakeEditor(join(tmp, "dir/file.txt"), project_path)
field = await_coroutine(field.with_context(editor))
test = make_completions_checker(field)
yield test, ".../", ["a.txt", "B file", "b.txt"], 4
with replattr(editor, "_project_path", project_path + "/"):
yield test, ".../", ["a.txt", "B file", "b.txt"], 4
yield test, "...//", ["a.txt", "B file", "b.txt"], 5
with replattr((editor, "_project_path", None),
(editor, "_file_path", join(tmp, "space dir/file")),
sigcheck=False):
yield test, "", ["file"], 0
yield test, "../", ["dir/", "file.doc", "file.txt",
"space dir/"], 3
# yield test, "..//", ["dir", "file.doc", "file.txt", "space dir"], 4
yield test, "../f", ["file.doc", "file.txt"], 3
yield test, "../dir/", ["a.txt", "B file", "b.txt"], 7
test = make_arg_string_checker(field)
yield test, "/str", "/str"
yield test, "/a b", '"/a b"'
yield test, os.path.expanduser("~/a b"), '"~/a b"'
yield test, join(tmp, "dir/file"), "file"
yield test, join(tmp, "dir/a b"), '"a b"'
yield test, join(tmp, "file"), join(tmp, "file")
yield test, "arg/", Error("not a file: path='arg/'")
test = make_consume_checker(field)
yield test, '', 0, (None, 1)
yield test, 'a', 0, (join(tmp, 'dir/a'), 2)
yield test, 'abc', 0, (join(tmp, 'dir/abc'), 4)
yield test, 'abc ', 0, (join(tmp, 'dir/abc'), 4)
yield test, 'file.txt', 0, (join(tmp, 'dir/file.txt'), 9)
yield test, '../file.txt', 0, (join(tmp, 'dir/../file.txt'), 12)
yield test, '/file.txt', 0, ('/file.txt', 10)
yield test, '~/file.txt', 0, (os.path.expanduser('~/file.txt'), 11)
yield test, '...', 0, (join(tmp, 'dir'), 4)
yield test, '.../file.txt', 0, (join(tmp, 'dir/file.txt'), 13)
yield test, '"ab c"', 0, (join(tmp, 'dir/ab c'), 6)
yield test, "'ab c'", 0, (join(tmp, 'dir/ab c'), 6)
yield test, "'ab c/'", 0, (join(tmp, 'dir/ab c/'), 7)
# completions
def expanduser(path):
if path.startswith("~"):
if len(path) == 1:
return tmp
assert path.startswith("~/"), path
return tmp + path[1:]
return path
@async_test
async def test(input, output):
if input.startswith("/"):
input = tmp + "/"
with replattr(os.path, "expanduser", expanduser):
arg = await mod.Arg(field, input, 0, None)
eq_(await field.get_completions(arg), output)
yield test, "", ["a.txt", "B file", "b.txt"]
yield test, "a", ["a.txt"]
yield test, "a.txt", ["a.txt"]
yield test, "b", ["B file", "b.txt"]
yield test, "B", ["B file"]
yield test, "..", ["../"]
yield test, "../", ["dir/", "file.doc", "file.txt", "space dir/"]
yield test, "../.", [".hidden"]
yield test, "...", [".../"]
yield test, ".../", ["a.txt", "B file", "b.txt"]
yield test, "../dir", ["dir/"]
yield test, "../dir/", ["a.txt", "B file", "b.txt"]
yield test, "../sp", ["space dir/"]
yield test, "../space\\ d", ["space dir/"]
yield test, "../space\\ dir", ["space dir/"]
yield test, "../space\\ dir/", ["file"]
yield test, "val", []
yield test, "/", ["dir/", "file.doc", "file.txt", "space dir/"]
yield test, "~", ["~/"]
yield test, "~/", ["dir/", "file.doc", "file.txt", "space dir/"]
# delimiter completion
@async_test
async def test(input, output, start=0):
arg = await mod.Arg(field, input, 0, None)
words = await field.get_completions(arg)
assert all(isinstance(w, CompleteWord) for w in words), \
repr([w for w in words if not isinstance(w, CompleteWord)])
eq_([w.complete() for w in words], output)
eq_([w.start for w in words], [start] * len(words), words)
yield test, "", ["a.txt ", "B\\ file ", "b.txt "]
yield test, "x", []
yield test, "..", ["../"]
yield test, "../", ["dir/", "file.doc ", "file.txt ",
"space\\ dir/"], 3
yield test, "../dir", ["dir/"], 3
yield test, "../di", ["dir/"], 3
yield test, "../sp", ["space\\ dir/"], 3
yield test, "../space\\ d", ["space\\ dir/"], 3
yield test, "../space\\ dir", ["space\\ dir/"], 3
yield test, ".../", ["a.txt ", "B\\ file ", "b.txt "], 4
yield test, "../space\\ dir/", ["file "], 14
yield test, "~", ["~/"], None
field = File('dir', directory=True)
eq_(str(field), 'dir')
eq_(repr(field), "File('dir', directory=True)")
field = await_coroutine(field.with_context(editor))
test = make_consume_checker(field)
yield test, '', 0, (None, 1)
yield test, 'a', 0, (join(tmp, 'dir/a'), 2)
yield test, 'abc', 0, (join(tmp, 'dir/abc'), 4)
yield test, 'abc ', 0, (join(tmp, 'dir/abc'), 4)
yield test, 'abc/', 0, (join(tmp, 'dir/abc/'), 5)
yield test, '...', 0, (join(tmp, 'dir'), 4)
yield test, '.../abc/', 0, (join(tmp, 'dir/abc/'), 9)
test = make_completions_checker(field)
yield test, "", [], 0
yield test, "a", [], 0
yield test, "..", ["../"], 0
yield test, "../", ["dir/", "space dir/"], 3
test = make_arg_string_checker(field)
yield test, "/a", "/a"
yield test, "/a/", "/a/"
yield test, "/dir/a", "/dir/a"
yield test, "/dir/a/", "/dir/a/"
field = File('dir', default="~/dir")
check = make_completions_checker(field)
def test(input, output, *args):
if input.startswith("/"):
input = tmp + "/"
with replattr(os.path, "expanduser", expanduser):
check(input, output, *args)
yield test, "", [], 0
test = make_placeholder_checker(field)
yield test, "", 0, ("", "~/dir")
yield test, " ", 0, ("~/dir", "")
def setup_module():
global _tmp, state_dir
_tmp = tempdir()
state_dir = _tmp.__enter__()