def test_process_processed_files(self):
"""Make sure that a project that's being processed or already
processed can't be processed again.
"""
project1 = Project(name="foo", path="/test-path",
status=Project.STATUS_PREPROCESSING)
rel = self.user.add_project(project1, role=ProjectsUsers.ROLE_ADMIN)
document_file = DocumentFile(path="foo/foo.xml")
structure_file = StructureFile(path="foo/foo.json")
project1.document_files = [document_file]
project1.structure_files = [structure_file]
project1.save()
data = {
"struc_id": structure_file.id,
}
result = self.client.post(application.config["PROCESS_ROUTE"] + str(project1.id), data=data)
assert '"status": "OK"' not in result.data
project1.status = Project.STATUS_DONE
project1.save()
result = self.client.post(application.config["PROCESS_ROUTE"] + str(project1.id), data=data)
assert '"status": "OK"' not in result.data
def test_project_show_delete(self, mock_os):
"""Test file deletion.
"""
mock_os.path.isdir.return_value = False
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
db.session.add_all([document_file1, document_file2])
document_file1.save()
document_file2.save()
result = self.client.post(application.config["DELETE_ROUTE"],
data={
"project_id": pid,
"obj_type": "doc",
"obj_id": document_file1.id
})
assert '"obj_type": "doc",' in result.data
assert '"obj_id": %s' % document_file1.id in result.data
mock_os.remove.assert_any_call("/test-path/1.xml")
# mock_os.remove.assert_any_call("/test-path/2.xml")
assert mock_os.remove.call_count == 1
def test_project_show_double_upload(self):
"""Try uploading two files with the same name to the project_show view.
"""
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
upload_dir = tempfile.mkdtemp()
application.config["UPLOAD_DIR"] = upload_dir
os.makedirs(os.path.join(upload_dir, pid))
result = self.client.post(
application.config["PROJECT_ROUTE"] + pid + "/upload",
data={
"uploaded_file":
(StringIO("<thing>Test file</thing>"), "test.xml")
})
result = self.client.post(
application.config["PROJECT_ROUTE"] + pid + "/upload",
data={
"uploaded_file":
(StringIO("<thing>Test file 2</thing>"), "test.xml")
})
assert "already exists" in result.data
def test_project_show_bad_process(self, mock_process_files):
"""Test processing an unprocessable group of files.
"""
project = Project(name="test", user=self.user, path="/foo")
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
document_file1.save()
document_file2.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-selection": ["1", "2"]
})
assert "must include exactly one" in result.data
structure_file = StructureFile(project=project, path="/foo/bar.json")
structure_file.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-structure_file": [str(structure_file.id)]
})
assert "at least one document" in result.data.lower()
def test_projects(self):
"""Test the projects view with a project present.
"""
new_project = Project(name="test", user=self.user)
new_project.save()
result = self.client.get("/projects/")
assert "/projects/1" in result.data
def test_logs(self):
"""Test to make sure that logs are being displayed.
"""
project1 = Project(name="foo", path="/test-path",
user=self.user)
project2 = Project(name="foob", path="/foobar",
user=self.user)
logs = [WarningLog(log_item="a", item_value="a", project=project1),
InfoLog(log_item="b", item_value="b", project=project1),
ErrorLog(log_item="c", item_value="c", project=project1)]
project1.document_files = [DocumentFile(path="foo")]
project2.document_files = [DocumentFile(path="foo")]
project1.save()
project2.save()
result = self.client.get("/projects/1")
assert "alert alert-warning" in result.data
assert "alert alert-info" in result.data
assert "alert alert-danger" in result.data
assert "<em>a</em>: a" in result.data
assert "<em>b</em>: b" in result.data
assert "<em>c</em>: c" in result.data
def test_project_show_delete(self, mock_os):
"""Test file deletion.
"""
mock_os.path.isdir.return_value = False
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
db.session.add_all([document_file1, document_file2])
document_file1.save()
document_file2.save()
result = self.client.post(application.config["DELETE_ROUTE"], data={
"project_id": pid,
"obj_type": "doc",
"obj_id": document_file1.id
})
assert '"obj_type": "doc",' in result.data
assert '"obj_id": %s' % document_file1.id in result.data
mock_os.remove.assert_any_call("/test-path/1.xml")
# mock_os.remove.assert_any_call("/test-path/2.xml")
assert mock_os.remove.call_count == 1
def test_project_show_upload(self):
"""Try uploading a file to the project_show view.
"""
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
upload_dir = tempfile.mkdtemp()
application.config["UPLOAD_DIR"] = upload_dir
os.makedirs(os.path.join(upload_dir, pid))
result = self.client.post(application.config["PROJECT_ROUTE"] + pid + "/upload", data={
"uploaded_file": (StringIO("<thing>Test file</thing>"), "test.xml")
})
data = loads(result.data)
# validation error contains "The file test.xml is not well-formed XML."
assert os.path.exists(os.path.join(upload_dir, pid, "test.xml"))
assert data["files"][0]["type"] == "doc"
assert data["files"][0]["filename"] == "test.xml"
uploaded_file = open(os.path.join(upload_dir, pid, "test.xml"))
assert uploaded_file.read() == "<thing>Test file</thing>"
def test_projects(self):
"""Test the projects view with a project present.
"""
new_project = Project(name="test", users=[self.user])
new_project.save()
result = self.client.get("/projects/")
assert "/projects/1" in result.data
def test_process_processed_files(self):
"""Make sure that a project that's being processed or already
processed can't be processed again.
"""
project1 = Project(name="foo",
path="/test-path",
status=Project.STATUS_PREPROCESSING)
rel = self.user.add_project(project1, role=ProjectsUsers.ROLE_ADMIN)
document_file = DocumentFile(path="foo/foo.xml")
structure_file = StructureFile(path="foo/foo.json")
project1.document_files = [document_file]
project1.structure_files = [structure_file]
project1.save()
data = {
"struc_id": structure_file.id,
}
result = self.client.post(application.config["PROCESS_ROUTE"] +
str(project1.id),
data=data)
assert '"status": "OK"' not in result.data
project1.status = Project.STATUS_DONE
project1.save()
result = self.client.post(application.config["PROCESS_ROUTE"] +
str(project1.id),
data=data)
assert '"status": "OK"' not in result.data
def test_project_show_upload(self):
"""Try uploading a file to the project_show view.
"""
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
upload_dir = tempfile.mkdtemp()
application.config["UPLOAD_DIR"] = upload_dir
os.makedirs(os.path.join(upload_dir, pid))
result = self.client.post(
application.config["PROJECT_ROUTE"] + pid + "/upload",
data={
"uploaded_file":
(StringIO("<thing>Test file</thing>"), "test.xml")
})
data = loads(result.data)
# validation error contains "The file test.xml is not well-formed XML."
assert os.path.exists(os.path.join(upload_dir, pid, "test.xml"))
assert data["files"][0]["type"] == "doc"
assert data["files"][0]["filename"] == "test.xml"
uploaded_file = open(os.path.join(upload_dir, pid, "test.xml"))
assert uploaded_file.read() == "<thing>Test file</thing>"
def test_no_project_show(self):
"""Make sure project_show says that there are no files.
"""
project = Project(name="test", user=self.user)
project.save()
result = self.client.get("/projects/1")
assert "test" in result.data
assert "There are no files in this project" in result.data
class AuthTests(unittest.TestCase):
"""Make sure that users can only see the pages and such that they
should be seeing.
"""
#TODO: can we make this a classmethod without SQLAlchemy complaining?
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project = Project(name="Bars project", user=self.user2)
self.project.save()
file_handle, file_path = tempfile.mkstemp()
file_handle = os.fdopen(file_handle, "r+")
file_handle.write("foobar")
self.file_path = os.path.join(file_path)
self.document_file = DocumentFile(projects=[self.project],
path=self.file_path)
self.document_file.save()
def test_list_projects(self):
"""Test to make sure that bar's projects aren't listed for foo.
"""
result = self.client.get("/projects/")
assert "Bars project" not in result.data
def test_view_project(self):
"""Test to make sure that foo can't see bar's project.
"""
result = self.client.get("/projects/" + str(self.project.id))
assert "Bars project" not in result.data
def test_view_document(self):
"""Test to make sure that foo can't see bar's file.
"""
result = self.client.get("/projects/" + str(self.project.id) +
"/documents/" + str(self.document_file.id))
assert "/uploads/" + str(self.document_file.id) not in result.data
def test_get_document(self):
"""Test to make sure that foo can't get bar's file.
"""
result = self.client.get("/uploads/" + str(self.document_file.id))
with open(self.file_path) as test_file:
assert result.data is not test_file.read()
def test_no_project_show(self):
"""Make sure project_show says that there are no files.
"""
project = Project(name="test", users=[self.user])
project.save()
result = self.client.get(application.config["PROJECT_ROUTE"] + str(project.id))
assert "test" in result.data
assert "There are no Documents in this project." in result.data
def setUpModule():
global project
project = Project()
project.save()
global mock_project_logger
with mock.patch("app.preprocessor.collectionprocessor.StringProcessor",
autospec=True), mock.patch("app.preprocessor.collectionprocessor.logger.ProjectLogger",
autospec=True):
global colproc
colproc = collectionprocessor.CollectionProcessor(project.id)
def test_no_project_show(self):
"""Make sure project_show says that there are no files.
"""
project = Project(name="test", users=[self.user])
project.save()
result = self.client.get(application.config["PROJECT_ROUTE"] +
str(project.id))
assert "test" in result.data
assert "There are no Documents in this project." in result.data
def test_log(self):
"""Test the log() method. These tests assume that get() works.
"""
project = Project()
project.save()
logger.log(project, "logtest", "true", logger.REPLACE)
self.failUnless(logger.get(project, "logtest") == "true")
logger.log(project, "logtest", "false", logger.UPDATE)
self.failUnless(logger.get(project, "logtest") == "true [false] ")
def test_log(self):
"""Test the log() method. These tests assume that get() works.
"""
project = Project()
project.save()
logger.log(project, "logtest", "true", logger.REPLACE)
self.failUnless(logger.get(project, "logtest") == "true")
logger.log(project, "logtest", "false", logger.UPDATE)
self.failUnless(logger.get(project, "logtest") == "true [false] ")
def setUpModule():
global project
project = Project()
project.save()
global mock_project_logger
with mock.patch(
"app.preprocessor.collectionprocessor.StringProcessor",
autospec=True), mock.patch(
"app.preprocessor.collectionprocessor.logger.ProjectLogger",
autospec=True):
global colproc
colproc = collectionprocessor.CollectionProcessor(project.id)
def test_projects_duplicate_create(self):
"""Test creating a project with the same name as another user's.
"""
project = Project(name="test", users=[User()])
project.save()
result = self.client.post("/projects/", data={
"create-submitted": "true",
"name": "test"
})
assert "already exists" not in result.data
def test_projects_bad_create(self):
"""Test creating an existing project.
"""
project = Project(name="test", user=self.user)
project.save()
result = self.client.post("/projects/", data={
"create-submitted": "true",
"create-name": "test"
})
assert "already exists" in result.data
def test_projects_bad_create(self):
"""Test creating an existing project.
"""
project = Project(name="test", users=[self.user])
project.save()
result = self.client.post("/projects/",
data={
"create-submitted": "true",
"create-name": "test"
})
assert "already exists" in result.data
def test_projects_duplicate_create(self):
"""Test creating a project with the same name as another user's.
"""
project = Project(name="test", users=[User()])
project.save()
result = self.client.post("/projects/",
data={
"create-submitted": "true",
"name": "test"
})
assert "already exists" not in result.data
def test_get(self):
"""Test the get() method.
"""
project = Project()
project.save()
entry = Log(project=project, item_value="true", log_item="logtest")
entry.save()
self.failUnless(logger.get(project, "logtest") ==
Log.query.filter(Log.log_item == "logtest").all()[0].item_value)
self.failUnless(logger.get(project, "fakerandomname") == "")
class LongSentencePlayTests(unittest.TestCase):
def setUp(self):
"""Parse the brief example"""
database.clean()
self.path = "tests/data/plays/"
self.structure_file = self.path + "structure.json"
self.input_file = self.path + "brief_example.xml"
self.input_project = Project()
t.project = self.input_project
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, t)
def test_long_speech(self):
"""Test long sentences in combined paragraphs with line breaks
"""
# run the parser
self.extractor.extract(self.input_file)
sentences = self.input_project.sentences
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[4].id).all()
self.assertEqual(words[3].surface, "forgeries")
self.assertEqual(words[3].space_before, " ")
self.assertEqual(words[7].surface, "And")
self.assertEqual(words[7].space_before, "\n")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[5].id).all()
self.assertEqual(words[10].surface, "As")
self.assertEqual(words[10].space_before, "\n")
self.assertEqual(words[11].surface, "in")
self.assertEqual(words[11].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[6].id).all()
self.assertEqual(words[4].surface, "land")
self.assertEqual(words[4].space_before, " ")
self.assertEqual(words[5].surface, "Have")
self.assertEqual(words[5].space_before, "\n")
def test_get(self):
"""Test the get() method.
"""
project = Project()
project.save()
entry = Log(project=project, item_value="true", log_item="logtest")
entry.save()
self.failUnless(
logger.get(project, "logtest") == Log.query.filter(
Log.log_item == "logtest").all()[0].item_value)
self.failUnless(logger.get(project, "fakerandomname") == "")
def test_projects_bad_process(self, mock_process_files):
"""Test processing an unprocessable project.
"""
project1 = Project(name="test1", user=self.user)
project1.save()
result = self.client.post("/projects/", data={
"action": "0",
"process-submitted": "true",
"process-selection": ["1"]
})
assert "include exactly one json file" in result.data
class ProjectTest(AppCase):
def setUp(self):
self.setup_app()
self.create_user()
self.create_issue()
self.project = Project(
name='foo bar',
repo='foo/bar',
author=self.test_user
)
self.project.issues.append(self.test_issue)
self.project.save()
self.test_issue.project = self.project
self.test_issue.save()
def tearDown(self):
self.teardown_dbs()
def test_process_attachments(self):
file = MagicMock()
file.filename = 'foo.jpg'
files = {'file0': file }
self.project.upload_file = MagicMock(return_value='12345')
self.project.process_attachments(files, self.test_issue)
attachment = self.project.attachments[0]
self.assertEqual(attachment.file_id, '12345')
self.assertEqual(attachment.project, self.project)
self.assertEqual(attachment.parent, self.test_issue)
def test_process_attachments_disallowed_extension(self):
file = MagicMock()
file.filename = 'foo.sh'
files = {'file0': file }
self.project.process_attachments(files, self.test_issue)
self.assertEqual(len(self.project.attachments), 0)
def test_pre_delete(self):
self.assertEqual(Issue.objects.count(), 1)
self.project.delete()
self.assertEqual(Issue.objects.count(), 0)
class AppCase(RequiresMocks):
def setup_app(self):
self.app = app.test_client()
# Mock the GitHub API
self.mock_github = self.create_patch('app.routes.oauth.github.api')
self.mock_github_api = MagicMock()
self.mock_github.return_value = self.mock_github_api
# Mock the Google API
self.mock_google_user_info = self.create_patch('app.routes.oauth.google.user_info')
self.mock_google_drive_api = self.create_patch('app.routes.oauth.google.drive_api')
self.mock_google_drive_api.return_value = MagicMock()
def teardown_dbs(self):
Issue.drop_collection()
User.drop_collection()
Project.drop_collection()
Attachment.drop_collection()
Comment.drop_collection()
def create_user(self):
self.test_user = User(
name='Numpy Ping',
google_id='12345',
picture='http://foo.com/image.png',
email='*****@*****.**'
)
self.test_user.save()
def create_project(self):
if not hasattr(self, 'test_user'):
self.create_user()
self.test_project = Project(
name='Proj',
repo='pub/bar',
author=self.test_user
)
self.test_project.save()
def create_issue(self):
if not hasattr(self, 'test_project'):
self.create_project()
self.test_issue = Issue(
title='Some important issue',
project=self.test_project,
author=self.test_user
)
self.test_issue.save()
def test_project_show(self):
"""Make sure project_show shows files.
"""
project = Project(name="test", user=self.user)
project.save()
document_file1 = DocumentFile(path="/test/doc1.xml", projects=[project])
document_file2 = DocumentFile(path="/test/doc2.xml", projects=[project])
document_file1.save()
document_file2.save()
result = self.client.get("/projects/1")
assert "doc1.xml" in result.data
assert "doc2.xml" in result.data
assert "/projects/1/documents/1" in result.data
assert "/projects/1/documents/2" in result.data
def test_project_show(self):
"""Make sure project_show shows files.
"""
project = Project(name="test", users=[self.user])
project.save()
document_file1 = DocumentFile(path="/test/doc1.xml", projects=[project])
document_file2 = DocumentFile(path="/test/doc2.xml", projects=[project])
document_file1.save()
document_file2.save()
result = self.client.get("/projects/1")
assert "doc1.xml" in result.data
assert "doc2.xml" in result.data
assert application.config["UPLOAD_ROUTE"] + "doc/%s" % document_file1.id in result.data
assert application.config["UPLOAD_ROUTE"] + "doc/%s" % document_file2.id in result.data
def test_projects_delete_no_perms(self):
"""Delete projects without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
project.save()
result = self.client.post(application.config["DELETE_ROUTE"], data={
"project_id": project.id,
"obj_type": "project",
"obj_id": project.id,
})
print result.data
assert "login?next=%2Fdelete%2F" in result.data
class LongSentencePlayTests(unittest.TestCase):
def setUp(self):
"""Parse the brief example"""
database.clean()
self.path = "tests/data/plays/"
self.structure_file = self.path + "structure.json"
self.input_file = self.path + "brief_example.xml"
self.input_project = Project()
t.project = self.input_project
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, t)
def test_long_speech(self):
"""Test long sentences in combined paragraphs with line breaks
"""
# run the parser
self.extractor.extract(self.input_file)
sentences = self.input_project.sentences
words = WordInSentence.query.filter(WordInSentence.sentence_id == sentences[4].id).all()
self.assertEqual(words[3].surface, "forgeries")
self.assertEqual(words[3].space_before, " ")
self.assertEqual(words[7].surface, "And")
self.assertEqual(words[7].space_before, "\n")
words = WordInSentence.query.filter(WordInSentence.sentence_id == sentences[5].id).all()
self.assertEqual(words[10].surface, "As")
self.assertEqual(words[10].space_before, "\n")
self.assertEqual(words[11].surface, "in")
self.assertEqual(words[11].space_before, " ")
words = WordInSentence.query.filter(WordInSentence.sentence_id == sentences[6].id).all()
self.assertEqual(words[4].surface, "land")
self.assertEqual(words[4].space_before, " ")
self.assertEqual(words[5].surface, "Have")
self.assertEqual(words[5].space_before, "\n")
def test_project_show_no_post(self):
"""Try sending an empty post to project_show.
"""
project = Project(name="test")
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
result = self.client.post("/projects/1",
data={"create-submitted": "true"})
assert "You must select a file" in result.data
result = self.client.post("/projects/1",
data={"process-submitted": "true"})
assert "You must select at least one document file"
def test_project_show_process_no_perms(self):
"""Delete files without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
document_file1 = DocumentFile(projects=[project], path="/foo/bar.xml")
structure_file1 = StructureFile(project=project, path="/foo/bar.json")
project.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "-1",
"process-selection": ["1"]
})
assert '"status": "OK",' not in result.data
def test_projects_delete_no_perms(self):
"""Delete projects without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
project.save()
result = self.client.post(application.config["DELETE_ROUTE"],
data={
"project_id": project.id,
"obj_type": "project",
"obj_id": project.id,
})
print result.data
assert "login?next=%2Fdelete%2F" in result.data
def test_projects_bad_delete(self):
"""Test deleting without a selection.
"""
project1 = Project(name="test1", user=self.user)
project2 = Project(name="test2", user=self.user)
project1.save()
project2.save()
result = self.client.post("/projects/", data={
"action": "-1",
"process-submitted": "true",
})
assert "must select" in result.data
assert "/projects/1" in result.data
assert "/projects/2" in result.data
def test_project_show_process_no_perms(self):
"""Delete files without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
document_file1 = DocumentFile(projects=[project], path="/foo/bar.xml")
structure_file1 = StructureFile(project=project, path="/foo/bar.json")
project.save()
result = self.client.post("/projects/1",
data={
"process-submitted": "true",
"action": "-1",
"process-selection": ["1"]
})
assert '"status": "OK",' not in result.data
def test_project_show_no_post(self):
"""Try sending an empty post to project_show.
"""
project = Project(name="test", user=self.user)
project.save()
result = self.client.post("/projects/1", data={
"create-submitted": "true"
})
assert "You must select a file" in result.data
result = self.client.post("/projects/1", data={
"process-submitted": "true"
})
assert "You must select at least one document file"
def test_document_show(self):
"""Test the detail document view.
"""
projxyz = Project(name="test project", path="/test-path/",
user=self.user)
docxyz = DocumentFile(path="/test-path/test-file.xml",
projects=[projxyz])
docxyz.save()
projxyz.save()
#TODO: why is this necessary? why does sqlalchemy complain otherwise
docid = docxyz.id
result = self.client.get("/projects/1/documents/1")
assert "/uploads/" + str(docid) in result.data
assert "test-file.xml" in result.data
def setUpClass(cls):
"""Reset the DB and create a dummy project and document.
"""
database.clean()
cls.client = application.test_client()
user = User()
db.session.add(user)
db.session.commit()
project = Project(name="Bars project", users=[user])
project.save()
cls.file_handle, cls.file_path = tempfile.mkstemp()
cls.file = os.fdopen(cls.file_handle, "r+")
cls.file.write("foobar")
cls.file_name = os.path.split(cls.file_path)[1]
document_file = DocumentFile(projects=[project], path=cls.file_path)
document_file.save()
def setUp(self):
"""Reset the DB and create a dummy project and document.
"""
database.clean()
self.client = application.test_client()
user = User()
db.session.add(user)
db.session.commit()
project = Project(name="Bars project", user=user)
project.save()
self.file_handle, self.file_path = tempfile.mkstemp()
self.file = os.fdopen(self.file_handle, "r+")
self.file.write("foobar")
self.file_name = os.path.split(self.file_path)[1]
document_file = DocumentFile(projects=[project], path=self.file_path)
document_file.save()
def add_project():
"""
添加公司项目
:return:
"""
data = request.get_json()
print(data)
project = Project()
project["name"] = data["project_name"]
company_name = "金峰测试"
company = Company.objects.filter(name=company_name).first_or_404()
project["company_id"] = company["id"]
project.save()
for device_id in data["device"]:
device = Device.objects.filter(device_id=device_id).first_or_404()
device["project_name"] = data["project_name"]
device["project_id"] = project["id"]
device.save()
return Success()
def test_project_show_process_no_perms(self):
"""Process files without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
document_file = DocumentFile(projects=[project], path="/foo/bar.xml")
structure_file = StructureFile(project=project, path="/foo/bar.json")
document_file.save()
structure_file.save()
project.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-selection": [str(document_file.id)],
"process-structure_file": str(structure_file.id)
})
assert "You can't do that" in result.data
def test_project_show(self):
"""Make sure project_show shows files.
"""
project = Project(name="test", users=[self.user])
project.save()
document_file1 = DocumentFile(path="/test/doc1.xml",
projects=[project])
document_file2 = DocumentFile(path="/test/doc2.xml",
projects=[project])
document_file1.save()
document_file2.save()
result = self.client.get("/projects/1")
assert "doc1.xml" in result.data
assert "doc2.xml" in result.data
assert application.config[
"UPLOAD_ROUTE"] + "doc/%s" % document_file1.id in result.data
assert application.config[
"UPLOAD_ROUTE"] + "doc/%s" % document_file2.id in result.data
def test_project_show_bad_process(self, mock_process_files):
"""Test processing an unprocessable group of files.
"""
project = Project(name="test", path="/foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
db.session.add(project)
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
db.session.add_all([document_file1, document_file2])
document_file1.save()
document_file2.save()
result = self.client.post(application.config["PROCESS_ROUTE"] +
str(project.id),
data={"struc_id": 555})
assert '"status": "OK",' not in result.data
def test_project_show_bad_delete(self):
"""Test a bad file delete request.
"""
project = Project(name="test", users=[self.user])
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
document_file1.save()
document_file2.save()
result = self.client.post(application.config["DELETE_ROUTE"], data={
"project_id": project.id,
"obj_type": "doc",
# missing the object id
})
assert '"status": "OK"' not in result.data
def test_project_show_bad_process(self, mock_process_files):
"""Test processing an unprocessable group of files.
"""
project = Project(name="test", path="/foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
db.session.add(project)
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
db.session.add_all([document_file1, document_file2])
document_file1.save()
document_file2.save()
result = self.client.post(application.config["PROCESS_ROUTE"] + str(project.id), data={
"struc_id": 555
})
assert '"status": "OK",' not in result.data
def test_project_show_process(self, mock_process_files):
"""Test processing a processable group of files.
"""
project = Project(name="test", user=self.user)
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.json")
document_file1.save()
document_file2.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-selection": ["1", "2"]
})
assert "Errors have occurred" not in result.data
def test_project_show_process(self, mock_process_files):
"""Test processing a processable group of files.
"""
#TODO: why is this passing?
project = Project(name="test", users=[self.user])
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.json")
document_file1.save()
document_file2.save()
result = self.client.post("/projects/1",
data={
"process-submitted": "true",
"action": "0",
"process-selection": ["1", "2"]
})
assert "Errors have occurred" not in result.data
def test_project_show_bad_delete(self):
"""Test a bad file delete request.
"""
project = Project(name="test", users=[self.user])
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
document_file1.save()
document_file2.save()
result = self.client.post(
application.config["DELETE_ROUTE"],
data={
"project_id": project.id,
"obj_type": "doc",
# missing the object id
})
assert '"status": "OK"' not in result.data
def test_project_show_process_no_perms(self):
"""Process files without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
document_file = DocumentFile(projects=[project], path="/foo/bar.xml")
structure_file = StructureFile(project=project, path="/foo/bar.json")
document_file.save()
structure_file.save()
project.save()
result = self.client.post("/projects/1",
data={
"process-submitted":
"true",
"action":
"0",
"process-selection":
[str(document_file.id)],
"process-structure_file":
str(structure_file.id)
})
assert "You can't do that" in result.data
def post(self):
"""
"""
current_user_id = get_jwt_identity()
current_user_roles = get_jwt_claims()['roles']
project_schema = ProjectSchema()
project_data = request.get_json()
validated_project_data, errors = project_schema.load(project_data)
if errors:
return dict(status='fail', message=errors), 400
if not has_role(current_user_roles, 'administrator'):
validated_project_data['owner_id'] = current_user_id
# check if project already exists
existing_project = Project.find_first(
name=validated_project_data['name'],
owner_id=validated_project_data['owner_id'])
if existing_project:
return dict(
status='fail',
message=
f'project with name {validated_project_data["name"]} already exists'
), 409
try:
validated_project_data['alias'] =\
create_alias(validated_project_data['name'])
namespace_name = validated_project_data['alias']
cluster_id = validated_project_data['cluster_id']
cluster = Cluster.get_by_id(cluster_id)
if not cluster:
return dict(status='fail',
message=f'cluster {cluster_id} not found'), 404
kube_host = cluster.host
kube_token = cluster.token
kube_client = create_kube_clients(kube_host, kube_token)
# create namespace in cluster
cluster_namespace = kube_client.kube.create_namespace(
client.V1Namespace(metadata=client.V1ObjectMeta(
name=namespace_name)))
# create project in database
if cluster_namespace:
ingress_name = f"{validated_project_data['alias']}-ingress"
ingress_meta = client.V1ObjectMeta(name=ingress_name)
ingress_default_rule = client.ExtensionsV1beta1IngressRule(
host="traefik-ui.cranecloud.io",
http=client.ExtensionsV1beta1HTTPIngressRuleValue(paths=[
client.ExtensionsV1beta1HTTPIngressPath(
path="/*",
backend=client.ExtensionsV1beta1IngressBackend(
service_name="traefik-web-ui-ext",
service_port=80))
]))
ingress_spec = client.ExtensionsV1beta1IngressSpec(
rules=[ingress_default_rule])
ingress_body = client.ExtensionsV1beta1Ingress(
metadata=ingress_meta, spec=ingress_spec)
kube_client.extension_api.create_namespaced_ingress(
namespace=namespace_name, body=ingress_body)
project = Project(**validated_project_data)
saved = project.save()
if not saved:
# delete the namespace
kube_client.kube.delete_namespace(namespace_name)
return dict(status='fail',
message='Internal Server Error'), 500
new_project_data, errors = project_schema.dump(project)
return dict(status='success',
data=dict(project=new_project_data)), 201
except client.rest.ApiException as e:
return dict(status='fail', message=e.body), e.status
except Exception as err:
return dict(status='fail', message=str(err)), 500
class ProjectPermissionsTests(unittest.TestCase):
"""Tests for the ProjectPermissions view.
"""
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project1 = Project(name="Foos project")
self.project2 = Project(name="Bars project")
self.user1.add_project(self.project1, ProjectsUsers.ROLE_ADMIN)
self.user2.add_project(self.project2, ProjectsUsers.ROLE_ADMIN)
self.project1.save()
self.project2.save()
def test_permissions_access(self):
"""Make sure users can only access their own permissions.
"""
result = self.client.get("/projects/2/permissions")
assert result.status_code == 302
assert "Permissions" not in result.data
result = self.client.get("/projects/1/permissions")
assert "Permissions" in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_create_nonexistent(self):
"""Add a nonexistent user.
"""
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "1",
"permissions-new_collaborator": "gjie"
})
assert result.status_code == 200
assert "This user does not exist." in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_create_existing(self):
"""Add an existing user.
"""
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "1",
"permissions-new_collaborator":
"*****@*****.**"
})
assert result.status_code == 200
assert "This user is already on this project" in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_create_proper(self):
"""Do a proper add.
"""
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "1",
"permissions-new_collaborator":
"*****@*****.**",
"permissions-create_permissions": "1"
})
assert result.status_code == 200
assert "[email protected]</label>" in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_delete_nonexistant(self):
"""Try to delete without a selection.
"""
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "-1",
})
assert result.status_code == 200
assert "must make a selection" in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_delete(self):
"""Try to delete a selection.
"""
rel = self.user2.add_project(self.project1, ProjectsUsers.ROLE_ADMIN)
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "-1",
"permissions-selection": [str(rel.id)]
})
assert result.status_code == 200
assert "*****@*****.**" not in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_update(self):
"""Try to update a user's permissions.
"""
rel = self.user2.add_project(self.project1, ProjectsUsers.ROLE_USER)
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "0",
"permissions-selection": [str(rel.id)],
"permissions-update_permissions": ["1"]
})
assert result.status_code == 200
assert "User</td>" not in result.data
def test_role_access(self):
"""Make sure that regular users can't see permissions of a project
they have.
"""
rel = self.user1.add_project(self.project1, ProjectsUsers.ROLE_USER)
result = self.client.get("/projects/2/permissions")
assert result.status_code == 302
assert "Bars project" not in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_final_delete(self):
"""Make sure that there is always at least one user with admin
privileges on a project.
"""
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "-1",
"permissions-selection": ["1"]
})
assert result.status_code == 200
assert "At least one user" in result.data
assert "[email protected]</label>" in result.data
@unittest.skip("test is getting CSRF error for some reason???")
def test_final_update(self):
"""Make sure that at least one user has admin privileges.
"""
result = self.client.post("/projects/1/permissions",
data={
"permissions-submitted": "true",
"action": "0",
"permissions-selection": ["1"],
"permissions-update_permissions": ["0"]
})
assert result.status_code == 200
assert "At least one user" in result.data
assert "User</td>" not in result.data
import sys
import database
collection_name = "articles"
if len(sys.argv) > 1 and sys.argv[1]:
collection_name = sys.argv[1]
collection_dir = os.path.join("tests", "data", collection_name)
extension = ".xml"
structure_file = os.path.join(collection_dir, "structure.json")
database.reset()
project = Project()
project.save()
user = User(email="test", password="******")
user.add_project(project)
user.save()
files = [
f for f in os.listdir(collection_dir)
if os.path.isfile(os.path.join(collection_dir, f))
]
for file_name in files:
if os.path.splitext(file_name)[1] == extension:
document_file = DocumentFile(
path=os.path.join(collection_dir, file_name))
project.document_files.append(document_file)
def mutate(self, info, name):
project = ProjectModel(name=name)
project.save()
return CreateProject(project=project)
class DocumentParserTests(unittest.TestCase):
"""Run tests on the DocumentParser.
"""
def setUp(self):
"""Get the documentparser instance.
"""
database.clean()
self.project = Project()
self.project.save()
self.mock_str_proc = MagicMock()
with patch("app.preprocessor.documentparser.SequenceProcessor"):
self.docparser = documentparser.DocumentParser(
self.mock_str_proc, self.project)
@unittest.skip("DocumentParser no longer calls StringProcessor")
@patch("app.preprocessor.documentparser.db", autospec=True)
def test_parse_document(self, mock_db, mock_logger):
"""Test the parse_document method.
This method supplies a document of sixty sentences to make sure that
everything works properly, and mocks out everything except for
parse_document().
"""
latest_sent = 5
runs = 108
# Mock out the write_and_parse method
self.docparser.write_and_parse = MagicMock()
# Simulate a logger
attrs = {"get.return_value": str(latest_sent)}
mock_logger.configure_mock(**attrs)
# Simulate a document with lots of sentences
mock_sents = []
for i in range(0, runs):
mock_sents.append(MagicMock(id=i, name="Sentence " + str(i)))
mock_doc = create_autospec(Document)
mock_doc.all_sentences = mock_sents
# Configure the mock parser
mock_products = MagicMock(name="Mock products")
attrs = {"parse.return_value": mock_products}
self.mock_str_proc.configure_mock(**attrs)
# Run the method
self.docparser.parse_document(mock_doc)
# Nothing should have been logged
self.failIf(mock_logger.log.called)
# The parser should have only been called on sentences with an id > 5
parse_calls = []
for i in range(latest_sent + 1, runs):
parse_calls.append(call(mock_sents[i], {}, {}))
self.mock_str_proc.parse.assert_has_calls(parse_calls, any_order=True)
# Every 50 sentences we commit to the database
assert len(mock_db.session.commit.call_args_list
) == (runs - latest_sent + 1) / 50 + 1
class LongSentenceTests(unittest.TestCase):
def setUp(self):
"""Parse the brief example"""
database.clean()
self.path = "tests/data/long_sentences/"
self.structure_file = self.path + "structure.json"
self.input_file = self.path + "document.xml"
self.input_project = Project()
t.project = self.input_project
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, t)
def test_long_sent_parsing(self):
"""test that long paragraphs are split and their spaces indexed properly by the parser
"""
# run the parser
self.extractor.extract(self.input_file)
sentences = self.input_project.sentences
# test short paragraph with normal sentences
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[0].id).all()
self.assertEqual(words[2].surface, "a")
self.assertEqual(words[2].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[1].id).all()
self.assertEqual(words[2].surface, "the")
self.assertEqual(words[2].space_before, " ")
# test long paragraph with normal sentences
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[3].id).all()
self.assertEqual(words[2].surface, "a")
self.assertEqual(words[2].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[4].id).all()
self.assertEqual(words[7].surface, "ipsum")
self.assertEqual(words[7].space_before, " ")
# test long sentence with punctuation
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[5].id).all()
self.assertEqual(words[7].surface, "long")
self.assertEqual(words[7].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[6].id).all()
self.assertEqual(words[6].surface, "consectetur")
self.assertEqual(words[6].space_before, " ")
# test long sentence with no punctuation
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[8].id).all()
self.assertEqual(words[3].surface, "hella")
self.assertEqual(words[3].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[9].id).all()
self.assertEqual(words[2].surface, "sodales")
self.assertEqual(words[2].space_before, " ")
# test no punctuation with recursion
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[10].id).all()
self.assertEqual(words[3].surface, "even")
self.assertEqual(words[3].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[11].id).all()
self.assertEqual(words[1].surface, "facilisis")
self.assertEqual(words[1].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[12].id).all()
self.assertEqual(words[2].surface, "Fusce")
self.assertEqual(words[2].space_before, " ")
