def setUp(self, path, structure_file, input_file):
"""Set up some common variables.
:param str path: The path that contains both the structure_file and
input_file.
:param str structure_file: The file with a JSON description of the
XML structure.
:param str input_file: The XML file to test.
"""
database.clean()
self.path = path
self.structure_file = path + structure_file
self.input_file = path + input_file
string_processor.project = Project()
self.input_project = Project()
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file,
string_processor)
def setUp(self, path, structure_file, input_file):
"""Set up some common variables.
:param str path: The path that contains both the structure_file and
input_file.
:param str structure_file: The file with a JSON description of the
XML structure.
:param str input_file: The XML file to test.
"""
database.clean()
self.path = path
self.structure_file = path + structure_file
self.input_file = path + input_file
string_processor.project = Project()
self.input_project = Project()
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, string_processor)
def setUpClass(self, text=""):
"""Set up some local variables.
"""
database.clean()
t.project = Project()
self.example = text
self.result = t.tokenize(self.example)
self.raw = t.parser.raw_parse(self.example)
def test_sanity(self):
"""Method should output the same result as running raw_parse directly
when run on a normal sentence text.
"""
database.clean()
text = "The fox is brown."
result = t.parse_with_error_handling(text)
expected_result = t.parser.raw_parse(text)
self.failUnless(result == expected_result)
def setUpClass(self, text=""):
"""Set up some local variables.
"""
database.clean()
t.project = Project()
self.example = text
# TODO: this method doesn't exist anymore, it was making
# a redundant parser call
# self.result = t.tokenize(self.example)
self.raw = t.parser.raw_parse(self.example)
def setUpClass(self, text=""):
"""Set up some local variables.
"""
database.clean()
t.project = Project()
self.example = text
# TODO: this method doesn't exist anymore, it was making
# a redundant parser call
# self.result = t.tokenize(self.example)
self.raw = t.parser.raw_parse(self.example)
def test_sanity(self):
"""Method should output the same result as running raw_parse directly
when run on a normal sentence text.
"""
database.clean()
text = "The fox is brown."
result = t.parse_with_error_handling(text)
expected_result = t.parser.raw_parse(text)
self.failUnless(result == expected_result)
def setUp(self):
"""Obtain a SequenceProcessor.
"""
database.clean()
self.project = mock.create_autospec(Project)
self.seq_proc = SequenceProcessor(self.project)
self.words = [Word(lemma="first", word="first"),
Word(lemma="second", word="second"),
Word(lemma="third", word="third")]
self.string = "first second third"
def setUpClass(cls):
database.clean()
cls.set = Set()
cls.sequenceset = SequenceSet()
cls.sentenceset = SentenceSet()
cls.documentset = DocumentSet()
db.session.add_all([cls.set, cls.sequenceset, cls.sentenceset,
cls.documentset])
db.session.commit()
def setUp(self):
"""Get the documentparser instance.
"""
database.clean()
self.project = Project()
self.project.save()
self.mock_str_proc = MagicMock()
with patch("app.preprocessor.documentparser.SequenceProcessor"):
self.docparser = documentparser.DocumentParser(
self.mock_str_proc, self.project)
def setUpClass(cls):
database.clean()
cls.set = Set()
cls.sequenceset = SequenceSet()
cls.sentenceset = SentenceSet()
cls.documentset = DocumentSet()
db.session.add_all(
[cls.set, cls.sequenceset, cls.sentenceset, cls.documentset])
db.session.commit()
def setUp(self):
"""Clear the database for the next unit test.
"""
self.client = application.test_client()
database.clean()
self.user = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user.id
sess["_fresh"] = True
def setUp(self):
"""Get the documentparser instance.
"""
database.clean()
self.project = Project()
self.project.save()
self.mock_str_proc = MagicMock()
with patch("app.preprocessor.documentparser.SequenceProcessor"):
self.docparser = documentparser.DocumentParser(
self.mock_str_proc,
self.project)
def setUp(self):
"""Clear the database for the next unit test.
"""
self.client = application.test_client()
database.clean()
self.user = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.add(self.user)
db.session.commit()
with self.client.session_transaction() as sess:
db.session.add(self.user)
sess["user_id"] = self.user.id
sess["_fresh"] = True
def setUpClass(cls):
database.clean()
cls.word1 = Word(lemma="bar")
cls.word2 = Word(lemma="baz")
cls.word3 = Word(lemma="bar")
cls.word4 = Word(lemma="qux")
sequence1 = Sequence(words=[cls.word1, cls.word2])
sequence2 = Sequence(words=[cls.word4])
sequence3 = Sequence(words=[cls.word3])
cls.sequenceset1 = SequenceSet(sequences=[sequence1, sequence2])
cls.sequenceset2 = SequenceSet(sequences=[sequence3])
db.session.add_all([sequence1, sequence2, sequence3, cls.sequenceset1,
cls.sequenceset2, cls.word1, cls.word2, cls.word3, cls.word4])
db.session.commit()
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project1 = Project(name="Foos project")
self.project2 = Project(name="Bars project")
self.user1.add_project(self.project1, ProjectsUsers.ROLE_ADMIN)
self.user2.add_project(self.project2, ProjectsUsers.ROLE_ADMIN)
self.project1.save()
self.project2.save()
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project1 = Project(name="Foos project")
self.project2 = Project(name="Bars project")
self.user1.add_project(self.project1, ProjectsUsers.ROLE_ADMIN)
self.user2.add_project(self.project2, ProjectsUsers.ROLE_ADMIN)
self.project1.save()
self.project2.save()
def setUpClass(cls):
"""Reset the DB and create a dummy project and document.
"""
database.clean()
cls.client = application.test_client()
user = User()
db.session.add(user)
db.session.commit()
project = Project(name="Bars project", users=[user])
project.save()
cls.file_handle, cls.file_path = tempfile.mkstemp()
cls.file = os.fdopen(cls.file_handle, "r+")
cls.file.write("foobar")
cls.file_name = os.path.split(cls.file_path)[1]
document_file = DocumentFile(projects=[project], path=cls.file_path)
document_file.save()
def setUp(self):
"""Reset the DB and create a dummy project and document.
"""
database.clean()
self.client = application.test_client()
user = User()
db.session.add(user)
db.session.commit()
project = Project(name="Bars project", user=user)
project.save()
self.file_handle, self.file_path = tempfile.mkstemp()
self.file = os.fdopen(self.file_handle, "r+")
self.file.write("foobar")
self.file_name = os.path.split(self.file_path)[1]
document_file = DocumentFile(projects=[project], path=self.file_path)
document_file.save()
def setUpClass(cls):
database.clean()
cls.word1 = Word(lemma="bar")
cls.word2 = Word(lemma="baz")
cls.word3 = Word(lemma="bar")
cls.word4 = Word(lemma="qux")
sequence1 = Sequence(words=[cls.word1, cls.word2])
sequence2 = Sequence(words=[cls.word4])
sequence3 = Sequence(words=[cls.word3])
cls.sequenceset1 = SequenceSet(sequences=[sequence1, sequence2])
cls.sequenceset2 = SequenceSet(sequences=[sequence3])
db.session.add_all([
sequence1, sequence2, sequence3, cls.sequenceset1,
cls.sequenceset2, cls.word1, cls.word2, cls.word3, cls.word4
])
db.session.commit()
def setUp(self):
database.clean()
colproc.parse_documents = mock.create_autospec(colproc.parse_documents)
#name="parse_documents",)
colproc.extract_record_metadata = mock.create_autospec(
colproc.extract_record_metadata)
#name="extract_record_metadata",)
# Set up the dict that is our "logging database" and set the arguments
# for calling process()
self.log_dict = {
"finished_recording_text_and_metadata": "true",
"finished_grammatical_processing": "true",
"finished_sequence_processing": "true",
"word_counts_done": "true",
"tfidf_done": "true",
"word_similarity_calculations_done": "true"
}
self.args = ["", "", "", False]
def setUp(self):
"""Parse the brief example"""
database.clean()
self.path = "tests/data/plays/"
self.structure_file = self.path + "structure.json"
self.input_file = self.path + "brief_example.xml"
self.input_project = Project()
t.project = self.input_project
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, t)
def setUp(self):
"""Parse the brief example"""
database.clean()
self.path = "tests/data/plays/"
self.structure_file = self.path + "structure.json"
self.input_file = self.path + "brief_example.xml"
self.input_project = Project()
t.project = self.input_project
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, t)
def setUp(self):
database.clean()
colproc.parse_documents = mock.create_autospec(
colproc.parse_documents)
#name="parse_documents",)
colproc.extract_record_metadata = mock.create_autospec(
colproc.extract_record_metadata)
#name="extract_record_metadata",)
# Set up the dict that is our "logging database" and set the arguments
# for calling process()
self.log_dict = {"finished_recording_text_and_metadata": "true",
"finished_grammatical_processing": "true",
"finished_sequence_processing": "true",
"word_counts_done": "true",
"tfidf_done": "true",
"word_similarity_calculations_done": "true"
}
self.args = ["", "", "", False]
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project = Project(name="Bars project")
self.user2.add_project(self.project, role=ProjectsUsers.ROLE_ADMIN)
file_handle, file_path = tempfile.mkstemp()
file_handle = os.fdopen(file_handle, "r+")
file_handle.write("foobar")
self.file_path = os.path.join(file_path)
self.document_file = DocumentFile(projects=[self.project],
path=self.file_path)
self.document_file.save()
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project = Project(name="Bars project", user=self.user2)
self.project.save()
file_handle, file_path = tempfile.mkstemp()
file_handle = os.fdopen(file_handle, "r+")
file_handle.write("foobar")
self.file_path = os.path.join(file_path)
self.document_file = DocumentFile(projects=[self.project],
path=self.file_path)
self.document_file.save()
t.start()
if __name__ == '__main__':
# Import Psyco if available
try:
import psyco
psyco.full()
except ImportError:
pass
if len(sys.argv) == 1:
usage()
elif len(sys.argv) > 1:
if sys.argv[1] == 'createdb':
database.create()
elif sys.argv[1] == 'wenxue':
wenxue()
elif sys.argv[1] == 'singtao':
singtao()
elif sys.argv[1] == 'list':
database.list_news()
elif sys.argv[1] == 'clean':
database.clean(sys.argv[2])
else:
fetch((sys.argv[1]), debug=False)
# wait all threads done
q.join()
import eupol
import eurojust
import f4e
import satcen
import ema
import cpvo
import esma
import bbi
import eba
import cdt
import eacea
import etf
import efsa
import eiopa
database.clean()
logging.basicConfig(
level=logging.INFO,
format='[%(levelname)s] (%(threadName)-10s) %(message)s',
)
epsoThread = threading.Thread(target=epso.scrapEPSO)
eupolThread = threading.Thread(target=eupol.scrapEUROPOL)
eujustThread = threading.Thread(target=eurojust.scrapEurojust)
emsaThread = threading.Thread(target=emsa.scrapEMSA)
f4eThread = threading.Thread(target=f4e.scrapF4E)
def setUp(self):
"""Mock out the parser for testing.
"""
database.clean()
def setUp(self):
"""Obtain a SequenceProcessor.
"""
database.clean()
self.project = Project()
self.seq_proc = SequenceProcessor(self.project)
def setUp(self):
database.clean()
def setUp(self):
"""Clean the database.
"""
database.clean()
import eupol
import eurojust
import f4e
import satcen
import ema
import cpvo
import esma
import bbi
import eba
import cdt
import eacea
import etf
import efsa
import eiopa
database.clean()
logging.basicConfig(level=logging.INFO,
format='[%(levelname)s] (%(threadName)-10s) %(message)s',
)
epsoThread = threading.Thread(target=epso.scrapEPSO)
eupolThread = threading.Thread(target=eupol.scrapEUROPOL)
eujustThread = threading.Thread(target= eurojust.scrapEurojust)
emsaThread = threading.Thread(target=emsa.scrapEMSA)
f4eThread = threading.Thread(target=f4e.scrapF4E)
def setUp(self):
"""Clean the current database.
"""
database.clean()
def setUp(self):
"""Clean the current database.
"""
database.clean()
Project.active_project = Project()
def setUp(self):
"""Clean the database.
"""
database.clean()
def setUp(self):
"""Clean the current database.
"""
database.clean()
def setUp(self):
database.clean()
def setUp(self):
"""Mock out the parser for testing.
"""
database.clean()
def setUp(self):
"""Clean the current database.
"""
database.clean()
Project.active_project = Project()
