def explore_data(project_id, dataset_id):
if not current_user.is_authenticated:
print('not logged in')
return redirect(url_for('login'))
content = None
data = None
data = Project.from_user(current_user.user_id)
if Project.check_auth(current_user.user_id, int(project_id)):
project_specific_data = Project.get_one(current_user.user_id,
int(project_id))
table_data, titles, numerical_vals = Analysis.get_coloums_stat(
app.config['UPLOAD_FOLDER'] + 'dataset/' + dataset_id)
try:
# try to match the pages defined in -> pages/<input file>
return render_template('pages/explore_data.html',
data=data,
project_specific_data=project_specific_data,
numerical_vals=numerical_vals,
table_data=table_data,
titles=titles,
active_dataset=dataset_id)
except:
return render_template('pages/error-404.html')
def test_projects(self):
"""Test the projects view with a project present.
"""
new_project = Project(name="test", user=self.user)
new_project.save()
result = self.client.get("/projects/")
assert "/projects/1" in result.data
def test_project_show_delete(self, mock_os):
"""Test file deletion.
"""
mock_os.path.isdir.return_value = False
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
db.session.add_all([document_file1, document_file2])
document_file1.save()
document_file2.save()
result = self.client.post(application.config["DELETE_ROUTE"], data={
"project_id": pid,
"obj_type": "doc",
"obj_id": document_file1.id
})
assert '"obj_type": "doc",' in result.data
assert '"obj_id": %s' % document_file1.id in result.data
mock_os.remove.assert_any_call("/test-path/1.xml")
# mock_os.remove.assert_any_call("/test-path/2.xml")
assert mock_os.remove.call_count == 1
def test_project_show_upload(self):
"""Try uploading a file to the project_show view.
"""
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
upload_dir = tempfile.mkdtemp()
application.config["UPLOAD_DIR"] = upload_dir
os.makedirs(os.path.join(upload_dir, pid))
result = self.client.post(application.config["PROJECT_ROUTE"] + pid + "/upload", data={
"uploaded_file": (StringIO("<thing>Test file</thing>"), "test.xml")
})
data = loads(result.data)
# validation error contains "The file test.xml is not well-formed XML."
assert os.path.exists(os.path.join(upload_dir, pid, "test.xml"))
assert data["files"][0]["type"] == "doc"
assert data["files"][0]["filename"] == "test.xml"
uploaded_file = open(os.path.join(upload_dir, pid, "test.xml"))
assert uploaded_file.read() == "<thing>Test file</thing>"
def test_project_show_bad_process(self, mock_process_files):
"""Test processing an unprocessable group of files.
"""
project = Project(name="test", user=self.user, path="/foo")
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
document_file1.save()
document_file2.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-selection": ["1", "2"]
})
assert "must include exactly one" in result.data
structure_file = StructureFile(project=project, path="/foo/bar.json")
structure_file.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-structure_file": [str(structure_file.id)]
})
assert "at least one document" in result.data.lower()
def create_project_sample(customer_name, user_id=None):
if not current_user.is_authenticated:
if user_id is None:
print('not logged in')
return redirect(url_for('login'))
if request.method == 'POST':
dataset = [
{'name':'sample.csv', 'type':'Forecasting Timeseries Data', 'file_attributes':[0,0], 'datasetID':'sample.csv', 'status':'Active'},
{'name':'sample.csv', 'type':'Item Attributes', 'file_attributes':[0,0], 'datasetID':'sample1.csv', 'status':'Active'},
]
pname = customer_name
#if pname is None or len(pname) == 0:
# pname = "Sample"
ptype = "Forecasting"
project_id = int(str(uuid.uuid4().int)[:6])
date = str(datetime.datetime.utcnow())
#project_id, user_id, pname, ptype, dataset, date
new_project = Project(project_id, user_id, pname, ptype, dataset, date)
new_project.save_to_mongo()
return redirect(url_for('project_dashboard', project_id=project_id))
def test_project_show_upload(self):
"""Try uploading a file to the project_show view.
"""
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
upload_dir = tempfile.mkdtemp()
application.config["UPLOAD_DIR"] = upload_dir
os.makedirs(os.path.join(upload_dir, pid))
result = self.client.post(
application.config["PROJECT_ROUTE"] + pid + "/upload",
data={
"uploaded_file":
(StringIO("<thing>Test file</thing>"), "test.xml")
})
data = loads(result.data)
# validation error contains "The file test.xml is not well-formed XML."
assert os.path.exists(os.path.join(upload_dir, pid, "test.xml"))
assert data["files"][0]["type"] == "doc"
assert data["files"][0]["filename"] == "test.xml"
uploaded_file = open(os.path.join(upload_dir, pid, "test.xml"))
assert uploaded_file.read() == "<thing>Test file</thing>"
def test_project_show_delete(self, mock_os):
"""Test file deletion.
"""
mock_os.path.isdir.return_value = False
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.xml")
db.session.add_all([document_file1, document_file2])
document_file1.save()
document_file2.save()
result = self.client.post(application.config["DELETE_ROUTE"],
data={
"project_id": pid,
"obj_type": "doc",
"obj_id": document_file1.id
})
assert '"obj_type": "doc",' in result.data
assert '"obj_id": %s' % document_file1.id in result.data
mock_os.remove.assert_any_call("/test-path/1.xml")
# mock_os.remove.assert_any_call("/test-path/2.xml")
assert mock_os.remove.call_count == 1
def test_project_show_double_upload(self):
"""Try uploading two files with the same name to the project_show view.
"""
project = Project(name="test")
db.session.add(project)
self.user.add_project(project, role=ProjectsUsers.ROLE_ADMIN)
project.save()
pid = str(project.id)
upload_dir = tempfile.mkdtemp()
application.config["UPLOAD_DIR"] = upload_dir
os.makedirs(os.path.join(upload_dir, pid))
result = self.client.post(
application.config["PROJECT_ROUTE"] + pid + "/upload",
data={
"uploaded_file":
(StringIO("<thing>Test file</thing>"), "test.xml")
})
result = self.client.post(
application.config["PROJECT_ROUTE"] + pid + "/upload",
data={
"uploaded_file":
(StringIO("<thing>Test file 2</thing>"), "test.xml")
})
assert "already exists" in result.data
def test_projects(self):
"""Test the projects view with a project present.
"""
new_project = Project(name="test", users=[self.user])
new_project.save()
result = self.client.get("/projects/")
assert "/projects/1" in result.data
def test_logs(self):
"""Test to make sure that logs are being displayed.
"""
project1 = Project(name="log test project", path="/log-test-path")
self.user.add_project(project1, role=ProjectsUsers.ROLE_ADMIN)
logs = [
WarningLog(log_item="a", item_value="a", project=project1),
InfoLog(log_item="b", item_value="b", project=project1),
ErrorLog(log_item="c", item_value="c", project=project1)
]
project1.document_files = [DocumentFile(path="foo")]
db.session.add(project1)
db.session.commit()
result = self.client.get(application.config["PROJECT_ROUTE"] +
str(project1.id))
print result.data
assert "log test project" in result.data
assert "processlog alert alert-warning" in result.data
assert "processlog alert alert-warning hidden" not in result.data
assert "processlog alert alert-info" in result.data
assert "processlog alert alert-info hidden" not in result.data
assert "processlog alert alert-danger" in result.data
assert "processlog alert alert-danger hidden" not in result.data
assert "<em>a</em>: a" in result.data
assert "<em>b</em>: b" in result.data
assert "<em>c</em>: c" in result.data
def predictions_dashboard(project_id):
if not current_user.is_authenticated:
print('not logged in')
return redirect(url_for('login'))
content = None
data = None
model_info = None
data = Project.from_user(current_user.user_id)
if Project.check_auth(current_user.user_id, int(project_id)):
project_specific_data = Project.get_one(current_user.user_id,
int(project_id))
if project_specific_data[0]['model_available']:
model_info = Database.find_one(
collection="models",
query={"project_id": project_specific_data[0]['project_id']})
print(model_info)
try:
# try to match the pages defined in -> pages/<input file>
return render_template('pages/prediction_dashboard.html',
data=data,
project_specific_data=project_specific_data,
model_info=model_info)
except:
return render_template('pages/error-404.html')
def start_train(project_id):
if not current_user.is_authenticated:
print('not logged in')
return redirect(url_for('login'))
content = None
data = None
data = Project.from_user(current_user.user_id)
project_specific_data = []
html = None
titles = None
if Project.check_auth(current_user.user_id, int(project_id)):
project_specific_data = Project.get_one(current_user.user_id,
int(project_id))
if project_specific_data[0]['model_available']:
return jsonify(result='trained')
q.enqueue(DLModel.train_model,
project_specific_data[0]['dataset'][0]['name'],
int(project_id), app.config['UPLOAD_FOLDER'])
Database.update_one(collection='projects',
query=[{
'project_id': int(project_id)
}, {
"$set": {
"in_training": True
}
}])
return jsonify(result='done')
else:
return jsonify(result='error')
def test_logs(self):
"""Test to make sure that logs are being displayed.
"""
project1 = Project(name="log test project", path="/log-test-path")
self.user.add_project(project1, role=ProjectsUsers.ROLE_ADMIN)
logs = [WarningLog(log_item="a", item_value="a", project=project1),
InfoLog(log_item="b", item_value="b", project=project1),
ErrorLog(log_item="c", item_value="c", project=project1)]
project1.document_files = [DocumentFile(path="foo")]
db.session.add(project1)
db.session.commit()
result = self.client.get(application.config["PROJECT_ROUTE"] + str(project1.id))
print result.data
assert "log test project" in result.data
assert "processlog alert alert-warning" in result.data
assert "processlog alert alert-warning hidden" not in result.data
assert "processlog alert alert-info" in result.data
assert "processlog alert alert-info hidden" not in result.data
assert "processlog alert alert-danger" in result.data
assert "processlog alert alert-danger hidden" not in result.data
assert "<em>a</em>: a" in result.data
assert "<em>b</em>: b" in result.data
assert "<em>c</em>: c" in result.data
def start_train(project_id):
"""
STRICT API To allow ADMIN user to force sart training for the Deep Leanring Model
Allows an ADMIN user to start traning the user Deep LEarning model
customer's activities. \n\n
API/URL must be accessed with GET request and supply project_id the URL\n
method: GET\n
Args:
project_id (str): ID of the poject/Customer need to be sent in url. It is made to do so via Front end href
Returns:
response: JSON object
On Success \n
response = {
'result': 'done'
}
\n
On Fail:\n
response = {
'result': 'error'
}
\n
"""
if not current_user.is_authenticated:
print('not logged in')
return redirect(url_for('login'))
content = None
data = None
data = Project.from_user(current_user.user_id)
project_specific_data = []
html = None
titles = None
if Project.check_auth(current_user.user_id, int(project_id)):
project_specific_data = Project.get_one(current_user.user_id,
int(project_id))
if project_specific_data[0]['model_available']:
return jsonify(result='trained')
q.enqueue(DLModel.train_model,
project_specific_data[0]['dataset'][0]['name'],
int(project_id), app.config['UPLOAD_FOLDER'])
Database.update_one(collection='projects',
query=[{
'project_id': int(project_id)
}, {
"$set": {
"in_training": True
}
}])
return jsonify(result='done')
else:
return jsonify(result='error')
def after_put(self, instance):
total_hours = instance.temp_alloc
if instance.temp_type == 'minus':
pass
else:
proj_params = Project.find_by_proj_key(instance.project_id)
rem_hours = Project.removeHours(proj_params, total_hours)
return rem_hours
def after_put(self, instance):
total_hours = instance.temp_alloc
if instance.temp_type == 'minus':
pass
else:
proj_params = Project.find_by_proj_key(instance.project_id)
rem_hours = Project.removeHours(proj_params, total_hours)
return rem_hours
def test_no_project_show(self):
"""Make sure project_show says that there are no files.
"""
project = Project(name="test", user=self.user)
project.save()
result = self.client.get("/projects/1")
assert "test" in result.data
assert "There are no files in this project" in result.data
class AuthTests(unittest.TestCase):
"""Make sure that users can only see the pages and such that they
should be seeing.
"""
#TODO: can we make this a classmethod without SQLAlchemy complaining?
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project = Project(name="Bars project", user=self.user2)
self.project.save()
file_handle, file_path = tempfile.mkstemp()
file_handle = os.fdopen(file_handle, "r+")
file_handle.write("foobar")
self.file_path = os.path.join(file_path)
self.document_file = DocumentFile(projects=[self.project],
path=self.file_path)
self.document_file.save()
def test_list_projects(self):
"""Test to make sure that bar's projects aren't listed for foo.
"""
result = self.client.get("/projects/")
assert "Bars project" not in result.data
def test_view_project(self):
"""Test to make sure that foo can't see bar's project.
"""
result = self.client.get("/projects/" + str(self.project.id))
assert "Bars project" not in result.data
def test_view_document(self):
"""Test to make sure that foo can't see bar's file.
"""
result = self.client.get("/projects/" + str(self.project.id) +
"/documents/" + str(self.document_file.id))
assert "/uploads/" + str(self.document_file.id) not in result.data
def test_get_document(self):
"""Test to make sure that foo can't get bar's file.
"""
result = self.client.get("/uploads/" + str(self.document_file.id))
with open(self.file_path) as test_file:
assert result.data is not test_file.read()
def project_dashboard(project_id):
"""
Admin Dashboard \n
ALlows ADMIN to access project/CUSTOMER user
ALlows ADMIN to access project/CUSTOMER user dashboard
with all the admin tools to vefiy user pipeline
this allows admin to oversee if the user data is present or not
Allow to see the status of user Deep Learning model.\n\n
method: GET\n
API/URL must be accessed with GET request and supply project_id in the URL\n
Args:
project_id (str): ID of the poject need to be sent in url. It is made to do so via Front end href
Returns:
view: a url VIEW the project's/CUSTOMER's required params such as
Dataset, dataset voliation if present, Deep leanring model status,
Deep learning model metrics, predication metreics. OR if the user is
not logged in or CUSTOMER user does not exists then 404 redirect
"""
if not current_user.is_authenticated:
print('not logged in')
return redirect(url_for('login'))
content = None
data = None
data = Project.from_user(current_user.user_id)
project_specific_data = []
html = None
titles = None
model_info = None
if Project.check_auth(current_user.user_id, int(project_id)):
project_specific_data = Project.get_one(current_user.user_id,
int(project_id))
print(project_specific_data)
if project_specific_data[0]['model_available']:
model_info = Database.find_one(
collection="models",
query={"project_id": project_specific_data[0]['project_id']})
print(model_info)
try:
# try to match the pages defined in -> pages/<input file>
return render_template('pages/project_dashboard.html',
data=data,
project_specific_data=project_specific_data,
model_info=model_info)
except:
return render_template('pages/error-404.html')
def test_no_project_show(self):
"""Make sure project_show says that there are no files.
"""
project = Project(name="test", users=[self.user])
project.save()
result = self.client.get(application.config["PROJECT_ROUTE"] + str(project.id))
assert "test" in result.data
assert "There are no Documents in this project." in result.data
def update_book(id_):
book = Book.query.get(id_)
if not book:
raise ApiError('书本不存在')
data = request.json
book = book.update(data)
book = book.to_dict()
project_ids = data.get('project_ids', [])
Project.generate_project(id_, project_ids)
return api_success(book)
def test_no_project_show(self):
"""Make sure project_show says that there are no files.
"""
project = Project(name="test", users=[self.user])
project.save()
result = self.client.get(application.config["PROJECT_ROUTE"] +
str(project.id))
assert "test" in result.data
assert "There are no Documents in this project." in result.data
def setUpModule():
global project
project = Project()
project.save()
global mock_project_logger
with mock.patch("app.preprocessor.collectionprocessor.StringProcessor",
autospec=True), mock.patch("app.preprocessor.collectionprocessor.logger.ProjectLogger",
autospec=True):
global colproc
colproc = collectionprocessor.CollectionProcessor(project.id)
def add_project():
try:
request.get_json()["status"] = True
request.get_json()["createAt"] = datetime.datetime.utcnow()
request.get_json()["lastUpdateTime"] = datetime.datetime.utcnow()
filtered_data = Project.filter_field(request.get_json(), use_set_default=True)
Project.insert(filtered_data)
return jsonify({'status': 'ok', 'data': '新建成功'})
except BaseException as e:
return jsonify({'status': 'failed', 'data': '新建失败 %s' % e})
def test_process_processed_files(self):
"""Make sure that a project that's being processed or already
processed can't be processed again.
"""
project1 = Project(name="foo",
path="/test-path",
status=Project.STATUS_PREPROCESSING)
rel = self.user.add_project(project1, role=ProjectsUsers.ROLE_ADMIN)
document_file = DocumentFile(path="foo/foo.xml")
structure_file = StructureFile(path="foo/foo.json")
project1.document_files = [document_file]
project1.structure_files = [structure_file]
project1.save()
data = {
"struc_id": structure_file.id,
}
result = self.client.post(application.config["PROCESS_ROUTE"] +
str(project1.id),
data=data)
assert '"status": "OK"' not in result.data
project1.status = Project.STATUS_DONE
project1.save()
result = self.client.post(application.config["PROCESS_ROUTE"] +
str(project1.id),
data=data)
assert '"status": "OK"' not in result.data
def setUp(self):
"""Get the documentparser instance.
"""
database.clean()
self.project = Project()
self.project.save()
self.mock_str_proc = MagicMock()
with patch("app.preprocessor.documentparser.SequenceProcessor"):
self.docparser = documentparser.DocumentParser(
self.mock_str_proc, self.project)
def test_log(self):
"""Test the log() method. These tests assume that get() works.
"""
project = Project()
project.save()
logger.log(project, "logtest", "true", logger.REPLACE)
self.failUnless(logger.get(project, "logtest") == "true")
logger.log(project, "logtest", "false", logger.UPDATE)
self.failUnless(logger.get(project, "logtest") == "true [false] ")
def test_log(self):
"""Test the log() method. These tests assume that get() works.
"""
project = Project()
project.save()
logger.log(project, "logtest", "true", logger.REPLACE)
self.failUnless(logger.get(project, "logtest") == "true")
logger.log(project, "logtest", "false", logger.UPDATE)
self.failUnless(logger.get(project, "logtest") == "true [false] ")
def create_project_sample(customer_name, user_id=None):
"""
Creates a sample customer user via ADMIN dashboard
Allows an ADMIN user to create sample customer
through its dashboard and set all DATA required
AUTOMATICALLY. It created the user and saves all the files
required in necessary order.\n\n
method: POST\n
POST: API usage, acts a api to login user\n\n
Args:
customer_name (str): name of the customer name need to be sent in url, like GET
user_id (str): Id of the the user need to be sent in url, like GET (optional)
Returns:
redirect: url for the admin dashboard on success
"""
if not current_user.is_authenticated:
if user_id is None:
print('not logged in')
return redirect(url_for('login'))
if request.method == 'POST':
dataset = [
{
'name': 'sample.csv',
'type': 'Forecasting Timeseries Data',
'file_attributes': [0, 0],
'datasetID': 'sample.csv',
'status': 'Active'
},
{
'name': 'sample.csv',
'type': 'Item Attributes',
'file_attributes': [0, 0],
'datasetID': 'sample1.csv',
'status': 'Active'
},
]
pname = customer_name
#if pname is None or len(pname) == 0:
# pname = "Sample"
ptype = "Forecasting"
project_id = int(str(uuid.uuid4().int)[:6])
date = str(datetime.datetime.utcnow())
#project_id, user_id, pname, ptype, dataset, date
new_project = Project(project_id, user_id, pname, ptype, dataset, date)
new_project.save_to_mongo()
return redirect(url_for('project_dashboard', project_id=project_id))
def patch(self, project_id):
"""
"""
try:
current_user_id = get_jwt_identity()
current_user_roles = get_jwt_claims()['roles']
project_schema = ProjectSchema(only=("name", "description"),
partial=True)
project_data = request.get_json()
validate_project_data, errors = project_schema.load(project_data)
existing_project = False
if errors:
return dict(status='fail', message=errors), 400
if 'name' in validate_project_data:
existing_project = Project.find_first(
name=validate_project_data['name'],
owner_id=current_user_id)
if existing_project:
return dict(
status='fail',
message=
f'project with name {validate_project_data["name"]} already exists'
), 409
project = Project.get_by_id(project_id)
if not project:
return dict(status='fail',
message=f'Project {project_id} not found'), 404
if not is_owner_or_admin(project, current_user_id,
current_user_roles):
return dict(status='fail', message='unauthorised'), 403
updated = Project.update(project, **validate_project_data)
if not updated:
return dict(status='fail',
message='internal server error'), 500
return dict(
status='success',
message=f'project {project_id} updated successfully'), 200
except Exception as e:
return dict(status='fail', message=str(e)), 500
def project_add():
""" 项目添加 """
data = request.get_json()
if Project.query.filter_by(swagger_url=data.get('swagger_url', None),
is_valid=True).first():
return bad_request('项目Swagger地址已存在')
data['created_at'] = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
project_data = Project()
project_data.from_dict(data)
db.session.add(project_data)
db.session.commit()
return jsonify({"status": 200, "data": "success"})
def setUpModule():
global project
project = Project()
project.save()
global mock_project_logger
with mock.patch(
"app.preprocessor.collectionprocessor.StringProcessor",
autospec=True), mock.patch(
"app.preprocessor.collectionprocessor.logger.ProjectLogger",
autospec=True):
global colproc
colproc = collectionprocessor.CollectionProcessor(project.id)
def create_project():
if not current_user.is_authenticated:
print('not logged in')
return redirect(url_for('login'))
data = Project.from_user(current_user.user_id)
all_data = []
html = None
project_id = None
titles = None
if request.method == 'POST':
f_1 = request.files['file_1']
f_2 = request.files['file_2']
filename_one = str(uuid.uuid4())[:8] + "." + f_1.filename.rsplit(
'.', 1)[1].lower()
if not f_2:
f_2 = f_1
filename_two = str(uuid.uuid4())[:8] + "." + f_2.filename.rsplit(
'.', 1)[1].lower()
f_1.save(app.config['UPLOAD_FOLDER'] + "dataset/" +
secure_filename(filename_one))
f_2.save(app.config['UPLOAD_FOLDER'] + "dataset/" +
secure_filename(filename_two))
dataset = [{
'name': filename_one,
'type': 'Forecasting Timeseries Data',
'file_attributes': [0, 0],
'datasetID': filename_one,
'status': 'Active'
}, {
'name': filename_two,
'type': 'Item Attributes',
'file_attributes': [0, 0],
'datasetID': filename_two,
'status': 'Active'
}]
pname = request.form['pname']
ptype = request.form['ptype']
user_id = current_user.user_id
project_id = int(str(uuid.uuid4().int)[:6])
date = str(datetime.datetime.utcnow())
#project_id, user_id, pname, ptype, dataset, date
new_project = Project(project_id, user_id, pname, ptype, dataset, date)
new_project.save_to_mongo()
return redirect(url_for('project_dashboard', project_id=project_id))
def test_projects_bad_create(self):
"""Test creating an existing project.
"""
project = Project(name="test", user=self.user)
project.save()
result = self.client.post("/projects/", data={
"create-submitted": "true",
"create-name": "test"
})
assert "already exists" in result.data
def test_projects_duplicate_create(self):
"""Test creating a project with the same name as another user's.
"""
project = Project(name="test", users=[User()])
project.save()
result = self.client.post("/projects/", data={
"create-submitted": "true",
"name": "test"
})
assert "already exists" not in result.data
def test_projects_duplicate_create(self):
"""Test creating a project with the same name as another user's.
"""
project = Project(name="test", users=[User()])
project.save()
result = self.client.post("/projects/",
data={
"create-submitted": "true",
"name": "test"
})
assert "already exists" not in result.data
def test_projects_bad_create(self):
"""Test creating an existing project.
"""
project = Project(name="test", users=[self.user])
project.save()
result = self.client.post("/projects/",
data={
"create-submitted": "true",
"create-name": "test"
})
assert "already exists" in result.data
def test_get(self):
"""Test the get() method.
"""
project = Project()
project.save()
entry = Log(project=project, item_value="true", log_item="logtest")
entry.save()
self.failUnless(logger.get(project, "logtest") ==
Log.query.filter(Log.log_item == "logtest").all()[0].item_value)
self.failUnless(logger.get(project, "fakerandomname") == "")
def update_project(project_id):
try:
filtered_data = Project.filter_field(request.get_json())
for key, value in filtered_data.items():
Project.update({"_id": ObjectId(project_id)},
{'$set': {key: value}})
update_response = Project.update({"_id": ObjectId(project_id)},
{'$set': {'lastUpdateTime': datetime.datetime.utcnow()}},)
if update_response["n"] == 0:
return jsonify({'status': 'failed', 'data': '未找到相应更新数据！'})
return jsonify({'status': 'ok', 'data': '更新成功'})
except BaseException as e:
return jsonify({'status': 'failed', 'data': '更新失败: %s' % e})
class LongSentencePlayTests(unittest.TestCase):
def setUp(self):
"""Parse the brief example"""
database.clean()
self.path = "tests/data/plays/"
self.structure_file = self.path + "structure.json"
self.input_file = self.path + "brief_example.xml"
self.input_project = Project()
t.project = self.input_project
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, t)
def test_long_speech(self):
"""Test long sentences in combined paragraphs with line breaks
"""
# run the parser
self.extractor.extract(self.input_file)
sentences = self.input_project.sentences
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[4].id).all()
self.assertEqual(words[3].surface, "forgeries")
self.assertEqual(words[3].space_before, " ")
self.assertEqual(words[7].surface, "And")
self.assertEqual(words[7].space_before, "\n")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[5].id).all()
self.assertEqual(words[10].surface, "As")
self.assertEqual(words[10].space_before, "\n")
self.assertEqual(words[11].surface, "in")
self.assertEqual(words[11].space_before, " ")
words = WordInSentence.query.filter(
WordInSentence.sentence_id == sentences[6].id).all()
self.assertEqual(words[4].surface, "land")
self.assertEqual(words[4].space_before, " ")
self.assertEqual(words[5].surface, "Have")
self.assertEqual(words[5].space_before, "\n")
def test_projects_bad_process(self, mock_process_files):
"""Test processing an unprocessable project.
"""
project1 = Project(name="test1", user=self.user)
project1.save()
result = self.client.post("/projects/", data={
"action": "0",
"process-submitted": "true",
"process-selection": ["1"]
})
assert "include exactly one json file" in result.data
def test_logs(self):
"""Test to make sure that logs are being displayed.
"""
project1 = Project(name="foo", path="/test-path",
user=self.user)
project2 = Project(name="foob", path="/foobar",
user=self.user)
logs = [WarningLog(log_item="a", item_value="a", project=project1),
InfoLog(log_item="b", item_value="b", project=project1),
ErrorLog(log_item="c", item_value="c", project=project1)]
project1.document_files = [DocumentFile(path="foo")]
project2.document_files = [DocumentFile(path="foo")]
project1.save()
project2.save()
result = self.client.get("/projects/1")
assert "alert alert-warning" in result.data
assert "alert alert-info" in result.data
assert "alert alert-danger" in result.data
assert "<em>a</em>: a" in result.data
assert "<em>b</em>: b" in result.data
assert "<em>c</em>: c" in result.data
def test_get(self):
"""Test the get() method.
"""
project = Project()
project.save()
entry = Log(project=project, item_value="true", log_item="logtest")
entry.save()
self.failUnless(
logger.get(project, "logtest") == Log.query.filter(
Log.log_item == "logtest").all()[0].item_value)
self.failUnless(logger.get(project, "fakerandomname") == "")
def test_process_processed_files(self):
"""Make sure that a project that's being processed or already
processed can't be processed again.
"""
project1 = Project(name="foo", path="/test-path",
status=Project.STATUS_PREPROCESSING)
rel = self.user.add_project(project1, role=ProjectsUsers.ROLE_ADMIN)
document_file = DocumentFile(path="foo/foo.xml")
structure_file = StructureFile(path="foo/foo.json")
project1.document_files = [document_file]
project1.structure_files = [structure_file]
project1.save()
data = {
"struc_id": structure_file.id,
}
result = self.client.post(application.config["PROCESS_ROUTE"] + str(project1.id), data=data)
assert '"status": "OK"' not in result.data
project1.status = Project.STATUS_DONE
project1.save()
result = self.client.post(application.config["PROCESS_ROUTE"] + str(project1.id), data=data)
assert '"status": "OK"' not in result.data
def predictions_dashboard(project_id):
"""
Predictions Dashboard \n
To allow ADMIN user see all necessary metrics for the project/CUSTOMER user
Allows an ADMIN user visualize the Deep Learning model's performance, along side
customer's activities.\n\n
method: GET\n
API/URL must be accessed with GET request and supply project_id in the URL\n
Args:
project_id (str): ID of the poject need to be sent in url. It is made to do so via Front end href
Returns:
view: a url VIEW the project's/CUSTOMER's all required prediction values and visulization data
give user login status. If project CUSTOMER not found then redirect 404.
"""
if not current_user.is_authenticated:
print('not logged in')
return redirect(url_for('login'))
content = None
data = None
model_info = None
data = Project.from_user(current_user.user_id)
if Project.check_auth(current_user.user_id, int(project_id)):
project_specific_data = Project.get_one(current_user.user_id,
int(project_id))
if project_specific_data[0]['model_available']:
model_info = Database.find_one(
collection="models",
query={"project_id": project_specific_data[0]['project_id']})
print(model_info)
try:
# try to match the pages defined in -> pages/<input file>
return render_template('pages/prediction_dashboard.html',
data=data,
project_specific_data=project_specific_data,
model_info=model_info)
except:
return render_template('pages/error-404.html')
class AppCase(RequiresMocks):
def setup_app(self):
self.app = app.test_client()
# Mock the GitHub API
self.mock_github = self.create_patch('app.routes.oauth.github.api')
self.mock_github_api = MagicMock()
self.mock_github.return_value = self.mock_github_api
# Mock the Google API
self.mock_google_user_info = self.create_patch('app.routes.oauth.google.user_info')
self.mock_google_drive_api = self.create_patch('app.routes.oauth.google.drive_api')
self.mock_google_drive_api.return_value = MagicMock()
def teardown_dbs(self):
Issue.drop_collection()
User.drop_collection()
Project.drop_collection()
Attachment.drop_collection()
Comment.drop_collection()
def create_user(self):
self.test_user = User(
name='Numpy Ping',
google_id='12345',
picture='http://foo.com/image.png',
email='*****@*****.**'
)
self.test_user.save()
def create_project(self):
if not hasattr(self, 'test_user'):
self.create_user()
self.test_project = Project(
name='Proj',
repo='pub/bar',
author=self.test_user
)
self.test_project.save()
def create_issue(self):
if not hasattr(self, 'test_project'):
self.create_project()
self.test_issue = Issue(
title='Some important issue',
project=self.test_project,
author=self.test_user
)
self.test_issue.save()
def test_project_show(self):
"""Make sure project_show shows files.
"""
project = Project(name="test", users=[self.user])
project.save()
document_file1 = DocumentFile(path="/test/doc1.xml", projects=[project])
document_file2 = DocumentFile(path="/test/doc2.xml", projects=[project])
document_file1.save()
document_file2.save()
result = self.client.get("/projects/1")
assert "doc1.xml" in result.data
assert "doc2.xml" in result.data
assert application.config["UPLOAD_ROUTE"] + "doc/%s" % document_file1.id in result.data
assert application.config["UPLOAD_ROUTE"] + "doc/%s" % document_file2.id in result.data
def test_project_show(self):
"""Make sure project_show shows files.
"""
project = Project(name="test", user=self.user)
project.save()
document_file1 = DocumentFile(path="/test/doc1.xml", projects=[project])
document_file2 = DocumentFile(path="/test/doc2.xml", projects=[project])
document_file1.save()
document_file2.save()
result = self.client.get("/projects/1")
assert "doc1.xml" in result.data
assert "doc2.xml" in result.data
assert "/projects/1/documents/1" in result.data
assert "/projects/1/documents/2" in result.data
def test_projects_delete_no_perms(self):
"""Delete projects without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
project.save()
result = self.client.post(application.config["DELETE_ROUTE"], data={
"project_id": project.id,
"obj_type": "project",
"obj_id": project.id,
})
print result.data
assert "login?next=%2Fdelete%2F" in result.data
class LongSentencePlayTests(unittest.TestCase):
def setUp(self):
"""Parse the brief example"""
database.clean()
self.path = "tests/data/plays/"
self.structure_file = self.path + "structure.json"
self.input_file = self.path + "brief_example.xml"
self.input_project = Project()
t.project = self.input_project
self.input_project.document_files.append(
DocumentFile(path=self.input_file))
self.input_project.save()
with open(self.structure_file) as f:
self.json = json.load(f)
self.xml = etree.parse(self.input_file)
self.extractor = StructureExtractor(self.input_project,
self.structure_file, t)
def test_long_speech(self):
"""Test long sentences in combined paragraphs with line breaks
"""
# run the parser
self.extractor.extract(self.input_file)
sentences = self.input_project.sentences
words = WordInSentence.query.filter(WordInSentence.sentence_id == sentences[4].id).all()
self.assertEqual(words[3].surface, "forgeries")
self.assertEqual(words[3].space_before, " ")
self.assertEqual(words[7].surface, "And")
self.assertEqual(words[7].space_before, "\n")
words = WordInSentence.query.filter(WordInSentence.sentence_id == sentences[5].id).all()
self.assertEqual(words[10].surface, "As")
self.assertEqual(words[10].space_before, "\n")
self.assertEqual(words[11].surface, "in")
self.assertEqual(words[11].space_before, " ")
words = WordInSentence.query.filter(WordInSentence.sentence_id == sentences[6].id).all()
self.assertEqual(words[4].surface, "land")
self.assertEqual(words[4].space_before, " ")
self.assertEqual(words[5].surface, "Have")
self.assertEqual(words[5].space_before, "\n")
def get(current_user, workspaceId, projectId):
"""
Get a service by service id or project id
:return: Http Json response
"""
serviceId = request.args.get('serviceId')
if serviceId:
# Get by Service ID
service = Service.get_by_id(serviceId)
if service:
return {
'serviceType': service.serviceType,
'id': service._id,
'createdBy': service.createdBy,
'createdOn': service.createdOn,
'isActive': service.isActive,
'serviceMeta': service.serviceMeta
}
else:
return response('failed', 'service not found', 404)
else:
# Get by Project ID
project = Project.get_by_id(projectId)
services = project.services
payload = []
for service_id in services:
service = Service.get_by_id(service_id)
payload.append({"id": service_id,
"serviceType": service.serviceType,
"isActive": service.isActive,
"serviceMeta": service.serviceMeta})
return {'services': payload}
def test_project_show_process_no_perms(self):
"""Delete files without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
document_file1 = DocumentFile(projects=[project], path="/foo/bar.xml")
structure_file1 = StructureFile(project=project, path="/foo/bar.json")
project.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "-1",
"process-selection": ["1"]
})
assert '"status": "OK",' not in result.data
def test_document_show(self):
"""Test the detail document view.
"""
projxyz = Project(name="test project", path="/test-path/",
user=self.user)
docxyz = DocumentFile(path="/test-path/test-file.xml",
projects=[projxyz])
docxyz.save()
projxyz.save()
#TODO: why is this necessary? why does sqlalchemy complain otherwise
docid = docxyz.id
result = self.client.get("/projects/1/documents/1")
assert "/uploads/" + str(docid) in result.data
assert "test-file.xml" in result.data
def test_project_show_no_post(self):
"""Try sending an empty post to project_show.
"""
project = Project(name="test", user=self.user)
project.save()
result = self.client.post("/projects/1", data={
"create-submitted": "true"
})
assert "You must select a file" in result.data
result = self.client.post("/projects/1", data={
"process-submitted": "true"
})
assert "You must select at least one document file"
def setUp(self):
database.clean()
self.client = application.test_client()
self.user1 = user_datastore.create_user(email="*****@*****.**",
password="******")
self.user2 = user_datastore.create_user(email="*****@*****.**",
password="******")
db.session.commit()
with self.client.session_transaction() as sess:
sess["user_id"] = self.user1.get_id()
sess["_fresh"] = True
self.project1 = Project(name="Foos project")
self.project2 = Project(name="Bars project")
self.user1.add_project(self.project1, ProjectsUsers.ROLE_ADMIN)
self.user2.add_project(self.project2, ProjectsUsers.ROLE_ADMIN)
self.project1.save()
self.project2.save()
def setUp(self):
"""Reset the DB and create a dummy project and document.
"""
database.clean()
self.client = application.test_client()
user = User()
db.session.add(user)
db.session.commit()
project = Project(name="Bars project", user=user)
project.save()
self.file_handle, self.file_path = tempfile.mkstemp()
self.file = os.fdopen(self.file_handle, "r+")
self.file.write("foobar")
self.file_name = os.path.split(self.file_path)[1]
document_file = DocumentFile(projects=[project], path=self.file_path)
document_file.save()
def create_project(self):
if not hasattr(self, 'test_user'):
self.create_user()
self.test_project = Project(
name='Proj',
repo='pub/bar',
author=self.test_user
)
self.test_project.save()
def test_project_show_process_no_perms(self):
"""Process files without proper permissions.
"""
project = Project(name="foo")
rel = self.user.add_project(project, role=ProjectsUsers.ROLE_USER)
document_file = DocumentFile(projects=[project], path="/foo/bar.xml")
structure_file = StructureFile(project=project, path="/foo/bar.json")
document_file.save()
structure_file.save()
project.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-selection": [str(document_file.id)],
"process-structure_file": str(structure_file.id)
})
assert "You can't do that" in result.data
def test_project_show_process(self, mock_process_files):
"""Test processing a processable group of files.
"""
project = Project(name="test", user=self.user)
project.save()
document_file1 = DocumentFile(projects=[project],
path="/test-path/1.xml")
document_file2 = DocumentFile(projects=[project],
path="/test-path/2.json")
document_file1.save()
document_file2.save()
result = self.client.post("/projects/1", data={
"process-submitted": "true",
"action": "0",
"process-selection": ["1", "2"]
})
assert "Errors have occurred" not in result.data
