def upload_remote_file():
print("test>>>>>")
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
# flash('No file part')
return redirect(request.url)
file = request.files['file']
print((file.filename))
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
# flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
# print(file.read())
# save file
filename = secure_filename(file.filename)
filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(filepath)
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
with open(filepath, 'r', encoding='utf-8') as f:
for line in f:
label, txt = line.split(" ", 1)
ca["test"].insert_one({"txt": txt, "label": label})
return jsonify(data={"status": "success"},
code=200,
message="load success")
return '''
def export_data():
"""
:return:
"""
filepath = request.args.get("filepath")
# # read file
ca = get_mongo_client()
with open("../../data/files/test.json", "w") as f:
# texts = list(ca["test"].find())
# print(texts)
# datas = [delattr(text, "_id") for text in texts if "_id" in text]
annotations = ca["test"].find({}).batch_size(50)
result = []
for annotation in annotations:
data = {
"label": annotation["label"],
"txt": annotation["txt"]
}
result.append(data)
json.dump(result, f)
#
# with open(filepath) as f:
# for line in f:
# label, txt = line.split(" ", 1)
# ca["test"].insert_one({"txt": txt, "label": label})
return send_from_directory('../../data/files', "test.json")
def upload_remote_file():
print("test>>>>>")
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
# flash('No file part')
return redirect(request.url)
file = request.files['file']
print((file.filename))
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
# flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
# print(file.read())
# save file
filename = secure_filename(file.filename)
filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(filepath)
# read file
ca = get_mongo_client()
with open(filepath) as f:
for line in f:
label, txt = line.split(" ", 1)
ca["test"].insert_one({"txt": txt, "label": label})
return redirect(url_for('uploaded_file', filename=filename))
return '''
def load_single_unlabeled():
"""
:return:
"""
# read file
ca = get_mongo_client()
text = ca["test"].find_one({"label": {"$exists": False}})
return jsonify(data={"text": text.get("txt")}, code=200, message="load success")
def check_offline_progress():
"""
:return:
"""
# read file
text = request.form.get("text", "")
label = request.form.get("label", "")
print(text)
print(label)
ca = get_mongo_client()
text = ca["test"].insert_one({"label": label, "text": text})
return jsonify(data={"progress": 50}, code=200, message="annotate success")
def annotate_single_unlabeled():
"""
:return:
"""
# read file
text = request.form.get("text", "")
label = request.form.get("label", "")
print(text)
print(label)
ca = get_mongo_client()
text = ca["test"].insert_one({"label": label, "text": text})
return jsonify(data={}, code=200, message="annotate success")
def upload_remote_file(request):
"""
load data from file to mongodb, this is the main interface to load data
:return:
"""
response = APIResponse()
response.data = {"status": "Failed"}
response.code = 302
if request.method == 'POST':
# check if the post request has the file part
if 'file' in request.FILES:
file = request.FILES['file']
print(file.name)
# if user does not select file, browser also
# submit a empty part without filename
if file.name != '':
if file and allowed_file(file.name):
# save file
filename = secure_filename(file.name)
file_path = os.path.join(UPLOAD_FOLDER, filename)
with open(file_path, 'wb+') as destination:
for chunk in file.chunks():
destination.write(chunk)
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
with open(file_path, 'r', encoding='utf-8') as f:
for line in f:
text = line.strip()
text_uuid = uuid.uuid1()
annotation_data = AnnotationData(text=text,
uuid=text_uuid)
annotation_data_serializer = AnnotationDataSerializer(
annotation_data)
ca["annotation_data"].insert_one(
annotation_data_serializer.data)
response.data = {"status": "success"}
response.code = 200
response.message = "Load SUCCESS"
else:
response.message = "only support 'txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif' file"
else:
response.message = "file name should not been empty"
else:
response.message = "no file has been upload"
else:
response.message = "Only support POST function"
api_serializer = APIResponseSerializer(response)
return JsonResponse(api_serializer.data)
def load_single_unlabeled():
"""
load one unlabeled text from Mongo DB to web
:return:
"""
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
text = ca["annotation_data"].find_one({"label": {"$exists": False}})
return jsonify(data={
"text": text.get("txt"),
"uuid": text.get("uuid")
},
code=200,
message="load success")
def load_local_dataset():
"""
load local unlabeled dataset
:return:
"""
filepath = request.args.get("filepath")
# read file
ca = get_mongo_client()
with open(filepath) as f:
for line in f:
# label, txt = line.split(" ", 1)
ca["test"].insert_one({"txt": line})
return jsonify(data={"status": "success"},
code=200,
message="load success")
def load_single_unlabeled(request):
"""
load one unlabeled text from Mongo DB to web
:return:
"""
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
text = ca["annotation_data"].find_one({"label": ""})
annotation_data = AnnotationData(text=text.get("text"),
uuid=text.get("uuid"))
annotation_data_serializer = AnnotationDataSerializer(annotation_data)
response = APIResponse()
response.data = annotation_data_serializer.data
response.code = 200
serializer = APIResponseSerializer(response)
return JsonResponse(serializer.data)
def load_local_dataset():
"""
load local unlabeled dataset
:return:
"""
if request.method == 'POST':
filepath = request.data.get("filepath")
else:
filepath = request.args.get("filepath")
print(filepath)
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
with open(filepath, 'r', encoding='utf-8') as f:
for line in f:
# label, txt = line.split(" ", 1)
print("get string %s" % line)
label, txt = line.split(" ", 1)
ca["test"].insert_one({"txt": txt, "label": label})
return jsonify(data={"status": "success"}, code=200, message="load success")
def upload_remote_file():
"""
load data from file to mongodb, this is the main interface to load data
:return:
"""
print("test>>>>>")
if request.method == 'POST':
# check if the post request has the file part
if 'file' not in request.files:
# flash('No file part')
return redirect(request.url)
file = request.files['file']
print((file.filename))
# if user does not select file, browser also
# submit a empty part without filename
if file.filename == '':
# flash('No selected file')
return redirect(request.url)
if file and allowed_file(file.filename):
# save file
filename = secure_filename(file.filename)
filepath = os.path.join(app.config['UPLOAD_FOLDER'], filename)
file.save(filepath)
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
with open(filepath, 'r', encoding='utf-8') as f:
for line in f:
txt = line.strip()
text_uuid = uuid.uuid1()
ca["annotation_data"].insert_one({
"txt": txt,
"uuid": text_uuid
})
return jsonify(data={"status": "success"},
code=200,
message="load success")
return jsonify(data={"status": "fail"},
code=302,
message="the upload file is incorrect")
def export_data():
"""
:return:
"""
filepath = request.args.get("filepath")
# # read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
with open("../../data/files/test.json", "w") as f:
# texts = list(ca["test"].find())
# print(texts)
# datas = [delattr(text, "_id") for text in texts if "_id" in text]
annotations = ca["test"].find({}).batch_size(50)
result = []
for annotation in annotations:
data = {"label": annotation["label"], "txt": annotation["txt"]}
result.append(data)
json.dump(result, f)
return send_from_directory('../../data/files', "test.json")
def load_local_dataset(request):
"""
load local unlabeled dataset
:return:
"""
if request.method == 'POST':
file_path = request.body.get("filepath")
else:
file_path = request.GET.get("filepath")
print(file_path)
response = APIResponse()
if os.path.exists(file_path):
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
with open(file_path, 'r', encoding='utf-8') as f:
for line in f:
# label, txt = line.split(" ", 1)
print("get string %s" % line)
text = line.strip()
text_uuid = uuid.uuid1()
annotation_data = AnnotationData(text=text, uuid=text_uuid)
annotation_data_serializer = AnnotationDataSerializer(
annotation_data)
ca["annotation_data"].insert_one(
annotation_data_serializer.data)
response.data = {"status": "success"}
response.code = 200
response.message = "Load SUCCESS"
else:
response.data = {"status": "Failed"}
response.code = 302
response.message = "the specified file is not exist"
serializer = APIResponseSerializer(response)
return JsonResponse(serializer.data)
def export_data(request):
"""
dump data to local user instance folder
:return:
"""
# read file
ca = get_mongo_client(uri='mongodb://localhost:27017/')
with open("../../data/files/data.json", "w") as f:
annotations = ca["annotation_data"].find({}).batch_size(50)
result = []
for annotation in annotations:
data = {
"label": annotation["label"],
"txt": annotation["txt"],
}
result.append(data)
json.dump(result, f)
response = APIResponse()
response.data = {"status": "success"}
response.code = 200
response.message = "export SUCCESS"
serializer = APIResponseSerializer(response)
return JsonResponse(serializer.data)
