def test_base(session, db):
request = Request(id=13, url='http://www.google.com', status='done')
keyword = Keyword(word='carros', requests=[request])
label = Label(name='veículos', restrict=True, keywords=[keyword])
db.create_all()
db.session.add(label)
db.session.commit()
#test create
assert session.query(Request).count() == 1
assert session.query(Keyword).count() == 1
assert session.query(Label).count() == 1
#test read
assert Request.query.filter_by(
id=13).first().url == "http://www.google.com"
assert Keyword.query.filter_by(word='carros').first().requests[0].id == 13
assert Label.query.filter_by(name='veículos').first().restrict == True
#test update
request.url = "http://www.twitch.tv"
db.session.commit()
assert Request.query.filter_by(id=13).first().url == "http://www.twitch.tv"
label.restrict = False
assert not Label.query.filter_by(name='veículos').first().restrict
#test delete
db.session.delete(label)
db.session.commit()
assert Label.query.filter_by(name='veículos').first() == None
assert Request.query.filter_by(id=13).first().url == "http://www.twitch.tv"
assert Keyword.query.filter_by(word='carros').first().requests[0].id == 13
def addLabel(labelInfo):
label = Label()
getId = False
if 'id' in labelInfo:
label = getLabelById(labelInfo['id'])
getId = True
else:
label.label_num = 0
if len(labelInfo['name']) > 30:
labelInfo['name'] = labelInfo['name'][0:30]
label.label_name = labelInfo['name']
if len(labelInfo['slug']) > 30:
labelInfo['slug'] = labelInfo['slug'][0:30]
label.label_slug = labelInfo['slug']
db.session.add(label)
if getId is True:
db.session.flush()
db.session.commit()
return label
def new_label():
label_data = request.get_json()
try:
label = Label(**label_data)
db.session.add(label)
db.session.commit()
return label.to_dict()
except exc.IntegrityError as ex:
db.session.rollback()
flask.abort(400, str(ex))
except ValueError as ex:
flask.abort(400, str(ex))
def create(request):
if request.method == 'POST':
dataset = Dataset()
dataset.owner = request.user
dataset.name = request.POST['name']
dataset.number_of_labels = request.POST['number_of_labels']
dataset.description = request.POST['description']
if not dataset.privacy_validation(request.POST['privacy']):
return render(request, '400.html', status=400)
dataset_file = request.FILES['dataset']
reader = csv.reader(dataset_file, encoding='utf-8')
header_list = reader.next()
label_name = request.POST.get('label_name', 'CLASS')
append_label_column = request.POST.get('append_label_column', False)
if not append_label_column:
label_index = header_list.index(label_name)
header_list.pop(label_index)
header_list.append(label_name)
dataset.header = csvlist_to_string(header_list).strip()
dataset.save()
samples_count = 0
for row_list in reader:
samples_count += 1
if not append_label_column:
label_string = row_list.pop(label_index)
row = csvlist_to_string(row_list).strip()
sample = Sample(dataset=dataset,
data=row,
original_index=samples_count)
sample.save()
if not append_label_column and label_string:
label = Label(owner=request.user,
sample=sample,
label=label_string)
label.save()
sample.times_labeled = 1
sample.save()
dataset.number_of_samples = samples_count
dataset.save()
return HttpResponseRedirect(
reverse('datasets_show', args=(dataset.id, )))
def create(request):
if request.method == 'POST':
dataset = Dataset()
dataset.owner = request.user
dataset.name = request.POST['name']
dataset.number_of_labels = request.POST['number_of_labels']
dataset.description = request.POST['description']
if not dataset.privacy_validation(request.POST['privacy']):
return render(request, '400.html', status=400)
dataset_file = request.FILES['dataset']
reader = csv.reader(dataset_file, encoding='utf-8')
header_list = reader.next()
label_name = request.POST.get('label_name', 'CLASS')
append_label_column = request.POST.get('append_label_column', False)
if not append_label_column:
label_index = header_list.index(label_name)
header_list.pop(label_index)
header_list.append(label_name)
dataset.header = csvlist_to_string(header_list).strip()
dataset.save()
samples_count = 0
for row_list in reader:
samples_count += 1
if not append_label_column:
label_string = row_list.pop(label_index)
row = csvlist_to_string(row_list).strip()
sample = Sample(dataset=dataset, data=row,
original_index=samples_count)
sample.save()
if not append_label_column and label_string:
label = Label(owner=request.user, sample=sample,
label=label_string)
label.save()
sample.times_labeled = 1
sample.save()
dataset.number_of_samples = samples_count
dataset.save()
return HttpResponseRedirect(
reverse('datasets_show', args=(dataset.id,)))
def test_label(app):
label = Label(
name='Some awesome label',
color='#ff0000'
)
assert label.name == 'Some awesome label'
assert label.color == '#ff0000'
def add_label():
form = LabelForm()
if form.validate_on_submit():
label = Label(name=form.name.data, description=form.description.data)
db.session.add(label)
db.session.commit()
flash('données enregistrées')
return redirect(url_for('admin.label'))
return render_template('admin/label/add_label.html', form=form)
def test_label(database):
label = Label(uid='blacklabel',
description='Special label for special society')
database.session.add(label)
database.session.commit()
assert label.uid == 'blacklabel'
assert label.description == 'Special label for special society'
assert label.created_at is not None
assert label.records == []
# missing all parameters
label = Label()
database.session.add(label)
database.session.commit()
assert label.uid is not None
assert label.description == ''
assert label.created_at is not None
assert label.records == []
def add_label(labels, blog):
for label_item in labels:
label = Label.query.filter_by(name=label_item).first()
# labels表中不存在指定名称的标签
if label is None:
# 创建新标签
label = Label(name=label_item)
label.blogs.append(blog)
db.session.add(label)
def add():
form = LabelForm()
if form.validate_on_submit():
label = Label(name=form.name.data, description=form.description.data)
db.session.add(label)
db.session.commit()
flash(_('Nouvelle etude ajouté avec succèss!'))
return redirect(url_for('label.detail', id=label.id))
return render_template('label/form.html', form=form)
def create_label():
payload = request.get_json()
label = Label(payload['name'])
db.session.add(label)
db.session.commit()
result = {'id': label.id, 'label': label_to_json(label)}
return jsonify(result=result)
def delete(label_dict=None, raw=False):
content = label_dict if label_dict else request.json
user_id = label_dict["user_id"] if label_dict else get_jwt_identity()
user = User.objects(id=user_id).first()
task = Task.objects(id=content["task_id"]).first()
if task in user.tasks:
label = Label.objects(id=content["id"]).first()
if label and label in task.labels:
task.labels.remove(label)
task.save()
return (jsonify(), 200) if not raw else None
abort(404)
def add(label_dict=None, raw=False):
content = label_dict if label_dict else request.json
user_id = label_dict["user_id"] if label_dict else get_jwt_identity()
user = User.objects(id=user_id).first()
task = Task.objects(id=content["task_id"]).first()
if task in user.tasks:
label = Label()
label.color = content["color"]
label.name = content["name"]
label.isthing = content["isthing"]
label.save()
task.labels.append(label)
task.save()
return (label.to_json(), 200) if not raw else label
abort(404)
def jsonToLabels(json):
labels = []
for item in json:
label = Label()
label.id = item.get('id')
label.supplier_name = item.get('supplier_name')
label.print_material_type = item.get('print_material_type')
label.print_material = item.get('print_material')
label.url = item.get('url')
label.width = item.get('width')
label.grammage_or_height = item.get('grammage_or_height')
if item.get('copies'):
for i in range(item.get('copies')):
labels.append(label)
else:
labels.append(label)
return labels
def import_label():
"""Import Data Label to Database"""
file = os.path.abspath('app/resources') + "/label.json"
json_data = open(file).read()
json_obj = json.loads(json_data)
for row in json_obj:
try:
data = Label(id=row['id'], item=row['item'])
db.session.add(data)
db.session.commit()
print('Import data label {} done.'.format(data.item))
except exc.IntegrityError:
print('Data label {} already exists'.format(data.item))
db.session.rollback()
def update(label_dict=None, raw=False):
content = label_dict if label_dict else request.json
user_id = label_dict["user_id"] if label_dict else get_jwt_identity()
user = User.objects(id=user_id).first()
task = Task.objects(id=content["task_id"]).first()
if task in user.tasks:
label = Label.objects(id=content["id"]).first()
if label and label in task.labels:
color = content.get("color", None)
name = content.get("name", None)
isthing = content.get("isthing", None)
label.color = color if color is not None else label.color
label.name = name if name is not None else label.name
label.isthing = isthing if isthing is not None else label.isthing
label.save()
return (label.to_json(), 200) if not raw else label
abort(404)
def import_row(row, type_id, language_ids):
imported_species = False
imported_common_name = False
language_id = language_ids.get(row['col:language'])
label = Label.query.filter(Label.name == row['col:scientificName']).first()
if not label: # scientific name not found in database
label = Label(name=row['col:scientificName'], type_id=type_id)
db.session.add(label)
imported_species = True
if row['col:name'] is not np.NaN: # common name is not blank
if language_id: # language exists in database
common_name = CommonName.query.join(Label).filter(Label.name == row['col:scientificName'], CommonName.name == row['col:name'], CommonName.language_id == language_id).first()
if not common_name: # common name not found in database
common_name = CommonName(name=row['col:name'], label=label, language_id=language_id)
db.session.add(common_name)
imported_common_name = True
return [imported_species, imported_common_name]
def add_label():
res = {
"code": 0,
"message": "",
"data": {},
}
try:
name = request.json['name']
type = request.json['type']
label = Label(name=name, type=type)
db.session.add(label)
db.session.commit()
except Exception as e:
res["code"] = 10000
res["message"] = str(e)
return jsonify(res)
from app import app, db
from app.models import Label, Memo
import random
import time
if __name__ == "__main__":
db.drop_all()
db.create_all()
for i in range(10):
label = Label("{}번째 라벨".format(i + 1))
db.session.add(label)
db.session.commit()
for i in range(30):
memo = Memo("{}번째 메모".format(i + 1), "내용내용내용내용내용내용내용내용내용",
int(time.time() * 1000 * 0.95), [])
db.session.add(memo)
db.session.commit()
for i in range(100):
label = random.choice(Label.query.all())
memo = random.choice(Memo.query.all())
if label not in memo.labels:
memo.labels.append(label)
db.session.add(memo)
db.session.commit()
app.run(debug=True)
def test_classifier(self):
default_req = [Request(id=0, url=' ', status='done')]
default_kw_armas = [
Keyword(word='arma', requests=[default_req[0]]),
Keyword(word='munição', requests=[default_req[0]]),
Keyword(word='calibre', requests=[default_req[0]]),
Keyword(word='revólver', requests=[default_req[0]]),
Keyword(word='cano', requests=[default_req[0]]),
Keyword(word='carabina', requests=[default_req[0]]),
Keyword(word='espingarda', requests=[default_req[0]])
]
default_kw_cigarros = [
Keyword(word='cigarro', requests=[default_req[0]]),
Keyword(word='vape', requests=[default_req[0]]),
Keyword(word='narguile', requests=[default_req[0]]),
Keyword(word='fumar', requests=[default_req[0]]),
Keyword(word='tragar', requests=[default_req[0]]),
Keyword(word='tabaco', requests=[default_req[0]]),
Keyword(word='nicotina', requests=[default_req[0]]),
Keyword(word='vaporizador', requests=[default_req[0]]),
Keyword(word='ervas', requests=[default_req[0]])
]
default_kw_prost = [
Keyword(word='sexo', requests=[default_req[0]]),
Keyword(word='prostituta', requests=[default_req[0]]),
Keyword(word='fetiche', requests=[default_req[0]]),
Keyword(word='cache', requests=[default_req[0]]),
Keyword(word='acompanhante', requests=[default_req[0]]),
Keyword(word='programa', requests=[default_req[0]]),
Keyword(word='seios', requests=[default_req[0]]),
Keyword(word='bunda', requests=[default_req[0]]),
Keyword(word='travesti', requests=[default_req[0]]),
Keyword(word='gostosa', requests=[default_req[0]])
]
default_kw_remedio = [
Keyword(word='remédio', requests=[default_req[0]]),
Keyword(word='aborto', requests=[default_req[0]]),
Keyword(word='comprimido', requests=[default_req[0]]),
Keyword(word='secundários', requests=[default_req[0]]),
Keyword(word='efeitos', requests=[default_req[0]])
]
default_kw_serv = [
Keyword(word='operadora', requests=[default_req[0]]),
Keyword(word='cabo', requests=[default_req[0]]),
Keyword(word='assinatura', requests=[default_req[0]]),
Keyword(word='liberação', requests=[default_req[0]]),
Keyword(word='sem', requests=[default_req[0]]),
Keyword(word='aparelhos', requests=[default_req[0]]),
Keyword(word='net', requests=[default_req[0]]),
Keyword(word='vivo', requests=[default_req[0]])
]
labels = [
Label(name='Armas de fogo',
restrict=True,
keywords=default_kw_armas),
Label(name='Cigarros', restrict=True,
keywords=default_kw_cigarros),
Label(name='Prostutuição',
restrict=True,
keywords=default_kw_prost),
Label(name='Remédios', restrict=True, keywords=default_kw_remedio),
Label(name='Serviços ilegais',
restrict=True,
keywords=default_kw_serv),
Label(name='Site permitido', restrict=False, keywords=[])
]
kws = default_kw_armas + default_kw_cigarros + default_kw_prost + default_kw_remedio + default_kw_serv
stat = []
for status in self.classifier.classify(url='https://www.uol.com.br/',
kws=kws,
labels=labels):
stat += [status]
self.assertEqual(stat[-1]['label'], "permitted")
def create_labels_from_text(text, publicity="public"):
labels = []
label = Label()
label.set_text(text)
label.type = "generic"
label.publicity = publicity
label_slug = label.create_slug()
existing_label = Label.query.filter_by(slug=label_slug)
if existing_label.count() > 0:
label = existing_label[0]
else:
label.slug = label_slug
db.session.add(label)
db.session.flush()
labels.append(label.id)
split_text = text.split(":")
if len(split_text) is 2 and len(
split_text[1]) > 0 and split_text[0] in SPECIAL_LABELS:
modifier_label = Label()
modifier_label.set_text(split_text[1])
modifier_label.type = split_text[0]
modifier_label.publicity = publicity
modifier_label_slug = modifier_label.create_slug()
existing_modifier_label = Label.query.filter_by(
slug=modifier_label_slug)
if existing_modifier_label.count() > 0:
modifier_label = existing_modifier_label[0]
else:
modifier_label.slug = modifier_label_slug
db.session.add(modifier_label)
db.session.flush()
labels.append(modifier_label.id)
db.session.commit()
return labels
def batch(request, batch_id):
if request.method == 'GET':
_labels = [
{
"id": "rec0",
"x": 290,
"y": 245,
"width": 650,
"height": 341,
"brand": "b0",
"model": "m0",
"color": "c0",
"nn": "n0"
},
]
_images = [
{
"src": "static/dataset/_00.jpg",
"labels": _labels
},
]
#x=Batch.objects.get(pk=batch_id)
BID = Batch.objects.get(pk=batch_id)
IMGS = Image.objects.filter(batch=BID)
data = []
for I in IMGS:
labels = []
for L in Label.objects.filter(image=I):
labels.append({
#"id":"rec%d"%L.id,
"x": L.x,
"y": L.y,
"width": L.width,
"height": L.height,
"brand": L.brand,
"model": L.model,
"color": L.color,
"nn": L.nickname,
})
item = {'src': I.src_path, 'labels': labels}
data.append(item)
return JsonResponse(data, safe=False)
if request.method == 'POST':
client_data = json.loads(request.POST.get('client_data'))
print client_data
images = client_data
for i in images:
print "i: %s" % i
m = Image.objects.get(src_path=i['src'])
Label.objects.filter(image=m).delete()
for j in i['labels']:
m = Image.objects.get(src_path=i['src'])
q = Label(image=m,
x=j['x'],
y=j['y'],
width=j['width'],
height=j['height'],
brand=j['brand'],
model=j['model'],
color=j['color'],
nickname=j['nn'])
q.save()
if request.POST.get('rework'):
b = Batch.objects.get(pk=batch_id)
b.num_rework = b.num_rework + 1
b.status = md.TAGGING
b.save()
else:
if request.POST.get('submission'):
if request.POST.get(
'submission'
) == '1' and 'username' in request.session: #reviewer
_username = request.session['username']
u = User.objects.get(username=_username)
mu = MyUser.objects.get(user=u)
b = Batch.objects.get(pk=batch_id)
b.reviewer = mu
b.status = md.DONE
b.save()
print "DONE:: batch: %s, status: %s, submission: %s" % (
b, b.status, request.POST.get('submission'))
else: #labeller
b = Batch.objects.get(pk=batch_id)
b.status = md.REVIEWING
b.save()
print "TO REVIEWER:: batch: %s, status: %s, submission: %s" % (
b, b.status, request.POST.get('submission'))
else:
# b=Batch.objects.get(pk=batch_id)
# if 'username' in request.session: #debug view92
# _username=request.session['username']
# print "#debug view92:: username: %s"%_username
# if b.labeller:
# print "#debug view92:: the batch being labelled"
# else: #labelling for the first time
# u=User.objects.get(username=_username)
# mu=MyUser.objects.get(user=u)
# b.status=md.TAGGING
# b.labeller=mu
# b.save()
print "batch: %s, status: %s" % (b, b.status)
return JsonResponse(images, safe=False)
def label_init_func(row):
c = Label(id=row['id'],
name=row['nom'],
description=row['description'])
return c
def label_fake():
labels = ['固定资产', '房租', '水电', '差旅', '培训', '招待费', '员工福利', '电话费', '运费', '补贴', '其他', '审计评估费', '税费', '广告费']
for label in labels:
l = Label(label_name=label)
db.session.add(l)
db.session.commit()
Keyword(word='secundários', requests=[default_req[0]]),
Keyword(word='efeitos', requests=[default_req[0]])
]
default_kw_serv = [
Keyword(word='operadora', requests=[default_req[0]]),
Keyword(word='cabo', requests=[default_req[0]]),
Keyword(word='assinatura', requests=[default_req[0]]),
Keyword(word='liberação', requests=[default_req[0]]),
Keyword(word='sem', requests=[default_req[0]]),
Keyword(word='aparelhos', requests=[default_req[0]]),
Keyword(word='net', requests=[default_req[0]]),
Keyword(word='vivo', requests=[default_req[0]])
]
labels = [
Label(name='Armas de fogo', restrict=True, keywords=default_kw_armas),
Label(name='Cigarros', restrict=True, keywords=default_kw_cigarros),
Label(name='Prostituição', restrict=True, keywords=default_kw_prost),
Label(name='Remédios', restrict=True, keywords=default_kw_remedio),
Label(name='Serviços ilegais', restrict=True, keywords=default_kw_serv),
Label(name='Site permitido', restrict=False, keywords=[])
]
with app.app_context():
db.create_all()
for label in labels:
db.session.add(label)
db.session.commit()
def upload_firebase():
with open(app.config['FIREBASE_DUMP']) as f:
firebase = json.loads(f.read())
for (person_id, person) in firebase['persons']['data'].items():
if 'name' not in person:
continue
new_person = Person()
new_person.first_name = person['name'].split(' ')[0].title()
if len(person['name'].split(' ')) > 1:
new_person.last_name = ' '.join(
person['name'].split(' ')[1:]).title()
else:
new_person.last_name = ' '
new_person.name = person['name']
new_person.slug = new_person.create_slug()
#if new_person.first_name == "David":
# pdb.set_trace()
if Person.query.filter(Person.first_name == new_person.first_name,
Person.last_name
== new_person.last_name).count() is 0:
print("adding person %s" % new_person.slug)
db.session.add(new_person)
new_tag = Tag()
new_tag = new_tag.initialize("Created from firebase",
new_person.id,
new_person.id,
subject_slug=new_person.slug,
originator_slug=new_person.slug,
type="metadata",
publicity="private")
db.session.add(new_tag)
firebase['persons']['data'][person_id]['id'] = new_person.id
firebase['persons']['data'][person_id]['slug'] = new_person.slug
else:
firebase['persons']['data'][person_id]['id'] = Person.query.filter(
Person.first_name == new_person.first_name,
Person.last_name == new_person.last_name)[0].id
firebase['persons']['data'][person_id][
'slug'] = Person.query.filter(
Person.first_name == new_person.first_name,
Person.last_name == new_person.last_name)[0].slug
for (tag_id, tag) in firebase['tags']['data'].items():
if 'subject' not in tag or tag['subject'] in [
'Climbing_Person7', 'Climbing_person8', '0', '1', '2', '3'
]:
continue
if tag['subject'] == 'Rachel_Zucker3':
tag['subject'] = 'Rachel_Zucker1'
if tag['subject'] == 'Seth_Berman16':
tag['subject'] = 'Seth_Berman10'
if tag['subject'] == 'Natalie_Dillon_14':
tag['subject'] = 'Natalie_Dillon11'
if tag['subject'] == 'Sasha_Sheng':
tag['subject'] = 'Sasha_Sheng12'
if tag['subject'] == 'Adrienne_Tran':
tag['subject'] = 'Adrienne_Tran0'
if tag['subject'] == 'Micah_Catlin':
tag['subject'] = 'Micah_Catlin14'
if tag['subject'] == 'Leo_Polovets9':
tag['subject'] = 'Leo_Polovets2'
new_tag = Tag()
publicity = tag['publicity']
type = find_tag_type(tag['label'])
originator_id = Person.query.filter(
Person.first_name == "Benjamin",
Person.last_name == "Reinhardt")[0].id
subject_id = firebase['persons']['data'][tag['subject']]['id']
subject_slug = firebase['persons']['data'][tag['subject']]['slug']
new_tag = new_tag.initialize(tag['label'],
originator_id,
subject_id,
subject_slug=subject_slug,
publicity=publicity,
type=type)
print(new_tag.to_deliverable())
if publicity == "public" and "metadata" not in decode_tag_types(
type) and Label.query.filter(
Label.text == tag['label']).count() == 0:
print("Creating new label %s" % tag['label'])
new_label = Label()
new_label.text = tag['label']
db.session.add(new_label)
new_tag.label = new_label.id
if Tag.query.filter(Tag.slug == new_tag.slug).count() == 0:
print("adding tag %s" % new_tag.slug)
db.session.add(new_tag)
else:
slug = Tag.query.filter(Tag.slug == new_tag.slug)[0].slug
print("tag %s already exists" % slug)
db.session.commit()
return jsonify(firebase)
