def submit(self):
self.app.submitPushButton.setEnabled(False)
fname = self.inputs['fname'].text()
mname = self.inputs['mname'].text()
lname = self.inputs['lname'].text()
sex = self.inputs['sex'].currentText()
age = self.inputs['age'].value()
job = SimpleHandler(self.dbu.AddRecordToTable, self.identityTable,
(fname, mname, lname, sex, age))
self.qu.put(job)
query_data = {
'first_name': fname,
'middle_name': mname,
'last_name': lname,
'sex': sex,
'age': str(age)
}
job2 = DBHandler(fn1=self.dbu.GetColumns,
fn2=self.dbu.Query,
tableName=self.identityTable,
data=query_data)
job2.signals.finished.connect(self.enableSubmitButton)
job2.signals.result.connect(self.update_tree)
self.qu.put(job2)
self.full_list_present = False
def addTweetRecieved():
requestJson = request.get_json()
#dump to console for debugging purposs
print(json.dumps(requestJson, indent=4, sort_keys=True))
tweetID = requestJson['tweetID']
twitterAPI = initApiObject()
r = twitterAPI.request('statuses/show/:%d' % int(tweetID))
print(r.json())
if r.status_code == 200:
#save tweet
isNew = DBHandler.addTweetIfNew(tweetID, r.json())
if isNew == True:
return Response(json.dumps({'message': 'Successfully added'}),
mimetype='application/json')
else:
return Response(json.dumps({'message': 'Already added'}),
mimetype='application/json')
else:
return Response(json.dumps({'message': 'Twitter API error'}),
mimetype='application/json')
return ('', HTTPStatus.OK)
def query(self):
input_data = {}
query_data = {}
input_data['first_name'] = self.inputs['fname'].text()
input_data['middle_name'] = self.inputs['mname'].text()
input_data['last_name'] = self.inputs['lname'].text()
input_data['sex'] = self.inputs['sex'].currentText()
input_data['age'] = str(self.inputs['age'].value())
if input_data['first_name'] != '':
query_data['first_name'] = input_data['first_name']
if input_data['middle_name'] != '':
query_data['middle_name'] = input_data['middle_name']
if input_data['last_name'] != '':
query_data['last_name'] = input_data['last_name']
if input_data['sex'] != 'Select':
query_data['sex'] = input_data['sex']
if input_data['age'] != '0':
query_data['age'] = input_data['age']
if len(query_data) > 0:
job2 = DBHandler(fn1=self.dbu.GetColumns,
fn2=self.dbu.Query,
tableName=self.identityTable,
data=query_data)
job2.signals.result.connect(self.update_tree)
self.qu.put(job2)
self.full_list_present = False
elif not self.full_list_present:
job = DBHandler(fn1=self.dbu.GetColumns,
fn2=self.dbu.GetTable,
tableName=self.identityTable)
job.signals.result.connect(self.update_tree)
self.qu.put(job)
self.full_list_present = True
def calculateProjectDif(project_id):
test_images = db.get_test_project_images(project_id)
dif = []
for image in test_images:
commits = db.get_image_commit(image.id)
accuracies = []
for commit in commits:
accuracies.append(commit.accuracy)
array = numpy.array(accuracies)
e = array.sum() / len(array)
var = (array * array).sum() / len(array) + 1
array.sort()
median = array[len(array) // 2]
if median < e:
flag = -1
else:
flag = 1
D = e / 0.2 + flag * 0.1 * (2 /
(1 + math.e**(-0.7 *
(len(array) / var))) - 1)
dif.append(5 - int(D))
return numpy.array(dif).sum() / len(dif)
def getAnswerFromTags(imageId):
session = db.setup_db()
tags = session.query(db.Tag).filter(db.Tag.image_id == imageId).all()
userIds = []
handler = xp.XMLParser()
for tag in tags:
xml_string = tag.xml_file
handler.parse(xml_string)
userIds.append(tag.worker_id)
session.close()
return handler, userIds
def updateAccuracyAndAbility(imageId, userIds, accuracy, efficiency):
session = db.setup_db()
commits = session.query(
db.CommitEvent).filter(db.CommitEvent.imageid == imageId).all()
for commit in commits:
for i in range(len(userIds)):
if commit.workerid == userIds[i]:
# commit.accuracy=accuracy[i]
acc = accuracy[i]
uid = userIds[i]
session.query(db.CommitEvent)\
.filter(db.CommitEvent.workerid==uid , db.CommitEvent.imageid==imageId)\
.update({"accuracy":float(acc),"efficiency":float(efficiency[i])})
'''根据projectId获得项目对象的label
转化为string列表
'''
projectId = session.query(
db.Image).filter(db.Image.id == imageId).one().project_id
labels = session.query(db.Project_Label).filter(
db.Project_Label.project_id == projectId).all()
labels = [x.labels for x in labels]
for label in labels:
for i in range(len(userIds)):
abilities = session.query(db.Ability).filter(
db.Ability.user_id == userIds[i],
db.Ability.label_name == label).all()
for ability in abilities:
if ability.label_name == label:
total = ability.label_history_num * ability.accuracy + accuracy[
i]
eff = ability.efficiency * ability.label_history_num + efficiency[
i]
count = ability.label_history_num + 1
total = total / count
eff = eff / count
session.query(db.Ability).filter(
db.Ability.user_id == userIds[i],
db.Ability.label_name == label).update({
db.Ability.accuracy:
float(total),
db.Ability.label_history_num:
int(count),
db.Ability.efficiency:
float(eff)
})
session.close()
def __init__(self,
app=None,
database=None,
id_table_name=None,
visit_table_name=None,
qu=None):
super(Entry, self).__init__()
self.app = app
self.dbu = database
self.identityTable = id_table_name
self.visitTable = visit_table_name
self.qu = qu
self.total_visits = self.dbu.GetTotalVisits(self.visitTable)[0][0]
self.inputs = {
'fname': self.app.fnameLineEdit,
'mname': self.app.mnameLineEdit,
'lname': self.app.lnameLineEdit,
'sex': self.app.sexComboBox,
'age': self.app.ageSpinBox
}
self.patient_id = None
self.full_list_present = True
self.app.submitPushButton.setEnabled(False)
self.inputs['sex'].addItems(['Select', 'M', 'F'])
self.app.clearPushButton.clicked.connect(self.reset_age)
self.app.clearPushButton.clicked.connect(self.reset_sex)
self.app.submitPushButton.clicked.connect(self.submit)
self.app.treeWidget.itemClicked.connect(self.save_for_queue)
self.app.addPushButton.clicked.connect(self.add_to_queue)
self.inputs['fname'].textChanged.connect(self.query)
self.inputs['mname'].textChanged.connect(self.query)
self.inputs['lname'].textChanged.connect(self.query)
self.inputs['sex'].currentIndexChanged.connect(self.query)
self.inputs['age'].valueChanged.connect(self.query)
job = DBHandler(fn1=self.dbu.GetColumns,
fn2=self.dbu.GetTable,
tableName=self.identityTable)
job.signals.result.connect(self.update_tree)
job.signals.finished.connect(self.enableSubmitButton)
self.qu.put(job)
def getLatestTweet():
latestTweet = DBHandler.getLatestTweet()
if latestTweet == False:
return Response(json.dumps({'message': 'Error retrieving'}),
mimetype='application/json')
else:
tweetURL = 'https://twitter.com/' + latestTweet['tweetData']['user'][
'screen_name'] + '/status/' + latestTweet['tweetData']['id_str']
r = requests.request(
"GET",
'https://publish.twitter.com/oembed?omit_script=true&hide_thread=true&url='
+ tweetURL)
print(r.json())
return Response(json.dumps(r.json()), mimetype='application/json')
from flask_bcrypt import Bcrypt
from wtforms import StringField, PasswordField, BooleanField, SubmitField
from wtforms.validators import DataRequired
from utils import DBHandler, log_parser, parse_string
app = Flask(__name__)
app.config['SECRET_KEY'] = os.environ.get('SECRET_TOKEN')
bcrypt = Bcrypt()
login_manager = LoginManager()
bcrypt.init_app(app)
login_manager.init_app(app)
login_manager.login_view = 'login'
login_manager.login_message_category = 'info'
db_handler = DBHandler('data.db')
@login_manager.user_loader
def load_user(user_id):
return User.get(user_id)
class User(UserMixin):
users = {0: None}
def __init__(self, username, password):
super().__init__()
self.username = username
self.password = bcrypt.generate_password_hash(password).decode('utf-8')
self.id = max(self.users) + 1
self.users[self.id] = self
def work(imageId, markmode):
print('imageID : ', imageId)
print('markMode : ', markmode)
session = db.setup_db()
image = session.query(db.Image).filter(db.Image.id == imageId).one()
# get workers' answers
handler, userIds = getAnswerFromTags(imageId)
usr_ans_rects = handler.allPoints
usr_ans_tags = handler.allTags
print('user_ans_rect = ', usr_ans_rects)
print('user_ans_tag = ', usr_ans_tags)
width = handler.pictureWidth
height = handler.pictureHeight
usr_ans_eff = handler.times
efficiency = []
projectId = -1
if image.is_test == True:
projectId = session.query(
db.Image).filter(db.Image.id == imageId).one().project_id
else:
print("is not test answer")
id = session.query(
db.Image).filter(db.Image.id == imageId).one().project_id
projectId = session.query(
db.Project).filter(db.Project.id == id).one().test_project_id
dif = pd.calculateProjectDif(projectId)
if math.isnan(dif):
# 该项目没有测试集，故难度是nan
dif = 1
print('difficulty = ', dif)
# for time in usr_ans_eff:
# eff_=
# efficiency.append(time/(6-dif))
#
# print('efficiency = ',efficiency)
try:
# generate answer
res_centers, res_labels, label_accuracy = generateResult(
handler, markmode)
if markmode == 0:
accuracy = []
for ans in usr_ans_rects:
tmp = clu.cal_rect_accuracy(ans, res_centers)
accuracy.append(tmp)
# update commit event & user ability
print('result rect = ', res_centers)
print('result label = ', res_labels)
print('rect accuracy = ', accuracy)
print('label accuracy = ', label_accuracy)
for i in range(len(accuracy)):
accuracy[i] = 0.8 * accuracy[i] + 0.2 * label_accuracy[i]
# 计算效率
for i in range(len(usr_ans_eff)):
eff_ = math.exp(-usr_ans_eff[i] * 0.1) * accuracy[i] / dif
efficiency.append(eff_)
print('efficiency = ', efficiency)
updateAccuracyAndAbility(imageId, userIds, accuracy, efficiency)
#generate tag object
max_rect_accuracy = 0.0
ptr = -1
for i in range(len(accuracy)):
if accuracy[i] > max_rect_accuracy:
max_rect_accuracy = accuracy[i]
ptr = i
if ptr != -1:
session.query(db.Tag).filter(db.Tag.worker_id==userIds[ptr],db.Tag.image_id==imageId) \
.update({db.Tag.is_result:True})
session.query(db.Image).filter(db.Image.id == imageId).update(
{db.Image.is_finished: True})
pass
elif markmode == 1:
max_rect_accuracy = 0.0
ptr = -1
for i in range(len(label_accuracy)):
if label_accuracy[i] > max_rect_accuracy:
max_rect_accuracy = label_accuracy[i]
ptr = i
if ptr != -1:
session.query(
db.Tag).filter(db.Tag.worker_id == userIds[ptr],
db.Tag.image_id == imageId).update(
{db.Tag.is_result: True})
session.query(db.Image).filter(db.Image.id == imageId).update(
{db.Image.is_finished: True})
tags = session.query(
db.Tag).filter(db.Tag.image_id == imageId).all()
answerTag = db.Tag
for tag in tags:
if tag.worker_id == userIds[ptr]:
answerTag = tag
break
accuracy = []
for j in range(0, len(tags)):
acc = cs.cal_similarity(tags[j], answerTag, width, height)
accuracy.append(acc * 0.8 + label_accuracy[j] * 0.2)
updateAccuracyAndAbility(imageId, userIds, accuracy, efficiency)
pass
# calculate accuracy and update the db
# modify CommitEvent
# modify Ability
# set Image to finished
# set Tag to isResult
pass
except Exception:
print('generate failed')
pass
pass
session.close()
def workOne(userId, imageId, markmode):
# get the answer
session = db.setup_db()
image = session.query(db.Image).filter(db.Image.id == imageId).one()
tag = session.query(db.Tag).filter(db.Tag.image_id == imageId,
db.Tag.is_result == True).one()
usrTag = session.query(db.Tag).filter(db.Tag.image_id == imageId,
db.Tag.worker_id == userId).one()
# calculate accuracy and update the db
print('already has result , calculate accuracy')
print('update db')
handler = xp.XMLParser()
handler.parse(tag.xml_file)
pointsAnswer = handler.allPoints
print(pointsAnswer)
labelAnswer = handler.allTags
handler = xp.XMLParser()
handler.parse(usrTag.xml_file)
accuracy = []
userIds = []
userIds.append(userId)
efficiency = []
projectId = -1
if image.is_test:
projectId = image.project_id
else:
id = session.query(
db.Image).filter(db.Image.id == imageId).one().project_id
projectId = session.query(
db.Project).filter(db.Project.id == id).one().test_project_id
dif = pd.calculateProjectDif(projectId)
efficiency.append(handler.times[0] / dif)
if markmode == 0:
usr_ans_rects = handler.allPoints
labels = labelAnswer
# print(labels)
while [] in labels:
labels.remove([])
names = []
values = []
for x in labels:
for y in x:
names.append(y[0])
values.append(y[1])
names = list(set(names))
values = list(set(values))
new_labels = []
for x in labels:
nx = []
for y in x:
nx.append([names.index(y[0]), values.index(y[1])])
new_labels.append(nx)
labels = handler.allTags
while [] in labels:
labels.remove([])
names = []
values = []
for x in labels:
for y in x:
names.append(y[0])
values.append(y[1])
names = list(set(names))
values = list(set(values))
usr_labels = []
for x in labels:
nx = []
for y in x:
nx.append([names.index(y[0]), values.index(y[1])])
usr_labels.append(nx)
acc = clu.cal_rect_accuracy(
usr_ans_rects[0], pointsAnswer[0]) * 0.8 + clu.cal_label_accuracy(
usr_labels[0], new_labels[0])
accuracy.append(acc)
updateAccuracyAndAbility(imageId, userIds, accuracy, efficiency)
return [accuracy[0], efficiency[0]]
elif markmode == 1:
usr_ans_rects = handler.allPoints
labels = labelAnswer
# print(labels)
while [] in labels:
labels.remove([])
names = []
values = []
for x in labels:
for y in x:
names.append(y[0])
values.append(y[1])
names = list(set(names))
values = list(set(values))
new_labels = []
for x in labels:
nx = []
for y in x:
nx.append([names.index(y[0]), values.index(y[1])])
new_labels.append(nx)
labels = handler.allTags
while [] in labels:
labels.remove([])
names = []
values = []
for x in labels:
for y in x:
names.append(y[0])
values.append(y[1])
names = list(set(names))
values = list(set(values))
usr_labels = []
for x in labels:
nx = []
for y in x:
nx.append([names.index(y[0]), values.index(y[1])])
usr_labels.append(nx)
acc = cs.cal_similarity(usrTag, tag, handler.pictureWidth, handler.
pictureHeight) * 0.8 + clu.cal_label_accuracy(
usr_labels[0], new_labels[0])
accuracy.append(acc)
updateAccuracyAndAbility(imageId, userIds, accuracy, efficiency)
return [accuracy[0], efficiency[0]]
pass
argparser.add_argument('-port', default=5000, help='Port to run dashboard on')
argparser.add_argument('-host', default='127.0.0.1', help='Host to run dashboard on')
args = argparser.parse_args()
logger = logging.getLogger('__main__')
logger.setLevel(logging.DEBUG)
stream_handler = logging.StreamHandler()
stream_handler.setLevel(logging.INFO)
stream_handler.setFormatter(logging.Formatter('[%(asctime)s] %(message)s', '%H:%M:%S'))
logger.addHandler(stream_handler)
a_date = datetime.date.today()
db_handler = DBHandler('data.db')
db_handler.create_db()
def set_file_logger():
logger.handlers = [handler for handler in logger.handlers if handler.name != 'file_handler']
if not os.path.exists('logs'): os.mkdir('logs')
file_handler = logging.FileHandler(os.path.join('logs', str(a_date) + '.log'))
formatter = logging.Formatter('%(asctime)s:%(module)s:%(levelname)s:%(message)s', '%H-%M-%S')
file_handler.setFormatter(formatter)
file_handler.set_name('file_handler')
logger.addHandler(file_handler)
from tornado.options import define, options
define("port", default=8000, help="run on the given port", type=int)
# DbHandle = DataBaseHandle.DataBaseHandle('127.0.0.1','root','root','ihome',3306)
# cc=DbHandle.selectDb('select ai_name from ih_area_info')
# # DbHandle.selectDb('select * from test')
# print("=======================")
# print(cc)
# print("=======================")
# DbHandle.closeDb()
DbHandle = DataBaseHandle.MysqlTools(
{
'host': '127.0.0.1',
'user': '******',
'passwd': 'root',
'db': 'ihome',
'port': 3306
}, "stream")
sAll = DbHandle.select_all("select * from ih_area_info")
print(sAll)
# client_key = '<YOUR-KEY-HERE>'
# client_secret = '<YOUR-SECRET-HERE>'
# key_secret = '{}:{}'.format(client_key, client_secret).encode('ascii')
# b64_encoded_key = base64.b64encode(key_secret)
# b64_encoded_key = b64_encoded_key.decode('ascii')
# base_url = 'https://api.twitter.com/'
# auth_url = '{}oauth2/token'.format(base_url)
# print(auth_url)
# auth_headers = {'Authorization': 'Basic {}'.format(b64_encoded_key),
import os
import json
from tgapi import TgBot
from utils import DBHandler, parse_string, log_parser
import logging
logger = logging.getLogger('__main__')
bot = TgBot(os.environ.get('BOT_TOKEN'), os.environ.get('CHAT_ID'))
db_handler = DBHandler('data.db')
with open('localization.json', 'r') as read_file:
loc_text = json.load(read_file)
try:
bot.context = {
user.chat_id: {
'lang': user.lang
}
for user in db_handler.get_users()
}
except:
bot.context = {}
def get_loc_text(text_name, chat_id):
return loc_text[text_name][bot.context.get(chat_id, {}).get('lang', 'rus')]
def loc_reply(message, text_name, **kwargs):
message_text = get_loc_text(text_name, message.chat_id)