def get_camera_group_data(cls, request):
camera_group_id = request.GET['camera_group_id']
if camera_group_id:
try:
check_group = CameraGroup.objects.get(
camera_group_id=camera_group_id,
status=DEFAULT_STATUS_ACTIVE
)
result = {
KEY_CAMERA_GROUP_ID: check_group.camera_group_id,
KEY_CAMERA_GROUP_NAME: check_group.camera_group_name,
KEY_CAMERA_GROUP_COLOR: check_group.camera_group_color,
}
return Result.success(
message='Get group success',
data=result
)
except CameraGroup.DoesNotExist:
return Result.failed(
message='Group doesnt exist!'
)
else:
return Result.failed(
message='Missing data in the request!'
)
def run(dataset):
n_features = len(meta[dataset]['val_name'])
result_online = Result('%s-%s' %(dataset, 'aws-online'), aws=True)
result_baseline = Result('%s-%s' %(dataset, 'aws-baseline'), aws=True)
result_active = Result('%s-%s' %(dataset, 'aws-active'), aws=True)
for repeat in range(0, n_repeat):
print 'Round %d of %d'% (repeat, n_repeat - 1)
ex = AWSOnline(meta[dataset]['model_id'], 1, 0, n_features, meta[dataset]['val_name'], ftype='uniform', error=.1)
test_x, test_y = load_svmlight_file('/Users/Fan/dev/ML/code/binary-classifiers/targets/%s/test.scale' % dataset, n_features)
test_x = test_x.todense()
test_y = [a if a == 1 else 0 for a in test_y]
train_x, train_y = [], []
for i in result_active.index:
q_by_u = result_active.Q_by_U[i]
print 'Active learning with budget %d / %d' % (q_by_u, q_by_u * (n_features + 1))
main = ActiveLearning(ex, (None, None), (test_x, test_y), n_features,
q_by_u * (n_features + 1), 5)
L_unif, L_test = main.do()
result_active.L_unif[i].append(L_unif)
result_active.L_test[i].append(L_test)
result_active.nquery[i].append(ex.get_n_query())
ex = AWSOnline(meta[dataset]['model_id'], 1, 0, n_features, meta[dataset]['val_name'], ftype='uniform', error=.1)
for i in result_online.index:
q_by_u = result_online.Q_by_U[i]
print 'collecting up to budget %d / %d' % (q_by_u, q_by_u * (n_features + 1))
ex.collect_up_to_budget(q_by_u * (n_features + 1))
train_x.extend(ex.pts_near_b)
train_y.extend(ex.pts_near_b_labels)
print 'retraining with %d points' % len(train_y)
# online
e = RBFKernelRetraining(ex.batch_predict, (train_x, train_y), (test_x, test_y), n_features)
L_unif, L_test = e.grid_retrain_in_x()
result_online.L_unif[i].append(L_unif)
result_online.L_test[i].append(L_test)
result_online.nquery[i].append(ex.get_n_query())
# baseline
e = Baseline(ex.batch_predict, (train_x, train_y), (test_x, test_y), n_features)
L_unif, L_test = e.do()
result_baseline.L_unif[i].append(L_unif)
result_baseline.L_test[i].append(L_test)
result_baseline.nquery[i].append(ex.get_n_query())
print result_online
print result_baseline
print result_active
def set_avatar(cls, request):
user_information = jwt.decode(
cls.parse_request_user_data(request).get('token'),
SECRET_TOKEN_KEY)
user_id = user_information[KEY_USER_ID]
avatar = request.FILES['avatar']
try:
XrefAvatarUser.objects.get(user_id=User.objects.get(
user_id=user_id)).delete()
except XrefAvatarUser.DoesNotExist:
return Result.failed(message='Some data error!')
_, serve_urls, exc = StaticServing.save_res(
[avatar], ['user_avatar', f'{user_id}'])
if exc is not None:
raise IOError(f'Save image file failed: {exc}')
img_url = serve_urls[0]
try:
result_create_avatar = XrefAvatarUser.objects.create(
user_id=User.objects.get(user_id=user_id),
avatar_file=img_url,
)
except Exception as exc:
# log.error(f'Save person image to db failed: {exc}')
return Result.failed(
message=f'Save person image to db failed: {exc}')
if not result_create_avatar:
return Result.failed(
message='Have some problem when add image data!')
return Result.success(message='Success set avatar!')
def delete_camera(cls, request):
result_delete_camera = cls.parse_delete_camera_data(request)
if result_delete_camera:
try:
check_camera = Camera.objects.get(
camera_id=result_delete_camera.get(KEY_CAMERA_ID),
project_id=result_delete_camera.get(KEY_PROJECT_ID))
xref_people_group = XrefCameraGroup.objects.select_related('camera_id') \
.filter(camera_id=result_delete_camera[KEY_CAMERA_ID])
xref_people_group.delete()
process_running = list(Process.objects.filter(
camera_id=Camera.objects.get(
camera_id=result_delete_camera.get(KEY_CAMERA_ID)
)
).values())
for process in process_running:
process_cur = Process.objects.get(
process_id=process['process_id']
)
process_cur.delete()
check_camera.status = DEFAULT_STATUS_DEACTIVE
check_camera.save()
return Result.success(
message='Success delete camera'
)
except Camera.DoesNotExist:
return Result.failed(
message='There is no camera with this ID!'
)
else:
return Result.failed(
message='Missing data in the request!'
)
def test_init_with_invalid_values(self):
with pytest.raises(Exception):
result = Result(value="ok", check_success=True, success=False)
with pytest.raises(Exception):
result = Result(
value="ok", check_success=True, success=True, error="something"
)
def add_person_to_group(cls, request):
request_data = cls.parse_add_person_to_group(request)
if request_data:
try:
check_person = People.objects.get(
person_id=request_data.get(KEY_PERSON_ID),
project_id=Project.objects.get(
project_id=request_data.get(KEY_PROJECT_ID)))
group_arr = request.data.get(KEY_GROUP_ARRAY)
for group in group_arr:
check_group = PeopleGroup.objects.get(
people_group_code=group,
project_id=Project.objects.get(
project_id=request_data.get(KEY_PROJECT_ID)))
check_person_xref = XrefPeopleGroup.objects.filter(
person_id=check_person, people_group_id=check_group)
if not check_person_xref:
result = XrefPeopleGroup.objects.create(
person_id=check_person,
people_group_id=check_group,
)
if not result:
return Result.failed(
message=
'Have some problem when add person to group!')
else:
return Result.failed(
message='This person already in this group!')
return Result.success(message='Add person to group success!')
except (People.DoesNotExist, PeopleGroup.DoesNotExist):
return Result.failed(
message='Person id or group id not exist!')
else:
return Result.failed(message='Missing data in the request!')
def choose_project(cls, request):
user_information = jwt.decode(
cls.parse_request_user_data(request).get('token'),
SECRET_TOKEN_KEY)
project_code = request.GET['project_code']
if project_code:
result_choose_project = Project.objects.filter(
project_code=project_code,
status=DEFAULT_STATUS_ACTIVE,
owner_user_id=user_information[KEY_USER_ID],
)
if result_choose_project:
chose_project = list(result_choose_project.values())[0]
result = {
KEY_PROJECT_ID:
chose_project.get(KEY_PROJECT_ID),
KEY_PROJECT_NAME:
chose_project.get(KEY_PROJECT_NAME),
KEY_PROJECT_CODE:
chose_project.get(KEY_PROJECT_CODE),
KEY_PROJECT_SECRET_KEY:
chose_project.get(KEY_PROJECT_SECRET_KEY),
}
return Result.success(message='Choose project success!',
data=result)
else:
return Result.failed(message='Project does not exist')
else:
return Result.failed(message='Failed to choose project!')
def read_result_from_file(result_file, docnolist_file):
docnolist = parse_corpus(docnolist_file)
docid_map = dict(zip(docnolist, range(len(docnolist))))
res_all = parse_corpus(result_file)
res_dict = OrderedDict()
prev_qid, runid = -1, -1
docid_list = []
score_list = []
for line in res_all:
tokens = line.split()
qid, docid, score, runid = int(tokens[0]), docid_map.get(
tokens[2]), float(tokens[4]), tokens[5]
if qid != prev_qid:
if len(docid_list) > 0:
result = Result(qid, docid_list, score_list, runid)
res_dict.update({prev_qid: result})
docid_list, score_list = [docid], [score]
prev_qid = qid
else:
docid_list.append(docid)
score_list.append(score)
res = Result(prev_qid, docid_list, score_list, runid)
res_dict.update({prev_qid: res})
return res_dict
def delete_person_from_group(cls, request):
request_data = cls.parse_delete_person_from_group_data(request)
if request_data:
try:
check_person = People.objects.get(
person_id=request_data.get(KEY_PERSON_ID),
project_id=Project.objects.get(
project_id=request_data.get(KEY_PROJECT_ID)))
check_group = PeopleGroup.objects.get(
people_group_id=request_data.get(KEY_PEOPLE_GROUP_ID))
result = XrefPeopleGroup.objects.filter(
person_id=check_person,
people_group_id=check_group,
)
if result:
result.delete()
return Result.success(
message='Delete person from group success!')
else:
return Result.failed(
message='Have some problem when delete person to group!'
)
except (People.DoesNotExist, PeopleGroup.DoesNotExist,
Project.DoesNotExist):
return Result.failed(
message='Person id or group id not exist!')
else:
return Result.failed(message='Missing data in the request!')
def delete_camera_group(cls, request):
try:
camera_group_id = request.GET['camera_group_id']
delete_group = CameraGroup.objects.get(camera_group_id=camera_group_id)
if delete_group:
try:
xref_camera_group = XrefCameraGroup.objects.select_related('camera_group_id') \
.filter(camera_group_id=camera_group_id)
delete_group.status = DEFAULT_STATUS_DEACTIVE
delete_group.save()
xref_camera_group.delete()
return Result.success(
message='Delete group success!',
)
except CameraGroup.DoesNotExist:
return Result.failed(
message='CameraGroup matching query does not exist'
)
else:
return Result.failed(
message='Group doesnt exist'
)
except CameraGroup.DoesNotExist:
return Result.failed(
message='CameraGroup matching query does not exist!'
)
except Exception as exc:
return Result.failed(
message=f'Missing data from request: {exc}'
)
def check_all_url(cls, request):
project_id = request.GET['project_id']
if project_id:
try:
all_camera = list(Camera.objects.filter(
status=DEFAULT_STATUS_ACTIVE,
project_id=Project.objects.get(project_id=project_id)
).values())
except Project.DoesNotExist:
return Result.failed(
message='Project doesnt exist!!'
)
list_status = []
for camera in all_camera:
item = {
KEY_CAMERA_ID: camera.get(KEY_CAMERA_ID),
'status': cls.check_camera_available(camera.get('stream_url'))
}
list_status.append(item)
print(list_status)
return Result.success(
message='Status of all cameras',
data=list_status
)
else:
return Result.failed(
message='Missing data from request!!'
)
def test_combine_with_error(self):
result_list = [
Result.Ok("a", True),
Result.Ok("b", True),
Result.Fail("c"),
]
result = Result.combine(result_list)
assert not result.success
def test_combine_without_error(self):
result_list = [
Result.Ok("a", True),
Result.Ok("b", True),
Result.Ok("c", False),
]
result = Result.combine(result_list)
assert result.success
def dango():
try:
data = fetch('SELECT * FROM 小团子酱杨雪')
logger.success('查询 "小团子酱 杨雪" 完成')
return Result.success(data)
except:
logger.success('查询 "小团子酱 杨雪" 失败')
return Result.error(400, '获取数据失败')
def test_combine_without_error_list_result(self):
result_list = [
Result.Ok("a", True),
Result.Ok("b", True),
Result.Ok(["c", "d"], False),
]
result = Result.combine(result_list)
assert result.success
assert result.value == ["a", "b", "c", "d"]
def stop_process(cls, request):
process_id = request.GET['process_id']
if process_id:
process_update = Process.objects.get(process_id=process_id, )
process_update.process_status = KEY_PROCESS_STOP
process_update.save()
result = list(
Process.objects.filter(process_id=process_id, ).values())[0]
return Result.success(message='Success stop process!', data=result)
else:
return Result.failed(message='Missing data in the request!')
def add_camera(cls, request):
result_camera = cls.parse_add_camera_data(request)
if result_camera:
check_camera = Camera.objects.filter(
camera_name=result_camera[KEY_CAMERA_NAME],
project_id=Project.objects.get(
project_id=result_camera[KEY_PROJECT_ID]),
).values()
if not check_camera:
result_create_camera = Camera.objects.create(
camera_name=result_camera[KEY_CAMERA_NAME],
stream_url=result_camera[KEY_STREAM_URL],
project_id=Project.objects.get(project_id=result_camera[KEY_PROJECT_ID]),
status=DEFAULT_STATUS_ACTIVE,
is_running=True,
)
if result_create_camera:
camera_just_created = list(Camera.objects.filter(
camera_name=result_camera[KEY_CAMERA_NAME],
status=DEFAULT_STATUS_ACTIVE,
).values())[0]
group_array = request.data['group']
if len(group_array) != 0:
for group in group_array:
result_create_group_camera = XrefCameraGroup.objects.create(
camera_id=Camera.objects.get(
camera_id=camera_just_created.get(KEY_CAMERA_ID),
status=DEFAULT_STATUS_ACTIVE,
),
camera_group_id=CameraGroup.objects.get(
camera_group_name=group,
status=DEFAULT_STATUS_ACTIVE,
)
)
if not result_create_group_camera:
return Result.failed(
message='Have some problem when create group!'
)
return Result.success(
message='Success add camera!'
)
else:
return Result.failed(
message='Have some problem when add camera!'
)
else:
return Result.failed(
message='Camera name already existed!'
)
else:
return Result.failed(
message='Missing data in the request!'
)
def delete_process(cls, request):
process_id = request.GET['process_id']
if process_id:
try:
check_process = Process.objects.select_related('process_id') \
.filter(process_id=request.GET['process_id'])
check_process.delete()
return Result.success(message='Success delete process')
except Camera.DoesNotExist:
return Result.failed(
message='There is no process with this ID!')
else:
return Result.failed(message='Missing data in the request!')
def update_process(cls, request):
process = cls.parse_update_process(request)
if process:
Process.objects.filter(process_id=process[KEY_PROCESS_ID]).update(
process_name=process[KEY_PROCESS_NAME],
process_config=process[KEY_PROCESS_CONFIG],
people_group_id=process[KEY_PROCESS_PEOPLE_GROUP_ID],
camera_id=Camera.objects.get(
camera_id=process[KEY_PROCESS_CAMERA_ID]))
return Result.success(message='Success edit process!')
else:
return Result.failed(message='Missing data in the request!')
def get_process(cls, request):
process_id = request.GET['process_id']
if process_id:
try:
process = Process.objects.filter(process_id=process_id, )
except Process.DoesNotExist:
return Result.failed(message='Process doesnt exist')
get_process = list(process.values())[0]
print(get_process.get('camera_id_id'))
return Result.success(message="Get process successful",
data=get_process)
else:
return Result.failed(message="Process id not exists", )
def check_camera(cls, request):
camera_id = request.GET['camera_id']
if camera_id:
camera_url = Camera.objects.get(camera_id=camera_id).stream_url
camera_status = cls.check_camera_available(camera_url)
return Result.success(
message='Status of all cameras',
data=camera_status
)
else:
return Result.failed(
message='Missing data from request!!'
)
def reset_master_admin_password(cls, request):
secret_key = request.data.get(KEY_SECRET_RESTORE_MASTER_ADMIN_PASSWORD)
master_admin = User.objects.filter(
username=DEFAULT_MASTER_ADMIN_USER_NAME).values()
if secret_key == SECRET_RESET_MASTER_ADMIN_PASSWORD_KEY:
new_password = cls.generate_password(DEFAULT_MASTER_ADMIN_PASSWORD)
if master_admin:
restore_password = User.objects.filter(
username=DEFAULT_MASTER_ADMIN_USER_NAME).update(
password=new_password)
if restore_password:
return Result.success(
message=
'Reset password for super administrator success!',
data={
KEY_USER_NAME: DEFAULT_MASTER_ADMIN_USER_NAME,
KEY_USER_PASSWORD: DEFAULT_MASTER_ADMIN_PASSWORD
})
else:
return Result.success(
message='Reset password for super administrator failed!'
)
else:
create_master_admin = User.objects.create(
username=DEFAULT_MASTER_ADMIN_USER_NAME,
password=new_password,
full_name=DEFAULT_MASTER_ADMIN_FULL_NAME,
role=DEFAULT_ADMIN_ROLE,
status=DEFAULT_USER_STATUS)
if create_master_admin:
return Result.success(
message=
'Reset password for super administrator success!',
data={
KEY_USER_NAME: DEFAULT_MASTER_ADMIN_USER_NAME,
KEY_USER_PASSWORD: DEFAULT_MASTER_ADMIN_PASSWORD
})
else:
return Result.success(
message='Reset password for super administrator failed!'
)
else:
return Result.failed(
message='The security key to reset the account is incorrect!')
def get_avatar(cls, request):
user_information = jwt.decode(
cls.parse_request_user_data(request).get('token'),
SECRET_TOKEN_KEY)
if not user_information:
return Result.failed(message='Missing data from request!')
try:
check_image = XrefAvatarUser.objects.get(user_id=User.objects.get(
user_id=user_information[KEY_USER_ID]))
link = check_image.avatar_file.split('/static')
result = cls.image_as_base64(STATIC_DIR + link[1])
return Result.success(message='Get image of person success!',
data=result)
except (User.DoesNotExist, XrefAvatarUser.DoesNotExist):
return Result.failed(message='User doesnt exist!')
def get_all_people_group(cls, request):
project_id = request.GET['project_id']
if project_id:
try:
all_people_group = PeopleGroup.objects.filter(
project_id=Project.objects.get(project_id=project_id),
status=DEFAULT_STATUS_ACTIVE)
except Project.DoesNotExist:
return Result.failed(message='Project doesnt exist')
list_all_people_group = list(all_people_group.values())
return Result.success(message='Get all group success',
data=list_all_people_group)
else:
return Result.failed(message='Missing data in the request!')
def get_all_camera(cls, request):
project_id = request.GET['project_id']
if project_id:
try:
all_camera = Camera.objects.filter(
project_id=Project.objects.get(project_id=project_id),
status=DEFAULT_STATUS_ACTIVE
)
except Project.DoesNotExist:
return Result.failed(
message='Project doesnt exist'
)
result = []
list_all_camera = list(all_camera.values())
for camera in list_all_camera:
try:
check_camera = Camera.objects.get(
camera_id=camera.get('camera_id'),
status=DEFAULT_STATUS_ACTIVE,
)
group_list = []
xref_camera_group = list(XrefCameraGroup.objects.select_related('camera_id')
.filter(camera_id=check_camera.camera_id).values())
for item in xref_camera_group:
group_item = \
list(CameraGroup.objects.select_related('camera_group_id')
.filter(camera_group_id=item.get('camera_group_id_id'),
project_id=Project.objects.get(project_id=project_id)).values())
if len(group_item) > 0:
group_list.append(group_item[0])
camera['group'] = group_list
result.append(camera)
except Camera.DoesNotExist:
return Result.failed(
message='Camera id not exist!'
)
return Result.success(
message='Get all camera success',
data=result
)
else:
return Result.failed(
message='Missing data in the request!'
)
def libsvm_run(dataset_name, n_features, Extractor, ftype, n_repeat=5):
n_features = int(n_features)
base_dir = os.path.join(os.getcwd(), '../targets/%s/' % dataset_name)
model_file = os.path.join(base_dir, 'train.scale.model')
result = Result('%s-%s' % (dataset_name, Extractor.__name__))
for repeat in range(0, n_repeat):
print 'Round %d of %d' % (repeat, n_repeat - 1)
# load model and collect QSV
ex = LibSVMOnline(dataset_name, model_file, (1, -1), n_features, ftype,
1e-1)
# generate test score
X_test, y_test = load_svmlight_file(
os.path.join(base_dir, 'test.scale'), n_features)
X_test = X_test.todense()
train_x, train_y = [], []
for i in result.index:
q_by_u = result.Q_by_U[i]
ex.collect_up_to_budget(q_by_u * (n_features + 1))
train_x.extend(ex.pts_near_b)
train_y.extend(ex.pts_near_b_labels)
main = Extractor(ex.batch_predict, (train_x, train_y),
(X_test, y_test), n_features)
L_unif, L_test = main.do()
result.L_unif[i].append(L_unif)
result.L_test[i].append(L_test)
result.nquery[i].append(ex.get_n_query())
# print ex.get_n_query() / (n_features + 1), ',', L_unif, ',', L_test
print result
def livefe(request):
try:
return StreamingHttpResponse(
gen(VideoCamera(request.GET['stream_url']), request),
content_type='multipart/x-mixed-replace;boundary=frame')
except: # This is bad! replace it with proper handling
return Result.failed(message='Have some problems!')
def run(dataset_name, n_features, n_repeat=5, n_learning_round=5):
base_dir = os.path.join(os.getcwd(), '../targets/%s/' % dataset_name)
model_file = os.path.join(base_dir, 'train.scale.model')
result = Result(dataset_name + '-' + 'active')
for repeat in range(0, n_repeat):
print 'Round %d of %d' % (repeat, n_repeat - 1)
ex = LibSVMOnline(dataset_name, model_file, (1, -1), n_features,
'uniform', 1e-1)
X_test, y_test = load_svmlight_file(
os.path.join(base_dir, 'test.scale'), n_features)
X_test = X_test.todense()
for i in result.index:
q_by_u = result.Q_by_U[i]
main = ActiveLearning(ex, (None, None),
(X_test, y_test), n_features,
q_by_u * (n_features + 1), n_learning_round)
L_unif, L_test = main.do()
result.L_unif[i].append(L_unif)
result.L_test[i].append(L_test)
result.nquery[i].append(ex.get_n_query())
print result
def eval_by_qid_list_helper(self, qid_list, pair_generator):
relevance_dict = load_pickle(self.config.relevance_dict_path)
qid_list = sorted(qid_list)
qualified_qid_list = []
res_dict = OrderedDict()
for qid in qid_list:
relevance = relevance_dict.get(qid)
supervised_docid_list = relevance.get_supervised_docid_list()
if len(supervised_docid_list) < self.config.nb_supervised_doc:
# cannot construct d2d feature, thus not need to be update
score_list = relevance.get_supervised_score_list()
res = Result(qid, supervised_docid_list, score_list,
self.config.runid)
res_dict.update({qid: res})
logging.warn("query {0} not to be rerank".format(qid))
else:
qualified_qid_list.append(qid)
# generate re rank score
dd_q, dd_d, score_gate, len_indicator = \
pair_generator.generate_list_batch(qualified_qid_list, self.config.rerank_topk)
return [dd_q, dd_d,
score_gate], len_indicator, res_dict, qualified_qid_list
def grid_search(self):
C_range = np.logspace(-5, 15, 21, base=2)
param_grid = dict(C=C_range)
cv = StratifiedShuffleSplit(self.y_ex,
n_iter=5,
test_size=0.2,
random_state=42)
grid = GridSearchCV(LinearSVC(dual=False, max_iter=10000),
param_grid=param_grid,
cv=cv,
n_jobs=1,
verbose=0)
logger.info('start grid search for Linear')
grid.fit(self.X_ex, self.y_ex)
logger.info('end grid search for Linear')
scores = [x[1] for x in grid.grid_scores_]
# final train
rbf_svc2 = grid.best_estimator_
pred_train = rbf_svc2.predict(self.X_ex)
pred_val = rbf_svc2.predict(self.val_x)
pred_test = rbf_svc2.predict(self.test_x)
r = Result(self.name + ' (X)', 'Linear', len(self.X_ex),
sm.accuracy_score(self.y_ex, pred_train),
sm.accuracy_score(self.val_y, pred_val),
sm.accuracy_score(self.test_y, pred_test))
return r
