def __init__(self, pattern):
# kafka 配置
self.consumer = KafkaConsumer("maxwell",
bootstrap_servers=[
'192.168.1.208:9092',
'192.168.1.212:9092',
'192.168.1.211:9092'
])
# es 配置
self.es = Elasticsearch([{
'host': '192.168.1.252',
'port': 9202
}, {
'host': '192.168.1.193',
'port': 9202
}, {
'host': '192.168.1.194',
'port': 9202
}])
# 一些正则表达式
self.pattern = pattern
# 分类程序，区别（物资/非物资；港航/非港航；拟建/招标/中标）
self.clf = Classification()
# 字段提取程序，提取第一中标候选人，中标金额，投资金额，业主，项目概况等信息
self.ifm = KafkaInformation(pattern)
# redis 配置， 暂时没有使用
self.redis = StrictRedis('192.168.1.60',
port=6379,
db=7,
password='******')
def __init__(self,
client_id,
client_secret,
callback_host='localhost',
callback_port=8080,
force_new_token=False,
session=None,
course_code=None,
semester=None,
group_code=None,
lang=None):
self.classification = Classification(client_id, client_secret,
callback_host, callback_port,
force_new_token, session)
self.course_code = course_code
self.semester = semester
self.group_code = group_code or 'ALL'
self.lang = lang
def loadTrainedFile(self, checked):
if os.path.isfile(self.trained_file_path.text()):
deep_learning_service.load_model(self.trained_file_path.text())
data_transporter_service.fire(WINDOW_CHANEL, Classification())
'ISIC_0014593_segmentation.png',
'ISIC_0014341_segmentation.png',
'ISIC_0014782_segmentation.png')
small_high_contrast_data = ('ISIC_0014158.jpg', 'ISIC_0014093.jpg',
'ISIC_0014028.jpg', 'ISIC_0014770.jpg')
small_high_contrast_ground = ('ISIC_0014158_segmentation.png',
'ISIC_0014093_segmentation.png',
'ISIC_0014028_segmentation.png',
'ISIC_0014770_segmentation.png')
############################################################################################################
############################################################################################################
feature = FE.get('ADV') # (MRG or ADV)
learning = LE.get('NN') # Only NN
classify = Classification(learning, feature)
############################################################################################################
############################################################################################################
print("Building dataset\n")
X_train, y_train = feature.get_data_set(train_data_set,
train_ground_set,
type="train")
print("Fitting Neural Network")
learning.fit(X_train, y_train)
print("Classifying bloom images")
acc_bloom = classify.accurate_and_segmentation(bloom_data,
bloom_ground,
set='test',
class Stream(object):
def __init__(self, pattern):
# kafka 配置
self.consumer = KafkaConsumer("maxwell",
bootstrap_servers=[
'192.168.1.208:9092',
'192.168.1.212:9092',
'192.168.1.211:9092'
])
# es 配置
self.es = Elasticsearch([{
'host': '192.168.1.252',
'port': 9202
}, {
'host': '192.168.1.193',
'port': 9202
}, {
'host': '192.168.1.194',
'port': 9202
}])
# 一些正则表达式
self.pattern = pattern
# 分类程序，区别（物资/非物资；港航/非港航；拟建/招标/中标）
self.clf = Classification()
# 字段提取程序，提取第一中标候选人，中标金额，投资金额，业主，项目概况等信息
self.ifm = KafkaInformation(pattern)
# redis 配置， 暂时没有使用
self.redis = StrictRedis('192.168.1.60',
port=6379,
db=7,
password='******')
# 主程序， 从kalfka读取数据，处理后直接存入es
def consume(self):
# 持续从kafka读取数据
for msg in self.consumer:
# 解析读取后的数据
key = json.loads(msg.key.decode())
value = msg.value.decode()
# 如果不是stang_bid_new, stang_cbid, stang_pifu这三个表的数据，直接丢弃
if not self.is_valid_table_name(key['table']):
pass
else:
try:
# 分类以及字段提取持续
res = self.job_run(value)
# 仅仅将需要的数据存入es
if res:
# 存入es的doc的id为 数据表名_id
doc_id = str(res['table_name']) + '_' + str(res['id'])
self.es.index(index='ccccltd',
doc_type='shipping',
body=res,
id=doc_id)
except Exception as e:
print(e)
# 判断是否是 stang_bid_new, stang_cbid 和 stang_pifu表中的数据
def is_valid_table_name(self, database_name):
if database_name is None:
return None
if not isinstance(database_name, str):
raise ValueError('data base table name is not valid')
if database_name in self.pattern.valid_database:
return True
else:
return False
# 将从kafka读取到的数据，转换为kafka_data对象
def get_data(self, value):
try:
dict_value = json.loads(value)
kafka_data = KafkaData(dict_value, self.pattern)
return kafka_data
except Exception as e:
print('Get kafka data object data wrong', e)
# 对于每一条数据的处理程序
def job_run(self, values):
# 获取kafka_data对象
kafka_data = self.get_data(values)
# 获取拟建\招标\中标以及所属八类别， 当不是需要的数据的时候会返回None, None(例如非八大类，非拟建招中标， 或者是物资的数据)
cate_id, label = self.clf.main(kafka_data.title,
kafka_data.get_info_text(),
kafka_data.cate_id)
# 当为有效的数据时候
if cate_id is not None:
# 所属类别
kafka_data.cate_id = cate_id
# 字段提取
first_bid_company, first_bid_money, invest_money, summary, owner = self.ifm.get_information(
kafka_data)
# 所属类别
kafka_data.data['data']['cate_id'] = int(cate_id)
# 八大类标记
kafka_data.data['data']['label'] = label
# 第一中标单位
kafka_data.data['data']['first_bid_company'] = first_bid_company
# 中标金额
kafka_data.data['data']['first_bid_money'] = first_bid_money
# 投资额
kafka_data.data['data']['invest_money'] = invest_money
# 项目概况
kafka_data.data['data']['summary'] = summary
# 业主
kafka_data.data['data']['owner'] = owner
# es的入库时间， 毫秒级
kafka_data.data['data']['add_time'] = int(round(time.time() *
1000))
# pubtime, 毫秒级别
kafka_data.data['data']['pub_time'] = int(
str(
str(
pd.Timestamp(
kafka_data.data['data']['pubtime']).value)[:13]))
# 获取需要插入es的数
res = kafka_data.get_the_data_to_es()
# print(res, '现在打印的是res')
return res
# 当不是需要的数据的时候返回None(例如非八大类，非拟建招中标， 或者是物资的数据)
else:
return None
class ClassificationParamsProxy:
"""This proxy class can store some parameters for API calls.
You can pass parameters listed below either during the creation
of this object or later via directly setting its attributes.
Later you can omit them in methods calls. Or supply them anyway
to override saved defaults.
Note:
This proxy can do everything that
:py:class:`~.classification.Classification` class can.
See its documentation for all the parameters. Here we describe
only attributes not present in the default implementation.
Warning:
Please note, that due to the fact that some parameters
can be stored internally, they default to ``None`` now.
However, they can still be needed to make API calls.
are not compulsory in method calls. Keep in mind that:
- If a required parameter is neither saved in proxy
nor passed to a method explicitly,
a :py:exc:`~classification.exceptions.MissingParameterError`
will be raised.
- The order of arguments in methods of the plain client
and its proxy version can be different.
If you prefer to do without named arguments in Python,
be sure to double check what you are
passing to a method.
Attributes:
classification (str): The implementation of the real library,
created automatically.
course_code (str): Stores the code of the course.
semester (str): Stores the semester identifier.
group_code (str): Stores the code of the group.
lang (str): Stores the language tag.
"""
PARAM_ERROR = 'The following parameter must be supplied: '
def __init__(self,
client_id,
client_secret,
callback_host='localhost',
callback_port=8080,
force_new_token=False,
session=None,
course_code=None,
semester=None,
group_code=None,
lang=None):
self.classification = Classification(client_id, client_secret,
callback_host, callback_port,
force_new_token, session)
self.course_code = course_code
self.semester = semester
self.group_code = group_code or 'ALL'
self.lang = lang
def reinit_session(self,
callback_host='localhost',
callback_port=8080,
force_new_token=False):
self.classification.reinit_session(callback_host, callback_port,
force_new_token)
def drop_session(self):
self.classification.drop_session()
# -----------------------------------------------
# ---------- CLASSIFICATION CONTROLLER ----------
# -----------------------------------------------
def delete_classification(self,
classification_id,
course_code=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.delete_classification(course_code, classification_id,
semester, **kwargs)
def find_classifications_for_course(self,
course_code=None,
semester=None,
lang=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
lang = self._get_param(lang, 'lang', False)
return self.classification \
.find_classifications_for_course(course_code, semester,
lang, **kwargs)
def save_classification(self,
course_code=None,
classification_dto=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
return self.classification \
.save_classification(course_code, classification_dto, **kwargs)
def change_order_of_classifications(self,
indexes,
course_code=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.change_order_of_classifications(course_code, indexes,
semester, **kwargs)
def find_classification(self,
identifier,
course_code=None,
semester=None,
lang=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
lang = self._get_param(lang, 'lang', False)
return self.classification \
.find_classification(course_code, identifier,
semester, lang, **kwargs)
def clone_classification_definitions(self, target_semester,
target_course_code, source_semester,
source_course_code, remove_existing,
**kwargs):
return self.classification \
.clone_classification_definitions(target_semester,
target_course_code,
source_semester,
source_course_code,
remove_existing,
**kwargs)
# -----------------------------------------------
# -------------- EDITOR CONTROLLER --------------
# -----------------------------------------------
def get_editors(self, course_code=None, **kwargs):
course_code = self._get_param(course_code, 'course_code', True)
return self.classification \
.get_editors(course_code, **kwargs)
def delete_editor(self, username, course_code=None, **kwargs):
course_code = self._get_param(course_code, 'course_code', True)
return self.classification \
.delete_editor(course_code, username, **kwargs)
def add_editor(self, username, course_code=None, **kwargs):
course_code = self._get_param(course_code, 'course_code', True)
return self.classification \
.add_editor(course_code, username, **kwargs)
# -----------------------------------------------
# ------------ EXPRESSION CONTROLLER ------------
# -----------------------------------------------
def evaluate_all(self, expressions_dto=None, **kwargs):
return self.classification \
.evaluate_all(expressions_dto, **kwargs)
def try_validity(self, expression=None, **kwargs):
return self.classification \
.try_validity(expression, **kwargs)
def get_functions(self, **kwargs):
return self.classification \
.get_functions(**kwargs)
# -----------------------------------------------
# ----------- NOTIFICATION CONTROLLER -----------
# -----------------------------------------------
def get_all_notifications(self,
username,
count=None,
page=None,
lang=None,
**kwargs):
lang = self._get_param(lang, 'lang', False)
return self.classification \
.get_all_notifications(username, count, page, lang, **kwargs)
def get_unread_notifications(self,
username,
count=None,
page=None,
lang=None,
**kwargs):
lang = self._get_param(lang, 'lang', False)
return self.classification \
.get_unread_notifications(username, count, page, lang, **kwargs)
def unread_all_notifications(self, username, **kwargs):
return self.classification \
.unread_all_notifications(username, **kwargs)
def read_all_notifications(self, username, **kwargs):
return self.classification \
.read_all_notifications(username, **kwargs)
def unread_notification(self, username, id, **kwargs):
return self.classification \
.unread_notification(username, id, **kwargs)
def read_notification(self, username, id, **kwargs):
return self.classification \
.read_notification(username, id, **kwargs)
# -----------------------------------------------
# ------------- SETTINGS CONTROLLER -------------
# -----------------------------------------------
def get_settings(self, semester=None, lang=None, **kwargs):
semester = self._get_param(semester, 'semester', False)
lang = self._get_param(lang, 'lang', False)
return self.classification \
.get_settings(semester, lang, **kwargs)
def save_my_settings(self, user_settings_dto=None, **kwargs):
return self.classification \
.save_my_settings(user_settings_dto, **kwargs)
def save_student_course_settings(self,
user_course_settings_dto=None,
semester=None,
**kwargs):
semester = self._get_param(semester, 'semester', False)
return self.classification \
.save_student_course_settings(user_course_settings_dto,
semester, **kwargs)
def save_teacher_course_settings(self,
user_course_settings_dto=None,
semester=None,
**kwargs):
semester = self._get_param(semester, 'semester', False)
return self.classification \
.save_teacher_course_settings(user_course_settings_dto,
semester, **kwargs)
# -----------------------------------------------
# ------ STUDENT CLASSIFICATION CONTROLLER ------
# -----------------------------------------------
def find_student_group_classifications(self,
course_code=None,
group_code=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
group_code = self._get_param(group_code, 'group_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.find_student_group_classifications(course_code, group_code,
semester, **kwargs)
def find_student_group_classifications_simple_s2t(self,
course_code=None,
group_code=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
group_code = self._get_param(group_code, 'group_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.find_student_group_classifications_simple_s2t(
course_code, group_code,
semester, **kwargs)
def find_student_group_classifications_simple_t2s(self,
course_code,
group_code=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
group_code = self._get_param(group_code, 'group_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.find_student_group_classifications_simple_t2s(
course_code, group_code,
semester, **kwargs)
def find_student_classifications_for_definitions(self,
identifier,
course_code=None,
group_code=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
group_code = self._get_param(group_code, 'group_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.find_student_classifications_for_definitions(course_code,
identifier,
group_code,
semester, **kwargs)
def save_student_classifications(self,
course_code=None,
student_classifications=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.save_student_classifications(course_code,
student_classifications,
semester, **kwargs)
def save_student_classifications_simple_s2t(self,
course_code=None,
student_to_tasks=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.save_student_classifications_simple_s2t(course_code,
student_to_tasks,
semester, **kwargs)
def save_student_classifications_simple_t2s(self,
course_code=None,
task_to_students=None,
semester=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
return self.classification \
.save_student_classifications_simple_t2s(course_code,
task_to_students,
semester, **kwargs)
def find_student_classification(self,
student_username,
course_code=None,
semester=None,
lang=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
lang = self._get_param(lang, 'lang', False)
return self.classification \
.find_student_classification(course_code, student_username,
semester, lang, **kwargs)
# -----------------------------------------------
# ---------- STUDENT GROUP CONTROLLER -----------
# -----------------------------------------------
def get_course_groups(self,
course_code=None,
semester=None,
lang=None,
**kwargs):
course_code = self._get_param(course_code, 'course_code', True)
semester = self._get_param(semester, 'semester', False)
lang = self._get_param(lang, 'lang', False)
return self.classification \
.get_course_groups(course_code, semester, lang, **kwargs)
# -----------------------------------------------
# -------------- HELPER FUNCTIONS ---------------
# -----------------------------------------------
def _get_param(self, supplied, var_name, required):
result = supplied or getattr(self, var_name)
if required and result is None:
raise MissingParameterError(self.PARAM_ERROR + var_name)
return result
size = cfg.nImage
index_train = np.random.randint(0, length, size)
index_test = np.random.randint(0, length, size)
train_data = [train_data_set[i] for i in index_train]
train_ground = [train_ground_set[i] for i in index_train]
test_data = [test_data_set[i] for i in index_test]
test_ground = [test_ground_set[i] for i in index_test]
############################################################################################################
############################################################################################################
feature = FE.get('MRG') # (MRG or ADV)
learning = LE.get('NN') # Only NN
classify = Classification(learning, feature)
############################################################################################################
############################################################################################################
print("Building dataset\n")
X_train, y_train = feature.get_data_set(train_data, train_ground, type="train")
#X_test, y_test = feature.get_data_set(test_data, test_ground, type="test")
print("Fitting Neural Network")
learning.fit(X_train, y_train)
print("Classifying")
data = test_data[0:3]
ground = test_ground[0:3]
acc = classify.accurate(data, ground, set='test')
from classification.classification import Classification
from algorithm.create_df.read_data_lib.data_base import DataSQL
c = Classification()
d = DataSQL()
data = d.read_sql(
'select title, info, cate_id FROM stang_cbid WHERE id = 6304465')[0]
print(data[0])
print(c.main(data[0], data[1], data[2]))
densenet_info = densenet_ref.get().to_dict()
if densenet_info['is_enable'] and (densenet_info['is_update']
or not os.path.isfile(MODEL_PATH)):
file_name = MODEL_PATH
file_url = densenet_info['url']
urllib.request.urlretrieve(file_url, file_name)
densenet_ref.set({
'is_enable': True,
'is_update': False,
'url': densenet_info['url'],
'labels': densenet_info['labels']
})
if os.path.isfile(MODEL_PATH):
classification = Classification(model_path=MODEL_PATH,
classes=densenet_info['labels'])
# 訓練用サーバーの URL を指定
train_url = sys.argv[1]
# base64画像 -> numpy画像
def base64_to_numpy(img_base64):
img_bytes = base64.b64decode(img_base64)
temp = np.frombuffer(img_bytes, np.uint8)
img_np = cv2.imdecode(temp, cv2.IMREAD_ANYCOLOR)
return img_np
# numpy画像 -> base64画像
def numpy_to_base64(img_np):