def get_systemcodes(): empty = [] if not request.args: # 没有指定id则返回全部 db_session = create_session() posts = db_session.query(SystemCode).all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty) elif 'code_main' in request.args and 'code_code' not in request.args: db_session = create_session() posts = db_session.query(SystemCode).filter( SystemCode.code_main == request.args['code_main']).all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty) else: db_session = create_session() post = db_session.query(SystemCode).filter( SystemCode.code_main == request.args['code_main'], SystemCode.code_code == request.args['code_code']).one() db_session.close() return jsonify(post.to_json())
def run(socketio, l_type, long_process_command): #发送任务到queue db_session = create_session() import_neo4j_install_dir_db = db_session.query(SystemPar).filter( SystemPar.par_code == 'import_neo4j_install_dir').one() start_time = datetime.datetime.now() import_neo4j_install_dir_value = import_neo4j_install_dir_db.par_value _queue_uuid = str(uuid.uuid1()) _queue = JobQueue(u_uuid=_queue_uuid, u_declare_key=l_type, u_body=long_process_command, u_publisher_id=l_type, u_publish_datetime=start_time, u_no_ack=False, u_start_datetime=None, u_complete_datetime=None, u_status='发布') db_session.add(_queue) db_session.flush() db_session.commit() db_session.close() #循环检测状态看是否导入成功 u_complete_datetime = None u_back_message = '' while u_complete_datetime == None: socketio.sleep(10) mem = psutil.virtual_memory() disk = psutil.disk_usage(import_neo4j_install_dir_value) socketio.emit('system_report', { 'platform': platform.platform(), 'disk_total': disk.total, 'disk_free': disk.free, 'cpu_percent': psutil.cpu_percent(), 'mem_total': mem.total, 'mem_used': mem.used, 'mem_free': mem.free }, broadcast=True) db_session_check = create_session() _queue_reload = db_session_check.query(JobQueue).filter( JobQueue.u_uuid == _queue_uuid).one() u_complete_datetime = _queue_reload.u_complete_datetime u_status = _queue_reload.u_status u_back_message = _queue_reload.u_back_message db_session_check.close() return u_back_message
def get_posts(): empty = [] if not request.args: db_session = create_session() posts = db_session.query(SystemPar).all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty) else: db_session = create_session() post = db_session.query(SystemPar).filter( SystemPar.par_code == request.args['par_code']).one() db_session.close() return jsonify(post.to_json())
def get_system_data(): empty = [] class System_data(TableRestful): def __init__(self, args, db_session): super(System_data, self).__init__(args, db_session) self.query = self.db_session.query(SystemData) def _query_parmeter(self): self.q_filter = self.q_join def _ordery_by(self): self.q_order = self.q_filter def _join(self): self.q_join = self.query db_session = create_session() pd = System_data(request.args, db_session) rs = pd.get_rs_all() for r in rs: empty.append({'system_data': r.to_json()}) #print(s) db_session.close() return jsonify(empty)
def opendb_getjob(u_declare_key, job): sys.path.append("python_common") sys.path.append("mosr_back_orm") try: db_session = create_session() queue = db_session.query(JobQueue).filter( JobQueue.u_status == '发布', JobQueue.u_declare_key == u_declare_key, JobQueue.u_publisher_id == u_declare_key).order_by( JobQueue.u_publish_datetime.desc()).first() if (queue != None): current = datetime.datetime.now() queue.u_start_datetime = current queue.u_status = '处理中' db_session.commit() current = job(db_session, queue, current) current = datetime.datetime.now() queue.u_complete_datetime = current queue.u_status = '处理完成' db_session.commit() except Exception as ex: db_session.rollback() print(ex) finally: db_session.close()
def update_import_data(uuid): if len(uuid) == 0: abort(404) if not request.json: abort(400) request.get_json(silent=True) u_status = request.json['u_status'] db_session = create_session() import_data = db_session.query(ImportData).filter( ImportData.u_uuid == uuid).one() import_data.u_status = u_status #删除文件 db_import_neo4j_install_dir = db_session.query(SystemPar).filter( SystemPar.par_code == 'import_neo4j_install_dir').one() import_neo4j_install_dir = db_import_neo4j_install_dir.par_value file_path = import_neo4j_install_dir + "import/" + import_data.u_queue_uuid #print(file_path) #print(os.path.exists(file_path)) if os.path.exists(file_path): print("start reomve") os.remove(file_path) db_session.commit() db_session.close() return jsonify({'result': True})
def neo4j_catlog_edgetypes(): empty = [] db_session = create_session() posts = db_session.query(CurrentEdgeTyps).all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty)
def my_templates(): empty = [] if not request.args: db_session = create_session() posts = db_session.query(QueryTemplate).all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty) else: db_session = create_session() posts = db_session.query(QueryTemplate).filter( QueryTemplate.qt_type == request.args['qt_type']).all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty)
def neo4j_catlog_nodelabels(): empty = [] db_session = create_session() posts = db_session.query(CurrentNodeLabels).all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty)
def import_data(): #print("import_get...............................") empty = [] if not request.args: db_session = create_session() posts = db_session.query(ImportData).filter( ImportData.u_status != '已删除').all() #print("import中国") for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty) else: db_session = create_session() posts = db_session.query(ImportData).filter( ImportData.u_status != '已删除').all() for post in posts: empty.append(post.to_json()) db_session.close() return jsonify(empty)
def delete_my_templates(uuid): if len(uuid) == 0: abort(404) db_session = create_session() post = db_session.query(QueryTemplate).filter( QueryTemplate.qt_uuid == uuid).delete() db_session.commit() db_session.close() return jsonify({'result': True})
def delete_import_data(uuid): if len(uuid) == 0: abort(404) db_session = create_session() post = db_session.query(ImportData).filter( ImportData.u_uuid == uuid).delete() db_session.commit() db_session.close() return jsonify({'result': True})
def neo4j_catlog_properties_code(): empty = None db_session = create_session() #print(request.args['u_label_type']) post = db_session.query(CurrentProperties).filter( CurrentProperties.u_label_type == '[\'' + request.args['u_label_type'] + '\']', CurrentProperties.u_column_type == '编码').one() empty = {'u_column_name': post.u_column_name} #print(post.u_column_name) db_session.close() return jsonify(empty)
def post_neo4j_catlog(): request.get_json(silent=True) if not request.json: abort(400) neno4jCatalog = Neno4jCatalog(nc_uuid=str(uuid.uuid1()), nc_update_datetime=datetime.datetime.now(), nc_type=request.json['nc_type'], nc_value=request.json['nc_value']) db_session = create_session() db_session.add(neno4jCatalog) pd_json = neno4jCatalog.to_json() db_session.commit() db_session.close() return jsonify({'neo4j_catlog': pd_json}), 201
def neo4j_data(): if 'neo4jgraph_cypher' in request.args: db_session = create_session() systemCodes = db_session.query(SystemCode).filter( SystemCode.code_main == 'node_color').all() colors = [] for sc in systemCodes: colors.append(sc.code_code) db_session.close() return getPath(Base64Uri.decode(request.args['neo4jgraph_cypher']), colors) else: pass
def post_process_detail(): request.get_json(silent=True) if not request.json: abort(400) data = request.get_json() processDetail = ProcessDetail(pd_uuid=str(uuid.uuid1()), pd_start_datetime=datetime.datetime.now(), pd_catalog=data['pd_catalog'], pd_command=data['pd_command']) db_session = create_session() db_session.add(processDetail) pd_json = processDetail.to_json() db_session.commit() db_session.close() return jsonify({'ProcessDetail': pd_json}), 201
def neo4j_catlog_properties(): empty = [] db_session = create_session() posts = db_session.query(CurrentProperties.u_type, CurrentProperties.u_column_name, CurrentProperties.u_column_type).group_by( CurrentProperties.u_type, CurrentProperties.u_column_name, CurrentProperties.u_column_type).all() for post in posts: empty.append({ 'u_type': post.u_type, 'u_column_name': post.u_column_name, 'u_column_type': post.u_column_type }) db_session.close() return jsonify(empty)
def readData(command_sql, records): db_session = create_session() uuid_str = str(uuid.uuid1()) #print(uuid_str) setId = -1 setSize = 0 #每个组的大小 set_size = 0 #有多少个组 for record in records: #print(record) #print(record['setId']) if setId != record['setId']: #新的setId if setId != -1: algorithmRsCCD = AlgorithmRsCCD(u_uuid=uuid_str, u_setId=setId, u_size=setSize) db_session.add(algorithmRsCCD) setId = record['setId'] setSize = 0 set_size += 1 algorithmRsCCDD = AlgorithmRsCCDD(u_uuid=uuid_str, u_setId=setId, u_nodeId=record['nodeId'], u_nodeName=record['node_name']) db_session.add(algorithmRsCCDD) setSize += 1 algorithmRsCCD = AlgorithmRsCCD(u_uuid=uuid_str, u_setId=setId, u_size=setSize) db_session.add(algorithmRsCCD) algorithmRsCCM = AlgorithmRsCCM(u_uuid=uuid_str, u_create_datetime=datetime.datetime.now(), u_queue_string=command_sql, u_set_size=set_size + 1) db_session.add(algorithmRsCCM) db_session.commit() db_session.close() return uuid_str
def union_find_node_count(): empty = [] db_session = create_session() count_rs = 0 if 'min_set_count' not in request.args: post = db_session.query( func.count(AlgorithmRsCCD.u_size).label("u_size")).filter( AlgorithmRsCCD.u_uuid == request.args['u_uuid']).one() count_rs = post.u_size else: post = db_session.query( func.count(AlgorithmRsCCD.u_size).label("u_size")).filter( AlgorithmRsCCD.u_uuid == request.args['u_uuid'], AlgorithmRsCCD.u_size >= request.args['min_set_count'], AlgorithmRsCCD.u_size <= request.args['max_set_count']).one() count_rs = post.u_size db_session.close() return {'set_sum': count_rs}
def neo4j_command(): db_session = create_session() queue = db_session.query(JobQueue).filter( JobQueue.u_status == '发布', JobQueue.u_declare_key == 'neo4j_command', JobQueue.u_publisher_id == 'neo4j_command').order_by( JobQueue.u_publish_datetime.desc()).first() job = queue if (queue != None): current = datetime.datetime.now() queue.u_start_datetime = current db_session.commit() body = queue.u_body print(body) command(body, None) current = datetime.datetime.now() queue.u_complete_datetime = current queue.u_status = '处理完成' db_session.commit() db_session.close()
def save_template(): request.get_json(silent=True) if not request.json: abort(400) qt_object = Base64Uri.decode(request.json['qt_object']) qt_cypher = Base64Uri.decode(request.json['qt_cypher']) queryTemplate = QueryTemplate(qt_uuid=str(uuid.uuid1()), qt_datetime=datetime.datetime.now(), qt_type=request.json['qt_type'], qt_object=qt_object, qt_cypher=qt_cypher, qt_title=request.json['qt_title'], qt_desc=request.json['qt_desc']) db_session = create_session() db_session.add(queryTemplate) pd_json = queryTemplate.to_json() db_session.commit() db_session.close() return jsonify({'queryTemplate': pd_json}), 201
def get_process_detail(): empty = [] class Process_detail(TableRestful): def __init__(self, args, db_session): super(Process_detail, self).__init__(args, db_session) self.query = self.db_session.query(ProcessDetail, SystemCode) def _query_parmeter(self): if 'start_datetime_begin' in self.args and 'start_datetime_end' in self.args: self.q_filter = self.q_join.filter( ProcessDetail.start_datetime >= request.args['start_datetime_begin'], ProcessDetail.start_datetime <= request.args['start_datetime_end']) else: self.q_filter = self.q_join self.q_filter = self.q_filter.filter( SystemCode.code_code == ProcessDetail.pd_catalog) def _ordery_by(self): self.q_order = self.q_filter.order_by( ProcessDetail.pd_start_datetime.desc()) def _join(self): self.q_join = self.query db_session = create_session() pd = Process_detail(request.args, db_session) rs = pd.get_rs_all() #print(rs) for r, s in rs: empty.append({ 'process_detail': r.to_json(), 'system_code': s.to_json() }) #print(s) db_session.close() return jsonify(empty)
def union_find_data(): empty = [] db_session = create_session() posts = db_session.query(AlgorithmRsCCD).filter( AlgorithmRsCCD.u_uuid == request.args['u_uuid'], AlgorithmRsCCD.u_size >= request.args['min_set_count'], AlgorithmRsCCD.u_size <= request.args['max_set_count']).order_by( AlgorithmRsCCD.u_size.desc()).all() for post in posts: posts2 = db_session.query(AlgorithmRsCCDD).filter( AlgorithmRsCCDD.u_uuid == request.args['u_uuid'], AlgorithmRsCCDD.u_setId == post.u_setId).all() line = [] for post2 in posts2: #line.append({'u_nodeId':post2.u_nodeId,'u_nodeName':post2.u_nodeName}) #line.append(post2.u_nodeName) line.append([post2.u_nodeId, post2.u_nodeName]) empty.append({'setId': post.u_setId, 'names': line}) posts2 = None db_session.close() return jsonify(empty)
import uuid import time from download_data import download_data from import_data import import_data from clean_neo4j import clean_neo4j from neo4j_command import neo4j_command from algoUnionFind import unionFind import sys sys.path.append("python_common") sys.path.append("mosr_back_orm") ps = 0 try: db_session = create_session() #读取执行的间隔 polling_second = db_session.query(SystemPar).filter( SystemPar.par_code == 'polling_second').one() ps = int(polling_second.par_value) db_session.commit() except: db_session.rollback() raise finally: db_session.close() print("连接数据库成功,轮询间隔" + str(ps)) if ps > 0: while True:
def wait_for_import_end(par_dict): """Example of how to send server generated events to clients.""" count = 0 while True: print("okokokoko") if background_importing_thread_flag: socketio.sleep(10) count += 1 socketio.emit('system_report', psutil.cpu_times(), broadcast=True) print('background_thread_importing') print(count) else: print("background_thread_imported") break socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '导入数据成功' }) #emit('neo4j_rebuild_process', {'message': '导入数据成功'}, broadcast=True) socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '开始启动分析服务器' }) #emit('neo4j_rebuild_process', {'message': '开始启动分析服务器'}, broadcast=True) #启动数据库 start_commonad = par_dict['import_neo4j_install_dir'] + ( 'bin/neo4j.bat' if system_type == 'Windows' else 'bin/neo4j') + ' start' print(start_commonad) r_start_commonad = os.popen( start_commonad).read() #subprocess.call(start_commonad) socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '分析服务器启动成功' }) #emit('neo4j_rebuild_process', {'message': '分析服务器启动成功'}, broadcast=True) #socketio.sleep(5) print("分析服务器启动成功") socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_end", 'message_info': '分析数据库重建完成' }) #emit('neo4j_rebuild_end', {'message': '分析数据库重建完成'}, broadcast=True) print("重建数据库完成") db_session = create_session() #跟新数据库数据 end_import_time = datetime.datetime.now() manage_import_data = par_dict['manage_import_data'] import_data = par_dict['import_data'] for index in range(len(manage_import_data)): if manage_import_data[index]: item = import_data[index] u_uuid = item['u_uuid'] import_data_db = db_session.query(ImportData).filter( ImportData.u_uuid == u_uuid).one() import_data_db.u_end_import_datetime = end_import_time #更新节点数量和关系数量 node_count_db = db_session.query(SystemPar).filter( SystemPar.par_code == 'node_count').one() node_count = par_dict['node_count'] node_count_db.par_value = str(node_count) edge_count_db = db_session.query(SystemPar).filter( SystemPar.par_code == 'edge_count').one() edge_count = par_dict['edge_count'] edge_count_db.par_value = str(edge_count) db_session.flush() #更新数据库中现有的节点、关系和属性 CurrentNodeLabels.delete_all(db_session) db_session.flush() node_labels = par_dict['node_labels'] for node_label in node_labels: currentNodeLabels = CurrentNodeLabels(labels=str(node_label[0]), label=node_label[1], create_datetime=end_import_time) db_session.add(currentNodeLabels) db_session.flush() CurrentEdgeTyps.delete_all(db_session) db_session.flush() edge_types = par_dict['edge_types'] for edge_type in edge_types: currentEdgeTyps = CurrentEdgeTyps(edge_type=edge_type, create_datetime=end_import_time) db_session.add(currentEdgeTyps) db_session.flush() CurrentProperties.delete_all(db_session) db_session.flush() properties = par_dict['properties'] for _property in properties: currentProperties = CurrentProperties(u_uuid=str(uuid.uuid1()), u_type=_property[0], u_label_type=_property[1], u_column_name=_property[2], u_column_type=_property[3], create_datetime=end_import_time) db_session.add(currentProperties) #设置其它的import_data的导入日期为NONE end_import_time db_session.commit() db_session.close()
def neo4j_rebuild(manage_import_data, import_data): socketio.start_background_task( long_time_process, { 'message_type': "neo4j_rebuild_start", 'message_info': '系统开始重建分析数据库,业务将暂时中止,需等待数据库重建完成' }) #emit('neo4j_rebuild_start', {'message': '系统开始重建分析数据库,业务将暂时中止,需等待数据库重建完成'}, broadcast=True) #print(manage_import_data) #print(import_data) #print(len(manage_import_data)) #如果两个列表的数量不一致,不能重建 if len(manage_import_data) != len(import_data): socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_end", 'messsage_info': '重建未成功' }) #emit('neo4j_rebuild_end', {'message': '重建未成功'}, broadcast=True) else: system_type = '' if platform.platform().find('Windows') >= 0: system_type = 'Windows' else: system_type = 'UNIX' import_command = '' db_session = create_session() import_neo4j_install_dir = db_session.query(SystemPar).filter( SystemPar.par_code == 'import_neo4j_install_dir').one() import_command += import_neo4j_install_dir.par_value + 'bin/' + ( 'neo4j-admin.bat' if system_type == 'Windows' else 'neo4j-admin') + ' import' #import_command+=' --mode csv --database graph.db ' #print(import_command) #节点数量和关系数量 node_count = 0 edge_count = 0 node_labels = [] #第一个字段是输入的lables,可能两个值,后面一个字段是明细 edge_types = [] properties = [] #第一个字段是类型,第二个字段是输入的lables或者type,第三个字段是名称,第四个字段是类型 start_import_time = datetime.datetime.now() for index in range(len(manage_import_data)): if manage_import_data[index]: item = import_data[index] #跟新数据库中的开始导入时间 u_uuid = item['u_uuid'] #print(u_uuid) import_data_db = db_session.query(ImportData).filter( ImportData.u_uuid == u_uuid).one() import_data_db.u_start_import_datetime = start_import_time #数据文件信息 u_queue_uuid = item['u_queue_uuid'] u_ndoe_edge = item['u_node_edge'] u_queue_uuid = item['u_queue_uuid'] u_label_items = '' u_edge_type = '' if u_ndoe_edge == 'node': #node u_label_items = item['u_label_items'].split(',') labels = '' for label in u_label_items: labels += ':' + label if [u_label_items, label] not in node_labels: node_labels.append([u_label_items, label]) import_command += ' --nodes' + labels + '=' + import_neo4j_install_dir.par_value + 'import/' + u_queue_uuid node_count += import_data_db.u_rowcount else: #edge u_edge_type = item['u_edge_type'] if u_edge_type not in edge_types: edge_types.append(u_edge_type) import_command += ' --relationships:' + u_edge_type + '=' + import_neo4j_install_dir.par_value + 'import/' + u_queue_uuid edge_count += import_data_db.u_rowcount #处理属性 u_column_items = item['u_column_items'] #print(u_column_items) #解析u_column_items为数组 array_cols = u_column_items.split(',') flag = 0 while (flag < len(array_cols)): col_item = [] col_item.append(u_ndoe_edge) col_item.append( str(u_label_items) if u_ndoe_edge == 'node' else u_edge_type) col_item.append(array_cols[flag]) flag += 1 #col_item.append(array_cols[flag]) flag += 1 col_item.append(array_cols[flag]) flag += 1 properties.append(col_item) #print(col_item) db_session.commit() import_command += ' --ignore-extra-columns=true --ignore-missing-nodes=true --ignore-duplicate-nodes=true' #停止Neo4j socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '开始停止分析服务器' }) #emit('neo4j_rebuild_process', {'message': '开始停止分析服务器'}, broadcast=True) stop_commonad = import_neo4j_install_dir.par_value + ( 'bin/neo4j.bat' if system_type == 'Windows' else 'bin/neo4j') + ' stop' print(stop_commonad) r_stop_commonad = os.popen( stop_commonad).read() #subprocess.call(stop_commonad) print(r_stop_commonad) #socketio.start_background_task(long_time_process,{'message_type':"neo4j_rebuild_process", 'message_info':'分析服务器成功停止'}) #emit('neo4j_rebuild_process', {'message': '分析服务器成功停止'}, broadcast=True) #删除原数据库 socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '开始清理服务器数据' }) #emit('neo4j_rebuild_process', {'message': '开始清理服务器数据'}, broadcast=True) long_run(socketio, 'clean_neo4j', '') ''' while True: mem=psutil.virtual_memory() disk=psutil.disk_usage(import_neo4j_install_dir.par_value) socketio.emit('system_report',{'platform':platform.platform(),'disk_total':disk.total,'disk_free':disk.free,'cpu_percent':psutil.cpu_percent(),'mem_total':mem.total,'mem_used':mem.used,'mem_free':mem.free}, broadcast=True) socketio.sleep(10) if system_type=='Windows': windows_path=import_neo4j_install_dir.par_value.replace("/", "\\") if os.path.exists(windows_path+'data\\databases\\graph.db\\temp.db\\temp.db'): del_db_command='del /q '+windows_path+'data\\databases\\graph.db\\temp.db\\temp.db' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db\\temp.db'): del_db_command='del /q '+windows_path+'data\\databases\\graph.db\\temp.db' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db\\index'): del_db_command='del /q '+windows_path+'data\\databases\\graph.db\\index' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db\\profiles'): del_db_command='del /q '+windows_path+'data\\databases\\graph.db\\profiles' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db'): del_db_command='del /q '+windows_path+'data\\databases\\graph.db' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) #删除目录 if os.path.exists(windows_path+'data\\databases\\graph.db\\temp.db\\temp.db'): del_db_command='rd '+windows_path+'data\\databases\\graph.db\\temp.db\\temp.db' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db\\temp.db'): del_db_command='rd '+windows_path+'data\\databases\\graph.db\\temp.db' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db\\index'): del_db_command='rd '+windows_path+'data\\databases\\graph.db\\index' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db\\profiles'): del_db_command='rd '+windows_path+'data\\databases\\graph.db\\profiles' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) if os.path.exists(windows_path+'data\\databases\\graph.db'): del_db_command='rd '+windows_path+'data\\databases\\graph.db' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) #测试是否删掉 if os.path.exists(windows_path+'data\\databases\\graph.db'): #数据库并未停止继续删除 pass else: break else: del_db_command='rm -Rf '+import_neo4j_install_dir.par_value+'data/databases/graph.db' print(del_db_command) r_del_db_command = os.popen(del_db_command).read() print(r_del_db_command) #测试是否删掉 if os.path.exists(import_neo4j_install_dir.par_value+'data/databases/graph.db'): #数据库并未停止继续删除 pass else: break ''' #socketio.start_background_task(long_time_process,{'message_type':"neo4j_rebuild_process", 'message_info':'服务器数据成功清理'}) #emit('neo4j_rebuild_process', {'message': '服务器数据成功清理'}, broadcast=True) #导入数据库 socketio.start_background_task( long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '开始导入数据并重建分析数据库,请等待' }) #emit('neo4j_rebuild_process', {'message': '开始导入数据并重建分析数据库,请等待'}, broadcast=True) #socketio.start_background_task(neo4j_import,{'import_command':import_command,'system_type':system_type,'properties':properties,'edge_types':edge_types,'node_labels':node_labels,'edge_count':edge_count,'node_count':node_count,'neo4j_import':neo4j_import, 'import_neo4j_install_dir':import_neo4j_install_dir.par_value,'manage_import_data':manage_import_data,'import_data':import_data}) #发送任务到queue long_run(socketio, 'import_data', import_command) ''' import_queue_uuid=str(uuid.uuid1()) import_queue=JobQueue(u_uuid=import_queue_uuid,u_declare_key='import_data',u_body=import_command,u_publisher_id='import_data',u_publish_datetime=start_import_time,u_no_ack=False,u_start_datetime=None,u_complete_datetime=None,u_status='发布') db_session.add(import_queue) db_session.flush() db_session.commit() #循环检测状态看是否导入成功 db_session_check=create_session() import_queue_reload_init=db_session_check.query(JobQueue).filter(JobQueue.u_uuid==import_queue_uuid).one() u_complete_datetime=import_queue_reload_init.u_complete_datetime db_session_check.close() while u_complete_datetime==None: socketio.sleep(10) mem=psutil.virtual_memory() disk=psutil.disk_usage(import_neo4j_install_dir.par_value) socketio.emit('system_report',{'platform':platform.platform(),'disk_total':disk.total,'disk_free':disk.free,'cpu_percent':psutil.cpu_percent(),'mem_total':mem.total,'mem_used':mem.used,'mem_free':mem.free}, broadcast=True) db_session_check=create_session() import_queue_reload=db_session_check.query(JobQueue).filter(JobQueue.u_uuid==import_queue_uuid).one() u_complete_datetime=import_queue_reload.u_complete_datetime u_status=import_queue_reload.u_status db_session_check.close() print(import_queue_uuid) print(u_status) print(u_complete_datetime) ''' socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '导入数据成功' }) #emit('neo4j_rebuild_process', {'message': '导入数据成功'}, broadcast=True) socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_process", 'message_info': '开始启动分析服务器' }) #emit('neo4j_rebuild_process', {'message': '开始启动分析服务器'}, broadcast=True) #启动数据库 start_commonad = import_neo4j_install_dir.par_value + ( 'bin/neo4j.bat' if system_type == 'Windows' else 'bin/neo4j') + ' start' print(start_commonad) r_start_commonad = os.popen( start_commonad).read() #subprocess.call(start_commonad) #socketio.start_background_task(long_time_process,{'message_type':"neo4j_rebuild_process", 'message_info':'分析服务器启动成功'}) #emit('neo4j_rebuild_process', {'message': '分析服务器启动成功'}, broadcast=True) #socketio.sleep(5) print("分析服务器启动成功") socketio.start_background_task(long_time_process, { 'message_type': "neo4j_rebuild_end", 'message_info': '分析数据库重建完成' }) #emit('neo4j_rebuild_end', {'message': '分析数据库重建完成'}, broadcast=True) print("重建数据库完成") #跟新数据库数据 end_import_time = datetime.datetime.now() #manage_import_data=par_dict['manage_import_data'] #import_data=par_dict['import_data'] for index in range(len(manage_import_data)): if manage_import_data[index]: item = import_data[index] u_uuid = item['u_uuid'] import_data_db = db_session.query(ImportData).filter( ImportData.u_uuid == u_uuid).one() import_data_db.u_end_import_datetime = end_import_time #更新节点数量和关系数量 node_count_db = db_session.query(SystemPar).filter( SystemPar.par_code == 'node_count').one() #node_count=par_dict['node_count'] node_count_db.par_value = str(node_count) edge_count_db = db_session.query(SystemPar).filter( SystemPar.par_code == 'edge_count').one() #edge_count=par_dict['edge_count'] edge_count_db.par_value = str(edge_count) db_session.flush() #更新数据库中现有的节点、关系和属性 CurrentNodeLabels.delete_all(db_session) db_session.flush() #node_labels=par_dict['node_labels'] for node_label in node_labels: currentNodeLabels = CurrentNodeLabels( labels=str(node_label[0]), label=node_label[1], create_datetime=end_import_time) db_session.add(currentNodeLabels) db_session.flush() CurrentEdgeTyps.delete_all(db_session) db_session.flush() #edge_types=par_dict['edge_types'] for edge_type in edge_types: currentEdgeTyps = CurrentEdgeTyps(edge_type=edge_type, create_datetime=end_import_time) db_session.add(currentEdgeTyps) db_session.flush() CurrentProperties.delete_all(db_session) db_session.flush() #properties=par_dict['properties'] for _property in properties: currentProperties = CurrentProperties( u_uuid=str(uuid.uuid1()), u_type=_property[0], u_label_type=_property[1], u_column_name=_property[2], u_column_type=_property[3], create_datetime=end_import_time) db_session.add(currentProperties) #设置其它数据的导入时间为NONE other_import_data_dbs = db_session.query(ImportData).filter( ImportData.u_end_import_datetime != end_import_time).all() for item in other_import_data_dbs: #item=other_import_data_dbs[i] item.u_start_import_datetime = None item.u_end_import_datetime = None db_session.commit() db_session.close()
def import_queue_upload(): db_type = request.form.get('db_type') db_address = request.form.get('db_address') db_port = request.form.get('db_port') db_name = request.form.get('db_name') db_username = request.form.get('db_username') db_password = request.form.get('db_password') select_table = request.form.get('select_table') column_items = request.form.get('column_items') label_items = request.form.get('label_items') node_edge = request.form.get('node_edge') edge_type = request.form.get('edge_type') #从数据库得到neo4j的安装路径 db_session = create_session() import_neo4j_install_dir = db_session.query(SystemPar).filter( SystemPar.par_code == 'import_neo4j_install_dir').one() db_session.close() #queue_uuid是queue的id,也是文件名 create_pub_time = datetime.datetime.now() queue_uuid = str(uuid.uuid1()) u_body = { 'db_type': db_type, 'db_address': db_address, 'db_port': db_port, 'db_name': db_name, 'db_username': db_username, 'db_password': db_password, 'select_table': select_table, 'column_items': column_items, 'label_items': label_items, 'node_edge': node_edge, 'edge_type': edge_type } print(edge_type) queue = JobQueue(u_uuid=queue_uuid, u_declare_key='download_data', u_body=str(u_body), u_publisher_id='download_data', u_publish_datetime=create_pub_time, u_no_ack=False, u_start_datetime=None, u_complete_datetime=None, u_status='发布') db_session = create_session() db_session.add(queue) db_session.commit() db_session.close() #数据导入列表 import_uuid = str(uuid.uuid1()) import_data = ImportData(u_uuid=import_uuid, u_create_datetime=create_pub_time, u_queue_uuid=queue_uuid, u_start_download_datetime=None, u_end_download_datetime=None, u_start_import_datetime=None, u_end_import_datetime=None, u_node_edge=node_edge, u_label_items=label_items, u_edge_type=edge_type, u_column_items=column_items, u_status='创建任务') db_session = create_session() db_session.add(import_data) db_session.commit() db_session.close() return jsonify({'status': 'success'})
def new_system_par(): systemPar = SystemPar.from_json(request.json) db_session = create_session() db_session.add(systemPar) db_session.close()