def get(self): return_status = None result = {} start = int(round(time.time() * 1000)) try: etl_metadata_obj = Metadata() id = request.args.get("id") data = etl_metadata_obj.select_workflows(id) return_status = 200 result = {"status": "success", "data": data} log.info("__") except: result = {} log.exception('Exception while submitting file processing Request') return_status = 500 result['status'] = 0 result[ 'message'] = 'Internal Error has Occurred while processing the File request' finally: resp = Response(json.dumps(result), status=return_status, mimetype="application/json") del etl_metadata_obj return resp
def run(self, request): """ This function stores the user feedback received from Ul server """ return_status = None result = {} try: total_rows = 0 # validate payload log.info(request.json) d = request.json current_cwd = os.getcwd() UPLOAD_FOLDER = current_cwd + "\\data\\" d['file_name'] = UPLOAD_FOLDER + d['file_name'].split( '.')[0] + ".csv" etl_obj = Fileprocess(d) file_type = etl_obj.validate() if not file_type in ("csv", "xlsx", "pdf"): raise ValueError(file_type) if request.args.get("metadata_ind", None) == "yes": data = etl_obj.file_metadata_process(file_type) elif request.args.get("pivot_ind", None) == "yes": file_name = request.json.get("file_name", None) filter_items = request.json.get("filter", None) columns = request.json.get("columns", None) rows = request.json.get("rows", None) values_section = request.json.get("values", None) download_ind = request.json.get("download_ind", None) data = etl_obj.file_process(file_type, file_name, filter_items, columns, rows, values_section, download_ind) else: data = etl_obj.sample_data(file_type) return_status = 200 result['status'] = "successful" result['data'] = data except ValueError as e: result = {} log.exception("Value Exception while submitting feedback") result['status'] = "failed" return_status = 400 result['message'] = e.args[0] except: result = {} log.exception("Exception while submitting feedback") return_status = 500 result['status'] = "failed" result[ 'message'] = 'internal error has occured while processing the request' return return_status, result
def submit(self, name, frames, parameters={}): job = self.create_job(name, frames, parameters, self.is_cali()) # Deadline integration log.info(f'Deadline submit shot: {self}') deadline_ids = submit_deadline(self._parent.name, self, job) if deadline_ids is None: log.error('Deadline submit server error!') job.remove() return log.info(f'Deadline submit done: {self}') ui.dispatch_event( UIEventType.NOTIFICATION, { 'title': f'[{self.name}] Submit Success', 'description': (f'Shot [{self.name}] submitted with {len(frames)} frames.') }) if self.state != 2: self.update({'state': 2}) job.update({'deadline_ids': deadline_ids})
def post(self): return_status = None result = {} start = int(round(time.time() * 1000)) try: log.info("api Request Initiated") fp = FilesourceMiddleware(filesource_parser) return_status, result = fp.run(request) log.info("__") except: result = {} log.exception('Exception while submitting file processing Request') return_status = 500 result['status'] = 0 result[ 'message'] = 'Internal Error has Occurred while processing the File request' finally: #resp = Response(json.dumps(result, cls=NpEncoder) ,status = return_status, mimetype ="application/json") resp = Response(json.dumps(result), status=return_status, mimetype="application/json") #metrics_logging(request, resp, int(round(time.time() * 100 ))start) return resp
def retrigger(self): log.info('Retrigger') message_manager.send_message( MessageType.RETRIGGER ) self.stop_capture()
def select_job(self, job): if job: log.info(f'Select job: {job}') if job.get_id() not in self._selected_jobs: self._selected_jobs[job.get_id()] = job else: log.info('Select job: Empty') self.current_job = job ui.dispatch_event(UIEventType.JOB_SELECTED, job)
def on_entity_event(self, event, entity): """聆聽實體事件 當專案與 Shot 有事件發生時的處理 Args: event: 實體事件 entity: 發生事件的實體 """ # 如果有實體刪除 if event is EntityEvent.REMOVE: # 專案: 同時在 self._projects 刪除並確認是否是 current 來清空 if isinstance(entity, ProjectEntity): log.info('Remove project: {}'.format(entity)) self._projects.remove(entity) ui.dispatch_event(UIEventType.PROJECT_MODIFIED, [*self._projects]) if self.current_project == entity: self.select_project(None) # Shot: 如果是 current 清空,並傳送給 slave 通知刪除檔案 elif isinstance(entity, ShotEntity): log.info('Remove shot: {}'.format(entity)) if self.current_shot == entity: self.select_shot(None) ui.dispatch_event(UIEventType.SHOT_MODIFIED, [*self.current_project.shots]) message_manager.send_message(MessageType.REMOVE_SHOT, {'shot_id': entity.get_id()}) elif isinstance(entity, JobEntity): log.info('Remove job: {}'.format(entity)) if self.current_job == entity: self.select_job(None) ui.dispatch_event(UIEventType.JOB_MODIFIED, [*self.current_shot.jobs]) # 如果有實體創建 elif event is EntityEvent.CREATE: log.info(f'Create {entity.print_name}: {entity}') if isinstance(entity, ShotEntity): ui.dispatch_event(UIEventType.SHOT_MODIFIED, [*self.current_project.shots]) self.select_shot(entity) elif isinstance(entity, JobEntity): ui.dispatch_event(UIEventType.JOB_MODIFIED, [*self.current_shot.jobs]) # 如果有實體更改 elif event is EntityEvent.MODIFY: log.info(f'Modify:\n {entity.print_name}')
def validate_user_connection(self,user_input): try: conn_values = user_input['connectionInfo'] source_id = user_input['sourceID'] login_keys = [] login_values= [] for i in conn_values: login_keys.append(i['fieldId']) login_values.append(i['value']) data =dict(zip(login_keys,login_values)) log.info(data) server =int(source_id) connection_name =data['1'] host = data['2'] server_id = data['3'] user = data['4'] password = data['5'] db_name = data['6'] port = data['7'] print(server,host,user,password,db_name) # postgrasql connection status = "successful" if server == 1: connect_str = f"dbname='{db_name}'user='******' host='{host}' password='******'" connection = psycopg2.connect(connect_str) #Mysql connection elif server == 2: connection = mysql.connector.connect(host=host, user=user, password=password,database=db_name) # Oracle connection elif server == 3: connection = cx_Oracle.connect(user=user, password=password,dsn=host) # SQL server connection elif server == 4: connection = pyodbc.connect(host=host,user=user, password =password,database =db_name,port=port,Driver='') #sql Lite elif server == 5: connection = sqlite3.connect(host =host) else: raise Exception("invalid server") if status =="successful": status= Connections.insert(self,user_input) except Exception as e: print("Error =%s"%str(e)) status="Failed" finally: connection.close() return status
def run(self): """執行 Qt""" app = QApplication(sys.argv) app.setWindowIcon(QIcon('source/icon/ico.svg')) apply_theme(app) log.info('Initialize UI') main_window = MainWindow() self._main = main_window self._lock.acquire() self._lock.notify() self._lock.release() result = app.exec_()
def record(self): """開關錄製 根據目前的 shot 選擇去啟動錄製 如果已在錄製中便關閉,並創建該 shot 的錄製回報蒐集器 """ self._is_recording = not self._is_recording ui.dispatch_event( UIEventType.RECORDING, self._is_recording ) parms = { 'is_start': self._is_recording } shot_id = project_manager.current_shot.get_id() is_cali = project_manager.current_shot.is_cali() # 開啟錄製 if self._is_recording: parms['shot_id'] = shot_id parms['is_cali'] = is_cali log.info('Start recording: {} / {}'.format( project_manager.current_project, project_manager.current_shot )) # 關閉錄製 else: log.info('Stop recording') # 取得這次錄製的相機參數 parameters = {} for name, parm in self._parameters.items(): parameters[name] = parm.get_value() # 建立錄製報告容器 self._report_collector.new_record_report_container( shot_id, parameters ) message_manager.send_message( MessageType.TOGGLE_RECORDING, parms ) if self._is_recording and is_cali: self.record()
def select_shot(self, shot): """選擇 Shot 選擇 Shot 後,會在 self._selected_shots 新增以便之後快速存取 """ if shot: log.info(f'Select shot: {shot}') if shot.get_id() not in self._selected_shots: self._selected_shots[shot.get_id()] = shot else: log.info('Select shot: Empty') self.current_shot = shot ui.dispatch_event(UIEventType.SHOT_SELECTED, shot)
def add_connection(self, conn): """增加連線 將新增的連線包成 node,納入管理 Args: conn: 連線的 socket """ name = conn.getpeername() # 取得連線名稱當作ID log.info(f'Connection established ({name[0]})') send_node = MessageSendNode(conn, name) receive_node = MessageReceiveNode(conn, name, put_inbox=self._put_inbox) self._nodes[name] = (send_node, receive_node)
def trigger(self): """觸發相機擷取""" log.info('Trigger camera capture') hardware_trigger.trigger() # 觸發後,將相機參數做統一設定 for name, parm in self._parameters.items(): self.change_parameter( name, parm.get_value() ) ui.dispatch_event( UIEventType.TRIGGER, True ) self._is_capturing = True
def submit_job(step, _job_info): """在 deadline 新增排程工作,並返回 job id Args: _job_info: 要覆蓋的 job 資料 plugin_info: 要覆蓋的 plugin 資料 """ extra_info_dict = { 'resolve_step': step, 'shot_path': shot_path, 'job_path': job_path, 'parameters': json.dumps(job.parameters, ensure_ascii=True) } if not is_cali: extra_info_dict['cali_path'] = f'{setting.submit_cali_path}{job.get_cali_id()}/' count = 0 extra_info = {} for key, value in extra_info_dict.items(): extra_info[f'ExtraInfoKeyValue{count}'] = f'{key}={value}' count += 1 job_info = { 'Plugin': '4DREC', 'BatchName': f'[4D][{label}] {project_name} - {shot.name} - {job.name} ' + f'({job.get_id()})', 'Name': f'{submit_job.count} - {step} ({job.get_id()})', 'UserName': '******', 'ChunkSize': '1', 'Pool': '4drec', 'Frames': ','.join([str(f) for f in job.frames]), 'OutputDirectory0': job_path, **_job_info, **extra_info } result = deadline.Jobs.SubmitJob(job_info, {}) if isinstance(result, dict) and '_id' in result: log.info(f"Deadline submitted: {job_info['Name']}") submit_job.count += 1 return result['_id'] return None
def live_view( self, toggle, scale_length=150, close_up=None ): """開關相機預覽 開關指定的相機預覽,同時設定串流品質跟尺寸 Args: camera_ids: [相機ID] quality: 串流品質 scale_length: 最長邊長度 """ if toggle: self._delay.execute( lambda: ( log.info('Toggle LiveView on'), message_manager.send_message( MessageType.TOGGLE_LIVE_VIEW, { 'quality': setting.jpeg.live_view.quality, 'scale_length': scale_length, 'close_up': close_up, 'toggle': toggle } ), ui.dispatch_event( UIEventType.LIVE_VIEW, True ) ) ) else: log.info('Toggle LiveView off') message_manager.send_message( MessageType.TOGGLE_LIVE_VIEW, { 'toggle': toggle } ) ui.dispatch_event( UIEventType.LIVE_VIEW, False )
def _initialize(self): """初始化相機系統 用 has_reset 去操作恢復原廠設定 另外檢查相機初始化的數量是否正確,如果不是正確數量,隔一段時間再重試 """ has_reset = True # 恢復原廠設定的開關,false 的話初始化都會重置一次 while True: log.info('Initialize camera system') self._camera_system = PySpin.System.GetInstance() self._camera_list = self._camera_system.GetCameras() current_cameras_count = self._camera_list.GetSize() setting_cameras_count = setting.get_slave_cameras_count() # 相機數量不對的狀況 if current_cameras_count != setting_cameras_count: log.error( ("Camera count didn't match setting" f' ({current_cameras_count}/{setting_cameras_count}),' ' try again after 30s')) time.sleep(10) self.clear(retry=True) continue # 需要恢復原廠設定的狀況 if not has_reset: log.info('Factory reset cameras') for i in range(current_cameras_count): camera = self._camera_list.GetByIndex(i) camera.Init() camera.FactoryReset() del camera self.clear(retry=True) has_reset = True continue # 正常初始化的狀況,將相機給 connector 納管 self._connectors = self._build_connectors() break
def _run_master(self): """master 的運作 在為 master 的情況,socket 建立聆聽後會轉交給另一個執行緒 MessageAccepter MessageAccepter 負責把產生連線的對方 socket 傳回 queue 去做監控 """ connect_queue = queue.Queue() sock = self._build_socket() sock.bind(('0.0.0.0', setting.host_address.port)) # 聆聽連線 sock.listen(6) log.info(f'Socket is Listening') # 建立連線交給 Accepter 處理 self._accepter = MessageAccepter(sock, connect_queue) while self._running: # 有新連線的狀況 if not connect_queue.empty(): conn = connect_queue.get() self._node.add_connection(conn) else: # 檢查已連線的 socket 是否有出錯 has_error = None for node in self._node.get_all(): if node.isFinished(): error = node.get_error() if error is not None: log.warning(f'<{node.get_name()[0]}> {error}') has_error = node self._node.remove(node) # 連線有出錯的狀況,送警告訊息給自己 if has_error is not None: self.send_message(MessageType.SLAVE_DOWN, {'node': node}, is_local=True) # 緩衝 loop 的間隔時間 time.sleep(0.01)
def start_master(): """master 總啟動程序""" from utility.message import message_manager from utility.logger import log from utility.define import MessageType from .ui import ui # 初始化 UI from .hardware_trigger import hardware_trigger from .camera import camera_manager from .resolve import resolve_manager log.info('Start Master') ui.show() while True: # Message 接收與觸發 message = message_manager.receive_message() if (message.type is MessageType.LIVE_VIEW_IMAGE or message.type is MessageType.SHOT_IMAGE): camera_manager.receive_image(message) elif message.type is MessageType.CAMERA_STATUS: camera_manager.update_status(message) elif message.type is MessageType.SLAVE_DOWN: camera_manager.stop_capture(message) elif message.type is MessageType.MASTER_DOWN: log.warning('Master closed') break elif message.type is MessageType.RECORD_REPORT: camera_manager.collect_report(message) elif message.type is MessageType.SUBMIT_REPORT: camera_manager.collect_report(message) elif message.type is MessageType.TRIGGER_REPORT: camera_manager.collect_report(message) # 關閉通訊 hardware_trigger.close() message_manager.stop()
def post(self): return_status = None result = {} start = int(round(time.time() * 1000)) try: file_name = "" file_name = request.json["file_name"] columns = request.json["columns"] if file_name: download_file = file_name.split('\\')[-1] download_file = r"C:\\Users\\venkats_mandadapu\\{}".format( download_file) chunk_size = 1000000 reader = pd.read_csv(file_name, header=0, iterator=True) chunks = [] loop = True while loop: try: chunk = reader.get_chunk(chunk_size)[columns] chunks.append(chunk) except StopIteration: loop = False print("Iteration is stopped") df_ac = pd.concat(chunks, ignore_index=True) df_ac.to_csv(download_file, index=False) return_status = 200 result = {"status": "success", "download_file": download_file} log.info("__") except: result = {} log.exception('Exception while submitting file processing Request') return_status = 500 result['status'] = 0 result[ 'message'] = 'Internal Error has Occurred while processing the File request' finally: resp = Response(json.dumps(result), status=return_status, mimetype="application/json") return resp
def _run_slave(self): """slave 的運作 在為 slave 的情況,socket 會去找 host 並連接 隨時監控連接是否斷訊並做相應處理 """ connected = False while self._running: try: sock = self._build_socket() sock.settimeout(1.0) sock.connect(self._address) sock.settimeout(None) # 連接上後的處理 connected = True self._node.add_connection(sock) self.send_message(MessageType.MASTER_UP, is_local=True) while self._running: # 監測 socket 是否有狀況 for node in self._node.get_all(): node.join(0.01) error = node.get_error() if error is not None: raise error except (ConnectionResetError, ConnectionRefusedError, ConnectionAbortedError, TimeoutError, OSError) as error: # 有狀況時的回報 if connected: log.warning(error) self.send_message(MessageType.MASTER_DOWN, is_local=True) connected = False # connected 的設置是避免不斷報錯 if error.errno == 10054: if current_process().name == 'MainProcess': log.info('hihi') self._node.clear()
def _setup_ui(self): log.info('create widgets') """創建介面物件""" from .custom_widgets import LayoutWidget, make_layout from .header import Header from .body import Body from .footer import Footer from .sidebar import Sidebar from .dialog import SecondScreenView self.setStyleSheet(self._default) log.info('finish widgets') # 尺寸 self.setGeometry(100, 80, 1600, 1100) # 版面 widget = LayoutWidget(horizon=False) hlayout = make_layout() vlayout = make_layout(horizon=False) vlayout.addLayout(Body()) vlayout.addWidget(Footer()) hlayout.addWidget(Sidebar()) hlayout.addLayout(vlayout) # 設定 widget.addWidget(Header()) widget.addLayout(hlayout) self.setCentralWidget(widget) # 置中 self.layout().invalidate() self.layout().activate() center = self.rect().center() dcenter = QApplication.desktop().screenGeometry().center() self.move(dcenter - center) # 全螢幕預覽 self._second_screen = SecondScreenView()
def submit_shot(self, name, frames, parameters): """到 deadline 放算""" shot = project_manager.current_shot log.info(f'Preparing to submit shot: {shot}') self._report_collector.new_submit_report_container( shot, name, frames, parameters ) # 通知 Slaves 傳輸轉檔 Shot message_manager.send_message( MessageType.SUBMIT_SHOT, { 'shot_id': shot.get_id(), 'job_name': name, 'frames': frames } )
def start_slave(): """Slave 總啟動程序""" log.info('Start slave') # 等待 Master 連接 log.info('Wait for master connecting...') if not message_manager.is_connected(): message = message_manager.receive_message() while message.type is not MessageType.MASTER_UP: continue # 相機系統初始化 camera_system = CameraSystem() camera_system.start() is_master_down = False while True: message = message_manager.receive_message() if message.type is MessageType.MASTER_DOWN: log.warning('Master Down !!') is_master_down = True break log.info('Stop all connectors') camera_system.stop() if is_master_down: restart()
def _summarize_report(self): """總結 找出所有失蹤格數與最大最小格數,並更新 Shot 的資料 """ # 總容量大小與最大、最小的格數 start_frames = [] end_frames = [] size = 0 for r in self._reports: start_frames.append(r['frame_range'][0]) end_frames.append(r['frame_range'][1]) size += r['size'] start_frame = max(start_frames) # 讓開始格數齊頭 end_frame = min(end_frames) # 讓結束格數齊尾 # 失蹤格數,藉由開始結尾跟相機編號所設立 missing_frames = {} for r in self._reports: frames = r['missing_frames'] missing_frames[r['camera_id']] = ([ f for f in frames if f >= start_frame and f <= end_frame ]) data = { 'frame_range': (start_frame, end_frame), 'size': size, 'missing_frames': missing_frames, 'camera_parameters': self._parameters, 'state': 1 } # 更新資料庫 log.info('Update shot with recorded data') shot = project_manager.get_shot(self._shot_id) shot.update(data)
def select_project(self, project): """選擇專案 選擇專案,如果專案已有 Shot,會選擇最新的 Shot """ self.current_project = project if project.get_id() not in self._selected_projects: self._selected_projects[project.get_id()] = project if project: log.info(f'Select project: {project}') if len(project.shots) != 0: self.select_shot(project.shots[0]) else: self.select_shot(None) else: log.info('Select project: Empty') if self.current_shot is not None: self.select_shot(None) ui.dispatch_event(UIEventType.PROJECT_SELECTED, project)
def run(self, request): """ This function stores the user feedback received from Ul server """ return_status = None result = {} log.info(f"args: (str(self.args)") try: # validate payload log.info(request.args) log.info(f"input file: {request.json['file']}") etl_obj = Fileprocess(request.json) msg = etl_obj.validate() if not etl_obj.validate() == "successful": raise ValueError(msg) if request.args["file_source_type"] == "excel": data = etl_obj.excel_file_process() else: data = etl_obj.non_excel_file_process() return_status = 200 result['status'] = 1 result['data'] = data except ValueError as e: result = {} log.exception("Value Excerticon while submitting feedback") result['status'] = 0 return_status = 400 result['message'] = e.args[0] except: result = {} log.exception("Exception while submitting feedback") return_status = 500 result['status'] = 0 result[ 'message'] = 'internal error has occured while processing the request' return return_status, result
def run(self, request, id=None, connectionid=None, post_request=False, active_connections_ind=False): """ This function stare the user feedback received from UI server """ return_status = None result = {} try: # validate payload log.info(request.args) if post_request: user_input = request.json data = self.etl_obj.validate_user_connection(user_input) elif active_connections_ind: data = self.etl_obj.get_connection_details() else: if connectionid: metadata_dict = { 'schemas': 'TABLE_SCHEMA', 'tables': 'TABLE_NAME', 'columns': 'COLUMN_NAME' } type = request.args.get("type", None) schema = request.args.get("schema", None) table = request.args.get("table", None) if request.args["type"] in metadata_dict: metadata = self.etl_obj.get_db_object_metadata( connectionid, type, schema, table) if len(metadata) > 0: data = list( set( map( lambda element_dict: element_dict[ metadata_dict[request.args['type'] ]], metadata)) )[int(request.args['offset']):int(request. args['limit'])] else: data = 'No data rows was found' else: source_id = None if id: source_id = int(id) data = self.etl_obj.get_datasource(source_id) return_status = 200 result['status'] = 1 result['data'] = data except ValueError as e: result = {} log.exception("Value Exception while submitting feedback") result['status'] = 0 return_status = 400 result['message'] = e.args[0] except Exception: result = {} log.exception("Exception while submitting feedback") return_status = 500 result['status'] = 0 result['message'] = ( 'Internal Error has occurred while processing the request') return return_status, result
#Defining api models from documentation model_400 = api.model('Errorresponse400', { 'message': fields.String, 'errors': fields.Raw }) model_500 = api.model('Errorresponse400', { 'message': fields.Integer, 'errors': fields.String }) model_health_200 = api.model('successResponse200', { 'success': fields.Boolean, 'status': fields.Integer }) log.info("AB-server api started Successfully") #---------------------------------------------------------- current_cwd = os.getcwd() UPLOAD_FOLDER = current_cwd + "\\data\\" app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER @app.route('/upload', methods=['POST']) def upload(): try: return_status = None result = None filename = request.args['filename']
def stop(self): for connector in self._connectors: connector.kill() log.info('Connector stop...') self.clear()