def get(self):
return_status = None
result = {}
start = int(round(time.time() * 1000))
try:
etl_metadata_obj = Metadata()
id = request.args.get("id")
data = etl_metadata_obj.select_workflows(id)
return_status = 200
result = {"status": "success", "data": data}
log.info("__")
except:
result = {}
log.exception('Exception while submitting file processing Request')
return_status = 500
result['status'] = 0
result[
'message'] = 'Internal Error has Occurred while processing the File request'
finally:
resp = Response(json.dumps(result),
status=return_status,
mimetype="application/json")
del etl_metadata_obj
return resp
def run(self, request):
"""
This function stores the user feedback received from Ul server
"""
return_status = None
result = {}
try:
total_rows = 0
# validate payload
log.info(request.json)
d = request.json
current_cwd = os.getcwd()
UPLOAD_FOLDER = current_cwd + "\\data\\"
d['file_name'] = UPLOAD_FOLDER + d['file_name'].split(
'.')[0] + ".csv"
etl_obj = Fileprocess(d)
file_type = etl_obj.validate()
if not file_type in ("csv", "xlsx", "pdf"):
raise ValueError(file_type)
if request.args.get("metadata_ind", None) == "yes":
data = etl_obj.file_metadata_process(file_type)
elif request.args.get("pivot_ind", None) == "yes":
file_name = request.json.get("file_name", None)
filter_items = request.json.get("filter", None)
columns = request.json.get("columns", None)
rows = request.json.get("rows", None)
values_section = request.json.get("values", None)
download_ind = request.json.get("download_ind", None)
data = etl_obj.file_process(file_type, file_name, filter_items,
columns, rows, values_section,
download_ind)
else:
data = etl_obj.sample_data(file_type)
return_status = 200
result['status'] = "successful"
result['data'] = data
except ValueError as e:
result = {}
log.exception("Value Exception while submitting feedback")
result['status'] = "failed"
return_status = 400
result['message'] = e.args[0]
except:
result = {}
log.exception("Exception while submitting feedback")
return_status = 500
result['status'] = "failed"
result[
'message'] = 'internal error has occured while processing the request'
return return_status, result
def submit(self, name, frames, parameters={}):
job = self.create_job(name, frames, parameters, self.is_cali())
# Deadline integration
log.info(f'Deadline submit shot: {self}')
deadline_ids = submit_deadline(self._parent.name, self, job)
if deadline_ids is None:
log.error('Deadline submit server error!')
job.remove()
return
log.info(f'Deadline submit done: {self}')
ui.dispatch_event(
UIEventType.NOTIFICATION, {
'title':
f'[{self.name}] Submit Success',
'description':
(f'Shot [{self.name}] submitted with {len(frames)} frames.')
})
if self.state != 2:
self.update({'state': 2})
job.update({'deadline_ids': deadline_ids})
def post(self):
return_status = None
result = {}
start = int(round(time.time() * 1000))
try:
log.info("api Request Initiated")
fp = FilesourceMiddleware(filesource_parser)
return_status, result = fp.run(request)
log.info("__")
except:
result = {}
log.exception('Exception while submitting file processing Request')
return_status = 500
result['status'] = 0
result[
'message'] = 'Internal Error has Occurred while processing the File request'
finally:
#resp = Response(json.dumps(result, cls=NpEncoder) ,status = return_status, mimetype ="application/json")
resp = Response(json.dumps(result),
status=return_status,
mimetype="application/json")
#metrics_logging(request, resp, int(round(time.time() * 100 ))start)
return resp
def retrigger(self):
log.info('Retrigger')
message_manager.send_message(
MessageType.RETRIGGER
)
self.stop_capture()
def select_job(self, job):
if job:
log.info(f'Select job: {job}')
if job.get_id() not in self._selected_jobs:
self._selected_jobs[job.get_id()] = job
else:
log.info('Select job: Empty')
self.current_job = job
ui.dispatch_event(UIEventType.JOB_SELECTED, job)
def on_entity_event(self, event, entity):
"""聆聽實體事件
當專案與 Shot 有事件發生時的處理
Args:
event: 實體事件
entity: 發生事件的實體
"""
# 如果有實體刪除
if event is EntityEvent.REMOVE:
# 專案: 同時在 self._projects 刪除並確認是否是 current 來清空
if isinstance(entity, ProjectEntity):
log.info('Remove project: {}'.format(entity))
self._projects.remove(entity)
ui.dispatch_event(UIEventType.PROJECT_MODIFIED,
[*self._projects])
if self.current_project == entity:
self.select_project(None)
# Shot: 如果是 current 清空，並傳送給 slave 通知刪除檔案
elif isinstance(entity, ShotEntity):
log.info('Remove shot: {}'.format(entity))
if self.current_shot == entity:
self.select_shot(None)
ui.dispatch_event(UIEventType.SHOT_MODIFIED,
[*self.current_project.shots])
message_manager.send_message(MessageType.REMOVE_SHOT,
{'shot_id': entity.get_id()})
elif isinstance(entity, JobEntity):
log.info('Remove job: {}'.format(entity))
if self.current_job == entity:
self.select_job(None)
ui.dispatch_event(UIEventType.JOB_MODIFIED,
[*self.current_shot.jobs])
# 如果有實體創建
elif event is EntityEvent.CREATE:
log.info(f'Create {entity.print_name}: {entity}')
if isinstance(entity, ShotEntity):
ui.dispatch_event(UIEventType.SHOT_MODIFIED,
[*self.current_project.shots])
self.select_shot(entity)
elif isinstance(entity, JobEntity):
ui.dispatch_event(UIEventType.JOB_MODIFIED,
[*self.current_shot.jobs])
# 如果有實體更改
elif event is EntityEvent.MODIFY:
log.info(f'Modify:\n {entity.print_name}')
def validate_user_connection(self,user_input):
try:
conn_values = user_input['connectionInfo']
source_id = user_input['sourceID']
login_keys = []
login_values= []
for i in conn_values:
login_keys.append(i['fieldId'])
login_values.append(i['value'])
data =dict(zip(login_keys,login_values))
log.info(data)
server =int(source_id)
connection_name =data['1']
host = data['2']
server_id = data['3']
user = data['4']
password = data['5']
db_name = data['6']
port = data['7']
print(server,host,user,password,db_name)
# postgrasql connection
status = "successful"
if server == 1:
connect_str = f"dbname='{db_name}'user='******' host='{host}' password='******'"
connection = psycopg2.connect(connect_str)
#Mysql connection
elif server == 2:
connection = mysql.connector.connect(host=host, user=user, password=password,database=db_name)
# Oracle connection
elif server == 3:
connection = cx_Oracle.connect(user=user, password=password,dsn=host)
# SQL server connection
elif server == 4:
connection = pyodbc.connect(host=host,user=user, password =password,database =db_name,port=port,Driver='')
#sql Lite
elif server == 5:
connection = sqlite3.connect(host =host)
else:
raise Exception("invalid server")
if status =="successful":
status= Connections.insert(self,user_input)
except Exception as e:
print("Error =%s"%str(e))
status="Failed"
finally:
connection.close()
return status
def run(self):
"""執行 Qt"""
app = QApplication(sys.argv)
app.setWindowIcon(QIcon('source/icon/ico.svg'))
apply_theme(app)
log.info('Initialize UI')
main_window = MainWindow()
self._main = main_window
self._lock.acquire()
self._lock.notify()
self._lock.release()
result = app.exec_()
def record(self):
"""開關錄製
根據目前的 shot 選擇去啟動錄製
如果已在錄製中便關閉，並創建該 shot 的錄製回報蒐集器
"""
self._is_recording = not self._is_recording
ui.dispatch_event(
UIEventType.RECORDING,
self._is_recording
)
parms = {
'is_start': self._is_recording
}
shot_id = project_manager.current_shot.get_id()
is_cali = project_manager.current_shot.is_cali()
# 開啟錄製
if self._is_recording:
parms['shot_id'] = shot_id
parms['is_cali'] = is_cali
log.info('Start recording: {} / {}'.format(
project_manager.current_project,
project_manager.current_shot
))
# 關閉錄製
else:
log.info('Stop recording')
# 取得這次錄製的相機參數
parameters = {}
for name, parm in self._parameters.items():
parameters[name] = parm.get_value()
# 建立錄製報告容器
self._report_collector.new_record_report_container(
shot_id, parameters
)
message_manager.send_message(
MessageType.TOGGLE_RECORDING,
parms
)
if self._is_recording and is_cali:
self.record()
def select_shot(self, shot):
"""選擇 Shot
選擇 Shot 後，會在 self._selected_shots 新增以便之後快速存取
"""
if shot:
log.info(f'Select shot: {shot}')
if shot.get_id() not in self._selected_shots:
self._selected_shots[shot.get_id()] = shot
else:
log.info('Select shot: Empty')
self.current_shot = shot
ui.dispatch_event(UIEventType.SHOT_SELECTED, shot)
def add_connection(self, conn):
"""增加連線
將新增的連線包成 node，納入管理
Args:
conn: 連線的 socket
"""
name = conn.getpeername() # 取得連線名稱當作ID
log.info(f'Connection established ({name[0]})')
send_node = MessageSendNode(conn, name)
receive_node = MessageReceiveNode(conn,
name,
put_inbox=self._put_inbox)
self._nodes[name] = (send_node, receive_node)
def trigger(self):
"""觸發相機擷取"""
log.info('Trigger camera capture')
hardware_trigger.trigger()
# 觸發後，將相機參數做統一設定
for name, parm in self._parameters.items():
self.change_parameter(
name, parm.get_value()
)
ui.dispatch_event(
UIEventType.TRIGGER,
True
)
self._is_capturing = True
def submit_job(step, _job_info):
"""在 deadline 新增排程工作，並返回 job id
Args:
_job_info: 要覆蓋的 job 資料
plugin_info: 要覆蓋的 plugin 資料
"""
extra_info_dict = {
'resolve_step': step,
'shot_path': shot_path,
'job_path': job_path,
'parameters': json.dumps(job.parameters, ensure_ascii=True)
}
if not is_cali:
extra_info_dict['cali_path'] = f'{setting.submit_cali_path}{job.get_cali_id()}/'
count = 0
extra_info = {}
for key, value in extra_info_dict.items():
extra_info[f'ExtraInfoKeyValue{count}'] = f'{key}={value}'
count += 1
job_info = {
'Plugin': '4DREC',
'BatchName': f'[4D][{label}] {project_name} - {shot.name} - {job.name} ' +
f'({job.get_id()})',
'Name': f'{submit_job.count} - {step} ({job.get_id()})',
'UserName': '******',
'ChunkSize': '1',
'Pool': '4drec',
'Frames': ','.join([str(f) for f in job.frames]),
'OutputDirectory0': job_path,
**_job_info,
**extra_info
}
result = deadline.Jobs.SubmitJob(job_info, {})
if isinstance(result, dict) and '_id' in result:
log.info(f"Deadline submitted: {job_info['Name']}")
submit_job.count += 1
return result['_id']
return None
def live_view(
self, toggle, scale_length=150, close_up=None
):
"""開關相機預覽
開關指定的相機預覽，同時設定串流品質跟尺寸
Args:
camera_ids: [相機ID]
quality: 串流品質
scale_length: 最長邊長度
"""
if toggle:
self._delay.execute(
lambda:
(
log.info('Toggle LiveView on'),
message_manager.send_message(
MessageType.TOGGLE_LIVE_VIEW,
{
'quality': setting.jpeg.live_view.quality,
'scale_length': scale_length,
'close_up': close_up,
'toggle': toggle
}
),
ui.dispatch_event(
UIEventType.LIVE_VIEW,
True
)
)
)
else:
log.info('Toggle LiveView off')
message_manager.send_message(
MessageType.TOGGLE_LIVE_VIEW,
{
'toggle': toggle
}
)
ui.dispatch_event(
UIEventType.LIVE_VIEW,
False
)
def _initialize(self):
"""初始化相機系統
用 has_reset 去操作恢復原廠設定
另外檢查相機初始化的數量是否正確，如果不是正確數量，隔一段時間再重試
"""
has_reset = True # 恢復原廠設定的開關，false 的話初始化都會重置一次
while True:
log.info('Initialize camera system')
self._camera_system = PySpin.System.GetInstance()
self._camera_list = self._camera_system.GetCameras()
current_cameras_count = self._camera_list.GetSize()
setting_cameras_count = setting.get_slave_cameras_count()
# 相機數量不對的狀況
if current_cameras_count != setting_cameras_count:
log.error(
("Camera count didn't match setting"
f' ({current_cameras_count}/{setting_cameras_count}),'
' try again after 30s'))
time.sleep(10)
self.clear(retry=True)
continue
# 需要恢復原廠設定的狀況
if not has_reset:
log.info('Factory reset cameras')
for i in range(current_cameras_count):
camera = self._camera_list.GetByIndex(i)
camera.Init()
camera.FactoryReset()
del camera
self.clear(retry=True)
has_reset = True
continue
# 正常初始化的狀況，將相機給 connector 納管
self._connectors = self._build_connectors()
break
def _run_master(self):
"""master 的運作
在為 master 的情況，socket 建立聆聽後會轉交給另一個執行緒 MessageAccepter
MessageAccepter 負責把產生連線的對方 socket 傳回 queue 去做監控
"""
connect_queue = queue.Queue()
sock = self._build_socket()
sock.bind(('0.0.0.0', setting.host_address.port))
# 聆聽連線
sock.listen(6)
log.info(f'Socket is Listening')
# 建立連線交給 Accepter 處理
self._accepter = MessageAccepter(sock, connect_queue)
while self._running:
# 有新連線的狀況
if not connect_queue.empty():
conn = connect_queue.get()
self._node.add_connection(conn)
else:
# 檢查已連線的 socket 是否有出錯
has_error = None
for node in self._node.get_all():
if node.isFinished():
error = node.get_error()
if error is not None:
log.warning(f'<{node.get_name()[0]}> {error}')
has_error = node
self._node.remove(node)
# 連線有出錯的狀況，送警告訊息給自己
if has_error is not None:
self.send_message(MessageType.SLAVE_DOWN, {'node': node},
is_local=True)
# 緩衝 loop 的間隔時間
time.sleep(0.01)
def start_master():
"""master 總啟動程序"""
from utility.message import message_manager
from utility.logger import log
from utility.define import MessageType
from .ui import ui # 初始化 UI
from .hardware_trigger import hardware_trigger
from .camera import camera_manager
from .resolve import resolve_manager
log.info('Start Master')
ui.show()
while True:
# Message 接收與觸發
message = message_manager.receive_message()
if (message.type is MessageType.LIVE_VIEW_IMAGE
or message.type is MessageType.SHOT_IMAGE):
camera_manager.receive_image(message)
elif message.type is MessageType.CAMERA_STATUS:
camera_manager.update_status(message)
elif message.type is MessageType.SLAVE_DOWN:
camera_manager.stop_capture(message)
elif message.type is MessageType.MASTER_DOWN:
log.warning('Master closed')
break
elif message.type is MessageType.RECORD_REPORT:
camera_manager.collect_report(message)
elif message.type is MessageType.SUBMIT_REPORT:
camera_manager.collect_report(message)
elif message.type is MessageType.TRIGGER_REPORT:
camera_manager.collect_report(message)
# 關閉通訊
hardware_trigger.close()
message_manager.stop()
def post(self):
return_status = None
result = {}
start = int(round(time.time() * 1000))
try:
file_name = ""
file_name = request.json["file_name"]
columns = request.json["columns"]
if file_name:
download_file = file_name.split('\\')[-1]
download_file = r"C:\\Users\\venkats_mandadapu\\{}".format(
download_file)
chunk_size = 1000000
reader = pd.read_csv(file_name, header=0, iterator=True)
chunks = []
loop = True
while loop:
try:
chunk = reader.get_chunk(chunk_size)[columns]
chunks.append(chunk)
except StopIteration:
loop = False
print("Iteration is stopped")
df_ac = pd.concat(chunks, ignore_index=True)
df_ac.to_csv(download_file, index=False)
return_status = 200
result = {"status": "success", "download_file": download_file}
log.info("__")
except:
result = {}
log.exception('Exception while submitting file processing Request')
return_status = 500
result['status'] = 0
result[
'message'] = 'Internal Error has Occurred while processing the File request'
finally:
resp = Response(json.dumps(result),
status=return_status,
mimetype="application/json")
return resp
def _run_slave(self):
"""slave 的運作
在為 slave 的情況，socket 會去找 host 並連接
隨時監控連接是否斷訊並做相應處理
"""
connected = False
while self._running:
try:
sock = self._build_socket()
sock.settimeout(1.0)
sock.connect(self._address)
sock.settimeout(None)
# 連接上後的處理
connected = True
self._node.add_connection(sock)
self.send_message(MessageType.MASTER_UP, is_local=True)
while self._running:
# 監測 socket 是否有狀況
for node in self._node.get_all():
node.join(0.01)
error = node.get_error()
if error is not None:
raise error
except (ConnectionResetError, ConnectionRefusedError,
ConnectionAbortedError, TimeoutError, OSError) as error:
# 有狀況時的回報
if connected:
log.warning(error)
self.send_message(MessageType.MASTER_DOWN, is_local=True)
connected = False # connected 的設置是避免不斷報錯
if error.errno == 10054:
if current_process().name == 'MainProcess':
log.info('hihi')
self._node.clear()
def _setup_ui(self):
log.info('create widgets')
"""創建介面物件"""
from .custom_widgets import LayoutWidget, make_layout
from .header import Header
from .body import Body
from .footer import Footer
from .sidebar import Sidebar
from .dialog import SecondScreenView
self.setStyleSheet(self._default)
log.info('finish widgets')
# 尺寸
self.setGeometry(100, 80, 1600, 1100)
# 版面
widget = LayoutWidget(horizon=False)
hlayout = make_layout()
vlayout = make_layout(horizon=False)
vlayout.addLayout(Body())
vlayout.addWidget(Footer())
hlayout.addWidget(Sidebar())
hlayout.addLayout(vlayout)
# 設定
widget.addWidget(Header())
widget.addLayout(hlayout)
self.setCentralWidget(widget)
# 置中
self.layout().invalidate()
self.layout().activate()
center = self.rect().center()
dcenter = QApplication.desktop().screenGeometry().center()
self.move(dcenter - center)
# 全螢幕預覽
self._second_screen = SecondScreenView()
def submit_shot(self, name, frames, parameters):
"""到 deadline 放算"""
shot = project_manager.current_shot
log.info(f'Preparing to submit shot: {shot}')
self._report_collector.new_submit_report_container(
shot,
name,
frames,
parameters
)
# 通知 Slaves 傳輸轉檔 Shot
message_manager.send_message(
MessageType.SUBMIT_SHOT,
{
'shot_id': shot.get_id(),
'job_name': name,
'frames': frames
}
)
def start_slave():
"""Slave 總啟動程序"""
log.info('Start slave')
# 等待 Master 連接
log.info('Wait for master connecting...')
if not message_manager.is_connected():
message = message_manager.receive_message()
while message.type is not MessageType.MASTER_UP:
continue
# 相機系統初始化
camera_system = CameraSystem()
camera_system.start()
is_master_down = False
while True:
message = message_manager.receive_message()
if message.type is MessageType.MASTER_DOWN:
log.warning('Master Down !!')
is_master_down = True
break
log.info('Stop all connectors')
camera_system.stop()
if is_master_down:
restart()
def _summarize_report(self):
"""總結
找出所有失蹤格數與最大最小格數，並更新 Shot 的資料
"""
# 總容量大小與最大、最小的格數
start_frames = []
end_frames = []
size = 0
for r in self._reports:
start_frames.append(r['frame_range'][0])
end_frames.append(r['frame_range'][1])
size += r['size']
start_frame = max(start_frames) # 讓開始格數齊頭
end_frame = min(end_frames) # 讓結束格數齊尾
# 失蹤格數，藉由開始結尾跟相機編號所設立
missing_frames = {}
for r in self._reports:
frames = r['missing_frames']
missing_frames[r['camera_id']] = ([
f for f in frames if f >= start_frame and f <= end_frame
])
data = {
'frame_range': (start_frame, end_frame),
'size': size,
'missing_frames': missing_frames,
'camera_parameters': self._parameters,
'state': 1
}
# 更新資料庫
log.info('Update shot with recorded data')
shot = project_manager.get_shot(self._shot_id)
shot.update(data)
def select_project(self, project):
"""選擇專案
選擇專案，如果專案已有 Shot，會選擇最新的 Shot
"""
self.current_project = project
if project.get_id() not in self._selected_projects:
self._selected_projects[project.get_id()] = project
if project:
log.info(f'Select project: {project}')
if len(project.shots) != 0:
self.select_shot(project.shots[0])
else:
self.select_shot(None)
else:
log.info('Select project: Empty')
if self.current_shot is not None:
self.select_shot(None)
ui.dispatch_event(UIEventType.PROJECT_SELECTED, project)
def run(self, request):
"""
This function stores the user feedback received from Ul server
"""
return_status = None
result = {}
log.info(f"args: (str(self.args)")
try:
# validate payload
log.info(request.args)
log.info(f"input file: {request.json['file']}")
etl_obj = Fileprocess(request.json)
msg = etl_obj.validate()
if not etl_obj.validate() == "successful":
raise ValueError(msg)
if request.args["file_source_type"] == "excel":
data = etl_obj.excel_file_process()
else:
data = etl_obj.non_excel_file_process()
return_status = 200
result['status'] = 1
result['data'] = data
except ValueError as e:
result = {}
log.exception("Value Excerticon while submitting feedback")
result['status'] = 0
return_status = 400
result['message'] = e.args[0]
except:
result = {}
log.exception("Exception while submitting feedback")
return_status = 500
result['status'] = 0
result[
'message'] = 'internal error has occured while processing the request'
return return_status, result
def run(self,
request,
id=None,
connectionid=None,
post_request=False,
active_connections_ind=False):
"""
This function stare the user feedback received from UI server
"""
return_status = None
result = {}
try:
# validate payload
log.info(request.args)
if post_request:
user_input = request.json
data = self.etl_obj.validate_user_connection(user_input)
elif active_connections_ind:
data = self.etl_obj.get_connection_details()
else:
if connectionid:
metadata_dict = {
'schemas': 'TABLE_SCHEMA',
'tables': 'TABLE_NAME',
'columns': 'COLUMN_NAME'
}
type = request.args.get("type", None)
schema = request.args.get("schema", None)
table = request.args.get("table", None)
if request.args["type"] in metadata_dict:
metadata = self.etl_obj.get_db_object_metadata(
connectionid, type, schema, table)
if len(metadata) > 0:
data = list(
set(
map(
lambda element_dict: element_dict[
metadata_dict[request.args['type']
]], metadata))
)[int(request.args['offset']):int(request.
args['limit'])]
else:
data = 'No data rows was found'
else:
source_id = None
if id:
source_id = int(id)
data = self.etl_obj.get_datasource(source_id)
return_status = 200
result['status'] = 1
result['data'] = data
except ValueError as e:
result = {}
log.exception("Value Exception while submitting feedback")
result['status'] = 0
return_status = 400
result['message'] = e.args[0]
except Exception:
result = {}
log.exception("Exception while submitting feedback")
return_status = 500
result['status'] = 0
result['message'] = (
'Internal Error has occurred while processing the request')
return return_status, result
#Defining api models from documentation
model_400 = api.model('Errorresponse400', {
'message': fields.String,
'errors': fields.Raw
})
model_500 = api.model('Errorresponse400', {
'message': fields.Integer,
'errors': fields.String
})
model_health_200 = api.model('successResponse200', {
'success': fields.Boolean,
'status': fields.Integer
})
log.info("AB-server api started Successfully")
#----------------------------------------------------------
current_cwd = os.getcwd()
UPLOAD_FOLDER = current_cwd + "\\data\\"
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
@app.route('/upload', methods=['POST'])
def upload():
try:
return_status = None
result = None
filename = request.args['filename']
def stop(self):
for connector in self._connectors:
connector.kill()
log.info('Connector stop...')
self.clear()
