def __init__(self, config):
super(Data, self).__init__()
self.config = config
self.train_batch_index = 0
self.test_seq_index = 0
self.batch_size = config['batch_size']
self.vox_res_x = config['vox_res_x']
self.vox_res_y = config['vox_res_y']
self.train_names = config['train_names']
self.test_names = config['test_names']
self.queue_train = queue(3)
self.stop_queue = False
self.X_train_files, self.Y_train_files = self.load_X_Y_files_paths_all(
self.train_names, label='train')
self.X_test_files, self.Y_test_files = self.load_X_Y_files_paths_all(
self.test_names, label='test')
print('X_train_files:', len(self.X_train_files))
print('X_test_files:', len(self.X_test_files))
self.total_train_batch_num = int(
len(self.X_train_files) // self.batch_size)
self.total_test_seq_batch = int(
len(self.X_test_files) // self.batch_size)
def __init__(self, addr, timeout, tries):
super(MemcacheClient, self).__init__()
self.daemon = True
self.queue = queue()
self.addr = addr
self.timeout = timeout
self.tries = tries
self.errors = 0
self.client = memcache.Client([self.addr], socket_timeout=self.timeout)
def __init__(self, info, args):
# set up all the statsd stuff
if args.statsd and not re.match(REGEX_STATSD_HOST, args.statsd):
raise Exception("Invalid statsd host provided")
self.statsd_host = self.statsd_port = None
self.statsd_disabled = True
if args.statsd:
self.statsd_host, self.statsd_port = args.statsd.rsplit(":", 1)
self.statsd_port = int(self.statsd_port)
self.statsd_disabled = False
self.statsd_connection = statsd.Connection(host=self.statsd_host,
port=self.statsd_port, sample_rate=0.5,
disabled=self.statsd_disabled)
self.statsd = statsd.Client("hydra_broker", self.statsd_connection)
self.statsd_timers = {}
self.statsd_counter = self.statsd.get_client(class_=statsd.Counter)
# set up statsd timers
self.timestamp = self.time(self.timestamp)
self.add_worker = self.time(self.add_worker)
self.remove_worker = self.time(self.remove_worker)
self.assign_job = self.time(self.assign_job)
self.mark_hostname = self.time(self.mark_hostname)
# initialize in general
self.MESSAGE_HEARTBEAT = make_message(BYTE_HEARTBEAT)
self.MESSAGE_HELLO = make_message(BYTE_HELLO, info)
self.threads = []
self.info = info
self.info_small = umsgpack.packb(info)
self.listening = True
self.workers = {}
self.jobs = {}
self.job_queue = queue(2)
self.print_queue = queue()
self.hostname_queue = queue()
self.info_socket = context.socket(zmq.REP)
self.info_socket.bind(args.bind_address or "tcp://0.0.0.0:22202")
database_info = info["d"]
if args.db_override:
database_info += " host=%s" % args.db_override
self.database = database.Database(database_info)
self.info["h2"] = self.info["h"]*2
def __init__(self, args):
if args.statsd and not re.match(REGEX_STATSD_HOST, args.statsd):
raise Exception("Invalid statsd host provided")
self.statsd_host = self.statsd_port = None
self.statsd_disabled = True
if args.statsd:
self.statsd_host, self.statsd_port = args.statsd.rsplit(":", 1)
self.statsd_port = int(self.statsd_port)
self.statsd_disabled = False
self.statsd_connection = statsd.Connection(host=self.statsd_host,
port=self.statsd_port, sample_rate=0.5,
disabled=self.statsd_disabled)
self.statsd = statsd.Client("hydra_worker", self.statsd_connection)
self.statsd_timers = {}
self.statsd_counter = self.statsd.get_client(class_=statsd.Counter)
self.get_info = self.time(self.get_info)
utils.find_urls = self.time(utils.find_urls)
self.get_jobs = self.time(self.get_jobs)
self.done_jobs = self.time(self.done_jobs)
self.get_tasks = self.time(self.get_tasks)
self.args = args
self.threads = []
self.working = True
self.break_now = False
self.jobs = {}
self.job_lock = threading.Lock()
self.broker_lock = threading.Lock()
self.insert_queue = queue()
self.print_queue = queue()
self.fill_queue = queue()
self.robots = RobotsCache()
self.worker_id = args.worker_id or uuid.uuid1().hex
self.socket = context.socket(zmq.REQ)
self.socket.connect(args.broker_address)
self.get_info()
self.headers = {"Accept-encoding": "gzip", "User-agent": self.info["u"]}
if args.db_override:
self.info["d"] += " host=%s" % args.db_override
self.database = database.Database(self.info["d"])
def __init__(self, bufsize=2048):
super(Monitor, self).__init__()
self.sock = socket.socket()
self.sock.connect((HOST, PORT))
# network header (contains: magic number, sequence number,
# version number, format number, source ID)
hdata = self.sock.recv(20, socket.MSG_WAITALL) # header of aer stream
self.hdata = struct.unpack('llbbh', hdata)
print('opened connection:', self.hdata)
self.q = mp.queue(bufsize)
self.maxsize = self.q._maxsize
self.qsize = 0
self.exit = mp.Event()
#self.daemon = True
self.start()
def __init__(self, parent=None):
QThread.__init__(self, parent)
self._is_rec_btn_pressed = False
self.q = queue()
self.robo = {}
SKARAB = 'skarab02030E-01'
f = casperfpga.CasperFpga(SKARAB)
# this will 'leak' sockets and you will run out of file descriptors
for loop in range(LOOP_LIMIT):
print('Loop {}'.format(loop))
sys.stdout.flush()
f.upload_to_ram_and_program(FILE)
# f.get_system_information(FILE)
def check_fpga_thing(result_queue, fpga):
fpga.upload_to_ram_and_program(FILE)
result_queue.put(True)
# this won't
results = queue()
for loop in range(LOOP_LIMIT):
print('Loop {}'.format(loop))
sys.stdout.flush()
p = Process(target=check_fpga_thing, args=(results, f))
p.start()
p.join(timeout=30)
result = results.get()
print(result)
sys.stdout.flush()
# end
class MelsecplcWorker(BaseWorker):
melsecPlc_Setting = settings['manager']['workers']['melsecplc']
gateway_id = None
device_name = None
if 'topic_subscription' in melsecPlc_Setting:
_, _, gateway_id, device_name, cmd_type, cmd = melsecPlc_Setting[
'topic_subscription'].split('/')
# Default values
devices = {}
# Payload that should be send when device is available
available_payload = 'home' # type: str
# Payload that should be send when device is unavailable
unavailable_payload = 'not_home' # type: str
# After what time (in seconds) we should inform that device is available (default: 0 seconds)
available_timeout = 0 # type: float
# After what time (in seconds) we should inform that device is unavailable (default: 60 seconds)
unavailable_timeout = 60 # type: float
scan_timeout = 10. # type: float
scan_passive = True # type: str or bool
start_to_collect = False #Global variable to control start to collect or not
flag_started = False #Flag to control BleDeviceStatus object creation
count = 0
# Defined Puhlish MQTT Topic
ReadData_Topic = "/IEW/{gateway}/{device}/ReplyData"
HeartBeat_Topic = "/IEW/{gateway}/{device}/Status/HeartBeat"
Alarm_Topic = "/IEW/{gateway}/{device}/Status/Alarm"
ReadData_job_id = '{}_interval_job'.format("ReadData")
Version = "1.0"
Status = "Init"
Job_queue = queue(maxsize=10)
_LOGGER.info("Version = " + Version)
_LOGGER.info("MelsecPLC Class Initial Funished")
_LOGGER.info("Setting Gateway_ID = " + gateway_id + ", Device_ID = " +
device_name)
_LOGGER.info("ReadData_Topic = " + ReadData_Topic)
_LOGGER.info("HeartBeat_Topic = " + HeartBeat_Topic)
def __init__(self, command_timeout, **kwargs):
super(MelsecplcWorker, self).__init__(command_timeout, **kwargs)
self._scheduler = BackgroundScheduler(timezone=utc)
self._scheduler.add_job(self.Read_PLC_Data,
'interval',
seconds=10,
id=self.ReadData_job_id)
self._scheduler.start()
# Apschedule 不要紀錄 Logger
logging.getLogger('apscheduler.executors.default').propagate = False
self.Status = "Init"
_LOGGER.info("MelsecplcWorker --> starts = " + self.Status)
self.report_alarm('0000', 'Info', 'MelsecplcWorker is Initial')
def run(self, mqtt):
while True:
time.sleep(1)
if self.Job_queue.empty() == True:
continue
else:
try:
SendOutMQTT = self.Job_queue.get()
mqtt.publish(SendOutMQTT)
except Exception as ee:
logger.log_exception(
_LOGGER, 'MelsecplcWorker --> Publish Error Msg : %s',
str(ee))
def create_devices(self):
if not self.start_to_collect:
self.last_status = []
for device_name, dev_info in self.devices.items():
_LOGGER.info(
"MelsecplcWorker --> create_devices : device_name = " +
device_name)
for d in dev_info:
_LOGGER.info(
"MelsecplcWorker --> create_devices : ip_addr = " +
d['IP_ADDR'])
_LOGGER.info(
"MelsecplcWorker --> create_devices : port_id = " +
d['PORT_ID'])
self.last_status = [
PLCDevice(self, d['IP_ADDR'], d['PORT_ID'],
device_name)
]
self.start_to_collect = True
def set_stop_flag(self):
self.start_to_collect = False
self.flag_started = False
def Read_PLC_Data(self):
if self.start_to_collect and self.flag_started:
self.count += 1
if self.count > 65535:
self.count = 1
_LOGGER.debug("MelsecplcWorker --> ReadPLC Data enters count = " +
str(self.count))
sendout_topic = self.ReadData_Topic.replace(
"{gateway}", self.gateway_id).replace("{device}",
self.device_name)
# 成功讀取狀態改為Run
try:
for status in self.last_status:
json_msg = status.read_data(self.addr_array)
status.set_status(status is not None)
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
_LOGGER.info("MelsecplcWorker --> Report Read PLC Data: " +
json_msg)
self.Status = "Run"
except Exception as ee:
logger.log_exception(
_LOGGER,
'MelsecplcWorker --> Read Data Excepting. Msg : %s',
str(ee))
self.Status = "Down"
# Read Data Error Send Error to MQTT
else:
_LOGGER.info(
"MelsecplcWorker --> status_update: Waiting for Collect Command!"
)
def read_payload_cmd_start(self, device_name, payload, topic):
try:
cmd_start = {}
cmd_start = json.loads(payload)
_LOGGER.info(
"MelsecplcWorker --> read_payload_cmd_start: payload = " +
payload)
_LOGGER.info(
"MelsecplcWorker --> read_payload_cmd_start: cmd_start = " +
str(cmd_start))
_LOGGER.info(
"MelsecplcWorker --> read_payload_cmd_start: cmd_start['Device_Info] = "
+ str(cmd_start['Device_Info']))
sendout_topic = topic + "/Ack"
if cmd_start['Cmd_Type'] == "Start":
connect_status = 'OK'
self.devices.update({device_name: cmd_start['Device_Info']})
# 連線設定完成
if not self.start_to_collect:
self.create_devices()
for status in self.last_status:
if status.p_device == None:
connect_status = 'NG'
ret_json = {}
ret_json.update({'Cmd_Result': connect_status})
ret_json.update({'Trace_ID': cmd_start['Trace_ID']})
json_msg = json.dumps(ret_json)
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
if connect_status == 'NG':
self.Status = "Down"
self.report_alarm('1001', 'ERROR', 'Connected PLC Faild')
else:
self.Status = "Ready"
self.report_alarm('0001', 'INFO',
'Connected PLC successed')
else:
ret_json = {}
ret_json.update({'Cmd_Result': "NG"})
ret_json.update({'Trace_ID': cmd_start['Trace_ID']})
json_msg = json.dumps(ret_json)
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
# 補送Alarm to MQTT
self.report_alarm('1002', 'ERROR', 'Cmd_Type not Start')
except Exception as ee:
logger.log_exception(_LOGGER, 'Cmd Start Exception Error Msg : %s',
str(ee))
self.Status = "Down"
ret_json = {}
ret_json.update({'Cmd_Result': "NG"})
ret_json.update({'Trace_ID': cmd_start['Trace_ID']})
json_msg = json.dumps(ret_json)
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
# 補送Alarm to MQTT
self.report_alarm(
'1003', 'ERROR',
'Cmd_Type Start happened exception error :' + str(ee))
def read_payload_cmd_readdata(self, device_name, payload, topic):
try:
cmd_read = {}
cmd_read = json.loads(payload)
_LOGGER.info(
"MelsecplcWorker --> read_payload_cmd_parameter: cmd_parameter = "
+ str(cmd_read))
sendout_topic = topic + "/Ack"
if cmd_read['Cmd_Type'] == "Collect":
self.addr_array = cmd_read['Address_Info']
interval = int(cmd_read['Report_Interval'])
self.flag_started = False
self._scheduler.pause()
self._scheduler.reschedule_job(job_id=self.ReadData_job_id,
trigger='interval',
seconds=interval)
self._scheduler.resume()
self.flag_started = True
ret_json = {}
ret_json.update({'Cmd_Result': "OK"})
ret_json.update({'Trace_ID': cmd_read['Trace_ID']})
json_msg = json.dumps(ret_json)
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
else:
ret_json = {}
ret_json.update({'Cmd_Result': "NG"})
ret_json.update({'Trace_ID': cmd_read['Trace_ID']})
json_msg = json.dumps(ret_json)
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
_LOGGER.error(
"MelsecplcWorker --> read_payload_cmd_NG tpye not Collect, cmd_read_type = "
+ str(cmd_read['Cmd_Type']))
# 補送Alarm to host
self.report_alarm('1004', 'ERROR', 'Type not Collect')
except Exception as ee:
logger.log_exception(_LOGGER, 'Cmd Start Exception Error Msg : %s',
str(ee))
ret_json = {}
ret_json.update({'Cmd_Result': "NG"})
ret_json.update({'Trace_ID': cmd_read['Trace_ID']})
json_msg = json.dumps(ret_json)
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
self.Status = "Down"
self.report_alarm(
'1005', 'ERROR',
'Cmd_Type Collect happened exception error :' + str(ee))
def read_payload_parameter_request(self, device_name, payload):
parameter_request = {}
parameter_request = json.loads(payload)
_LOGGER.info(
"MelsecplcWorker --> read_payload_parameter_request: parameter_request = "
+ parameter_request)
def cmd_stop(self, value):
if value == KILL_ME:
sys.exit("END")
elif value == STOP_COLLECT:
self._scheduler.pause()
for status in self.last_status:
self.set_stop_flag()
self.last_status = None
def status_update(self):
_LOGGER.info("MelsecplcWorker --> Heartbit Report")
now = datetime.datetime.now()
sendout_topic = self.HeartBeat_Topic.replace(
"{gateway}", self.gateway_id).replace("{device}", self.device_name)
HB_json = {}
HB_json.update({'Version': self.Version})
HB_json.update({'Status': self.Status})
HB_json.update({'HBDatetime': now.strftime("%Y%m%d%H%M%S%f")[:-3]})
json_msg = json.dumps(HB_json)
_LOGGER.debug("Heartbit Report : " + json_msg)
ret = []
messages = []
messages.append(MqttMessage(topic=sendout_topic, payload=json_msg))
ret += messages
return ret
def report_alarm(self, code, level, desc):
now = datetime.datetime.now()
report_alarm_Topic = self.Alarm_Topic.replace(
"{gateway}", self.gateway_id).replace("{device}", self.device_name)
Alarm_json = {}
Alarm_json.update({'AlarmCode': code})
Alarm_json.update({'AlarmLevel': level})
Alarm_json.update({'AlarmApp': 'WORKER'})
Alarm_json.update({'DateTime': now.strftime("%Y%m%d%H%M%S%f")[:-3]})
Alarm_json.update({'AlarmDesc': desc})
json_msg = json.dumps(Alarm_json)
self.Job_queue.put(
[MqttMessage(topic=report_alarm_Topic, payload=json_msg)])
_LOGGER.error("MelsecplcWorker --> Report Alarm to Host Msg : " +
str(json_msg))
def on_command(self, topic, value):
value = value.decode('utf-8')\
# 設定改在其他地方
_, _, gateway_id, device_name, cmd_type, cmd = topic.split('/')
if cmd_type == "Cmd":
_LOGGER.info("MelsecplcWorker --> on_command: topic = " + topic)
_LOGGER.info("MelsecplcWorker --> on_command: gateway_id = " +
gateway_id)
_LOGGER.info("MelsecplcWorker --> on_command: device_name = " +
device_name)
_LOGGER.info("MelsecplcWorker --> on_command: cmd_type = " +
cmd_type)
_LOGGER.info("MelsecplcWorker --> on_command: cmd = " + cmd)
if cmd == "Start":
self.read_payload_cmd_start(device_name, value, topic)
elif cmd == "ReadData":
self.read_payload_cmd_readdata(device_name, value, topic)
elif cmd == "Stop":
self.cmd_stop(value)
_LOGGER.error("MelsecplcWorker --> on_command = " + cmd +
"Stop Collect Data")
self.Status = "Idle"
elif cmd == "OTA":
cmd_OTA = json.loads(value)
if cmd_OTA["Cmd"] == "OTA":
OTA_json = {}
OTA_json.update({'Trace_ID': cmd_OTA["Trace_ID"]})
OTA_json.update({'Version': self.Version})
OTA_json.update({'Status': self.Status})
OTA_json.update(
{'Datetime': now.strftime("%Y%m%d%H%M%S%f")[:-3]})
OTA_json.update({'ProcrssID': os.getpid()})
json_msg = json.dumps(OTA_json)
sendout_topic = topic + "/Ack"
self.Job_queue.put(
[MqttMessage(topic=sendout_topic, payload=json_msg)])
time.sleep(5)
self.cmd_stop("kill")
elif cmd_type == "Parameter":
if cmd == "Request":
self.read_payload_parameter_request(device_name, value)