def main_without_kanban(opt: Options):
lprint("start main_without_kanban()")
# get cache kanban
conn = opt.get_conn()
num = opt.get_number()
#kanban: Kanban = conn.set_kanban(SERVICE_NAME, num)
kanban = conn.get_one_kanban(SERVICE_NAME, num)
device_name = os.environ["TRANSFER_DEVICE"]
######### main function #############
yesterday = date.today() - timedelta(days=1)
file_name = yesterday.strftime('%Y%m%d') + '.json'
backup_dir = f'/var/lib/aion/Data/{SERVICE_NAME}_{num}/output/'
backup_file = backup_dir + file_name
os.makedirs(backup_dir, exist_ok=True)
subprocess.run([
'mongoexport', '-h', 'mongo', '--db', 'AionCore', '--collection',
'kanban', '-o', backup_file, '--query',
r'{"finishAt": /^' + yesterday.strftime('%Y-%m-%d') + '/}'
])
# output after kanban
conn.output_kanban(
result=True,
connection_key="default",
file_list=[backup_file],
metadata={"file_name": file_name},
device_name=device_name,
)
def get_all_header_list_by_json(json_path):
header_list = {}
conf = read_config_json(json_path)
if conf is None:
return {}
for data in conf:
command = data.get("command")
dec_data_library = import_module(
"src.plc_data.decoder.jtekt_decoder_0x" + command)
decoder_class = dec_data_library.PlcData
if data.get("expire_time") is None or data.get("interval") is None:
lprint(
f"there is no expire_time or interval (command:{command})")
continue
headers = decoder_class.create_request(data)
if headers is None:
lprint(f"cant get header data (command:{command})")
continue
header_list[command] = {
"headers": headers,
"interval": data.get("interval"),
"expire_time": data.get("expire_time"),
"decoder": decoder_class,
}
return header_list
def main():
try:
server = grpc.server(futures.ThreadPoolExecutor(max_workers=5))
if CURRENT_DEVICE_NAME in ['tartarus', 'poseidon', 'lib']:
template_matcning_summary_pb2_grpc.add_TemplateMatchingSummaryServicer_to_server(
VehicleSummaryServer(),
server,
)
elif CURRENT_DEVICE_NAME in ['deneb', 'elpis', 'neo', 'moca']:
template_matcning_summary_pb2_grpc.add_TemplateMatchingSummaryServicer_to_server(
TriggerSummaryServer(),
server,
)
else:
raise TemplateMatchingSummaryServerError("Device Name " +
CURRENT_DEVICE_NAME +
" is wrong.")
server.add_insecure_port('[::]:' + str(SERVER_PORT))
server.start()
server.wait_for_termination()
except Exception as e:
lprint(e)
sys.exit(1)
def main(opt: Options):
# get cache kanban
conn = opt.get_conn()
num = opt.get_number()
#kanban = conn.get_one_kanban(SERVICE_NAME, num)
kanban: Kanban = conn.set_kanban(SERVICE_NAME, num)
data_path = f"/var/lib/aion/Data/{SERVICE_NAME}_{num}/data"
# open config file
config = None
with open(os.path.join(CONFIG_PATH, "config.json")) as f:
config = json.load(f)
result_status = True
JST = timezone(timedelta(hours=+9), 'JST')
backup_time = datetime.now(JST)
backup_dir = os.path.join(data_path, backup_time.strftime('%Y%m%d%H%M%S'))
backup_file = config.get('ftp-backup-file')
backup_save_path = {}
for bf in backup_file:
backup_save_path[bf] = os.path.join(backup_dir, bf)
host = config.get('ftp-server')
user = config.get('ftp-user')
passwd = config.get('ftp-passwd')
# make backup directory
for bsp in backup_save_path.values():
dirname = os.path.dirname(bsp)
lprint(f"make backup directory: {dirname}")
os.makedirs(dirname, exist_ok=True)
### don't mkdir backup_dir becaus dirs are defined in backup_file ###
try:
ftpc = FtpClient(host, user, passwd)
for bf, bsp in backup_save_path.items():
ftpc.get(bf, bsp)
except ftplib.all_errors as error:
lprint_exception(f"ftp connection failed : {error}")
result_status = False
# write backup history to db
try:
with RobotBackupDB() as db:
backup_state = 1 if result_status else 2 # 1:succeeded 2:failed
for bsp in backup_save_path.values():
db.set_backup_to_db(bsp,
backup_time.strftime('%Y-%m-%d %H:%M:%S'),
backup_state)
finally:
# output after kanban
conn.output_kanban(
result=True,
connection_key="key",
output_data_path=data_path,
process_number=num,
)
return
async def output_status_json(self, conn):
while True:
data_list, timestamp = await self.rcv_queue.get()
robot_data = data_list.to_json()
metadata_sets = {"RobotData": robot_data, "timestamp": timestamp}
await conn.output_kanban(result=True, metadata=metadata_sets)
lprint(f'[client] output kanban')
def main(opt: Options):
conn = opt.get_conn()
num = opt.get_number()
try:
for kanban in conn.get_kanban_itr(SERVICE_NAME, num):
metadata = kanban.get_metadata()
template_matching_summary = metadata['TemplateMatchingSummary']
if CURRENT_DEVICE_NAME in ["deneb", "elpis", "moca"]:
t = TemplateMatchingSummaryWatch("trigger")
t.print_template_matching(template_matching_summary)
elif CURRENT_DEVICE_NAME in ["lib", "neo", "poseidon", "tartarus"]:
t = TemplateMatchingSummaryWatch("vehicle")
t.print_template_matching(template_matching_summary)
if template_matching_summary['end']['status']:
templates = metadata["TemplateMatchingSetTemplates"]
if NEXT_DEVICE_NAME:
conn.output_kanban(
result=True,
connection_key="default",
metadata=templates,
device_name=NEXT_DEVICE_NAME,
)
lprint(
f"output_kanban to other device {NEXT_DEVICE_NAME} {templates}"
)
else:
lprint("next_device_name is none")
except Exception:
traceback.print_exc()
sys.exit(1)
def main_without_kanban(opt: Options):
lprint("start main_without_kanban()")
# get cache kanban
conn = opt.get_conn()
num = opt.get_number()
kanban: Kanban = conn.set_kanban(SERVICE_NAME, num)
######### main function #############
client = TemperatureClient(host='stream-usb-thermo-by-grpc-server-001-srv',
port=50051)
while True:
# temp = client.get_temperature()
temp = client.get()
# output after kanban
if temp is not None:
# image_list = temp.img.tolist()
# temp_list = temp.temps.tolist()
metadata = {
"img": temp.image,
"temp": temp.temperatures,
"timestamp": temp.timestamp,
}
# NOTE: can not send decode image because size is too big
# reciever need to decode image
conn.output_kanban(
result=True,
connection_key="default",
process_number=num,
metadata=metadata,
)
time.sleep(INTERVAL)
def work_ng(self):
"""
同一セッションにて、workが連続 x[s]見つからない場合を検知する
"""
works = self.get_work_detection_result()
if not works:
lprint("no work detection result")
return False
works.sort(key=lambda x: x['timestamp'])
total = None
for pre_work, next_work in zip(works, works[1:]):
if not pre_work.get('is_work') and not next_work.get('is_work'):
pre_t = datetime.strptime(pre_work.get('timestamp'),
"%Y%m%d%H%M%S%f")
next_t = datetime.strptime(next_work.get('timestamp'),
"%Y%m%d%H%M%S%f")
diff_t = next_t - pre_t
if total:
total += diff_t
else:
total = diff_t
if total and total >= timedelta(seconds=WORK_NG_TIME):
lprint(f"work was not found for {str(total)}")
return True
else:
total = None
return False
def main_with_kanban(opt: Options):
conn = opt.get_conn()
num = opt.get_number()
try:
for kanban in conn.get_kanban_itr(config.MICROSERVICE_NAME, num):
metadata = kanban.get_metadata()
# Check metadata
keys = [
'deviceName', 'projectName', 'microserviceName',
'projectCommitId', 'status', 'error'
]
for key in keys:
if key not in metadata:
raise RuntimeError(f"Not found '{key}' in metadadata.")
main.main(metadata['deviceName'], metadata['projectName'],
metadata['microserviceName'],
metadata['projectCommitId'], metadata['status'],
metadata['error'])
conn.output_kanban(metadata=metadata)
lprint(f'Output kanban {metadata}.')
except Exception as e:
lprint(str(e))
def main(opt: Options):
conn = opt.get_conn()
num = opt.get_number()
lprint("DISPLAY: ", os.environ.get('DISPLAY'))
try:
while True:
kanban = conn.get_one_kanban(SERVICE_NAME, num)
metadata = kanban.get_metadata()
screen_name = metadata.get('screen_name')
screenshot_id = metadata.get('screenshot_id')
if screen_name is None:
screen_name = "NO_NAME"
lprint(f"screen_name: {screen_name}")
lprint(f"screenshot_id : {screenshot_id}")
timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
screen = ScreenShotService()
if screen.check_new_id(screenshot_id):
screen.shot_and_save(SAVE_DIRECTORY,
f"{screen_name}_{timestamp}")
screen.write_result(screenshot_id)
else:
lprint("not new id")
except KanbanNotFoundError:
lprint("kanban not found finish")
sys.exit(0)
except Exception as e:
lprint_exception(e)
def _disable_pods(self, my_etcd: MyEtcd):
lprint("start disable_pods thread...")
while True:
try:
for pod_on_etcd, _ in my_etcd.get_alive_my_pods(
self.node_name):
_pod_on_etcd = json.loads(pod_on_etcd.decode("utf-8"))
pod_name_on_etcd = _pod_on_etcd["podName"]
is_pod = False
for pod_on_kube in MyPod(self.kube_client, self.node_name,
self.datetime_fmt):
if pod_name_on_etcd == pod_on_kube["podName"]:
is_pod = True
break
if is_pod is False:
my_etcd.disable_pod(_pod_on_etcd["deviceNameFk"],
pod_name_on_etcd, _pod_on_etcd,
self.datetime_fmt)
except Exception as e:
lprint(f"failed to disabled_pods. {e}")
finally:
sleep(self.disable_interval)
def main(opt: Options):
# get cache kanban
conn = opt.get_conn()
num = opt.get_number()
kanban = conn.get_one_kanban(SERVICE_NAME, num)
# get output data path
data_path = kanban.get_data_path()
# get metadata
metadata = kanban.get_metadata()
schedule_list = metadata.get('schedule_list')
######### main function #############
job_id = int(schedule_list['job_id'])
new_schedules = schedule_list['schedule']
microservice_name = ''
# get db conn
with JobSchedulerDB() as db:
microservice_name = db.get_microservice_name(job_id)
if microservice_name == '':
lprint(f'microservice is not set for the job: {job_id}')
return
cj = CronJob()
cm = ConfigMap()
# get old schedule from k8s-api
old_schedule_list = cj.getCurrentCronjobs(microservice_name)
# stop job schedule from cronjob
for old_schedule_name in old_schedule_list:
cronjob_name = f'{old_schedule_name}'
envoy_name = f'envoy-config-{old_schedule_name}'
cj.deleteCronJob(cronjob_name)
cm.deleteEnvoy(envoy_name)
sleep(10)
# set job schedule to cronjob
for row in new_schedules:
schedule_id = int(row['schedule_id'])
cron_date = row['cron_date']
cronjob_name = f'{microservice_name}-job{schedule_id:03}'
envoy_name = f'envoy-config-{microservice_name}-job{schedule_id:03}'
cm.createEnvoy(envoy_name)
cj.createCronJob(job_id, cronjob_name, microservice_name, cron_date,
schedule_id, envoy_name)
# output after kanban
conn.output_kanban(
result=True,
connection_key="default",
output_data_path=data_path,
process_number=num,
)
def __init__(self, dir_path):
lprint("watch dir ", dir_path)
if not os.path.isdir(dir_path):
lprint("Error: transcript data is None")
sys.exit(1)
self.dir_path = dir_path
self.search_jpg = NewerFileList("jpg", dir_path)
def remove_container(self, container):
try:
container_obj = self.get_container_by_name(container['name'])
if container_obj:
container_obj.remove()
lprint(f"{container_obj} removed")
except docker.errors.APIError as e:
lprint(f" {container_obj} can't remove contaier for running")
def _create_pv(self, name, path, storage, namespace):
body = self._get_pv_body(name, path, storage, namespace)
try:
return self.k8s_v1.create_persistent_volume(body)
except ApiException as e:
lprint(e)
raise RuntimeError(e)
def _create(self, name, container_port, node_port, namespace):
body = self._get_body(name, container_port, node_port, namespace)
try:
return self.k8s_v1.create_namespaced_service(namespace, body)
except ApiException as e:
lprint(e)
raise RuntimeError(e)
def get_session(self):
with SummrizeCollection(DB_NAME, DB_COLLECTION) as db:
session_id = db.find_one_latest_session_id()
if session_id is None:
lprint("not found session id")
return None
return session_id
def __init__(self, dir_path):
if not os.path.isdir(dir_path):
lprint("Error: transcript data is None")
sys.exit(1)
self.dir_path = dir_path
self.before_time = dt.now(JST)
self.search_jpg = NewerFileList("jpg", dir_path)
def apply(self, name):
try:
is_namespace = self._is(name)
if is_namespace is False:
return self._create(name)
except ApiException as e:
lprint(e)
raise RuntimeError(e)
def update_temperature(self, temperatures, image, timestamp):
try:
ret, enimg = cv2.imencode('.jpg', image)
self.image = base64.b64encode(enimg)
self.temperatures = base64.b64encode(temperatures)
self.timestamp = str(timestamp)
except Exception as e:
lprint("fail to encode data")
def _create_pvc(self, name, storage, namespace):
body = self._get_pvc_body(name, storage, namespace)
try:
return self.k8s_v1.create_namespaced_persistent_volume_claim(
namespace, body)
except ApiException as e:
lprint(e)
raise RuntimeError(e)
def write_result(self, _id):
is_success = 1 if self.save_is_success else 0
save_path = self.save_path if self.save_path else ""
with UsbSql(DATABASE) as db:
try:
db.write_screenshot_save_result(_id, save_path, is_success)
lprint(f"write result to mysql: {_id}, {save_path}")
except Exception as e:
lprint_exception(e)
lprint(f"{e} : fail to write result to mysql")
def get_focus_absolute(self):
command = ['v4l2-ctl', '-d', self.device_path, '-C', 'focus_absolute']
res = subprocess.run(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out = res.stdout.decode("utf8")
out = out[:-1]
focus_absolute = out.replace('focus_absolute: ', '')
lprint("current focus_absoulte: " + focus_absolute)
return int(focus_absolute)
def stop_and_remove_container_by_name(self, container_name):
container = self.get_container_by_name(container_name)
if container is not None:
container.stop()
container.remove()
lprint(f"success stop and remove container: {container_name}")
return True
else:
lprint(f"cant stop and remove container: {container_name}")
return False
def _create(self, name):
try:
ret = self.k8s_v1.create_namespace(
client.V1Namespace(api_version="v1",
kind="Namespace",
metadata=client.V1ObjectMeta(name=name)))
return ret
except ApiException as e:
lprint(e)
raise RuntimeError(e)
def _is(self, name):
try:
ret = self.k8s_v1.list_namespace()
for item in ret.items:
if item.metadata.name == name:
return True
return False
except ApiException as e:
lprint(e)
raise RuntimeError(e)
def main_without_kanban(opt: Options):
lprint("start main_without_kanban()")
# get cache kanban
conn = opt.get_conn()
num = opt.get_number()
kanban: Kanban = conn.set_kanban(SERVICE_NAME, num)
copy(
'./json/C_MongoBackup.json',
'/var/lib/aion/Data/direct-next-service_1/C_MongoBackup.json')
def __init__(self, host, user, passwd):
if user and passwd:
# login with ftp user
lprint(f"connect to ftp server: {host}:{user}")
self.ftp = ftplib.FTP(host, user, passwd, timeout=FTP_TIMEOUT)
else:
# login with anonymous
lprint(f"connect to ftp server: {host}")
self.ftp = ftplib.FTP(host=host, timeout=FTP_TIMEOUT)
self.ftp.cwd('MEMCARD1')
def _update(self, image_name, name, container_port, envs, volume_mounts, volumes, service_account_name,
prior_device_name, namespace):
body = self._get_body(image_name, name, container_port, envs, volume_mounts, volumes, service_account_name,
prior_device_name, namespace)
try:
return self.k8s_apps_v1.replace_namespaced_deployment(name, namespace, body)
except ApiException as e:
lprint(e)
raise RuntimeError(e)
def disable_pod(self, device_name, pod_name, my_pod, datetime_fmt):
before_key = "/Pod/" + device_name + "/" + self.alive_status_index + "/" + pod_name
self.client.delete(before_key)
# change status 0 -> 1
my_pod["status"] = self.dead_status_index
my_pod["updateAt"] = dt.now().strftime(datetime_fmt)
after_key = "/Pod/" + device_name + "/" + self.dead_status_index + "/" + pod_name
self.client.put(after_key, json.dumps(my_pod))
lprint(f"disabled status: {before_key} -> {after_key}")