class NetworkConnector:
"""Class to define an accessing to the around networks ability of tracking system.
This class provides necessary initiations and functions named :func:`t_system.audition.Audition.listen_async`
as the loop for asynchronous collecting audio data to the vision ability, named :func:`t_system.audition.Audition.listen_sync`
for the synchronous collecting audio data to the vision ability and named :func:`t_system.audition.Audition.start_recording`
as entry point from vision ability for starting recording processes.
"""
def __init__(self, args):
"""Initialization method of :class:`t_system.accession.NetworkManager` class.
Args:
args: Command-line arguments.
"""
self.folder = dot_t_system_dir + "/network"
if not os.path.exists(self.folder):
os.mkdir(self.folder)
self.table = DBFetcher(self.folder, "db", "login").fetch()
self.wpa_supplicant = WpaSupplicant(args)
self.wlan = args["wlan"]
self.current_cells = []
self.current_available_networks = []
self.known_networks = []
set_local_ip_address(args["wlan"], args["static_ip"])
self.scan()
self.set_available_networks()
self.refresh_known_networks()
def scan(self, wlan=None):
"""The high-level method to scan around for searching available networks.
Args:
wlan: wi-fi interface that will be used to create hotSpot.
"""
if wlan:
self.wlan = wlan
self.current_cells = list(Cell.all(self.wlan))
def set_available_networks(self):
"""The low-level method to setting available networks with their ssid and passwords.
"""
self.current_available_networks.clear()
for cell in self.current_cells:
network = {"ssid": cell.ssid}
self.current_available_networks.append(network)
def add_network(self, ssid, password):
"""The high-level method to set network parameter for reaching it.
Args:
ssid: The name of the surrounding access point.
password: The password of the surrounding access point.
"""
admin_id = check_secret_root_entry(ssid, password)
if admin_id:
return True, admin_id
status = False
for network in self.current_available_networks:
if ssid == network["ssid"]:
self.db_upsert(ssid, password)
self.refresh_known_networks()
status = True
return status, admin_id
def delete_network(self, ssid):
"""The high-level method to set network parameter for reaching it.
Args:
ssid: The name of the surrounding access point.
"""
self.table.remove((Query().ssid == ssid))
@dispatch()
def connect(self):
"""The high-level method to try to connect to one of the available networks using wpa_supplicant.conf file that is restarted by subprocess.
"""
result = False
self.wpa_supplicant.restart_ws_service()
time.sleep(5)
if self.is_network_online():
print("Network connection established!")
# result = True
return result
@dispatch(str, str)
def connect(self, ssid, password):
"""The high-level method to try connect to one of available networks with using `wifi` library.
Args:
ssid (str): The name of the surrounding access point.
password (str): The password of the surrounding access point.
"""
result = False
for cell in self.current_cells:
if cell.ssid == ssid:
try:
scheme = Scheme.for_cell(self.wlan, ssid, cell, password)
scheme.activate()
result = True
except Exception as e:
print(e)
return result
def try_creating_connection(self):
"""The high-level method to try connect to one of available networks.
"""
for network in self.known_networks:
if self.connect(network["ssid"], network["password"]):
return True
return False
def db_upsert(self, ssid, password, force_insert=False):
"""Function to insert(or update) the position to the database.
Args:
ssid: The name of the surrounding access point.
password: The password of the surrounding access point.
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.table.search((Query().ssid == ssid)):
if force_insert:
# self.already_exist = False
self.table.update({'password': password, 'wlan': self.wlan}, Query().ssid == ssid)
else:
# self.already_exist = True
return "Already Exist"
else:
self.table.insert({
'wlan': self.wlan,
'ssid': ssid,
'password': password
}) # insert the given data
return ""
def refresh_known_networks(self):
"""The low-level method to refreshing known networks from the database (and creating objects for them.)
"""
self.known_networks.clear()
self.wpa_supplicant.create_wsc()
for login in self.table.all():
network = {"ssid": login["ssid"], "password": login["password"], "wlan": login["wlan"]}
self.known_networks.append(network)
self.wpa_supplicant.add_network_to_wsc(login["ssid"], login["password"])
@staticmethod
def is_network_online():
"""The top-level method to check the internet access of the current network connection via sending request to Google.
Returns:
bool: status.
"""
url = 'http://www.google.com/'
timeout = 5
try:
_ = requests.get(url, timeout=timeout)
return True
except requests.ConnectionError:
# print("Internet connection could not be established.")
pass
return False
class ArmModeler:
"""Class to define the D-H matrix modeler of T_System arm .
This class provides necessary initiations and a function named :func:`t_system.motion.arm.ArmModeler.create`
for the create D-H model of the given arm.
"""
def __init__(self):
"""Initialization method of :class:`t_system.motion.arm.modelisation.ArmModeler` class.
"""
self.name = None
self.config_file = f'{T_SYSTEM_PATH}/motion/arm/config.json'
self.db = DBFetcher(f'{T_SYSTEM_PATH}/motion/arm', "model").fetch()
self.joint_count = 0
self.alpha = None
self.a = None
self.q = None
self.d = None
self.dh_params = {}
self.tf_matrices = []
self.jacobian_matrix = None
def create(self, arm_name):
"""Method to create D-H model of given arm.
Args:
arm_name (str): A robotic arm name in to the config.json file.
"""
try:
with open(self.config_file) as conf_file:
arm_configs = json.load(conf_file)[
arm_name] # config file returns the arms.
except KeyError:
raise Exception(f'{arm_name} is not exit in configuration file.')
joint_configs = arm_configs["joints"]
self.joint_count = len(joint_configs)
self.__prepare_dh_params()
self.__set_dh_params(joint_configs)
self.__calc_jacobian_matrix()
self.__db_upsert(force_insert=True)
def get(self, arm_name=None):
"""Method to create D-H model of given arm.
Args:
arm_name (str): A robotic arm name in to the config.json file.
"""
if arm_name:
model = self.db.search((Query().name == arm_name))
if model:
return {
"alpha":
loads(model[0]["alpha"].encode("raw_unicode_escape")),
"a":
loads(model[0]["a"].encode("raw_unicode_escape")),
"q":
loads(model[0]["q"].encode("raw_unicode_escape")),
"d":
loads(model[0]["d"].encode("raw_unicode_escape")),
"dh_params":
loads(model[0]["dh_params"].encode("raw_unicode_escape")),
"transform_matrices":
loads(model[0]["transform_matrices"].encode(
"raw_unicode_escape")),
"jacobian_matrix":
loads(model[0]["jacobian_matrix"].encode(
"raw_unicode_escape"))
}
return None
else:
arms = []
for model in self.db.all():
arms.append({
"alpha":
loads(model[0]["alpha"].encode("raw_unicode_escape")),
"a":
loads(model[0]["a"].encode("raw_unicode_escape")),
"q":
loads(model[0]["q"].encode("raw_unicode_escape")),
"d":
loads(model[0]["d"].encode("raw_unicode_escape")),
"dh_params":
loads(model[0]["dh_params"].encode("raw_unicode_escape")),
"transform_matrices":
loads(model[0]["transform_matrices"].encode(
"raw_unicode_escape")),
"jacobian_matrix":
loads(model[0]["jacobian_matrix"].encode(
"raw_unicode_escape"))
})
return arms
def show(self, arm_name=None):
"""Method to show model by given name parameter. If there is no name, print all arms models to the screen.
Args:
arm_name (str): A robotic arm name in to the config.json file.
"""
if arm_name:
model = self.get(arm_name)
if model:
for key, value in model.items():
print(f'{key}: \n{value}')
return True
logger.error(f'There is no model for arm {arm_name}')
return False
else:
models = self.get()
if models:
for model in models:
for key, value in model.items():
print(f'{key}: \n{value}')
return True
logger.error(f'There is no any created model.')
return False
def __prepare_dh_params(self):
"""Method to preparing D-H parameters of Arm.
"""
self.alpha = symbols('alpha0:' + str(self.joint_count))
self.a = symbols('a0:' + str(self.joint_count))
self.q = symbols('q1:' + str(self.joint_count + 1))
self.d = symbols('d1:' + str(self.joint_count + 1))
def __set_dh_params(self, joints):
"""Method to setting joint's D-H parameters.
Args:
joints (list): The arm's joints list for preparing parameters of Denavit-Hartenberg chart.
"""
self.dh_params = {}
for i in range(len(joints)):
self.dh_params[self.alpha[i]] = joints[i]["alpha"]
self.dh_params[self.a[i]] = joints[i]["a"]
if joints[i]["structure"] == "revolute":
self.dh_params[self.q[i]] = self.q[i]
self.dh_params[self.d[i]] = joints[i]["init_d"]
elif joints[i]["structure"] == "prismatic":
self.dh_params[self.q[i]] = joints[i]["init_q"]
self.dh_params[self.d[i]] = self.d[i]
elif joints[i]["structure"] == "constant":
self.dh_params[self.q[i]] = joints[i]["init_q"]
self.dh_params[self.d[i]] = joints[i]["init_d"]
self.__set_transform_matrices()
def show_dh_params(self):
"""Method to getting D-H parameters of joints of Arm as string message.
"""
print(f'DH Parameters are: {self.dh_params}')
def __set_transform_matrices(self):
"""Method to setting D-H transform matrices.
"""
self.tf_matrices = []
transform_matrix = eye(
4) # creates a unit matrix via passing argument.
for i in range(self.joint_count):
transform_matrix = transform_matrix * self.__create_tf_matrix(
self.alpha[i], self.a[i], self.d[i], self.q[i]).subs(
self.dh_params)
self.tf_matrices.append(transform_matrix)
def show_transform_matrices(self):
"""Method to getting D-H parameters of joints of Arm as string message.
"""
print(f'Transform Matrices are: {self.tf_matrices}')
@staticmethod
def __create_tf_matrix(alpha, a, d, q):
"""Method to calculate transform matrix of Denavit-Hartenberg Method.
Args:
alpha: The twist angle. Axis angle between consecutive two axes.
a: The limb length between consecutive two axis.
d: link offset. The displacement along the same axis.
q: The rotation theta angle about the joint axis.
Returns:
object: The Denavit-Hartenberg transform matrix object.
"""
tf_matrix = Matrix([[cos(q), -sin(q), 0., a],
[
sin(q) * cos(alpha),
cos(q) * cos(alpha), -sin(alpha),
-sin(alpha) * d
],
[
sin(q) * sin(alpha),
cos(q) * sin(alpha),
cos(alpha),
cos(alpha) * d
], [0., 0., 0., 1.]])
return tf_matrix
def __calc_jacobian_matrix(self):
"""Method to calculate jacobian matrix of Arm's General Denavit-Hartenberg Transform Matrix.
"""
tf_matrix_first_to_last = self.tf_matrices[-1]
self.jacobian_matrix = [
diff(tf_matrix_first_to_last[:3, -1], self.q[i]).reshape(1, 3)
for i in range(len(self.q))
]
self.jacobian_matrix = Matrix(
self.jacobian_matrix).T # .T returns the transpose of matrix.
def __db_upsert(self, force_insert=False):
"""Function to insert(or update) the record to the database.
Args:
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.db.search((Query().name == self.name)):
if force_insert:
self.db.update(
{
'name':
self.name,
'alpha':
dumps(self.alpha).decode("raw_unicode_escape"),
'a':
dumps(self.a).decode("raw_unicode_escape"),
'q':
dumps(self.q).decode("raw_unicode_escape"),
'd':
dumps(self.d).decode("raw_unicode_escape"),
'dh_params':
dumps(self.dh_params).decode("raw_unicode_escape"),
'transform_matrices':
dumps(self.tf_matrices).decode("raw_unicode_escape"),
'jacobian_matrix':
dumps(
self.jacobian_matrix).decode("raw_unicode_escape")
},
Query().name == self.name)
else:
return "Already Exist"
else:
self.db.insert({
'name':
self.name,
'alpha':
dumps(self.alpha).decode("raw_unicode_escape"),
'a':
dumps(self.a).decode("raw_unicode_escape"),
'q':
dumps(self.q).decode("raw_unicode_escape"),
'd':
dumps(self.d).decode("raw_unicode_escape"),
'dh_params':
dumps(self.dh_params).decode("raw_unicode_escape"),
'transform_matrices':
dumps(self.tf_matrices).decode("raw_unicode_escape"),
'jacobian_matrix':
dumps(self.jacobian_matrix).decode("raw_unicode_escape")
}) # insert the given data
return ""
class StreamWebSite:
"""Class to define website that will used as live stream platform.
This class provides necessary initiations and functions named :func:`t_system.online_stream.StreamWebSite.upsert_stream_key`
as the new account adding point and existing account updating point.
"""
def __init__(self,
name,
url,
server,
to_be_used=False,
stream_ids=None,
active_stream_id=None,
id=None):
"""Initialization method of :class:`t_system.online_stream.OnlineStream` class.
Args:
name: Name of the WebSite. youtube, facebook etc.
url: Website's page URL.
server: Website's Live stream server RTMP URL.
to_be_used (bool): To be used flag that specify usage status of website on live stream.
stream_ids: hexadecimal stream keys of the website.
active_stream_id: hexadecimal stream key that use in current stream of the website.
id: Unique ID of the website.
"""
self.id = id
if not id:
self.id = str(uuid.uuid1())
self.stream_ids = stream_ids
if not stream_ids:
self.stream_ids = []
self.active_stream_id = active_stream_id
if not active_stream_id:
self.active_stream_id = {}
self.name = name
self.url = url
self.server = server
self.to_be_used = to_be_used
self.streaming_folder = f'{dot_t_system_dir}/streaming'
self.keys_folder = f'{self.streaming_folder}/keys'
self.parent_folder = f'{self.streaming_folder}/websites'
self.folder = f'{self.parent_folder}/{self.name}'
self.key_file = f'{self.keys_folder}/{self.name}.key'
self.__check_folders()
self.table = DBFetcher(self.streaming_folder, "db", "websites").fetch()
self.__db_upsert()
def set_usage_stat(self, to_be_used):
"""Method to set website as to be used or not to be used.
Args:
to_be_used (bool): To be used flag that specify usage status of website on live stream.
"""
self.to_be_used = to_be_used
self.__db_upsert(force_insert=True)
def activate_stream_key(self, account_name):
"""Method to set given stream key for using on the current live stream for the website.
Args:
account_name (str): Name of the website's account.
"""
for stream_id in self.stream_ids:
if stream_id["is_active"]:
stream_id["is_active"] = False
break
for stream_id in self.stream_ids:
if stream_id["account_name"] == account_name:
with open(self.key_file, "w+") as key_file:
key_file.write(stream_id["key"])
key_file.close()
stream_id["is_active"] = True
self.active_stream_id = stream_id
self.__db_upsert(force_insert=True)
return True
return False
def upsert_stream_key(self, account_name, key):
"""Method to insert(or update) new stream key and its account name.
Args:
account_name (str): Name of the website's account.
key (str): Hexadecimal live stream key of the websites's account.
"""
is_update = False
for stream_id in self.stream_ids:
if account_name == stream_id["account_name"]:
stream_id["key"] = key
is_update = True
break
if not is_update:
self.stream_ids.append({
"account_name": account_name,
"key": key,
"key_file": f'{self.folder}/{account_name}.key',
"is_active": False
})
self.__db_upsert(force_insert=True)
self.__set_key_files(account_name)
return True
def remove_stream_key(self, account_name):
"""Method to remove existing stream key and its account name.
Args:
account_name (str): Name of the website's account.
"""
for stream_id in self.stream_ids:
if account_name == stream_id["account_name"]:
self.stream_ids.remove(
stream_id) # for removing object from list
self.__db_upsert(force_insert=True)
os.remove(stream_id["key_file"])
return True
return False
def update_self(self, url, server):
"""Method to update website itself.
"""
if url:
self.url = url
if server:
self.server = server
self.__db_upsert(force_insert=True)
def delete_self(self):
"""Method to delete website itself.
"""
self.table.remove((Query().id == self.id))
def __db_upsert(self, force_insert=False):
"""Function to insert(or update) the record to the database.
Args:
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.table.search((Query().id == self.id)):
if force_insert:
self.table.update(
{
'id': self.id,
'name': self.name,
'url': self.url,
'server': self.server,
'to_be_used': self.to_be_used,
'stream_ids': self.stream_ids,
'active_stream_id': self.active_stream_id
},
Query().id == self.id)
else:
return "Already Exist"
else:
self.table.insert({
'id': self.id,
'name': self.name,
'url': self.url,
'server': self.server,
'to_be_used': self.to_be_used,
'stream_ids': self.stream_ids,
'active_stream_id': self.active_stream_id
}) # insert the given data
return ""
def __set_key_files(self, account_name=None):
"""Method to crete `.key` files that keeps stream keys of the websites live stream entry.
Args:
account_name (str): Name of the website's account.
"""
result = False
stream_ids = []
if account_name:
for stream_id in self.stream_ids:
if account_name == stream_id["account_name"]:
stream_ids.append(stream_id)
result = True
break
else:
stream_ids = self.stream_ids
for stream_id in stream_ids:
result = True
with open(stream_id["key_file"], "w+") as key_file:
key_file.write(stream_id["key"])
key_file.close()
return result
def __check_folders(self):
"""Method to checking the necessary folders created before. If not created creates them.
"""
if not os.path.exists(self.parent_folder):
os.mkdir(self.parent_folder)
if not os.path.exists(self.keys_folder):
os.mkdir(self.keys_folder)
if not os.path.exists(self.folder):
os.mkdir(self.folder)
class NetworkConnector:
"""Class to define an accessing to the around networks ability of tracking system.
This class provides necessary initiations and functions named :func:`t_system.accession.NetworkConnector.connect`
for the connecting with wpa_supplicant.conf or wifi.Scheme.
"""
def __init__(self, args):
"""Initialization method of :class:`t_system.accession.NetworkConnector` class.
Args:
args: Command-line arguments.
"""
self.folder = f'{dot_t_system_dir}/network'
if not os.path.exists(self.folder):
os.mkdir(self.folder)
self.login_table = DBFetcher(self.folder, "db", "login").fetch()
self.status_table = DBFetcher(self.folder, "db", "status").fetch()
self.activity = None
self.__refresh_status()
self.wpa_supplicant = WpaSupplicant(args)
self.wlan = args["wlan"]
self.interface_ec = 0
self.current_cells = []
self.current_available_networks = []
self.known_networks = []
if args["static_ip"]:
set_local_ip_address(args["wlan"], args["static_ip"])
if self.activity:
self.scan()
self.__set_available_networks()
self.__refresh_known_networks()
def scan(self, wlan=None):
"""Method to scan around for searching available networks.
Args:
wlan: wi-fi interface that will be used to create hotSpot.
"""
if wlan:
self.wlan = wlan
try:
self.current_cells = list(Cell.all(self.wlan))
except exceptions.InterfaceError:
if self.interface_ec < 1:
logger.warning(
f'InterfaceError for {self.interface_ec} times...')
self.interface_ec += 1
restart_interface(self.wlan)
self.scan()
else:
logger.error(
exceptions.InterfaceError(f'Error can not resolved!'))
self.current_cells = []
def __set_available_networks(self):
"""Method to setting available networks with their ssid and passwords.
"""
self.current_available_networks.clear()
for cell in self.current_cells:
network = {"ssid": cell.ssid}
self.current_available_networks.append(network)
def add_network(self, ssid, password):
"""Method to set network parameter for reaching it.
Args:
ssid: The name of the surrounding access point.
password: The password of the surrounding access point.
"""
admin_id = check_secret_root_entry(ssid, password)
if admin_id:
return True, admin_id
status = False
for network in self.current_available_networks:
if ssid == network["ssid"]:
self.login_upsert(ssid, password)
self.__refresh_known_networks()
self.wpa_supplicant.add_network_to_wsc(ssid, password)
self.__restart_networking_service()
status = True
break
return status, admin_id
def delete_network(self, ssid):
"""Method to set network parameter for reaching it.
Args:
ssid: The name of the surrounding access point.
"""
self.login_table.remove((Query().ssid == ssid))
self.wpa_supplicant.create_wsc()
for login in self.login_table.all():
self.wpa_supplicant.add_network_to_wsc(login["ssid"],
login["password"])
@dispatch()
def connect(self):
"""Method to try to connect to one of the available networks using wpa_supplicant.conf file that is restarted by subprocess.
"""
if self.activity:
if self.are_networks_accessible():
self.wpa_supplicant.restart_ws_service()
time.sleep(5)
if self.is_connected_to_network():
logger.info("Connected to a network.")
return True
return False
@dispatch(str, str)
def connect(self, ssid, password):
"""Method to try connect to one of available networks with using `wifi` library.
Args:
ssid (str): The name of the surrounding access point.
password (str): The password of the surrounding access point.
"""
result = False
if self.activity:
for cell in self.current_cells:
if cell.ssid == ssid:
try:
scheme = Scheme.for_cell(self.wlan, ssid, cell,
password)
scheme.activate()
result = True
except Exception as e:
logger.error(e)
return result
def try_creating_connection(self):
"""Method to try connect to one of available networks.
"""
if self.activity:
for network in self.known_networks:
if self.connect(network["ssid"], network["password"]):
return True
return False
def login_upsert(self, ssid, password, force_insert=False):
"""Function to insert(or update) the connection info of new networks to the database.
Args:
ssid: The name of the surrounding access point.
password: The password of the surrounding access point.
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.login_table.search((Query().ssid == ssid)):
if force_insert:
# self.already_exist = False
self.login_table.update(
{
'password': password,
'wlan': self.wlan
},
Query().ssid == ssid)
else:
# self.already_exist = True
return "Already Exist"
else:
self.login_table.insert({
'wlan': self.wlan,
'ssid': ssid,
'password': password
}) # insert the given data
return ""
def status_upsert(self, activity, force_insert=False):
"""Function to insert(or update) the status of NetworkConnector to the database.
Args:
activity (bool): Activity flag of the NetworkConnector. If False, NetworkConnector not try connecting to surround networks.
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
status = self.status_table.all()
if status:
self.status_table.update({'activity': activity})
else:
self.status_table.insert({
'activity': activity,
}) # insert the given data
return ""
def __refresh_known_networks(self):
"""Method to refresh known networks from the database (and creating objects for them.)
"""
self.known_networks.clear()
for login in self.login_table.all():
network = {
"ssid": login["ssid"],
"password": login["password"],
"wlan": login["wlan"]
}
self.known_networks.append(network)
@staticmethod
def __restart_networking_service():
"""Method to to restart networking.service
"""
call("systemctl restart networking.service", shell=True)
def __refresh_status(self):
"""Method to refresh status from the database.
"""
status = self.status_table.all()
if status:
self.activity = status[0]["activity"]
else:
self.activity = True
self.status_upsert(self.activity)
def change_status(self, activity):
"""high-level method to change status of NetworkConnector via given parameters.
Args:
activity (bool): Activity flag of the NetworkConnector. If False, NetworkConnector not try connecting to surround networks.
Returns:
str: Response.
"""
self.status_upsert(activity)
self.__refresh_status()
@staticmethod
def is_network_online():
"""The top-level method to check the internet access of the current network connection via sending request to Google.
Returns:
bool: status.
"""
url = 'http://www.google.com/'
timeout = 5
try:
_ = requests.get(url, timeout=timeout)
return True
except requests.ConnectionError:
logger.info("No internet access!")
pass
return False
@staticmethod
def is_connected_to_network():
"""The top-level method to check network connection status using `hostname` command via subprocess.
Returns:
bool: status.
"""
wifi_ip = check_output(['hostname', '-I'])
if wifi_ip:
return True
return False
def are_networks_accessible(self):
"""The top-level method to check if T_System has the necessary information to connect to surrounding networks.
Returns:
bool: status.
"""
for known_network in self.known_networks:
if known_network["ssid"] in self.current_available_networks:
return True
return False
class DropBox:
"""Class to define a file synchronizer to an Dropbox account.
This class provides necessary initiations and functions named :func:`t_system.r_sync.DropBox.sync`
to provide synchronizing recorded videos with the Dropbox account.
"""
def __init__(self, to_be_used=False, accounts=None, active_account=None):
"""Initialization method of :class:`t_system.r_sync.DropBox` class.
Args:
to_be_used (bool): To be used flag that specify usage status of service on folder synchronization.
accounts: DropBox account owner name and account API key list.
active_account: hexadecimal stream key that use in current stream of the website.
"""
self.accounts = accounts
if not accounts:
self.accounts = []
self.active_account = active_account
if not active_account:
self.active_account = {}
self.name = "Dropbox"
self.to_be_used = to_be_used
self.dbx = None
self.folder = f'{dot_t_system_dir}/r_sync'
self.sync_sou_dir = f'{dot_t_system_dir}/records'
self.sync_tar_dir = "Media-from-T_System"
self.table = DBFetcher(self.folder, "db", "services").fetch()
self.__db_upsert()
def sync(self):
"""Method to synchronizing folder that keeps recorded videos with user's Dropbox account.
"""
if self.__prepare_sync():
for dn, dirs, files in os.walk(self.sync_sou_dir):
sub_folder = dn[len(self.sync_sou_dir):].strip(os.path.sep)
listing = self.list_folder(self.sync_tar_dir, sub_folder)
logger.info(f'Descending into {sub_folder}...')
# First do all the files.
for file in files:
file_path = os.path.join(dn, file)
if isinstance(file, bytes):
file = file.decode('utf-8')
n_name = unicodedata.normalize('NFC', file)
if file.startswith('.'):
logger.info(f'Skipping dot file: {file}')
elif file.startswith('@') or file.endswith('~'):
logger.info(f'Skipping temporary file: {file}')
elif file.endswith('.pyc') or file.endswith('.pyo'):
logger.info(f'Skipping generated file: {file}')
elif file.endswith('.json'):
logger.info(f'Skipping database file: {file}')
elif n_name in listing:
md = listing[n_name]
mtime = os.path.getmtime(file_path)
mtime_dt = datetime.datetime(*time.gmtime(mtime)[:6])
size = os.path.getsize(file_path)
if isinstance(
md, dropbox.files.FileMetadata
) and mtime_dt == md.client_modified and size == md.size:
logger.info(
f' {file} is already synced [stats match]')
else:
logger.info(
file,
f'{file} exists with different stats, downloading'
)
res = self.download(self.sync_tar_dir, sub_folder,
file)
with open(file_path) as f:
data = f.read()
if res == data:
logger.info(
f'{file} is already synced [content match]'
)
else:
logger.info(
f'{file} has changed since last sync')
self.upload(file_path,
self.sync_tar_dir,
sub_folder,
file,
overwrite=True)
else:
self.upload(file_path, self.sync_tar_dir, sub_folder,
file)
# Then choose which subdirectories to traverse.
keep = []
for directory in dirs:
if directory.startswith('.'):
logger.info(f'Skipping dot directory: {directory}')
elif directory.startswith('@') or directory.endswith('~'):
logger.info(
f'Skipping temporary directory: {directory}')
elif directory == '__pycache__':
logger.info(
f'Skipping generated directory: {directory}')
else:
logger.info(f'Keeping directory: {directory}')
keep.append(directory)
dirs[:] = keep
def __prepare_sync(self):
"""Method to prepare DropBoxer to folder synchronization.
"""
if self.activate_account:
self.dbx = dropbox.Dropbox(self.active_account["key"])
return True
return False
def list_folder(self, folder, sub_folder):
"""Method to list a folder.
Args:
folder (str): Top-folder that contains synchronized items.
sub_folder (str): Sub-folder that belongs to top-folder.
Returns:
a dict mapping unicode filenames to FileMetadata|FolderMetadata entries.
"""
path = '/%s/%s' % (folder, sub_folder.replace(os.path.sep, '/'))
while '//' in path:
path = path.replace('//', '/')
path = path.rstrip('/')
try:
res = self.dbx.files_list_folder(path)
except dropbox.exceptions.ApiError as err:
logger.error(
f'Folder listing failed for {path} -- assumed empty: {err}')
return {}
else:
rv = {}
for entry in res.entries:
rv[entry.name] = entry
return rv
def download(self, folder, sub_folder, file_name):
"""Method to download a file from Dropbox account.
Args:
folder (str): Top-folder that contains synchronized items.
sub_folder (str): Sub-folder that belongs to top-folder.
file_name (str): The name of the file that will be uploaded.
Returns:
The bytes of the file, or None if it doesn't exist.
"""
path = '/%s/%s/%s' % (folder, sub_folder.replace(os.path.sep,
'/'), file_name)
while '//' in path:
path = path.replace('//', '/')
try:
md, res = self.dbx.files_download(path)
except dropbox.exceptions.HttpError as err:
print('*** HTTP error', err)
return None
data = res.content
logger.info(f'{len(data)} bytes; md: {md}')
return data
def upload(self,
file_path,
folder,
sub_folder,
file_name,
overwrite=False):
"""Method to upload a file to Dropbox account.
Args:
file_path (str): The path of the file that will be uploaded.
folder (str): Top-folder that contains synchronized items.
sub_folder (str): Sub-folder that belongs to top-folder.
file_name (str): The name of the file that will be uploaded.
overwrite (bool): The overwriting flag of file that will be uploaded.
Returns:
The request response, or None in case of error.
"""
path = '/%s/%s/%s' % (folder, sub_folder.replace(os.path.sep,
'/'), file_name)
while '//' in path:
path = path.replace('//', '/')
mode = (dropbox.files.WriteMode.overwrite
if overwrite else dropbox.files.WriteMode.add)
mtime = os.path.getmtime(file_path)
with open(file_path, 'rb') as f:
data = f.read()
try:
res = self.dbx.files_upload(
data,
path,
mode,
client_modified=datetime.datetime(*time.gmtime(mtime)[:6]),
mute=True)
except dropbox.exceptions.ApiError as err:
logger.error(f'API error {err}')
return None
logger.info(f'uploaded as {res.name.encode("utf8")}')
return res
def set_usage_stat(self, to_be_used):
"""Method to set website as to be used or not to be used.
Args:
to_be_used (bool): To be used flag that specify usage status of service on folder synchronization.
"""
self.to_be_used = to_be_used
self.__db_upsert(force_insert=True)
def activate_account(self, account_name):
"""Method to set given API key of given account for using on the current folder synchronization.
Args:
account_name (str): Name of the Dropbox account.
"""
for account in self.accounts:
if account["is_active"]:
account["is_active"] = False
break
for account in self.accounts:
if account["name"] == account_name:
account["is_active"] = True
self.active_account = account
self.__db_upsert(force_insert=True)
return True
return False
def upsert_account(self, name, key):
"""Method to insert(or update) new API key and its account name for reaching the Dropbox.
Args:
name (str): Name of the Dropbox account.
key (str): API key of Dropbox account.
"""
is_update = False
for account in self.accounts:
if name == account["name"]:
account["key"] = key
is_update = True
break
if not is_update:
self.accounts.append({
"name": name,
"key": key,
"is_active": False
})
self.__db_upsert(force_insert=True)
return True
def remove_account(self, name):
"""Method to remove existing stream key and its account name.
Args:
name (str): Name of the Dropbox account.
"""
for account in self.accounts:
if name == account["name"]:
self.accounts.remove(account) # for removing object from list
self.__db_upsert(force_insert=True)
return True
return False
def delete_self(self):
"""Method to delete website itself.
"""
self.table.remove((Query().name == self.name))
def __db_upsert(self, force_insert=False):
"""Function to insert(or update) the record to the database.
Args:
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.table.search((Query().name == self.name)):
if force_insert:
self.table.update(
{
'info': {
'to_be_used': self.to_be_used,
'accounts': self.accounts,
'active_account': self.active_account
}
},
Query().name == self.name)
else:
return "Already Exist"
else:
self.table.insert({
'name': self.name,
'info': {
'to_be_used': self.to_be_used,
'accounts': self.accounts,
'active_account': self.active_account
}
}) # insert the given data
return ""
class Record:
"""Class to define records of t_systems vision.
This class provides necessary initiations and functions named :func:`t_system.recordation.Record.__db_upsert`
for saving records to the database safely.
"""
def __init__(self,
d_m_y,
h_m_s,
scope,
record_formats,
id=None,
name=None,
length=None):
"""Initialization method of :class:`t_system.recordation.Record` class.
Args:
d_m_y (str): Date that is day_mount_year format.
h_m_s (str): Date that is hour_minute_second format.
scope (str): The working type during recording.
record_formats (dict): Formats of the records for video, audio and merged.
id (str): The id of the record.
name (str): The name of the record.
length (str): The length of the record as m:s.
"""
self.id = id
if not id:
self.id = str(uuid.uuid1())
self.name = name
if not name:
self.name = h_m_s
self.date = d_m_y # table name at the same time
self.time = h_m_s
self.scope = scope
self.record_formats = record_formats
self.length = length
self.records_folder = f'{dot_t_system_dir}/records'
self.parent_folder = f'{self.records_folder}/{self.date}'
self.folder = f'{self.parent_folder}/{self.time}'
self.video_file = f'{self.folder}/{self.time}.{self.record_formats["video"]}'
self.audio_file = f'{self.folder}/{self.time}.{self.record_formats["audio"]}'
self.merged_file = f'{self.folder}/{self.time}.{self.record_formats["merged"]}'
self.db = DBFetcher(self.records_folder, "db").fetch()
self.__check_folders()
if length is None:
self.length = self.__calc_length()
self.__db_upsert()
def __db_upsert(self, force_insert=False):
"""Function to insert(or update) the record to the database.
Args:
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.db.search((Query().id == self.id)):
if force_insert:
# self.already_exist = False
self.db.update(
{
'id': self.id,
'name': self.name,
'time': self.time,
'date': self.date,
'scope': self.scope,
'record_formats': self.record_formats,
'length': self.length
},
Query().id == self.id)
else:
# self.already_exist = True
return "Already Exist"
else:
self.db.insert({
'id': self.id,
'name': self.name,
'time': self.time,
'date': self.date,
'scope': self.scope,
'record_formats': self.record_formats,
'length': self.length
}) # insert the given data
return ""
def update_name(self, name):
"""Method to updating self name via by given parameter.
Args:
name (str): The name of the record.
"""
self.name = name
self.__db_upsert(True)
def remove_self(self):
"""Method to remove face itself.
"""
rmtree(self.folder)
self.db.remove((Query().id == self.id))
def __calc_length(self):
"""Method to calculating length of record with using OpenCV.
"""
if os.path.exists(self.merged_file):
import cv2
cap = cv2.VideoCapture(self.merged_file)
fps = cap.get(
cv2.CAP_PROP_FPS) # OpenCV2 version 2 used "CV_CAP_PROP_FPS"
frame_count = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = frame_count / fps
minutes = int(duration / 60)
seconds = round(duration % 60)
length = f'{minutes}:{seconds}'
cap.release()
return length
return None
def __check_folders(self):
"""Method to checking the necessary folders created before. If not created creates them.
"""
if not os.path.exists(self.parent_folder):
os.mkdir(self.parent_folder)
if not os.path.exists(self.folder):
os.mkdir(self.folder)
class Shot:
"""Class to define shots of t_systems vision.
This class provides necessary initiations and functions named :func:`t_system.recordation.Shot.__db_upsert`
for saving shots to the database safely.
"""
def __init__(self, d_m_y, h_m_s, shot_format, id=None, name=None, size=None):
"""Initialization method of :class:`t_system.recordation.Record` class.
Args:
d_m_y (str): Date that is day_mount_year format.
h_m_s (str): Date that is hour_minute_second format.
shot_format (str): Format of the shot. (jpg, png etc.)
id (str): The id of the record.
name (str): The name of the record.
size (str): The length of the record as m:s.
"""
self.id = id
if not id:
self.id = str(uuid.uuid1())
self.name = name
if not name:
self.name = h_m_s
self.date = d_m_y # table name at the same time
self.time = h_m_s
self.shot_format = shot_format
self.size = size
self.records_folder = f'{dot_t_system_dir}/records'
self.parent_folder = f'{self.records_folder}/{self.date}'
self.folder = f'{self.parent_folder}/{self.time}'
self.shot_file = f'{self.folder}/{self.time}.{self.shot_format}'
self.table = DBFetcher(self.records_folder, "db", "shots").fetch()
self.__check_folders()
if size is None:
self.size = self.__calc_size()
self.__db_upsert()
def __db_upsert(self, force_insert=False):
"""Function to insert(or update) the record to the database.
Args:
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.table.search((Query().id == self.id)):
if force_insert:
# self.already_exist = False
self.table.update({'id': self.id, 'name': self.name, 'time': self.time, 'date': self.date, 'shot_format': self.shot_format, 'size': self.size}, Query().id == self.id)
else:
# self.already_exist = True
return "Already Exist"
else:
self.table.insert({
'id': self.id,
'name': self.name,
'time': self.time,
'date': self.date,
'shot_format': self.shot_format,
'size': self.size
}) # insert the given data
return ""
def update_name(self, name):
"""Method to updating self name via by given parameter.
Args:
name (str): The name of the record.
"""
self.name = name
self.__db_upsert(True)
def remove_self(self):
"""Method to remove face itself.
"""
rmtree(self.folder)
self.table.remove((Query().id == self.id))
def __calc_size(self):
"""Method to calculating length of record with using OpenCV.
"""
if os.path.exists(self.shot_file):
return os.path.getsize(self.shot_file) / 1000 # in kB unit.
return None
def __check_folders(self):
"""Method to checking the necessary folders created before. If not created creates them.
"""
if not os.path.exists(self.parent_folder):
os.mkdir(self.parent_folder)
if not os.path.exists(self.folder):
os.mkdir(self.folder)
class Face:
"""Class to define a face that has dataset and encoding pickle file.
This class provides necessary initiations and a function named :func:`t_system.face_encoding.Face.create_dataset_from_base_sf_photos`
for the provide creating dataset images from the given photos list that is contains photos names and their bas64 encoded form.
"""
def __init__(self, name, id=None):
"""Initialization method of :class:`t_system.face_encoder.Face` class.
Args:
name (str): The name of the man who has face in dataset.
id (str): The id of the face.
"""
self.name = name
self.id = id
self.id = id
if not id:
self.id = str(uuid.uuid1())
self.recognition_folder = f'{dot_t_system_dir}/recognition'
self.dataset_folder = f'{self.recognition_folder}/dataset/{self.name}'
self.pickle_file = f'{self.recognition_folder}/encodings/{self.name}_encoding.pickle'
self.__check_folders()
self.table = DBFetcher(self.recognition_folder, "db", "faces").fetch()
self.image_names = []
self.refresh_image_names()
self.__db_upsert()
def copy_images_to(self, dest):
"""Method to copying image inside the dataset to the given destination folder.
Args:
dest (str): Destination folder to copying images those are inside the dataset.
"""
if not os.path.exists(dest):
os.mkdir(dest)
rmtree(dest)
src_files = os.listdir(self.dataset_folder)
for file_name in src_files:
full_file_name = os.path.join(self.dataset_folder, file_name)
if os.path.isfile(full_file_name):
copy(full_file_name, dest)
def __db_upsert(self, force_insert=False):
"""Function to insert(or update) the face to the database.
Args:
force_insert (bool): Force insert flag.
Returns:
str: Response.
"""
if self.table.search((Query().id == self.id)):
if force_insert:
# self.already_exist = False
self.table.update(
{
'name': self.name,
'image_names': self.image_names
},
Query().id == self.id)
else:
# self.already_exist = True
return "Already Exist"
else:
self.table.insert({
'id': self.id,
'name': self.name,
'image_names': self.image_names
}) # insert the given data
return ""
def refresh_image_names(self, use_db=False):
"""The top-level method to reload the image_names from given source flag.
Args:
use_db (bool): Refreshing source flag. False is for using directly by scanning dataset folder
"""
if use_db:
face = self.table.search((Query().id == self.id))[0]
self.image_names = face["image_names"]
else:
self.image_names = [
f for f in listdir(self.dataset_folder)
if isfile(join(self.dataset_folder, f))
]
def delete_images(self, image_names):
"""Method to deleting images via given image names.
Args:
image_names (list): The name list of the images those inside the dataset.
"""
for image_name in image_names:
self.image_names.remove(image_name)
os.remove(f'{self.dataset_folder}/{image_name}')
self.__db_upsert(force_insert=True)
def create_dataset_from_base_sf_photos(self, photos):
"""Method to creating image that will be dataset for recognizing person's face later from base64 encoded string photo data.
Args:
photos (list): The person's raw photo data list. Contains list of {"name": "photo_name", "base_sf": "Base64_encoded_data"}.
Returns:
str: dataset.
"""
for photo in photos:
with open(f'{self.dataset_folder}/{photo["name"]}',
"wb") as dataset_image:
try:
dataset_image.write(b64decode(photo["base_sf"]))
except binascii.Error:
raise Exception("no correct base64")
self.refresh_image_names(use_db=True)
self.__db_upsert(force_insert=True)
def create_dataset_from_file_storage_object(self, photos):
"""Method to creating image that will be dataset for recognizing person's face later from FileStorage object that is coming from HTML input form element via Flask.
Args:
photos (list): The FileStorage object list.
Returns:
str: dataset.
"""
for photo in photos:
photo.save(f'{self.dataset_folder}/{photo.filename}')
self.refresh_image_names()
self.__db_upsert(force_insert=True)
def __check_folders(self):
"""The low-level method to checking the necessary folders created before. If not created creates them.
"""
if not os.path.exists(self.recognition_folder):
os.mkdir(self.recognition_folder)
if not os.path.exists(self.dataset_folder):
os.mkdir(self.dataset_folder)
def remove_self(self):
"""Method to remove face itself.
"""
rmtree(self.dataset_folder)
os.remove(self.pickle_file)
self.table.remove((Query().id == self.id))