def _yaml_dump(self, data, path_file, **kwargs) -> None:
""" dump YAML file
:param data: the data to persist
:param path_file: the name and path of the file
:param default_flow_style: (optional) if to include the default YAML flow style
"""
module_name = 'yaml'
if HandlerFactory.check_module(module_name=module_name):
module = HandlerFactory.get_module(module_name=module_name)
else:
raise ModuleNotFoundError(
f"The required module {module_name} has not been installed. "
f"Please pip install the appropriate package in order to complete this action"
)
encoding = kwargs.pop('encoding', 'utf-8')
default_flow_style = kwargs.pop('default_flow_style', False)
with self._lock:
# make sure the dump is clean
try:
with closing(open(path_file, mode='w',
encoding=encoding)) as ymlfile:
module.safe_dump(data=data,
stream=ymlfile,
default_flow_style=default_flow_style,
**kwargs)
except IOError as e:
raise IOError(
f"The yaml file {path_file} failed to open with: {e}")
# check the file was created
return
def _yaml_load(path_file, **kwargs) -> dict:
""" loads the YAML file
:param path_file: the name and path of the file
:return: a dictionary
"""
module_name = 'yaml'
if HandlerFactory.check_module(module_name=module_name):
module = HandlerFactory.get_module(module_name=module_name)
else:
raise ModuleNotFoundError(
f"The required module {module_name} has not been installed. "
f"Please pip install the appropriate package in order to complete this action"
)
encoding = kwargs.pop('encoding', 'utf-8')
try:
with closing(open(path_file, mode='r',
encoding=encoding)) as ymlfile:
rtn_dict = module.safe_load(ymlfile)
except IOError as e:
raise IOError(
f"The yaml file {path_file} failed to open with: {e}")
if not isinstance(rtn_dict, dict) or not rtn_dict:
raise TypeError(
f"The yaml file {path_file} could not be loaded as a dict type"
)
return rtn_dict
def has_changed(self) -> bool:
""" returns the status of the change_flag indicating if the file has changed since last load or reset"""
if not self.exists():
return False
# maintain the change flag
_cc = self.connector_contract
if _cc.schema.startswith('http') or _cc.schema.startswith('git'):
if not isinstance(self.connector_contract, ConnectorContract):
raise ValueError(
"The Pandas Connector Contract has not been set")
module_name = 'requests'
_address = _cc.address.replace("git://", "https://")
if HandlerFactory.check_module(module_name=module_name):
module = HandlerFactory.get_module(module_name=module_name)
state = module.head(_address).headers.get('last-modified', 0)
else:
raise ModuleNotFoundError(
f"The required module {module_name} has not been installed. Please pip "
f"install the appropriate package in order to complete this action"
)
else:
state = os.stat(_cc.address).st_mtime_ns
if state != self._file_state:
self._changed_flag = True
self._file_state = state
return self._changed_flag
def _json_load(path_file: str, **kwargs) -> [dict, pd.DataFrame]:
""" loads a pickle file """
if path_file.startswith('http'):
module_name = 'requests'
if HandlerFactory.check_module(module_name=module_name):
module = HandlerFactory.get_module(module_name=module_name)
username = kwargs.get('username', None)
password = kwargs.get('password', None)
auth = (username, password) if username and password else None
r = module.get(path_file, auth=auth)
return r.json()
with closing(open(path_file, mode='r')) as f:
return json.load(f, **kwargs)
def exists(self) -> bool:
""" Returns True is the file exists """
if not isinstance(self.connector_contract, ConnectorContract):
raise ValueError("The Pandas Connector Contract has not been set")
_cc = self.connector_contract
if _cc.schema.startswith('http') or _cc.schema.startswith('git'):
module_name = 'requests'
_address = _cc.address.replace("git://", "https://")
if HandlerFactory.check_module(module_name=module_name):
module = HandlerFactory.get_module(module_name=module_name)
return module.get(_address).status_code == 200
raise ModuleNotFoundError(
f"The required module {module_name} has not been installed. "
f"Please pip install the appropriate package in order to complete this action"
)
if os.path.exists(_cc.address):
return True
return False
def _pickle_load(path_file: str, **kwargs) -> [dict, pd.DataFrame]:
""" loads a pickle file """
fix_imports = kwargs.pop('fix_imports', True)
encoding = kwargs.pop('encoding', 'ASCII')
errors = kwargs.pop('errors', 'strict')
if path_file.startswith('http'):
module_name = 'requests'
if HandlerFactory.check_module(module_name=module_name):
module = HandlerFactory.get_module(module_name=module_name)
username = kwargs.get('username', None)
password = kwargs.get('password', None)
auth = (username, password) if username and password else None
r = module.get(path_file, auth=auth)
return r.content
with closing(open(path_file, mode='rb')) as f:
return pickle.load(f,
fix_imports=fix_imports,
encoding=encoding,
errors=errors)