def __init__(self):
self.config = Config()
dotenv_file = dotenv.find_dotenv()
if not dotenv_file:
with open("./.env", "w"):
pass
self.last_error_message = ""
def __init__(self):
if hasattr(self, '_hash_json_data'
) is False: # only set these values first time around
self.config = Config()
self.storage = Storage()
self._hash_json_data = {}
self.load()
def __init__(self, index_name='temp_index', time_field=None, id_key=None):
self.config = Config()
self.id_key = id_key
self.index_name = index_name
self.index_pattern_name = index_name
self.time_field = time_field
self.enabled = False
def setUp(self) -> None:
self.config = Config()
self.repotrs_path = os.path.join(self.config.hd2_location, "reports")
self.processed_path = os.path.join(self.config.hd2_location,
self.config.hd2_processed_location)
self.pre_processor = Pre_Processor()
self.loops = Loops()
def __init__(self):
self.config = Config()
self.storage = Storage()
self.folder = os.path.join(self.config.hd2_location, "status")
self.analysis_data = {}
self.id = 0
self.get_from_file()
def __init__(self, file_hash=None):
self.config = Config()
self.storage = Storage()
self.process_status = Status()
self.metadata_utils = Metadata_Utils()
self.path_hd1 = self.storage.hd1()
self.data = self.default_data()
self.file_hash = file_hash
def setUpClass(cls) -> None:
super().setUpClass()
cls.test_file = Test_Data().create_test_pdf(text=random_text(prefix="some random text: "))
cls.test_file_name = file_name(cls.test_file)
cls.config = Config()
#cls.temp_root = folder_create('/tmp/temp_root') # temp_folder()
#cls.config.set_root_folder(root_folder=cls.temp_root)
cls.meta_service = Metadata_Service()
cls.metadata = cls.meta_service.create_metadata(cls.test_file)
cls.analysis_json = Analysis_Json()
def __init__(self):
self.config = Config()
self.hd1_base_location = self.config.hd1_location
self.hd2_base_location = self.config.hd2_location
self.hd3_base_location = self.config.hd3_location
self.zip_folder = os.path.join(os.getcwd(),"zip_folder")
self.storage = Storage()
folder_delete_all(self.zip_folder)
create_folder(self.zip_folder)
def setUp(self) -> None:
self.config = Config()
Setup_Testing().configure_config(self.config)
self.host = self.config.kibana_host
self.port = self.config.kibana_port
self.kibana = Kibana(host=self.host, port=self.port).setup()
if self.kibana.enabled is False:
pytest.skip('Elastic server not available')
self.dashboard_name = 'temp_dashbobard'
self.dashboard = Dashboard(kibana=self.kibana,
dashboard_name=self.dashboard_name)
def __init__(self):
self.config = Config()
self.meta_service = Metadata_Service()
self.status = Status()
self.storage = Storage()
self.file_name = None # set in process() method
self.current_path = None
self.base_folder = None
self.dst_folder = None
self.dst_file_name = None
self.status = Status()
self.status.reset()
def setUp(self) -> None:
self.config = Config()
Setup_Testing().configure_config(self.config)
self.host = self.config.kibana_host
self.port = self.config.kibana_port
self.kibana = Kibana(host=self.host, port=self.port).setup()
if self.kibana.enabled is False:
pytest.skip('Elastic server not available')
self.pattern_name = 'temp_index_pattern'
self.index_pattern = Index_Pattern(kibana=self.kibana,
pattern_name=self.pattern_name)
def setUp(self) -> None:
self.config = Config()
Setup_Testing().configure_config(self.config)
self.host = self.config.kibana_host
self.port = self.config.kibana_port
self.index_name = 'temp_index'
self.kibana = Kibana(index_name=self.index_name,
host=self.host,
port=self.port)
self.kibana.setup()
if self.kibana.enabled is False:
pytest.skip('Kibana server not available')
def __init__(self, events_log, events_elastic, report_elastic,
analysis_elastic, meta_service):
self.meta_service = meta_service
self.events_log = events_log
self.events_elastic = events_elastic
self.storage = Storage()
self.config = Config()
self.status = Status()
self.hash_json = Hash_Json()
self.report_elastic = report_elastic
self.sdk_api_version = "Not available"
self.sdk_engine_version = "Not available"
self.analysis_json = Analysis_Json()
self.analysis_elastic = analysis_elastic
def __init__(self):
self.use_es = False
self.config = Config()
self.status = Status()
self.storage = Storage()
self.hash_json = Hash_Json()
self.events = Events_Log(self.config.hd2_status_location)
self.events_elastic = Events_Log_Elastic()
self.hash = None
self.report_elastic = Report_Elastic()
self.analysis_elastic = Analysis_Elastic()
self.report_elastic.setup()
self.analysis_elastic.setup()
create_folder(self.storage.hd2_processed())
create_folder(self.storage.hd2_not_processed())
class Server_Status:
def __init__(self):
self.config = Config()
def now(self):
data = {
"config": self.config.values(),
"date": datetime_now(),
"check_logging": self.check_logging()
}
return data
def check_logging(self):
#server_online = logging.elastic().server_online()
return {'server_online': 'not implemented'}
class Configure_Env:
def __init__(self):
self.config = Config()
dotenv_file = dotenv.find_dotenv()
if not dotenv_file:
with open("./.env", "w"):
pass
self.last_error_message = ""
def reset_last_error(self):
self.last_error_message = ""
def configure(self, hd1_path=None, hd2_path=None, hd3_path=None):
self.reset_last_error()
try:
dotenv_file = dotenv.find_dotenv()
if hd1_path:
if path.exists(hd1_path):
environ['HD1_LOCATION'] = hd1_path
dotenv.set_key(dotenv_file, "HD1_LOCATION",
environ["HD1_LOCATION"])
else:
self.last_error_message = f"hd1_path did not exist: {hd1_path}"
log_error(message=f"hd1_path did not exist",
data={"path": hd1_path})
return -1
if hd2_path:
if not path.exists(hd2_path):
folder_create(hd2_path)
folder_create(path_combine(hd2_path,
DEFAULT_HD2_DATA_NAME))
folder_create(
path_combine(hd2_path, DEFAULT_HD2_STATUS_NAME))
environ['HD2_LOCATION'] = hd2_path
dotenv.set_key(dotenv_file, "HD2_LOCATION",
environ["HD2_LOCATION"])
if hd3_path:
if not path.exists(hd3_path):
folder_create(hd3_path)
environ['HD3_LOCATION'] = hd3_path
dotenv.set_key(dotenv_file, "HD3_LOCATION",
environ["HD3_LOCATION"])
self.config.load_values()
return self.env_details()
except Exception as e:
self.last_error_message = f'Configure_Env : configure : {e}'
log_error(f'Configure_Env : configure : {e}')
raise ValueError(str(e))
def env_details(self):
self.reset_last_error()
try:
return {
"hd1_path": environ.get('HD1_LOCATION'),
"hd2_path": environ.get('HD2_LOCATION'),
"hd3_path": environ.get('HD3_LOCATION')
}
except Exception as e:
self.last_error_message = f'Configure_Env : env_details : {e}'
log_error(f'Configure_Env : env_details : {e}')
raise ValueError(str(e))
def configure_endpoints(self, endpoint_string):
self.reset_last_error()
try:
dotenv_file = dotenv.find_dotenv()
valid_endpoint_string = self.get_valid_endpoints(endpoint_string)
if valid_endpoint_string:
environ['ENDPOINTS'] = valid_endpoint_string
logger.info(f"ENDPOINTS : {environ['ENDPOINTS']}")
dotenv.set_key(dotenv_file, "ENDPOINTS", environ["ENDPOINTS"])
self.config.load_values()
return json.loads(environ['ENDPOINTS'])
else:
self.last_error_message = f"No valid endpoint found in: {endpoint_string}"
log_error(f"No valid endpoint found in",
data={"enpoints": endpoint_string})
return -1
except Exception as e:
self.last_error_message = f'Configure_Env : configure_endpoints : {e}'
log_error(f'Configure_Env : configure_endpoints : {e}')
raise ValueError(str(e))
def get_valid_endpoints(self, endpoint_string):
self.reset_last_error()
try:
valid_endpoints = {'Endpoints': []}
endpoint_json = json.loads(endpoint_string)
endpoint_count = len(endpoint_json['Endpoints'])
for idx in range(endpoint_count):
server_url = "http://" + endpoint_json['Endpoints'][idx]['IP'] + ":" + \
endpoint_json['Endpoints'][idx]['Port']
response = self.gw_sdk_healthcheck(server_url)
if response:
if response.status_code == 200:
valid_endpoints['Endpoints'].append(
endpoint_json['Endpoints'][idx])
valid_endpoints_count = len(valid_endpoints['Endpoints'])
if valid_endpoints_count == 0:
return None
return json.dumps(valid_endpoints)
except Exception as e:
self.last_error_message = f'Configure_Env : get_valid_endpoints : {e}'
log_error(f'Configure_Env : get_valid_endpoints : {e}')
raise ValueError(str(e))
def gw_sdk_healthcheck(self, server_url):
self.reset_last_error()
try:
api_route = "api/health/"
url = urljoin(server_url, api_route)
response = requests.request("GET", url, verify=False, timeout=10)
return response
except Exception as e:
self.last_error_message = f'Configure_Env : gw_sdk_healthcheck : {e}'
log_error(f'Configure_Env : gw_sdk_healthcheck : {e}')
return None
def __init__(self, folder):
self.config = Config()
self.folder = folder
self.data = {"events": []}
self.get_from_file()
self.unique_id = str(uuid.uuid4())
def test_now(self):
result = self.status.now()
assert list_set(result) == ['check_logging', 'config', 'date']
assert result.get('config') == Config().values()
def config():
return Config().values()
def __init__(self):
self.file_path = None
self.metadata_folder = None
self.metadata = None
self.config = Config()
self.metadata_elastic = Metadata_Elastic().setup()
def __init__(self):
self.config = Config()
def get_config(self):
from cdr_plugin_folder_to_folder.common_settings.Config import Config # needs to be here due to circular references with this class and Config
return Config()
def setUp(self) -> None:
self.config = Config()
self.local_storage = Storage()
