def measure_pvc_creation_time(interface, pvc_name): """ Measure PVC creation time based on logs Args: interface (str): The interface backed the PVC pvc_name (str): Name of the PVC for creation time measurement Returns: float: Creation time for the PVC """ format = '%H:%M:%S.%f' # Get the correct provisioner pod based on the interface if interface == constants.CEPHBLOCKPOOL: pod_name = pod.get_rbd_provisioner_pod().name else: pod_name = pod.get_cephfs_provisioner_pod().name # get the logs from the csi-provisioner container logs = pod.get_pod_logs(pod_name, 'csi-provisioner') logs = logs.split("\n") # Extract the starting time for the PVC provisioning start = [ i for i in logs if re.search(f"provision.*{pvc_name}.*started", i) ][0].split(' ')[1] # Extract the end time for the PVC provisioning end = [ i for i in logs if re.search(f"provision.*{pvc_name}.*succeeded", i) ][0].split(' ')[1] total = (datetime.datetime.strptime(end, format) - datetime.datetime.strptime(start, format)) return total.total_seconds()
def get_end_creation_time(interface, pvc_name): """ Get the ending creation time of a PVC based on provisioner logs Args: interface (str): The interface backed the PVC pvc_name (str): Name of the PVC for creation time measurement Returns: datetime object: End time of PVC creation """ format = '%H:%M:%S.%f' # Get the correct provisioner pod based on the interface if interface == constants.CEPHBLOCKPOOL: pod_name = pod.get_rbd_provisioner_pod().name else: pod_name = pod.get_cephfs_provisioner_pod().name # get the logs from the csi-provisioner container logs = pod.get_pod_logs(pod_name, 'csi-provisioner') logs = logs.split("\n") # Extract the starting time for the PVC provisioning end = [ i for i in logs if re.search(f"provision.*{pvc_name}.*succeeded", i) ] end = end[0].split(' ')[1] return datetime.datetime.strptime(end, format)
def test_noobaa_kms_validation(self): """ Validate from logs that there is successfully used NooBaa with KMS integration. """ operator_pod = pod.get_pods_having_label( label=constants.NOOBAA_OPERATOR_POD_LABEL, namespace=defaults.ROOK_CLUSTER_NAMESPACE, )[0] operator_logs = pod.get_pod_logs( pod_name=operator_pod["metadata"]["name"]) assert "found root secret in external KMS successfully" in operator_logs
def test_deployer_logs_not_empty(self): """ Test that the logs of manager container of ocs-osd-controller-manager pod are not empty """ deployer_pod = pod.get_pods_having_label( constants.MANAGED_CONTROLLER_LABEL, constants.OPENSHIFT_STORAGE_NAMESPACE)[0] deployer_logs = pod.get_pod_logs( pod_name=deployer_pod["metadata"]["name"], container="manager") log_lines = deployer_logs.split("\n") for line in log_lines: if "ERR" in line: log.info(f"{line}") log.info(f"Deployer log has {len(log_lines)} lines.") assert len(log_lines) > 100
def wait_for_active_pods(job, desired_count, timeout=3): """ Wait for job to load desired number of active pods in time specified in timeout. Args: job (obj): OCS job object desired_count (str): Number of desired active pods for provided job timeout (int): Number of seconds to wait for the job to get into state Returns: bool: If job has desired number of active pods """ job_name = job.name log.info(f"Checking number of active pods for job {job_name}") def _retrieve_job_state(): job_obj = job.ocp.get(resource_name=job_name, out_yaml_format=True) return job_obj["status"]["active"] try: for state in TimeoutSampler(timeout=timeout, sleep=3, func=_retrieve_job_state): if state == desired_count: return True else: log.debug(f"Number of active pods for job {job_name}: {state}") except TimeoutExpiredError: log.error( f"Job {job_name} doesn't have correct number of active pods ({desired_count})" ) job_pods = pod.get_pods_having_label(f"job-name={job_name}", job.namespace) for job_pod in job_pods: log.info( f"Description of job pod {job_pod['metadata']['name']}: {job_pod}" ) pod_logs = pod.get_pod_logs( job_pod["metadata"]["name"], namespace=job_pod["metadata"]["namespace"]) log.info(f"Logs from job pod {job_pod['metadata']['name']} are " f"available on DEBUG level") log.debug( f"Logs from job pod {job_pod['metadata']['name']}: {pod_logs}") return False
def test_verify_noobaa_db_service(mcg_obj_session): """ Validates whether MCG cli and noobaa db logs does not check 'noobaa-db' """ # Get noobaa status status = mcg_obj_session.exec_mcg_cmd("status").stderr assert ( 'Service "noobaa-db"' not in status ), "Error in MCG Cli status regarding non-existent noobaa-db service" log.info( "Verified: noobaa status does not contain error related to `noobaa-db` service." ) # verify noobaa db logs pattern = "Not found: Service noobaa-db" noobaa_db_log = get_pod_logs(pod_name=constants.NB_DB_NAME_47_AND_ABOVE) assert (re.search(pattern=pattern, string=noobaa_db_log) is None), f"Error: {pattern} msg found in the noobaa db logs."
def test_pod_log_after_upgrade(): """ Check OSD/MON/MGR pod logs after upgrade and verify the expected log exist """ pod_objs = get_osd_pods() + get_mon_pods() + get_mgr_pods() pod_names = [osd_pod_obj.name for osd_pod_obj in pod_objs] expected_log_after_upgrade = "set uid:gid to 167:167 (ceph:ceph)" logging.info(f"Check that the log '{expected_log_after_upgrade}' " f"appears after the osd/mon/mg pod is initialized") for pod_name in pod_names: pod_logs = get_pod_logs(pod_name=pod_name, all_containers=True) assert expected_log_after_upgrade in pod_logs, ( f"The expected log after upgrade '{expected_log_after_upgrade}' does not exist" f" on pod {pod_name}") logging.info( f"The log '{expected_log_after_upgrade}' appears in all relevant pods." )
def test_provider_server_logs(self): """ Test that the logs of ocs-provider-server pod have entries for each consumer """ provider_pod = pod.get_pods_having_label( constants.PROVIDER_SERVER_LABEL, constants.OPENSHIFT_STORAGE_NAMESPACE)[0] provider_logs = pod.get_pod_logs( pod_name=provider_pod["metadata"]["name"]) log_lines = provider_logs.split("\n") consumer_names = managedservice.get_consumer_names() for consumer_name in consumer_names: expected_log = ( f'successfully Enabled the StorageConsumer resource "{consumer_name}"' ) log_found = False for line in log_lines: if expected_log in line: log_found = True log.info( f"'{expected_log}' found in ocs-provider-server logs") break assert log_found, f"'{expected_log}' not found in ocs-provider-server logs"
def wait_for_workload(self, workload_id, sleep=1, timeout=60): """ Waits for the cosbench workload to complete Args: workload_id (str): ID of cosbench workload sleep: sleep in seconds timeout: timeout in seconds to check if mirroring Returns: bool: Whether cosbench workload processed successfully """ logger.info(f"Waiting for workload {workload_id} to be processed") pattern = f"sucessfully processed workload {workload_id}" try: for ret in TimeoutSampler( timeout=timeout, sleep=sleep, func=get_pod_logs, pod_name=self.cosbench_pod.name, namespace=self.namespace, ): if re.search(pattern=pattern, string=ret): break logger.info( f"Verified: Workload {workload_id} processed successfully") return True except TimeoutExpiredError: logger.error( f"Workload {workload_id} did not complete. Dumping cosbench pod log" ) # Log cosbench pod for debugging purpose cosbench_log = get_pod_logs(pod_name=self.cosbench_pod.name, namespace=self.namespace) logger.debug(cosbench_log) return False
def test_recovery_from_volume_deletion(self, nodes, pvc_factory, pod_factory): """ Test cluster recovery from disk deletion from the platform side. Based on documented procedure detailed in https://bugzilla.redhat.com/show_bug.cgi?id=1823183 """ logger.info("Picking a PV which to be deleted from the platform side") osd_pvs = get_deviceset_pvs() osd_pv = random.choice(osd_pvs) osd_pv_name = osd_pv.name # get the claim name logger.info(f"Getting the claim name for OSD PV {osd_pv_name}") claim_name = osd_pv.get().get("spec").get("claimRef").get("name") # Get the backing volume name logger.info(f"Getting the backing volume name for PV {osd_pv_name}") backing_volume = nodes.get_data_volumes(pvs=[osd_pv])[0] # Get the corresponding PVC logger.info(f"Getting the corresponding PVC of PV {osd_pv_name}") osd_pvcs = get_deviceset_pvcs() osd_pvcs_count = len(osd_pvcs) osd_pvc = [ ds for ds in osd_pvcs if ds.get().get("metadata").get("name") == claim_name ][0] # Get the corresponding OSD pod and ID logger.info(f"Getting the OSD pod using PVC {osd_pvc.name}") osd_pods = get_osd_pods() osd_pods_count = len(osd_pods) osd_pod = [ osd_pod for osd_pod in osd_pods if osd_pod.get().get("metadata").get("labels").get( constants.CEPH_ROOK_IO_PVC_LABEL) == claim_name ][0] logger.info(f"OSD_POD {osd_pod.name}") osd_id = osd_pod.get().get("metadata").get("labels").get("ceph-osd-id") # Get the node that has the OSD pod running on logger.info( f"Getting the node that has the OSD pod {osd_pod.name} running on") osd_node = get_pod_node(osd_pod) osd_prepare_pods = get_osd_prepare_pods() osd_prepare_pod = [ pod for pod in osd_prepare_pods if pod.get().get("metadata").get( "labels").get(constants.CEPH_ROOK_IO_PVC_LABEL) == claim_name ][0] osd_prepare_job_name = (osd_prepare_pod.get().get("metadata").get( "labels").get("job-name")) osd_prepare_job = get_job_obj(osd_prepare_job_name) # Get the corresponding OSD deployment logger.info(f"Getting the OSD deployment for OSD PVC {claim_name}") osd_deployment = [ osd_pod for osd_pod in get_osd_deployments() if osd_pod.get().get("metadata").get("labels").get( constants.CEPH_ROOK_IO_PVC_LABEL) == claim_name ][0] osd_deployment_name = osd_deployment.name # Delete the volume from the platform side logger.info(f"Deleting {backing_volume} from the platform side") nodes.detach_volume(backing_volume, osd_node) # Scale down OSD deployment logger.info(f"Scaling down OSD deployment {osd_deployment_name} to 0") ocp.OCP().exec_oc_cmd( f"scale --replicas=0 deployment/{osd_deployment_name}") # Force delete OSD pod if necessary osd_pod_name = osd_pod.name logger.info(f"Waiting for OSD pod {osd_pod.name} to get deleted") try: osd_pod.ocp.wait_for_delete(resource_name=osd_pod_name) except TimeoutError: osd_pod.delete(force=True) osd_pod.ocp.wait_for_delete(resource_name=osd_pod_name) # Run ocs-osd-removal job ocp_version = float(get_ocp_version()) if ocp_version >= 4.6: cmd = f"process ocs-osd-removal -p FAILED_OSD_IDS={osd_id} -o yaml" else: cmd = f"process ocs-osd-removal -p FAILED_OSD_ID={osd_id} -o yaml" logger.info(f"Executing OSD removal job on OSD-{osd_id}") ocp_obj = ocp.OCP(namespace=config.ENV_DATA["cluster_namespace"]) osd_removal_job_yaml = ocp_obj.exec_oc_cmd(cmd) osd_removal_job = OCS(**osd_removal_job_yaml) osd_removal_job.create(do_reload=False) # Get ocs-osd-removal pod name logger.info("Getting the ocs-osd-removal pod name") osd_removal_pod_name = get_osd_removal_pod_name(osd_id) osd_removal_pod_obj = get_pod_obj(osd_removal_pod_name, namespace="openshift-storage") osd_removal_pod_obj.ocp.wait_for_resource( condition=constants.STATUS_COMPLETED, resource_name=osd_removal_pod_name) # Verify OSD removal from the ocs-osd-removal pod logs logger.info( f"Verifying removal of OSD from {osd_removal_pod_name} pod logs") logs = get_pod_logs(osd_removal_pod_name) pattern = f"purged osd.{osd_id}" assert re.search(pattern, logs) osd_pvc_name = osd_pvc.name if ocp_version < 4.6: # Delete the OSD prepare job logger.info(f"Deleting OSD prepare job {osd_prepare_job_name}") osd_prepare_job.delete() osd_prepare_job.ocp.wait_for_delete( resource_name=osd_prepare_job_name, timeout=120) # Delete the OSD PVC logger.info(f"Deleting OSD PVC {osd_pvc_name}") osd_pvc.delete() osd_pvc.ocp.wait_for_delete(resource_name=osd_pvc_name) # Delete the OSD deployment logger.info(f"Deleting OSD deployment {osd_deployment_name}") osd_deployment.delete() osd_deployment.ocp.wait_for_delete( resource_name=osd_deployment_name, timeout=120) else: # If ocp version is '4.6' and above the osd removal job should # delete the OSD prepare job, OSD PVC, OSD deployment logger.info( f"Verifying deletion of OSD prepare job {osd_prepare_job_name}" ) osd_prepare_job.ocp.wait_for_delete( resource_name=osd_prepare_job_name, timeout=30) logger.info(f"Verifying deletion of OSD PVC {osd_pvc_name}") osd_pvc.ocp.wait_for_delete(resource_name=osd_pvc_name, timeout=30) logger.info( f"Verifying deletion of OSD deployment {osd_deployment_name}") osd_deployment.ocp.wait_for_delete( resource_name=osd_deployment_name, timeout=30) # Delete PV logger.info(f"Verifying deletion of PV {osd_pv_name}") try: osd_pv.ocp.wait_for_delete(resource_name=osd_pv_name) except TimeoutError: osd_pv.delete() osd_pv.ocp.wait_for_delete(resource_name=osd_pv_name) if ocp_version < 4.6: # Delete the rook ceph operator pod to trigger reconciliation rook_operator_pod = get_operator_pods()[0] logger.info( f"deleting Rook Ceph operator pod {rook_operator_pod.name}") rook_operator_pod.delete() # Delete the OSD removal job logger.info(f"Deleting OSD removal job ocs-osd-removal-{osd_id}") osd_removal_job = get_job_obj(f"ocs-osd-removal-{osd_id}") osd_removal_job.delete() osd_removal_job.ocp.wait_for_delete( resource_name=f"ocs-osd-removal-{osd_id}") timeout = 600 # Wait for OSD PVC to get created and reach Bound state logger.info( "Waiting for a new OSD PVC to get created and reach Bound state") assert osd_pvc.ocp.wait_for_resource( timeout=timeout, condition=constants.STATUS_BOUND, selector=constants.OSD_PVC_GENERIC_LABEL, resource_count=osd_pvcs_count, ), (f"Cluster recovery failed after {timeout} seconds. " f"Expected to have {osd_pvcs_count} OSD PVCs in status Bound. Current OSD PVCs status: " f"{[pvc.ocp.get_resource(pvc.get().get('metadata').get('name'), 'STATUS') for pvc in get_deviceset_pvcs()]}" ) # Wait for OSD pod to get created and reach Running state logger.info( "Waiting for a new OSD pod to get created and reach Running state") assert osd_pod.ocp.wait_for_resource( timeout=timeout, condition=constants.STATUS_RUNNING, selector=constants.OSD_APP_LABEL, resource_count=osd_pods_count, ), (f"Cluster recovery failed after {timeout} seconds. " f"Expected to have {osd_pods_count} OSD pods in status Running. Current OSD pods status: " f"{[osd_pod.ocp.get_resource(pod.get().get('metadata').get('name'), 'STATUS') for pod in get_osd_pods()]}" ) # We need to silence the old osd crash warning due to BZ https://bugzilla.redhat.com/show_bug.cgi?id=1896810 # This is a workaround - issue for tracking: https://github.com/red-hat-storage/ocs-ci/issues/3438 if ocp_version >= 4.6: silence_osd_crash = cluster.wait_for_silence_ceph_osd_crash_warning( osd_pod_name) if not silence_osd_crash: logger.info("Didn't find ceph osd crash warning") # Validate cluster is still functional self.sanity_helpers.health_check(tries=100) self.sanity_helpers.create_resources(pvc_factory, pod_factory)
def test_rgw_kafka_notifications(self, bucket_factory): """ Test to verify rgw kafka notifications """ # Get sc sc = default_storage_class(interface_type=constants.CEPHBLOCKPOOL) # Deploy amq cluster self.amq.setup_amq_cluster(sc.name) # Create topic self.kafka_topic = self.amq.create_kafka_topic() # Create Kafkadrop pod ( self.kafkadrop_pod, self.kafkadrop_pod, self.kafkadrop_route, ) = self.amq.create_kafkadrop() # Get the kafkadrop route kafkadrop_host = self.kafkadrop_route.get().get("spec").get("host") # Create bucket bucketname = bucket_factory(amount=1, interface="RGW-OC")[0].name # Get RGW credentials rgw_obj = RGW() rgw_endpoint, access_key, secret_key = rgw_obj.get_credentials() # Clone notify repo notify_path = clone_notify() # Initialise to put objects data = "A random string data to write on created rgw bucket" obc_obj = OBC(bucketname) s3_resource = boto3.resource( "s3", verify=retrieve_verification_mode(), endpoint_url=rgw_endpoint, aws_access_key_id=obc_obj.access_key_id, aws_secret_access_key=obc_obj.access_key, ) s3_client = s3_resource.meta.client # Initialize notify command to run notify_cmd = ( f"python {notify_path} -e {rgw_endpoint} -a {obc_obj.access_key_id} " f"-s {obc_obj.access_key} -b {bucketname} -ke {constants.KAFKA_ENDPOINT} -t {self.kafka_topic.name}" ) log.info(f"Running cmd {notify_cmd}") # Put objects to bucket assert s3_client.put_object(Bucket=bucketname, Key="key-1", Body=data), "Failed: Put object: key-1" exec_cmd(notify_cmd) # Validate rgw logs notification are sent # No errors are seen pattern = "ERROR: failed to create push endpoint" rgw_pod_obj = get_rgw_pods() rgw_log = get_pod_logs(pod_name=rgw_pod_obj[0].name, container="rgw") assert re.search(pattern=pattern, string=rgw_log) is None, ( f"Error: {pattern} msg found in the rgw logs." f"Validate {pattern} found on rgw logs and also " f"rgw bucket notification is working correctly") assert s3_client.put_object(Bucket=bucketname, Key="key-2", Body=data), "Failed: Put object: key-2" exec_cmd(notify_cmd) # Validate message are received Kafka side using curl command # A temporary way to check from Kafka side, need to check from UI curl_command = ( f"curl -X GET {kafkadrop_host}/topic/{self.kafka_topic.name} " "-H 'content-type: application/vnd.kafka.json.v2+json'") json_output = run_cmd(cmd=curl_command) new_string = json_output.split() messages = new_string[new_string.index("messages</td>") + 1] if messages.find("1") == -1: raise Exception( "Error: Messages are not recieved from Kafka side." "RGW bucket notification is not working as expected.") # Validate the timestamp events ocs_version = config.ENV_DATA["ocs_version"] if Version.coerce(ocs_version) >= Version.coerce("4.8"): cmd = ( f"bin/kafka-console-consumer.sh --bootstrap-server {constants.KAFKA_ENDPOINT} " f"--topic {self.kafka_topic.name} --from-beginning --timeout-ms 20000" ) pod_list = get_pod_name_by_pattern( pattern="my-cluster-zookeeper", namespace=constants.AMQ_NAMESPACE) zookeeper_obj = get_pod_obj(name=pod_list[0], namespace=constants.AMQ_NAMESPACE) event_obj = zookeeper_obj.exec_cmd_on_pod(command=cmd) log.info(f"Event obj: {event_obj}") event_time = event_obj.get("Records")[0].get("eventTime") format_string = "%Y-%m-%dT%H:%M:%S.%fZ" try: datetime.strptime(event_time, format_string) except ValueError as ef: log.error( f"Timestamp event {event_time} doesnt match the pattern {format_string}" ) raise ef log.info( f"Timestamp event {event_time} matches the pattern {format_string}" )
def test_rgw_kafka_notifications(self, bucket_factory): """ Test to verify rgw kafka notifications """ # Get sc sc = default_storage_class(interface_type=constants.CEPHBLOCKPOOL) # Deploy amq cluster self.amq.setup_amq_cluster(sc.name) # Create topic self.kafka_topic = self.amq.create_kafka_topic() # Create Kafkadrop pod ( self.kafkadrop_pod, self.kafkadrop_pod, self.kafkadrop_route, ) = self.amq.create_kafkadrop() # Get the kafkadrop route kafkadrop_host = self.kafkadrop_route.get().get("spec").get("host") # Create bucket bucketname = bucket_factory(amount=1, interface="RGW-OC")[0].name # Get RGW credentials rgw_obj = RGW() rgw_endpoint, access_key, secret_key = rgw_obj.get_credentials() # Clone notify repo notify_path = clone_notify() # Initialise to put objects data = "A random string data to write on created rgw bucket" obc_obj = OBC(bucketname) s3_resource = boto3.resource( "s3", verify=retrieve_verification_mode(), endpoint_url=rgw_endpoint, aws_access_key_id=obc_obj.access_key_id, aws_secret_access_key=obc_obj.access_key, ) s3_client = s3_resource.meta.client # Initialize notify command to run notify_cmd = ( f"python {notify_path} -e {rgw_endpoint} -a {obc_obj.access_key_id} " f"-s {obc_obj.access_key} -b {bucketname} -ke {constants.KAFKA_ENDPOINT} -t {self.kafka_topic.name}" ) log.info(f"Running cmd {notify_cmd}") # Put objects to bucket assert s3_client.put_object(Bucket=bucketname, Key="key-1", Body=data), "Failed: Put object: key-1" exec_cmd(notify_cmd) # Validate rgw logs notification are sent # No errors are seen pattern = "ERROR: failed to create push endpoint" rgw_pod_obj = get_rgw_pods() rgw_log = get_pod_logs(pod_name=rgw_pod_obj[0].name, container="rgw") assert re.search(pattern=pattern, string=rgw_log) is None, ( f"Error: {pattern} msg found in the rgw logs." f"Validate {pattern} found on rgw logs and also " f"rgw bucket notification is working correctly") assert s3_client.put_object(Bucket=bucketname, Key="key-2", Body=data), "Failed: Put object: key-2" exec_cmd(notify_cmd) # Validate message are received Kafka side using curl command # A temporary way to check from Kafka side, need to check from UI curl_command = ( f"curl -X GET {kafkadrop_host}/topic/{self.kafka_topic.name} " "-H 'content-type: application/vnd.kafka.json.v2+json'") json_output = run_cmd(cmd=curl_command) new_string = json_output.split() messages = new_string[new_string.index("messages</td>") + 1] if messages.find("1") == -1: raise Exception( "Error: Messages are not recieved from Kafka side." "RGW bucket notification is not working as expected.")
def measure_obc_deletion_time(obc_name_list, timeout=60): """ Measure OBC deletion time Args: obc_name_list (list): List of obc names to measure deletion time timeout (int): Wait time in second before collecting log Returns: obc_dict (dict): Dictionary of obcs and deletion time in second """ # Get obc deletion logs nb_pod_name = get_pod_name_by_pattern("noobaa-operator-") nb_pod_log = pod.get_pod_logs( pod_name=nb_pod_name[0], namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) nb_pod_log = nb_pod_log.split("\n") loop_cnt = 0 while True: no_data = list() for obc_name in obc_name_list: start = [ i for i in nb_pod_log if re.search(f"removing ObjectBucket.*{obc_name}", i) ] end = [ i for i in nb_pod_log if re.search(f"ObjectBucket deleted.*{obc_name}", i) ] if not start or not end: no_data.append(obc_name) if no_data: time.sleep(timeout) nb_pod_name = get_pod_name_by_pattern("noobaa-operator-") nb_pod_log = pod.get_pod_logs( pod_name=nb_pod_name[0], namespace=constants.OPENSHIFT_STORAGE_NAMESPACE) nb_pod_log = nb_pod_log.split("\n") loop_cnt += 1 if loop_cnt >= 10: log.info("Waited for more than 10 mins but still no data") raise UnexpectedBehaviour( f"There is no obc deletion data in noobaa-operator logs for {no_data}" ) continue else: break obc_dict = dict() this_year = str(datetime.datetime.now().year) for obc_name in obc_name_list: # Extract obc deletion start time start_item = [ i for i in nb_pod_log if re.search(f"removing ObjectBucket.*{obc_name}", i) ] mon_day = " ".join(start_item[0].split(" ")[0:2]) start = f"{this_year} {mon_day}" dt_start = datetime.datetime.strptime(start, "%Y I%m%d %H:%M:%S.%f") # Extract obc deletion end time end_item = [ i for i in nb_pod_log if re.search(f"ObjectBucket deleted.*{obc_name}", i) ] mon_day = " ".join(end_item[0].split(" ")[0:2]) end = f"{this_year} {mon_day}" dt_end = datetime.datetime.strptime(end, "%Y I%m%d %H:%M:%S.%f") total = dt_end - dt_start log.info(f"{obc_name}: {total.total_seconds()} sec") obc_dict[obc_name] = total.total_seconds() return obc_dict