def list_log_files(fs, devices, start_times, verbose=True):
"""Given a list of device paths, list log files from specified filesystem.
Data is loaded based on the list of start datetimes
"""
import canedge_browser, mdf_iter
log_files = []
if len(start_times):
for idx, device in enumerate(devices):
start = start_times[idx]
log_files_device = canedge_browser.get_log_files(fs, [device], start_date=start)
# exclude the 1st log file if the last timestamp is before the start timestamp
if len(log_files_device) > 0:
with fs.open(log_files_device[0], "rb") as handle:
mdf_file = mdf_iter.MdfFile(handle)
df_raw = mdf_file.get_data_frame()
end_time = df_raw.index[-1]
if end_time < start:
log_files_device = log_files_device[1:]
log_files.extend(log_files_device)
if verbose:
print(f"Found {len(log_files)} log files\n")
return log_files
def get_raw_data(self, log_file):
"""Extract a df of raw data and device ID from log file
"""
import mdf_iter
with self.fs.open(log_file, "rb") as handle:
mdf_file = mdf_iter.MdfFile(handle)
device_id = self.get_device_id(mdf_file)
df_raw = mdf_file.get_data_frame()
return df_raw, device_id
def example_iterator():
"""Simple example which extracts all CAN records in a pandas DataFrame.
"""
# Get a handle to the remote file.
fs = setup_fs()
log_file = r"LOG/EEEE0001/00000001/00000001.MF4"
with fs.open(log_file, "rb") as handle:
# Load the file.
mdf_file = mdf_iter.MdfFile(handle)
# Extract a dataframe with all the CAN measurements.
result = mdf_file.get_data_frame()
print(result)
return
def get_raw_data(self, log_file, lin=False, passwords={}):
"""Extract a df of raw data and device ID from log file.
Optionally include LIN bus data by setting lin=True
"""
import mdf_iter
with self.fs.open(log_file, "rb") as handle:
mdf_file = mdf_iter.MdfFile(handle, passwords=passwords)
device_id = self.get_device_id(mdf_file)
if lin:
df_raw_lin = mdf_file.get_data_frame_lin()
df_raw_lin["IDE"] = 0
df_raw_can = mdf_file.get_data_frame()
df_raw = df_raw_can.append(df_raw_lin)
else:
df_raw = mdf_file.get_data_frame()
return df_raw, device_id
def example_first_timestamp():
"""Simple example which get the timestamp of the first measurement in the log file.
"""
# Get a handle to the remote file.
fs = setup_fs()
log_file = r"LOG/EEEE0001/00000001/00000001.MF4"
with fs.open(log_file, "rb") as handle:
# Load the file.
mdf_file = mdf_iter.MdfFile(handle)
# Extract the first timestamp in nanoseconds.
timestamp_raw = mdf_file.get_first_measurement()
# Convert to datetime and set the timezone to UTC.
timestamp = datetime.utcfromtimestamp(timestamp_raw * 1E-9)
timestamp = timestamp.replace(tzinfo=timezone.utc)
print("First measurement at {}".format(timestamp))
return
base_path = Path(__file__).parent
fs = canedge_browser.LocalFileSystem(base_path=base_path)
log_files = canedge_browser.get_log_files(fs, devices)
print(f"Found a total of {len(log_files)} log files")
s3 = boto3.client(
"s3", endpoint_url=endpoint, aws_access_key_id=key, aws_secret_access_key=secret, config=Config(signature_version="s3v4"),
)
transfer = S3Transfer(s3, TransferConfig(multipart_threshold=9999999999999999, max_concurrency=10, num_download_attempts=10,))
# for each log file, extract header information, create S3 key and upload
for log_file in log_files:
with fs.open(log_file, "rb") as handle:
mdf_file = mdf_iter.MdfFile(handle)
header = "HDComment.Device Information"
device_id = mdf_file.get_metadata()[f"{header}.serial number"]["value_raw"]
session = mdf_file.get_metadata()[f"{header}.File Information.session"]["value_raw"]
session = f"{(int(session) + session_offset):08}"
split = int(mdf_file.get_metadata()[f"{header}.File Information.split"]["value_raw"])
split = f"{split:08}"
ext = log_file.split(".")[-1]
s3_meta_fw = mdf_file.get_metadata()[f"{header}.firmware version"]["value_raw"]
s3_meta_timestamp = mdf_file.get_data_frame().index.min().strftime("%Y%m%dT%H%M%S")
s3_key = f"{device_id}/{session}/{split}.{ext}"
s3_meta = {"Metadata": {"Fw": s3_meta_fw, "Timestamp": s3_meta_timestamp}}
