def setup_fs(s3, key="", secret="", endpoint="", cert=""):
"""Given a boolean specifying whether to use local disk or S3, setup filesystem
Syntax examples: AWS (http://s3.us-east-2.amazonaws.com), MinIO (http://192.168.0.1:9000)
The cert input is relevant if you're using MinIO with TLS enabled, for specifying the path to the certficiate
"""
if s3:
import s3fs
if "amazonaws" in endpoint:
fs = s3fs.S3FileSystem(key=key, secret=secret)
elif cert != "":
fs = s3fs.S3FileSystem(key=key,
secret=secret,
client_kwargs={
"endpoint_url": endpoint,
"verify": cert
})
else:
fs = s3fs.S3FileSystem(
key=key,
secret=secret,
client_kwargs={"endpoint_url": endpoint},
)
else:
from pathlib import Path
import canedge_browser
base_path = Path(__file__).parent
fs = canedge_browser.LocalFileSystem(base_path=base_path)
return fs
def setup_fs(s3, key="", secret="", endpoint="", cert="", passwords={}):
"""Given a boolean specifying whether to use local disk or S3, setup filesystem
Syntax examples: AWS (http://s3.us-east-2.amazonaws.com), MinIO (http://192.168.0.1:9000)
The cert input is relevant if you're using MinIO with TLS enabled, for specifying the path to the certficiate.
The block_size is set to accomodate files up to 55 MB in size. If your log files are larger, adjust this value accordingly
"""
if s3:
import s3fs
block_size = 55 * 1024 * 1024
if "amazonaws" in endpoint:
fs = s3fs.S3FileSystem(key=key,
secret=secret,
default_block_size=block_size)
elif cert != "":
fs = s3fs.S3FileSystem(
key=key,
secret=secret,
client_kwargs={
"endpoint_url": endpoint,
"verify": cert
},
default_block_size=block_size,
)
else:
fs = s3fs.S3FileSystem(
key=key,
secret=secret,
client_kwargs={"endpoint_url": endpoint},
default_block_size=block_size,
)
else:
from pathlib import Path
import canedge_browser
base_path = Path(__file__).parent
fs = canedge_browser.LocalFileSystem(base_path=base_path,
passwords=passwords)
return fs
# specify devices to process from local disk
devices = ["LOG/958D2219"]
session_offset = 0 # optionally offset the session counter for the uploaded files
# specify target S3 bucket details
key = "s3_key"
secret = "s3_secret"
endpoint = "s3_endpoint" # e.g. https://s3.eu-central-1.amazonaws.com
bucket = "s3_bucket"
# ----------------------------------
# load all log files from local folder
base_path = Path(__file__).parent
fs = canedge_browser.LocalFileSystem(base_path=base_path)
log_files = canedge_browser.get_log_files(fs, devices)
print(f"Found a total of {len(log_files)} log files")
s3 = boto3.client(
"s3", endpoint_url=endpoint, aws_access_key_id=key, aws_secret_access_key=secret, config=Config(signature_version="s3v4"),
)
transfer = S3Transfer(s3, TransferConfig(multipart_threshold=9999999999999999, max_concurrency=10, num_download_attempts=10,))
# for each log file, extract header information, create S3 key and upload
for log_file in log_files:
with fs.open(log_file, "rb") as handle:
mdf_file = mdf_iter.MdfFile(handle)
header = "HDComment.Device Information"