def load_simdjson(file, data):
import simdjson
if file is not None:
return simdjson.load(file)
with open(data, 'r') as f:
return simdjson.load(f)
def load_simdjson(file, data):
import simdjson
if file is not None:
args['verbose'] and print(f"Loading file using simdjson")
return simdjson.load(file)
with open(data, 'r') as f:
return simdjson.load(f)
def _get_pred_attr_cache(self, gid):
op = dag_utils.report_graph(self._graph, types_pb2.PRED_ATTR_BY_GID, gid=gid)
archive = op.eval()
gid = archive.get_uint64()
fp = io.BytesIO(archive.get_bytes())
pred_attr_cache = simdjson.load(fp)
return gid, pred_attr_cache
def _test_load():
"""Ensure basic usage of load is the same."""
# We don't use a binary file here because pre-py3.6 the built-in couldn't
# handle bytes.
with open('jsonexamples/canada.json', 'r') as fin:
builtin_json = json.load(fin)
with open('jsonexamples/canada.json', 'rb') as fin:
simd_json = simdjson.load(fin)
assert builtin_json == simd_json
def load_p4(p4_file, p4_version=16, add_attrs=True):
p4_file = os.path.expandvars(p4_file)
json_filename = hlir16.hlir.p4_to_json(p4_file)
import simdjson
with open(json_filename, 'r') as json:
json_contents = simdjson.load(json)
hlir = hlir16.hlir.walk_json_from_top(json_contents)
if type(error_code := hlir) is not P4Node:
print(f"Could not load P4 file {p4_file}, error code: {error_code}")
sys.exit(error_code)
def read_simdjson(filepath: str):
try:
with open(filepath) as fp:
return simdjson.load(fp)
except:
return None
def read_json(path: Union[str, Path]):
"""
Read a json file from a string path
"""
with open(path) as f:
return simdjson.load(f)
def auth_cloud(gcs=None, s3=None):
# expects a json file for gcs
global _gcs_available, _s3_available, s3_client, gcp_client, gcp_storage_client
if gcs:
try:
gcp_client = service_account.Credentials.service_account_info = json.load(open(gcs, 'r'))
gcp_storage_client = storage.Client.from_service_account_json(gcs)
_gcs_available = True
except Exception as e:
print(f'Unable to Authenticate GCS: {str(e)}')
_gcs_available = False
if s3:
if _io_type(s3) == 'str' and os.path.exists(s3):
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = s3
try:
s3_client = boto3.Session()
_s3_available = True
except Exception as e:
print(f'Unable to Authenticate S3: {str(e)}')
_s3_available = False
elif _io_type(s3) == 'dict':
if 'aws_access_key_id' in s3 and 'aws_secret_access_key' in s3:
try:
s3_client = boto3.Session(aws_access_key_id=s3['aws_access_key_id'], aws_secret_access_key=s3['aws_secret_access_key'])
_s3_available = True
except Exception as e:
print(f'Unable to Authenticate S3: {str(e)}')
_s3_available = False
elif 'aws_access_key_id'.upper() in s3 and 'aws_secret_access_key'.upper() in s3:
try:
s3_client = boto3.Session(aws_access_key_id=s3['aws_access_key_id'.upper()], aws_secret_access_key=s3['aws_secret_access_key'.upper()])
_s3_available = True
except Exception as e:
print(f'Unable to Authenticate S3: {str(e)}')
_s3_available = False
elif 'AWS_SHARED_CREDENTIALS_FILE' in s3:
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = s3
try:
s3_client = boto3.Session()
_s3_available = True
except Exception as e:
print(f'Unable to Authenticate S3: {str(e)}')
_s3_available = False
else:
_S3_ID = os.environ.get("AWS_ACCESS_KEY_ID", None)
_S3_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", None)
_S3_PATH = os.environ.get("AWS_SHARED_CREDENTIALS_FILE", None)
_GCS_PATH = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", None)
if not _s3_available:
if _S3_ID and _S3_KEY:
try:
s3_client = boto3.Session(aws_access_key_id=_S3_ID, aws_secret_access_key=_S3_KEY)
_s3_available = True
except Exception as e:
print(f'Unable to Authenticate S3: {str(e)}')
_s3_available = False
elif _S3_PATH:
try:
os.environ['AWS_SHARED_CREDENTIALS_FILE'] = _S3_PATH
s3_client = boto3.Session()
_s3_available = True
except Exception as e:
print(f'Unable to Authenticate S3: {str(e)}')
_s3_available = False
if not _gcs_available and _GCS_PATH:
try:
gcp_client = service_account.Credentials.service_account_info = json.load(open(_GCS_PATH, 'r'))
gcp_storage_client = storage.Client.from_service_account_json(_GCS_PATH)
_gcs_available = True
except Exception as e:
print(f'Unable to Authenticate GCS: {str(e)}')
_gcs_available = False
if _gcs_available:
print('Authenticated GCS')
else:
gcp_client = None
gcp_storage_client = None
if _s3_available:
print('Authenticated S3')
else:
s3_client = None
_tqdm_available = False
try:
import numpy as np
_numpy_available = True
except ImportError:
_numpy_available = False
try:
from google.cloud import storage
from google.oauth2 import service_account
USE_GCS = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", None)
if USE_GCS:
if os.path.exists(USE_GCS):
gcp_client = service_account.Credentials.service_account_info = json.load(open(USE_GCS, 'r'))
gcp_storage_client = storage.Client.from_service_account_json(USE_GCS)
_gcs_available = True
else:
gcp_client, gcp_storage_client = None, None
_gcs_available = False
else:
_gcs_available = False
except ImportError:
gcp_client, gcp_storage_client = None, None
_gcs_available = False
try:
import boto3
_S3_ID = os.environ.get("AWS_ACCESS_KEY_ID", None)
_S3_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", None)
def jsonload(cls, filename):
return json.load(gfile(filename, 'r'))
def read_simdjson(filepath: str):
with open(filepath) as fp:
return simdjson.load(fp)
