def test_roundtrip(p):
obj, is_binary, indent = p
out = BytesIO()
dump(obj, out, binary=is_binary, indent=indent)
out.seek(0)
res = load(out)
_assert_roundtrip(obj, res)
def _comparison_params(vector_type, directory_path):
for file in _list_files(directory_path):
with _open(file) as vector:
sequences = load(vector, single_value=False)
for sequence in sequences:
for param in vector_type.comparison_param_generator(file, sequence):
yield param
def read_data(filename: str):
# Build the shared symbol table from the analytics symbols
# Don't know how much time this takes but I presume that this only needs to be done once
catalog = ion_symbols.SymbolTableCatalog()
symbols = ion_symbols.SymbolTable(ion_symbols.SHARED_TABLE_TYPE, table,
"foxtel.engagement.format", 1)
catalog.register(symbols)
# Pull in the file and transition from ION format to the internal data-model
# from here we can either generate XML, JSON or send events to Segment.
# Rather than performing class inspection it maybe better to add handlers to
# the reporting event model.
with open(filename, "rb") as read_file:
data = simpleion.load(read_file, catalog, single_value=True)
event_model = decode_ion_data(data)
events = event_model.events
for event in events:
if isinstance(event, ViewingStopEvent):
pass
if isinstance(event, RecordingEvent):
pass
if isinstance(event, BookContentActionEvent):
pass
if isinstance(event, DeviceContextEvent):
pass
return event_model
def test_roundtrip(p):
obj, is_binary = p
out = BytesIO()
dump(obj, out, binary=is_binary)
out.seek(0)
res = load(out)
_assert_roundtrip(obj, res)
def test_dump_load_text(p):
# test dump
out = BytesIO()
dump(p.obj, out, binary=False, sequence_as_stream=p.stream)
res = out.getvalue()
if not p.has_symbols:
assert (b'$ion_1_0 ' + p.expected) == res
else:
# The payload contains a LST. The value comes last, so compare the end bytes.
assert p.expected == res[len(res) - len(p.expected):]
# test load
out.seek(0)
res = load(out, single_value=(not p.stream))
def equals():
if ion_equals(p.obj, res):
return True
if isinstance(p.obj, SymbolToken):
expected_token = p.obj
if p.obj.text is None:
assert p.obj.sid is not None
# System symbol IDs are mapped correctly in the text format.
token = SYSTEM_SYMBOL_TABLE.get(p.obj.sid)
assert token is not None # User symbols with unknown text won't be successfully read.
expected_token = token
return expected_token == res
else:
try:
return isnan(p.obj) and isnan(res)
except TypeError:
return False
if not equals():
assert ion_equals(p.obj,
res) # Redundant, but provides better error message.
def _comparison_params(vector_type, directory_path):
for file in _list_files(directory_path):
with _open(file) as vector:
sequences = load(vector, single_value=False)
for sequence in sequences:
for param in vector_type.comparison_param_generator(file, sequence):
yield param
def read_data(filename: str): catalog = ion_symbols.SymbolTableCatalog() symbols = ion_symbols.SymbolTable(ion_symbols.SHARED_TABLE_TYPE, table, "foxtel.engagement.format", 1) catalog.register(symbols) with open(filename, "rb") as read_file: data = simpleion.load(read_file, catalog, single_value=True) with open(filename + '.json', 'w', encoding='utf-8') as outfile: json.dump(data, outfile, ensure_ascii=False, indent=4, cls=JSONEncoderForIonTypes) print(json.dumps(data, ensure_ascii=False, indent=4, cls=JSONEncoderForIonTypes))
def get_file_rates(cache, contract_name, dt):
try:
return cache['contract_names'][contract_name][dt]
except KeyError:
try:
contract_names = cache['contract_names']
except KeyError:
contract_names = {}
cache['contract_names'] = contract_names
try:
cont = contract_names[contract_name]
except KeyError:
cont = {}
contract_names[contract_name] = cont
try:
return cont[dt]
except KeyError:
rscripts_path = os.path.join(
root_path, 'rate_scripts', contract_name)
for rscript_fname in sorted(os.listdir(rscripts_path)):
if not rscript_fname.endswith('.ion'):
continue
try:
start_str, finish_str = \
rscript_fname.split('.')[0].split('_')
except ValueError:
raise Exception(
"The rate script " + rscript_fname +
" in the directory " + rscripts_path +
" should consist of two dates separated by an " +
"underscore.")
start_date = to_utc(Datetime.strptime(start_str, "%Y%m%d%H%M"))
if finish_str == 'ongoing':
finish_date = to_utc(Datetime.max)
else:
finish_date = to_utc(
Datetime.strptime(finish_str, "%Y%m%d%H%M"))
if start_date <= dt <= finish_date:
with open(
os.path.join(rscripts_path, rscript_fname),
'r') as f:
rscript = load(f)
FOUR_WEEKS = timedelta(weeks=4)
range_start = hh_max(start_date, dt - FOUR_WEEKS)
range_finish = hh_min(finish_date, dt + FOUR_WEEKS)
for hh_date in hh_range(range_start, range_finish):
cont[hh_date] = rscript
break
try:
return cont[dt]
except KeyError:
raise BadRequest(
"Can't find a rate script at " + hh_format(dt))
def errors(self):
if self.__errors is None:
if os.path.isfile(self.error_location):
errors_in = FileIO(self.error_location, mode='rb')
try:
errors_stream = simpleion.load(errors_in, single_value=False)
finally:
errors_in.close()
self.__errors = IonPyList.from_value(IonType.LIST, errors_stream)
else:
self.__errors = IonPyList.from_value(IonType.LIST, [])
return self.__errors
def comparison_report(self):
if self.__comparison_report is None:
if os.path.isfile(self.output_location):
comparisons_in = FileIO(self.output_location, mode='rb')
try:
comparison_failure_stream = simpleion.load(comparisons_in, single_value=False)
finally:
comparisons_in.close()
self.__comparison_report = IonPyList.from_value(IonType.LIST, comparison_failure_stream)
else:
self.__comparison_report = IonPyList.from_value(IonType.LIST, [])
return self.__comparison_report
def to_ion_hash(algorithm):
if 'ion' in ion_test:
value = ion_test['ion']
if '10n' in ion_test:
ba = bytearray(_IVM_BYTES)
for byte in ion_test['10n']:
ba.append(byte)
value = ion.load(BytesIO(ba))
return value.ion_hash(hash_function_provider=hash_function_provider(
algorithm, _actual_updates, _actual_digests))
def test_dump_load_binary(p):
# test dump
out = BytesIO()
dump(p.obj, out, binary=True, sequence_as_stream=p.stream)
res = out.getvalue()
if not p.has_symbols:
assert (_IVM + p.expected) == res
else:
# The payload contains a LST. The value comes last, so compare the end bytes.
assert p.expected == res[len(res) - len(p.expected):]
# test load
out.seek(0)
res = load(out, single_value=(not p.stream))
assert ion_equals(p.obj, res)
def _to_buffer(ion_test, binary):
if 'ion' in ion_test:
v = ion.dumps(ion_test['ion'], binary=binary)
if '10n' in ion_test:
v = bytearray(_IVM_BYTES)
for byte in ion_test['10n']:
v.append(byte)
if not binary:
value = ion.load(BytesIO(v))
v = ion.dumps(value, binary=False)
if binary:
return BytesIO(v)
else:
return StringIO(v)
def good():
with _open(file) as vector:
load(vector, single_value=False)
def _roundtrip(value, is_binary):
out = BytesIO()
dump(value, out, binary=is_binary)
out.seek(0)
roundtripped = load(out)
return roundtripped
def preprocess(element):
if is_embedded:
assert isinstance(element, six.text_type)
element = load(six.StringIO(element), single_value=False)
return element
def bad():
with _open(file) as vector:
with raises((IonException, ValueError, TypeError)):
load(vector, single_value=False)
def good():
with _open(file) as vector:
load(vector, single_value=False)
def _roundtrip(value, is_binary):
out = BytesIO()
dump(value, out, binary=is_binary)
out.seek(0)
roundtripped = load(out)
return roundtripped
def test_single_value_with_stream_fails(is_binary):
out = BytesIO()
dump(['foo', 123], out, binary=is_binary, sequence_as_stream=True)
out.seek(0)
with raises(IonException):
load(out, single_value=True)
def test_reading_simpleion_load():
# http://amzn.github.io/ion-docs/guides/cookbook.html#reading-and-writing-ion-data
data = BytesIO(b'{hello: "world"}')
value = simpleion.load(data)
assert u'hello world' == u'hello %s' % value[u'hello']
# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
# express or implied. See the License for the specific language governing
# permissions and limitations under the License.
import sys
import amazon.ion.simpleion as ion
import ionhash
if len(sys.argv) < 3:
print("Utility that prints the Ion Hash of the top-level values in a file.")
print()
print("Usage:")
print(" ion-hash [algorithm] [filename]")
print()
print("where [algorithm] is a hash function such as sha256")
print()
sys.exit()
algorithm = sys.argv[1]
input_file = sys.argv[2]
with open(input_file, 'rb') as f:
values = ion.load(f, single_value=False)
for value in values:
try:
print(''.join('%02x ' % x for x in value.ion_hash(algorithm))[0:-1])
except Exception as e:
print('[unable to digest: ' + str(e) + ']')
def bad():
with _open(file) as vector:
with raises((IonException, ValueError, TypeError)):
load(vector, single_value=False)
def preprocess(element):
if is_embedded:
assert isinstance(element, six.text_type)
element = load(six.StringIO(element), single_value=False)
return element
def req_ion(name):
try:
return load(StringIO(req_str(name)))
except IonException as e:
raise BadRequest(
"Problem parsing the field " + name + " as ION " + str(e) + ".")
def compare_two_files(first_file, second_file):
first = simpleion.load(FileIO(first_file))
second = simpleion.load(FileIO(second_file))
return ion_equals(first, second)
def run():
with open("data.ion", 'rb') as f:
data = simpleion.load(f, single_value=False)
print(data)
def _simple_loads(data, *args, **kw):
buf = BytesIO()
buf.write(data)
buf.seek(0)
return load(buf, *args, **kw)
def loads(ion_str):
try:
return load(StringIO(ion_str))
except IonException as e:
raise BadRequest("Problem with ION: " + str(e))
def test_single_value_with_stream_fails(is_binary):
out = BytesIO()
dump(['foo', 123], out, binary=is_binary, sequence_as_stream=True)
out.seek(0)
with raises(IonException):
load(out, single_value=True)
def _simple_loads(data, *args, **kw):
buf = BytesIO()
buf.write(data)
buf.seek(0)
return load(buf, *args, **kw)