def __getitem__( self, key ):
try:
# vvvv -- Process an EtherNet/IP CIP Read [Tag [Fragmented]].
#
# We'll just access the (previously written) saved data here, and output the read
# request (and the value returned) to our time-series history file.
#
value = super( Attribute_historize, self ).__getitem__( key )
self.__logger.write( { 'read': value }, serial=(self.name, (
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
)))
# ^^^^
return value
except Exception as exc:
# vvvv -- Process an EtherNet/IP CIP Read [Tag [Fragmented]] Exception.
#
# Something went wrong with the Read request processing. Log something intelligent and
# re-raise the exception, to return a failure to the EtherNet/IP client.
#
self.__logger.comment(
"%s: PLC I/O Read Tag %20s[%5s-%-5s] Exception: %s" % (
history.timestamp(), self.name,
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
exc ))
# ^^^^
raise
def __setitem__( self, key, value ):
try:
# vvvv -- Process an EtherNet/IP CIP Write [Tag [Fragmented]].
#
# We'll just store the value, and output the write request (and the value written) to
# our time-series history file.
#
super( Attribute_historize, self ).__setitem__( key, value )
self.__logger.write( { 'write': value }, serial=(self.name, (
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
)))
# ^^^^
except Exception as exc:
# vvvv -- Process an EtherNet/IP CIP Write [Tag [Fragmented]] Exception.
#
# Something went wrong with the Write request processing. Log something intelligent and
# re-raise the exception, to return a failure to the EtherNet/IP client.
#
self.__logger.comment(
"%s: PLC I/O Write Tag %20s[%5s-%-5s] Exception: %s" % (
history.timestamp(), self.name,
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
exc ))
# ^^^^
raise
def __getitem__( self, key ):
try:
# vvvv -- Process an EtherNet/IP CIP Read [Tag [Fragmented]].
#
# We'll just access the (previously written) saved data here, and output the read
# request (and the value returned) to our time-series history file.
#
value = super( Attribute_historize, self ).__getitem__( key )
self.__logger.write( { 'read': value }, serial=(self.name, (
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
)))
# ^^^^
return value
except Exception as exc:
# vvvv -- Process an EtherNet/IP CIP Read [Tag [Fragmented]] Exception.
#
# Something went wrong with the Read request processing. Log something intelligent and
# re-raise the exception, to return a failure to the EtherNet/IP client.
#
self.__logger.comment(
"%s: PLC I/O Read Tag %20s[%5s-%-5s] Exception: %s" % (
history.timestamp(), self.name,
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
exc ))
# ^^^^
raise
def __setitem__( self, key, value ):
try:
# vvvv -- Process an EtherNet/IP CIP Write [Tag [Fragmented]].
#
# We'll just store the value, and output the write request (and the value written) to
# our time-series history file.
#
super( Attribute_historize, self ).__setitem__( key, value )
self.__logger.write( { 'write': value }, serial=(self.name, (
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
)))
# ^^^^
except Exception as exc:
# vvvv -- Process an EtherNet/IP CIP Write [Tag [Fragmented]] Exception.
#
# Something went wrong with the Write request processing. Log something intelligent and
# re-raise the exception, to return a failure to the EtherNet/IP client.
#
self.__logger.comment(
"%s: PLC I/O Write Tag %20s[%5s-%-5s] Exception: %s" % (
history.timestamp(), self.name,
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
exc ))
# ^^^^
raise
def __setitem__( self, key, value ):
try:
super( Attribute_historize, self ).__setitem__( key, value )
self.__logger.write( { 'write': value }, serial=(self.name, (
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
)))
except Exception as exc:
self.__logger.comment(
"%s: PLC I/O Write Tag %20s[%5s-%-5s] Exception: %s" % (
history.timestamp(), self.name,
key.indices( len( self ))[0] if isinstance( key, slice ) else key,
key.indices( len( self ))[1]-1 if isinstance( key, slice ) else key,
exc ))
raise
def test_history_timestamp_abbreviations():
"""Test timezone abbreviation support. """
abbrev = timestamp.support_abbreviations( 'CA', reset=True )
assert sorted( abbrev ) == ['ADT', 'AST', 'CDT', 'CST', 'EDT', 'EST', 'MDT', 'MST', 'NDT', 'NST', 'PDT', 'PST']
# Perform all the remaining timezone abbreviation tests relative to a known range of times, to
# avoid differences in the future due to timezone changes.
ts = timestamp( "2014-04-24 08:00:00 MDT" )
assert near( ts.value, 1398348000.0 )
try:
abbrev = timestamp.support_abbreviations( 'America' )
assert False, "Many zones should have been ambiguously abbreviated"
except AmbiguousTimeZoneError as exc:
assert "America/Mazatlan" in str( exc )
abbrev = timestamp.support_abbreviations( 'America',
exclude=['America/Mazatlan', 'America/Merida', 'America/Mexico_City', 'America/Monterrey',
'America/Bahia_Banderas', 'America/Cancun', 'America/Chihuahua', 'America/Havana',
'America/Santa_Isabel', 'America/Grand_Turk'] )
#print( sorted( abbrev ))
#print( reprlib.repr( timestamp._tzabbrev ))
assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HADT', 'HAST',
'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
abbrev = timestamp.support_abbreviations( 'Europe/Berlin' )
assert sorted( abbrev ) == ['CEST', 'CET']
assert 'CEST' in timestamp._tzabbrev
assert 'EEST' not in timestamp._tzabbrev
abbrev = timestamp.support_abbreviations( 'Europe', exclude=[ 'Europe/Simferopol', 'Europe/Istanbul', 'Europe/Minsk' ] )
#print( sorted( abbrev ))
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'FET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']
assert 'EEST' in timestamp._tzabbrev
try:
timestamp.support_abbreviations( 'Asia' )
assert False, "Asia/Jerusalem IST should have mismatched Europe/Dublin IST"
except AmbiguousTimeZoneError as exc:
assert "Asia/Jerusalem" in str( exc )
assert near( parse_offset( '< 1:00:00.001' ), -3600.001 )
assert near( parse_offset( '<:1.001' ), -1.001 )
assert near( parse_offset( '>1:0.001' ), 60.001 )
assert near( parse_offset( '>1' ), 1 )
# While Asia is internally very inconsistent (eg. EEST), countries should be internally consisent
abbrev = timestamp.support_abbreviations( 'JO', reset=True ) # Jordan
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'EEST', 'EET']
z,dst,off = timestamp._tzabbrev['EEST']
assert str(z) == 'Asia/Amman' and dst == True and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
abbrev = timestamp.support_abbreviations( 'IE', reset=True ) # Israel
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'GMT', 'IST' ]
# Jordan, Israel and Lebanon only work if we pick a region to exclude, for one EEST definition
abbrev = timestamp.support_abbreviations( ['JO', 'IE', 'LB'],
exclude=[ 'Asia/Amman' ], reset=True )
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'EEST', 'EET', 'GMT', 'IST' ]
z,dst,off = timestamp._tzabbrev['EEST']
assert str(z) == 'Asia/Beirut' and dst == True and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
# Australia zones incompatible with a bunch of other timezone abbreviations, eg. CST; reset
abbrev = timestamp.support_abbreviations( 'Australia', reset=True )
#print( sorted( abbrev ))
#print( repr( timestamp._tzabbrev ))
assert sorted( abbrev ) == ['ACDT', 'ACST', 'ACWST', 'AEDT', 'AEST', 'AWST', 'LHDT', 'LHST']
z,dst,off = timestamp._tzabbrev['LHST']
assert str(z) == 'Australia/Lord_Howe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == ">10:30:00"
# Ensure that non-ambiguous (DST-specific) zone abbreviations override ambiguous (no longer
# relevant, as pytz >= 2014.7 no longer contains dst == None for some of the Australian zones
# without DST)
abbrev = timestamp.support_abbreviations( [ 'Australia/Adelaide' ], reset=True )
#print( sorted( abbrev )) # ['ACDT', 'ACST']
z,dst,off = timestamp._tzabbrev['ACST']
assert str(z) == 'Australia/Adelaide' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
abbrev = timestamp.support_abbreviations( [ 'Australia/Adelaide', 'Australia/Darwin' ], reset=True )
#print( sorted( abbrev ))
#print( repr( timestamp._tzabbrev ))
z,dst,off = timestamp._tzabbrev['ACST']
assert str(z) in ( 'Australia/Darwin',
'Australia/Adelaide' ) and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
# Check that zones with complete, permanent offset changes (not just DST) are handled. We know
# that within a year of 2014-04-28, the America/Eirunepe (west Amazonas) zone had such a change.
abbrev = timestamp.support_abbreviations( [ 'America/Eirunepe' ], at=datetime.datetime( 2014, 4, 28 ))
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'ACT', 'AMT' ]
z,dst,off = timestamp._tzabbrev['ACT']
assert str(z) == 'America/Eirunepe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 5:00:00"
z,dst,off = timestamp._tzabbrev['AMT']
assert str(z) == 'America/Eirunepe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 4:00:00"
def test_history_timestamp():
"""Test timestamp, ensuring comparison deals in UTC only. Supports testing in local timezones:
Canada/Edmonton -- A generic, ambiguous DST/non-DST timezone
MST -- A DST-specific non-DST timezone
UTC -- UTC
"""
trtab = ( string
if sys.version_info[0] < 3
else str ).maketrans( ":-.", " " )
def utc_strp( loctime ):
if '.' in loctime:
unaware = datetime.datetime.strptime( loctime, timestamp._fmt + ".%f" )
else:
unaware = datetime.datetime.strptime( loctime, timestamp._fmt )
return pytz.utc.localize( unaware )
def utc_trns( loctime ):
terms = loctime.translate( trtab ).split()
if len( terms ) == 7:
# convert .123 into 123000 microseconds
terms[6] += '0' * ( 6 - len( terms[6] ))
return datetime.datetime( *map( int, terms ), tzinfo=pytz.utc )
# Basic millisecond hygiene. Comparisons are by standard UTC format to 3 sub-second decimal
# places of precision. Unfortunately, the Python 2/3 strftime microsecond formatters are
# different, so we don't use them. If no precision, we do NOT round; we truncate, to avoid the
# surprising effect of formatting a UNIX value manually using strftime produces a different
# second than formatting it using render() with no sub-second precision.
assert timestamp( 1399326141.999836 ) >= timestamp( 1399326141.374836 )
assert timestamp( 1399326141.999836 ).render( ms=False ) == '2014-05-05 21:42:21'
assert timestamp( 1399326141.999836 ).render( ms=5 ) == '2014-05-05 21:42:21.99984'
assert timestamp( 1399326141.999836 ).render() == '2014-05-05 21:42:22.000'
# Type caste support
assert abs( float( timestamp( 1399326141.999836 )) - 1399326141.999836 ) < 1e-6
assert int( timestamp( 1399326141.999836 )) == 1399326141
# Adjust timestamp default precision and comparison epsilon.
save = timestamp._precision,timestamp._epsilon
try:
ts = timestamp( 1399326141.999836 )
for p in range( 0, 7 ):
timestamp._precision= p
timestamp._epsilon = 10**-p if p else 0
assert ts.render( ms=True ) == {
0: '2014-05-05 21:42:21', # Truncates at 0 digits of sub-second precision
1: '2014-05-05 21:42:22.0',
2: '2014-05-05 21:42:22.00',
3: '2014-05-05 21:42:22.000',
4: '2014-05-05 21:42:21.9998',
5: '2014-05-05 21:42:21.99984',
6: '2014-05-05 21:42:21.999836',
}[timestamp._precision]
# For p == 0, try exact precision. 1e-6 is the smallest delta that can be reliably
# added to a typical UNIX timestamp (eg. 1399326141.999836) in a double and still
# expect it to affect the value (can store 15-17 decimal digits of precision).
s,l = (timestamp._epsilon*f for f in (0.9,1.1)) if p else (0,10**-6)
assert ts == ts + s
assert ts == ts - s
assert not(ts == ts + l)
assert not(ts == ts - l)
assert ts != ts + l
assert ts != ts - l
assert not(ts < ts + s)
assert not(ts < ts - s)
assert ts < ts + l
assert not(ts < ts - l)
assert ts <= ts + s
assert ts <= ts - s
assert ts <= ts + l
assert not(ts <= ts - l)
assert not(ts > ts + s)
assert not(ts > ts - s)
assert not(ts > ts + l)
assert ts > ts - l
assert ts >= ts + s
assert ts >= ts - s
assert not(ts >= ts + l)
assert ts >= ts - l
finally:
timestamp._precision,timestamp._epsilon = save
# Maintain DST specificity when rendering in DST-specific timezones? Nope, only when using
# specially constructed non-DST versions of timezones, when they are made available by pytz.
timestamp.support_abbreviations( None, reset=True )
assert timestamp.timezone_info('MST') == (pytz.timezone( 'MST' ),None)
assert timestamp( 1399326141.999836 ).render(
tzinfo='MST', ms=False ) == '2014-05-05 14:42:21 MST'
# Get MST/MDT etc., and CET/CEST abbreviations
timestamp.support_abbreviations( ['CA','Europe/Berlin'], reset=True )
assert timestamp.timezone_info('MST') == (pytz.timezone( 'America/Edmonton' ),False)
assert timestamp( 1399326141.999836 ).render(
tzinfo='MST', ms=False ) == '2014-05-05 15:42:21 MDT'
# $ TZ=UTC date --date=@1388559600
# Wed Jan 1 07:00:00 UTC 2014
# 1396531199
# Thu Apr 3 07:19:59 MDT 2014
assert '2014-01-02 03:04:55.123'.translate( trtab ) == '2014 01 02 03 04 55 123'
cnt = 10000
beg = timer()
for _ in range( cnt ):
utc1 = utc_strp( '2014-01-02 03:04:55.123' )
dur1 = timer() - beg
beg = timer()
for _ in range( cnt ):
utc2 = utc_trns( '2014-01-02 03:04:55.123' )
dur2 = timer() - beg
beg = timer()
for _ in range( cnt ):
utc3 = timestamp.datetime_from_string( '2014-01-02 03:04:55.123' )
dur3 = timer() - beg
assert utc1.strftime( timestamp._fmt ) \
== utc2.strftime( timestamp._fmt ) \
== utc3.strftime( timestamp._fmt ) == '2014-01-02 03:04:55'
logging.detail( "strptime: %d/s, translate: %d/s, timestamp: %d/s", cnt/dur1, cnt/dur2, cnt/dur3 )
now = timer()
assert timestamp( now ) < timestamp( now + 1 )
# From a numeric timestamp
ts = timestamp( 1396531199 )
assert ts.utc == '2014-04-03 13:19:59.000' == str( ts )
assert ts.local in ( '2014-04-03 07:19:59 MDT',
'2014-04-03 06:19:59 MST',
'2014-04-03 13:19:59 UTC' )
# From a string UTC time
dt = timestamp.datetime_from_string( '2014-01-01 07:00:00.0' )
assert str( dt ) == '2014-01-01 07:00:00+00:00'
assert repr( dt ) == 'datetime.datetime(2014, 1, 1, 7, 0, tzinfo=<UTC>)'
#assert dt.strftime( '%s' ) != '1388559600' # !? (will fail if machine is in UTC timezone )
#assert pytz.utc.normalize( dt ).strftime( '%s' ) != '1388559600' # !?
assert 1388559559.999999 < timestamp.number_from_datetime( dt ) < 1388559600.000001 # ok
ts = timestamp( '2014-01-01 07:00:00.0' )
assert 1388559559.999999 < ts.value < 1388559600.000001
assert ts.utc == '2014-01-01 07:00:00.000' == str( ts )
assert ts.local in ( '2014-01-01 00:00:00 MST',
'2014-01-01 07:00:00 UTC' )
# OK, now try a UTC time where the local timezone is in MDT
ts.utc = '2014-04-01 07:00:00.000'
assert ts.local in ( '2014-04-01 01:00:00 MDT',
'2014-04-01 00:00:00 MST',
'2014-04-01 07:00:00 UTC' )
# Make sure that local times are unambiguous over daylight savings time
# Mar 9 02:00 -> 03:00 1394355540 == Mar 9 2014 01:59
# Nov 2 02:00 -> 01:00 1414915140 == Nov 2 2014 01:59
ts = timestamp( 1394355540 )
assert ts.local in ( '2014-03-09 01:59:00 MST',
'2014-03-09 08:59:00 UTC' )
ts += 61
assert ts.local in ( '2014-03-09 03:00:01 MDT',
'2014-03-09 02:00:01 MST',
'2014-03-09 09:00:01 UTC' )
ts = timestamp( 1414915140 )
assert ts.local in ( '2014-11-02 01:59:00 MDT',
'2014-11-02 00:59:00 MST',
'2014-11-02 07:59:00 UTC' )
ts += 61
assert ts.local in ( '2014-11-02 01:00:01 MST',
'2014-03-09 02:00:01 MST',
'2014-11-02 08:00:01 UTC' )
# Now try converting a few strings that have a specific timezone. We can use either .utc =
# ... or .local = ...; they just default to the UTC or (local) timezone, respectively. Using a
# DST-specific timezone such as MST/MDT, we can unambiguously specify whether a time is inside
# or outside DST.
try:
ts.local = '2014-03-09 02:00:01 America/Edmonton' # Just inside MDT 2014
assert False, """Should have failed -- time doesn't exist during "spring ahead" """
except Exception as exc:
assert "NonExistentTimeError" in str( exc )
ts.local = '2014-03-09 03:00:01 MDT' # Just inside MDT 2014
assert 1394355600.999999 < ts.value < 1394355601.000001
assert ts.utc == '2014-03-09 09:00:01.000' # MDT == UCT-6:00
assert ts.local in ( '2014-03-09 03:00:01 MDT',
'2014-03-09 02:00:01 MST',
'2014-03-09 09:00:01 UTC' )
# However, we CAN use a specifically non-DST timezone to specify times non-existent in DST
ts.local = '2014-03-09 02:00:01 MST' # No such time in MDT!!
assert 1394355600.999999 < ts.value < 1394355601.000001
assert ts.utc == '2014-03-09 09:00:01.000'
assert ts.local in ( '2014-03-09 03:00:01 MDT',
'2014-03-09 02:00:01 MST',
'2014-03-09 09:00:01 UTC' )
ts.local = '2014-11-02 01:00:01 MST' # 1 second after the end of DST
assert 1414915200.999999 < ts.value < 1414915201.000001
assert ts.utc == '2014-11-02 08:00:01.000'
assert ts.local in ( '2014-11-02 01:00:01 MST',
'2014-11-02 00:59:59 MST',
'2014-11-02 08:00:01 UTC' )
ts -= 2 # Go back 2 seconds, into DST
assert ts.utc == '2014-11-02 07:59:59.000'
assert ts.local in ( '2014-11-02 01:59:59 MDT',
'2014-11-02 00:59:59 MST',
'2014-11-02 07:59:59 UTC' )
ts.local = '2014-11-02 01:59:58 MDT' # 2 seconds before end of DST
assert 1414915197.999999 < ts.value < 1414915198.000001
assert ts.utc == '2014-11-02 07:59:58.000'
assert ts.local in ( '2014-11-02 01:59:58 MDT',
'2014-11-02 00:59:58 MST',
'2014-11-02 07:59:58 UTC' )
# Using a canonical timezone such as 'America/Edmonton', an "ambiguous" time (eg. during the
# overlap in the fall) cannot be specified. Using a DST-specific timezone, we can.
try:
ts.local = '2014-11-02 01:00:01 America/Edmonton' # Inside DST?
except Exception as exc:
assert "AmbiguousTimeError" in str( exc )
ts.local = '2014-11-02 00:59:59 America/Edmonton' # 2 seconds before end of DST
assert 1414911598.999999 < ts.value < 1414911599.000001
assert ts.utc == '2014-11-02 06:59:59.000'
assert ts.local in ( '2014-11-02 00:59:59 MDT',
'2014-11-01 23:59:59 MST',
'2014-11-02 06:59:59 UTC' )
after = timestamp( '2014-11-02 01:02:03.123 MST' ) # (Nov 2 2014 -- 1:02 *after* DST ended)
before = timestamp( '2014-11-02 01:02:03.456 MDT' ) # (Nov 2 2014 -- :58 *before* DST ends)
assert before < after
assert before.utc == '2014-11-02 07:02:03.456'
assert before.local in ( '2014-11-02 01:02:03 MDT',
'2014-11-02 00:02:03 MST',
'2014-11-02 07:02:03 UTC' )
assert after.utc == '2014-11-02 08:02:03.123'
assert after.local in ( '2014-11-02 01:02:03 MST',
'2014-11-02 08:02:03 UTC' )
after = timestamp( '2014-10-26 02:01:00.123 CET' ) # (Nov 26 2014 -- 1:02 *after* DST ended)
before = timestamp( '2014-10-26 02:01:00.456 CEST' ) # (Nov 26 2014 -- :58 *before* DST ends)
assert before < after
assert before.utc == '2014-10-26 00:01:00.456'
assert before.local in ( '2014-10-25 18:01:00 MDT',
'2014-10-25 17:01:00 MST',
'2014-10-26 00:01:00 UTC' )
assert after.utc == '2014-10-26 01:01:00.123'
assert after.local in ( '2014-10-25 19:01:00 MDT',
'2014-10-25 18:01:00 MST',
'2014-10-26 01:01:00 UTC' )
def __init__( self, *args, **kwds ):
super( Attribute_historize, self ).__init__( *args, **kwds )
self.__logger.comment( "%s: Started recording Tag: %s" % ( history.timestamp(), self.name ))
1: logging.NORMAL,
2: logging.DETAIL,
3: logging.INFO,
4: logging.DEBUG,
}
cpppo.log_cfg['level'] = ( levelmap[args.verbose]
if args.verbose in levelmap
else logging.DEBUG )
if args.log:
cpppo.log_cfg['filename'] = args.log
logging.basicConfig( **cpppo.log_cfg )
def attribute_operations( paths ):
for op in client.parse_operations( paths ):
if 'attribute' in op['path'][-1]:
op['method'] = 'get_attribute_single'
else:
op['method'] = 'get_attributes_all'
yield op
with client.connector( host=args.address, timeout=timeout ) as conn:
idx = -1
start = cpppo.timer()
for idx,dsc,op,rpy,sts,val in conn.pipeline(
operations=attribute_operations( tags ), depth=depth,
multiple=False, timeout=timeout ):
print( "%s: %3d: %s == %s" % ( timestamp(), idx, dsc, val ))
elapsed = cpppo.timer() - start
logging.normal( "%3d requests in %7.2fs at pipeline depth %2s; %5.1f TPS" % (
idx+1, elapsed, args.depth, idx / elapsed ))
def test_history_unparsable():
"""Test history files rendered unparsable due to dropouts. This should be handled with no problem
except if the initial frame of register data on the first file is missing.
"""
for _ in range( 3 ):
path = "/tmp/test_unparsable_%d" % random.randint( 100000, 999999 )
if os.path.exists( path ):
continue
assert not os.path.exists( path ), "Couldn't find an unused name: %s" % path
files = []
try:
# Create a series of history files with decreasing timestamps as the numeric extension
# increases, containing records that are invalid.
now = timer()
v = 10000
secs = 10
secs_ext = 1.0 # adjust range of history to target out by this +/-
basisext = .5 # adjust start basis time from now by this +/-
minfactor = .25
maxfactor = 2.0
maxlatency = .25
# 1/N file lines corrupted (kills 2 records; the current and following). 0 --> no errors
maxerror = random.choice( [ None, 3, 10, 100 ] )
oldest = None
newest = None
logging.normal( "Corrupting %s of all history lines", None if not maxerror else "1/%d" % maxerror )
for e in range( secs ):
f = path + (( '.%d' % e ) if e else '') # 0'th file has no extension
files.append( f )
with logger( f ) as l:
ssend = 100
for ss in range( 0, ssend ): # subseconds up to but not including ssend...
js = json.dumps( { 40001: v + e * 1000 + (ss * 1000 // ssend) } ) + '\n'
if maxerror and not random.randint( 0, maxerror ):
# Truncate some of the records (as would occur in a filesystem full or halt)
js = js[:random.randint( 0, len( js ) - 1)]
ts = timestamp( now - e + ss/ssend )
if oldest is None or ts < oldest:
oldest = ts
if newest is None or ts > newest:
newest = ts
l._append( '\t'.join( (str( ts ),json.dumps( None ),js) ) )
# Load the historical records. This will be robust against all errors except if the first
# line of the first history file opened is corrupt, and we therefore cannot get the initial
# frame of register data.
historical = timestamp( now - random.uniform( -secs_ext, secs + secs_ext ))
basisdelay = random.uniform( -basisext, +basisext )
basis = now + basisdelay
factor = random.uniform( minfactor, maxfactor )
lookahead = 1.0
on_bad_iframe = random.choice( (loader.RAISE, loader.FAIL, loader.SUPPRESS, loader.SUPPRESS, loader.SUPPRESS) )
on_bad_data = random.choice( (loader.RAISE, loader.FAIL, loader.SUPPRESS, loader.SUPPRESS, loader.SUPPRESS) )
logging.normal( "Playback starts %s (%.1f%%) of history %s-%s, in %.3fs, at x %.2f rate w/%.1fs lookahead, on_bad_iframe=%s, on_bad_data=%s",
historical, ( historical.value - oldest.value ) * 100 / ( newest.value - oldest.value ),
oldest, newest, basisdelay, factor, lookahead,
"SUPPRESS" if on_bad_iframe == loader.SUPPRESS else "FAIL" if on_bad_iframe == loader.FAIL else "RAISE",
"SUPPRESS" if on_bad_data == loader.SUPPRESS else "FAIL" if on_bad_data == loader.FAIL else "RAISE" )
ld = loader( path,
historical=historical, basis=basis, factor=factor, lookahead=lookahead )
dur = basisext + ( secs_ext + secs + secs_ext ) / factor + basisext + 2*maxlatency # Don't be tooo strict
beg = timer()
count = 0
while ld:
assert timer() - beg < dur, "The loader should have ended"
cur,events = ld.load( on_bad_iframe=on_bad_iframe, on_bad_data=on_bad_data )
count += len( events )
logging.normal( "%s loaded up to %s; %d future, %d values: %d events: %s",
ld, cur, len( ld.future ), len( ld.values ), len( events ),
repr( events ) if logging.root.isEnabledFor( logging.DEBUG ) else reprlib.repr( events ))
time.sleep( random.uniform( 0.0, maxlatency ))
if on_bad_data == ld.FAIL or on_bad_iframe == ld.FAIL:
assert ld.state in (ld.COMPLETE, ld.FAILED)
else:
assert ld.state == ld.COMPLETE
except IframeError as exc:
logging.warning( "Detected error on initial frame of registers in first history file; failure expected: %s", exc )
assert ld.state == ld.FAILED and count == 0, "Shouldn't have loaded any events -- only iframe failures expected"
except DataError as exc:
logging.warning( "Detected error on registers data in a history file; failure expected: %s", exc )
assert ld.state == ld.FAILED
except Exception as exc:
logging.normal( "Test failed: %s", exc )
for f in files:
if os.path.exists( f ):
logging.normal( "%s:\n %s", f, " ".join( l for l in open( f )))
else:
logging.warning( "%s: Couldn't find file", f )
raise
finally:
for f in files:
logging.detail( "unlinking %s", f )
try:
os.unlink( f )
except:
pass
def test_history_timestamp():
"""Test timestamp, ensuring comparison deals in UTC only. Supports testing in local timezones:
Canada/Edmonton -- A generic, ambiguous DST/non-DST timezone
MST -- A DST-specific non-DST timezone
UTC -- UTC
"""
trtab = ( string
if sys.version_info[0] < 3
else str ).maketrans( ":-.", " " )
def utc_strp( loctime ):
if '.' in loctime:
unaware = datetime.datetime.strptime( loctime, timestamp._fmt + ".%f" )
else:
unaware = datetime.datetime.strptime( loctime, timestamp._fmt )
return pytz.utc.localize( unaware )
def utc_trns( loctime ):
terms = loctime.translate( trtab ).split()
if len( terms ) == 7:
# convert .123 into 123000 microseconds
terms[6] += '0' * ( 6 - len( terms[6] ))
return datetime.datetime( *map( int, terms ), tzinfo=pytz.utc )
# Basic millisecond hygiene. Comparisons are by standard UTC format to 3 sub-second decimal
# places of precision. Unfortunately, the Python 2/3 strftime microsecond formatters are
# different, so we don't use them. If no precision, we do NOT round; we truncate, to avoid the
# surprising effect of formatting a UNIX value manually using strftime produces a different
# second than formatting it using render() with no sub-second precision.
assert timestamp( 1399326141.999836 ) >= timestamp( 1399326141.374836 )
assert timestamp( 1399326141.999836 ).render( ms=False ) == '2014-05-05 21:42:21'
assert timestamp( 1399326141.999836 ).render( ms=5 ) == '2014-05-05 21:42:21.99984'
assert timestamp( 1399326141.999836 ).render() == '2014-05-05 21:42:22.000'
# Type caste support
assert abs( float( timestamp( 1399326141.999836 )) - 1399326141.999836 ) < 1e-6
assert int( timestamp( 1399326141.999836 )) == 1399326141
# Adjust timestamp default precision and comparison epsilon.
save = timestamp._precision,timestamp._epsilon
try:
ts = timestamp( 1399326141.999836 )
for p in range( 0, 7 ):
timestamp._precision= p
timestamp._epsilon = 10**-p if p else 0
assert ts.render( ms=True ) == {
0: '2014-05-05 21:42:21', # Truncates at 0 digits of sub-second precision
1: '2014-05-05 21:42:22.0',
2: '2014-05-05 21:42:22.00',
3: '2014-05-05 21:42:22.000',
4: '2014-05-05 21:42:21.9998',
5: '2014-05-05 21:42:21.99984',
6: '2014-05-05 21:42:21.999836',
}[timestamp._precision]
# For p == 0, try exact precision. 1e-6 is the smallest delta that can be reliably
# added to a typical UNIX timestamp (eg. 1399326141.999836) in a double and still
# expect it to affect the value (can store 15-17 decimal digits of precision).
s,l = (timestamp._epsilon*f for f in (0.9,1.1)) if p else (0,10**-6)
assert ts == ts + s
assert ts == ts - s
assert not(ts == ts + l)
assert not(ts == ts - l)
assert ts != ts + l
assert ts != ts - l
assert not(ts < ts + s)
assert not(ts < ts - s)
assert ts < ts + l
assert not(ts < ts - l)
assert ts <= ts + s
assert ts <= ts - s
assert ts <= ts + l
assert not(ts <= ts - l)
assert not(ts > ts + s)
assert not(ts > ts - s)
assert not(ts > ts + l)
assert ts > ts - l
assert ts >= ts + s
assert ts >= ts - s
assert not(ts >= ts + l)
assert ts >= ts - l
finally:
timestamp._precision,timestamp._epsilon = save
# Maintain DST specificity when rendering in DST-specific timezones? Nope, only when using
# specially constructed non-DST versions of timezones, when they are made available by pytz.
timestamp.support_abbreviations( None, reset=True )
assert timestamp.timezone_info('MST') == (pytz.timezone( 'MST' ),None)
assert timestamp( 1399326141.999836 ).render(
tzinfo='MST', ms=False ) == '2014-05-05 14:42:21 MST'
# Get MST/MDT etc., and CET/CEST abbreviations
timestamp.support_abbreviations( ['CA','Europe/Berlin'], reset=True )
assert timestamp.timezone_info('MST') == (pytz.timezone( 'America/Edmonton' ),False)
assert timestamp( 1399326141.999836 ).render(
tzinfo='MST', ms=False ) == '2014-05-05 15:42:21 MDT'
# $ TZ=UTC date --date=@1388559600
# Wed Jan 1 07:00:00 UTC 2014
# 1396531199
# Thu Apr 3 07:19:59 MDT 2014
assert '2014-01-02 03:04:55.123'.translate( trtab ) == '2014 01 02 03 04 55 123'
cnt = 10000
beg = timer()
for _ in range( cnt ):
utc1 = utc_strp( '2014-01-02 03:04:55.123' )
dur1 = timer() - beg
beg = timer()
for _ in range( cnt ):
utc2 = utc_trns( '2014-01-02 03:04:55.123' )
dur2 = timer() - beg
beg = timer()
for _ in range( cnt ):
utc3 = timestamp.datetime_from_string( '2014-01-02 03:04:55.123' )
dur3 = timer() - beg
assert utc1.strftime( timestamp._fmt ) \
== utc2.strftime( timestamp._fmt ) \
== utc3.strftime( timestamp._fmt ) == '2014-01-02 03:04:55'
logging.detail( "strptime: %d/s, translate: %d/s, timestamp: %d/s", cnt/dur1, cnt/dur2, cnt/dur3 )
now = timer()
assert timestamp( now ) < timestamp( now + 1 )
# From a numeric timestamp
ts = timestamp( 1396531199 )
assert ts.utc == '2014-04-03 13:19:59.000' == str( ts )
assert ts.local in ( '2014-04-03 07:19:59 MDT',
'2014-04-03 06:19:59 MST',
'2014-04-03 13:19:59 UTC' )
# From a string UTC time
dt = timestamp.datetime_from_string( '2014-01-01 07:00:00.0' )
assert str( dt ) == '2014-01-01 07:00:00+00:00'
assert repr( dt ) == 'datetime.datetime(2014, 1, 1, 7, 0, tzinfo=<UTC>)'
#assert dt.strftime( '%s' ) != '1388559600' # !? (will fail if machine is in UTC timezone )
#assert pytz.utc.normalize( dt ).strftime( '%s' ) != '1388559600' # !?
assert 1388559559.999999 < timestamp.number_from_datetime( dt ) < 1388559600.000001 # ok
ts = timestamp( '2014-01-01 07:00:00.0' )
assert 1388559559.999999 < ts.value < 1388559600.000001
assert ts.utc == '2014-01-01 07:00:00.000' == str( ts )
assert ts.local in ( '2014-01-01 00:00:00 MST',
'2014-01-01 07:00:00 UTC' )
# OK, now try a UTC time where the local timezone is in MDT
ts.utc = '2014-04-01 07:00:00.000'
assert ts.local in ( '2014-04-01 01:00:00 MDT',
'2014-04-01 00:00:00 MST',
'2014-04-01 07:00:00 UTC' )
# Make sure that local times are unambiguous over daylight savings time
# Mar 9 02:00 -> 03:00 1394355540 == Mar 9 2014 01:59
# Nov 2 02:00 -> 01:00 1414915140 == Nov 2 2014 01:59
ts = timestamp( 1394355540 )
assert ts.local in ( '2014-03-09 01:59:00 MST',
'2014-03-09 08:59:00 UTC' )
ts += 61
assert ts.local in ( '2014-03-09 03:00:01 MDT',
'2014-03-09 02:00:01 MST',
'2014-03-09 09:00:01 UTC' )
ts = timestamp( 1414915140 )
assert ts.local in ( '2014-11-02 01:59:00 MDT',
'2014-11-02 00:59:00 MST',
'2014-11-02 07:59:00 UTC' )
ts += 61
assert ts.local in ( '2014-11-02 01:00:01 MST',
'2014-03-09 02:00:01 MST',
'2014-11-02 08:00:01 UTC' )
# Now try converting a few strings that have a specific timezone. We can use either .utc =
# ... or .local = ...; they just default to the UTC or (local) timezone, respectively. Using a
# DST-specific timezone such as MST/MDT, we can unambiguously specify whether a time is inside
# or outside DST.
try:
ts.local = '2014-03-09 02:00:01 America/Edmonton' # Just inside MDT 2014
assert False, """Should have failed -- time doesn't exist during "spring ahead" """
except Exception as exc:
assert "NonExistentTimeError" in str( exc )
ts.local = '2014-03-09 03:00:01 MDT' # Just inside MDT 2014
assert 1394355600.999999 < ts.value < 1394355601.000001
assert ts.utc == '2014-03-09 09:00:01.000' # MDT == UCT-6:00
assert ts.local in ( '2014-03-09 03:00:01 MDT',
'2014-03-09 02:00:01 MST',
'2014-03-09 09:00:01 UTC' )
# However, we CAN use a specifically non-DST timezone to specify times non-existent in DST
ts.local = '2014-03-09 02:00:01 MST' # No such time in MDT!!
assert 1394355600.999999 < ts.value < 1394355601.000001
assert ts.utc == '2014-03-09 09:00:01.000'
assert ts.local in ( '2014-03-09 03:00:01 MDT',
'2014-03-09 02:00:01 MST',
'2014-03-09 09:00:01 UTC' )
ts.local = '2014-11-02 01:00:01 MST' # 1 second after the end of DST
assert 1414915200.999999 < ts.value < 1414915201.000001
assert ts.utc == '2014-11-02 08:00:01.000'
assert ts.local in ( '2014-11-02 01:00:01 MST',
'2014-11-02 00:59:59 MST',
'2014-11-02 08:00:01 UTC' )
ts -= 2 # Go back 2 seconds, into DST
assert ts.utc == '2014-11-02 07:59:59.000'
assert ts.local in ( '2014-11-02 01:59:59 MDT',
'2014-11-02 00:59:59 MST',
'2014-11-02 07:59:59 UTC' )
ts.local = '2014-11-02 01:59:58 MDT' # 2 seconds before end of DST
assert 1414915197.999999 < ts.value < 1414915198.000001
assert ts.utc == '2014-11-02 07:59:58.000'
assert ts.local in ( '2014-11-02 01:59:58 MDT',
'2014-11-02 00:59:58 MST',
'2014-11-02 07:59:58 UTC' )
# Using a canonical timezone such as 'America/Edmonton', an "ambiguous" time (eg. during the
# overlap in the fall) cannot be specified. Using a DST-specific timezone, we can.
try:
ts.local = '2014-11-02 01:00:01 America/Edmonton' # Inside DST?
except Exception as exc:
assert "AmbiguousTimeError" in str( exc )
ts.local = '2014-11-02 00:59:59 America/Edmonton' # 2 seconds before end of DST
assert 1414911598.999999 < ts.value < 1414911599.000001
assert ts.utc == '2014-11-02 06:59:59.000'
assert ts.local in ( '2014-11-02 00:59:59 MDT',
'2014-11-01 23:59:59 MST',
'2014-11-02 06:59:59 UTC' )
after = timestamp( '2014-11-02 01:02:03.123 MST' ) # (Nov 2 2014 -- 1:02 *after* DST ended)
before = timestamp( '2014-11-02 01:02:03.456 MDT' ) # (Nov 2 2014 -- :58 *before* DST ends)
assert before < after
assert before.utc == '2014-11-02 07:02:03.456'
assert before.local in ( '2014-11-02 01:02:03 MDT',
'2014-11-02 00:02:03 MST',
'2014-11-02 07:02:03 UTC' )
assert after.utc == '2014-11-02 08:02:03.123'
assert after.local in ( '2014-11-02 01:02:03 MST',
'2014-11-02 08:02:03 UTC' )
after = timestamp( '2014-10-26 02:01:00.123 CET' ) # (Nov 26 2014 -- 1:02 *after* DST ended)
before = timestamp( '2014-10-26 02:01:00.456 CEST' ) # (Nov 26 2014 -- :58 *before* DST ends)
assert before < after
assert before.utc == '2014-10-26 00:01:00.456'
assert before.local in ( '2014-10-25 18:01:00 MDT',
'2014-10-25 17:01:00 MST',
'2014-10-26 00:01:00 UTC' )
assert after.utc == '2014-10-26 01:01:00.123'
assert after.local in ( '2014-10-25 19:01:00 MDT',
'2014-10-25 18:01:00 MST',
'2014-10-26 01:01:00 UTC' )
kwargs={
'via': via,
'cycle': cycle,
'process': process,
'failure': failure,
'params': params,
})
]
poller[-1].start()
# Monitor the values and failed containers (updated in another Thread)
try:
while True:
while values:
par, (tmr, val) = values.popitem()
print("%s: %-32s == %r" % (timestamp(tmr), par, val))
while failed:
tmr, exc = failed.pop(0)
print("%s: %s" % (timestamp(tmr), exc))
time.sleep(.1)
# Every 2 seconds-ish, fall thru and update 'Motor Velocity'
if random.randint(0, 2 * 10 - 1):
continue
try:
param = 'Motor Velocity = (REAL)%s' % (round(
random.uniform(0, 100), 3))
with via: # establish gateway, detects Exception (closing gateway)
val, = via.write(via.parameter_substitution(param),
checking=True)
print("%s: %-32s == %s" % (timestamp(), param, val))
except Exception as exc:
help="Hostname of target Controller")
ap.add_argument('-t',
'--timeout',
default=None,
help="I/O timeout seconds (default: None)")
ap.add_argument('tags', nargs='+', help="Tags to read/write")
args = ap.parse_args()
depth = int(args.depth)
multiple = int(args.multiple)
repeat = int(args.repeat)
operations = client.parse_operations(args.tags * repeat)
timeout = None
if args.timeout is not None:
timeout = float(args.timeout)
with client.connector(host=args.address, timeout=timeout) as conn:
start = cpppo.timer()
num, idx = -1, -1
for num, (idx, dsc, op, rpy, sts, val) in enumerate(
conn.pipeline(operations=operations,
depth=depth,
multiple=multiple,
timeout=timeout)):
print("%s: %3d: %s" % (timestamp(), idx, val))
elapsed = cpppo.timer() - start
print(
"%3d operations using %3d requests in %7.2fs at pipeline depth %2s; %5.1f TPS"
% (num + 1, idx + 1, elapsed, args.depth, num / elapsed))
def __init__( self, *args, **kwds ):
super( Attribute_historize, self ).__init__( *args, **kwds )
self.__logger.comment( "%s: Started recording Tag: %s" % ( history.timestamp(), self.name ))
def test_history_timestamp_abbreviations():
"""Test timezone abbreviation support. """
abbrev = timestamp.support_abbreviations("CA", reset=True)
assert sorted(abbrev) == ["ADT", "AST", "CDT", "CST", "EDT", "EST", "MDT", "MST", "NDT", "NST", "PDT", "PST"]
# Perform all the remaining timezone abbreviation tests relative to a known range of times, to
# avoid differences in the future due to timezone changes.
ts = timestamp("2014-04-24 08:00:00 MDT")
assert near(ts.value, 1398348000.0)
try:
abbrev = timestamp.support_abbreviations("America")
assert False, "Many zones should have been ambiguously abbreviated"
except AmbiguousTimeZoneError as exc:
assert "America/Mazatlan" in str(exc)
abbrev = timestamp.support_abbreviations(
"America",
exclude=[
"America/Mazatlan",
"America/Merida",
"America/Mexico_City",
"America/Monterrey",
"America/Bahia_Banderas",
"America/Cancun",
"America/Chihuahua",
"America/Havana",
"America/Santa_Isabel",
"America/Grand_Turk",
"America/Cayman",
],
)
# print( sorted( abbrev ))
# print( reprlib.repr( timestamp._tzabbrev ))
if tuple(map(int, pytz.__version__.split("."))) < (2015, 4):
logging.warning("pytz < 2015.4; HADT/HAST vs. HDT/HST")
assert sorted(abbrev) == [
"ACT",
"AKDT",
"AKST",
"AMST",
"AMT",
"ART",
"BOT",
"BRST",
"BRT",
"CLST",
"CLT",
"COT",
"ECT",
"EGST",
"EGT",
"FNT",
"GFT",
"GMT",
"GYT",
"HADT",
"HAST",
"PET",
"PMDT",
"PMST",
"PYST",
"PYT",
"SRT",
"UYST",
"UYT",
"VET",
"WGST",
"WGT",
]
else:
assert sorted(abbrev) == [
"ACT",
"AKDT",
"AKST",
"AMST",
"AMT",
"ART",
"BOT",
"BRST",
"BRT",
"CLST",
"CLT",
"COT",
"ECT",
"EGST",
"EGT",
"FNT",
"GFT",
"GMT",
"GYT",
"HDT",
"HST",
"PET",
"PMDT",
"PMST",
"PYST",
"PYT",
"SRT",
"UYST",
"UYT",
"VET",
"WGST",
"WGT",
]
abbrev = timestamp.support_abbreviations("Europe/Berlin")
assert sorted(abbrev) == ["CEST", "CET"]
assert "CEST" in timestamp._tzabbrev
assert "EEST" not in timestamp._tzabbrev
abbrev = timestamp.support_abbreviations(
"Europe", exclude=["Europe/Simferopol", "Europe/Istanbul", "Europe/Minsk", "Europe/Chisinau"]
)
# print( sorted( abbrev ))
assert sorted(abbrev) == ["BST", "EEST", "EET", "IST", "MSK", "SAMT", "WEST", "WET"]
assert "EEST" in timestamp._tzabbrev
try:
timestamp.support_abbreviations("Asia")
assert False, "Asia/Jerusalem IST should have mismatched Europe/Dublin IST"
except AmbiguousTimeZoneError as exc:
assert "Asia/Jerusalem" in str(exc)
assert near(parse_offset("< 1:00:00.001"), -3600.001)
assert near(parse_offset("<:1.001"), -1.001)
assert near(parse_offset(">1:0.001"), 60.001)
assert near(parse_offset(">1"), 1)
# While Asia is internally very inconsistent (eg. EEST), countries should be internally consisent
abbrev = timestamp.support_abbreviations("JO", reset=True) # Jordan
# print( sorted( abbrev ))
assert sorted(abbrev) == ["EEST", "EET"]
z, dst, off = timestamp._tzabbrev["EEST"]
assert (
str(z) == "Asia/Amman" and dst == True and format_offset(timedelta_total_seconds(off), ms=None) == "> 3:00:00"
)
abbrev = timestamp.support_abbreviations("IE", reset=True) # Israel
# print( sorted( abbrev ))
assert sorted(abbrev) == ["GMT", "IST"]
# Jordan, Israel and Lebanon only work if we pick a region to exclude, for one EEST definition
abbrev = timestamp.support_abbreviations(["JO", "IE", "LB"], exclude=["Asia/Amman"], reset=True)
# print( sorted( abbrev ))
assert sorted(abbrev) == ["EEST", "EET", "GMT", "IST"]
z, dst, off = timestamp._tzabbrev["EEST"]
assert (
str(z) == "Asia/Beirut" and dst == True and format_offset(timedelta_total_seconds(off), ms=None) == "> 3:00:00"
)
# Australia zones incompatible with a bunch of other timezone abbreviations, eg. CST; reset
abbrev = timestamp.support_abbreviations("Australia", reset=True)
# print( sorted( abbrev ))
# print( repr( timestamp._tzabbrev ))
assert sorted(abbrev) == ["ACDT", "ACST", "ACWST", "AEDT", "AEST", "AWST", "LHDT", "LHST"]
z, dst, off = timestamp._tzabbrev["LHST"]
assert (
str(z) == "Australia/Lord_Howe"
and dst == False
and format_offset(timedelta_total_seconds(off), ms=None) == ">10:30:00"
)
# Ensure that non-ambiguous (DST-specific) zone abbreviations override ambiguous (no longer
# relevant, as pytz >= 2014.7 no longer contains dst == None for some of the Australian zones
# without DST)
abbrev = timestamp.support_abbreviations(["Australia/Adelaide"], reset=True)
# print( sorted( abbrev )) # ['ACDT', 'ACST']
z, dst, off = timestamp._tzabbrev["ACST"]
assert (
str(z) == "Australia/Adelaide"
and dst == False
and format_offset(timedelta_total_seconds(off), ms=None) == "> 9:30:00"
)
abbrev = timestamp.support_abbreviations(["Australia/Adelaide", "Australia/Darwin"], reset=True)
# print( sorted( abbrev ))
# print( repr( timestamp._tzabbrev ))
z, dst, off = timestamp._tzabbrev["ACST"]
assert (
str(z) in ("Australia/Darwin", "Australia/Adelaide")
and dst == False
and format_offset(timedelta_total_seconds(off), ms=None) == "> 9:30:00"
)
# Check that zones with complete, permanent offset changes (not just DST) are handled. We know
# that within a year of 2014-04-28, the America/Eirunepe (west Amazonas) zone had such a change.
abbrev = timestamp.support_abbreviations(["America/Eirunepe"], at=datetime.datetime(2014, 4, 28))
# print( sorted( abbrev ))
assert sorted(abbrev) == ["ACT", "AMT"]
z, dst, off = timestamp._tzabbrev["ACT"]
assert (
str(z) == "America/Eirunepe"
and dst == False
and format_offset(timedelta_total_seconds(off), ms=None) == "< 5:00:00"
)
z, dst, off = timestamp._tzabbrev["AMT"]
assert (
str(z) == "America/Eirunepe"
and dst == False
and format_offset(timedelta_total_seconds(off), ms=None) == "< 4:00:00"
)
def test_history_unparsable():
"""Test history files rendered unparsable due to dropouts. This should be handled with no problem
except if the initial frame of register data on the first file is missing.
"""
for _ in range( 3 ):
path = "/tmp/test_unparsable_%d" % random.randint( 100000, 999999 )
if os.path.exists( path ):
continue
assert not os.path.exists( path ), "Couldn't find an unused name: %s" % path
files = []
try:
# Create a series of history files with decreasing timestamps as the numeric extension
# increases, containing records that are invalid.
now = timer()
v = 10000
secs = 10
secs_ext = 1.0 # adjust range of history to target out by this +/-
basisext = .5 # adjust start basis time from now by this +/-
minfactor = .25
maxfactor = 2.0
maxlatency = .25
# 1/N file lines corrupted (kills 2 records; the current and following). 0 --> no errors
maxerror = random.choice( [ None, 3, 10, 100 ] )
oldest = None
newest = None
logging.normal( "Corrupting %s of all history lines", None if not maxerror else "1/%d" % maxerror )
for e in range( secs ):
f = path + (( '.%d' % e ) if e else '') # 0'th file has no extension
files.append( f )
with logger( f ) as l:
ssend = 100
for ss in range( 0, ssend ): # subseconds up to but not including ssend...
js = json.dumps( { 40001: v + e * 1000 + (ss * 1000 // ssend) } ) + '\n'
if maxerror and not random.randint( 0, maxerror ):
# Truncate some of the records (as would occur in a filesystem full or halt)
js = js[:random.randint( 0, len( js ) - 1)]
ts = timestamp( now - e + ss/ssend )
if oldest is None or ts < oldest:
oldest = ts
if newest is None or ts > newest:
newest = ts
l._append( '\t'.join( (str( ts ),json.dumps( None ),js) ) )
# Load the historical records. This will be robust against all errors except if the first
# line of the first history file opened is corrupt, and we therefore cannot get the initial
# frame of register data.
historical = timestamp( now - random.uniform( -secs_ext, secs + secs_ext ))
basisdelay = random.uniform( -basisext, +basisext )
basis = now + basisdelay
factor = random.uniform( minfactor, maxfactor )
lookahead = 1.0
on_bad_iframe = random.choice( (loader.RAISE, loader.FAIL, loader.SUPPRESS, loader.SUPPRESS, loader.SUPPRESS) )
on_bad_data = random.choice( (loader.RAISE, loader.FAIL, loader.SUPPRESS, loader.SUPPRESS, loader.SUPPRESS) )
logging.normal( "Playback starts %s (%.1f%%) of history %s-%s, in %.3fs, at x %.2f rate w/%.1fs lookahead, on_bad_iframe=%s, on_bad_data=%s",
historical, ( historical.value - oldest.value ) * 100 / ( newest.value - oldest.value ),
oldest, newest, basisdelay, factor, lookahead,
"SUPPRESS" if on_bad_iframe == loader.SUPPRESS else "FAIL" if on_bad_iframe == loader.FAIL else "RAISE",
"SUPPRESS" if on_bad_data == loader.SUPPRESS else "FAIL" if on_bad_data == loader.FAIL else "RAISE" )
ld = loader( path,
historical=historical, basis=basis, factor=factor, lookahead=lookahead )
dur = basisext + ( secs_ext + secs + secs_ext ) / factor + basisext + 2*maxlatency # Don't be tooo strict
beg = timer()
count = 0
while ld:
assert timer() - beg < dur, "The loader should have ended"
cur,events = ld.load( on_bad_iframe=on_bad_iframe, on_bad_data=on_bad_data )
count += len( events )
logging.normal( "%s loaded up to %s; %d future, %d values: %d events: %s",
ld, cur, len( ld.future ), len( ld.values ), len( events ),
repr( events ) if logging.root.isEnabledFor( logging.DEBUG ) else reprlib.repr( events ))
time.sleep( random.uniform( 0.0, maxlatency ))
if on_bad_data == ld.FAIL or on_bad_iframe == ld.FAIL:
assert ld.state in (ld.COMPLETE, ld.FAILED)
else:
assert ld.state == ld.COMPLETE
except IframeError as exc:
logging.warning( "Detected error on initial frame of registers in first history file; failure expected: %s", exc )
assert ld.state == ld.FAILED and count == 0, "Shouldn't have loaded any events -- only iframe failures expected"
except DataError as exc:
logging.warning( "Detected error on registers data in a history file; failure expected: %s", exc )
assert ld.state == ld.FAILED
except Exception as exc:
logging.normal( "Test failed: %s", exc )
for f in files:
if os.path.exists( f ):
logging.normal( "%s:\n %s", f, " ".join( l for l in open( f )))
else:
logging.warning( "%s: Couldn't find file", f )
raise
finally:
for f in files:
logging.detail( "unlinking %s", f )
try:
os.unlink( f )
except:
pass
def test_history_performance():
try:
tracemalloc.start()
except:
pass
for _ in range( 3 ):
path = "/tmp/test_performance_%d" % random.randint( 100000, 999999 )
if os.path.exists( path ):
continue
assert not os.path.exists( path ), "Couldn't find an unused name: %s" % path
files = []
try:
day = 24*60*60
dur = 3*day # a few days worth of data
regstps = 0.0,5.0 # 0-5secs between updates
numfiles = dur//day+1 # ~1 file/day, but at least 2
values = {} # Initial register values
regscount = 1000 # Number of different registers
regschanged = 1,10 # From 1-25 registers per row
regsbase = 40001
start = timer()
now = beg = start - dur
linecnt = 0
for e in reversed( range( numfiles )):
f = path + (( '.%d' % e ) if e else '') # 0'th file has no extension
files.append( f )
with logger( f ) as l:
if values:
l.write( values, now=now ); linecnt += 1
while now < beg + len(files) * dur/numfiles:
lst = now
now += random.uniform( *regstps )
assert now >= lst
assert timestamp( now ) >= timestamp( lst ), "now: %s, timestamp(now): %s" % ( now, timestamp( now ))
updates = {}
for _ in range( random.randint( *regschanged )):
updates[random.randint( regsbase, regsbase + regscount - 1 )] = random.randint( 0, 1<<16 - 1 )
values.update( updates )
l.write( updates, now=now ); linecnt += 1
lst = now
now += random.uniform( *regstps )
assert now >= lst
assert timestamp( now ) >= timestamp( lst )
if e:
# Compress .1 onward using a random format; randomly delete origin uncompressed file
# so sometimes both files exist
if random.choice( (True, False, False, False) ):
continue # Don't make a compressed version of some files
fz = f + '.%s' % random.choice( ('gz', 'bz2', 'xz') )
files.append( fz )
with opener( fz, mode='wb' ) as fd:
fd.write( open( f, 'rb' ).read() )
if random.choice( (True, False, False) ):
continue # Don't remove some of the uncompressed files
os.unlink( f )
files.pop( files.index( f ))
logging.warning( "Generated data in %.3fs; lines: %d", timer() - start, linecnt )
# Start somewhere within 0-1% the dur of the beg, forcing the load the look back to
# find the first file. Try to do it all in the next 'playback' second (just to push it to
# the max), in 'chunks' pieces.
historical = timestamp( random.uniform( beg + dur*0/100, beg + dur*1/100 ))
basis = timer()
playback = 2.0 * dur/day # Can sustain ~2 seconds / day of history on a single CPU
chunks = 1000
factor = dur / playback
lookahead = 60.0
duration = None
if random.choice( (True,False) ):
duration = random.uniform( dur * 98/100, dur * 102/100 )
begoff = historical.value - beg
endoff = 0 if duration is None else (( historical.value + duration ) - ( beg + dur ))
logging.warning( "Playback starts at beginning %s %s, duration %s, ends at ending %s %s",
timestamp( beg ), format_offset( begoff, ms=False ),
None if duration is None else format_offset( duration, ms=False, symbols='-+' ),
timestamp( beg + dur ), format_offset( endoff, ms=False ))
ld = loader(
path, historical=historical, basis=basis, factor=factor, lookahead=lookahead, duration=duration )
eventcnt = 0
slept = 0
cur = None
while ld:
once = False
while ld.state < ld.AWAITING or not once:
once = True
upcoming = None
limit = random.randint( 0, 250 )
if random.choice( (True,False) ):
upcoming = ld.advance()
if random.choice( (True,False) ) and cur:
# ~25% of the time, provide an 'upcoming' timestamp that is between the
# current advancing historical time and the last load time.
upcoming-= random.uniform( 0, upcoming.value - cur.value )
cur,events = ld.load( upcoming=upcoming, limit=limit )
eventcnt += len( events )
advance = ld.advance()
offset = advance.value - cur.value
logging.detail( "%s loaded up to %s (%s w/ upcoming %14s); %4d future, %4d values: %4d events / %4d limit" ,
ld, cur, format_offset( offset ),
format_offset( upcoming.value - advance.value ) if upcoming is not None else None,
len( ld.future ), len( ld.values ), len( events ), limit )
logging.warning( "%s loaded up to %s; %3d future, %4d values: %6d events total",
ld, cur, len( ld.future ), len( ld.values ), eventcnt )
try:
snapshot = tracemalloc.take_snapshot()
display_top( snapshot, limit=10 )
except:
pass
time.sleep( playback/chunks )
slept += playback/chunks
elapsed = timer() - basis
eventtps = eventcnt // ( elapsed - slept )
logging.error( "Playback in %.3fs (slept %.3fs); events: %d ==> %d historical records/sec",
elapsed, slept, eventcnt, eventtps )
if not logging.getLogger().isEnabledFor( logging.NORMAL ):
# Ludicrously low threshold, to pass tests on very slow machines
assert eventtps >= 1000, \
"Historical event processing performance low: %d records/sec" % eventtps
try:
display_biggest_traceback()
except:
pass
except Exception as exc:
logging.normal( "Test failed: %s", exc )
'''
for f in files:
logging.normal( "%s:\n %s", f, " ".join( l for l in open( f )))
'''
raise
finally:
for f in files:
logging.detail( "unlinking %s", f )
try:
os.unlink( f )
except:
pass
if __name__ == "__main__":
ap = argparse.ArgumentParser()
ap.add_argument( '-d', '--depth', default=0, help="Pipelining depth" )
ap.add_argument( '-m', '--multiple', default=0, help="Multiple Service Packet size limit" )
ap.add_argument( '-r', '--repeat', default=1, help="Repeat requests this many times" )
ap.add_argument( '-a', '--address', default='localhost', help="Hostname of target Controller" )
ap.add_argument( '-t', '--timeout', default=None, help="I/O timeout seconds (default: None)" )
ap.add_argument( 'tags', nargs='+', help="Tags to read/write" )
args = ap.parse_args()
depth = int( args.depth )
multiple = int( args.multiple )
repeat = int( args.repeat )
operations = client.parse_operations( args.tags * repeat )
timeout = None
if args.timeout is not None:
timeout = float( args.timeout )
with client.connector( host=args.address, timeout=timeout ) as conn:
start = cpppo.timer()
num,idx = -1,-1
for num,(idx,dsc,op,rpy,sts,val) in enumerate( conn.pipeline(
operations=operations, depth=depth,
multiple=multiple, timeout=timeout )):
print( "%s: %3d: %s" % ( timestamp(), idx, val ))
elapsed = cpppo.timer() - start
print( "%3d operations using %3d requests in %7.2fs at pipeline depth %2s; %5.1f TPS" % (
num+1, idx+1, elapsed, args.depth, num / elapsed ))
def test_history_timestamp_abbreviations():
"""Test timezone abbreviation support. """
abbrev = timestamp.support_abbreviations( 'CA', reset=True )
assert sorted( abbrev ) == ['ADT', 'AST', 'CDT', 'CST', 'EDT', 'EST', 'MDT', 'MST', 'NDT', 'NST', 'PDT', 'PST']
# Perform all the remaining timezone abbreviation tests relative to a known range of times, to
# avoid differences in the future due to timezone changes.
ts = timestamp( "2014-04-24 08:00:00 MDT" )
assert near( ts.value, 1398348000.0 )
# Try to add all of the Americas to the CA abbreviations already supported; can't be done (too
# many inconsistencies)
try:
abbrev = timestamp.support_abbreviations( 'America' )
assert False, "Many zones should have been ambiguously abbreviated"
except AmbiguousTimeZoneError as exc:
assert "America/Mazatlan" in str( exc )
exclude = [
'America/Mazatlan', 'America/Merida', 'America/Mexico_City', 'America/Monterrey',
'America/Bahia_Banderas', 'America/Cancun', 'America/Chihuahua', 'America/Havana',
'America/Santa_Isabel', 'America/Grand_Turk', 'America/Cayman', 'America/Port-au-Prince',
'America/Metlakatla',
]
#print()
#print( "America, w/o %r" % ( exclude ))
abbrev = timestamp.support_abbreviations( 'America', exclude=exclude )
#print( sorted( abbrev ))
#print( reprlib.repr( timestamp._tzabbrev ))
pytz_version = tuple( map( int, pytz.__version__.split( '.' )))
if pytz_version < (2015,4):
logging.warning( "pytz < 2015.4; HADT/HAST vs. HDT/HST" )
assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HADT', 'HAST',
'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
elif pytz_version < (2015,7):
logging.warning( "pytz < 2015.7; had UYST" )
assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HDT', 'HST',
'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
elif pytz_version < (2017,2):
assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HDT', 'HST',
'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYT', 'VET', 'WGST', 'WGT']
else:
# As of pytz 2017.2, alot of these zones are now using time zones consistent with CA; only a few added.
assert sorted( abbrev ) == ['AKDT', 'AKST', 'GMT', 'HDT', 'HST']
# We *can* add Europe/Berlin
abbrev = timestamp.support_abbreviations( 'Europe/Berlin' )
assert sorted( abbrev ) == ['CEST', 'CET']
assert 'CEST' in timestamp._tzabbrev
assert 'EEST' not in timestamp._tzabbrev
# And all of Europe, w/o some troublesome time zones
exclude = [ 'Europe/Simferopol', 'Europe/Istanbul', 'Europe/Minsk', 'Europe/Chisinau', 'Europe/Dublin' ]
#print()
#print( "Europe, w/o %r" % ( exclude ))
abbrev = timestamp.support_abbreviations( 'Europe', exclude=exclude )
#print( sorted( abbrev ))
if pytz_version < (2015,2):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'MSK', 'SAMT', 'WEST', 'WET']
elif pytz_version < (2016,3):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']
elif pytz_version < (2016,7):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']
elif pytz_version < (2018,5):
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'WEST', 'WET']
else:
assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'MSK', 'WEST', 'WET']
assert 'EEST' in timestamp._tzabbrev
try:
timestamp.support_abbreviations( 'Asia' )
assert False, "Asia/Jerusalem IST should have mismatched Europe/Dublin IST"
except AmbiguousTimeZoneError as exc:
assert "Asia/Jerusalem" in str( exc )
assert near( parse_offset( '< 1:00:00.001' ), -3600.001 )
assert near( parse_offset( '<:1.001' ), -1.001 )
assert near( parse_offset( '>1:0.001' ), 60.001 )
assert near( parse_offset( '>1' ), 1 )
# While Asia is internally very inconsistent (eg. EEST), countries should be internally consisent
abbrev = timestamp.support_abbreviations( 'JO', reset=True ) # Jordan
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'EEST', 'EET']
z,dst,off = timestamp._tzabbrev['EEST']
assert str(z) == 'Asia/Amman' and dst == True and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
abbrev = timestamp.support_abbreviations( 'IE', reset=True ) # Israel
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'GMT', 'IST' ]
# Jordan, Israel and Lebanon only work if we pick a region to exclude, for one EEST definition
abbrev = timestamp.support_abbreviations( ['JO', 'IE', 'LB'],
exclude=[ 'Asia/Amman' ], reset=True )
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'EEST', 'EET', 'GMT', 'IST' ]
z,dst,off = timestamp._tzabbrev['EEST']
assert str(z) == 'Asia/Beirut' and dst == True and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
# Australia zones incompatible with a bunch of other timezone abbreviations, eg. CST; reset
abbrev = timestamp.support_abbreviations( 'Australia', reset=True )
#print( sorted( abbrev ))
#print( repr( timestamp._tzabbrev ))
if pytz_version < (2017,2):
assert sorted( abbrev ) == ['ACDT', 'ACST', 'ACWST', 'AEDT', 'AEST', 'AWST', 'LHDT', 'LHST']
z,dst,off = timestamp._tzabbrev['LHST']
assert str(z) == 'Australia/Lord_Howe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == ">10:30:00"
else:
assert sorted( abbrev ) == ['ACDT', 'ACST', 'AEDT', 'AEST', 'AWST']
# Ensure that non-ambiguous (DST-specific) zone abbreviations override ambiguous (no longer
# relevant, as pytz >= 2014.7 no longer contains dst == None for some of the Australian zones
# without DST)
abbrev = timestamp.support_abbreviations( [ 'Australia/Adelaide' ], reset=True )
assert sorted( abbrev ) == [ 'ACDT', 'ACST' ]
z,dst,off = timestamp._tzabbrev['ACST']
assert str(z) == 'Australia/Adelaide' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
abbrev = timestamp.support_abbreviations( [ 'Australia/Adelaide', 'Australia/Darwin' ], reset=True )
#print( sorted( abbrev ))
#print( repr( timestamp._tzabbrev ))
z,dst,off = timestamp._tzabbrev['ACST']
assert str(z) in ( 'Australia/Darwin',
'Australia/Adelaide' ) and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
# Check that zones with complete, permanent offset changes (not just DST) are handled. We know
# that within a year of 2014-04-28, the America/Eirunepe (west Amazonas) zone had such a change
# (pre pytz 2017.2, anyway...)
if pytz_version < (2017,2):
abbrev = timestamp.support_abbreviations( [ 'America/Eirunepe' ], at=datetime.datetime( 2014, 4, 28 ), reset=True)
#print( sorted( abbrev ))
assert sorted( abbrev ) == [ 'ACT', 'AMT' ]
z,dst,off = timestamp._tzabbrev['ACT']
assert str(z) == 'America/Eirunepe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 5:00:00"
z,dst,off = timestamp._tzabbrev['AMT']
assert str(z) == 'America/Eirunepe' and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 4:00:00"
values[p] = (cpppo.timer(),v)
process.done = False
poller = []
via = powerflex_750_series(
host=address[0], port=address[1], timeout=timeout )
for cycle,params in targets.items():
poller += [ threading.Thread( target=poll.run, kwargs={
'via': via,
'cycle': cycle,
'process': process,
'failure': failure,
'params': params,
})]
poller[-1].start()
# Monitor the values and failed containers (updated in another Thread)
try:
while True:
while values:
par,(tmr,val) = values.popitem()
print( "%s: %16s == %r" % ( timestamp( tmr ), par, val ))
while failed:
tmr,exc = failed.pop( 0 )
print( "%s: %s" %( timestamp( tmr ), exc ))
time.sleep( .1 )
finally:
process.done = True
for p in poller:
p.join()
def test_history_performance():
try:
tracemalloc.start()
except:
pass
for _ in range( 3 ):
path = "/tmp/test_performance_%d" % random.randint( 100000, 999999 )
if os.path.exists( path ):
continue
assert not os.path.exists( path ), "Couldn't find an unused name: %s" % path
files = []
try:
day = 24*60*60
dur = 3*day # a few days worth of data
regstps = 0.0,5.0 # 0-5secs between updates
numfiles = dur//day+1 # ~1 file/day, but at least 2
values = {} # Initial register values
regscount = 1000 # Number of different registers
regschanged = 1,10 # From 1-25 registers per row
regsbase = 40001
start = timer()
now = beg = start - dur
linecnt = 0
for e in reversed( range( numfiles )):
f = path + (( '.%d' % e ) if e else '') # 0'th file has no extension
files.append( f )
with logger( f ) as l:
if values:
l.write( values, now=now ); linecnt += 1
while now < beg + len(files) * dur/numfiles:
lst = now
now += random.uniform( *regstps )
assert now >= lst
assert timestamp( now ) >= timestamp( lst ), "now: %s, timestamp(now): %s" % ( now, timestamp( now ))
updates = {}
for _ in range( random.randint( *regschanged )):
updates[random.randint( regsbase, regsbase + regscount - 1 )] = random.randint( 0, 1<<16 - 1 )
values.update( updates )
l.write( updates, now=now ); linecnt += 1
lst = now
now += random.uniform( *regstps )
assert now >= lst
assert timestamp( now ) >= timestamp( lst )
if e:
# Compress .1 onward using a random format; randomly delete origin uncompressed file
# so sometimes both files exist
if random.choice( (True, False, False, False) ):
continue # Don't make a compressed version of some files
fz = f + '.%s' % random.choice( ('gz', 'bz2', 'xz') )
files.append( fz )
with opener( fz, mode='wb' ) as fd:
with open( f, 'rb' ) as rd:
fd.write( rd.read() )
if random.choice( (True, False, False) ):
continue # Don't remove some of the uncompressed files
os.unlink( f )
files.pop( files.index( f ))
logging.warning( "Generated data in %.3fs; lines: %d", timer() - start, linecnt )
# Start somewhere within 0-1% the dur of the beg, forcing the load the look back to
# find the first file. Try to do it all in the next 'playback' second (just to push it to
# the max), in 'chunks' pieces.
historical = timestamp( random.uniform( beg + dur*0/100, beg + dur*1/100 ))
basis = timer()
playback = 2.0 * dur/day # Can sustain ~2 seconds / day of history on a single CPU
chunks = 1000
factor = dur / playback
lookahead = 60.0
duration = None
if random.choice( (True,False) ):
duration = random.uniform( dur * 98/100, dur * 102/100 )
begoff = historical.value - beg
endoff = 0 if duration is None else (( historical.value + duration ) - ( beg + dur ))
logging.warning( "Playback starts at beginning %s %s, duration %s, ends at ending %s %s",
timestamp( beg ), format_offset( begoff, ms=False ),
None if duration is None else format_offset( duration, ms=False, symbols='-+' ),
timestamp( beg + dur ), format_offset( endoff, ms=False ))
ld = loader(
path, historical=historical, basis=basis, factor=factor, lookahead=lookahead, duration=duration )
eventcnt = 0
slept = 0
cur = None
while ld:
once = False
while ld.state < ld.AWAITING or not once:
once = True
upcoming = None
limit = random.randint( 0, 250 )
if random.choice( (True,False) ):
upcoming = ld.advance()
if random.choice( (True,False) ) and cur:
# ~25% of the time, provide an 'upcoming' timestamp that is between the
# current advancing historical time and the last load time.
upcoming-= random.uniform( 0, upcoming.value - cur.value )
cur,events = ld.load( upcoming=upcoming, limit=limit )
eventcnt += len( events )
advance = ld.advance()
offset = advance.value - cur.value
logging.detail( "%s loaded up to %s (%s w/ upcoming %14s); %4d future, %4d values: %4d events / %4d limit" ,
ld, cur, format_offset( offset ),
format_offset( upcoming.value - advance.value ) if upcoming is not None else None,
len( ld.future ), len( ld.values ), len( events ), limit )
logging.warning( "%s loaded up to %s; %3d future, %4d values: %6d events total",
ld, cur, len( ld.future ), len( ld.values ), eventcnt )
try:
snapshot = tracemalloc.take_snapshot()
display_top( snapshot, limit=10 )
except:
pass
time.sleep( playback/chunks )
slept += playback/chunks
elapsed = timer() - basis
eventtps = eventcnt // ( elapsed - slept )
logging.error( "Playback in %.3fs (slept %.3fs); events: %d ==> %d historical records/sec",
elapsed, slept, eventcnt, eventtps )
if not logging.getLogger().isEnabledFor( logging.NORMAL ):
# Ludicrously low threshold, to pass tests on very slow machines
assert eventtps >= 1000, \
"Historical event processing performance low: %d records/sec" % eventtps
try:
display_biggest_traceback()
except:
pass
except Exception as exc:
logging.normal( "Test failed: %s", exc )
'''
for f in files:
logging.normal( "%s:\n %s", f, " ".join( l for l in open( f )))
'''
raise
finally:
for f in files:
logging.detail( "unlinking %s", f )
try:
os.unlink( f )
except:
pass
for cycle,params in targets.items():
poller += [ threading.Thread( target=poll.run, kwargs={
'via': via,
'cycle': cycle,
'process': process,
'failure': failure,
'params': params,
})]
poller[-1].start()
# Monitor the values and failed containers (updated in another Thread)
try:
while True:
while values:
par,(tmr,val) = values.popitem()
print( "%s: %-32s == %r" % ( timestamp( tmr ), par, val ))
while failed:
tmr,exc = failed.pop( 0 )
print( "%s: %s" %( timestamp( tmr ), exc ))
time.sleep( .1 )
# Every 2 seconds-ish, fall thru and update 'Motor Velocity'
if random.randint( 0, 2*10-1 ):
continue
try:
param = 'Motor Velocity = (REAL)%s' % (
round( random.uniform( 0, 100 ), 3 ))
with via: # establish gateway, detects Exception (closing gateway)
val, = via.write(
via.parameter_substitution( param ), checking=True )
print( "%s: %-32s == %s" % ( timestamp(), param, val ))
except Exception as exc:
