コード例 #1
0
ファイル: history_test.py プロジェクト: srdgame/cpppo
def test_history_performance():
    try:
        tracemalloc.start()
    except:
        pass

    for _ in range( 3 ):
        path		= "/tmp/test_performance_%d" % random.randint( 100000, 999999 )
        if os.path.exists( path ):
            continue
    assert not os.path.exists( path ), "Couldn't find an unused name: %s" % path 

    files		= []
    try:
        day		= 24*60*60
        dur		= 3*day		# a few days worth of data
        regstps		= 0.0,5.0	# 0-5secs between updates
        numfiles	= dur//day+1	# ~1 file/day, but at least 2
        values		= {}		# Initial register values
        regscount	= 1000		# Number of different registers
        regschanged	= 1,10		# From 1-25 registers per row
        regsbase	= 40001

        start		= timer()

        now = beg	= start - dur
        linecnt		= 0
        for e in reversed( range( numfiles )):
            f		= path + (( '.%d' % e ) if e else '') # 0'th file has no extension
            files.append( f )
            with logger( f ) as l:
                if values:
                    l.write( values, now=now ); linecnt += 1
                while now < beg + len(files) * dur/numfiles:
                    lst	= now
                    now += random.uniform( *regstps )
                    assert now >= lst
                    assert timestamp( now ) >= timestamp( lst ), "now: %s, timestamp(now): %s" % ( now, timestamp( now ))
                    updates = {}
                    for _ in range( random.randint( *regschanged )):
                        updates[random.randint( regsbase, regsbase + regscount - 1 )] = random.randint( 0, 1<<16 - 1 )
                    values.update( updates )
                    l.write( updates, now=now ); linecnt += 1
                lst 	= now
                now    += random.uniform( *regstps )
                assert now >= lst
                assert timestamp( now ) >= timestamp( lst )
            if e:
                # Compress .1 onward using a random format; randomly delete origin uncompressed file
                # so sometimes both files exist
                if random.choice( (True, False, False, False) ):
                    continue # Don't make a compressed version of some files
                fz	 = f + '.%s' % random.choice( ('gz', 'bz2', 'xz') )
                files.append( fz )
                with opener( fz, mode='wb' ) as fd:
                    with open( f, 'rb' ) as rd:
                        fd.write( rd.read() )
                if random.choice( (True, False, False) ):
                    continue # Don't remove some of the uncompressed files
                os.unlink( f )
                files.pop( files.index( f ))

        logging.warning( "Generated data in %.3fs; lines: %d", timer() - start, linecnt )

        # Start somewhere within 0-1% the dur of the beg, forcing the load the look back to
        # find the first file.  Try to do it all in the next 'playback' second (just to push it to
        # the max), in 'chunks' pieces.
        historical	= timestamp( random.uniform( beg + dur*0/100, beg + dur*1/100 ))
        basis		= timer()
        playback	= 2.0 * dur/day # Can sustain ~2 seconds / day of history on a single CPU
        chunks		= 1000
        factor		= dur / playback
        lookahead	= 60.0
        duration	= None
        if random.choice( (True,False) ):
            duration	= random.uniform( dur * 98/100, dur * 102/100 )

        begoff		= historical.value - beg
        endoff		= 0 if duration is None else (( historical.value + duration ) - ( beg + dur ))
        logging.warning( "Playback starts at beginning %s %s, duration %s, ends at ending %s %s",
                         timestamp( beg ), format_offset( begoff, ms=False ),
                         None if duration is None else format_offset( duration, ms=False, symbols='-+' ),
                         timestamp( beg + dur ), format_offset( endoff, ms=False ))

        ld		= loader(
            path, historical=historical, basis=basis, factor=factor, lookahead=lookahead, duration=duration )
        eventcnt	= 0
        slept		= 0
        cur		= None
        while ld:
            once	= False
            while ld.state < ld.AWAITING or not once:
                once		= True
                upcoming	= None
                limit		= random.randint( 0, 250 )
                if random.choice( (True,False) ):
                    upcoming	= ld.advance()
                    if random.choice( (True,False) ) and cur:
                        # ~25% of the time, provide an 'upcoming' timestamp that is between the
                        # current advancing historical time and the last load time.
                        upcoming-= random.uniform( 0, upcoming.value - cur.value )
                cur,events	= ld.load( upcoming=upcoming, limit=limit )
                eventcnt       += len( events )
                advance		= ld.advance()
                offset		= advance.value - cur.value
                logging.detail( "%s loaded up to %s (%s w/ upcoming %14s); %4d future, %4d values: %4d events / %4d limit" ,
                                ld, cur, format_offset( offset ),
                                format_offset( upcoming.value - advance.value ) if upcoming is not None else None,
                                len( ld.future ), len( ld.values ), len( events ), limit )

            logging.warning( "%s loaded up to %s; %3d future, %4d values: %6d events total",
                                ld, cur, len( ld.future ), len( ld.values ), eventcnt )
            try:
                snapshot	= tracemalloc.take_snapshot()
                display_top( snapshot, limit=10 )
            except:
                pass

            time.sleep( playback/chunks )
            slept	       += playback/chunks

        elapsed		= timer() - basis
        eventtps	= eventcnt // ( elapsed - slept )
        logging.error( "Playback in %.3fs (slept %.3fs); events: %d ==> %d historical records/sec",
                       elapsed, slept, eventcnt, eventtps )
        if not logging.getLogger().isEnabledFor( logging.NORMAL ):
            # Ludicrously low threshold, to pass tests on very slow machines
            assert eventtps >= 1000, \
                "Historical event processing performance low: %d records/sec" % eventtps
        try:
            display_biggest_traceback()
        except:
            pass

    except Exception as exc:
        logging.normal( "Test failed: %s", exc )
        '''
        for f in files:
            logging.normal( "%s:\n    %s", f, "    ".join( l for l in open( f )))
        '''
        raise

    finally:
        for f in files:
            logging.detail( "unlinking %s", f )
            try:
                os.unlink( f )
            except:
                pass
コード例 #2
0
ファイル: history_test.py プロジェクト: srdgame/cpppo
def test_history_timestamp_abbreviations():
    """Test timezone abbreviation support. """
    abbrev			= timestamp.support_abbreviations( 'CA', reset=True )
    assert sorted( abbrev ) == ['ADT', 'AST', 'CDT', 'CST', 'EDT', 'EST', 'MDT', 'MST', 'NDT', 'NST', 'PDT', 'PST']

    # Perform all the remaining timezone abbreviation tests relative to a known range of times, to
    # avoid differences in the future due to timezone changes.
    ts				= timestamp( "2014-04-24 08:00:00 MDT" )
    assert near( ts.value, 1398348000.0 )

    # Try to add all of the Americas to the CA abbreviations already supported; can't be done (too
    # many inconsistencies)
    try:
        abbrev			= timestamp.support_abbreviations( 'America' )
        assert False, "Many zones should have been ambiguously abbreviated"
    except AmbiguousTimeZoneError as exc:
        assert "America/Mazatlan" in str( exc )

    exclude			= [
        'America/Mazatlan', 'America/Merida', 'America/Mexico_City', 'America/Monterrey',
        'America/Bahia_Banderas', 'America/Cancun', 'America/Chihuahua', 'America/Havana',
        'America/Santa_Isabel', 'America/Grand_Turk', 'America/Cayman', 'America/Port-au-Prince',
        'America/Metlakatla',
    ]
    #print()
    #print( "America, w/o %r" % ( exclude ))
    abbrev			= timestamp.support_abbreviations( 'America', exclude=exclude )
    #print( sorted( abbrev ))
    #print( reprlib.repr( timestamp._tzabbrev ))
    pytz_version		= tuple( map( int, pytz.__version__.split( '.' )))
    if pytz_version < (2015,4):
        logging.warning( "pytz < 2015.4; HADT/HAST vs. HDT/HST" )
        assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
                                    'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HADT', 'HAST',
                                    'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
    elif pytz_version < (2015,7):
        logging.warning( "pytz < 2015.7; had UYST" )
        assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
                                    'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HDT', 'HST',
                                    'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
    elif pytz_version < (2017,2):
        assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
                                    'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HDT', 'HST',
                                    'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYT', 'VET', 'WGST', 'WGT']
    else:
        # As of pytz 2017.2, alot of these zones are now using time zones consistent with CA; only a few added.
        assert sorted( abbrev ) == ['AKDT', 'AKST', 'GMT', 'HDT', 'HST']

    # We *can* add Europe/Berlin
    abbrev			= timestamp.support_abbreviations( 'Europe/Berlin' )
    assert sorted( abbrev ) == ['CEST', 'CET']
    assert 'CEST' in timestamp._tzabbrev
    assert 'EEST' not in timestamp._tzabbrev

    # And all of Europe, w/o some troublesome time zones
    exclude			= [ 'Europe/Simferopol', 'Europe/Istanbul', 'Europe/Minsk', 'Europe/Chisinau', 'Europe/Dublin' ]
    #print()
    #print( "Europe, w/o %r" % ( exclude ))
    abbrev			= timestamp.support_abbreviations( 'Europe', exclude=exclude )
    #print( sorted( abbrev ))
    if pytz_version < (2015,2):
        assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'MSK', 'SAMT', 'WEST', 'WET']
    elif pytz_version < (2016,3):
        assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']
    elif pytz_version < (2016,7):
        assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']
    elif pytz_version < (2018,5):
        assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'IST', 'MSK', 'WEST', 'WET']
    else:
        assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'MSK', 'WEST', 'WET']
        
    assert 'EEST' in timestamp._tzabbrev
    try:
        timestamp.support_abbreviations( 'Asia' )
        assert False, "Asia/Jerusalem IST should have mismatched Europe/Dublin IST"
    except AmbiguousTimeZoneError as exc:
        assert "Asia/Jerusalem" in str( exc )

    assert near( parse_offset( '< 1:00:00.001' ),	-3600.001 )
    assert near( parse_offset( '<:1.001' ), 		   -1.001 )
    assert near( parse_offset( '>1:0.001' ),		   60.001 )
    assert near( parse_offset( '>1' ), 			    1 )

    # While Asia is internally very inconsistent (eg. EEST), countries should be internally consisent
    abbrev			= timestamp.support_abbreviations( 'JO', reset=True ) # Jordan
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == [ 'EEST', 'EET']
    z,dst,off			= timestamp._tzabbrev['EEST']
    assert str(z) == 'Asia/Amman'	and dst == True  and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
    abbrev			= timestamp.support_abbreviations( 'IE', reset=True ) # Israel
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == [ 'GMT', 'IST' ]
    # Jordan, Israel and Lebanon only work if we pick a region to exclude, for one EEST definition
    abbrev			= timestamp.support_abbreviations( ['JO', 'IE', 'LB'],
                                                                   exclude=[ 'Asia/Amman' ], reset=True )
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == [ 'EEST', 'EET', 'GMT', 'IST' ]
    z,dst,off			= timestamp._tzabbrev['EEST']
    assert str(z) == 'Asia/Beirut'	and dst == True  and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"

    # Australia zones incompatible with a bunch of other timezone abbreviations, eg. CST; reset
    abbrev			= timestamp.support_abbreviations( 'Australia', reset=True )
    #print( sorted( abbrev ))
    #print( repr( timestamp._tzabbrev ))
    if pytz_version < (2017,2):
        assert sorted( abbrev ) == ['ACDT', 'ACST', 'ACWST', 'AEDT', 'AEST', 'AWST', 'LHDT', 'LHST']
        z,dst,off		= timestamp._tzabbrev['LHST']
        assert str(z) == 'Australia/Lord_Howe'	and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == ">10:30:00"
    else:
        assert sorted( abbrev ) ==  ['ACDT', 'ACST', 'AEDT', 'AEST', 'AWST']


    # Ensure that non-ambiguous (DST-specific) zone abbreviations override ambiguous (no longer
    # relevant, as pytz >= 2014.7 no longer contains dst == None for some of the Australian zones
    # without DST)
    abbrev			= timestamp.support_abbreviations( [ 'Australia/Adelaide' ], reset=True )
    assert sorted( abbrev ) == [ 'ACDT', 'ACST' ]
    z,dst,off			= timestamp._tzabbrev['ACST']
    assert str(z) == 'Australia/Adelaide'	and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
    abbrev			= timestamp.support_abbreviations( [ 'Australia/Adelaide', 'Australia/Darwin' ], reset=True )
    #print( sorted( abbrev ))
    #print( repr( timestamp._tzabbrev ))
    z,dst,off			= timestamp._tzabbrev['ACST']
    assert str(z) in ( 'Australia/Darwin',
                       'Australia/Adelaide' ) and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"

    # Check that zones with complete, permanent offset changes (not just DST) are handled.  We know
    # that within a year of 2014-04-28, the America/Eirunepe (west Amazonas) zone had such a change
    # (pre pytz 2017.2, anyway...)
    if pytz_version < (2017,2):
        abbrev			= timestamp.support_abbreviations( [ 'America/Eirunepe' ], at=datetime.datetime( 2014, 4, 28 ), reset=True)
        #print( sorted( abbrev ))
        assert sorted( abbrev ) == [ 'ACT', 'AMT' ]
        z,dst,off			= timestamp._tzabbrev['ACT']
        assert str(z) == 'America/Eirunepe'		and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 5:00:00"
        z,dst,off			= timestamp._tzabbrev['AMT']
        assert str(z) == 'America/Eirunepe'		and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 4:00:00"
コード例 #3
0
ファイル: history_test.py プロジェクト: ekw/cpppo
def test_history_timestamp_abbreviations():
    """Test timezone abbreviation support. """
    abbrev			= timestamp.support_abbreviations( 'CA', reset=True )
    assert sorted( abbrev ) == ['ADT', 'AST', 'CDT', 'CST', 'EDT', 'EST', 'MDT', 'MST', 'NDT', 'NST', 'PDT', 'PST']

    # Perform all the remaining timezone abbreviation tests relative to a known range of times, to
    # avoid differences in the future due to timezone changes.
    ts				= timestamp( "2014-04-24 08:00:00 MDT" )
    assert near( ts.value, 1398348000.0 )

    try:
        abbrev			= timestamp.support_abbreviations( 'America' )
        assert False, "Many zones should have been ambiguously abbreviated"
    except AmbiguousTimeZoneError as exc:
        assert "America/Mazatlan" in str( exc )
    abbrev			= timestamp.support_abbreviations( 'America', 
        exclude=['America/Mazatlan', 'America/Merida', 'America/Mexico_City', 'America/Monterrey',
                 'America/Bahia_Banderas', 'America/Cancun', 'America/Chihuahua', 'America/Havana',
                 'America/Santa_Isabel', 'America/Grand_Turk'] )
    #print( sorted( abbrev ))
    #print( reprlib.repr( timestamp._tzabbrev ))
    assert sorted( abbrev ) == ['ACT', 'AKDT', 'AKST', 'AMST', 'AMT', 'ART', 'BOT', 'BRST', 'BRT', 'CLST', 'CLT',
                                'COT', 'ECT', 'EGST', 'EGT', 'FNT', 'GFT', 'GMT', 'GYT', 'HADT', 'HAST',
                                'PET', 'PMDT', 'PMST', 'PYST', 'PYT', 'SRT', 'UYST', 'UYT', 'VET', 'WGST', 'WGT']
    abbrev			= timestamp.support_abbreviations( 'Europe/Berlin' )
    assert sorted( abbrev ) == ['CEST', 'CET']
    assert 'CEST' in timestamp._tzabbrev
    assert 'EEST' not in timestamp._tzabbrev
    abbrev			= timestamp.support_abbreviations( 'Europe', exclude=[ 'Europe/Simferopol', 'Europe/Istanbul', 'Europe/Minsk' ] )
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == ['BST', 'EEST', 'EET', 'FET', 'IST', 'MSK', 'SAMT', 'WEST', 'WET']

    assert 'EEST' in timestamp._tzabbrev
    try:
        timestamp.support_abbreviations( 'Asia' )
        assert False, "Asia/Jerusalem IST should have mismatched Europe/Dublin IST"
    except AmbiguousTimeZoneError as exc:
        assert "Asia/Jerusalem" in str( exc )

    assert near( parse_offset( '< 1:00:00.001' ),	-3600.001 )
    assert near( parse_offset( '<:1.001' ), 		   -1.001 )
    assert near( parse_offset( '>1:0.001' ),		   60.001 )
    assert near( parse_offset( '>1' ), 			    1 )

    # While Asia is internally very inconsistent (eg. EEST), countries should be internally consisent
    abbrev			= timestamp.support_abbreviations( 'JO', reset=True ) # Jordan
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == [ 'EEST', 'EET']
    z,dst,off			= timestamp._tzabbrev['EEST']
    assert str(z) == 'Asia/Amman'	and dst == True  and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"
    abbrev			= timestamp.support_abbreviations( 'IE', reset=True ) # Israel
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == [ 'GMT', 'IST' ]
    # Jordan, Israel and Lebanon only work if we pick a region to exclude, for one EEST definition
    abbrev			= timestamp.support_abbreviations( ['JO', 'IE', 'LB'],
                                                                   exclude=[ 'Asia/Amman' ], reset=True )
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == [ 'EEST', 'EET', 'GMT', 'IST' ]
    z,dst,off			= timestamp._tzabbrev['EEST']
    assert str(z) == 'Asia/Beirut'	and dst == True  and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 3:00:00"

    # Australia zones incompatible with a bunch of other timezone abbreviations, eg. CST; reset
    abbrev			= timestamp.support_abbreviations( 'Australia', reset=True )
    #print( sorted( abbrev ))
    #print( repr( timestamp._tzabbrev ))
    assert sorted( abbrev ) == ['ACDT', 'ACST', 'ACWST', 'AEDT', 'AEST', 'AWST', 'LHDT', 'LHST']
    z,dst,off			= timestamp._tzabbrev['LHST']
    assert str(z) == 'Australia/Lord_Howe'	and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == ">10:30:00"

    # Ensure that non-ambiguous (DST-specific) zone abbreviations override ambiguous (no longer
    # relevant, as pytz >= 2014.7 no longer contains dst == None for some of the Australian zones
    # without DST)
    abbrev			= timestamp.support_abbreviations( [ 'Australia/Adelaide' ], reset=True )
    #print( sorted( abbrev )) # ['ACDT', 'ACST']
    z,dst,off			= timestamp._tzabbrev['ACST']
    assert str(z) == 'Australia/Adelaide'	and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"
    abbrev			= timestamp.support_abbreviations( [ 'Australia/Adelaide', 'Australia/Darwin' ], reset=True )
    #print( sorted( abbrev ))
    #print( repr( timestamp._tzabbrev ))
    z,dst,off			= timestamp._tzabbrev['ACST']
    assert str(z) in ( 'Australia/Darwin',
                       'Australia/Adelaide' ) and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "> 9:30:00"

    # Check that zones with complete, permanent offset changes (not just DST) are handled.  We know
    # that within a year of 2014-04-28, the America/Eirunepe (west Amazonas) zone had such a change.
    abbrev			= timestamp.support_abbreviations( [ 'America/Eirunepe' ], at=datetime.datetime( 2014, 4, 28 ))
    #print( sorted( abbrev ))
    assert sorted( abbrev ) == [ 'ACT', 'AMT' ]
    z,dst,off			= timestamp._tzabbrev['ACT']
    assert str(z) == 'America/Eirunepe'		and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 5:00:00"
    z,dst,off			= timestamp._tzabbrev['AMT']
    assert str(z) == 'America/Eirunepe'		and dst == False and format_offset( timedelta_total_seconds( off ), ms=None ) == "< 4:00:00"
コード例 #4
0
ファイル: history_test.py プロジェクト: ekw/cpppo
def test_history_performance():
    try:
        tracemalloc.start()
    except:
        pass

    for _ in range( 3 ):
        path		= "/tmp/test_performance_%d" % random.randint( 100000, 999999 )
        if os.path.exists( path ):
            continue
    assert not os.path.exists( path ), "Couldn't find an unused name: %s" % path 

    files		= []
    try:
        day		= 24*60*60
        dur		= 3*day		# a few days worth of data
        regstps		= 0.0,5.0	# 0-5secs between updates
        numfiles	= dur//day+1	# ~1 file/day, but at least 2
        values		= {}		# Initial register values
        regscount	= 1000		# Number of different registers
        regschanged	= 1,10		# From 1-25 registers per row
        regsbase	= 40001

        start		= timer()

        now = beg	= start - dur
        linecnt		= 0
        for e in reversed( range( numfiles )):
            f		= path + (( '.%d' % e ) if e else '') # 0'th file has no extension
            files.append( f )
            with logger( f ) as l:
                if values:
                    l.write( values, now=now ); linecnt += 1
                while now < beg + len(files) * dur/numfiles:
                    lst	= now
                    now += random.uniform( *regstps )
                    assert now >= lst
                    assert timestamp( now ) >= timestamp( lst ), "now: %s, timestamp(now): %s" % ( now, timestamp( now ))
                    updates = {}
                    for _ in range( random.randint( *regschanged )):
                        updates[random.randint( regsbase, regsbase + regscount - 1 )] = random.randint( 0, 1<<16 - 1 )
                    values.update( updates )
                    l.write( updates, now=now ); linecnt += 1
                lst 	= now
                now    += random.uniform( *regstps )
                assert now >= lst
                assert timestamp( now ) >= timestamp( lst )
            if e:
                # Compress .1 onward using a random format; randomly delete origin uncompressed file
                # so sometimes both files exist
                if random.choice( (True, False, False, False) ):
                    continue # Don't make a compressed version of some files
                fz	 = f + '.%s' % random.choice( ('gz', 'bz2', 'xz') )
                files.append( fz )
                with opener( fz, mode='wb' ) as fd:
                    fd.write( open( f, 'rb' ).read() )
                if random.choice( (True, False, False) ):
                    continue # Don't remove some of the uncompressed files
                os.unlink( f )
                files.pop( files.index( f ))

        logging.warning( "Generated data in %.3fs; lines: %d", timer() - start, linecnt )

        # Start somewhere within 0-1% the dur of the beg, forcing the load the look back to
        # find the first file.  Try to do it all in the next 'playback' second (just to push it to
        # the max), in 'chunks' pieces.
        historical	= timestamp( random.uniform( beg + dur*0/100, beg + dur*1/100 ))
        basis		= timer()
        playback	= 2.0 * dur/day # Can sustain ~2 seconds / day of history on a single CPU
        chunks		= 1000
        factor		= dur / playback
        lookahead	= 60.0
        duration	= None
        if random.choice( (True,False) ):
            duration	= random.uniform( dur * 98/100, dur * 102/100 )

        begoff		= historical.value - beg
        endoff		= 0 if duration is None else (( historical.value + duration ) - ( beg + dur ))
        logging.warning( "Playback starts at beginning %s %s, duration %s, ends at ending %s %s",
                         timestamp( beg ), format_offset( begoff, ms=False ),
                         None if duration is None else format_offset( duration, ms=False, symbols='-+' ),
                         timestamp( beg + dur ), format_offset( endoff, ms=False ))

        ld		= loader(
            path, historical=historical, basis=basis, factor=factor, lookahead=lookahead, duration=duration )
        eventcnt	= 0
        slept		= 0
        cur		= None
        while ld:
            once	= False
            while ld.state < ld.AWAITING or not once:
                once		= True
                upcoming	= None
                limit		= random.randint( 0, 250 )
                if random.choice( (True,False) ):
                    upcoming	= ld.advance()
                    if random.choice( (True,False) ) and cur:
                        # ~25% of the time, provide an 'upcoming' timestamp that is between the
                        # current advancing historical time and the last load time.
                        upcoming-= random.uniform( 0, upcoming.value - cur.value )
                cur,events	= ld.load( upcoming=upcoming, limit=limit )
                eventcnt       += len( events )
                advance		= ld.advance()
                offset		= advance.value - cur.value
                logging.detail( "%s loaded up to %s (%s w/ upcoming %14s); %4d future, %4d values: %4d events / %4d limit" ,
                                ld, cur, format_offset( offset ),
                                format_offset( upcoming.value - advance.value ) if upcoming is not None else None,
                                len( ld.future ), len( ld.values ), len( events ), limit )

            logging.warning( "%s loaded up to %s; %3d future, %4d values: %6d events total",
                                ld, cur, len( ld.future ), len( ld.values ), eventcnt )
            try:
                snapshot	= tracemalloc.take_snapshot()
                display_top( snapshot, limit=10 )
            except:
                pass

            time.sleep( playback/chunks )
            slept	       += playback/chunks

        elapsed		= timer() - basis
        eventtps	= eventcnt // ( elapsed - slept )
        logging.error( "Playback in %.3fs (slept %.3fs); events: %d ==> %d historical records/sec",
                       elapsed, slept, eventcnt, eventtps )
        if not logging.getLogger().isEnabledFor( logging.NORMAL ):
            # Ludicrously low threshold, to pass tests on very slow machines
            assert eventtps >= 1000, \
                "Historical event processing performance low: %d records/sec" % eventtps
        try:
            display_biggest_traceback()
        except:
            pass

    except Exception as exc:
        logging.normal( "Test failed: %s", exc )
        '''
        for f in files:
            logging.normal( "%s:\n    %s", f, "    ".join( l for l in open( f )))
        '''
        raise

    finally:
        for f in files:
            logging.detail( "unlinking %s", f )
            try:
                os.unlink( f )
            except:
                pass
コード例 #5
0
ファイル: history_test.py プロジェクト: raphaelbertozzi/cpppo
def test_history_timestamp_abbreviations():
    """Test timezone abbreviation support. """
    abbrev = timestamp.support_abbreviations("CA", reset=True)
    assert sorted(abbrev) == ["ADT", "AST", "CDT", "CST", "EDT", "EST", "MDT", "MST", "NDT", "NST", "PDT", "PST"]

    # Perform all the remaining timezone abbreviation tests relative to a known range of times, to
    # avoid differences in the future due to timezone changes.
    ts = timestamp("2014-04-24 08:00:00 MDT")
    assert near(ts.value, 1398348000.0)

    try:
        abbrev = timestamp.support_abbreviations("America")
        assert False, "Many zones should have been ambiguously abbreviated"
    except AmbiguousTimeZoneError as exc:
        assert "America/Mazatlan" in str(exc)
    abbrev = timestamp.support_abbreviations(
        "America",
        exclude=[
            "America/Mazatlan",
            "America/Merida",
            "America/Mexico_City",
            "America/Monterrey",
            "America/Bahia_Banderas",
            "America/Cancun",
            "America/Chihuahua",
            "America/Havana",
            "America/Santa_Isabel",
            "America/Grand_Turk",
            "America/Cayman",
        ],
    )
    # print( sorted( abbrev ))
    # print( reprlib.repr( timestamp._tzabbrev ))
    if tuple(map(int, pytz.__version__.split("."))) < (2015, 4):
        logging.warning("pytz < 2015.4; HADT/HAST vs. HDT/HST")
        assert sorted(abbrev) == [
            "ACT",
            "AKDT",
            "AKST",
            "AMST",
            "AMT",
            "ART",
            "BOT",
            "BRST",
            "BRT",
            "CLST",
            "CLT",
            "COT",
            "ECT",
            "EGST",
            "EGT",
            "FNT",
            "GFT",
            "GMT",
            "GYT",
            "HADT",
            "HAST",
            "PET",
            "PMDT",
            "PMST",
            "PYST",
            "PYT",
            "SRT",
            "UYST",
            "UYT",
            "VET",
            "WGST",
            "WGT",
        ]
    else:
        assert sorted(abbrev) == [
            "ACT",
            "AKDT",
            "AKST",
            "AMST",
            "AMT",
            "ART",
            "BOT",
            "BRST",
            "BRT",
            "CLST",
            "CLT",
            "COT",
            "ECT",
            "EGST",
            "EGT",
            "FNT",
            "GFT",
            "GMT",
            "GYT",
            "HDT",
            "HST",
            "PET",
            "PMDT",
            "PMST",
            "PYST",
            "PYT",
            "SRT",
            "UYST",
            "UYT",
            "VET",
            "WGST",
            "WGT",
        ]
    abbrev = timestamp.support_abbreviations("Europe/Berlin")
    assert sorted(abbrev) == ["CEST", "CET"]
    assert "CEST" in timestamp._tzabbrev
    assert "EEST" not in timestamp._tzabbrev
    abbrev = timestamp.support_abbreviations(
        "Europe", exclude=["Europe/Simferopol", "Europe/Istanbul", "Europe/Minsk", "Europe/Chisinau"]
    )
    # print( sorted( abbrev ))
    assert sorted(abbrev) == ["BST", "EEST", "EET", "IST", "MSK", "SAMT", "WEST", "WET"]

    assert "EEST" in timestamp._tzabbrev
    try:
        timestamp.support_abbreviations("Asia")
        assert False, "Asia/Jerusalem IST should have mismatched Europe/Dublin IST"
    except AmbiguousTimeZoneError as exc:
        assert "Asia/Jerusalem" in str(exc)

    assert near(parse_offset("< 1:00:00.001"), -3600.001)
    assert near(parse_offset("<:1.001"), -1.001)
    assert near(parse_offset(">1:0.001"), 60.001)
    assert near(parse_offset(">1"), 1)

    # While Asia is internally very inconsistent (eg. EEST), countries should be internally consisent
    abbrev = timestamp.support_abbreviations("JO", reset=True)  # Jordan
    # print( sorted( abbrev ))
    assert sorted(abbrev) == ["EEST", "EET"]
    z, dst, off = timestamp._tzabbrev["EEST"]
    assert (
        str(z) == "Asia/Amman" and dst == True and format_offset(timedelta_total_seconds(off), ms=None) == "> 3:00:00"
    )
    abbrev = timestamp.support_abbreviations("IE", reset=True)  # Israel
    # print( sorted( abbrev ))
    assert sorted(abbrev) == ["GMT", "IST"]
    # Jordan, Israel and Lebanon only work if we pick a region to exclude, for one EEST definition
    abbrev = timestamp.support_abbreviations(["JO", "IE", "LB"], exclude=["Asia/Amman"], reset=True)
    # print( sorted( abbrev ))
    assert sorted(abbrev) == ["EEST", "EET", "GMT", "IST"]
    z, dst, off = timestamp._tzabbrev["EEST"]
    assert (
        str(z) == "Asia/Beirut" and dst == True and format_offset(timedelta_total_seconds(off), ms=None) == "> 3:00:00"
    )

    # Australia zones incompatible with a bunch of other timezone abbreviations, eg. CST; reset
    abbrev = timestamp.support_abbreviations("Australia", reset=True)
    # print( sorted( abbrev ))
    # print( repr( timestamp._tzabbrev ))
    assert sorted(abbrev) == ["ACDT", "ACST", "ACWST", "AEDT", "AEST", "AWST", "LHDT", "LHST"]
    z, dst, off = timestamp._tzabbrev["LHST"]
    assert (
        str(z) == "Australia/Lord_Howe"
        and dst == False
        and format_offset(timedelta_total_seconds(off), ms=None) == ">10:30:00"
    )

    # Ensure that non-ambiguous (DST-specific) zone abbreviations override ambiguous (no longer
    # relevant, as pytz >= 2014.7 no longer contains dst == None for some of the Australian zones
    # without DST)
    abbrev = timestamp.support_abbreviations(["Australia/Adelaide"], reset=True)
    # print( sorted( abbrev )) # ['ACDT', 'ACST']
    z, dst, off = timestamp._tzabbrev["ACST"]
    assert (
        str(z) == "Australia/Adelaide"
        and dst == False
        and format_offset(timedelta_total_seconds(off), ms=None) == "> 9:30:00"
    )
    abbrev = timestamp.support_abbreviations(["Australia/Adelaide", "Australia/Darwin"], reset=True)
    # print( sorted( abbrev ))
    # print( repr( timestamp._tzabbrev ))
    z, dst, off = timestamp._tzabbrev["ACST"]
    assert (
        str(z) in ("Australia/Darwin", "Australia/Adelaide")
        and dst == False
        and format_offset(timedelta_total_seconds(off), ms=None) == "> 9:30:00"
    )

    # Check that zones with complete, permanent offset changes (not just DST) are handled.  We know
    # that within a year of 2014-04-28, the America/Eirunepe (west Amazonas) zone had such a change.
    abbrev = timestamp.support_abbreviations(["America/Eirunepe"], at=datetime.datetime(2014, 4, 28))
    # print( sorted( abbrev ))
    assert sorted(abbrev) == ["ACT", "AMT"]
    z, dst, off = timestamp._tzabbrev["ACT"]
    assert (
        str(z) == "America/Eirunepe"
        and dst == False
        and format_offset(timedelta_total_seconds(off), ms=None) == "< 5:00:00"
    )
    z, dst, off = timestamp._tzabbrev["AMT"]
    assert (
        str(z) == "America/Eirunepe"
        and dst == False
        and format_offset(timedelta_total_seconds(off), ms=None) == "< 4:00:00"
    )