def run_one_test(typ, fixedLen, value):
    """Run a single test."""

    # Create the template to be tested.
    nullable = 'null' in typ
    if fixedLen:
        skip = 'skip(%d)%s' % (fixedLen, ' null' if  nullable else '')
    else:
        skip = 'skip%s' % (' null' if nullable else '')
    template = '(int64,%s,string)' % skip

    # People want to know: which test case is this?
    tc_name = 'tc_%s' % typ.replace(' ', '_')
    print "---- Test case:", tc_name, template

    # Write the T_center.dat file.
    global _files
    _files.append('/tmp/{0}_center_{1}.dat'.format(tc_name, _args.run_id))
    if typ.startswith('binary'):
        # Python runtime hates exec'ing the embedded NUL horror, so we
        # need to craft the .dat file by hand.
        eor = ''.join((struct.pack("<I", len('eor\x00')), 'eor\x00'))
        with open(_files[-1], 'wb') as W:
            for i in xrange(DIM_LO, DIM_HI+1):
                if nullable:
                    W.write(struct.pack("<qbI", i+1, -1, len(value)))
                else:
                    W.write(struct.pack("<qI", i+1, len(value)))
                W.write(value)
                W.write(eor)
    else:
        query = """save(project(apply({0}, center, {1}), left, center, right),
                   '{2}', -2, '(int64,{3},string)')""".format(
            REF, str(value), _files[-1], typ)
        ok(iquery(['-naq', query]))

    # Recreate target array.
    iquery(['-naq', 'remove(%s)' % TARGET]) # may fail, no ok() wrapper
    ok(iquery(['-aq', 'create temp array %s %s' % (TARGET, SCHEMA)]))

    # Load target from T_center.dat and verify == to reference array.
    ok(iquery(['-naq', "load({0}, '{1}', -2, '{2}')".format(
                    TARGET, _files[-1], template)]))
    check_vs_ref_array(TARGET)

    # Save using same template.
    _files.append('/tmp/{0}_pad_{1}.dat'.format(tc_name, _args.run_id))
    ok(iquery(['-naq', "save({0}, '{1}', -2, '{2}')".format(
                    TARGET, _files[-1], template)]))

    # Recreate target array.
    iquery(['-naq', 'remove(%s)' % TARGET])
    ok(iquery(['-aq', 'create temp array %s %s' % (TARGET, SCHEMA)]))

    # Load target from T_pad.dat and verify == to reference array.
    ok(iquery(['-naq', "load({0}, '{1}', -2, '{2}')".format(
                    TARGET, _files[-1], template)]))
    check_vs_ref_array(TARGET)

    return None                 # Done.
Beispiel #2
0
def run_tests():
    """Run the test cases.  Only two of them."""
    query = "consume(input({0}, '{1}', -2, '(binary)'))".format(SCHEMA, _fifo)

    # Test 1: A big chunk, but not too big.
    log("Starting big chunk test ({0} blobs of {1} bytes)".format(
        CHUNK_INTERVAL, BIG_BLOB_SIZE))
    p = Process(target=blob_writer, args=(BIG_BLOB_SIZE, CHUNK_INTERVAL))
    p.start()
    with ElapsedTimer():
        ret = ok(iquery(['-aq', query]))
    p.join()
    if ret:
        return ret

    # Test 2: A ***HUGE*** chunk!
    log("Starting HUGE chunk test ({0} blobs of {1} bytes)".format(
        CHUNK_INTERVAL, HUGE_BLOB_SIZE))
    p = Process(target=blob_writer, args=(HUGE_BLOB_SIZE, CHUNK_INTERVAL))
    p.start()
    with ElapsedTimer():
        ret = fail(iquery(['-aq', query]), LE_CHUNK_TOO_HUGE,
                   not _args.verbose)
    p.join()
    return ret
def check_vs_ref_array(ary):
    """Run queries whose output should always be the same if ary
    matches the reference array.  (Test harness will detect any
    difference.)
    """
    ok(iquery(['-aq', "filter(%s, left<>i+1 or right <> 'eor')" % ary]))
    ok(iquery(['-aq', "aggregate(%s, count(*))" % ary]))
def cleanup():
    """Remove files and arrays we created elsewhere."""
    for f in _files:
        try:
            os.unlink(f)
        except:
            pass
    for a in (REF, TARGET):
        iquery(['-aq', 'remove(%s)' % a])
Beispiel #5
0
def log_mem_info():
    """Log the memory configuration etc. to big_chunk.out .

    This is to get a better understanding of what memory
    configurations allow the test to pass cleanly (as opposed to
    having to silently tolerate SCIDB_LE_CANT_REALLOCATE_MEMORY
    failures).
    """
    for opt in ('mem-array-threshold',
                'max-memory-limit',
                'chunk-size-limit-mb',
                'redim-chunk-overhead-limit-mb',
                'small-memalloc-size',
                'large-memalloc-limit',
                'network-buffer',
                'async-io-buffer'):
        val = iquery(['-otsv:l', '-aq', "_setopt('%s')" % opt]).splitlines()[1]
        log(opt, '=', val)
    log("Contents of /proc/meminfo:")
    try:
        with open('/proc/meminfo') as F:
            while True:
                x = F.read(4096)
                if not x:
                    break
                sys.stderr.write(x)
    except IOError as e:
        log("Cannot read /proc/meminfo:", e)
Beispiel #6
0
def run_tests():
    """Run the test cases.  Only two of them."""
    query = "consume(input({0}, '{1}', -2, '(binary)'))".format(SCHEMA, _fifo)

    # Test 1: A big chunk, but not too big.
    log("Starting big chunk test ({0} blobs of {1} bytes)".format(
        CHUNK_INTERVAL, BIG_BLOB_SIZE))
    p = Process(target=blob_writer, args=(BIG_BLOB_SIZE, CHUNK_INTERVAL))
    p.start()
    with ElapsedTimer():
        ret = ok(iquery(['-aq', query]))
    p.join()
    if ret:
        return ret

    # Test 2: A ***HUGE*** chunk!
    log("Starting HUGE chunk test ({0} blobs of {1} bytes)".format(
        CHUNK_INTERVAL, HUGE_BLOB_SIZE))
    p = Process(target=blob_writer, args=(HUGE_BLOB_SIZE, CHUNK_INTERVAL))
    p.start()
    with ElapsedTimer():
        ret = fail(iquery(['-aq', query]), LE_CHUNK_TOO_HUGE, not _args.verbose)
    p.join()
    return ret
Beispiel #7
0
def log_mem_info():
    """Log the memory configuration etc. to big_chunk.out .

    This is to get a better understanding of what memory
    configurations allow the test to pass cleanly (as opposed to
    having to silently tolerate SCIDB_LE_CANT_REALLOCATE_MEMORY
    failures).
    """
    for opt in ('mem-array-threshold', 'max-memory-limit',
                'chunk-size-limit-mb', 'redim-chunk-overhead-limit-mb',
                'small-memalloc-size', 'large-memalloc-limit',
                'network-buffer', 'async-io-buffer'):
        val = iquery(['-otsv:l', '-aq', "setopt('%s')" % opt]).splitlines()[1]
        log(opt, '=', val)
    log("Contents of /proc/meminfo:")
    try:
        with open('/proc/meminfo') as F:
            while True:
                x = F.read(4096)
                if not x:
                    break
                sys.stderr.write(x)
    except IOError as e:
        log("Cannot read /proc/meminfo:", e)
Beispiel #8
0
def main(argv=None):
    if argv is None:
        argv = sys.argv

    global _args
    parser = argparse.ArgumentParser(
        description='The store_01 test script.')
    parser.add_argument('-c', '--host', default='localhost',
                        help="The SciDB host address.")
    parser.add_argument('-p', '--port', type=int, default=1239,
                        help="The TCP port for connecting to SciDB.")
    parser.add_argument('-r', '--run-id', default="", help="""
        Uniquifier (such as $HPID) to use in naming files etc.""")
    parser.add_argument('-t', '--use-temp-array', default=False,
                        action='store_true', help="""
        Use a temporary array as the bounded target.""")
    parser.add_argument('-v', '--verbose', default=False, action='store_true',
                        help="""Print timings and full error descriptions.""")
    _args = parser.parse_args(argv[1:])

    t_other_utils.IQUERY_HOST = _args.host
    t_other_utils.IQUERY_PORT = _args.port

    BOUNDED_SCHEMA = "<value:int64>[row=0:499,100,0,col=0:99,100,0]"
    BOUNDED_ARRAY = "bounded_%s" % _args.run_id
    STORE_QUERY = 'store(%s, {0})'.format(BOUNDED_ARRAY)
    # Temp arrays take a different code path where, for historical
    # reasons I guess, the short error code is different.
    # Storing SG takes yet another path. So, we will just ignore the short error.
    # The semantic menaing is in the long error ayway.
    # In verbose mode the entire error string can be examined.
    LONG_ERROR = "SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES"

    print 'Create%s bounded array.' % (
        ' temporary' if _args.use_temp_array else '')
    print iquery(['-aq', 'create%s array %s %s' % (
                ' temp' if _args.use_temp_array else '',
                BOUNDED_ARRAY,
                BOUNDED_SCHEMA)])

    fails = 0
    quiet = not _args.verbose
    if quiet:
        ElapsedTimer.enabled = False

    print '\nEasy store...'
    with ElapsedTimer():
        fails += ok(iquery(['-naq', STORE_QUERY % make_grid(10, 10, 60, 30)]))

    print '\nRight up against the row limit...'
    with ElapsedTimer():
        fails += ok(iquery(['-naq', STORE_QUERY % make_grid(450, 10, 499, 20)]))

    print '\nOne step over the line...'
    with ElapsedTimer():
        fails += fail(iquery(['-naq', STORE_QUERY % make_grid(450, 10, 500, 20)]),
                      LONG_ERROR,
                      quiet)

    print '\nWay over the line...'
    with ElapsedTimer():
        fails += fail(iquery(['-naq', STORE_QUERY % make_grid(480, 10, 520, 20)]),
                      LONG_ERROR,
                      quiet)

    print '\nRight up against the column limit...'
    with ElapsedTimer():
        fails += ok(iquery(['-naq', STORE_QUERY % make_grid(10, 80, 50, 99)]))

    print '\nOne step over the column limit...'
    with ElapsedTimer():
        fails += fail(iquery(['-naq', STORE_QUERY % make_grid(10, 80, 50, 100)]),
                      LONG_ERROR,
                      quiet)

    print '\nPartially over both limits...'
    with ElapsedTimer():
        fails += fail(iquery(['-naq', STORE_QUERY % make_grid(480, 95, 500, 100)]),
                      LONG_ERROR,
                      quiet)

    print '\nWay over both limits...'
    with ElapsedTimer():
        fails += fail(iquery(['-naq', STORE_QUERY % make_grid(510, 120, 530, 140)]),
                      LONG_ERROR,
                      quiet)

    print "\nCleanup."
    with ElapsedTimer():
        iquery(['-naq', 'remove(%s)' % BOUNDED_ARRAY])

    if fails:
        print fails, "test case failures"
    else:
        print "All test cases passed."

    # By returning 0 even for failure, we prefer a FILES_DIFFER error
    # to an EXECUTOR_FAILED error.  Seems slightly more accurate.
    return 0
Beispiel #9
0
def main(argv=None):
    if argv is None:
        argv = sys.argv

    global _args
    parser = argparse.ArgumentParser(
        description='The insert_09 test script.')
    parser.add_argument('-c', '--host', default='localhost',
                        help="The SciDB host address.")
    parser.add_argument('-p', '--port', type=int, default=1239,
                        help="The TCP port for connecting to SciDB.")
    parser.add_argument('-r', '--run-id', default="", help="""
        Uniquifier (such as $HPID) to use in naming files etc.""")
    _args = parser.parse_args(argv[1:])

    t_other_utils.IQUERY_HOST = _args.host
    t_other_utils.IQUERY_PORT = _args.port

    BOUNDED_SCHEMA = "<value:int64>[row=0:499,100,0,col=0:99,100,0]"
    BOUNDED_ARRAY = "bounded_%s" % _args.run_id

    print 'Create bounded array.'
    print iquery(['-aq', 'create array %s %s' % (BOUNDED_ARRAY,BOUNDED_SCHEMA)])
    INSERT_QUERY = 'insert(%s, {0})'.format(BOUNDED_ARRAY)

    fails = 0

    print '\nEasy insert...'
    fails += ok(iquery(['-naq', INSERT_QUERY % make_grid(10, 10, 60, 30)]))

    print '\nRight up against the row limit...'
    fails += ok(iquery(['-naq', INSERT_QUERY % make_grid(450, 10, 499, 20)]))

    print '\nOne step over the line...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(450, 10, 500, 20)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nWay over the line...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(480, 10, 520, 20)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nRight up against the column limit...'
    fails += ok(iquery(['-naq', INSERT_QUERY % make_grid(10, 80, 50, 99)]))

    print '\nOne step over the column limit...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(10, 80, 50, 100)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nPartially over both limits...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(480, 95, 500, 100)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nWay over both limits...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(510, 120, 530, 140)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nCleanup.'
    iquery(['-naq', 'remove(%s)' % BOUNDED_ARRAY])

    if fails:
        print fails, "test case failures"
    else:
        print "All test cases passed."

    # By returning 0 even for failure, we prefer a FILES_DIFFER error
    # to an EXECUTOR_FAILED error.  Seems slightly more accurate.
    return 0
Beispiel #10
0
def main(argv=None):
    if argv is None:
        argv = sys.argv

    global _args
    parser = argparse.ArgumentParser(description='The insert_09 test script.')
    parser.add_argument('-c',
                        '--host',
                        default='localhost',
                        help="The SciDB host address.")
    parser.add_argument('-p',
                        '--port',
                        type=int,
                        default=1239,
                        help="The TCP port for connecting to SciDB.")
    parser.add_argument('-r',
                        '--run-id',
                        default="",
                        help="""
        Uniquifier (such as $HPID) to use in naming files etc.""")
    _args = parser.parse_args(argv[1:])

    t_other_utils.IQUERY_HOST = _args.host
    t_other_utils.IQUERY_PORT = _args.port

    BOUNDED_SCHEMA = "<value:int64>[row=0:499,100,0,col=0:99,100,0]"
    BOUNDED_ARRAY = "bounded_%s" % _args.run_id

    print 'Create bounded array.'
    print iquery(
        ['-aq', 'create array %s %s' % (BOUNDED_ARRAY, BOUNDED_SCHEMA)])
    INSERT_QUERY = 'insert(%s, {0})'.format(BOUNDED_ARRAY)

    fails = 0

    print '\nEasy insert...'
    fails += ok(iquery(['-naq', INSERT_QUERY % make_grid(10, 10, 60, 30)]))

    print '\nRight up against the row limit...'
    fails += ok(iquery(['-naq', INSERT_QUERY % make_grid(450, 10, 499, 20)]))

    print '\nOne step over the line...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(450, 10, 500, 20)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nWay over the line...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(480, 10, 520, 20)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nRight up against the column limit...'
    fails += ok(iquery(['-naq', INSERT_QUERY % make_grid(10, 80, 50, 99)]))

    print '\nOne step over the column limit...'
    fails += fail(iquery(['-naq', INSERT_QUERY % make_grid(10, 80, 50, 100)]),
                  "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nPartially over both limits...'
    fails += fail(
        iquery(['-naq', INSERT_QUERY % make_grid(480, 95, 500, 100)]),
        "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nWay over both limits...'
    fails += fail(
        iquery(['-naq', INSERT_QUERY % make_grid(510, 120, 530, 140)]),
        "SCIDB_SE_OPERATOR::SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES")

    print '\nCleanup.'
    iquery(['-naq', 'remove(%s)' % BOUNDED_ARRAY])

    if fails:
        print fails, "test case failures"
    else:
        print "All test cases passed."

    # By returning 0 even for failure, we prefer a FILES_DIFFER error
    # to an EXECUTOR_FAILED error.  Seems slightly more accurate.
    return 0
Beispiel #11
0
def make_ref_array():
    ok(iquery(['-aq', 'create temp array %s %s' % (REF, SCHEMA)]))
    ok(iquery(['-naq' "store(apply(build(<left:int64>%s, i+1), right, 'eor'), %s)" % (DIM, REF)]))
Beispiel #12
0
def main(argv=None):
    if argv is None:
        argv = sys.argv

    global _args
    parser = argparse.ArgumentParser(description='The store_01 test script.')
    parser.add_argument('-c',
                        '--host',
                        default='localhost',
                        help="The SciDB host address.")
    parser.add_argument('-p',
                        '--port',
                        type=int,
                        default=1239,
                        help="The TCP port for connecting to SciDB.")
    parser.add_argument('-r',
                        '--run-id',
                        default="",
                        help="""
        Uniquifier (such as $HPID) to use in naming files etc.""")
    parser.add_argument('-t',
                        '--use-temp-array',
                        default=False,
                        action='store_true',
                        help="""
        Use a temporary array as the bounded target.""")
    parser.add_argument('-v',
                        '--verbose',
                        default=False,
                        action='store_true',
                        help="""Print timings and full error descriptions.""")
    _args = parser.parse_args(argv[1:])

    t_other_utils.IQUERY_HOST = _args.host
    t_other_utils.IQUERY_PORT = _args.port

    BOUNDED_SCHEMA = "<value:int64>[row=0:499,100,0,col=0:99,100,0]"
    BOUNDED_ARRAY = "bounded_%s" % _args.run_id
    STORE_QUERY = 'store(%s, {0})'.format(BOUNDED_ARRAY)
    # Temp arrays take a different code path where, for historical
    # reasons I guess, the short error code is different.
    # Storing SG takes yet another path. So, we will just ignore the short error.
    # The semantic menaing is in the long error ayway.
    # In verbose mode the entire error string can be examined.
    LONG_ERROR = "SCIDB_LE_CHUNK_OUT_OF_BOUNDARIES"

    print 'Create%s bounded array.' % (' temporary'
                                       if _args.use_temp_array else '')
    print iquery([
        '-aq',
        'create%s array %s %s' % (' temp' if _args.use_temp_array else '',
                                  BOUNDED_ARRAY, BOUNDED_SCHEMA)
    ])

    fails = 0
    quiet = not _args.verbose
    if quiet:
        ElapsedTimer.enabled = False

    print '\nEasy store...'
    with ElapsedTimer():
        fails += ok(iquery(['-naq', STORE_QUERY % make_grid(10, 10, 60, 30)]))

    print '\nRight up against the row limit...'
    with ElapsedTimer():
        fails += ok(iquery(['-naq',
                            STORE_QUERY % make_grid(450, 10, 499, 20)]))

    print '\nOne step over the line...'
    with ElapsedTimer():
        fails += fail(
            iquery(['-naq', STORE_QUERY % make_grid(450, 10, 500, 20)]),
            LONG_ERROR, quiet)

    print '\nWay over the line...'
    with ElapsedTimer():
        fails += fail(
            iquery(['-naq', STORE_QUERY % make_grid(480, 10, 520, 20)]),
            LONG_ERROR, quiet)

    print '\nRight up against the column limit...'
    with ElapsedTimer():
        fails += ok(iquery(['-naq', STORE_QUERY % make_grid(10, 80, 50, 99)]))

    print '\nOne step over the column limit...'
    with ElapsedTimer():
        fails += fail(
            iquery(['-naq', STORE_QUERY % make_grid(10, 80, 50, 100)]),
            LONG_ERROR, quiet)

    print '\nPartially over both limits...'
    with ElapsedTimer():
        fails += fail(
            iquery(['-naq', STORE_QUERY % make_grid(480, 95, 500, 100)]),
            LONG_ERROR, quiet)

    print '\nWay over both limits...'
    with ElapsedTimer():
        fails += fail(
            iquery(['-naq', STORE_QUERY % make_grid(510, 120, 530, 140)]),
            LONG_ERROR, quiet)

    print "\nCleanup."
    with ElapsedTimer():
        iquery(['-naq', 'remove(%s)' % BOUNDED_ARRAY])

    if fails:
        print fails, "test case failures"
    else:
        print "All test cases passed."

    # By returning 0 even for failure, we prefer a FILES_DIFFER error
    # to an EXECUTOR_FAILED error.  Seems slightly more accurate.
    return 0