def test_scidb_afl_module(): """Testing all public methods in scidblib.scidb_afl.""" print '*** testing scidblib.scidb_afl...' class TmpArgs: def __init__(self): self.host = '' self.port = '' args = TmpArgs() iquery_cmd = scidb_afl.get_iquery_cmd(args) scidb_afl.execute_it_return_out_err('ls') scidb_afl.afl(iquery_cmd, 'list()') print 'time_afl(..., \'list()\') =', scidb_afl.time_afl( iquery_cmd, 'list()') print 'single_cell_afl(..., \'build(<v:int64>[i=0:0,1,0], 5)\', 1) =', \ scidb_afl.single_cell_afl(iquery_cmd, 'build(<v:int64>[i=0:0,1,0], 5)', 1) print 'single_cell_afl(..., \'apply(build(<v:int64>[i=0:0,1,0], 5), v2, 6)\', 2) =', \ scidb_afl.single_cell_afl(iquery_cmd, 'apply(build(<v:int64>[i=0:0,1,0], 5), v2, 6)', 2) print 'get_num_instances(...) =', scidb_afl.get_num_instances(iquery_cmd) print 'get_array_names(...) =', scidb_afl.get_array_names(iquery_cmd) print
def query(self, cmd, \ want_output=True, \ tolerate_error=False, \ verbose=True): return scidb_afl.afl( \ self._iquery_cmd, \ cmd, \ want_output, \ tolerate_error, \ verbose)
def test_scidb_afl_module(): """Testing all public methods in scidblib.scidb_afl.""" print '*** testing scidblib.scidb_afl...' class TmpArgs: def __init__(self): self.host = '' self.port = '' args = TmpArgs() iquery_cmd = scidb_afl.get_iquery_cmd(args) scidb_afl.execute_it_return_out_err('ls') scidb_afl.afl(iquery_cmd, 'list()') print 'time_afl(..., \'list()\') =', scidb_afl.time_afl(iquery_cmd, 'list()') print 'single_cell_afl(..., \'build(<v:int64>[i=0:0,1,0], 5)\', 1) =', \ scidb_afl.single_cell_afl(iquery_cmd, 'build(<v:int64>[i=0:0,1,0], 5)', 1) print 'single_cell_afl(..., \'apply(build(<v:int64>[i=0:0,1,0], 5), v2, 6)\', 2) =', \ scidb_afl.single_cell_afl(iquery_cmd, 'apply(build(<v:int64>[i=0:0,1,0], 5), v2, 6)', 2) print 'get_num_instances(...) =', scidb_afl.get_num_instances(iquery_cmd) print 'get_array_names(...) =', scidb_afl.get_array_names(iquery_cmd) print
def query(self, cmd, \ want_output=True, \ tolerate_error=False, \ verbose=False): """ Execute a SciDb AFL query @param cmd The SciDB command @param want_output If true we want the output @param tolerate_error If true, tolerate errors @param verbose If true, provide a higher level of verbosity """ return scidb_afl.afl( \ self._iquery_cmd, \ cmd, \ want_output, \ tolerate_error, \ verbose)
def main(): """The main function gets command-line argument as a pattern, and removes all arrays with that pattern. """ parser = argparse.ArgumentParser( description='Remove all SciDB arrays whose names match a given pattern.', epilog= 'assumptions:\n' + ' - iquery is in your path.', formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-f', '--force', action='store_true', help='Forced removal of the arrays without asking for confirmation.') parser.add_argument('-c', '--host', help='Host name to be passed to iquery.') parser.add_argument('-p', '--port', help='Port number to be passed to iquery.') parser.add_argument('-t', '--temp-only', action='store_true', help='Limiting the candidates to temp arrays.') parser.add_argument('-U', '--user-name', help='User name to be passed to iquery.') parser.add_argument('-P', '--user-password', help='User password to be passed to iquery.') parser.add_argument('-v', '--verbose ', default=True, help='display verbose output.') parser.add_argument('regex', metavar='REGEX', type=str, nargs='?', default='.*', help='''Regular expression to match against array names. The utility will remove arrays whose names match the regular expression. Default is '.+', meaning to remove all arrays, because the pattern matches all names. The regular expression must match the full array name. For instance, '.*s' will match array 'dogs' because it ends with 's', but will not match array 'moose' because it does not end with 's'.''' ) args = parser.parse_args() try: iquery_cmd = scidb_afl.get_iquery_cmd(args) namespaces = scidb_afl.get_namespace_names(iquery_cmd) for namespace in namespaces: print "\nSearching namespace: ", namespace names = scidb_afl.get_array_names( iquery_cmd = iquery_cmd, temp_only=args.temp_only, namespace=namespace) names_to_remove = [] for name in names: match_name = re.match('^'+args.regex+'$', name) if match_name: names_to_remove.append(name) if not names_to_remove: print "There are no arrays to remove in namespace", namespace continue if not args.force: print 'The following arrays are about to be removed from namespace ' + namespace + ':' print names_to_remove proceed = scidb_psf.confirm(prompt='Are you sure you want to remove?', resp=False) if not proceed: return for name in names_to_remove: scidb_afl.remove_array(name, namespace, iquery_cmd) if namespace != 'public': names = scidb_afl.get_array_names( iquery_cmd=iquery_cmd, temp_only=args.temp_only, namespace=namespace) if not names: scidb_afl.afl( iquery_cmd, 'drop_namespace(\'' + namespace + '\');') print "namespace " + namespace + " removed" print 'Number of arrays removed =', len(names_to_remove) except Exception, e: print >> sys.stderr, '------ Exception -----------------------------' print >> sys.stderr, e if args.verbose: print >> sys.stderr, '------ Traceback (for debug purpose) ---------' traceback.print_exc() print >> sys.stderr, '----------------------------------------------' sys.exit(-1) # upon an exception, throw -1
def my_test(args, num_chunks, chunk_length, initial_values_per_chunk, new_values_per_chunk, type_name): """This function does the testing of appending alternate values to the end of every chunk of an array. @param args command-line parameters. @param num_chunks how many chunks are there. @param chunk_length the chunk length. @param initial_values_per_chunk the number of initial values per chunk @param new_values_per_chunk how many value to insert into each chunk. @param type_name the data type. @return 0 """ # Set even_value and odd_value. even_value = "0" odd_value = "1" if type_name=="bool": even_value = "true" odd_value = "false" # Initialize the ProgressTracker progress_tracker = scidb_progress.ProgressTracker(if_print_start = args.verbose, if_print_end = args.verbose) progress_tracker.register_step('initial', 'Load initial values.') progress_tracker.register_step('new', 'Insert new values.') # Remove the array if exists. iquery_cmd = scidb_afl.get_iquery_cmd(args) my_remove_arrays(iquery_cmd, tolerate_error=True) # Create the array. cmd = "create temp array %s <v:%s>[i=0:%d,%d,0]" % (array_name, type_name, chunk_length*num_chunks-1, chunk_length) scidb_afl.afl(iquery_cmd, cmd) # Load initial values. # The algorithm is to create an array that describes the ranges for the initial values, # then use cross_between to filter out values from a fully-populated array. progress_tracker.start_step('initial') cmd = "create temp array %s <low:int64, high:int64>[i=0:%d,%d,0]" % (ranges_array_name, num_chunks-1, num_chunks) scidb_afl.afl(iquery_cmd, cmd) for c in xrange(num_chunks): cmd = ("insert(redimension(apply(build(<adummyattribute:bool>[adummydim=0:0,1,0],true), i, %d, low, %d, high, %d), %s), %s)" % (c, c*chunk_length, c*chunk_length+initial_values_per_chunk-1, ranges_array_name, ranges_array_name)) scidb_afl.afl(iquery_cmd, cmd) cmd = ("store(cross_between(build(%s, iif(i%%2=0, %s(%s), %s(%s))), %s), %s)" % (array_name, type_name, even_value, type_name, odd_value, ranges_array_name, array_name)) scidb_afl.afl(iquery_cmd, cmd) progress_tracker.end_step('initial') # Load the additional values. progress_tracker.start_step('new') if args.verbose: print "In each of the %d batches, one value will be appended to each of the %d chunks." % (new_values_per_chunk, num_chunks) print "Batch\tTime" for i in xrange(new_values_per_chunk): start_time = datetime.datetime.now() for c in xrange(num_chunks): index = c*chunk_length+i+initial_values_per_chunk value = type_name+"("+even_value+")" if index%2==0 else type_name+"("+odd_value+")" cmd = "op_set_cell_attr_1D(%s, i, %d, v, %s)" % (array_name, index, value) scidb_afl.afl(iquery_cmd, cmd) if args.verbose: seconds = scidb_progress.timedelta_total_seconds(datetime.datetime.now() - start_time) print "%d\t%f" % (i+1, seconds) progress_tracker.end_step('new') # Remove the array. my_remove_arrays(iquery_cmd, tolerate_error=False) # Return 0 return 0
def my_remove_arrays(iquery_cmd, tolerate_error): cmd = "remove(%s)" % (array_name); scidb_afl.afl(iquery_cmd, cmd, tolerate_error=tolerate_error) cmd = "remove(%s)" % (ranges_array_name); scidb_afl.afl(iquery_cmd, cmd, tolerate_error=tolerate_error)
def main(): """The main function gets command-line argument as a pattern, and removes all arrays with that pattern. """ parser = argparse.ArgumentParser( description= 'Remove all SciDB arrays whose names match a given pattern.', epilog='assumptions:\n' + ' - iquery is in your path.', formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '-f', '--force', action='store_true', help='Forced removal of the arrays without asking for confirmation.') parser.add_argument('-c', '--host', help='Host name to be passed to iquery.') parser.add_argument('-p', '--port', help='Port number to be passed to iquery.') parser.add_argument('-t', '--temp-only', action='store_true', help='Limiting the candidates to temp arrays.') parser.add_argument('-U', '--user-name', help='User name to be passed to iquery.') parser.add_argument('-P', '--user-password', help='User password to be passed to iquery.') parser.add_argument('-v', '--verbose ', default=True, help='display verbose output.') parser.add_argument( 'regex', metavar='REGEX', type=str, nargs='?', default='.*', help='''Regular expression to match against array names. The utility will remove arrays whose names match the regular expression. Default is '.+', meaning to remove all arrays, because the pattern matches all names. The regular expression must match the full array name. For instance, '.*s' will match array 'dogs' because it ends with 's', but will not match array 'moose' because it does not end with 's'.''' ) args = parser.parse_args() try: iquery_cmd = scidb_afl.get_iquery_cmd(args) namespaces = scidb_afl.get_namespace_names(iquery_cmd) for namespace in namespaces: print "\nSearching namespace: ", namespace names = scidb_afl.get_array_names(iquery_cmd=iquery_cmd, temp_only=args.temp_only, namespace=namespace) names_to_remove = [] for name in names: match_name = re.match('^' + args.regex + '$', name) if match_name: names_to_remove.append(name) if not names_to_remove: print "There are no arrays to remove in namespace", namespace continue if not args.force: print 'The following arrays are about to be removed from namespace ' + namespace + ':' print names_to_remove proceed = scidb_psf.confirm( prompt='Are you sure you want to remove?', resp=False) if not proceed: return for name in names_to_remove: scidb_afl.remove_array(name, namespace, iquery_cmd) if namespace != 'public': names = scidb_afl.get_array_names(iquery_cmd=iquery_cmd, temp_only=args.temp_only, namespace=namespace) if not names: scidb_afl.afl(iquery_cmd, 'drop_namespace(\'' + namespace + '\');') print "namespace " + namespace + " removed" print 'Number of arrays removed =', len(names_to_remove) except Exception, e: print >> sys.stderr, '------ Exception -----------------------------' print >> sys.stderr, e if args.verbose: print >> sys.stderr, '------ Traceback (for debug purpose) ---------' traceback.print_exc() print >> sys.stderr, '----------------------------------------------' sys.exit(-1) # upon an exception, throw -1
def my_test(args, num_chunks, chunk_length, initial_values_per_chunk, new_values_per_chunk, type_name): """This function does the testing of appending alternate values to the end of every chunk of an array. @param args command-line parameters. @param num_chunks how many chunks are there. @param chunk_length the chunk length. @param initial_values_per_chunk the number of initial values per chunk @param new_values_per_chunk how many value to insert into each chunk. @param type_name the data type. @return 0 """ # Set even_value and odd_value. even_value = "0" odd_value = "1" if type_name == "bool": even_value = "true" odd_value = "false" # Initialize the ProgressTracker progress_tracker = scidb_progress.ProgressTracker( if_print_start=args.verbose, if_print_end=args.verbose) progress_tracker.register_step('initial', 'Load initial values.') progress_tracker.register_step('new', 'Insert new values.') # Remove the array if exists. iquery_cmd = scidb_afl.get_iquery_cmd(args) my_remove_arrays(iquery_cmd, tolerate_error=True) # Create the array. cmd = "create temp array %s <v:%s>[i=0:%d,%d,0]" % ( array_name, type_name, chunk_length * num_chunks - 1, chunk_length) scidb_afl.afl(iquery_cmd, cmd) # Load initial values. # The algorithm is to create an array that describes the ranges for the initial values, # then use cross_between to filter out values from a fully-populated array. progress_tracker.start_step('initial') cmd = "create temp array %s <low:int64, high:int64>[i=0:%d,%d,0]" % ( ranges_array_name, num_chunks - 1, num_chunks) scidb_afl.afl(iquery_cmd, cmd) for c in xrange(num_chunks): cmd = ( "insert(redimension(apply(build(<adummyattribute:bool>[adummydim=0:0,1,0],true), i, %d, low, %d, high, %d), %s), %s)" % (c, c * chunk_length, c * chunk_length + initial_values_per_chunk - 1, ranges_array_name, ranges_array_name)) scidb_afl.afl(iquery_cmd, cmd) cmd = ( "store(cross_between(build(%s, iif(i%%2=0, %s(%s), %s(%s))), %s), %s)" % (array_name, type_name, even_value, type_name, odd_value, ranges_array_name, array_name)) scidb_afl.afl(iquery_cmd, cmd) progress_tracker.end_step('initial') # Load the additional values. progress_tracker.start_step('new') if args.verbose: print "In each of the %d batches, one value will be appended to each of the %d chunks." % ( new_values_per_chunk, num_chunks) print "Batch\tTime" for i in xrange(new_values_per_chunk): start_time = datetime.datetime.now() for c in xrange(num_chunks): index = c * chunk_length + i + initial_values_per_chunk value = type_name + "(" + even_value + ")" if index % 2 == 0 else type_name + "(" + odd_value + ")" cmd = "op_set_cell_attr_1D(%s, i, %d, v, %s)" % (array_name, index, value) scidb_afl.afl(iquery_cmd, cmd) if args.verbose: seconds = scidb_progress.timedelta_total_seconds( datetime.datetime.now() - start_time) print "%d\t%f" % (i + 1, seconds) progress_tracker.end_step('new') # Remove the array. my_remove_arrays(iquery_cmd, tolerate_error=False) # Return 0 return 0
def my_remove_arrays(iquery_cmd, tolerate_error): cmd = "remove(%s)" % (array_name) scidb_afl.afl(iquery_cmd, cmd, tolerate_error=tolerate_error) cmd = "remove(%s)" % (ranges_array_name) scidb_afl.afl(iquery_cmd, cmd, tolerate_error=tolerate_error)
def main(): """The main function gets command-line argument as a pattern, and removes all arrays with that pattern. @exception AppError if something goes wrong. @note If print_traceback_upon_exception (defined at the top of the script) is True, stack trace will be printed. This is helpful during debugging. """ parser = argparse.ArgumentParser( description= 'Remove all SciDB arrays whose names match a given pattern.', epilog='assumptions:\n' + ' - iquery is in your path.', formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '-f', '--force', action='store_true', help='Forced removal of the arrays without asking for confirmation.') parser.add_argument('-c', '--host', help='Host name to be passed to iquery.') parser.add_argument('-p', '--port', help='Port number to be passed to iquery.') parser.add_argument('-t', '--temp-only', action='store_true', help='Limiting the candidates to temp arrays.') parser.add_argument( 'regex', metavar='REGEX', type=str, nargs='?', default='.*', help='''Regular expression to match against array names. The utility will remove arrays whose names match the regular expression. Default is '.+', meaning to remove all arrays, because the pattern matches all names. The regular expression must match the full array name. For instance, '.*s' will match array 'dogs' because it ends with 's', but will not match array 'moose' because it does not end with 's'.''' ) args = parser.parse_args() try: names = scidb_afl.get_array_names(temp_only=args.temp_only) iquery_cmd = scidb_afl.get_iquery_cmd(args) names_to_remove = [] for name in names: match_name = re.match('^' + args.regex + '$', name) if match_name: names_to_remove.append(name) if not names_to_remove: print 'There is no array to remove.' sys.exit(0) if not args.force: print 'The following arrays are about to be removed:' print names_to_remove proceed = scidb_psf.confirm( prompt='Are you sure you want to remove?', resp=False) if not proceed: sys.exit(0) for name in names_to_remove: scidb_afl.afl(iquery_cmd, 'remove(' + name + ')') print 'Number of arrays removed =', len(names_to_remove) except Exception, e: print >> sys.stderr, '------ Exception -----------------------------' print >> sys.stderr, e if _print_traceback_upon_exception: print >> sys.stderr, '------ Traceback (for debug purpose) ---------' traceback.print_exc() print >> sys.stderr, '----------------------------------------------' sys.exit(-1) # upon an exception, throw -1
def main(): """The main function gets command-line argument as a pattern, and removes all arrays with that pattern. @exception AppError if something goes wrong. @note If print_traceback_upon_exception (defined at the top of the script) is True, stack trace will be printed. This is helpful during debugging. """ parser = argparse.ArgumentParser( description='Remove all SciDB arrays whose names match a given pattern.', epilog= 'assumptions:\n' + ' - iquery is in your path.', formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('-f', '--force', action='store_true', help='Forced removal of the arrays without asking for confirmation.') parser.add_argument('-c', '--host', help='Host name to be passed to iquery.') parser.add_argument('-p', '--port', help='Port number to be passed to iquery.') parser.add_argument('-t', '--temp-only', action='store_true', help='Limiting the candidates to temp arrays.') parser.add_argument('regex', metavar='REGEX', type=str, nargs='?', default='.*', help='''Regular expression to match against array names. The utility will remove arrays whose names match the regular expression. Default is '.+', meaning to remove all arrays, because the pattern matches all names. The regular expression must match the full array name. For instance, '.*s' will match array 'dogs' because it ends with 's', but will not match array 'moose' because it does not end with 's'.''' ) args = parser.parse_args() try: names = scidb_afl.get_array_names(temp_only=args.temp_only) iquery_cmd = scidb_afl.get_iquery_cmd(args) names_to_remove = [] for name in names: match_name = re.match('^'+args.regex+'$', name) if match_name: names_to_remove.append(name) if not names_to_remove: print 'There is no array to remove.' sys.exit(0) if not args.force: print 'The following arrays are about to be removed:' print names_to_remove proceed = scidb_psf.confirm(prompt='Are you sure you want to remove?', resp=False) if not proceed: sys.exit(0) for name in names_to_remove: scidb_afl.afl(iquery_cmd, 'remove('+name+')') print 'Number of arrays removed =', len(names_to_remove) except Exception, e: print >> sys.stderr, '------ Exception -----------------------------' print >> sys.stderr, e if _print_traceback_upon_exception: print >> sys.stderr, '------ Traceback (for debug purpose) ---------' traceback.print_exc() print >> sys.stderr, '----------------------------------------------' sys.exit(-1) # upon an exception, throw -1
def main(): """The main function gets command-line argument as a pattern, and removes all arrays with that pattern. Note: Empty namespaces will NOT be removed. """ parser = argparse.ArgumentParser( description= 'Remove all SciDB arrays whose names match a given pattern.', epilog='assumptions:\n' + ' - iquery is in your path.', formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument( '-f', '--force', action='store_true', help='Forced removal of the arrays without asking for confirmation.') parser.add_argument('-c', '--host', help='Host name to be passed to iquery.') parser.add_argument('-p', '--port', help='Port number to be passed to iquery.') parser.add_argument('-t', '--temp-only', action='store_true', help='Limiting the candidates to temp arrays.') parser.add_argument('-A', '--auth-file', help='Authentication file to be passed to iquery.') parser.add_argument( '-U', '--user-name', help= 'Deprecated: Use --auth-file instead. User name to be passed to iquery.' ) parser.add_argument( '-P', '--user-password', help= 'Deprecated: Use --auth-file instead. User password to be passed to iquery.' ) parser.add_argument('-v', '--verbose', default=True, help='display verbose output.') parser.add_argument( 'regex', metavar='REGEX', type=str, nargs='?', default='.*', help='''Regular expression to match against array names. The utility will remove arrays whose names match the regular expression. Default is '.+', meaning to remove all arrays, because the pattern matches all names. The regular expression must match the full array name. For instance, '.*s' will match array 'dogs' because it ends with 's', but will not match array 'moose' because it does not end with 's'.''' ) _temp_auth_file = None _arrays_removed = 0 args = parser.parse_args() try: if args.verbose == True: print >> sys.stderr, "args={0}".format(args) if args.user_name and args.user_password and (args.auth_file == None): print >> sys.stderr, '\nWARNING: --user-name and --user-password are deprecated. Use --auth-file instead.\n' _temp_auth_file = create_auth_file(args.user_name, args.user_password) args.auth_file = _temp_auth_file.name args.user_name = None args.user_password = None iquery_cmd = scidb_afl.get_iquery_cmd(args) namespaces = scidb_afl.get_namespace_names(iquery_cmd) if args.verbose == True: print >> sys.stderr, "namespaces={0}".format(namespaces) _arrays_removed = 0 for namespace in namespaces: if args.verbose == True: print >> sys.stderr, "\nSearching namespace: ", namespace names = scidb_afl.get_array_names(iquery_cmd=iquery_cmd, temp_only=args.temp_only, namespace=namespace) if args.verbose == True: print >> sys.stderr, "names={0}".format(names) names_to_remove = [] for name in names: match_name = re.match('^' + args.regex + '$', name) if match_name: if args.verbose == True: print >> sys.stderr, "Schedule {0}.{1} to be removed".format( namespace, name) names_to_remove.append(name) if not names_to_remove: if args.verbose == True: print "There are no arrays to remove in namespace", namespace continue if not args.force: print 'The following arrays are about to be removed from namespace {0}:'.format( namespace) print names_to_remove proceed = scidb_psf.confirm( prompt='Are you sure you want to remove?', resp=False) if not proceed: return for name in names_to_remove: scidb_afl.remove_array(name, namespace, iquery_cmd) if args.verbose == True: print >> sys.stderr, "array {0}.{1} removed".format( namespace, name) _arrays_removed += 1 if namespace != 'public': names = scidb_afl.get_array_names(iquery_cmd=iquery_cmd, temp_only=args.temp_only, namespace=namespace) if not names: scidb_afl.afl(iquery_cmd, "drop_namespace('{0}');".format(namespace)) if args.verbose == True: print >> sys.stderr, "namespace {0} removed".format( namespace) if args.verbose == True: print >> sys.stderr, 'Number of arrays removed =', _arrays_removed if _temp_auth_file: _temp_auth_file.close() _temp_auth_file = None except Exception, e: print >> sys.stderr, '------ Exception -----------------------------' print >> sys.stderr, e if args.verbose == True: print >> sys.stderr, 'Number of arrays removed =', _arrays_removed if args.verbose == True: print >> sys.stderr, '------ Traceback (for debug purpose) ---------' traceback.print_exc() if _temp_auth_file: _temp_auth_file.close() _temp_auth_file = None print >> sys.stderr, '----------------------------------------------' sys.exit(-1) # upon an exception, throw -1