def stop_arkouda_server(): """ Shutdown the Arkouda server. """ server_info = get_server_info() logging.info('Stopping the arkouda server') arkouda.connect(server_info.host, server_info.port) arkouda.shutdown() server_info.process.wait(20)
def stream(self, records): self.logger.debug('Arkouda_Search_Command: %s', self) # logs command line ak_server = self.arkouda_server ak_port = self.arkouda_port data_set = self.data_set data_type = self.data_type search_string = self.search_string ak.connect(ak_server, ak_port) search = ak.load(data_set, dataset=data_type) for record in records: result = ak.min(search) record['Search_results'] = result yield record
def stream(self, records): self.logger.debug('Arkouda_Search_Command: %s', self) # logs command line ak_server = self.arkouda_server ak_port = self.arkouda_port data_set = self.data_set data_type = self.data_type search_string = self.search_string ak.connect(ak_server,ak_port) #search=ak.load(data_set, dataset=data_type) load_array = [] array_position=0 for record in records: load_array[array_position]=record array_position=array_position+1 A = ak.range(1, array_position+1,load_array[])
help='Dtype of arrays (unused)') parser.add_argument( '--numpy', default=False, action='store_true', help='Run the same operation in NumPy to compare performance.') parser.add_argument('--correctness-only', default=False, action='store_true', help='Only check correctness, not performance.') return parser if __name__ == "__main__": import sys parser = create_parser() args = parser.parse_args() ak.verbose = False ak.connect(args.hostname, args.port) if args.correctness_only: check_correctness() sys.exit(0) print("number of trials = ", args.trials) time_ak_noop(args.trials) if args.numpy: time_np_noop(args.size) sys.exit(0)
akA = {k: ak.array(v) for k, v in npA.items()} npB = {'int64': np.random.randint(10, 20, SIZE), 'float64': np.random.randn(SIZE)+10, 'bool': np.random.randint(0, 2, SIZE, dtype='bool')} akB = {k: ak.array(v) for k, v in npB.items()} npCond = np.random.randint(0, 2, SIZE, dtype='bool') akCond = ak.array(npCond) scA = {'int64': 42, 'float64': 2.71828, 'bool': True} scB = {'int64': -1, 'float64': 3.14159, 'bool': False} dtypes = set(npA.keys()) failures = 0 tests = 0 for dtype in dtypes: for (ak1, ak2), (np1, np2) in zip(product((akA, scA), (akB, scB)), product((npA, scA), (npB, scB))): tests += 1 akres = ak.where(akCond, ak1[dtype], ak2[dtype]).to_ndarray() npres = np.where(npCond, np1[dtype], np2[dtype]) if not np.allclose(akres, npres, equal_nan=True): warnings.warn("{} !=\n{}".format(akres, npres)) failures += 1 print("{} failures in {} tests".format(failures, tests)) if __name__ == '__main__': import sys if len(sys.argv) != 3: print(f"Usage: {sys.argv[0]} <server_name> <port>") ak.connect(server=sys.argv[1], port=int(sys.argv[2])) run_tests() sys.exit()
pdextrema = df[vname][pdvals] akextrema = akdf[vname][ak.array(akvals)].to_ndarray() if not np.allclose(pdextrema, akextrema): print(f"Different argmin/argmax: Arkouda failed to find an extremum") print("pd: ", pdextrema) print("ak: ", akextrema) failures += 1 else: # if not np.allclose(pdkeys, akkeys): # print(f"Different keys") # failures += 1 failures += compare_keys(pdkeys, akkeys, levels, pdvals, akvals) # elif not np.allclose(pdvals, akvals): # print(f"Different values (abs diff = {np.where(np.isfinite(pdvals) & np.isfinite(akvals), np.abs(pdvals - akvals), 0).sum()})") # failures += 1 print(f"\n{failures} failures in {tests} tests ({not_impl} not implemented)") if __name__ == '__main__': import sys if len(sys.argv) != 4: print(f"Usage: {sys.argv[0]} <server> <port> <levels=1|2>") sys.exit() levels = int(sys.argv[3]) if levels not in (1, 2): print(f"Levels must be 1 or 2") sys.exit() ak.connect(sys.argv[1], int(sys.argv[2])) run_test(levels) ak.disconnect() sys.exit()
parser.add_argument('--Ne_per_v', type=int, default=10, help='number of edges per vertex') parser.add_argument('--prob', type=float, default=0.01, help='prob of quadrant-0') parser.add_argument('--perm', default=False, action='store_true', help='permute vertex indices/names') args = parser.parse_args() ak.v = False ak.connect(server=args.hostname, port=args.port) print((args.lgNv, args.Ne_per_v, args.prob, args.perm)) (ii, jj) = gen_rmat_edges(args.lgNv, args.Ne_per_v, args.prob, perm=args.perm) print("ii = ", (ii.size, ii)) print("ii(min,max) = ", (ii.min(), ii.max())) print("jj = ", (jj.size, jj)) print("jj(min,max) = ", (jj.min(), jj.max())) nda_ii = ii.to_ndarray() # convert to ndarray for plotting nda_jj = jj.to_ndarray() # convert to ndarray for plotting plt.scatter(nda_ii, nda_jj)
import numpy as np import argparse, sys, gc, time parser = argparse.ArgumentParser( description="Example of cosine distance/similarity in arkouda") parser.add_argument('--server', default="localhost", help='server/Hostname of arkouda server') parser.add_argument('--port', type=int, default=5555, help='Port of arkouda server') args = parser.parse_args() ak.v = False ak.connect(server=args.server, port=args.port) u1 = [1, 0, 0] v1 = [0, 1, 0] d1 = ak_cos_dist(ak.array(u1), ak.array(v1)) print("d1 = ", d1) # d1 should be 1.0 assert (np.allclose(d1, distance.cosine(u1, v1))) u2 = [100, 0, 0] d2 = ak_cos_dist(ak.array(u2), ak.array(v1)) print("d2 = ", d2) # d2 should be 1.0 assert (np.allclose(d2, distance.cosine(u2, v1))) u3 = [1, 1, 0]
#!/usr/bin/env python3 import importlib import numpy as np import math import gc import sys import arkouda as ak print(">>> Sanity checks on the arkouda_server") ak.v = False if len(sys.argv) > 1: ak.connect(server=sys.argv[1], port=sys.argv[2]) else: ak.connect() N = 1_000_000 def pass_fail(f): return ("Passed" if f else "Failed") def check_arange(N): # create np version a = ak.array(np.arange(N)) # create ak version b = ak.arange(N) # print(a,b)
# printCComp flag to print the connected components as they are found # edges needs to be symmetric/undirected def conn_comp(src, dst, printCComp=False, printLayers=False): unvisited = ak.unique(src) if printCComp: print("unvisited size = ", unvisited.size, unvisited) components = [] while unvisited.size > 0: # use lowest numbered vertex as representative vertex rep_vertex = unvisited[0] # bfs from rep_vertex layers, visited = bfs(src, dst, ak.array([rep_vertex]), printLayers) # add verticies in component to list of components components.append(visited) # subtract out visited from unvisited vertices unvisited = ak.setdiff1d(unvisited, visited) if printCComp: print(" visited size = ", visited.size, visited) if printCComp: print("unvisited size = ", unvisited.size, unvisited) return components ak.connect(server="localhost", port=5555) (ii, jj) = gen_rmat_edges(20, 2, 0.03, perm=True) src = ak.concatenate((ii, jj)) # make graph undirected/symmetric dst = ak.concatenate( (jj, ii)) # graph needs to undirected for connected components to work components = conn_comp(src, dst, printCComp=False, printLayers=False) # find components print("number of components = ", len(components)) print("representative vertices = ", [c[0] for c in components]) ak.shutdown()
#!/usr/bin/env python3 import arkouda as ak import sys, os saveone = '/tmp/ak_save.hdf' saveall = '/tmp/ak_save_all.hdf' if len(sys.argv) < 4: print("Usage: {} <hostname> <port> <HDF5_filenames>".format(sys.argv[0])) sys.exit() ak.connect(sys.argv[1], sys.argv[2]) onefile = sys.argv[3] print(ak.ls_hdf(onefile)) allfiles = sys.argv[3:] print(f"srcIP = ak.read_hdf('srcIP', {onefile})") srcIP = ak.read_hdf('srcIP', onefile) print(f"srcIP.save({saveone}, 'srcIP')") srcIP.save(saveone, 'srcIP') print(f"srcIP2 = ak.load({saveone}, 'srcIP')") srcIP2 = ak.load(saveone, 'srcIP') assert (srcIP == srcIP2).all() del srcIP del srcIP2 print(f"df = ak.read_all(['srcPort', 'proto', 'packets'], {allfiles})") df = ak.read_all(['srcPort', 'proto', 'packets'], allfiles) print(f"ak.save_all(df, {saveall})") ak.save_all(df, saveall) print(f"newdf = ak.load_all({saveall})") newdf = ak.load_all(saveall) print(newdf)
#!/usr/bin/python3 # coding=utf-8 from __future__ import absolute_import, division, print_function, unicode_literals import os, sys import time import json import arkouda as ak arkouda_server = '10.50.150.98' arkouda_port = '5555' data_set = 'inital_range' data_type = 'array' search_string = '500493' ak_server = arkouda_server ak_port = arkouda_port data_set = data_set data_type = data_type search_string = search_string ak.connect(ak_server, ak_port) dropped = sys.stdout error = sys.stderr sys.stdout = open('temp.txt', 'w') sys.stdout = dropped sys.stderr = open('temp.txt', 'w') sys.stderr = error search = ak.load(data_set, dataset=data_type) print(search_string) print(search) print(ak.min(search))
def _stop_arkouda_server(host, port): arkouda.connect(host, port) arkouda.shutdown()