def stop_arkouda_server():
"""
Shutdown the Arkouda server.
"""
server_info = get_server_info()
logging.info('Stopping the arkouda server')
arkouda.connect(server_info.host, server_info.port)
arkouda.shutdown()
server_info.process.wait(20)
def stream(self, records):
self.logger.debug('Arkouda_Search_Command: %s',
self) # logs command line
ak_server = self.arkouda_server
ak_port = self.arkouda_port
data_set = self.data_set
data_type = self.data_type
search_string = self.search_string
ak.connect(ak_server, ak_port)
search = ak.load(data_set, dataset=data_type)
for record in records:
result = ak.min(search)
record['Search_results'] = result
yield record
def stream(self, records):
self.logger.debug('Arkouda_Search_Command: %s', self) # logs command line
ak_server = self.arkouda_server
ak_port = self.arkouda_port
data_set = self.data_set
data_type = self.data_type
search_string = self.search_string
ak.connect(ak_server,ak_port)
#search=ak.load(data_set, dataset=data_type)
load_array = []
array_position=0
for record in records:
load_array[array_position]=record
array_position=array_position+1
A = ak.range(1, array_position+1,load_array[])
help='Dtype of arrays (unused)')
parser.add_argument(
'--numpy',
default=False,
action='store_true',
help='Run the same operation in NumPy to compare performance.')
parser.add_argument('--correctness-only',
default=False,
action='store_true',
help='Only check correctness, not performance.')
return parser
if __name__ == "__main__":
import sys
parser = create_parser()
args = parser.parse_args()
ak.verbose = False
ak.connect(args.hostname, args.port)
if args.correctness_only:
check_correctness()
sys.exit(0)
print("number of trials = ", args.trials)
time_ak_noop(args.trials)
if args.numpy:
time_np_noop(args.size)
sys.exit(0)
akA = {k: ak.array(v) for k, v in npA.items()}
npB = {'int64': np.random.randint(10, 20, SIZE),
'float64': np.random.randn(SIZE)+10,
'bool': np.random.randint(0, 2, SIZE, dtype='bool')}
akB = {k: ak.array(v) for k, v in npB.items()}
npCond = np.random.randint(0, 2, SIZE, dtype='bool')
akCond = ak.array(npCond)
scA = {'int64': 42, 'float64': 2.71828, 'bool': True}
scB = {'int64': -1, 'float64': 3.14159, 'bool': False}
dtypes = set(npA.keys())
failures = 0
tests = 0
for dtype in dtypes:
for (ak1, ak2), (np1, np2) in zip(product((akA, scA), (akB, scB)),
product((npA, scA), (npB, scB))):
tests += 1
akres = ak.where(akCond, ak1[dtype], ak2[dtype]).to_ndarray()
npres = np.where(npCond, np1[dtype], np2[dtype])
if not np.allclose(akres, npres, equal_nan=True):
warnings.warn("{} !=\n{}".format(akres, npres))
failures += 1
print("{} failures in {} tests".format(failures, tests))
if __name__ == '__main__':
import sys
if len(sys.argv) != 3:
print(f"Usage: {sys.argv[0]} <server_name> <port>")
ak.connect(server=sys.argv[1], port=int(sys.argv[2]))
run_tests()
sys.exit()
pdextrema = df[vname][pdvals]
akextrema = akdf[vname][ak.array(akvals)].to_ndarray()
if not np.allclose(pdextrema, akextrema):
print(f"Different argmin/argmax: Arkouda failed to find an extremum")
print("pd: ", pdextrema)
print("ak: ", akextrema)
failures += 1
else:
# if not np.allclose(pdkeys, akkeys):
# print(f"Different keys")
# failures += 1
failures += compare_keys(pdkeys, akkeys, levels, pdvals, akvals)
# elif not np.allclose(pdvals, akvals):
# print(f"Different values (abs diff = {np.where(np.isfinite(pdvals) & np.isfinite(akvals), np.abs(pdvals - akvals), 0).sum()})")
# failures += 1
print(f"\n{failures} failures in {tests} tests ({not_impl} not implemented)")
if __name__ == '__main__':
import sys
if len(sys.argv) != 4:
print(f"Usage: {sys.argv[0]} <server> <port> <levels=1|2>")
sys.exit()
levels = int(sys.argv[3])
if levels not in (1, 2):
print(f"Levels must be 1 or 2")
sys.exit()
ak.connect(sys.argv[1], int(sys.argv[2]))
run_test(levels)
ak.disconnect()
sys.exit()
parser.add_argument('--Ne_per_v',
type=int,
default=10,
help='number of edges per vertex')
parser.add_argument('--prob',
type=float,
default=0.01,
help='prob of quadrant-0')
parser.add_argument('--perm',
default=False,
action='store_true',
help='permute vertex indices/names')
args = parser.parse_args()
ak.v = False
ak.connect(server=args.hostname, port=args.port)
print((args.lgNv, args.Ne_per_v, args.prob, args.perm))
(ii, jj) = gen_rmat_edges(args.lgNv,
args.Ne_per_v,
args.prob,
perm=args.perm)
print("ii = ", (ii.size, ii))
print("ii(min,max) = ", (ii.min(), ii.max()))
print("jj = ", (jj.size, jj))
print("jj(min,max) = ", (jj.min(), jj.max()))
nda_ii = ii.to_ndarray() # convert to ndarray for plotting
nda_jj = jj.to_ndarray() # convert to ndarray for plotting
plt.scatter(nda_ii, nda_jj)
import numpy as np
import argparse, sys, gc, time
parser = argparse.ArgumentParser(
description="Example of cosine distance/similarity in arkouda")
parser.add_argument('--server',
default="localhost",
help='server/Hostname of arkouda server')
parser.add_argument('--port',
type=int,
default=5555,
help='Port of arkouda server')
args = parser.parse_args()
ak.v = False
ak.connect(server=args.server, port=args.port)
u1 = [1, 0, 0]
v1 = [0, 1, 0]
d1 = ak_cos_dist(ak.array(u1), ak.array(v1))
print("d1 = ", d1)
# d1 should be 1.0
assert (np.allclose(d1, distance.cosine(u1, v1)))
u2 = [100, 0, 0]
d2 = ak_cos_dist(ak.array(u2), ak.array(v1))
print("d2 = ", d2)
# d2 should be 1.0
assert (np.allclose(d2, distance.cosine(u2, v1)))
u3 = [1, 1, 0]
#!/usr/bin/env python3
import importlib
import numpy as np
import math
import gc
import sys
import arkouda as ak
print(">>> Sanity checks on the arkouda_server")
ak.v = False
if len(sys.argv) > 1:
ak.connect(server=sys.argv[1], port=sys.argv[2])
else:
ak.connect()
N = 1_000_000
def pass_fail(f):
return ("Passed" if f else "Failed")
def check_arange(N):
# create np version
a = ak.array(np.arange(N))
# create ak version
b = ak.arange(N)
# print(a,b)
# printCComp flag to print the connected components as they are found
# edges needs to be symmetric/undirected
def conn_comp(src, dst, printCComp=False, printLayers=False):
unvisited = ak.unique(src)
if printCComp: print("unvisited size = ", unvisited.size, unvisited)
components = []
while unvisited.size > 0:
# use lowest numbered vertex as representative vertex
rep_vertex = unvisited[0]
# bfs from rep_vertex
layers, visited = bfs(src, dst, ak.array([rep_vertex]), printLayers)
# add verticies in component to list of components
components.append(visited)
# subtract out visited from unvisited vertices
unvisited = ak.setdiff1d(unvisited, visited)
if printCComp: print(" visited size = ", visited.size, visited)
if printCComp: print("unvisited size = ", unvisited.size, unvisited)
return components
ak.connect(server="localhost", port=5555)
(ii, jj) = gen_rmat_edges(20, 2, 0.03, perm=True)
src = ak.concatenate((ii, jj)) # make graph undirected/symmetric
dst = ak.concatenate(
(jj, ii)) # graph needs to undirected for connected components to work
components = conn_comp(src, dst, printCComp=False,
printLayers=False) # find components
print("number of components = ", len(components))
print("representative vertices = ", [c[0] for c in components])
ak.shutdown()
#!/usr/bin/env python3
import arkouda as ak
import sys, os
saveone = '/tmp/ak_save.hdf'
saveall = '/tmp/ak_save_all.hdf'
if len(sys.argv) < 4:
print("Usage: {} <hostname> <port> <HDF5_filenames>".format(sys.argv[0]))
sys.exit()
ak.connect(sys.argv[1], sys.argv[2])
onefile = sys.argv[3]
print(ak.ls_hdf(onefile))
allfiles = sys.argv[3:]
print(f"srcIP = ak.read_hdf('srcIP', {onefile})")
srcIP = ak.read_hdf('srcIP', onefile)
print(f"srcIP.save({saveone}, 'srcIP')")
srcIP.save(saveone, 'srcIP')
print(f"srcIP2 = ak.load({saveone}, 'srcIP')")
srcIP2 = ak.load(saveone, 'srcIP')
assert (srcIP == srcIP2).all()
del srcIP
del srcIP2
print(f"df = ak.read_all(['srcPort', 'proto', 'packets'], {allfiles})")
df = ak.read_all(['srcPort', 'proto', 'packets'], allfiles)
print(f"ak.save_all(df, {saveall})")
ak.save_all(df, saveall)
print(f"newdf = ak.load_all({saveall})")
newdf = ak.load_all(saveall)
print(newdf)
#!/usr/bin/python3
# coding=utf-8
from __future__ import absolute_import, division, print_function, unicode_literals
import os, sys
import time
import json
import arkouda as ak
arkouda_server = '10.50.150.98'
arkouda_port = '5555'
data_set = 'inital_range'
data_type = 'array'
search_string = '500493'
ak_server = arkouda_server
ak_port = arkouda_port
data_set = data_set
data_type = data_type
search_string = search_string
ak.connect(ak_server, ak_port)
dropped = sys.stdout
error = sys.stderr
sys.stdout = open('temp.txt', 'w')
sys.stdout = dropped
sys.stderr = open('temp.txt', 'w')
sys.stderr = error
search = ak.load(data_set, dataset=data_type)
print(search_string)
print(search)
print(ak.min(search))
def _stop_arkouda_server(host, port):
arkouda.connect(host, port)
arkouda.shutdown()
