def testEnableDisableVerbose(self):
loggerOne = getArkoudaLogger(name='loggerOne', logLevel=LogLevel.INFO)
loggerTwo = getArkoudaLogger('loggerTwo', logLevel=LogLevel.INFO)
loggerOne.debug('loggerOne before enableVerbose')
loggerTwo.debug('loggerTwo before enableVerbose')
ak.enableVerbose()
loggerOne.debug('loggerOne after enableVerbose')
loggerTwo.debug('loggerTwo after enableVerbose')
ak.disableVerbose()
loggerOne.debug('loggerOne after disableVerbose')
loggerTwo.debug('loggerTwo after disableVerbose')
def __init__(self, keys : List[Union[pdarray,np.int64,Strings]],
assume_sorted : bool=False, hash_strings : bool=True) -> None:
self.logger = getArkoudaLogger(name=self.__class__.__name__)
self.assume_sorted = assume_sorted
self.hash_strings = hash_strings
self.keys = keys
if isinstance(keys, pdarray):
self.nkeys = 1
self.size = keys.size
if assume_sorted:
self.permutation = arange(self.size)
else:
self.permutation = argsort(keys)
# for Strings or Categorical
elif hasattr(keys, "group"):
self.nkeys = 1
self.size = keys.size
if assume_sorted:
self.permutation = arange(self.size)
else:
self.permutation = keys.group()
else:
self.nkeys = len(keys)
self.size = keys[0].size
for k in keys:
if k.size != self.size:
raise ValueError("Key arrays must all be same size")
if assume_sorted:
self.permutation = arange(self.size)
else:
self.permutation = coargsort(keys)
# self.permuted_keys = self.keys[self.permutation]
self.find_segments()
def __init__(self, values, **kwargs) -> None:
self.logger = getArkoudaLogger(name=__class__.__name__) # type: ignore
if 'codes' in kwargs and 'categories' in kwargs:
# This initialization is called by Categorical.from_codes()
# The values arg is ignored
self.codes = kwargs['codes']
self.categories = kwargs['categories']
if 'permutation' in kwargs:
self.permutation = cast(pdarray, kwargs['permutation'])
if 'segments' in kwargs:
self.segments = cast(pdarray, kwargs['segments'])
else:
# Typical initialization, called with values
if not isinstance(values, Strings):
raise ValueError(("Categorical: inputs other than " +
"Strings not yet supported"))
g = GroupBy(values)
self.categories = g.unique_keys
self.codes = g.broadcast(arange(self.categories.size),
permute=True)
self.permutation = cast(pdarray, g.permutation)
self.segments = g.segments
# Always set these values
self.size: int_scalars = self.codes.size
self.nlevels = self.categories.size
self.ndim = self.codes.ndim
self.shape = self.codes.shape
self.name: Optional[str] = None
def testErrorHandling(self):
logger = getArkoudaLogger(name='VerboseLogger', logLevel=LogLevel('INFO'))
with self.assertRaises(ValueError):
logger.getHandler('not-a-handler')
with self.assertRaises(TypeError):
logger.disableVerbose(logLevel='INFO')
def testArkoudaLogger(self):
handler = StreamHandler()
handler.name = 'streaming'
logger = getArkoudaLogger(name=self.__class__.__name__,
handlers=[handler])
self.assertEqual(DEBUG, logger.level)
self.assertEqual('LoggerTest', logger.name)
self.assertIsNotNone(logger.getHandler('streaming'))
logger.debug('debug message')
def testUpdateArkoudaLoggerLogLevel(self):
logger = getArkoudaLogger(name='UpdateLogger')
self.assertEqual(DEBUG, logger.level)
logger.debug('debug before level change')
logger.changeLogLevel(LogLevel.WARN)
self.assertEqual(WARN, logger.handlers[0].level)
logger.debug('debug after level change')
handlerOne = StreamHandler()
handlerOne.name = 'handler-one'
handlerOne.setLevel(DEBUG)
handlerTwo = FileHandler(filename='output.txt')
handlerTwo.name = 'handler-two'
handlerTwo.setLevel(INFO)
logger = getArkoudaLogger(name='UpdateLogger',
handlers=[handlerOne,handlerTwo])
logger.changeLogLevel(level=LogLevel.WARN, handlerNames=['handler-one'])
self.assertEqual(WARN,handlerOne.level)
self.assertEqual(INFO, handlerTwo.level)
def __init__(self, offset_attrib: Union[pdarray, np.ndarray],
bytes_attrib: Union[pdarray, np.ndarray]) -> None:
"""
Initializes the Strings instance by setting all instance
attributes, some of which are derived from the array parameters.
Parameters
----------
offset_attrib : Union[pdarray, np.ndarray,array]
the array containing the offsets
bytes_attrib : Union[pdarray, np.ndarray,array]
the array containing the string values
Returns
-------
None
Raises
------
RuntimeError
Raised if there's an error converting a Numpy array or standard
Python array to either the offset_attrib or bytes_attrib
ValueError
Raised if there's an error in generating instance attributes
from either the offset_attrib or bytes_attrib parameter
"""
if isinstance(offset_attrib, pdarray):
# print("In Strings init 1 offset_attrib={}".format(offset_attrib))
self.offsets = offset_attrib
else:
try:
# print("In Strings init 2 offset_attrib={}".format(offset_attrib))
self.offsets = create_pdarray(offset_attrib)
except Exception as e:
raise RuntimeError(e)
if isinstance(bytes_attrib, pdarray):
# print("In Strings init 1 bytes_attrib={}".format(bytes_attrib))
self.bytes = bytes_attrib
else:
try:
# print("In Strings init 1 bytes_attrib={}".format(bytes_attrib))
self.bytes = create_pdarray(bytes_attrib)
except Exception as e:
raise RuntimeError(e)
try:
self.size = self.offsets.size
self.nbytes = self.bytes.size
self.ndim = self.offsets.ndim
self.shape = self.offsets.shape
except Exception as e:
raise ValueError(e)
self.dtype = akstr
self.logger = getArkoudaLogger(name=__class__.__name__) # type: ignore
def testVerbosityControls(self):
logger = getArkoudaLogger(name='VerboseLogger', logLevel=LogLevel('INFO'))
self.assertEqual(INFO, logger.getHandler('console-handler').level)
logger.debug('non-working debug message')
logger.enableVerbose()
self.assertEqual(DEBUG, logger.getHandler('console-handler').level)
logger.debug('working debug message')
logger.disableVerbose()
self.assertEqual(INFO, logger.getHandler('console-handler').level)
logger.debug('next non-working debug message')
def __init__(self, offset_attrib: Union[pdarray, str],
bytes_attrib: Union[pdarray, str]) -> None:
"""
Initializes the Strings instance by setting all instance
attributes, some of which are derived from the array parameters.
Parameters
----------
offset_attrib : Union[pdarray, str]
the array containing the offsets
bytes_attrib : Union[pdarray, str]
the array containing the string values
Returns
-------
None
Raises
------
RuntimeError
Raised if there's an error converting a server-returned str-descriptor
or pdarray to either the offset_attrib or bytes_attrib
ValueError
Raised if there's an error in generating instance attributes
from either the offset_attrib or bytes_attrib parameter
"""
if isinstance(offset_attrib, pdarray):
self.offsets = offset_attrib
else:
try:
self.offsets = create_pdarray(offset_attrib)
except Exception as e:
raise RuntimeError(e)
if isinstance(bytes_attrib, pdarray):
self.bytes = bytes_attrib
else:
try:
self.bytes = create_pdarray(bytes_attrib)
except Exception as e:
raise RuntimeError(e)
try:
self.size = self.offsets.size
self.nbytes = self.bytes.size
self.ndim = self.offsets.ndim
self.shape = self.offsets.shape
except Exception as e:
raise ValueError(e)
self.dtype = npstr
self.name: Optional[str] = None
self.logger = getArkoudaLogger(name=__class__.__name__) # type: ignore
def __init__(self,
keys: Union[pdarray, Strings, 'Categorical',
List[Union[pdarray, np.int64, Strings]]],
assume_sorted: bool = False,
hash_strings: bool = True) -> None:
from arkouda.categorical import Categorical
self.logger = getArkoudaLogger(name=self.__class__.__name__)
self.assume_sorted = assume_sorted
self.hash_strings = hash_strings
self.keys: Union[pdarray, Strings, Categorical]
if isinstance(keys, pdarray):
if keys.dtype != int64:
raise TypeError(
'GroupBy only supports pdarrays with a dtype int64')
self.keys = cast(pdarray, keys)
self.nkeys = 1
self.size = cast(int, keys.size)
if assume_sorted:
self.permutation = cast(pdarray, arange(self.size))
else:
self.permutation = cast(pdarray, argsort(keys))
elif hasattr(keys, "group"): # for Strings or Categorical
self.nkeys = 1
self.keys = cast(Union[Strings, Categorical], keys)
self.size = cast(int, self.keys.size) # type: ignore
if assume_sorted:
self.permutation = cast(pdarray, arange(self.size))
else:
self.permutation = cast(Union[Strings, Categorical],
keys).group()
else:
self.keys = cast(Union[pdarray, Strings, Categorical], keys)
self.nkeys = len(keys)
self.size = cast(int, keys[0].size) # type: ignore
for k in keys:
if k.size != self.size:
raise ValueError("Key arrays must all be same size")
if assume_sorted:
self.permutation = cast(pdarray, arange(self.size))
else:
self.permutation = cast(
pdarray, coargsort(cast(Sequence[pdarray], keys)))
# self.permuted_keys = self.keys[self.permutation]
self.find_segments()
def __init__(self, pattern: str_scalars, parent_entry_name: str) -> None:
self.objtype = type(self).__name__
try:
self.pattern = pattern
re.compile(self.pattern)
except Exception as e:
raise ValueError(e)
if re.search(self.pattern, ''):
# TODO remove once changes from chapel issue #18639 are in arkouda
raise ValueError("regex operations with a pattern that matches the empty string are not currently supported")
self.parent_entry_name = parent_entry_name
self.num_matches: pdarray
self.starts: pdarray
self.lengths: pdarray
self.indices: pdarray
self.search_bool: pdarray
self.search_ind: pdarray
self.match_bool: pdarray
self.match_ind: pdarray
self.full_match_bool: pdarray
self.full_match_ind: pdarray
self.populated = False
self.logger = getArkoudaLogger(name=__class__.__name__) # type: ignore
from typeguard import typechecked
from arkouda.client import generic_msg, get_config
from arkouda.pdarrayclass import pdarray, create_pdarray
from arkouda.pdarraycreation import zeros, zeros_like, array
from arkouda.sorting import argsort
from arkouda.strings import Strings
from arkouda.logger import getArkoudaLogger
Categorical = ForwardRef('Categorical')
__all__ = [
"unique", "in1d", "concatenate", "union1d", "intersect1d", "setdiff1d",
"setxor1d"
]
logger = getArkoudaLogger(name='pdarraysetops')
@typechecked
def unique(
pda: Union[pdarray, Strings, 'Categorical'], # type: ignore
return_counts: bool = False
) -> Union[Union[pdarray, Strings, 'Categorical'], # type: ignore
Tuple[Union[pdarray, Strings, 'Categorical'],
Optional[pdarray]]]: #type: ignore
"""
Find the unique elements of an array.
Returns the unique elements of an array, sorted if the values are integers.
There is an optional output in addition to the unique elements: the number
of times each unique value comes up in the input array.
socket = context.socket(zmq.REQ)
connected = False
# username and token for when basic authentication is enabled
username = ''
token = ''
# verbose flag for arkouda module
verboseDefVal = False
verbose = verboseDefVal
# threshold for __iter__() to limit comms to arkouda_server
pdarrayIterThreshDefVal = 100
pdarrayIterThresh = pdarrayIterThreshDefVal
maxTransferBytesDefVal = 2**30
maxTransferBytes = maxTransferBytesDefVal
AllSymbols = "__AllSymbols__"
logger = getArkoudaLogger(name='Arkouda Client')
# reset settings to default values
def set_defaults() -> None:
"""
Sets client variables including verbose, maxTransferBytes and
pdarrayIterThresh to default values.
Returns
-------
None
"""
global verbose, maxTransferBytes, pdarrayIterThresh
verbose = verboseDefVal
pdarrayIterThresh = pdarrayIterThreshDefVal
socket = context.socket(zmq.REQ)
connected = False
serverConfig = None
# username and token for when basic authentication is enabled
username = ''
token = ''
# verbose flag for arkouda module
verboseDefVal = False
verbose = verboseDefVal
# threshold for __iter__() to limit comms to arkouda_server
pdarrayIterThreshDefVal = 100
pdarrayIterThresh = pdarrayIterThreshDefVal
maxTransferBytesDefVal = 2**30
maxTransferBytes = maxTransferBytesDefVal
logger = getArkoudaLogger(name='Arkouda Client')
clientLogger = getArkoudaLogger(name='Arkouda User Logger', logFormat='%(message)s')
# Print splash message
print('{}'.format(pyfiglet.figlet_format('Arkouda')))
print('Client Version: {}'.format(__version__)) # type: ignore
# reset settings to default values
def set_defaults() -> None:
"""
Sets client variables including verbose, maxTransferBytes and
pdarrayIterThresh to default values.
Returns
-------
None
from typing import Tuple, Union
from typeguard import typechecked
import json, struct
import numpy as np
from arkouda.client import generic_msg, verbose, maxTransferBytes, pdarrayIterThresh
from arkouda.dtypes import *
from arkouda.dtypes import structDtypeCodes, NUMBER_FORMAT_STRINGS
from arkouda.logger import getArkoudaLogger
__all__ = [
"pdarray", "info", "clear", "any", "all", "is_sorted", "sum", "prod",
"min", "max", "argmin", "argmax", "mean", "var", "std", "mink", "maxk",
"argmink", "argmaxk"
]
logger = getArkoudaLogger(name='pdarray')
@typechecked
def parse_single_value(msg: str) -> object:
"""
Attempt to convert a scalar return value from the arkouda server to a
numpy scalar in Python. The user should not call this function directly.
Parameters
----------
msg : str
scalar value in string form to be converted to a numpy scalar
Returns
-------
