def blz_llvec_fun(act, pred, output_vector_type ="numpy"):
epsilon = 1e-15
pred[pred<epsilon] = epsilon
pred[pred > (1-epsilon) ] = 1-epsilon
bt = blz.btable([act,pred],names=["y","py"])
if output_vector_type =="numpy":
return bt.eval("-y*log(py) - (1-y)*log(1-py)",out_flavor="numpy")
else:
return bt.eval("-y*log(py) - (1-y)*log(1-py)")
def test07(self):
"""Testing `wherechunks` method with a `limit`, `skip` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=N-M-2, skip=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - M - 2)
self.assert_(s == np.arange(M+1, N-1).sum())
def test05(self):
"""Testing `wherechunks` method with a `limit` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == M)
self.assert_(s == M * ((M + 1) / 2)) # Gauss summation formula
def test00(self):
"""Testing `wherechunks` method with only an expression"""
N = int(1e4)
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2'):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - 1)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test00(self):
"""Testing `wherechunks` method with only an expression"""
N = int(1e4)
ra = np.fromiter(((i, i * 2., i * 3) for i in xrange(N)),
dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2'):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - 1)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test07(self):
"""Testing `wherechunks` method with a `limit`, `skip` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i * 2., i * 3) for i in xrange(N)),
dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=N - M - 2, skip=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == N - M - 2)
self.assert_(s == np.arange(M + 1, N - 1).sum())
def test03(self):
"""Testing `wherechunks` method with a `outfields` with 1 field"""
N = int(1e4)
ra = np.fromiter(((i, i, i * 3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', outfields=('f1', )):
self.assert_(block.dtype.names == ('f1', ))
l += len(block)
s += block['f1'].sum()
self.assert_(l == N - 1)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test02(self):
"""Testing `wherechunks` method with a `outfields` with 2 fields"""
N = int(1e4)
ra = np.fromiter(((i, i, i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', outfields=('f1','f2')):
self.assert_(block.dtype.names == ('f1','f2'))
l += len(block)
s += block['f1'].sum()
self.assert_(l == N - 1)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test05(self):
"""Testing `wherechunks` method with a `limit` parameter"""
N, M = int(1e4), 101
ra = np.fromiter(((i, i * 2., i * 3) for i in xrange(N)),
dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f1 < f2', limit=M):
l += len(block)
s += block['f0'].sum()
self.assert_(l == M)
self.assert_(s == M * ((M + 1) / 2)) # Gauss summation formula
def modelPredictor(modelsPath_modelIndex_dataPath_colNames_tuple):
"""
Input: A tuple, with following two attributes (with order):
modelsPath: string, the path to the trained models. (pickle file)
modelIndex: integer, the index of the model to predict.
dataPath: string, the path to the data.
colNames: a list of strings, column names of the output table. It should be like ["Id", "V1", ...]
Output: A btable, consists of Id column, Predicted column and the data.
Notes:
modelPredictor will create following directories for you if they do not exist.
1. Model_No{modelIndex}_predicted_array: it will be under the dataPath.
"""
# Set up necessary constance.
divideN = 300000
modelsPath, modelIndex, dataPath, colNames = modelsPath_modelIndex_dataPath_colNames_tuple
def data_abspath(colname):
return os.path.abspath(os.path.join(dataPath, colname))
with open(modelsPath, "rb") as rf:
models = pickle.load(rf)
model = models[modelIndex]
del models
# Read in data with btable.
Id = blz.open(os.path.join(dataPath, colNames[0]))
totalN = len(Id)
if totalN % divideN == 0:
nodes_list = [i * divideN for i in range(totalN / divideN + 1)]
else:
nodes_list = [i * divideN for i in range(totalN / divideN + 1)] + [totalN]
nodes_pair_list = zip(nodes_list[:-1], nodes_list[1:])
# Prediction.
y_predict = np.zeros(totalN)
print "[Model No.{modelIndex}] Prediction process begins.".format(modelIndex = modelIndex)
for begin, end in nodes_pair_list:
print "[Model No.{modelIndex}] Processing {begin} ~ {end} observations.".format(modelIndex=modelIndex, begin = begin + 1, end = end)
columns = [blz.open(os.path.join(dataPath, colname))[begin:end] for colname in colNames[1:]]
X = np.column_stack(columns)
temp = model.predict(X)
y_predict[begin:end] = temp
columns = [Id, blz.barray(y_predict)]
data_rootdir = os.path.join(dataPath, "Model_No{modelIndex}_predicted_array".format(modelIndex = modelIndex))
if data_rootdir in os.listdir(dataPath):
print "Removing Old result_table directory for new btable."
command = "rm -rf " + data_rootdir
os.system(command)
final_table = blz.btable(columns = columns, names = ["Id", "Predict"], rootdir = data_rootdir)
print "The result_table btable rootdir is under {path}".format(path=data_rootdir)
def create_kaggle_submit_csv(self, submit_format="%d,%.6f"):
assert "predict_proba" in self.list_all_predictions["testing"]
prediction_prob = self.load_prediction_blz(datatype="testing", valuetype="predict_proba")[:,1]
ids_barray = blz.open(os.path.join(tools.TESTING_BLZ_PATH,TESTING_COLUMN_NAMES[0]))
bt = blz.btable(columns=[ids_barray,prediction_prob], names=["Id","Predicted"])
all_results = [submit_format % tuple(xx) for xx in bt.iter()]
all_results_string = "\n".join([",".join(bt.names)] + all_results)
submit_filename = "%s_%s.csv" % (self.model_id, datetime.datetime.now().strftime("%Y%m%d%H%M%S"))
submit_filepath = os.path.join(SUBMITS_PATH,submit_filename)
with open(submit_filepath,"w") as wf:
wf.write(all_results_string)
def test01(self):
"""Testing `wherechunks` method with a `blen`"""
N = int(1e4)
ra = np.fromiter(((i, i*2., i*3) for i in xrange(N)), dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f0 <= f1', blen=100):
l += len(block)
# All blocks should be of length 100, except the last one,
# which should be 0
self.assert_(len(block) in (0, 100))
s += block['f0'].sum()
self.assert_(l == N)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def test01(self):
"""Testing `wherechunks` method with a `blen`"""
N = int(1e4)
ra = np.fromiter(((i, i * 2., i * 3) for i in xrange(N)),
dtype='i4,f8,i8')
t = blz.btable(ra)
l, s = 0, 0
for block in blz.whereblocks(t, 'f0 <= f1', blen=100):
l += len(block)
# All blocks should be of length 100, except the last one,
# which should be 0
self.assert_(len(block) in (0, 100))
s += block['f0'].sum()
self.assert_(l == N)
self.assert_(s == (N - 1) * (N / 2)) # Gauss summation formula
def groupby(sreader, key, val, dtype, path=None, lines_per_chunk=LPC):
"""Group the `val` field in `sreader` stream of lines by `key` index.
Parameters
----------
sreader : iterator
Iterator over a stream of CSV lines.
key : string
The name of the field to be grouped by.
val : string
The field name with the values that have to be grouped.
dtype : dynd dtype
The DyND data type with all the fields of the CSV lines,
including the `key` and `val` names.
path : string
The path of the file where the BLZ array with the final
grouping will be stored. If None (default), the BLZ will be
stored in-memory (and hence non-persistent).
lines_per_chunk : int
The number of chunks that have to be read to be grouped by
in-memory. For optimal perfomance, some experimentation
should be needed. The default value should work reasonably
well, though.
Returns
-------
output : BLZ table
Returns a BLZ table with column names that are the groups
resulting from the groupby operation. The columns are filled
with the `val` field of the lines delivered by `sreader`.
"""
try:
nptype = get_nptype(dtype, val)
except ValueError:
raise ValueError("`val` should be a valid field")
# Start reading chunks
prev_keys = set()
while True:
ndbuf = nd.array(islice(sreader, lines_per_chunk), dtype)
if len(ndbuf) == 0: break # CSV data exhausted
# Do the groupby for this chunk
keys = getattr(ndbuf, key)
if val is None:
vals = ndbuf
else:
vals = getattr(ndbuf, val)
sby = nd.groupby(vals, keys)
lkeys = nd.as_py(sby.groups)
skeys = set(lkeys)
# BLZ does not understand dynd objects (yet)
sby = nd.as_py(sby.eval())
if len(prev_keys) == 0:
# Add the initial keys to a BLZ table
columns = [np.array(sby[i], nptype) for i in range(len(lkeys))]
ssby = blz.btable(columns=columns, names=lkeys, rootdir=path,
mode='w')
else:
# Have we new keys?
new_keys = skeys.difference(prev_keys)
for new_key in new_keys:
# Get the index of the new key
idx = lkeys.index(new_key)
# and add the values as a new columns
ssby.addcol(sby[idx], new_key, dtype=nptype)
# Now fill the pre-existing keys
existing_keys = skeys.intersection(prev_keys)
for existing_key in existing_keys:
# Get the index of the existing key
idx = lkeys.index(existing_key)
# and append the values here
ssby[existing_key].append(sby[idx])
# Add the new keys to the existing ones
prev_keys |= skeys
# Before returning, flush all data into disk
if path is not None:
ssby.flush()
return ssby
def groupby(sreader, key, val, dtype, path=None, lines_per_chunk=LPC):
"""Group the `val` field in `sreader` stream of lines by `key` index.
Parameters
----------
sreader : iterator
Iterator over a stream of CSV lines.
key : string
The name of the field to be grouped by.
val : string
The field name with the values that have to be grouped.
dtype : dynd dtype
The DyND data type with all the fields of the CSV lines,
including the `key` and `val` names.
path : string
The path of the file where the BLZ array with the final
grouping will be stored. If None (default), the BLZ will be
stored in-memory (and hence non-persistent).
lines_per_chunk : int
The number of chunks that have to be read to be grouped by
in-memory. For optimal perfomance, some experimentation
should be needed. The default value should work reasonably
well, though.
Returns
-------
output : BLZ table
Returns a BLZ table with column names that are the groups
resulting from the groupby operation. The columns are filled
with the `val` field of the lines delivered by `sreader`.
"""
try:
nptype = get_nptype(dtype, val)
except ValueError:
raise ValueError("`val` should be a valid field")
# Start reading chunks
prev_keys = set()
while True:
ndbuf = nd.array(islice(sreader, lines_per_chunk), dtype)
if len(ndbuf) == 0: break # CSV data exhausted
# Do the groupby for this chunk
keys = getattr(ndbuf, key)
if val is None:
vals = ndbuf
else:
vals = getattr(ndbuf, val)
sby = nd.groupby(vals, keys)
lkeys = nd.as_py(sby.groups)
skeys = set(lkeys)
# BLZ does not understand dynd objects (yet)
sby = nd.as_py(sby.eval())
if len(prev_keys) == 0:
# Add the initial keys to a BLZ table
columns = [np.array(sby[i], nptype) for i in range(len(lkeys))]
ssby = blz.btable(columns=columns,
names=lkeys,
rootdir=path,
mode='w')
else:
# Have we new keys?
new_keys = skeys.difference(prev_keys)
for new_key in new_keys:
# Get the index of the new key
idx = lkeys.index(new_key)
# and add the values as a new columns
ssby.addcol(sby[idx], new_key, dtype=nptype)
# Now fill the pre-existing keys
existing_keys = skeys.intersection(prev_keys)
for existing_key in existing_keys:
# Get the index of the existing key
idx = lkeys.index(existing_key)
# and append the values here
ssby[existing_key].append(sby[idx])
# Add the new keys to the existing ones
prev_keys |= skeys
# Before returning, flush all data into disk
if path is not None:
ssby.flush()
return ssby
N = 1e7 # the number of elements in x M = 100000 # the elements to get clevel = 1 # the compression level print "Creating inputs with %d elements..." % N bparams = blz.bparams(clevel) #x = np.arange(N) x = np.zeros(N, dtype="f8") y = x.copy() z = x.copy() cx = blz.barray(x, bparams=bparams) cy = cx.copy() cz = cx.copy() ct = blz.btable((cx, cy, cz), names=['x','y','z']) t = ct[:] print "Starting benchmark now for getting %d elements..." % M # Retrieve from a ndarray t0 = time() vals = [x[i] for i in xrange(0, M, 3)] print "Time for array--> %.3f" % (time()-t0,) print "vals-->", len(vals) #blz.set_num_threads(blz.ncores//2) # Retrieve from a barray t0 = time() cvals = [cx[i] for i in xrange(0, M, 3)] #cvals = cx[:M:3][:].tolist()
N = 1e7 # the number of elements in x M = 100000 # the elements to get clevel = 1 # the compression level print "Creating inputs with %d elements..." % N bparams = blz.bparams(clevel) #x = np.arange(N) x = np.zeros(N, dtype="f8") y = x.copy() z = x.copy() cx = blz.barray(x, bparams=bparams) cy = cx.copy() cz = cx.copy() ct = blz.btable((cx, cy, cz), names=['x', 'y', 'z']) t = ct[:] print "Starting benchmark now for getting %d elements..." % M # Retrieve from a ndarray t0 = time() vals = [x[i] for i in xrange(0, M, 3)] print "Time for array--> %.3f" % (time() - t0, ) print "vals-->", len(vals) #blz.set_num_threads(blz.ncores//2) # Retrieve from a barray t0 = time() cvals = [cx[i] for i in xrange(0, M, 3)] #cvals = cx[:M:3][:].tolist()
def array(obj, dshape=None, ddesc=None):
"""Create a Blaze array.
Parameters
----------
obj : array_like
Initial contents for the array.
dshape : datashape
The datashape for the resulting array. By default the
datashape will be inferred from data. If an explicit dshape is
provided, the input data will be coerced into the provided
dshape.
ddesc : data descriptor instance
This comes with the necessary info for storing the data. If
None, a DyND_DDesc will be used.
Returns
-------
out : a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
if ((obj is not None) and (not inspect.isgenerator(obj))
and (dshape is not None)):
dt = ndt.type(str(dshape))
if dt.ndim > 0:
obj = nd.array(obj, type=dt, access='rw')
else:
obj = nd.array(obj, dtype=dt, access='rw')
if obj is None and ddesc is None:
raise ValueError('you need to specify at least `obj` or `ddesc`')
if isinstance(obj, Array):
return obj
elif isinstance(obj, DDesc):
if ddesc is None:
ddesc = obj
return Array(ddesc)
else:
raise ValueError(('you cannot specify `ddesc` when `obj` '
'is already a DDesc instance'))
if ddesc is None:
# Use a dynd ddesc by default
try:
array = nd.asarray(obj, access='rw')
except:
raise ValueError(('failed to construct a dynd array from '
'object %r') % obj)
ddesc = DyND_DDesc(array)
return Array(ddesc)
# The DDesc has been specified
if isinstance(ddesc, DyND_DDesc):
if obj is not None:
raise ValueError(('you cannot specify simultaneously '
'`obj` and a DyND `ddesc`'))
return Array(ddesc)
elif isinstance(ddesc, BLZ_DDesc):
if inspect.isgenerator(obj):
dt = None if dshape is None else to_numpy_dtype(dshape)
# TODO: Generator logic could go inside barray
ddesc.blzarr = blz.fromiter(obj,
dtype=dt,
count=-1,
rootdir=ddesc.path,
mode=ddesc.mode,
**ddesc.kwargs)
else:
if isinstance(obj, nd.array):
obj = nd.as_numpy(obj)
if dshape and isinstance(dshape.measure, datashape.Record):
ddesc.blzarr = blz.btable(obj,
rootdir=ddesc.path,
mode=ddesc.mode,
**ddesc.kwargs)
else:
ddesc.blzarr = blz.barray(obj,
rootdir=ddesc.path,
mode=ddesc.mode,
**ddesc.kwargs)
elif isinstance(ddesc, HDF5_DDesc):
if isinstance(obj, nd.array):
obj = nd.as_numpy(obj)
with tb.open_file(ddesc.path, mode=ddesc.mode) as f:
where, name = split_path(ddesc.datapath)
if dshape and isinstance(dshape.measure, datashape.Record):
# Convert the structured array to unaligned dtype
# We need that because PyTables only accepts unaligned types,
# which are the default in NumPy
obj = np.array(obj, datashape.to_numpy_dtype(dshape.measure))
f.create_table(where, name, filters=ddesc.filters, obj=obj)
else:
f.create_earray(where, name, filters=ddesc.filters, obj=obj)
ddesc.mode = 'a' # change into 'a'ppend mode for further operations
return Array(ddesc)
def array(obj, dshape=None, ddesc=None):
"""Create a Blaze array.
Parameters
----------
obj : array_like
Initial contents for the array.
dshape : datashape
The datashape for the resulting array. By default the
datashape will be inferred from data. If an explicit dshape is
provided, the input data will be coerced into the provided
dshape.
ddesc : data descriptor instance
This comes with the necessary info for storing the data. If
None, a DyND_DDesc will be used.
Returns
-------
out : a concrete blaze array.
"""
dshape = _normalize_dshape(dshape)
if ((obj is not None) and
(not inspect.isgenerator(obj)) and
(dshape is not None)):
dt = ndt.type(str(dshape))
if dt.ndim > 0:
obj = nd.array(obj, type=dt, access='rw')
else:
obj = nd.array(obj, dtype=dt, access='rw')
if obj is None and ddesc is None:
raise ValueError('you need to specify at least `obj` or `ddesc`')
if isinstance(obj, Array):
return obj
elif isinstance(obj, DDesc):
if ddesc is None:
ddesc = obj
return Array(ddesc)
else:
raise ValueError(('you cannot specify `ddesc` when `obj` '
'is already a DDesc instance'))
if ddesc is None:
# Use a dynd ddesc by default
try:
array = nd.asarray(obj, access='rw')
except:
raise ValueError(('failed to construct a dynd array from '
'object %r') % obj)
ddesc = DyND_DDesc(array)
return Array(ddesc)
# The DDesc has been specified
if isinstance(ddesc, DyND_DDesc):
if obj is not None:
raise ValueError(('you cannot specify simultaneously '
'`obj` and a DyND `ddesc`'))
return Array(ddesc)
elif isinstance(ddesc, BLZ_DDesc):
if inspect.isgenerator(obj):
dt = None if dshape is None else to_numpy_dtype(dshape)
# TODO: Generator logic could go inside barray
ddesc.blzarr = blz.fromiter(obj, dtype=dt, count=-1,
rootdir=ddesc.path, mode=ddesc.mode,
**ddesc.kwargs)
else:
if isinstance(obj, nd.array):
obj = nd.as_numpy(obj)
if dshape and isinstance(dshape.measure, datashape.Record):
ddesc.blzarr = blz.btable(
obj, rootdir=ddesc.path, mode=ddesc.mode, **ddesc.kwargs)
else:
ddesc.blzarr = blz.barray(
obj, rootdir=ddesc.path, mode=ddesc.mode, **ddesc.kwargs)
elif isinstance(ddesc, HDF5_DDesc):
if isinstance(obj, nd.array):
obj = nd.as_numpy(obj)
with tb.open_file(ddesc.path, mode=ddesc.mode) as f:
where, name = split_path(ddesc.datapath)
if dshape and isinstance(dshape.measure, datashape.Record):
# Convert the structured array to unaligned dtype
# We need that because PyTables only accepts unaligned types,
# which are the default in NumPy
obj = np.array(obj, datashape.to_numpy_dtype(dshape.measure))
f.create_table(where, name, filters=ddesc.filters, obj=obj)
else:
f.create_earray(where, name, filters=ddesc.filters, obj=obj)
ddesc.mode = 'a' # change into 'a'ppend mode for further operations
return Array(ddesc)
