def get_weight_matrix(cites, indices):
id2index = {}
for i, id in enumerate(indices):
id2index[id] = i
pair_cnt = dd(int)
cited = dd(list)
for c1, c2s in cites.iteritems():
for c2 in c2s:
cited[c2].append(c1)
for ii in c2s:
for jj in c2s:
if ii == jj or ii not in id2index or jj not in id2index: continue
i, j = id2index[ii], id2index[jj]
pair_cnt[(i, j)] += 1
for c1, c2s in cited.iteritems():
for ii in c2s:
for jj in c2s:
if ii == jj or ii not in id2index or jj not in id2index: continue
i, j = id2index[ii], id2index[jj]
pair_cnt[(i, j)] += 1
row, col, data = [], [], []
for k, v in pair_cnt.iteritems():
i, j = k
row.append(i)
col.append(j)
data.append(v)
row, col, data = np.array(row), np.array(col), np.array(data, dtype = np.float32)
w = sparse.coo_matrix((data, (row, col)), shape = (len(indices), len(indices))).tocsr()
return w
def indices_to_one_hot(context, nb_classes):
"""Convert an iterable of indices to one-hot encoded labels."""
data=[]
for i in range(nb_classes):
data.append(i)
targets = np.array(data).reshape(-1)
return np.eye(nb_classes)[targets]
def evalWords(self, encoder, decoder, n, printWords=False):
lines = open('../../Data/Zulu/zulu.clean.test.conll', encoding='utf-8').\
read().strip().split('\n')
data = []
correct = 0
for line in lines:
line = line.split(" | ")
ortho = self.removeTags(line[2])
data.append(line[0] + "\t" + ortho)
for i in range(n):
source, target = data[i].split('\t')
output_words = self.evaluateModel(encoder, decoder, source.lower())
output = ''
for char in output_words[0]:
output += char
output = output[:-1]
if target == output:
correct += 1
if printWords == True:
print('>', source)
print('=', target)
print('<', output)
print('')
print("Accuracy: ", correct / n)
return (correct / n)
def _preprocess(self, sentences):
data = []
unk_id = self.dictionary.unk_id
for sent in sentences:
tokens = word_tokenize(sent)
data.append(
np.array([
self.dictionary.word2idx[token]
if token in self.dictionary.word2idx else unk_id
for token in tokens
]))
seq_lens = np.array([sent.shape[0] + 2 for sent in data])
seq_lens_idx = np.argsort(-seq_lens)
seq_lens = seq_lens[seq_lens_idx]
padded_data = np.zeros((len(data), seq_lens[0]), dtype='int64')
padded_data = padded_data + self.dictionary.eos_id
for i, sent in enumerate(data):
padded_data[i, 1:sent.shape[0] + 1] = sent
padded_data[:, 0] = self.dictionary.bos_id
padded_data = padded_data[seq_lens_idx]
batch = self.tt.from_numpy(padded_data.T).contiguous()
batch = Variable(batch)
seq_lens = Variable(self.tt.from_numpy(seq_lens))
return batch, seq_lens
def quadratic_line(self, start, stop, *argv, calc='x', plot=False):
# gets each pixle value for a quadratic over the image
# start : start value for the quadratic
# stop : end value for the quadratic
# calc : what quadratic variable do calculate
# quad_values : list of quadratic variables
# result : list of x and y coordinates
data = []
count = 0
array = np.array([n for n in range(start, stop)])
for arg_vect in argv:
if len(arg_vect) != 3:
raise 'there must be 3 quadratic values supplied'
data.append(arg_vect[0] * array**2 + arg_vect[1] * array +
arg_vect[2])
count += 1
# make the last list the average of all the lists
data.append(np.sum(data, axis=0) / count)
lines = np.array(data)
self.left_pts = lines[0]
self.right_pts = lines[1]
self.center_pts = lines[2]
self.y_pts = array
if plot:
self.plot_graph(array, lines)
return lines, array
def flatten_dimension(self, d):
n = len(d)
step = 100 / n
numbers = np.arange(0, 100, step)
# okay so we want to snap our d to numbers
# but we need to remember our original position
# and we need to fuzz the numbers to avoid bad correlations
structured_d = []
for i, value in enumerate(d):
structured_d.append({
"index": i,
"value": value + random.uniform(-.4, .4)
})
structured_d.sort(key=lambda x: x["value"])
for i, value in enumerate(structured_d):
structured_d[i]["flattened_value"] = numbers[i]
structured_d.sort(key=lambda x: x["index"])
d = []
for value in structured_d:
d.append(value["flattened_value"])
return d
def train_m(V, r, k, e):
m, n = np.shape(V)
W = np.mat(np.random.random((m, r)))
H = np.mat(np.random.random((r, n)))
data = []
for x in range(k):
V_pre = np.dot(W, H)
E = V - V_pre
err = 0.0
err = np.sum(np.square(E))
data.append(err)
if err < e: # threshold
break
a = np.dot(W.T, V) # Hkj
b = np.dot(np.dot(W.T, W), H)
for i_1 in range(r):
for j_1 in range(n):
if b[i_1, j_1] != 0:
H[i_1, j_1] = H[i_1, j_1] * a[i_1, j_1] / b[i_1, j_1]
c = np.dot(V, H.T)
d = np.dot(np.dot(W, H), H.T)
for i_2 in range(m):
for j_2 in range(r):
if d[i_2, j_2] != 0:
W[i_2, j_2] = W[i_2, j_2] * c[i_2, j_2] / d[i_2, j_2]
W = norm(W)
return W, H, data
def run(nets, loader, iterations, train):
if train:
[net.train() for net in nets]
else:
[net.eval() for net in nets]
optimizers = [torch.optim.Adam(net.parameters(), lr=0.01) for net in nets]
loss_function = nn.CrossEntropyLoss()
data = []
tq = tqdm(loader, total=iterations, ncols=0, position=2, desc='train' if train else 'val')
for i, (a, b, c) in enumerate(tq):
if i >= iterations:
break
a = Variable(a.cuda(async=True), requires_grad=False)
b = Variable(b.cuda(async=True).transpose(1, 2).contiguous(), requires_grad=False)
c = Variable(c.cuda(async=True), requires_grad=False)
pred_cs = [net(a, b) for net in nets]
losses = [loss_function(pred_c, c) for pred_c in pred_cs]
if train:
[optimizer.zero_grad() for optimizer in optimizers]
[loss.backward() for loss in losses]
[optimizer.step() for optimizer in optimizers]
data.append(extract_plins(nets[0]))
else:
acc = [(pred_c.data.max(dim=1)[1] == c.data).float().mean() for pred_c in pred_cs]
data.append(acc)
data = list(zip(*data))
if not train:
data = [np.mean(d) for d in data]
return data
def run_ensemble_pipeline(models,
models_names,
data_locations,
models_build_params,
models_fun_params,
prediction_mode,
random_percentage=1,
ensemble_name="ensemble"):
#checks first
assert len(data_locations)==len(models), \
"Usage error: you need to provide one data path for each model"
#loading data
abs_path = os.path.abspath(os.path.dirname(__file__))
print("Loading data")
data = []
for data_location in data_locations:
data_matrix = np.load(os.path.join(abs_path, data_location))['arr_0']
data.append(data_matrix)
#loading model functions
models_funcs = [model_pipeline_fun[model] for model in models]
#getting the ensemble model
get_ensemble(models=models_funcs,
data=data,
models_names=models_names,
models_build_params=models_build_params,
models_fun_params=models_fun_params,
random_percentage=random_percentage,
prediction_mode=prediction_mode,
ensemble_name=ensemble_name)
def save(self): #save our data
self.g.save.set_prop("player", "pos", self.tile_pos[:]) #store our position
self.g.save.set_prop("player", "direction", self.direction) #and direction
#save pokemon
data = []
for mon in self.party:
data.append(mon.save())
self.g.save.set_prop("player", "pokemon", data)
def load_data(file_path):
""" Read the file into textblob understandable format"""
with open(file_path) as f:
data = []
for line in f:
line = line.decode('utf-8').strip()
if line:
data.append(tuple(line.rsplit(',',1)))
return data
def contents(upload, joint=''):
xs = [chunk.splitlines() for chunk in upload.file.chunks()]
data = []
for x in xs:
if isinstance(x, list):
data.extend(x)
else:
data.append(x)
return joint.join(data)
def construct(x, res, k):
data, list_i, list_j = [], [], []
for i in range(x.shape[0]):
for j in range(x.indptr[i], x.indptr[i + 1]):
data.append(x.data[j])
list_i.append(i)
list_j.append(res[x.indices[j]])
data = np.array(data, dtype = np.float32)
return sp.coo_matrix((data, (list_i, list_j)), shape = (x.shape[0], k)).tocsr()
def contents(upload, joint=''):
xs = [chunk.splitlines() for chunk in upload.file.chunks()]
data = []
for x in xs:
if isinstance(x, list):
data.extend(x)
else:
data.append(x)
return joint.join(data)
def save(self): #save our data
self.g.save.set_prop("player", "pos",
self.tile_pos[:]) #store our position
self.g.save.set_prop("player", "direction",
self.direction) #and direction
#save pokemon
data = []
for mon in self.party:
data.append(mon.save())
self.g.save.set_prop("player", "pokemon", data)
def list_vms():
conn = connect()
other_domains = conn.listDefinedDomains()
running_domains = conn.listDomainsID()
message = "Got a list of domains."
create_log(message, 1)
data = []
for id in running_domains:
data.append(conn.lookupByID(id))
return data, other_domains
def readtrack( filename ) :
"""
Read a track file, return a numpy array.
"""
data = []
for line in open( filename ) :
if line.__contains__( '\"' ) :
continue
data.append( float( line.split()[2] ) )
if len( data ) == 0 :
raise PiqueException( 'Empty input file or parsing error : ' + filename )
return numpy.array( data )
def movie_user_df(input_data, user1, user2):
data = []
for movie in input_data[user1].keys():
if movie in input_data[user2].keys():
try:
data.append((movie
,input_data[user1][movie]
,input_data[user2][movie]) )
except:
pass
return pd.DataFrame(data = data, columns = ['movie', user1,
user2])
def readtrack( filename ) :
"""
Read a track file, return a numpy array.
"""
data = []
for line in open( filename ) :
if line.__contains__( '\"' ) :
continue
data.append( float( line.split()[2] ) )
if len( data ) == 0 :
raise PiqueException( 'Empty input file or parsing error : ' + filename )
return numpy.array( data )
def convert_to_index(data_raw, track_dic):
data = []
for session in data_raw:
b=[]
for track in session:
try:
b.append(track_dic_index[int(track)])
except ValueError:
b
data.append(b)
data = torch.Tensor(np.array(data)).long()
return data
def make_dataset(low_data, n_prev=100):
data, target = [], []
# 21データずつ分割 == 下記のcと同じ値とする
maxlen = 21
for i in range(len(low_data) - maxlen):
data.append(low_data[i:i + maxlen])
target.append(low_data[i + maxlen])
re_data = np.array(data).reshape(len(data), maxlen, 1)
re_target = np.array(target).reshape(len(data), 1)
return re_data, re_target
def output_features(rnn, feature_stractor, data_loader, json_dir):
''' set model to evaluate mode '''
rnn.eval()
feature_stractor.eval()
iters = 0
with torch.no_grad(): # do not need to caculate information for gradient during eval
data = []
for idx, (video, video_path) in enumerate(data_loader):
print(iters)
iters += 1
batch_img = []
batch_gt = []
for i in range(len(video_path)):
frames = readShortVideo(video_path[i], video.get('Video_category')[i], video.get('Video_name')[i])
vid = []
for j in range(frames.shape[0]):
im = transforms_array(frames[j])
vid.append(im)
vid = torch.stack(vid).cuda()
with torch.no_grad():
feature = feature_stractor(vid)
batch_img.append(feature)
gt = (int(video.get('Action_labels')[i]))
batch_gt.append(gt)
sequence, label, n_frames = batch_padding(batch_img, batch_gt)
# print(sequence.shape)
feat, _ = rnn(sequence, n_frames)
features_flt = []
for imgs in feat:
imgs_feature = []
for fea in imgs:
imgs_feature.append(float(fea))
features_flt.append(list(imgs_feature))
##strore the values of the pred.
for i in range(0, len(features_flt)):
data.append([list(features_flt[i]), batch_gt[i]])
data = list(data)
with open(json_dir, 'w') as outfile:
json.dump(data, outfile)
def plot_rtt(plot_data, ax, no, mec):
ax.grid(True)
ax_list = (ax1, ax7, ax13, ax19)
axx_list = {ax6: 4, ax12: 5, ax18: 6, ax24: 7}
list_dict = list(plot_data.keys())
data = []
for j in plot_data:
i = plot_data[j]
data.append(i[-1])
style_id = list_dict.index(j)
mv = _mov_avg(i)
pt = mv[0:len(mv):int((len(mv) / 10)) + 1]
if pt[-1] != mv[-1]:
pt.append(mv[-1])
for i in pt:
if i > 5:
a = pt.index(i)
pt[a] = pt[a + 1] + 0.3
a = list(range(0, len(mv)))
ptx = a[0:len(a):int((len(a) / 10)) + 1]
if ptx[-1] != a[-1]:
ptx.append(a[-1])
ax.plot(ptx,
pt,
style[style_id],
linewidth=2,
label=j)
avg_set = avg_rtt(data)
avg.append(avg_set)
#print(avg_set)
if mec == 4:
ax.set_title(algo_dict[names[no]], fontdict=font)
ax.set_xlabel('Time Period', fontdict=font1)
if ax in ax_list:
ax.set_ylabel('RTT (ms)', fontdict=font1)
# ax.set_ylabel('RTT ')
# ax.legend()
ax.set_ylim(top=4.2)
ax.set_ylim(bottom=0.5)
axx = ax.twinx()
axx.set_yticklabels([])
axx.set_yticks([])
if ax in axx_list:
axx.set_ylabel(f'{axx_list[ax]} MECs', rotation=0, fontdict=font, labelpad=38)
def read_corpus(file_path, source):
""" Read file, where each sentence is dilineated by a `\n`.
@param file_path (str): path to file containing corpus
@param source (str): "tgt" or "src" indicating whether text
is of the source language or target language
"""
data = []
for line in open(file_path):
sent = line.strip().split(' ')
# only append <s> and </s> to the target sentence
if source == 'tgt':
sent = ['<s>'] + sent + ['</s>']
data.append(sent)
return data
def measure_background(run_desc):
Izero = run_desc['I offset']
data = []
for i in range(50):
if STOP_FLAG:
break
V = i * run_desc['Vsafe'] / 50.0
run_desc['f_set'](V)
time.sleep(run_desc['pause'])
I = run_desc['f_read']() * run_desc['iscale'] - Izero
data.append([I, V])
run_desc['mydatabase'].add_data_point([V, I], run_desc['step'])
run_desc['step'] = run_desc['step'] + 1
fit = linear_fit(numarray.array(data))
return (fit[0], fit[1], fit[2], fit[3])
def prepareResultsData(results):
data = []
labels = ['test_id', 'pups', 'juveniles', 'adult_females', 'subadult_males', 'adult_males']
data.append(labels)
filenames = results.keys()
for filename in filenames:
counts = results[filename]
pups = str(int(counts['pups']))
juveniles = str(int(counts['juveniles']))
adult_females = str(int(counts['adult_females']))
subadult_males = str(int(counts['subadult_males']))
adult_males = str(int(counts['adult_males']))
entry = [filename, pups, juveniles, adult_females, subadult_males, adult_males]
data.append(entry)
return data
def measure_background(run_desc):
Izero = run_desc['I offset']
data = []
for i in range(50):
if STOP_FLAG:
break
V = i * run_desc['Vsafe']/50.0
run_desc['f_set'](V)
time.sleep(run_desc['pause'])
I = run_desc['f_read']() * run_desc['iscale'] - Izero
data.append([I, V])
run_desc['mydatabase'].add_data_point([V, I], run_desc['step'])
run_desc['step'] = run_desc['step'] + 1
fit = linear_fit(numarray.array(data))
return (fit[0], fit[1], fit[2], fit[3])
def bicycle_acc(self):
from connect import SensorDataReceiver, BicycleDataReceiver
from data import CSVMaker
import sys
mysocket = SensorDataReceiver.make_mysocket()
SensorDataReceiver.bind_mysocket(mysocket)
data = []
while True:
try:
acc = BicycleDataReceiver.receive_acc_data(mysocket)
data.append(acc)
except KeyboardInterrupt:
CSVMaker().write_bicycle_acc(data)
sys.exit()
except Exception:
pass
def test(build_model, dataset, hparams, logdir):
# Check if the given directory already contains model
if os.path.exists(os.path.join(logdir, "stats.json")):
# then mark this as the directory to load weights from
model_dir = logdir
else:
# Raise Error
raise RuntimeError(f"No valid model stats file found in {logdir}")
model_path = os.path.join(model_dir, "weights.h5")
# Build model
model = build_model(hparams, **dataset.preprocessing.kwargs)
# Compile model
model.compile(optimizer=optimizers.make_optimizer(hparams.optimizer,
hparams.opt_param),
loss="categorical_crossentropy",
metrics=["categorical_accuracy"])
# Print Summary of models
lq.models.summary(model)
# Load model weights from the specified file
print("Before loading...")
for l in model.layers:
for _w in l.trainable_weights:
print("{:40s}".format(l.name, _w.name),
tf.keras.backend.get_value(_w).flatten()[:3])
# model.load_weights(model_path)
utils.load_weights(model, model_path)
print("After loading...")
for l in model.layers:
for _w in l.trainable_weights:
print("{:25s}".format(l.name, _w.name),
tf.keras.backend.get_value(_w).flatten()[:3])
# Test this model
test_log = model.evaluate(dataset.test_data(hparams.batch_size),
steps=dataset.test_examples //
hparams.batch_size)
data = [["Metric", "Value"]]
for (idx, metric) in enumerate(model.metrics_names):
data.append([metric, test_log[idx]])
from terminaltables import AsciiTable
print(AsciiTable(data, title="Test Statistics").table)
def execute(self, data):
events = self.function.list()
data = []
for e in events:
item_title = '%s - %s' % (e.get_nice_time(), e.title)
item_actions = {
'Update':
'reminder update %s %%Timestamp{{%s}} %%Title{{%s}}' %
(e.id, e.get_input_time(), e.title)
}
item_meta = {'id': e.id, 'context': 'reminder item %s' % e.id}
data.append([item_title, None, item_actions, item_meta])
actions = [('Add new...', 'reminder add %Timestamp %Title')]
return function.response(function.STATE_SUCCESS, 'Upcoming reminders',
data, actions)
def load(path, is_train=True, maxlen=128, factor=1.0, pairwise=False):
with open(path, 'r', encoding='utf-8') as reader:
data = []
cnt = 0
for line in reader:
sample = json.loads(line)
sample['factor'] = factor
cnt += 1
if is_train:
if pairwise and (len(sample['token_id'][0]) > maxlen
or len(sample['token_id'][1]) > maxlen):
continue
if (not pairwise) and (len(sample['token_id']) > maxlen):
continue
data.append(sample)
print('Loaded {} samples out of {}'.format(len(data), path))
return data
def execute(self, data):
events = self.function.list()
data = []
for e in events:
item_title = '%s - %s' % (e.get_nice_time(), e.title)
item_actions = {
'Update': 'reminder update %s %%Timestamp{{%s}} %%Title{{%s}}' % (e.id, e.get_input_time(), e.title)
}
item_meta = {
'id': e.id,
'context': 'reminder item %s' % e.id
}
data.append([item_title, None, item_actions, item_meta])
actions = [('Add new...', 'reminder add %Timestamp %Title')]
return function.response(function.STATE_SUCCESS, 'Upcoming reminders', data, actions)
def output_features(classi, feaStract, data_loader, json_dir):
''' set model to evaluate mode '''
classi.eval()
feaStract.eval()
with torch.no_grad(): # do not need to caculate information for gradient during eval
data = []
for idx, (video, video_path) in enumerate(data_loader):
features = []
clss = []
print('Preprocessing the data')
for i in range(len(video_path)):
print('working ', i)
frames = readShortVideo(video_path[i], video.get('Video_category')[i], video.get('Video_name')[i])
frames_res = torch.from_numpy(frames)
frames_res.resize_(len(frames), 3, 240, 240)
frames_res = frames_res.float().cuda()
print(feaStract(frames_res).shape) # , end="\r")
features.append(torch.mean(feaStract(frames_res), 0).cpu().detach().numpy())
clss.append(int(video.get('Action_labels')[i]))
features = torch.from_numpy(np.asarray(features))
clss = torch.from_numpy(np.asarray(clss))
# FC
print('Classifier')
features = features.cuda()
feat, _ = classi(features)
features_flt = []
for imgs in feat:
imgs_feature = []
for fea in imgs:
imgs_feature.append(float(fea))
features_flt.append(list(imgs_feature))
##strore the values of the pred.
for i in range(0, len(features_flt)):
data.append([list(features_flt[i]), clss[i]])
data = list(data)
with open(json_dir, 'w') as outfile:
json.dump(data, outfile)
def test(build_model, dataset, hparams, logdir):
# Check if the given directory already contains model
if os.path.exists(os.path.join(logdir, "stats.json")):
# then mark this as the directory to load weights from
model_dir = logdir
else:
# Raise Error
raise RuntimeError(f"No valid model stats file found in {logdir}")
model_path = os.path.join(model_dir, "weights.h5")
# Build model
model = build_model(hparams, **dataset.preprocessing.kwargs)
# Custom metric
import math
def PSNR(y_true, y_pred):
max_pixel = 1.0
return 10.0 * (1.0 / math.log(10)) * K.log(
(max_pixel**2) / (K.mean(K.square(y_pred - y_true))))
# Compile model
model.compile(optimizer=optimizers.make_optimizer(hparams.optimizer,
hparams.opt_param),
loss="mse",
metrics=[PSNR])
# Print Summary of models
lq.models.summary(model)
# # Load model weights from the specified file
model.load_weights(model_path)
# Test this model
test_log = model.evaluate(dataset.test_data(hparams.batch_size),
steps=dataset.test_examples //
hparams.batch_size)
data = [["Metric", "Value"]]
for (idx, metric) in enumerate(model.metrics_names):
data.append([metric, test_log[idx]])
from terminaltables import AsciiTable
print(AsciiTable(data, title="Test Statistics").table)
def audio2tfr(hparams):
wav_load_fn = lambda x: load_wav(x, hparams)
txt_encode_fn = lambda x: encode_txt(x, hparams.vocab)
for mode in ["eval", "train"]:
with open(hparams["{}_files".format(mode)], "r") as f:
meta_data = [x.strip().split("|") for x in f.readlines()]
head = meta_data[0]
body = meta_data[1:]
data = []
for d in body:
fname = get_value_from_query("fname", head, d)
txt = get_value_from_query("txt", head, d)
sid = int(get_value_from_query("sid", head, d))
data.append((fname, txt, sid))
save_tfrecords(hparams.tfr_dir, "%s_%s" % (hparams.tfr_prefix, mode),
data, wav_load_fn, txt_encode_fn, _NUM_SHARDS[mode])
def csv_pre_train(filename): f = file(filename, 'rU') reader = csv.reader(f) headers = reader.next() data = [] for index, row in enumerate(reader): temp = [] for item in row: item = item.strip() temp.append(item) data.append(temp[:]) len_pre_train = int(len(data)*1/4) s1 = filename[:-4] + "Pre.csv" f1 = open(s1, 'ab') csvWriter1 = csv.writer(f1) csvWriter1.writerow(headers) for i in range(len(data[:len_pre_train])): csvWriter1.writerow(data[i]) return s1
def create_course():
with open('json/course.json', 'r') as f:
data = json.load(f)
data.append(dict(request.args))
with open('json/course.json', 'w') as f:
json.dump(data, f)
status = 201
message = dict(request.args)
response = Response(json.dumps(message),
status,
mimetype='application/json')
return response
"""Create a course.
:return: The course object (see the challenge notes for examples)
:rtype: object
"""
"""
def run(self): dataType = 0 while self.conn.isOpen(): if self.conn.inWaiting() > 0: tmp = self.conn.read() if ord(tmp) == 255: tmp = self.conn.read() if ord(tmp) == 123: tmp = self.conn.read() if ord(tmp) == 10: dataType = self.conn.read() length = self.getDataLength(dataType)/8 data = [] for i in range(length): data.append(self.conn.read()) #print data self.parse(dataType,data) time.sleep(0.001)
def parse_wild(self, wild): #parse wild pokemon data
when = wild.getAttribute("for") #get when the data will be used
data = []
for node in wild.childNodes: #loop through node data to get wild pokemon
if node.localName != "pokemon": continue #loop if it's not what we want
name = node.getAttribute("type") #get type of pokemon
levels = node.getAttribute("level").replace(" ", "") #and level
#and rarity
rarity = 1 if node.getAttribute("rarity") == "" else int(node.getAttribute("rarity"))
level_list = [] #we need to generate a list of possible levels
for level in levels.split("|"): #loop through different level groups
if "-" not in level: #if there's no range separator
level_list.append(int(level)) #add it as normal
else: #if we found one
t = level.split("-") #get both parts of range
start, end = int(t[0]), int(t[1])+1 #parse it
level_list.extend(range(start, end)) #add range to levels
t = [name, level_list] #generate data
for x in xrange(rarity): #add it once for each rarity
data.append(t)
self.wild_pokemon[when] = data #add generated list to wild data
def parse_wild(self, wild): #parse wild pokemon data
when = wild.getAttribute("for") #get when the data will be used
data = []
for node in wild.childNodes: #loop through node data to get wild pokemon
if node.localName != "pokemon": continue #loop if it's not what we want
name = node.getAttribute("type") #get type of pokemon
levels = node.getAttribute("level").replace(" ", "") #and level
#and rarity
rarity = 1 if node.getAttribute("rarity") == "" else int(node.getAttribute("rarity"))
level_list = [] #we need to generate a list of possible levels
for level in levels.split("|"): #loop through different level groups
if "-" not in level: #if there's no range separator
level_list.append(int(level)) #add it as normal
else: #if we found one
t = level.split("-") #get both parts of range
start, end = int(t[0]), int(t[1])+1 #parse it
level_list.extend(range(start, end)) #add range to levels
t = [name, level_list] #generate data
for x in xrange(rarity): #add it once for each rarity
data.append(t)
self.wild_pokemon[when] = data #add generated list to wild data
def constuct_dataset(features, labels, label_set, indices, maxf, cites = None, cite2index = None):
row, col, data = [], [], []
for i, id in enumerate(indices):
for f, v in features[id]:
row.append(i)
col.append(f)
data.append(v)
if cites is not None:
for c2 in cites[id]:
row.append(i)
col.append(maxf + 1 + cite2index[c2])
data.append(1.0)
row, col, data = np.array(row), np.array(col), np.array(data, dtype = np.float32)
add_len = len(cite2index) if cites else 0
trainx = sp.coo_matrix((data, (row, col)), shape = (len(indices), maxf + 1 + add_len))
label_list = list(label_set)
trainy = np.zeros((len(indices), len(label_list)), dtype = np.int32)
for i, id in enumerate(indices):
trainy[i, label_list.index(labels[id])] = 1
return trainx, trainy
def command_list(self):
plots = []
data = []
for i, curve in enumerate(self.__curves):
title = self.__title(curve)
plots.append("'-' title %s with lp ls %d pt %d" %
(self.__string(title), i+1, i+1))
plots.append("'-' title '' with errorbars ls %d pt %d" % (i+1, i+1))
for row in curve["$curve"]:
if row.get("$mean") is None:
data.append("")
else:
data.append("%s %s" % (row[self.__x], row["$mean"]))
data.append("e")
for row in curve["$curve"]:
if row.get("$mean") is not None:
data.append("%s %s %s %s" % (row[self.__x], row["$mean"],
row["$min"], row["$max"]))
data.append("e")
return ["set xlabel %s" % self.__string(self.__x),
"set ylabel %s" % self.__string(self.__y),
"plot %s" % ",".join(plots)] + data
def read():
data = []
try:
file = open("Data.txt", "r")
except:
file = open("Data.txt", "w")
file.close()
file = open("Data.txt", "r")
i = 0
while True:
data.append(file.readline())
print(data[i])
if data[i][0:1] == " ":
data[i] = data[i][1:]
try:
data.index("")
data.remove("")
break
except:
print(".")
i = i + 1
return io._data2MainData(data)
def Read(path, filename, validators):
os.chdir(workingDir)
location = os.path.join(os.getcwd(), path)
os.chdir(location)
print(os.listdir('.'))
if filename in os.listdir('.'):
book = xlrd.open_workbook(os.path.join(location, filename))
sheet = book.sheet_by_index(0)
print("rows: {} , columns: {}".format(sheet.nrows, sheet.ncols))
if sheet.nrows < 1:
print("No content")
return "This file has no content"
columns = []
for x in range(0, sheet.ncols):
columns.append(sheet.cell_value(0, x))
if str(columns) == validators:
data = []
for y in range(1, sheet.nrows):
row = []
for x in range(0, sheet.ncols):
try:
if x == 4:
value = str(int(sheet.cell_value(y, x)))
if value[0] == "7":
val = "{}{}".format("+254", value)
else:
val = int(sheet.cell_value(y, x))
except:
val = sheet.cell_value(y, x)
row.append(val)
data.append(row)
os.remove(filename)
os.chdir(workingDir)
return data
else:
print("No such file")
def cones(self, windows=[30, 60, 90, 120], quantiles=[0.25, 0.75]):
'''Plots volatility cones
Parameters
----------
windows : [int, int, ...]
List of rolling windows for which to calculate the estimator cones
quantiles : [lower, upper]
List of lower and upper quantiles for which to plot the cones
'''
if len(windows) < 2:
raise ValueError('Two or more window periods required')
if len(quantiles) != 2:
raise ValueError('A two element list of quantiles is required, lower and upper')
if quantiles[0] + quantiles[1] != 1.0:
raise ValueError('The sum of the quantiles must equal 1.0')
if quantiles[0] > quantiles[1]:
raise ValueError('The lower quantiles (first element) must be less than the upper quantile (second element)')
max = []
min = []
top_q = []
median = []
bottom_q = []
realized = []
data = []
for w in windows:
estimator = self._get_estimator(w)
max.append(estimator.max())
top_q.append(estimator.quantile(quantiles[1]))
median.append(estimator.median())
bottom_q.append(estimator.quantile(quantiles[0]))
min.append(estimator.min())
realized.append(estimator[-1])
data.append(estimator)
if self._type is "Skew" or self._type is "Kurtosis":
f = lambda x: "%i" % round(x, 0)
else:
f = lambda x: "%i%%" % round(x*100, 0)
'''
Figure args
'''
fig = plt.figure(figsize=(8, 6))
left, width = 0.07, 0.65
bottom, height = 0.2, 0.7
bottom_h = left_h = left+width+0.02
rect_cones = [left, bottom, width, height]
rect_box = [left_h, bottom, 0.17, height]
cones = plt.axes(rect_cones)
box = plt.axes(rect_box)
'''
Cones plot args
'''
# set the plots
cones.plot(windows, max, label="Max")
cones.plot(windows, top_q, label=str(int(quantiles[1]*100)) + " Prctl")
cones.plot(windows, median, label="Median")
cones.plot(windows, bottom_q, label=str(int(quantiles[0]*100)) + " Prctl")
cones.plot(windows, min, label="Min")
cones.plot(windows, realized, 'r-.', label="Realized")
# set the x ticks and limits
cones.set_xticks((windows))
cones.set_xlim((windows[0]-5, windows[-1]+5))
# set and format the y-axis labels
locs = cones.get_yticks()
cones.set_yticklabels(map(f, locs))
# turn on the grid
cones.grid(True, axis='y', which='major', alpha=0.5)
# set the title
cones.set_title(self._type + ' (' + self._ticker + ', daily ' + self._start.strftime("%Y-%m-%d") + ' to ' + self._end.strftime("%Y-%m-%d") + ')')
# set the legend
pos = cones.get_position() #
cones.legend(loc='upper center', bbox_to_anchor=(0.5, -0.05), ncol=3)
'''
Box plot args
'''
# set the plots
box.boxplot(data, notch=1, sym='+')
box.plot([i for i in range(1, len(windows)+1)], realized, color='r', marker='*', markeredgecolor='k')
# set and format the y-axis labels
locs = box.get_yticks()
box.set_yticklabels(map(f, locs))
# move the y-axis ticks on the right side
box.yaxis.tick_right()
# turn on the grid
box.grid(True, axis='y', which='major', alpha=0.5)
return fig, plt
