def test():
f = Features(**PARAMS)
for i in range(300, 400):
path = 'test/' + str(i) + '.png'
im = Image(path)
f.set_image(im)
f.extract_blobs()
def get_actual_value(row, column, key, has_fill_rows=True):
row, column = merge_data.parse_coordinates(row, column)
if not has_fill_rows:
row += merge_data.NO_FILL_ROW_OFFSET
# Handle both a string key and a Features key.
key = Features(key).value
return OUTPUT_CONTENTS.get(row, {}).get(column, {}).get(key, '')
def __init__(self, feat=SURF(), cls=RF(n_estimators=40), verbose=True):
self._ft = Features(feat)
self._da = Data(self._ft)
self._fm = FitBoVW(cls)
self._verbose = verbose
self._cl = cls
self._vq = None
def test():
f = Features(**PARAMS)
minx,maxx = 0,0
miny,maxy = 0,0
while True:
path = 'test/tmp.png'
screenshot(path, region=REGION)
im = Image(path)
f.set_image(im)
blobs = f.extract_blobs()
if not blobs[0]:
break
dl = f.small.dl()
bottom_right_corner = blobs[0].bottomRightCorner()
dl.circle(bottom_right_corner, 5, Color.RED)
if blobs[1]:
top_right = blobs[1].topRightCorner()
x = bottom_right_corner[0] - top_right[0]
y = bottom_right_corner[1] - top_right[1]
if x < minx:
minx = x
elif x > maxx:
maxx = x
if y < miny:
miny = y
elif y > maxy:
maxy = y
dl.circle(top_right, 5, Color.RED)
f.small.show()
print 'minx, maxx', minx, maxx
print 'miny, maxy', miny, maxy
def __init__(self, exchanges, logger, db_prices, db_other, db_client):
self.exchanges = exchanges
self.logger = logger
self.db_prices = db_prices
self.db_other = db_other
self.db_client = db_client
self.db_collections_price = {
i.get_name(): db_prices[i.get_name()]
for i in self.exchanges
}
# Save-later queue.
self.signals_save_to_db = queue.Queue(0)
# DataFrame container: data[exchange][symbol][timeframe].
self.data = {}
self.init_dataframes(empty=True)
# Strategy models.
self.models = self.load_models(self.logger)
# Signal container: signals[exchange][symbol][timeframe].
self.signals = {}
# persistent reference to features library.
self.feature_ref = Features()
def __init__(self, host_tcp_port, host_address, mode):
self.mode = mode
self.locally_reserved_ports = []
self.local_udp_port = 0
self.client_ip = ""
self.client_udp_port = 0
self.client_address = (self.client_ip, self.client_udp_port)
self.host_ip = host_address
self.host_tcp_port = host_tcp_port
self.host_udp_port = 0
self.host_address = (self.host_ip, self.host_udp_port)
self.server_udp_socket = socket(AF_INET, SOCK_DGRAM)
self.server_tcp_socket = socket(AF_INET, SOCK_STREAM)
self.tcp_connection_socket = socket(AF_INET, SOCK_STREAM)
self.multipart_message_buffer = ""
self.multipart_message_handling_over = True
self.features = Features()
if mode == 'client':
self. add_multipart()
self.init_connections()
#self.udp_client_socket = socket(AF_INET, SOCK_DGRAM)
self.start_udp_communication()
if mode == 'proxy':
self.init_connections()
self.start_proxy_service()
self.client_ip = ('', 0)
def preparation(x_list, y_list,feature_symbols,Func = None, ref = None, **kwargs):
"""
feature symbols are a list of interested feature names (Not with the prefix g_ or H_ )
"""
global perparation_called_times
features.DT = 0.1
print(perparation_called_times)
xy_vector = np.array(list(x_list) + list(y_list))
assert(len(x_list)==len(y_list))
# ref = ref_bottom if x_list[0]<1000 else ref_top
ref = ref_select(x_list,y_list) if ref is None else ref
if(Func is None):
feature = Features(vec=xy_vector, referenceCurv=ref,**kwargs)
else:
feature = featureFuncWrapper(vec=xy_vector, Func = Func, referenceCurv=ref,**kwargs)
ftrvalues = [np.array([feature.featureValue(f) for f in feature_symbols])]
return_list = []
for k in feature_symbols:
print("symbol:",k)
g,H,N = feature.featureGradJacobNormalizer(k)
return_list.append(g)
return_list.append(H)
return_list.append(N)
perparation_called_times +=1
return return_list, ftrvalues
def __init__(self, mode='svm'):
if mode != 'svm' and mode != 'xgboost':
raise ValueError('Unknown mode for CarModel')
self._f = Features()
self._model = None
self._mode = mode
self.input_shape = None
async def gather_features_for_prediction(image_slice: ImageSlice,
wbs: WindowBoxSlice,
search_params: SearchParams):
features = Features([])
gather_co_list = []
window = wbs.bbox_slice
async def gather_hog_features(window):
return image_slice.window_hog_features(window)
async def gather_spatial_features(window):
return image_slice.window_bin_spatial_features(window)
async def gather_color_hist_features(window):
return image_slice.window_color_hist_features(window)
if search_params.hog_feat is True:
gather_co_list.append(gather_hog_features(window))
if search_params.spatial_feat is True:
gather_co_list.append(gather_spatial_features(window))
if search_params.hist_feat is True:
gather_co_list.append(gather_color_hist_features(window))
for feature in await asyncio.gather(*gather_co_list):
features += feature
float_values = features.values.astype(np.float64)
return float_values
def __init__(self, feature_extractor, parameters, unique_templates):
self.feature_extractor = feature_extractor
self.parameters = parameters
self.unique_templates = unique_templates
feature_count = len(feature_extractor.ordered_features)
self.features = Features(feature_extractor.ordered_features,
[0 for _ in range(feature_count)])
def window_hog_features(self, window):
# add all the features from all hog channel windows
features = Features([])
for hif in self.hog_features_list:
features += hif.window_hog_features(window)
return features
def read_images():
galaxy_ids = get_galaxy_ids()
for galaxy_id in galaxy_ids:
image_file = '.'.join([str(galaxy_id), 'jpg'])
feature_vector = [galaxy_id]
features = Features(os.path.join(cfg.TRAIN_IMAGE_DIR, image_file))
feature_vector.extend(features.get_feature_vector())
yield feature_vector
def grab_data(path):
# grab the data and extract the MFCC feature of every music audio
print "grab_data"
for filename in os.listdir(path):
print filename
mfcc = Features(path + '/' + filename, feature_params)
datamfcc = mfcc.MFCC # MFCC features
dataset.append((datamfcc, filename))
def __init__(self, conf, poseidon, hermes):
self.conf = conf
self.poseidon = poseidon
self.processor = Processor(conf)
self.hermes = hermes
self.features = Features()
self.thunder = Thunder(self.conf, self.features, self.processor, self.poseidon, self.hermes)
self.scepter = Scepter(conf, self.hermes, self.processor, self.features, self.thunder)
def __init__(self, cls_path=None):
""" Init function"""
if cls_path is not None:
self.cls = load_cls(cls_path)
else:
self.cls = None
self.feat = Features()
self.percent_training = 80
def calibrate(self, reps=3, skip_time=2, hold_time=5, gap_time=0.25):
feat_len = int(hold_time / gap_time)
features = Features(feat_len, reps)
reps_completed = 0
printed = False
extended = True
index_middle_finger = 2
# array of middle finger averages of lengths. size is equal to number of reps
middle_len = []
time_elapsed = 0
feat_index = 0
while self.controller.is_connected:
if reps_completed == reps:
print "Calibration is finished!"
self.middle_len = np.mean(middle_len)
self.write_calibration()
return
else:
frame = self.controller.frame()
hands = frame.hands
if len(hands) == 0:
feat_index = 0
time_elapsed = 0
if not printed:
print 'Bring hand in view and extend all the fingers'
printed = True
extended = True
elif feat_index < feat_len:
for hand in hands:
# only for right hand as of now
if hand.is_right and time_elapsed > skip_time:
pointables = frame.pointables
if len(pointables.extended()) != 5:
print "Please extend all the fingers for calibration"
extended = True
else:
if extended:
print "Good! Calibration is starting. Do NOT move the hand..."
#time.sleep(10 * gap_time)
extended = False
# Relative origin(used to calculate the relative distances)
hand_center = hand.stabilized_palm_position
for pointable in pointables:
finger = Leap.Finger(pointable)
pointable_pos = pointable.stabilized_tip_position
relative_pos = pointable_pos - hand_center
features.finger_lengths[feat_index][finger.type] = relative_pos.magnitude
print "Finger lengths", features.finger_lengths[feat_index]
feat_index += 1
elif feat_index == feat_len:
feat_index += 1
middle_len.append(np.mean(features.finger_lengths, axis=0)[index_middle_finger])
reps_completed += 1
print "Remove hand from view"
printed = False
extended = False
time.sleep(gap_time)
time_elapsed += gap_time
def parse_harvest_data(lines, cells):
labels = [Features(v) for v in lines[0][3:]]
for line in lines[1:]:
row, column = line[2], line[1]
row = parse_coordinate(row) + NO_FILL_ROW_OFFSET
cell = cells.get(row, column)
cell.add_data(Features.PLOT_ID, line[0])
for i, value in enumerate(line[3:]):
cell.add_data(labels[i], value)
def window_color_hist_features(self, window):
sp = self.__search_params
# return CameraImage.color_hist(self.slice_window(window),
# nbins=search_params.hist_bins,
# bins_range=search_params.hist_range)
return Features(
ColorHistFeatures.color_hist_features(self.slice_window(window),
nbins=sp.hist_bins,
bins_range=sp.hist_range))
def setUp(self):
# 訓練データ
self.train_word_data = [["Peter", "Daniel", "Blackburn"], ["1966", "World", "Cup"]]
self.train_pos_data = [["NNP", "NNP", "NNP"], ["CD", "NNP", "NNP"]]
word_set = set([word for words in self.train_word_data for word in words])
self.pos_set = set([pos for poses in self.train_pos_data for pos in poses])
self.features = Features(list(self.pos_set))
self.features.create_feature(word_set)
# テストデータ
self.test_word_data = [["Peter", "Daniel", "Blackburn"], ["1980", "World"]]
def computeAtomicFeatures(self):
for node in self.node_list:
iFeatures = Features()
lemma = node.conllLine.lemma
iFeatures.addFeature("lemma", lemma)
label = node.conllLine.deprel
iFeatures.addFeature("label",
self.getVectorRepresentation(label, "dep"))
pos = node.conllLine.pos
iFeatures.addFeature("pos",
self.getVectorRepresentation(pos, "pos"))
children = self.getChildren(node)
grandChildren = self.getGrandChildren(children)
num_children = len(children)
iFeatures.addFeature("num-children", num_children)
num_grandChildren = len(grandChildren)
iFeatures.addFeature("num-grandchildren", num_grandChildren)
labels_children = []
pos_children = []
for child in children:
labels_children.append(child.conllLine.deprel)
pos_children.append(child.conllLine.pos)
# [numberNmods, numberPmods, .... totalNumberOfDeps] same with pos
lenPos = len(self.dictPos)
lenDep = len(self.dictDep)
i = 0
j = 0
posVector = []
depVector = []
while i < lenPos:
posVector.append(0)
i += 1
while j < lenDep:
depVector.append(0)
j += 1
for labelChild in labels_children:
position = self.dictDep[labelChild]
depVector[position] += 1
for posChild in pos_children:
position = self.dictPos[posChild]
posVector[position] += 1
iFeatures.addFeature("labels-children",
"".join(map(str, depVector)))
iFeatures.addFeature("pos-children", "".join(map(str, posVector)))
node.atomicFeatures = iFeatures
def predict():
spam_detect = Spam_Detect()
features_extract = Features(vocabulary_file)
if request.method == 'POST':
if 'train' in request.form:
print('Predict and Train')
train_nb_spam()
email = request.form['email']
data = [email]
featurevectors = features_extract.extract(data)
my_prediction = spam_detect.detect(featurevectors)
return render_template('result.html', prediction=my_prediction)
def test_binary_class():
stock_d = testdata()
ti = TechnicalIndicators(stock_d)
ti.calc_ret_index()
ret_index = ti.stock['ret_index']
f = Features()
train_X, train_y = f.binary_class(ret_index, range=90)
expected = [
1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0,
1, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 0,
0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 0,
1, 1, 0
]
for r, e in zip(train_y, expected):
eq_(r, e)
r = round(train_X[-1][-1], 5)
expected = 1.35486
eq_(r, expected)
r = round(train_X[0][0], 5)
expected = 1.19213
eq_(r, expected)
expected = 14
r = len(train_X[0])
eq_(r, expected)
expected = 75
r = len(train_X)
eq_(r, expected)
train_X, train_y = f.binary_class(ret_index)
expected = 0
eq_(train_y[0], expected)
expected = 1
eq_(len(train_y), expected)
r = round(train_X[0][0], 5)
expected = 1.30311
eq_(r, expected)
expected = 14
r = len(train_X[0])
eq_(r, expected)
expected = 1
r = len(train_X)
eq_(r, expected)
def __init__(self, cfg):
super(FeaturesPerceptronRanker, self).__init__(cfg)
if not cfg:
cfg = {}
self.feats = ['bias: bias']
self.vectorizer = None
self.normalizer = None
self.binarize = cfg.get('binarize', False)
# initialize feature functions
if 'features' in cfg:
self.feats.extend(cfg['features'])
self.feats = Features(self.feats, cfg.get('intermediate_features', []))
def get_data():
files = os.listdir('./MealNoMealData')
meal_data_files = []
no_meal_data_files = []
for file in files:
if 'Nomeal' in file:
no_meal_data_files.append(os.path.join('./MealNoMealData', file))
else:
meal_data_files.append(os.path.join('./MealNoMealData', file))
data = []
labels = []
for meal_data_file, no_meal_data_file in zip(meal_data_files,
no_meal_data_files):
preprocess_obj = Preprocess(meal_data_file)
meal_df = preprocess_obj.get_dataframe()
meal_features = Features(meal_df)
meal_features.compute_features()
# temp_meal_features = meal_features.pca_decomposition().tolist()
temp_meal_features = meal_features.get_features()
labels += [1] * len(temp_meal_features)
preprocess_obj_ = Preprocess(no_meal_data_file)
no_meal_df = preprocess_obj_.get_dataframe()
no_meal_features = Features(no_meal_df)
no_meal_features.compute_features()
no_meal_features_ = no_meal_features.get_features()
# no_meal_final_features = meal_features.pca.transform(no_meal_features_).tolist()
no_meal_final_features = no_meal_features_
labels += [0] * len(no_meal_features_)
for no_meal_feature in no_meal_final_features:
temp_meal_features.append(no_meal_feature)
for meal_no_meal_feature in temp_meal_features:
data.append(meal_no_meal_feature)
return data, labels
def __init__(self, path: str, extractor: str):
"""
Parameters
----------
path: str
The path to this video
extractor: str
The name of the feature extractor currently used
"""
self._path = path
self.reader = decord.VideoReader(self._path)
self.features = Features(self._path, extractor)
def build_features(self, player_id, postseason):
f = Features()
df = self.get_player_stats(player_id, postseason)
new_season = self.add_new_season(df)
f.merged = df.append(new_season, sort=False)
f.merged = f.merged[[x for x in f.merged if '_lag' not in x]]
f.merged.drop([f'{x}_{y}' for x in f.aggregates for y in f.stats_columns], axis=1, inplace=True)
f.merged['draft_entry'] = f.merged['draft_entry'].astype(float)
f.get_age_at_season()
f.get_stats_per_game()
f.get_lags()
f.get_aggregates()
return f.merged
def train_ctc_model(train_file, test_file):
""" Function of training Code Recognizer """
# training and test dataset (default)
train_file = parameters_ctc['train_file']
test_file = parameters_ctc['test_file']
# extract features from two language models trained on Gigaword and StackOverflow
features = Features(RESOURCES)
train_tokens, train_features, train_labels = features.get_features(train_file, True)
test_tokens, test_features, test_labels = features.get_features(test_file, False)
# fastText embedding
vocab_size, word_to_id, id_to_word, word_to_vec = get_word_dict_pre_embeds(train_file, test_file)
train_ids, test_ids = get_train_test_word_id(train_file, test_file, word_to_id)
# transform each ngram probability into a k-dimensional vector using Gaussian binning
word_embeds = np.random.uniform(-np.sqrt(0.06), np.sqrt(0.06), (vocab_size, parameters_ctc['word_dim']))
for word in word_to_vec:
word_embeds[word_to_id[word]]=word_to_vec[word]
# concatenate the outputs with fastText embedding
ctc_classifier = NeuralClassifier(len(train_features[0]), max(train_labels) + 1, vocab_size, word_embeds)
ctc_classifier.to(device)
# binary classifier
optimizer = torch.optim.Adam(ctc_classifier.parameters(), lr=parameters_ctc["LR"])
step_lr_scheduler = lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.8)
# prepare dataset
train_x = Variable(torch.FloatTensor(train_features).to(device))
train_x_words = Variable(torch.LongTensor(train_ids).to(device))
train_y = Variable(torch.LongTensor(train_labels).to(device))
test_x = Variable(torch.FloatTensor(test_features).to(device))
test_x_words = Variable(torch.LongTensor(test_ids).to(device))
test_y = Variable(torch.LongTensor(test_labels).to(device))
# training
for epoch in range(parameters_ctc['epochs']):
loss = ctc_classifier.CrossEntropy(train_features, train_x_words, train_y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
train_scores, train_preds = ctc_classifier(train_features, train_x_words)
test_scores, test_preds = ctc_classifier(test_features, test_x_words)
eval(test_preds, test_labels, "test")
return ctc_classifier, vocab_size, word_to_id, id_to_word, word_to_vec, features
def __init__(self):
self.action_space = spaces.Discrete(len(ACTION_LOOKUP))
#xss样本特征集合
#self.samples=[]
#当前处理的样本
self.current_sample = ""
#self.current_state=0
self.features_extra = Features()
self.waf_checker = Waf_Check()
#根据动作修改当前样本免杀
self.xss_manipulatorer = Xss_Manipulator()
self._reset()
def add_data(self, key, value, append_if_mismatch=False):
if value == '':
return
key = Features(key) # Ensure key is a Features instance.
if key in self._data and value != self._data[key]:
if append_if_mismatch:
self._data[key] = csv_utils.append_value(
self._data[key], value)
return
raise Exception('Unexpected mismatch in existing value:', key,
self._data[key], value)
self._data[key] = value
def classify(self):
for i, x in enumerate(self.channel_decode):
feature_obj = Features(self.data_raw[int(x) - 1],
self.sampling_freq, self.features_id)
features = feature_obj.extract_features()
try:
prediction = self.clf[i].predict([features]) - 1
if prediction != (self.prediction >> i
& 1): # if prediction changes
self.prediction = self.output(i, prediction,
self.prediction)
print('Prediction: %s' % format(self.prediction, 'b'))
except ValueError:
print('prediction failed...')
