def get_extended_observation(self):
self._observation = []
if not self._ft_obs_only:
if self.action_dim > 3:
pos, orn = self.member_pose[0], self.member_pose[1]
self._observation.extend(pos)
self._observation.extend(orn)
else:
pos = self.member_pose[0]
self._observation.extend(pos)
self.force_torque = self.get_force_torque()
if WRITE_CSV:
util.write_csv([self._env_step_counter] + self.force_torque,
'ft_reading.csv', False)
if self._limit_force_torque:
self.check_ft_limit(self.force_torque)
# if self._force_torque_violations != [0]*len(self.force_torque):
# util.prRed(self._force_torque_violations)
self._observation.extend(self.force_torque)
return self._observation
def write_measurements_to_csv(name, measurements):
rows = [['latitude', 'longitude', 'velocity', 'heading']] + [[
measurement.latitude, measurement.longitude, measurement.velocity,
measurement.heading
] for measurement in measurements]
write_csv(name, rows)
# if (__name__ == "__main__"):
# name = 'track_2018-03-14_134847'
# gpx = load_gpx_file('data/{}.gpx'.format(name))
# measurements = extract_gps_measurements(gpx)
# write_measurements_to_csv(name, measurements)
# for measurement in list(measurements)[:10]:
# print(measurement.latitude, measurement.longitude, measurement.velocity, measurement.heading)
def step(self, action):
if len(action) > 3:
delta_lin = np.array(action[0:3]) * self._max_vel * self._time_step
delta_rot = np.array(action[3:6]) * self._max_rad * self._time_step
delta = np.append(delta_lin, delta_rot)
else:
delta = np.array(action) * self._max_vel * self._time_step
if self._limit_force_torque:
self.constrain_velocity_for_ft(delta)
if WRITE_CSV:
util.write_csv([self._env_step_counter] + list(delta),
'data_out.csv', False)
return self.step2(delta)
def pos_dist_to_target(self):
self.member_pose = self.get_member_pose()
# log
if WRITE_CSV:
util.write_csv([self._env_step_counter] + self.member_pose[0] +
self.member_pose[1], 'member_pose.csv', False)
member_pos = list(self.member_pose[0])
target_pose = self.get_target_pose()
target_pos = list(target_pose[0])
dist_pos = np.linalg.norm(np.subtract(member_pos,
target_pos)) # linear dist in m
# util.prGreen("pos dist: {}".format(dist_pos))
return dist_pos
def write_pred_to_csv(file_names, model_preds, path="data/submission.csv"):
csv_list = []
for i in range(len(model_preds)):
csv_row = ['', '']
csv_row[0] = file_names[i]
s = 'new_whale' # string containing the five whale names separated by blanks
for j in range(len(model_preds[i]) -
1): # run over 5 ordered predictions
# if j>0:
s = s + ' '
s = s + model_preds[i][j]
# print("next_s", s)
csv_row[1] = s
csv_list.append(csv_row)
# print("csv_list", csv_list)
print("write csv file")
ut.write_csv(csv_list, path)
print("done writing csv file")
def main():
# create the base pre-trained model
base_model = InceptionV3(weights='imagenet', include_top=False)
# add a global spatial average pooling layer
x = base_model.output
x = GlobalAveragePooling2D()(x)
# let's add a fully-connected layer
x = Dense(1024, activation='relu')(x)
# and a logistic layer
num_classes = len(os.listdir(INPUT_DIRECTORY))
predictions = Dense(num_classes, activation='softmax')(x)
# this is the model we will train
model = Model(inputs=base_model.input, outputs=predictions)
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
for layer in base_model.layers:
layer.trainable = False
# compile the model (should be done *after* setting layers to non-trainable)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy')
# define image generator
train_gen = image.ImageDataGenerator()
# train the model on the new data for a few epochs
model.fit_generator(train_gen.flow_from_directory(INPUT_DIRECTORY),
steps_per_epoch=3,
epochs=1,
verbose=2)
# let's predict the test set to see a rough score
labels = make_label_dict()
test_gen = image.ImageDataGenerator()
flow = test_gen.flow_from_directory(INPUT_DIRECTORY, class_mode=None)
predictions = model.predict_generator(flow, verbose=1) # steps=15611//32)
top_k = predictions.argsort()[:, -4:][:, ::-1]
classes = [" ".join([labels[i] for i in line]) for line in top_k]
filenames = flow.filenames # [os.path.basename(f) for f in flow.filenames]
csv_list = zip(filenames, classes)
write_csv(csv_list, file_name=OUTPUT_FILE)
a = 0
for action in actions:
a += 1
associated = []
f = 0
for filename in iterate_directory(
'/home/user/Projects/Python/Recipes/Cookbook'):
f += 1
for step in get_steps(
get_lines('/home/user/Projects/Python/Recipes/Cookbook/' +
filename)):
s = 0
for sentence in get_sentences(step):
s += 1
if action in get_words(sentence):
for word in get_words(sentence):
for ingredient in ingredients:
if ingredient in word or word == ingredient:
associated.append(ingredient)
print('Action: ' + str(a) + ' File: ' + str(f) +
' Sentence: ' + str(s),
end='\r')
occurance = {}
try:
occurance = get_occurances(associated)
except:
occurance['all'] = 0
write_csv('information/associations/' + action + '.csv', occurance)
usleg_db = client["US_LEGISLATOR"]
usleg_db.authenticate("readonly", "smappnyu")
# Query for tweets - more complex than we're used to. Using AND.
# (NOTE that mongo explicitly ANDs multiple comma-separated search clauses,
# however, if two clauses on the same field with the same operator, must use $and)
start = datetime(2013, 8, 21)
end = datetime(2013, 9, 7)
results = usleg_db.legislator_tweets.find({
"timestamp": {"$gte": start, "$lt": end},
"$and": [
{"text": {"$regex": "syria", "$options": "i"}},
{"text": {"$regex": "interven", "$options": "i"}}
]
})
print "Found {0} tweets on topic. Writing to CSV file".format(
results.count(with_limit_and_skip=True))
write_csv(results, "usleg-syria-intervene.csv")
def __init__(self,
time_step=None,
max_steps=None,
step_limit=None,
action_dim=None,
max_vel=None,
max_rad=None,
ft_obs_only=None,
limit_ft=None,
max_ft=None,
max_position_range=None,
dist_threshold=None):
super().__init__()
self._max_step = max_steps
self._step_limit = step_limit
# max linear and rotational velocity command
self._max_vel = max_vel
self._max_rad = max_rad
# only use force torque as observation
self._ft_obs_only = ft_obs_only
self._time_step = time_step
self._observation = []
self._env_step_counter = 0
self._num_success = 0
self._limit_force_torque = limit_ft
self._max_force_torque = max_ft
self._force_torque_violations = [0.0] * len(self._max_force_torque)
self._ft_range_ratio = 1
""" Define Gym Spaces for observations and actions """
self._max_pos_range = max_position_range
if self._ft_obs_only: # no pose observation
self.observation_dim = len(self._max_force_torque)
observation_high = np.array(self._max_force_torque)
observation_low = -observation_high
elif action_dim == 6: # 6 DOF
self.observation_dim = 7 + len(self._max_force_torque)
observation_orn_high = [1] * 4
observation_high = np.array(self._max_pos_range +
observation_orn_high +
self._max_force_torque)
observation_low = -observation_high
else: # 3 DOF
self.observation_dim = 3 + len(self._max_force_torque)
observation_high = np.array(self._max_pos_range +
self._max_force_torque)
observation_low = -observation_high
self.observation_space = spaces.Box(observation_low, observation_high)
self._action_bound = 1
action_high = np.array([self._action_bound] * action_dim)
self.action_space = spaces.Box(-action_high, action_high)
self.action_dim = action_dim
self.member_pose = []
self.force_torque = []
self.dist_threshold = dist_threshold
# csv headers
if WRITE_CSV:
util.write_csv([
"step_member_pose", "pos_X", "pos_Y", "pos_Z", "qX", "qY",
"qZ", "qW"
], 'member_pose.csv', True)
util.write_csv(["step_ft", "Fx", "Fy", "Fz", "Tx", "Ty", "Tz"],
'ft_reading.csv', True)
if self.action_dim == 3:
util.write_csv(["step_actions", "vel_X", "vel_Y", "vel_Z"],
'data_out.csv', True)
else:
util.write_csv([
"step_actions", "vel_X", "vel_Y", "vel_Z", "rot_vel_X",
"rot_vel_Y", "rot_vel_Z"
], 'data_out.csv', True)
def main():
stocks = config.stocks
get_data_csv_name = config.get_data_csv_name
db_csv_name = config.db_csv_name
subprocess.call(["rm", get_data_csv_name])
subprocess.call(["rm", db_csv_name])
number = 1
for stock_index in range(len(stocks) - 1):
qiita_api = os.environ['QIITA_API']
url = "https://qiita.com/api/v2/items"
h = {"Authorization": "Bearer " + qiita_api}
p = {
'per_page':
100,
'query':
'stocks:<{} stocks:>{}'.format(str(int(stocks[stock_index]) + 1),
stocks[stock_index + 1])
}
response = requests.get(url, params=p, headers=h)
response_list = json.loads(response.text)
for index, item in enumerate(response_list):
created_at = response_list[index]["created_at"]
article_id = response_list[index]["id"]
likes_count = response_list[index]["likes_count"]
tags = []
for tag_index in range(5):
try:
tags.append(
response_list[index]["tags"][tag_index]["name"])
except IndexError:
tags.append(None)
title = response_list[index]["title"]
updated_at = response_list[index]["updated_at"]
url = response_list[index]["url"]
user_id = response_list[index]["user"]["id"]
number = utilities.write_csv(get_data_csv_name, number, article_id,
user_id, title, likes_count, url,
tags, created_at, updated_at)
utilities.delete_csv_row(get_data_csv_name, db_csv_name)
dt_now = datetime.datetime.now()
file = open('update_log.txt', 'a')
file.write(str(dt_now) + "\n")
file.close()
conn = utilities.get_connection()
cur = conn.cursor()
cur.execute('DELETE FROM update_time')
cur.execute('INSERT INTO update_time VALUES (' + str(dt_now.year) + ',' +
str(dt_now.month) + ',' + str(dt_now.day) + ')')
cur.execute('DELETE FROM articles')
f = open('db.csv', 'r')
cur.copy_from(f, 'articles', sep=',', null='\\N')
conn.commit()
cur.close()
conn.close()
for item in all_items:
try:
for file in iterate_directory(
'/home/user/Projects/Python/Recipes/Cookbook'):
ings = generate_ingredients(
'/home/user/Projects/Python/Recipes/Cookbook/' + file)
r = recipe(ings)
if item in r.items:
for it in r.items:
total_associations.append(it)
log = open('log_connect.txt', 'w')
log.write('item: ' + item + ', file: ' + file +
', associations: ' + str(len(total_associations)))
log.close()
global dict
dict = {}
if len(total_associations) > 0:
dict = get_occurances(total_associations)
name = '_'.join(item.strip(' "\'').split(' ')).strip(' "\'').strip(
"'").strip('"') + 'csv'
if len(total_associations) < 1:
for item in all_items:
dict[item] = 0
write_csv('information/associations/' + name, dict)
except:
pee = 'poo'
#path3.tolist()
#print(path3)
#print(path3)
kf = KFold(n_splits=5)
kf2 = StratifiedKFold(n_splits=5, shuffle=True)
i = 0
for train, test in kf2.split(path3, label):
#print(test)
#print(test.shape)
example = path3[test]
example = list(example)
print(example)
FILENAME = LABEL_PATH + '/fold_' + str(i) + '.csv'
UT.write_csv(FILENAME, example)
#FILENAME = LABEL_PATH + '/fold_' + str(i) + '.csv'
df = pd.read_csv(FILENAME, header=None)
data = df.values
data = list(map(list, zip(*data)))
data = pd.DataFrame(data)
data.to_csv(FILENAME, header=0, index=0)
i = i + 1
#print(example)
#print(example)
filenames = [
LABEL_PATH + '/fold_0.csv', LABEL_PATH + '/fold_1.csv',
usleg_db.authenticate("readonly", "smappnyu")
# Query for tweets - more complex than we're used to. Using AND.
# (NOTE that mongo explicitly ANDs multiple comma-separated search clauses,
# however, if two clauses on the same field with the same operator, must use $and)
start = datetime(2013, 8, 21)
end = datetime(2013, 9, 7)
results = usleg_db.legislator_tweets.find({
"timestamp": {
"$gte": start,
"$lt": end
},
"$and": [{
"text": {
"$regex": "syria",
"$options": "i"
}
}, {
"text": {
"$regex": "interven",
"$options": "i"
}
}]
})
print "Found {0} tweets on topic. Writing to CSV file".format(
results.count(with_limit_and_skip=True))
write_csv(results, "usleg-syria-intervene.csv")
for item in all_items:
try:
all_props = {}
for file in iterate_directory(
'/home/user/Projects/Python/Recipes/Cookbook'):
temp = {}
log = open('log_prop.txt', 'w')
log.write('item: ' + item + ', file: ' + file)
log.close()
ingredients = generate_ingredients(
'/home/user/Projects/Python/Recipes/Cookbook/' + file)
rec = recipe(ingredients)
if item in rec.items:
for food in rec.items:
temp[food] = rec.items[item] / (rec.items[food] + 0.000001)
else:
for food in rec.items:
temp[food] = 0
for food in temp:
if food not in all_props:
all_props[food] = temp[food]
else:
if temp[food] != 0:
all_props[food] = (all_props[food] + temp[food]) / 2
name = '_'.join(item.strip(' "\'').split(' ')).strip(' "\'').strip(
"'").strip('"') + '.csv'
write_csv('information/proportions/' + name, all_props)
print('done')
except:
pee = 'poo'
# input_data_file = APP_DIR + '/data/input/debug.txt'
input_data_file = APP_DIR + '/data/input/word_list.txt'
input_word_list = load_input_word_list(input_data_file)
load_vn_dict()
vn_dict = VNDict.get_instance()
# analyze
results = {WordTypeEnum.VERB: [], WordTypeEnum.ADJ: []}
for input_word in input_word_list:
words_were_found = vn_dict.look_up(input_word.txt,
w_kind=input_word.kind)
if not words_were_found:
continue
nearest_word = find_nearest_word(input_word, words_were_found)
results[nearest_word.type].append(nearest_word)
if Setting.GREEDY:
additional_verbs, additional_adjectives = greedy(
results, Setting.GREEDY_ALGORITHMS)
results[
WordTypeEnum.VERB] = results[WordTypeEnum.VERB] + additional_verbs
results[WordTypeEnum.
ADJ] = results[WordTypeEnum.ADJ] + additional_adjectives
write_csv(APP_DIR + '/data/output/{}.txt'.format(WordTypeEnum.VERB.name),
results[WordTypeEnum.VERB])
write_csv(APP_DIR + '/data/output/{}.txt'.format(WordTypeEnum.ADJ.name),
results[WordTypeEnum.ADJ])
