def get_matrix():
with Timer('Get cosine distance matrix from requisites'):
with Timer('Load JSON'):
with open(args.input_duplicates_file) as f:
data = json.load(f)
with Timer('Compute distances'):
params = {'organization': 'UPC', 'threshold': 0.1}
response = requests.post(COMPUTE_CLUSTERS_ENDPOINT,
params=params,
json=data)
assert response.ok
response_matrix = response.json()['dependencies']
assert len(response_matrix) == len(data['requirements'])
with Timer('Create mapping'):
mapping_dict = {}
for idx, req in enumerate(data['requirements']):
mapping_dict[idx] = req
with open(args.output_matrix_file, 'w') as f:
json.dump(mapping_dict, f)
with Timer('Save matrix'):
with open(args.output_mapping_file, 'w') as f:
json.dump(response_matrix, f)
def build_api_model():
with Timer('Build model calling API'):
with Timer('Load JSON'):
with open(args.input_requirements_file) as f:
data = json.load(f)
with Timer('Build model'):
params = {'organization': 'UPC'}
response = requests.post(BUILD_MODEL_ENDPOINT, params=params, json=data)
assert response.ok
def on_connection_ready(self, conn):
import functools
from src.util.timer import Timer
from proto.msg.heart_beat_msg import HeartBeatMsg
self.add_timer(
Timer(TestClient.HEART_BEAT_INTERNAL,
functools.partial(self.send_msg, HeartBeatMsg())))
def get_matrix():
with Timer('Load Validation duplicates JSON'):
with open(args.validation_duplicates_file) as f:
duplicates_requisites = json.load(f)
duplicates_requisites = duplicates_requisites['requirements']
duplicates_requisites = _chunks(duplicates_requisites, 2)
with Timer('Load clusters JSON'):
with open(args.clusters_file) as f:
clusters = json.load(f)
with Timer('Eval clusters'):
correct = 0
total_groups = len(duplicates_requisites)
for dup in tqdm(duplicates_requisites, total=total_groups):
correct += _eval_dup(dup, clusters)
accuracy = (correct / total_groups) * 100
log.info('ACCURACY: {}%'.format(accuracy))
def _search(index):
with Timer('Load JSON'):
json_path_score = os.path.join(args.data_path, JSON_FILE_NAME)
json_path_mapping = os.path.join(args.data_path, MAPPING_FILE_NAME)
mapping = helper.load_json(json_path_mapping)
score_matrix = helper.load_json(json_path_score)
clusters = {}
next_id = 0
req_to_cl = {}
progress_bar = tqdm(total=len(score_matrix))
for i, vector_arrays in enumerate(score_matrix):
vector_arrays = helper.to_nparray(vector_arrays)
closest, distances = index.query(vector_arrays, NEIGHBOURHOOD_AMOUNT)
similar = {i}
for j, dist in zip(closest, distances):
if j != i:
if dist < DISTANCE_THRESHOLD:
similar.add(int(j))
matching_clusters = set()
for j in similar:
if j in req_to_cl:
matching_clusters.add(req_to_cl[j])
for cl in matching_clusters:
similar.update(clusters.pop(cl))
cl_id = next_id
next_id += 1
clusters[cl_id] = similar
for j in similar:
req_to_cl[j] = cl_id
progress_bar.update(1)
similar_clusters = [list(cl) for cl in clusters.values()]
for i in range(0, len(similar_clusters)):
for j in range(0, len(similar_clusters[i])):
similar_clusters[i][j] = mapping[str(similar_clusters[i][j])]
return similar_clusters
assert len(factors) == len(anneal_steps)
assert len(factors) == len(max_iters)
base_expr = fa.Expression(("0.0", "X*x[1]"),
element=pde.V.ufl_element(),
X=MAX_DISP)
for Xi1, Xi2 in zip(Xi1s, Xi2s):
print("Pores: {}, {}".format(Xi1, Xi2))
boundary_data = torch.zeros(len(cem.global_coords), 2)
boundary_data[:, 1] = MAX_DISP * torch.Tensor(cem.global_coords)[:, 1]
params = torch.zeros(RVES_WIDTH * RVES_WIDTH, 2)
params[:, 0] = Xi1
params[:, 1] = Xi2
with Timer() as t:
surr_soln, traj_u, traj_f, traj_g = cem.solve(
params,
boundary_data,
cem_constraint_mask,
force_data,
step_size=0.25,
opt_steps=10000,
return_intermediate=True,
)
surr_time = t.interval
surr_numel = surr_soln.numel()
surr_E = cem.energy(surr_soln, params, force_data)
surr_coords = surr_soln.data.cpu().numpy()
fem_times = []
for msg_handler in msg_handlers:
msg_handler.run(message)
@bot.register(
chats=[bot.file_helper],
# msg_types=setting.MSG_TYPES,
except_self=False,
run_async=True)
def file_helper(msg):
msg_handlers = [
'src.handlers.textMessageHandler.TextMessageHandler',
'src.handlers.userHandler.UserHandler',
'src.handlers.searchArticleHandler.SearchArticleHandler'
]
msg_handlers = container.singletens(msg_handlers)
for msg_handler in msg_handlers:
msg_handler.run(msg)
timers.append(Timer(KeepAliveTask(bot), 1200, True))
timers.append(Timer(SyncUserTask(bot), 3600*3, True))
for t in timers:
t.start()
bot.join()
shutdown(0, timers)
content = []
for t in parser.table:
content.append(self._parseRow(t))
data = pd.DataFrame(content)
data.columns = self.COLS
data = data.set_index('Ticker')
return data
def parse(self):
for date in self.dates:
self.hist[date] = self._parse(date)
print self.hist[date]
def dump(self):
for dt, df in self.hist.iteritems():
year, month= dt[:4], dt[4:6]
directory = os.path.join(self.dumpbase, year, month)
if not os.path.exists(directory):
os.makedirs(directory)
path = os.path.join(directory, dt)
df.to_csv(path)
from src.util.timer import Timer
crawler = CoinMkttCapHistCrawler()
with Timer('Load'):
crawler.parse()
with Timer('Dump'):
crawler.dump()
def run(episodes=10,
experiment='swat',
max_actions=2**26,
knn=0.0001,
in_coeff=0.01,
ext_coeff=1,
expl=0.7,
flag=1,
l_rate=0.0001,
Discrete=True,
RND=True,
i=1):
env = cps()
print(env.observation_space)
print(env.action_space)
steps = 1000
if Discrete:
agent = WolpertingerAgent(env,
max_actions=max_actions,
k_ratio=knn,
l=l_rate)
else:
agent = DDPGAgent(env, l_rate=l_rate)
timer = Timer()
data = src.util.data.Data(flag)
data.set_agent(agent.get_name(),
int(agent.action_space.get_number_of_actions()),
agent.k_nearest_neighbors, 3)
data.set_experiment(experiment, agent.low.tolist(), agent.high.tolist(),
episodes)
agent.add_data_fetch(data)
print(data.get_file_name())
full_epoch_timer = Timer()
reward_sum = 0
for ep in range(episodes):
timer.reset()
observation = env.reset()
logging.debug('episodes:{}'.format(ep))
total_reward = 0
end = False
print('Episode ', ep, '/', episodes - 1, 'started...', end='')
for t in range(steps):
if np.random.uniform() > expl: # exploration
action = np.array(
np.random.randint(2, size=agent.action_space_size))
else:
action = agent.act(observation)
data.set__action(np.array([action]).tolist())
prev_observation = observation
observation_, ext_reward, done, f = env.step(
np.array([action]), prev_observation, flag, t)
data.set__state(np.array(observation_).tolist())
if RND:
in_reward = agent.get_in_reward(observation_)
else:
in_reward = 0.0
data.set_i_reward(in_reward)
data.set_e_reward(ext_reward)
reward = in_reward * in_coeff + ext_reward * ext_coeff
data.set_reward(reward)
observation = observation_
episode = {
'obs': np.array(prev_observation),
'action': np.array(action),
'reward': reward,
'obs2': np.array(observation),
'done': done,
'i_reward': in_reward,
't': t
}
agent.observe(episode)
total_reward += reward
if done:
end = True
if done or (t == steps - 1):
data.set_step(t)
t += 1
reward_sum += total_reward
time_passed = timer.get_time()
data.set_ep_time(time_passed / 1000)
print('Reward:{} Steps:{} t:{} ({}/step) Cur avg={}'.format(
total_reward, t, time_passed, round(time_passed / t),
np.round(reward_sum / (ep + 1))))
time = full_epoch_timer.get_time()
print("time:", time / 1000)
data.finish_and_store_episode(exp=expl,
l_r=l_rate,
flag=flag,
i=i)
break
if end:
break
time = full_epoch_timer.get_time()
print('Run {} episodes in {} seconds and got {} average reward'.format(
ep, time / 1000, reward_sum / (ep + 1)))
data.save(exp=expl, l_r=l_rate, flag=flag, j=i)
# encoding=utf8
import sys, os
sys.path.append(os.path.realpath('.'))
if __name__ == '__main__':
import time
from src.net.loop import EventLoop
from src.util.timer import Timer, TimerQueue
from src.util.logger import Logger
Logger.start_logger_service()
timer_queue = TimerQueue(EventLoop(0.01))
def test_func():
print 'hello'
def test_func1():
print 'hello repeat'
timer_ins = Timer(0, test_func)
print timer_ins.timer_id
timer_ins1 = Timer(1, test_func1)
print timer_ins1.timer_id
timer_queue.add_timer(timer_ins)
timer_queue.add_timer(timer_ins1)
while 1:
timer_queue.schedule()
time.sleep(1)
pass