def __call__(self, weights, update, gradients, staleness, epoch):
"""
:param weights: Copy of the model weights
:param update: Curent update
:param gradients: List of gradients
:param staleness: Staleness of each gradient
:param epoch: Current epoch
"""
lrr = -self.learning_rate_func(epoch, update) * len(gradients)
print(lrr)
logger.debug("the learning rate is:", lrr)
start_time = time.time()
if not self.staleness_aware:
gradient = gradients
else:
gradient = [np.divide(g, s) for g, s in zip(gradients, staleness)]
grad = np.mean(gradient, axis=0)
for i, g in enumerate(grad):
if self.momentum != 0:
buf = self.buf[i]
np.add(np.multiply(self.momentum, buf, out=buf), g, out=buf)
if self.nesterov:
np.add(np.multiply(self.momentum, buf), g, out=g)
else:
g = buf
np.add(self.weight[i], np.multiply(lrr, g), out=self.weight[i])
end_time = time.time()
if self.flags.time_program:
self.times.append(end_time - start_time)
return self.weight
def create_weights(flags, agent_function, test_data, test_labels, batch_size,pipe):
"""
Initializes the weights of the model
:param agent_function: Implementation of the abstract AgentFunction class
:param test_data: data of the test set
:param test_labels: Labels ot the test set
:param batch_size: batch size to be used
:param pipe: pipe to send back the data
"""
if flags.load_save:
logger.state("Loading saved weights at:", flags.saved_weights)
f = f = open(flags.saved_weights, "rb")
ret = pickle.load(f)
weights = ret[0][0]
else:
logger.debug("Before calling the agent_function")
af = agent_function(flags, test_data, test_labels, batch_size, 0,1, True, None, 1)
logger.debug("After calling the agent_function")
weights = af.get_weight()
af.close()
if flags.eamsgd:
weights.append(np.array([-0.05],np.float32)) #inital learning rate for eamsgd
pipe.send(weights)
pipe.close()
def __call__(self, weights, update, gradients, staleness, epoch):
"""
:param weights: Copy of the model weights
:param update: Curent update
:param gradients: List of gradients
:param staleness: Staleness of each gradient
:param epoch: Current epoch
"""
lrr = self.learning_rate_func(epoch, update) * len(gradients)
#print(lrr, flush=True)
start_time = time.time()
if not self.staleness_aware:
gradient = gradients
else:
logger.debug("optimizer staleness list", staleness)
for i, k in enumerate(staleness):
if k < self.agents:
staleness[i] = self.agents
gradient = [np.divide(g, s) for g, s in zip(gradients, staleness)]
grad = np.mean(gradient, axis=0)
for l, w in enumerate(weights):
np.subtract(w, lrr * grad[l], out=w)
# print("release lock opt", flush=True)
end_time = time.time()
if self.flags.time_program:
self.times.append(end_time - start_time)
return weights
def learn(self, result, delta, learning_rate):
super(BiasedNeuralLayer, self).learn(result, delta, learning_rate)
Logger.debug("delta: " + str(delta))
Logger.debug("biases: " + str(self.biases))
tmp = -(learning_rate * delta)
self.biases = tmp + np.atleast_2d(self.biases)
self.biases[self.biases < 0] = 0
def __call__(self, epoch, update):
count = 0
ii = 1
gradients_list = []
metrics_list = []
from_list = []
step_list = []
global_update_list = []
if self.mode == 1:
agnt_nr = self.holdback.pop()
logger.state("poped", agnt_nr, flush=True)
if len(self.holdback) == 0:
self.mode = 0
return None, [agnt_nr], None, None, None, 0, 1
while True:
i, p = next(self.gen)
if p.poll():
grads = []
for i, fs in enumerate(self.float_sizes):
w = p.recv_bytes(fs * 4)
grads.append(np.ndarray(self.shapes[i], np.float32, w))
last_update, step, agnt_nr, metrics = p.recv()
sending = 2
if self.mode == 0:
if self.updates_per_learner[agnt_nr-1] >= self.min + self.max_difference:
self.holdback.append(agnt_nr)
sending = 0
logger.debug("holding back", agnt_nr, flush=True)
else:
self.updates_per_learner[agnt_nr - 1] += 1
new_min = np.min(self.updates_per_learner)
if new_min != self.min:
self.min = new_min
if len(self.holdback) != 0:
self.mode = 1
logger.debug("reg send", agnt_nr, flush=True)
count += 1
gradients_list.append(grads)
metrics_list.append(metrics)
from_list.append(agnt_nr)
global_update_list.append(last_update)
step_list.append(1)
else:
ii += 1
if ii % self.learners == 0:
time.sleep(0.0001)
if count == self.num:
if self.timing:
self.counter += 1
return gradients_list, from_list, global_update_list,step_list, metrics_list, 0, sending
def get_delta(self, out_data, last_delta, last_weights):
"""calculate delta for layer before"""
#Logger.DEBUG = True
Logger.debug("Get delta: ")
Logger.debug("out: " + str(np.shape(out_data)))
Logger.debug("last_delta: " + str(np.shape(last_delta)))
Logger.debug("last_weights: " + str(np.shape(last_weights)))
dot = np.dot(last_weights.T, last_delta)
Logger.debug("dot shape: " + str(np.shape(dot)))
delta = dot * self.activation_deriv(out_data)
#Logger.debug("delta shape: " + str(np.shape(delta)))
#Logger.DEBUG = False
return delta
def get_weight(self):
"""
:return: the current weight of the agent
"""
list = []
counter = 0
for p in self.resnet.parameters():
#print(p)
#p.cpu()
list.append(p.cpu().detach().numpy())
counter +=1
logger.debug("counter:", counter)
return list
def __init__(self, in_size, out_size, activation_fn=None, activation_fn_deriv=None):
Logger.debug("create Neural Layer: \n" + \
" in_size: " + str(in_size) + "\n" + \
" out_size: " + str(out_size))
# initialize weight matrix
self.weights = np.random.uniform(-0.001, 0.001, (out_size, in_size)).astype(np.float64)
# set biases if not already set (by child class for example)
if not hasattr(self, 'biases'):
self.biases = np.zeros(out_size).astype(np.float64)
# set activation function and derivate
self.activation = activation_fn or NeuralLayer.activation_linear
self.activation_deriv = activation_fn_deriv or NeuralLayer.activation_linear_deriv
# set size and input size
self.size = out_size
self.in_size = in_size
def __del__(self):
logger.debug("del PS")
if not self.orderly:
for p in self.shards_list:
if p.is_alive():
try:
p.kill()
except Exception:
raise Exception
for p in self.threads:
if p.is_alive():
try:
p.kill()
except Exception:
raise Exception
def test_evaluation(self):
"""
Does the evaluation of the test set
"""
logger.debug("starting evaluation")
weights = self.test_weights_unified
other_vars = self.test_averages_unified
list1 = []
for i in range(0, self.evaluation_steps):
res = self.eval_obj.evaluate(weights, other_vars)
list1.append(res)
tmp1 = np.mean(list1, axis=0)
return tmp1
def elastic(self, bins_updates):
# based on https://arxiv.org/pdf/1412.6651.pdf
logger.debug("elastic", bins_updates, flush=True)
weights = self.flags.bin_weights
bins_updates = [u * v for (u, v) in zip(bins_updates, weights)]
sum = np.sum(bins_updates)
bins_updates = [i / sum for i in bins_updates]
logger.debug("elastic", bins_updates, flush=True)
new_weight = [
np.multiply(self.weights[i], m) for i, m in enumerate(bins_updates)
]
new_weight = np.sum(new_weight, axis=0)
for i, w in enumerate(new_weight):
for wb in self.weights:
wb[i][:] = w[:]
def __call__(self, epoch, update):
"""
Implementation with the learners waiting n/learners others to arrive
"""
ii=1
count = 0
list = []
gradients_list = []
metrics_list = []
from_list = []
step_list = []
global_update_list = []
while True:
i, p = next(self.gen)
if p.poll():
grads = []
for i, fs in enumerate(self.float_sizes):
w = p.recv_bytes(fs * 4)
grads.append(np.ndarray(self.shapes[i], np.float32, w))
last_update, step, agnt_nr, metrics = p.recv()
count += 1
gradients_list.append(grads)
metrics_list.append(metrics)
from_list.append(agnt_nr)
global_update_list.append(last_update)
step_list.append(1)
staleness = update - last_update
else:
ii += 1
if ii % self.learners == 0:
time.sleep(0.0001)
if count == self.num:
binning = 0
for i in self.bins:
if staleness >= i:
binning += 1
else:
break
self.bin_counts[binning] += 1
logger.debug("staleness", staleness, "put in bin", binning, flush=True)
return gradients_list, from_list, global_update_list, step_list, metrics_list, binning, 2
class FatDetector(object):
def __init__(self):
self.model = resnet50(num_classes=2, pretrained=False)
self.model.cuda().eval()
self.model.load_state_dict(torch.load('ckpt/fat/fat.pth'))
self.image_size = [(672, 224), (448, 224), (224, 224)]
self.logger = Logger()
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
self.transform = transforms.Compose([
transforms.ScalePad(self.image_size),
transforms.ToTensor(), normalize
])
def preprocessing(self, image):
return Variable(self.transform(image).unsqueeze(0),
volatile=True).cuda()
image = image.resize(self.image_size)
image = np.asarray(image, dtype=np.uint8)
image = torch.ByteTensor(image)
image = image.float().div(255)
image[:, :, 0] = (image[:, :, 0] - 0.485) / 0.229
image[:, :, 1] = (image[:, :, 1] - 0.456) / 0.224
image[:, :, 2] = (image[:, :, 2] - 0.406) / 0.225
image = image.unsqueeze(0).permute(0, 3, 1, 2).contiguous()
return Variable(image, volatile=True).cuda()
def detect(self, image):
start = time.time()
inputs = self.preprocessing(image)
self.logger.debug('preprocessing: %s', time.time() - start)
start = time.time()
#bs, ncrops, c, h, w = inputs.size()
#logits = self.model(inputs.view(-1, c, h, w))
#logits = logits.view(bs, ncrops, -1).mean(dim=1)
logits = self.model(inputs)
self.logger.debug('forward: %s', time.time() - start)
preds = F.softmax(logits)
return preds.data.cpu().numpy()[0][1]
def _get_browse_tag(url):
# 다음 세부분류 category로의 링크들
browse_url_list = []
# 특정 Category로 채용 공고를 찾는 링크들
search_url_list = []
soup = BeautifulSoup(_get_html(url), 'html.parser')
main_content = soup.find("table", {"id": "main_content"})
if main_content == None:
main_content = soup.find("table", {"id": "browsejobs_main_content"})
for a_tag in main_content.find_all("a"):
# print(a_tag)
if "browsejobs" in a_tag["href"]:
browse_url_list.append(a_tag["href"])
Logger.debug("browsejobs = " + a_tag.text)
else:
search_url_list.append(a_tag["href"])
Logger.debug("search = " + a_tag.text)
return browse_url_list, search_url_list
def get_emp_key(url, index):
# 해당 페이지에 "다음" 글씨가 있는지 여부
# next_page_exist=True
req_url = url + "&start=" + str(index * 50)
Logger.debug("get_emp_key request URL = " + url)
html = _get_html(url)
soup = BeautifulSoup(html, 'html.parser')
job_key_list = []
# a_tag 중 채용공고에 대한 Tag를 찾는 방식
# -> 일반 채용공고 뿐만 아니라 유료 광고도 불러옴 (일반 채용 공고 10개 + 유료 광고 6개)
for a_tag in soup.find_all("a", {"data-tn-element": "jobTitle"}):
split_id = a_tag.parent["id"].split("_")
id_type = split_id[0]
emp_id = split_id[1]
job_key_list.append(emp_id)
return job_key_list
def write_search_url_list(flush_num, max_sleep):
browse_url_list, _ = _get_browse_tag(INDEED_CATEGORY_URL)
file_index = 0
search_url_list = []
Logger.info("browse url list length = " + str(len(browse_url_list)) +
" search url list length = " + str(len(search_url_list)))
# browse_url 없을 때 까지 search url 계속 추가하면서 찾음
while len(browse_url_list) != 0:
Logger.debug("checking browse url = " + browse_url_list[0])
new_br_list, new_search_list = _get_browse_tag(INDEED_URL +
browse_url_list[0])
browse_url_list = browse_url_list + new_br_list
search_url_list = search_url_list + new_search_list
del browse_url_list[0]
# return search_url_list
Logger.info("browse url list length = " + str(len(browse_url_list)) +
" search url list length = " + str(len(search_url_list)))
time.sleep(uniform(0.1, max_sleep))
if len(search_url_list) >= flush_num:
file_path = config.CATEGORY_URL_FILE_PATH + "search_url_list_" + str(
file_index) + ".txt"
# with open(pkl_name,"wb") as f:
# pkl.dump(search_url_list,f)
with open(file_path, "w") as f:
f.write("\n".join(search_url_list))
Logger.info("searurl " + str(len(search_url_list)) +
" is written to " + file_path + "\n")
search_url_list = []
file_index += 1
def learn(self, result, delta, learning_rate):
#raw_input("press Enter")
# apply learning rule
#Logger.DEBUG = True
Logger.debug("NeuralLayer:learn")
Logger.debug("result: " + str(np.shape(result)))
Logger.debug("delta: " + str(np.shape(delta)))# + "\nresult shape:" + str(np.shape(result)))
delta_weights = learning_rate * np.outer(delta, result)
#Logger.debug("delta weights shape:" + str(np.shape(delta_weights)))
#Logger.log(str(delta_weights))
self.weights += delta_weights
def feed(self, input_data):
# calculate activation of layer for given inputs
#Logger.DEBUG = True
Logger.debug("NeuralLayer:feed")
Logger.debug("input: " + str(np.shape(input_data)))
dot = np.dot(self.weights, input_data)
result = self.activation(dot + np.atleast_2d(self.biases).T)
Logger.debug("output: " + str(np.shape(result)))
#Logger.debug("weights: " + str(np.shape(self.weights)))
#Logger.debug("dot shape: " + str(np.shape(dot)))
#Logger.DEBUG = False
return result
def close(self):
"""
need to close the data loaders otherwise they keep running
"""
logger.debug("close resnet torch", flush=True)
if self.timess != []:
logger.state("time is: ",np.mean(self.timess, axis = 0), flush=True )
a = getattr(self, "test_set_load", None)
b = getattr(self,"train_set_load", None)
if a != None:
logger.debug("shut them down",flush=True)
# del(a)
a._shutdown_workers()
if b != None:
logger.debug("shut them down")
b._shutdown_workers()
def work(self):
"""
Trains for one batch using the self.agent_function and then sends the gradient back to
the parameter shards and gets an up-to-date weight back
It logs the time when it finished each update.
in a file stored at self.flags.time_stamps_file_name
"""
# store the global update of each weight
if self.flags.load_save:
self.batch_size = self.flags.batch_size // self.num_agents
# number of iteration for the total batch size (Batch size if there were one agent)
self.iterations = self.flags.total_iterations
# how many iterations there are per epoch
if self.flags.drop_remainder:
self.iterations_in_epoch = math.floor(
self.flags.train_set_size /
(self.num_agents * self.batch_size))
else:
self.iterations_in_epoch = math.ceil(
self.flags.train_set_size /
(self.num_agents * self.batch_size))
self.iterations_in_epoch //= 1
global_update = [
self.flags.starting_epoch * self.iterations_in_epoch *
self.num_agents
] * self.num_shards
else:
global_update = [0] * self.num_shards
#list that sotres the size of each weight in bytes, needed for sending them
float_sizes = []
total_size = []
shapes = []
weights = []
for w in self.weights:
weights += w
shap = []
fs = []
ts = 0
for i in w:
l = 1
shap.append(i.shape)
for r in i.shape:
l *= r
fs.append(l)
ts += l
shapes.append(shap)
float_sizes.append(fs)
total_size.append(ts)
print(total_size[0], flush=True)
print(float_sizes[0], flush=True)
print(shapes[0], flush=True)
logger.debug("Agent nr",
self.agent_nr,
"pid is: ",
getpid(),
flush=True)
if self.data is not None:
logger.debug("Agent nr",
self.agent_nr,
self.data.shape,
self.labels.shape,
self.num_labels,
self.batch_size,
flush=True)
agent = self.agent_function(self.flags, self.data, self.labels,
self.batch_size, self.gpu_nr,
self.fraction, False, weights,
self.agent_nr)
del (self.data)
del (self.labels)
itr = True
step = 0
while itr:
start_time = time.time()
weights = []
for w in self.weights:
weights += w
grads_, metrics = agent.train(weights)
agent_part_finished = time.time()
if self.slow_down != None:
if type(self.slow_down) is tuple:
if self.flags.slow_down_type == "gauss":
r = self.random.gauss(self.slow_down[0],
self.slow_down[1])
if r <= 0:
time.sleep(0)
else:
time.sleep(r)
else:
if self.flags.slow_down_type == "ber":
r = self.random.uniform(0, 1)
if self.p >= r:
print(r)
time.sleep(self.slow_down)
elif self.flags.slow_down_type == "time":
time.sleep(self.slow_down)
offset = 0
for c, q in enumerate(self.pipes):
new_offset = offset + len(self.weights[c])
for bits in grads_[offset:new_offset]:
q.send_bytes(bits)
q.send((global_update[c], step, self.agent_nr, metrics))
offset = new_offset
logger.debug("agent", self.agent_nr, "sent to", c)
itr_inner = 0
while itr_inner < self.num_shards:
for c, p in enumerate(self.pipes):
if p.poll():
while True:
logger.debug("Agent", self.agent_nr, "is recv")
tmp = p.recv()
logger.debug("Agent", self.agent_nr, "has recv")
if tmp[0] == "update weights":
global_update[c] = p.recv()
weight = []
for fsi, fs in enumerate(float_sizes[c]):
w = p.recv_bytes(fs * 4)
weight.append(
np.ndarray(shapes[c][fsi], np.float32,
w))
self.weights[c] = weight
itr_inner += 1
break
elif tmp[0] == "stop":
logger.debug("Agent nr",
self.agent_nr,
"got stoped",
flush=True)
self.pipes[c].close()
del (self.pipes[c])
self.shut_down()
itr = False
itr_inner = self.num_shards
break
elif tmp[0] == "globals":
logger.debug("agent is sending globals")
glob = agent.get_globals()
p.send(glob)
else:
time.sleep(0.0001)
self.log[step, 0] = time.time()
self.log[step, 1] = time.clock()
self.saver.save_1D(self.log[step, :])
step += 1
end_time = time.time()
if self.timing:
self.times.append(
[agent_part_finished - start_time, end_time - start_time])
logger.debug("Agent nr",
self.agent_nr,
"loop actually finished",
flush=True)
agent.close()
for p in self.pipes:
p.close()
if self.timing:
t = np.average(self.times, axis=0)
logger.state("Agent nr",
self.agent_nr,
"running agent function:",
t[0],
"total time",
t[1],
flush=True)
def __call__(self, weights, update, gradients, staleness, epoch):
"""
:param weights: Copy of the model weights
:param update: Curent update
:param gradients: List of gradients
:param staleness: Staleness of each gradient
:param epoch: Current epoch
"""
self.learning_rate_func(epoch, update)
lr = torch.tensor(-self.learning_rate,
dtype=torch.float,
device=self.device)
start_time = time.time()
if not self.use_exp:
gradient = gradients
else:
lr_list = []
def exp(x):
return self.equalizer * np.exp(-(x - self.mean)**2 /
(2 * self.std**2))
for i, s in enumerate(staleness):
# print(s)
pos = self.staleness_counter % self.ring_size
self.staleness_ring[pos] = s
lrexp = exp(s)
self.lr_ring[pos] = lrexp
lr_list.append(lrexp)
self.staleness_counter += 1
if pos == self.ring_size - 1: ##is this at the end of the ring
std = np.std(self.staleness_ring)
self.std = self.decay * self.std + self.learn * std
if self.std == 0: # not numerically correct
self.std = 0.2
mean = np.mean(self.staleness_ring)
self.mean = self.decay * self.mean + self.learn * mean
sum = np.sum(
self.lr_ring
) #want sum to be about (self.ring_size/self.agents)
equalizer = self.equalizer * (
self.ring_size / self.agents
) / sum # problme that we change the equalizer and avgs while iterating through the stalenesses???????
self.equalizer = self.decay * self.equalizer + self.learn * equalizer
logger.debug("mean",
self.mean,
"std",
self.std,
"stalleness ring",
self.staleness_ring,
"lr ring",
self.lr_ring,
"sum",
sum,
"equalizer",
self.equalizer,
flush=True)
if self.staleness_counter <= self.ring_size * 5:
gradient = gradients
else:
gradient = [
np.multiply(g, s) for g, s in zip(gradients, lr_list)
]
grad = np.mean(gradient, axis=0)
grads = []
for g in grad:
grads.append(torch.tensor(g, device=self.device))
# followig code snipped is based on https://github.com/pytorch/pytorch/blob/master/torch/optim/sgd.py
for i, p in enumerate(self.weights):
d_p = grads[i].data
if self.momentum != 0:
buf = self.buf[i]
lr_grad = d_p.mul(lr)
buf.mul_(self.momentum).add_(
lr_grad) #corrected the version of lr
#buf.mul_(self.momentum).add_(d_p)
if self.nesterov:
d_p = buf.mul(self.momentum).add_(d_p.mul(lr))
#d_p = d_p.add(self.momentum, buf)
else:
d_p = buf
#p.data.add_(lr,d_p)
p.data.add_(d_p)
end_time = time.time()
if self.flags.time_program:
self.times.append(end_time - start_time)
w = []
for wg in self.weights:
w.append(wg.data.cpu().numpy())
return w
def __init__(self, flags, data, labels, batch_size, gpu_nr, fraction, testing, weights=None, agent_nr=1):
"""
:param flags: flags set by the user
:param data: sampels if None have to be loaded in the by the actual implementation
:param labels: lables if None have to be loaded in the by the actual implementation
:param batch_size: batch size to be used
:param gpu_nr: Number of the GPU, the agent should run on
:param fraction: fraction of the GPU memory the agent should maximally use
:param testing: True if we run a test set, False for training
"""
super().__init__(flags, data, labels, batch_size, gpu_nr, fraction, testing, weights, agent_nr)
seed = (int(time.time()*100)*agent_nr) % (2**32 - 1)
self.timess = []
logger.debug("seed", seed)
np.random.seed(seed)
#random.seed(123)
torch.manual_seed(seed) #set randomness for everything that torch uses the pythono random for. Here padding, image order
self.flags = flags
self.resnet = ResNet_cifar(depth=44)#num_classes = 100
#self.resnet = ResNet_imagenet(block=BasicBlock,
# layers=[2, 2, 2, 2],
# expansion=1)
self.loss = CrossEntropyLoss()
self.batch_size = batch_size
self.gpu_nr = gpu_nr
logger.debug("number of gpus", torch.cuda.device_count())
if 'cuda' in self.flags.device:
self.device = torch.device(self.gpu_nr)
logger.debug("using device ", torch.cuda.get_device_name(self.device),"am Agent ",agent_nr)
else:
self.device = torch.device('cpu')
self.cpu = torch.device('cpu')
self.resnet.to(self.device, dtype = torch.float)
self.loss.to(self.device)
path = os.environ['HOME']
path = path + '/cifar10_data/'
logger.debug("gpu_nr", gpu_nr)
# copied and modified from
# convNet.pytorch/utils/regularization Regularizer class
# and convNet.pytorch/resnet: weight_decay_config function
self._named_parameters = list(
FilterParameters(self.resnet, **{'parameter_name': lambda n: not n.endswith('bias'),
'module': lambda m: not isinstance(m, nn.BatchNorm2d)}).named_parameters())
# copid and modified from:
# convNet.pytorch/main.py
if not testing:
self.train_set = DataRegime(getattr(self.resnet, 'data_regime', None),
defaults={'datasets_path':path , 'name': 'cifar10', 'split': 'train',
'augment': True,
'input_size': None, 'batch_size': batch_size, 'shuffle': True,
'num_workers': 1, 'pin_memory': True, 'drop_last': True,
'distributed': False, 'duplicates': 1, #batch augmentation
'cutout': {'holes': 1, 'length': 16} if True else None})
self.train_epoch = 0
self.train_iterator = self._train_generator()
else:
self.test_set = DataRegime(getattr(self.resnet, 'data_eval_regime', None),
defaults={'datasets_path': path, 'name': 'cifar10', 'split': 'val',
'augment': False,
'input_size': None, 'batch_size': batch_size,
'shuffle': False,
'num_workers': 1, 'pin_memory': True, 'drop_last': False})
self.test_epoch = 0
self.test_iterator = self._test_generator()
#end
if flags.load_save:
import pickle
f = f = open(flags.saved_weights, "rb")
ret= pickle.load(f)
glob = ret[1][0]
for i, p in enumerate(self.resnet.buffers()):
if np.isscalar(glob[i]):
p.data = torch.tensor(glob[i])
else:
p.data[:] = torch.tensor(glob[i], device=self.device)[:]
if self.flags.eamsgd and not testing:
weights = weights[0:-1]
if (self.flags.correction or self.flags.eamsgd) and not testing:
if weights != None:
self.velocity = []
self.velocity_grad = []
self.momentum = torch.tensor(0.9, dtype=torch.float, device = self.device)
for w in weights:
self.velocity.append(torch.zeros(w.shape,dtype=torch.float, device = self.device))
class MultiTaskDetector(object):
def __init__(self):
self.model = MultiTask(pretrained=False, backend='resnext101_32d')
self.model.cuda().eval()
self.model.load_state_dict(torch.load('ckpt/multi_task/multi_task.pth'))
self.category_mapping = yaml.load(open('config/category_mapping.yml'))
self.category_trans = yaml.load(open('config/category_trans.yml'))
self.attr_mapping = yaml.load(open('config/attr_mapping.yml'))
self.attr_trans = yaml.load(open('config/attr_trans.yml'))
self.image_size = (224, 224)
self.logger = Logger()
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
self.transform = transforms.Compose([
# transforms.Scale(self.image_size),
# transforms.ThreeCrop(self.image_size),
# transforms.Lambda(lambda crops: torch.stack([normalize(transforms.ToTensor()(crop)) for crop in crops]))
transforms.Resize(self.image_size),
transforms.ToTensor(),
normalize,
])
def preprocessing(self, image):
return Variable(self.transform(image).unsqueeze(0), volatile=True).cuda()
image = image.resize(self.image_size)
image = np.asarray(image, dtype=np.uint8)
image = torch.ByteTensor(image)
image = image.float().div(255)
image[:, :, 0] = (image[:, :, 0] - 0.485) / 0.229
image[:, :, 1] = (image[:, :, 1] - 0.456) / 0.224
image[:, :, 2] = (image[:, :, 2] - 0.406) / 0.225
image = image.unsqueeze(0).permute(0, 3, 1, 2).contiguous()
return Variable(image, volatile=True).cuda()
def detect(self, image, cls_threshold=0.3, attr_threshold=0.3):
start = time.time()
inputs = self.preprocessing(image)
self.logger.debug('preprocessing: %s', time.time() - start)
start = time.time()
#bs, ncrops, c, h, w = inputs.size()
#logits = self.model(inputs.view(-1, c, h, w))
#logits = logits.view(bs, ncrops, -1).mean(dim=1)
cls_out, attr_out = self.model(inputs)
self.logger.debug('forward: %s', time.time() - start)
cls_preds = F.softmax(cls_out)
cls_probs, cls_labels = cls_preds.topk(1, 1)
cls_label = cls_labels.data.cpu().tolist()[0][0]
cls_prob = cls_probs.data.cpu().tolist()[0][0]
attr_preds = (F.sigmoid(attr_out)).view(1, -1).data.cpu().tolist()[0]
attrs = [(self.attr_mapping.get(x[0], x[0]), x[1]) for x in enumerate(attr_preds)]
attrs = [(self.attr_trans.get(x[0], x[0]), x[1]) for x in attrs]
attrs = sorted(filter(lambda x: x[1] >= attr_threshold, attrs), key=lambda x: x[1], reverse=True)
if cls_prob > cls_threshold:
cls_name = self.category_mapping.get(cls_label)
category = (self.category_trans.get(cls_name, cls_name), cls_prob)
return category, attrs
else:
return None, attrs
class MultAccount:
def __init__(self, name, id_list):
"""
:param id_list: string id list
"""
self._name = name
self._date_list = []
self._account_list = []
self._len = len(id_list)
for id in id_list:
self._account_list.append(Account(id))
self._id_list = id_list
self._nav_tail = -1
self._pos_maxval = 0
self._pos_dd = 0
self._sum_investment = 0
self._mult_account_weight_list = [1.0 / self._len] \
* self._len
self._market_value = 0
self._pos_list = []
self._cum_ret_list = []
self._final_cum_ret = None
self._invest_start = False
self._logger = Logger()
@property
def name(self):
return self._name
@property
def account_list(self):
return self._account_list
@property
def account_num(self):
return self._len
def __iter__(self):
for accout in self._account_list:
yield accout
def find_maxdown(cls, pos_val, pos_maxval, mdd):
down_val = pos_maxval - pos_val
dd = down_val / (pos_maxval + 1e-10)
if down_val < 0:
pos_maxval = pos_val
elif dd > mdd:
mdd = dd
return pos_maxval, mdd
def update_datenav(self, date, nav_list):
market_value_sum = 0
# obtain all accounts' remaining weight
market_value_list = []
for account, nav in list(zip(self._account_list, nav_list)):
account.update_nav(nav)
market_value_sum += account.market_value
market_value_list.append(account.market_value)
if market_value_sum > 0:
self._mult_account_weight_list = [
v / market_value_sum for v in market_value_list
]
self._market_value = market_value_sum
self._logger.debug(
module='account',
file='multacct.py',
content=" " * 4 +
"multi accounts, market value now is %.5f." % market_value_sum)
self._date_list.append(date)
self._pos_list.append(market_value_sum)
self._pos_maxval, self._pos_dd = \
self.find_maxdown(market_value_sum,
self._pos_maxval,
self._pos_dd)
self._cum_ret_list.append(self.cum_ret)
self._logger.debug(module='account',
file='multacct.py',
content=" " * 4 +
"return rate %s." % str(self.cum_ret))
def update_allocation(self, weight_list):
if 0.995 < sum(weight_list) < 1.005:
pass
else:
self._logger.debug(module='account',
file='multacct.py',
content="" * 4 +
"weights sum not equal to 1 is not allowed.")
return
practical_invest = 0
market_value_sum = 0
if self._invest_start:
pos_tail = self._pos_list[-1]
for account, weight in list(zip(self._account_list, weight_list)):
assert (isinstance(account, Account))
single_invest = weight * (pos_tail if pos_tail > 0 else 1)
adjust_invest = single_invest - account.market_value
if abs(adjust_invest) / (account.market_value + 1e-10) > 0.01:
if adjust_invest > 0:
practical_invest += adjust_invest
account.buy(adjust_invest)
elif adjust_invest < 0:
account.sell(abs(adjust_invest))
practical_invest -= account.surplus_value
account.extract_surplus()
market_value_sum += account.market_value
else:
for account, weight in list(zip(self._account_list, weight_list)):
practical_invest += weight
account.buy(weight)
market_value_sum += account.market_value
self._invest_start = True
self._sum_investment += practical_invest
self._market_value = market_value_sum
self._logger.debug(
module='account',
file='multacct.py',
content=" " * 4 +
"this transaction cost extra invest %.5f." % practical_invest)
self._logger.debug(
module='account',
file='multacct.py',
content=" " * 4 +
"cumulative investment is %.5f." % self._sum_investment)
self._logger.debug(
module='account',
file='multacct.py',
content=" " * 4 +
"multi accounts, market value after transaction is %.5f." %
market_value_sum)
def stop(self):
return_value_sum = 0
for account in self._account_list:
account.sell(account.market_value)
return_value_sum += account.extract_surplus()
self._final_cum_ret = (return_value_sum - self._sum_investment) / \
self._sum_investment
def logging(self):
self._logger.info(module='account',
file='multacct.py',
content={
'backtest_time': str(self._date_list[-1]),
'account_name': self._name,
'content': 'return rate is %.4f' % self.cum_ret
})
@property
def cum_ret(self):
if self._final_cum_ret:
return self._final_cum_ret
else:
return (self._market_value - self._sum_investment) / \
(self._sum_investment + 1e-10)
@property
def cum_ret_avg(self):
return sum(self._cum_ret_list) / len(self._cum_ret_list)
@property
def cum_ret_list(self):
return self._cum_ret_list
@property
def pos_list(self):
return self._pos_list
@property
def mult_account_weight_list(self):
return self._mult_account_weight_list
@property
def investment_sum(self):
return self._sum_investment
@property
def return_value(self):
return self.investment_sum * self.cum_ret
@property
def pos_dd(self):
return self._pos_dd
class Account:
def __init__(self, id, sub_discount=0.0015, red_discount=0.005):
self._id = id
self._nav = 1
self._volume = 0
self._market_value = 0
self._surplus_value = 0
self._sub_discount = sub_discount
self._red_discount = red_discount
self._logger = Logger()
@property
def volume(self):
return self._volume
@property
def market_value(self):
return self._market_value
@property
def surplus_value(self):
return self._surplus_value
def _update_market_value(self):
self._market_value = self._volume * self._nav
def update_nav(self, nav):
self._logger.debug(module='account',
file='singleacct.py',
content="----" * 2 +
"account %s, org market value is %.5f" %
(self._id, self._market_value))
self._nav = nav
self._update_market_value()
self._logger.debug(module='account',
file='singleacct.py',
content="----" * 3 +
"account %s, now market value is %.5f" %
(self._id, self._market_value))
def buy(self, weight):
self._logger.debug(module='account',
file='singleacct.py',
content="----" * 1 +
"account %s, buy weight %.5f, cut off %.5f" %
(self._id, weight, weight * self._sub_discount))
weight *= 1 - self._sub_discount
volume = weight / self._nav
self._volume += volume
self._update_market_value()
def sell(self, weight):
self._logger.debug(module='account',
file='singleacct.py',
content="----" * 1 +
"account %s, sell weight %.5f, cut off %.5f" %
(self._id, weight, weight * self._red_discount))
volume = weight / self._nav
self._volume -= volume
self._update_market_value()
self._surplus_value += \
volume * self._nav * (1 - self._red_discount)
def extract_surplus(self):
surplus_value = self._surplus_value
self._surplus_value = 0
return surplus_value
def write_emp_detail(url_list, max_sleep):
crawling_data_num = 0
start_time = time.time()
# job detail을 불러와 xml파일로 쓴 job key들
key_dict = {}
# 요청시 실패한 key list 저장
failed_key_list = []
url_index = 0
# URL 당 최대 1000개의 job detail적음
for url in url_list:
Logger.info(
str(url_index + 1) + "/" + str(len(url_list)) +
" job list request URL = " + url.replace("\n", ""))
url_index += 1
job_key_list = []
# 채용 공고 목록 페이지에서 job key들을 받아옴
# 50개씩 최대 20번 불러옴
try:
req_url = config.INDEED_URL + url + "&limit=50"
prev_key_list = []
for index in range(20):
cur_key_list = get_emp_key(req_url, index)
# 페이지 index가 넘어간 경우 계속 똑같은 list를 반복하므로 이 경우 더 이상 job key를 받아오지 않음
if prev_key_list == cur_key_list:
break
else:
prev_key_list = cur_key_list
job_key_list = job_key_list + cur_key_list
rand_sleep(max_sleep)
except:
continue
Logger.info("job key list length = " + str(len(job_key_list)))
not_exist_cnt = 0
# job key들의 채용 공고 detail 받아옴
for key in job_key_list:
# 해당 key에 대해서 아직 Crwaling 안한 경우
# Crawling 수행하고 xml 파일 적음
if not key in key_dict:
key_dict[key] = True
try:
xml = get_job_detail(key)
except:
Logger.warn(key + " parsing failed")
failed_key_list.append(key)
rand_sleep(max_sleep)
continue
file_path = config.JOB_DETAIL_FILE_PATH + key + ".xml"
_write_xml(xml, file_path)
Logger.debug(file_path + " is written")
not_exist_cnt += 1
crawling_data_num += 1
rand_sleep(max_sleep)
else:
Logger.debug("key " + key + " already exist")
# url
Logger.info("total data num =" + str(crawling_data_num) +
" added data num =" + str(not_exist_cnt) +
" elapsed time = " + str(time.time() - start_time) + "\n")
# 현재까지 가져온 모든 key들을 pkl로 저장
key_dict_path = "key_dict.pkl"
# 적은 job key의 dictionary를 pickle 파일로 저장
with open(key_dict_path, "wb") as f:
pkl.dump(key_dict, f)
Logger.info(key_dict_path + "has been written")
def __init__(self, flags, update_function, agent_function, data_loader, num_agents, slow_down, algorithm):
"""
:param flags: Flags set by the user.
:param update_function: Implementation of the UpdateFunction abstract class.
:param agent_function: Implementation of the abstract AgentFunction class.
:param data_loader: Function which returns ((train_data, train_labels),(test_data, test_labels),
number of distinct labels (Categories)).
If left None, then the data needs to be loaded by the agent_function.
:param num_agents: The number of Agents to be simulated.
:param slow_down: How much the agent gets slowed down, either a tuple (float a, float b),
a single float b or None for no slowdown.
:param algorithm: Implementation of the AlgorithmFunction abstract class.
"""
# flag used to figure out if the PS finished without an interruption
self.orderly = False
self.flags = flags
self.batch_size = self.flags.batch_size // num_agents
self.num_agents = num_agents
self.slow_down = slow_down
self.iterations = self.flags.total_iterations
logger.state("Number of iterations per agent " + str(self.iterations))
self.total_updates = self.iterations*self.num_agents
self.algorithm = algorithm
self.shards = self.flags.shards
self.number_of_labels = self.flags.number_of_labels
#if Dataset should get distributed
self.distribute = self.flags.distribute
self.update_rule_uninitiated = update_function
self.agent_function = agent_function
self.data_loader = data_loader
# list that stores gradients of the agents
self.gradients_list = []
# pipes to communicate with the agent processes
self.pipes = []
# Each agent gets an process same order as self.agents
self.threads = []
# list of the weights
self.weights_numpy = []
# Calculate what fraction of the gpu memory each Agnet and Shard gets to use
if flags.gpu_number == 1:
self.fraction = (1 - 0.01) / (self.num_agents + self.shards)
if flags.gpu_number > 1:
half = math.ceil((self.num_agents + self.shards) / flags.gpu_number)
self.fraction = (1 - 0.01) / (half)
if self.distribute:
((ts, tl), (es, el), number_of_labels) = self.data_loader()
self.data = ts
self.labels = tl
self.test_data = es
self.test_labels = el
self.number_of_labels = number_of_labels
#Distributing the data set such that each Agent gets the same number of distinct samples
self.data_list, self.labels_list = self._distribute_data()
else:
self.data_list = [None for i in range(0, self.num_agents)]
self.labels_list = [None for i in range(0, self.num_agents)]
self.test_data=None
self.test_labels=None
# getting the initial weighs
# We use a seperate process here because loading tensorflow in this process sometimes causes problems later on.
(ps_end, agent_end) = Pipe()
if self.distribute:
p = Process(target=create_weights, args= (self.flags, self.agent_function, self.test_data, self.test_labels
, self.flags.batch_size,agent_end))
else:
p = Process(target=create_weights, args=(self.flags, self.agent_function, None,None
, self.flags.batch_size, agent_end))
p.daemon = True
p.start()
agent_end.close()
self.weights_numpy = ps_end.recv()
p.join()
ps_end.close()
# list that stores the portion of the weight each shard gets
self.weights_numpy_list = []
# number of trainable weights each shard gets, the remainder gets distributed to the first few shards
size = len(self.weights_numpy) // self.shards
remainder = len(self.weights_numpy) % self.shards
offset = 0
for i in range(0, self.shards):
if i < remainder:
new_offset = offset + size + 1
else:
new_offset = offset + size
self.weights_numpy_list.append(self.weights_numpy[offset:new_offset])
offset = new_offset
self.pipes_list = []
# Creating the agent processes
for i in range(0, self.num_agents):
pipes_for_agent = []
pipes_for_shars = []
for j in range(0, self.shards):
(ps_end, agent_end) = Pipe()
pipes_for_agent.append(agent_end)
pipes_for_shars.append(ps_end)
self.pipes_list.append(pipes_for_shars)
p = Process(target=Agent, args=(self.flags, self.num_agents, i + 1, self.data_list[i],
self.labels_list[i], self.number_of_labels, self.batch_size,
self.weights_numpy_list, self.total_updates, pipes_for_agent,
self.agent_function,
self.fraction, self.slow_down[i], os.getpid()))
#p.daemon = True, cannot set this because Pytorch uses processes fo its own.
p.start()
agent_end.close()
self.threads.append(p)
self.shards_list = []
self.shard_pipes = []
# running the parameter server shards
for i in range(0,self.shards):
(ps_end, shard_end) = Pipe()
self.shard_pipes.append(ps_end)
pipes = []
for j in range(0,self.num_agents):
pipes.append(self.pipes_list[j][i])
p = Process(target=Shard, args=(self.flags, pipes,self.weights_numpy_list[i] ,i, self.algorithm,
self.update_rule_uninitiated, self.flags.train_set_size, shard_end,
self.fraction, os.getpid()))
p.start()
shard_end.close()
self.shards_list.append(p)
for i in self.pipes_list:
for p in i:
p.close()
# delete the reference to the data such that it doesn't use up memory, is not used here anymore
self.data_list = []
self.labels_list = []
self.data = None
self.csv_name = os.path.sep + self.flags.log_file_names + ".log"
saver = Saver(self.flags.log_dir + self.csv_name)
# Receiving the weights, other global variables and metrics from each shard
# list that stores the weights at the end of each epoch for the test set evaluation
self.test_weights = []
# other globals variables needed for the test set evaluation
self.test_globals = []
print("getting the test weights", flush=True)
batchsize = self.flags.evaluation_batchsize
self.evaluation_steps = self.flags.test_set_size // batchsize
self.eval_obj = self.agent_function(self.flags, self.test_data, self.test_labels, batchsize, 0, 1, True)
log = []
self.weights = []
self.globals = []
self.metrics = []
itr = True
if self.flags.load_save:
self.counter = self.flags.starting_epoch
else:
self.counter = 0
while itr:
self.counter += 1
self.test_globals = [[]] * self.shards
self.test_weights = [[]] * self.shards
c = 0
while c<self.shards:
for i,p in enumerate(self.shard_pipes):
if p.poll():
logger.debug("polled true",flush=True)
tmp = p.recv()
nr = i
c += 1
print("nr", nr, flush=True)
if tmp == 'end':
p.close()
del(self.shard_pipes[i])
self.shut_down_shards()
itr = False
break
weights_list, moving_avg_list = tmp
self.test_globals[nr] = moving_avg_list
self.test_weights[nr] = weights_list
break
else:
#logger.debug("not polled any", flush=True)
time.sleep(0.05)
if itr ==False:
break
if itr:
for p in self.shard_pipes:
p.send("beginning")
self.test_weights_unified = self.test_weights[0]
for i in range(1, self.shards):
self.test_weights_unified += self.test_weights[i]
self.test_averages_unified = self.test_globals[0] # could also take the mean of the list
if self.flags.eval_at_end:
self.weights.append(self.test_weights_unified)
self.globals.append(self.test_averages_unified)
else:
res = self.test_evaluation()
# stroing the weights, globals, and metrics just in case we need them at a later point again
if self.flags.save_weights:
if self.counter > 5:
os.remove(self.flags.saves_dir + os.path.sep + self.flags.dump_file_name + '_' + str(
self.counter - 5) + ".pkl")
f = open(self.flags.saves_dir + os.path.sep + self.flags.dump_file_name + '_' + str(
self.counter) + ".pkl", 'wb')
pickle.dump([self.test_weights, self.test_globals], f)
for p in self.shard_pipes:
p.send("finished")
for p in self.shard_pipes:
train = p.recv()
if self.flags.eval_at_end:
self.metrics.append(train)
else:
loglist = [self.counter]
for t in train:
loglist.append(t)
res = res.tolist()
for r in res:
loglist.append(r)
log.append(loglist)
saver.save_1D(loglist)
logger.results(*loglist)
if self.flags.eval_at_end:
for epoch, weight in enumerate(self.weights):
self.test_weights_unified = weight
self.test_averages_unified = self.globals[epoch]
res = self.test_evaluation()
loglist = [epoch]
for t in self.metrics[epoch]:
loglist.append(t)
res = res.tolist()
for r in res:
loglist.append(r)
log.append(loglist)
saver.save_1D(loglist)
logger.results(*loglist)
for t in self.shards_list:
t.join()
for t in self.threads:
t.join()
self.eval_obj.close()
saver.close()
logger.debug("at the end", flush=True)
self.orderly = True
def get_job_detail(key):
job_dict = {}
req_url = config.INDEED_JOB_SEARCH_URL + key
html = _get_html(req_url)
soup = BeautifulSoup(html, 'html.parser')
# job header 내용 Parsing
job_header = soup.find("div", {"data-tn-component": "jobHeader"})
Logger.debug("request URL =" + req_url)
# print(job_header)
job_title = job_header.find("b", {
"class": "jobtitle"
}).contents[0].get_text()
company = job_header.find("span", {"class": "company"}).get_text()
location = job_header.find("span", {"class": "location"}).get_text()
cmp_description = ""
soup.find("span", {"class": "company"})
# company description이 없는 페이지도 있음
try:
cmp_description = soup.find("div", {
"class": "cmp_description"
}).get_text()
except AttributeError:
Logger.debug("cmp_description does not exists")
job_summary = soup.find("span", {"id": "job_summary"}).get_text()
Logger.debug("jobtitle = " + job_title)
Logger.debug("company = " + company)
Logger.debug("location = " + location)
Logger.debug("cmp_description = " + cmp_description)
Logger.debug("job_summary = " + job_summary)
xml = ET.Element("root")
ET.SubElement(xml, "job_key").text = key
ET.SubElement(xml, "jobtitle").text = job_title
ET.SubElement(xml, "company").text = company
ET.SubElement(xml, "location").text = location
ET.SubElement(xml, "cmp_description").text = cmp_description
ET.SubElement(xml, "job_summary").text = job_summary
Logger.debug(ET.tostring(xml, encoding="utf=8"))
_indent(xml)
return xml
def rand_sleep(sec):
sleep_len = random.uniform(0.1, sec)
Logger.debug("sleep " + str(sleep_len) + "sec")
time.sleep(sleep_len)
def __init__(self, flags, pipes, weights, shard_nr, algorithm,
update_function, dataset_size, pipe, fraction, ps_pid):
"""
Simulates a shard
:param flags: Flags set by the user
:param pipes: Pipes connecting agents and shards (one pipe per agent) len(pipes) == #agents
:param weights: Initial weights for this shard
:param shard_nr: Shard number (starting form 0)
:param algorithm: Implementation of the abstract AlgorithmFunction class.
:param update_function: Implementation of the abstract AgentFunction class
:param dataset_size: Number of samples in the data-set
:param pipe: Pipe to communicate with the ParameterServer
:param fraction: Fraction of the GPU allowed to use
:param ps_pid: PID of the ParameterServer, used to shut down the program in case of an error.
"""
# flag used to figure out if the shard finished without an interruption
self.orderly = False
os.sched_setaffinity(0, list(range(0, flags.threads)))
logger.state("affinity shard", os.sched_getaffinity(0))
self._ps_pid = ps_pid
self.flags = flags
self.pipes = pipes
self.shard_nr = shard_nr
self.num_agents = len(self.pipes) # constraint to be imposed
self.update_function_uninitiated = update_function
self.times = []
self.dataset_size = dataset_size
self.pipe = pipe
self.fraction = fraction
self.timing = self.flags.time_program # The time calls them self not put into an if because it almost safes no
# time if timing is false, and costs if timing True
self.printing = self.flags.printing
self.print_interval = self.flags.print_interval
#period after which bins get averaged
self.bins_period = self.flags.bins_period
#Variables used to calculate the staleness
self.steps_for_learner = [0] * self.num_agents
self.max_staleness = 0
# Logs the training metrics, used to send them back at the end to the PS
self.training_metrics_log = []
# Logs the weights, used to send them back at the end to the PS
self.evaluation_weights = []
self.bins = self.flags.bins
# Putting the weights into shared memory such that the update_function and Communicator can use the same copy
self.weights = []
for i in range(0, self.bins):
self.weights.append(self.create_copy(weights))
self.float_sizes = []
self.total_size = 0
self.shapes = []
for i in self.weights[0]:
self.shapes.append(i.shape)
l = 1
for r in i.shape:
l *= r
self.float_sizes.append(l)
self.total_size += l
self.algorithm = algorithm(self.pipes, self.float_sizes, self.shapes)
self.batch_size = self.flags.batch_size // self.num_agents
# number of iteration for the total batch size (Batch size if there were one agent)
self.iterations = self.flags.total_iterations
# number of iteration we do in total (taking number of agents into account)
self.total_updates = self.iterations * self.num_agents
# how many iterations there are per epoch
if self.flags.drop_remainder:
self.iterations_in_epoch = math.floor(
self.dataset_size / (self.num_agents * self.batch_size))
else:
self.iterations_in_epoch = math.ceil(
self.dataset_size / (self.num_agents * self.batch_size))
logger.state("Shard {0} is doing epochs:{1}".format(
shard_nr, self.iterations // self.iterations_in_epoch))
# here for testing
self.iterations_in_epoch //= 1 # if changed here also has to be changed in the agent
self.csv_name_staleness = os.path.sep + self.flags.staleness_file_name + "_shard" + str(
self.shard_nr) + ".log"
self.saver = Saver(self.flags.log_dir + self.csv_name_staleness)
if not self.flags.load_save:
self.saver.header(
["Agent", "staleness", "phase_staleness", "global_staleness"])
logger.state("Shard {0} set up saving".format(shard_nr))
logger.state("Shard {0} set up the communicator".format(shard_nr))
logger.debug("affinity shard", os.sched_getaffinity(0))
self._run_shard()
self.shut_down()
self.pipe.send("end")
if self.bins > 1:
logger.state("the bins were", self.algorithm.bin_counts)
# self.queue.close()
for p in self.pipes:
p.close()
logger.debug(p.closed)
self.pipe.close()
logger.debug(self.pipe.closed)
self.orderly = True
def stop():
Logger.debug("Cancelling all future events for Phase One.")
scheduler.stop()
def _run_shard(self):
# staleness log, stores the staleness of each update
staleness_log = np.empty((self.iterations * self.num_agents, 4),
np.int64)
update_rule = []
for i in range(0, self.bins):
update_rule.append(
self.update_function_uninitiated(
self.flags, self.num_agents,
(self.shard_nr) % self.flags.gpu_number, self.fraction,
self.weights[i]))
if self.flags.load_save:
grad_update = self.flags.starting_epoch * self.iterations_in_epoch * self.num_agents
epoch = self.flags.starting_epoch + 1 # Start at one, for the update function
validation_checker = 0
else:
grad_update = 0
epoch = 1 # Start at one, for the update function
# second counter that gets reset once we finished an epoch
validation_checker = grad_update
# used to do stuff during the processing of the first update
first = True
# how many updates were done for each bin
bins_update = [0] * self.bins
# flag set if we need to wait for PS
wait_ps = False
# how many metrics have been added up
metrics_count = 0
# number of updates in an epoch
updates_until_validation = self.iterations_in_epoch * self.num_agents
print_abs = updates_until_validation // self.print_interval
if print_abs == 0:
print_abs = 1
start_time = time.time()
while True:
logger.debug("shard", self.shard_nr, " does next iter")
before_weights_time = time.time()
#calling the algorithm to get the gradients
gradients_list, from_list, tag_list, step_list, metrics_list,\
binning, sending = self.algorithm(epoch, grad_update)
logger.debug("shard", self.shard_nr, " after algorithm")
if sending != 1:
got_weights_time = time.time()
if first:
# list for all the moving average variant of the metrics
moving_averages = [0] * len(metrics_list[0])
first = False
logger.debug("shard", self.shard_nr, "before loop")
staleness_list = []
for j, anr in enumerate(from_list):
self.steps_for_learner[anr - 1] += 1
staleness_list.append(grad_update - tag_list[j] + 1)
self.max_staleness = np.max(self.steps_for_learner)
send_back = [agnr - 1 for agnr in from_list]
updates = len(from_list)
for j, s in enumerate(from_list):
staleness_log[
grad_update + j,
3] = self.max_staleness - self.steps_for_learner[s - 1]
staleness_log[grad_update + j, 2] = step_list[j]
staleness_log[grad_update + j, 1] = staleness_list[j] - 1
staleness_log[grad_update + j,
0] = from_list[j] # agent nr
self.saver.save_1D(staleness_log[grad_update + j, :])
logger.debug("shard", self.shard_nr, " after loop")
# Perform the update of the weights, note the weights and update are in shared memory and get updated in-suto
ww = update_rule[binning](self.weights[binning], grad_update,
gradients_list, staleness_list,
epoch)
if not self.flags.eamsgd:
self.weights[binning] = ww
logger.debug("shard", self.shard_nr, " after update rule")
bins_update[binning] += 1
# updates stores how many gradients get used in the same update
grad_update += updates
validation_checker += updates
# calculating moving average of the metrics
for metrics in metrics_list:
for mi, mv in enumerate(metrics):
moving_averages[mi] += mv
metrics_count += updates
#printing the current average train metrics
if self.printing and self.shard_nr == 0 and validation_checker % print_abs == 0:
logger.results(
str(grad_update) + "/" + str(updates_until_validation),
*np.divide(moving_averages, metrics_count),
staleness_list[-1])
if self.flags.bins > 1 and grad_update % self.bins_period == 0:
self.elastic(bins_update)
bins_update = [0] * self.bins
if validation_checker >= updates_until_validation:
weights_copy = self.weights[binning]
self.evaluation_weights.append(weights_copy)
self.training_metrics_log.append(
np.divide(moving_averages, metrics_count))
validation_checker -= updates_until_validation
logger.results(epoch,
*np.divide(moving_averages, metrics_count))
for metrics in metrics_list:
for mi, mv in enumerate(metrics):
moving_averages[mi] = 0
metrics_count = 0
logger.state("Time/updates:",
(time.time() - start_time) / grad_update,
flush=True)
epoch += 1
i = send_back[0]
p = self.pipes[i]
logger.debug("shard", self.shard_nr,
"before sending to Agent 'globals' to", i)
p.send(["globals", weights_copy])
logger.debug("shard", self.shard_nr, "after send")
glob = p.recv()
logger.debug("shard", self.shard_nr, "after recv")
self.pipe.send([weights_copy, glob])
logger.debug("shard", self.shard_nr, "sent to ps")
wait_ps = True
before_comm = time.time()
if sending != 0:
send_back = [agnr - 1 for agnr in from_list]
if self.flags.eamsgd:
logger.state("eamsgd shard", flush=True)
weights_copy = ww
else:
weights_copy = self.weights[binning]
self.send_updates(send_back, "update weights", grad_update,
weights_copy)
if sending != 1:
end_time = time.time()
if self.timing:
self.times.append([
end_time - before_weights_time,
end_time - got_weights_time, end_time - before_comm
])
if wait_ps:
while True:
a = self.pipe.recv()
if a == "beginning":
logger.debug("received beginning")
if a == "finished":
logger.debug("received finished")
self.pipe.send(self.training_metrics_log[-1])
logger.debug("sent training metrics")
wait_ps = False
break
if not (grad_update < self.total_updates):
break
for i in range(0, self.bins):
update_rule[i].close()
if self.timing:
t = np.average(self.times, axis=0)
logger.state("Shard, total time:", t[0],
"without waiting for gradients:", t[1], "comm time: ",
t[2])
def __del__(self):
logger.debug("del Agent")
if not self.orderly:
os.system('pkill -TERM -P ' + str(self._ps_pid))
pass
class Mail(object):
"""model for the Mail."""
id_is_valid = staticmethod(lambda num: 0 < int(num) <= 1L << 31)
def __init__(self, env, id=None, db=None, messageid=None, row=None):
self.env = env
self.db = db
self.log = Logger(env)
if id is not None:
self.resource = Resource('mailarchive', str(id), None)
self._fetch_mail(id)
elif messageid is not None:
self._fetch_mail_by_messageid(messageid)
self.resource = Resource('mailarchive', self.id, None)
elif row is not None:
self._fetch_mail_by_row(row)
self.resource = Resource('mailarchive', self.id, None)
else:
self.messageid = ''
self.subject = ''
self.utcdate = 0
self.localdate = ''
self.zoneoffset = 0
self.body = ''
def __eq__(self, other):
if isinstance(other, Mail):
return self.messageid == other.messageid
return super.__eq__(other)
def _get_db(self):
if self.db:
return self.db
else:
return self.env.get_db_cnx()
def _get_db_for_write(self):
if self.db:
return (self.db, False)
else:
return (self.env.get_db_cnx(), True)
def get_sanitized_fromaddr(self):
return self.fromaddr.replace('@',
self.env.config.get('mailarchive',
'replaceat', '@'))
def get_fromtext(self):
return get_author(self.fromname, self.fromaddr)
def get_category(self):
yearmonth = time.strftime("%Y%m", time.gmtime(self.utcdate))
category = self.mlid + yearmonth
return category.encode('utf-8')
def get_plain_body(self):
return self._sanitize(self.env, self.body)
def get_html_body(self, req):
# for HTML Mail
if self.body.lstrip().startswith('<'):
return Markup(self.body)
contentlines = self.body.splitlines()
htmllines = ['',]
#customize!
#http://d.hatena.ne.jp/ohgui/20090604/1244114483
wikimode = req.args.get('wikimode', 'on')
for line in contentlines:
if self.env.config.get('mailarchive', 'wikiview',' enabled') == 'enabled' and wikimode == 'on':
htmllines.append(wiki_to_oneliner(line, self.env, self.db, False, False, req))
else:
htmllines.append(Markup(Markup().escape(line).replace(' ',' ')))
content = Markup('<br/>').join(htmllines)
return content
def _sanitize(self, env, text):
return text.replace('@', env.config.get('mailarchive', 'replaceat','_at_') )
def _fetch_mail(self, id):
row = None
if self.id_is_valid(id):
db = self._get_db()
cursor = db.cursor()
cursor.execute(SELECT_FROM_MAILARC + " WHERE id=%s", (id,))
row = cursor.fetchone()
if not row:
raise ResourceNotFound('Mail %s does not exist.' % id,
'Invalid Mail Number')
self._fetch_mail_by_row(row)
def _fetch_mail_by_messageid(self, messageid):
row = None
db = self._get_db()
cursor = db.cursor()
cursor.execute(SELECT_FROM_MAILARC + " WHERE messageid=%s",
(messageid,))
row = cursor.fetchone()
if not row:
raise ResourceNotFound('Mail messageid %s does not exist.' % messageid,
'Invalid Mail messageid Number')
self._fetch_mail_by_row(row)
def _fetch_mail_by_row(self, row):
self.id = row[0]
self.messageid = row[1]
self.utcdate = row[2]
self.zoneoffset = row[3]
self.subject = row[4]
self.fromname = row[5]
self.fromaddr = row[6]
self.header =row[7]
self.body = row[8]
self.thread_root = row[9]
self.thread_parent = row[10]
self.zone = self._to_zone(self.zoneoffset)
self.localdate = self._to_localdate(self.utcdate, self.zoneoffset)
def _to_localdate(self, utcdate, zoneoffset):
return time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(utcdate + zoneoffset))
def _to_zone(self, zoneoffset):
#zone and date
zone = ''
if zoneoffset == '':
zoneoffset = 0
if zoneoffset > 0:
zone = ' +' + time.strftime('%H%M', time.gmtime(zoneoffset))
elif zoneoffset < 0:
zone = ' -' + time.strftime('%H%M', time.gmtime(-1 * zoneoffset))
return zone
def get_href(self, req):
return req.href.mailarchive(self.id)
def get_subject(self):
if is_empty(self.subject):
return '(no subject)'
else:
return self.subject
def get_senddate(self):
return self.localdate + self.zone
def get_thread_root(self):
if self.thread_root == '':
return self
try:
root_mail = Mail(self.env, messageid=self.thread_root)
except ResourceNotFound:
return self
#self.thread_rootはオリジナル版だと親のメールになってしまっている。
#互換性維持のため、ルートではない場合は自力で探しにいくロジックを走らす
if root_mail.thread_root == '':
return root_mail
else:
if self.thread_parent != '':
root_id = MailFinder.find_root_id(self.env, self.messageid)
return Mail(self.env, messageid=root_id)
def get_thread_parent_id(self):
if self.thread_parent != '':
return self.thread_parent.split(' ')[0]
return None
def get_thread_parent(self):
if self.thread_parent != '':
return Mail(self.env, db=self.db, messageid=self.get_thread_parent_id())
return self
def get_children(self, desc=False, cached_mails=None):
if cached_mails:
self.log.debug("[%s] mail's threads is cached." % self.id)
return [x for x in cached_mails if x.get_thread_parent_id() == self.messageid]
db = self._get_db()
cursor = db.cursor()
sql = SELECT_FROM_MAILARC + " WHERE threadparent LIKE %s ORDER BY utcdate"
if desc:
sql += " DESC"
cursor.execute(sql, ('%s %%' % self.messageid,))
children = []
for row in cursor:
child_mail = Mail(self.env, row=row, db=self.db)
children.append(child_mail)
return children
def get_thread_mails(self, desc=False):
root = self.get_thread_root()
db = self._get_db()
cursor = db.cursor()
sql = SELECT_FROM_MAILARC + " WHERE threadroot = %s ORDER BY utcdate"
if desc:
sql += " DESC"
cursor.execute(sql, (root.messageid,))
mails = []
for row in cursor:
mails.append(Mail(self.env, row=row, db=self.db))
return mails
def has_children(self, cached_mails=None):
rtn = len(self.get_children(cached_mails=cached_mails)) > 0
return rtn
def get_related_tickets(self, req):
db = self._get_db()
return get_related_tickets(self.env, req, db, self.id)
def has_attachments(self, req):
attachment = MailArchiveAttachment(self.env, self.id)
return attachment.has_attachments(req)
def populate(self, author, msg, mlid):
"""Populate the mail with 'suitable' values from a message"""
if 'message-id' not in msg:
raise 'Illegal Format Mail!'
self.is_new_mail = False
self.mlid = mlid
self._parse_messageid(msg)
self._parse_date(msg)
self._parse_subject(msg)
if msg.is_multipart():
self._parse_multipart(author, msg)
else:
self._parse_body(msg)
ref_messageid = self._parse_reference(msg)
self._make_thread(ref_messageid)
def update_or_save(self):
if self.messageid is None or self.messageid == '':
raise "Can't save mail to database."
db, has_tran = self._get_db_for_write()
cursor = db.cursor()
yearmonth = time.strftime("%Y%m", time.gmtime(self.utcdate))
category = self.mlid + yearmonth
cursor.execute("SELECT category, mlid, yearmonth, count FROM mailarc_category WHERE category=%s",
(category.encode('utf-8'),))
row = cursor.fetchone()
count = 0
if row:
count = row[3]
pass
else:
cursor.execute("INSERT INTO mailarc_category (category, mlid, yearmonth, count) VALUES(%s, %s, %s, %s)",
(category.encode('utf-8'),
self.mlid.encode('utf-8'),
yearmonth,
0))
if self.is_new_mail:
count = count + 1
cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s",
(count, category.encode('utf-8')))
# insert or update mailarc
#self.log.debug(
# "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id),
# category.encode('utf-8'),
# messageid,
# utcdate,
# zoneoffset,
# subject.encode('utf-8'), fromname.encode('utf-8'),
# fromaddr.encode('utf-8'),'','',
# thread_root,thread_parent))
cursor.execute("DELETE FROM mailarc where messageid=%s",
(self.messageid,))
cursor.execute("INSERT INTO mailarc ("
"id, category, messageid, utcdate, zoneoffset, subject,"
"fromname, fromaddr, header, text, threadroot, threadparent) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(str(self.id),
category.encode('utf-8'),
self.messageid,
self.utcdate,
self.zoneoffset,
self.subject.encode('utf-8'), self.fromname.encode('utf-8'),
self.fromaddr.encode('utf-8'), '', self.body.encode('utf-8'),
self.thread_root, self.thread_parent))
if has_tran:
db.commit()
def _parse_messageid(self, msg):
self.messageid = msg['message-id'].strip('<>')
#check messageid is unique
self.log.debug("Creating new mailarc '%s'" % 'mailarc')
db = self._get_db()
cursor = db.cursor()
cursor.execute("SELECT id from mailarc WHERE messageid=%s", (self.messageid,))
row = cursor.fetchone()
id = None
if row:
id = row[0]
if id == None or id == "":
# why? get_last_id return 0 at first.
#id = db.get_last_id(cursor, 'mailarc')
self.is_new_mail = True
cursor.execute("SELECT Max(id)+1 as id from mailarc")
row = cursor.fetchone()
if row and row[0] != None:
id = row[0]
else:
id = 1
self.id = int(id) # Because id might be 'n.0', int() is called.
def _parse_date(self, msg):
if 'date' in msg:
datetuple_tz = email.Utils.parsedate_tz(msg['date'])
localdate = calendar.timegm(datetuple_tz[:9]) #toDB
zoneoffset = datetuple_tz[9] # toDB
utcdate = localdate - zoneoffset # toDB
#make zone ( +HHMM or -HHMM
zone = ''
if zoneoffset > 0:
zone = '+' + time.strftime('%H%M', time.gmtime(zoneoffset))
elif zoneoffset < 0:
zone = '-' + time.strftime('%H%M', time.gmtime(-1 * zoneoffset))
#self.log.debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9]))
self.log.debug(time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(utcdate)))
self.log.debug(time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(localdate)))
self.log.debug(zone)
fromname, fromaddr = email.Utils.parseaddr(msg['from'])
self.fromname = self._decode_to_unicode(fromname)
self.fromaddr = self._decode_to_unicode(fromaddr)
self.zone = zone
self.utcdate = utcdate
self.zoneoffset = zoneoffset
self.localdate = self._to_localdate(utcdate, zoneoffset)
self.log.info(' ' + self.localdate + ' ' + zone +' '+ fromaddr)
def _parse_subject(self, msg):
if 'subject' in msg:
self.subject = self._decode_to_unicode(msg['subject'])
def _parse_reference(self, msg):
# make thread infomations
ref_messageid = ''
if 'in-reply-to' in msg:
ref_messageid = ref_messageid + msg['In-Reply-To'] + ' '
self.log.debug('In-Reply-To:%s' % ref_messageid )
if 'references' in msg:
ref_messageid = ref_messageid + msg['References'] + ' '
m = re.findall(r'<(.+?)>', ref_messageid)
ref_messageid = ''
for text in m:
ref_messageid = ref_messageid + "'%s'," % text
ref_messageid = ref_messageid.strip(',')
self.log.debug('RefMessage-ID:%s' % ref_messageid)
return ref_messageid
def _parse_multipart(self, author, msg):
body = ''
# delete all attachement at message-id
Attachment.delete_all(self.env, 'mailarchive', self.id, self.db)
for part in msg.walk():
content_type = part.get_content_type()
self.log.debug('Content-Type:' + content_type)
file_counter = 1
if content_type == 'multipart/mixed':
pass
elif content_type == 'text/html' and self._is_file(part) == False:
if body != '':
body += "\n------------------------------\n\n"
body = part.get_payload(decode=True)
charset = part.get_content_charset()
self.log.debug('charset:' + str(charset))
# Todo:need try
if charset != None:
body = self._to_unicode(body, charset)
elif content_type == 'text/plain' and self._is_file(part) == False:
#body = part.get_payload(decode=True)
if body != '':
body += "\n------------------------------\n\n"
current_body = part.get_payload(decode=True)
charset = part.get_content_charset()
self.log.debug('charset:' + str(charset))
# Todo:need try
if charset != None:
#body = self._to_unicode(body, charset)
body += self._to_unicode(current_body, charset)
else:
body += current_body
elif part.get_payload(decode=True) == None:
pass
# file attachment
else:
self.log.debug(part.get_content_type())
# get filename
# Applications should really sanitize the given filename so that an
# email message can't be used to overwrite important files
filename = self._get_filename(part)
if not filename:
import mimetypes
ext = mimetypes.guess_extension(part.get_content_type())
if not ext:
# Use a generic bag-of-bits extension
ext = '.bin'
filename = 'part-%03d%s' % (file_counter, ext)
file_counter += 1
self.log.debug("filename:" + filename.encode(OUTPUT_ENCODING))
# make attachment
tmp = os.tmpfile()
tempsize = len(part.get_payload(decode=True))
tmp.write(part.get_payload(decode=True))
tmp.flush()
tmp.seek(0,0)
attachment = Attachment(self.env, 'mailarchive', self.id)
attachment.description = '' # req.args.get('description', '')
attachment.author = author #req.args.get('author', '')
attachment.ipnr = '127.0.0.1'
try:
attachment.insert(filename,
tmp, tempsize, None, self.db)
except Exception, e:
try:
ext = filename.split('.')[-1]
if ext == filename:
ext = '.bin'
else:
ext = '.' + ext
filename = 'part-%03d%s' % (file_counter, ext)
file_counter += 1
attachment.description += ', Original FileName: %s' % filename
attachment.insert(filename,
tmp, tempsize, None, self.db)
self.log.warn('As name is too long, the attached file is renamed : ' + filename)
except Exception, e:
self.log.error('Exception at attach file of Message-ID:' + self.messageid)
traceback.print_exc(e)
tmp.close()
def stop():
Logger.debug("Cancelling all future events for Phase Five.")
def __del__(self):
logger.debug("del Shard", self.shard_nr)
if not self.orderly:
os.system('pkill -TERM -P ' + str(self._ps_pid))