def set(self, content):
self.save_targets.clear()
content_atoms = {}
for target, data in content.items():
if not isinstance(data, (str, ByteString, type(None))):
raise TypeError('Unsupported data type:\n{}'.format(
repr(data)))
with ThreadingTimeout(0.05) as timeout:
try:
target_atom = self.display.intern_atom(target)
except RuntimeError as e:
ErrorReporter.print(e)
raise BrokenConnection('Failed to intern atom') from e
if timeout.state == timeout.TIMED_OUT:
raise BrokenConnection('Interning atoms timed out')
if data:
self.save_targets.append(target_atom)
content_atoms[target_atom] = data
if self.content_set:
with ThreadingTimeout(0.05) as timeout:
self.window.send_event(self.selection_clear,
onerror=errHandler)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
raise BrokenConnection('Failed to flush events') from e
if errHandler.get_error():
raise BrokenConnection('Failed to send events')
if timeout.state == timeout.TIMED_OUT:
raise BrokenConnection('Sending event timed out')
self.outbox.put_nowait(content_atoms)
def _load_image_sequence(self, segment: AbstractSegment) -> torch.Tensor:
cache_directory = self.dataset_directory + "/segment_image_tensor_cache"
self._create_cache_dir(cache_directory)
try:
with ThreadingTimeout(2.0) as timeout_ctx1:
images = torch.load("{}/{}.pkl".format(cache_directory, segment.__hash__()))
if not bool(timeout_ctx1):
CometLogger.print('Took too long when loading a cache image. '
'We will load the image directly form the dataset instead.')
raise Exception()
except:
image_sequence = []
with ThreadingTimeout(3600.0) as timeout_ctx2:
for img_as_img in segment.get_images():
img_as_tensor = self.transformer(img_as_img)
if self.minus_point_5:
img_as_tensor = img_as_tensor - 0.5 # from [0, 1] -> [-0.5, 0.5]
img_as_tensor = self.normalizer(img_as_tensor)
img_as_tensor = img_as_tensor.unsqueeze(0)
image_sequence.append(img_as_tensor)
images = torch.cat(image_sequence, 0)
if not bool(timeout_ctx2):
CometLogger.fatalprint('Encountered fatal delay when reading the uncached images from the dataset')
free = -1
try:
with ThreadingTimeout(2.0) as timeout_ctx3:
_, _, free = shutil.disk_usage(cache_directory)
if not bool(timeout_ctx3):
CometLogger.print('Took too long to measure disk space. Skipping caching.')
except Exception as e:
print("Warning: unable to cache the segment's image tensor, there was an error while getting "
"disk usage: {}".format(e), file=sys.stderr)
if free == -1:
pass
elif free // (2**30) > 1:
try:
with ThreadingTimeout(5.0) as timeout_ctx4:
torch.save(images, "{}/{}.pkl".format(cache_directory, segment.__hash__()))
if not bool(timeout_ctx4):
CometLogger.print('Took too long when saving to cache folder. Deadlock possible. Skipping caching.')
except Exception as e:
print("Warning: unable to cache the segment's image tensor: {}".format(e), file=sys.stderr)
else:
pass
if self.augment_dataset:
images = self._augment_image_sequence(images)
return images
def base_schedule(timeout, runners, controllers):
with ThreadingTimeout(timeout) as ctx_timeout:
cntrl = controllers[0]
runner = runners[0]
runner.push_task(cntrl.get, [None], 3)
res = runner.schedule_tasks(3, timeout)
return ctx_timeout, res
def __getitem__(self, item: int):
with ThreadingTimeout(3600.0) as timeout_ctx1:
try:
segment, image_sequence = super().__getitem__(item)
except Exception as e:
CometLogger.print(str(e))
raise e
if not bool(timeout_ctx1):
CometLogger.fatalprint(
'Encountered fatal delay while getting the image sequence')
with ThreadingTimeout(3600.0) as timeout_ctx2:
pose = self._get_segment_pose(segment)
if not bool(timeout_ctx2):
CometLogger.fatalprint(
'Encountered fatal delay while getting the pose of the sequence'
)
return image_sequence, pose
def _worker(self, queue):
while not queue.empty():
channel, kwargs = queue.get()
self._logger.debug(
('Sending notification using channel: {channel} '
'with args:{args}').format(channel=channel.name, args=kwargs))
try:
with ThreadingTimeout(TIMEOUT):
result = channel.notify(**kwargs)
self._results.append({channel.name: result})
except TimeoutException:
self._logger.error(('The worker reached the time limit '
'({} secs)').format(TIMEOUT))
except Exception:
self._logger.exception(('Exception caught on sending on '
'channel:{}').format(channel.name))
queue.task_done()
def handle_get(timeout, runners, controllers):
with ThreadingTimeout(timeout) as ctx_timeout:
cntrl_first = controllers[0]
cntrl_subsequent = controllers[1]
runner = runners[0]
runner.push_task(cntrl_first.get, [None], 1)
try:
res = runner.schedule_tasks(1, 0.3)
except TimeoutException as e:
log.info(e)
runner.push_task(cntrl_subsequent.get, [None], 2)
res = runner.schedule_tasks(2, timeout)
if ctx_timeout.state == ctx_timeout.TIMED_OUT:
return timeout_error
elif ctx_timeout.state == ctx_timeout.EXECUTED:
if not res:
return timeout_error
return res
def get_sympathy_urls_from_post(self, blog_url):
try:
with ThreadingTimeout(5.0) as timeout_ctx:
assert timeout_ctx.state == timeout_ctx.EXECUTING
sympathy_urls = naver_crawler.get_sympathy_urls(driver=self.driver, blog_url=blog_url)
# except ThreadingTimeout:
# print('ThreadingTimeout')
# raise ThreadingTimeout
except AttributeError:
print('AttributeError')
raise AttributeError
except NoSuchElementException:
print('NoSuchElementException')
raise NoSuchElementException
except BaseException:
print('BaseException')
raise BaseException
return sympathy_urls
def crawl_wrapper(self, fuzzable_request):
"""
Wrapper around the crawl method in order to perform some generic tasks.
"""
# I copy the fuzzable request, to avoid cross plugin contamination
# in other words, if one plugin modified the fuzzable request object
# INSIDE that plugin, I don't want the next plugin to suffer from that
fuzzable_request_copy = safe_deepcopy(fuzzable_request)
# Crawl with timeout
try:
with ThreadingTimeout(self.PLUGIN_TIMEOUT, swallow_exc=False):
return self.crawl(fuzzable_request_copy)
except TimeoutException:
msg = '[timeout] The "%s" plugin took more than %s seconds to'\
' complete the crawling of "%s", killing it!'
om.out.debug(msg % (self.get_name(), self.PLUGIN_TIMEOUT,
fuzzable_request.get_url()))
def schedule_tasks(self, task_count, timeout):
with ThreadingTimeout(timeout, swallow_exc=False) as func_timeout:
log.info(f"Smart runner task timeout status {func_timeout.state}")
futures = {}
with ThreadPoolExecutor() as executor:
for item in itertools.islice(iter(self.tasks), task_count):
futures.update(
{executor.submit(item["task"]): item["task"]})
self.tasks = []
log.info(futures)
while futures:
log.info(len(futures) == 2)
done, running = wait(futures,
timeout=timeout,
return_when=FIRST_COMPLETED)
for fut in done:
futures.pop(fut)
if type(fut.result()) == dict:
executor.shutdown(wait=False)
return fut.result()
def crawl_wrapper(self, fuzzable_request):
"""
Wrapper around the crawl method in order to perform some generic tasks.
"""
# I copy the fuzzable request, to avoid cross plugin contamination
# in other words, if one plugin modified the fuzzable request object
# INSIDE that plugin, I don't want the next plugin to suffer from that
fuzzable_request_copy = safe_deepcopy(fuzzable_request)
# Crawl with timeout
try:
with ThreadingTimeout(self.PLUGIN_TIMEOUT, swallow_exc=False):
return self.crawl(fuzzable_request_copy)
except FourOhFourDetectionException, ffde:
# We simply ignore any exceptions we find during the 404 detection
# process. FYI: This doesn't break the xurllib error handling which
# happens at lower layers.
#
# https://github.com/andresriancho/w3af/issues/8949
om.out.debug('%s' % ffde)
def audit_with_copy(self, fuzzable_request, orig_resp):
"""
:param freq: A FuzzableRequest
:param orig_resp: The HTTP response we get from sending the freq
Copy the FuzzableRequest before auditing.
I copy the fuzzable request, to avoid cross plugin contamination.
In other words, if one plugins modified the fuzzable request object
INSIDE that plugin, I don't want the next plugin to suffer from that.
"""
fuzzable_request = copy.deepcopy(fuzzable_request)
try:
with ThreadingTimeout(self.PLUGIN_TIMEOUT, swallow_exc=False):
return self.audit(fuzzable_request, orig_resp)
except TimeoutException:
msg = '[timeout] The "%s" plugin took more than %s seconds to'\
' complete the analysis of "%s", killing it!'
om.out.debug(msg % (self.get_name(), self.PLUGIN_TIMEOUT,
fuzzable_request.get_url()))
def exposed_transaction_request(self, transaction):
transaction = obtain(transaction)
print("DB-%s: %s requested" % (str(Database().id), transaction))
if self.check_valid(transaction):
r = Database().pending_transactions
r[transaction] = {}
else:
return False
threads = []
for addr, port in Database().registrar.discover("DB"):
if port != Database().port:
r[transaction][addr, port] = False
threads.append(
Timer(random.uniform(1., 5.),
self.can_commit,
args=((addr, port), transaction)))
threads[-1].start()
# bool(timeout_ctx) is True if execution completed normally and False if timed out
with ThreadingTimeout(5.) as timeout_ctx:
for t in threads:
t.join()
if timeout_ctx and all(r[transaction].values()):
for addr, port in r[transaction]:
conn = rpyc.connect(addr, port)
conn.root.complete_transaction(transaction)
else:
self.exposed_complete_transaction(transaction)
return True
else:
for addr, port in r[transaction]:
conn = rpyc.connect(addr, port)
conn.root.rollback_transaction(transaction)
else:
self.exposed_rollback_transaction(transaction)
return False
def get_id_from_blog(self, blog_url, original_poster=True):
# 네이버 모듈에서 개별 블로그에서 유저 아이디 찾기 호출
try:
with ThreadingTimeout(5.0) as timeout_ctx:
assert timeout_ctx.state == timeout_ctx.EXECUTING
user_id = naver_crawler.get_naver_blog_id(driver=self.driver, blog_url=blog_url)
# except ThreadingTimeout:
# print('ThreadingTimeout')
# raise ThreadingTimeout
except AttributeError:
print('AttributeError')
raise AttributeError
except NoSuchElementException:
print('NoSuchElementException')
raise NoSuchElementException
except BaseException:
print('BaseException')
raise BaseException
return {
'user_id': user_id,
'original_poster': original_poster
}
def audit_with_copy(self, fuzzable_request, orig_resp):
"""
:param freq: A FuzzableRequest
:param orig_resp: The HTTP response we get from sending the freq
Copy the FuzzableRequest before auditing.
I copy the fuzzable request, to avoid cross plugin contamination.
In other words, if one plugins modified the fuzzable request object
INSIDE that plugin, I don't want the next plugin to suffer from that.
"""
fuzzable_request = safe_deepcopy(fuzzable_request)
try:
with ThreadingTimeout(self.PLUGIN_TIMEOUT, swallow_exc=False):
return self.audit(fuzzable_request, orig_resp)
except FourOhFourDetectionException, ffde:
# We simply ignore any exceptions we find during the 404 detection
# process. FYI: This doesn't break the xurllib error handling which
# happens at lower layers.
#
# https://github.com/andresriancho/w3af/issues/8949
om.out.debug('%s' % ffde)
def parse(self, response):
# 키워드 입력 시 블로그 글 검색 결과 (목록)
BLOG_ITEM_SELECTOR = '#wrap #container .pack_group #main_pack .blog ul li dl dt'
self.driver.get(response.url)
resp = TextResponse(url='', body=self.driver.page_source, encoding='utf-8')
html = self.driver.page_source
soup = BeautifulSoup(html, 'html.parser')
next_page = soup.find('a', class_='next')
print('next page: ' + next_page.text.strip() + ", link: " + next_page['href'])
# 개별 블로그 글
for blog_item in resp.css(BLOG_ITEM_SELECTOR):
# 목표한 아이디 개수가 채워졌으면 크롤러 종료
if self.close_down:
raise CloseSpider(reason='목표 아이디 개수 달성')
TITLE_SELECTOR = 'a::attr(title)'
URL_SELECTOR = 'a::attr(href)'
title = blog_item.css(TITLE_SELECTOR).get()
blog_url = blog_item.css(URL_SELECTOR).get()
print("URL: " + blog_url)
# 개별 블로그에서 아이디 추출, 글 게시자=True
try:
with ThreadingTimeout(5.0) as timeout_ctx:
assert timeout_ctx.state == timeout_ctx.EXECUTING
yield self.get_id_from_blog(blog_url=blog_url, original_poster=True)
# except ThreadingTimeout:
# print('ThreadingTimeout')
except AttributeError:
print('AttributeError')
except NoSuchElementException:
print('NoSuchElementException')
except BaseException:
print('BaseException')
commenter_urls = []
# 댓글 작성자 블로그 URL 추출
try:
with ThreadingTimeout(5.0) as timeout_ctx:
assert timeout_ctx.state == timeout_ctx.EXECUTING
commenter_urls = self.get_commenter_urls_from_post(blog_url=blog_url)
# except ThreadingTimeout:
# print('ThreadingTimeout')
except AttributeError:
print('AttributeError')
except NoSuchElementException:
print('NoSuchElementException')
except BaseException:
print('BaseException')
print("comment urls: " + str(commenter_urls))
sympathy_urls = []
# 공감 누른 사람 블로그 URL 추출
try:
with ThreadingTimeout(5.0) as timeout_ctx:
assert timeout_ctx.state == timeout_ctx.EXECUTING
sympathy_urls = self.get_sympathy_urls_from_post(blog_url=blog_url)
# except ThreadingTimeout:
# print('ThreadingTimeout')
except AttributeError:
print('AttributeError')
except NoSuchElementException:
print('NoSuchElementException')
except BaseException:
print('BaseException')
print("sympathy urls: " + str(sympathy_urls))
# 개별 댓글 작성자마다 블로그로 들어가서 아이디 추출
for commenter_url in commenter_urls:
# 목표한 아이디 개수가 채워졌으면 크롤러 종료
if self.close_down:
raise CloseSpider(reason='목표 아이디 개수 달성')
# 개별 블로그에서 아이디 추출, 글 게시자=False (댓글 작성자이기 때문)
try:
with ThreadingTimeout(5.0) as timeout_ctx:
assert timeout_ctx.state == timeout_ctx.EXECUTING
yield self.get_id_from_blog(blog_url=commenter_url, original_poster=False)
# except ThreadingTimeout:
# print('ThreadingTimeout')
# continue
except AttributeError:
print('AttributeError')
continue
except NoSuchElementException:
print('NoSuchElementException')
continue
except BaseException:
print('BaseException')
continue
# 개별 공감 누른 사람마다 블로그로 들어가서 아이디 추출
for sympathy_url in sympathy_urls:
# 목표한 아이디 개수가 채워졌으면 크롤러 종료
if self.close_down:
raise CloseSpider(reason='목표 아이디 개수 달성')
# 개별 블로그에서 아이디 추출, 글 게시자=False (댓글 작성자이기 때문)
try:
with ThreadingTimeout(5.0) as timeout_ctx:
assert timeout_ctx.state == timeout_ctx.EXECUTING
yield self.get_id_from_blog(blog_url=sympathy_url, original_poster=False)
# except ThreadingTimeout:
# print('ThreadingTimeout')
# continue
except AttributeError:
print('AttributeError')
continue
except NoSuchElementException:
print('NoSuchElementException')
continue
except BaseException:
print('BaseException')
continue
# 크롤링 할 블로그 검색 결과 페이지 수
# if self.page_count < 1:
if True:
self.page_count += 1
yield scrapy.Request(
urlparse.urljoin(self.url_head, next_page['href']),
callback=self.parse
)
def _train(self) -> tuple:
timer_start_time = time.time()
self.model.train()
losses_sum = 0
benchmark_losses_sum = 0
for i, (input, target) in enumerate(self.train_dataloader):
CometLogger.get_experiment().log_metric("Current batch", i + 1)
CometLogger.get_experiment().log_metric("Total nbr of batches",
len(self.train_dataloader))
# Only log this if we are NOT in a multiprocessing session
if CometLogger.gpu_id is None:
print("--> processing batch {}/{} of size {}".format(
i + 1, len(self.train_dataloader), len(input)))
if cuda_is_available():
with ThreadingTimeout(14400.0) as timeout_ctx1:
input = input.cuda(
non_blocking=self.train_dataloader.pin_memory)
target = target.cuda(
non_blocking=self.train_dataloader.pin_memory)
if not bool(timeout_ctx1):
CometLogger.fatalprint(
'Encountered fatally long delay when moving tensors to GPUs'
)
prediction = self.model.forward(input)
with ThreadingTimeout(14400.0) as timeout_ctx3:
if type(prediction) is tuple:
benchmark_loss = self.benchmark_MSE_loss.compute(
prediction[0], target)
else:
benchmark_loss = self.benchmark_MSE_loss.compute(
prediction, target)
if not bool(timeout_ctx3):
CometLogger.fatalprint(
'Encountered fatally long delay during computation of benchmark loss'
)
with ThreadingTimeout(14400.0) as timeout_ctx4:
benchmark_losses_sum += float(
benchmark_loss.data.cpu().numpy())
if not bool(timeout_ctx4):
CometLogger.fatalprint(
'Encountered fatally long delay during summation of benchmark losses'
)
with ThreadingTimeout(14400.0) as timeout_ctx4:
loss = self.custom_loss.compute(prediction, target)
if not bool(timeout_ctx4):
CometLogger.fatalprint(
'Encountered fatally long delay during computation of the custom loss'
)
self._backpropagate(loss)
with ThreadingTimeout(14400.0) as timeout_ctx6:
losses_sum += float(loss.data.cpu().numpy())
if not bool(timeout_ctx6):
CometLogger.fatalprint(
'Encountered fatally long delay during loss addition')
timer_end_time = time.time()
CometLogger.get_experiment().log_metric(
"Epoch training time", timer_end_time - timer_start_time)
return losses_sum / len(
self.train_dataloader), benchmark_losses_sum / len(
self.train_dataloader)
def get_logger(setting_getter, name, fail_to_local=False, filter=None):
global got_logger
if got_logger:
# Yes, this means that if you try to change your logging within an
# application, it won't work. This is intentional. You shouldn't do
# that.
return got_logger
if filter:
def log_filter(r, h):
if server_pipe_log_filter_re.search(r.message):
return False
return filter(r, h)
else:
def log_filter(r, h):
return not server_pipe_log_filter_re.search(r.message)
logger_name = 'penguindome-' + name
file_safe_logger_name = logger_name.replace(os.sep, '_')
logger = logbook.Logger(logger_name)
internal_log_dir = os.path.join(var_dir, 'log')
internal_log_file = os.path.join(internal_log_dir,
file_safe_logger_name + '.log')
os.makedirs(internal_log_dir, 0o0700, exist_ok=True)
# We always do local debug logging, regardless of whether we're also
# logging elsewhere.
logbook.RotatingFileHandler(internal_log_file,
bubble=True,
filter=log_filter).push_application()
handler_name = setting_getter('logging:handler')
if handler_name:
handler_name = handler_name.lower()
handler_name += 'handler'
handler_name = next(d for d in dir(logbook)
if d.lower() == handler_name)
handler = logbook.__dict__[handler_name]
kwargs = {'bubble': True, 'filter': log_filter}
level = setting_getter('logging:level')
kwargs['level'] = logbook.__dict__[level.upper()]
if handler_name == 'SyslogHandler':
kwargs['facility'] = setting_getter('logging:syslog:facility')
hostname = setting_getter('logging:syslog:host')
if hostname:
port = setting_getter('logging:syslog:port')
try:
addrinfo = socket.getaddrinfo(hostname, port,
socket.AF_INET,
socket.SOCK_STREAM)[0]
except:
if not fail_to_local:
raise
logger.warn(
'Failed to resolve {}:{}, falling back to '
'local-only logging', hostname, port)
handler = None
else:
kwargs['socktype'] = addrinfo[1]
kwargs['address'] = addrinfo[4]
if handler:
if fail_to_local:
try:
with ThreadingTimeout(5, swallow_exc=False):
handler = handler(**kwargs)
except:
logger.warn(
'Failed to create {}, falling back to '
'local-only logging', handler_name)
else:
handler.push_application()
else:
handler(**kwargs).push_application()
logbook.compat.redirect_logging()
got_logger = logger
return got_logger