def onReadTimeout(self):
if self.readSomeData:
timeout = SOCKET_READ_TIMEOUT
else:
timeout = SOCKET_INITIAL_READ_TIMEOUT
if clock() < self.lastClock + timeout:
self.readTimeout = eventloop.add_timeout(self.lastClock + timeout - clock(), self.onReadTimeout,
"AsyncSocket.onReadTimeout")
else:
self.readTimeout = None
self.timedOut = True
self.handleEarlyClose('read')
def check_subprocess_hung(self):
task_status = self.responder.movie_data_task_status
logging.warn("check_subprocess_hung: %s (time: %s)", task_status,
clock.clock())
if (task_status is not None and
clock.clock() - task_status.start_time > 90):
logging.warn("Worker process is hanging on a movie data task.")
error_result = TaskResult(task_status.task_id,
SubprocessTimeoutError())
self.responder.handle_task_result(error_result)
self.restart()
else:
self.schedule_check_subprocess_hung()
def onReadTimeout(self):
if self.readSomeData:
timeout = SOCKET_READ_TIMEOUT
else:
timeout = SOCKET_INITIAL_READ_TIMEOUT
if clock() < self.lastClock + timeout:
self.readTimeout = eventloop.add_timeout(
self.lastClock + timeout - clock(), self.onReadTimeout,
"AsyncSocket.onReadTimeout")
else:
self.readTimeout = None
self.timedOut = True
self.handleEarlyClose('read')
def __init__(self, url=None, item=None, restore=None):
self.metainfo = None
self.torrent = None
self.rate = self.eta = 0
self.upRate = self.uploaded = 0
self.activity = None
self.fastResumeData = None
self.retryDC = None
self.channelName = None
self.uploadedStart = 0
self.restarting = False
self.seeders = -1
self.leechers = -1
self.last_fast_resume_update = clock()
self.metainfo_updated = self.fast_resume_data_updated = False
if restore is not None:
self.firstTime = False
self.restore_state(restore)
else:
self.firstTime = True
BGDownloader.__init__(self, url, item)
self.run_downloader()
self.item = item
self._last_reannounce_time = time.time()
def startReadTimeout(self):
if self.disable_read_timeout:
return
self.lastClock = clock()
if self.readTimeout is not None:
return
self.readTimeout = eventloop.add_timeout(SOCKET_INITIAL_READ_TIMEOUT, self.onReadTimeout,
"AsyncSocket.onReadTimeout")
def add_timeout(self, delay, function, name, args=None, kwargs=None):
if args is None:
args = ()
if kwargs is None:
kwargs = {}
scheduled_time = clock() + delay
dc = DelayedCall(function, "timeout (%s)" % (name,), args, kwargs)
heapq.heappush(self.heap, (scheduled_time, dc))
return dc
def startReadTimeout(self):
if self.disable_read_timeout:
return
self.lastClock = clock()
if self.readTimeout is not None:
return
self.readTimeout = eventloop.add_timeout(SOCKET_INITIAL_READ_TIMEOUT,
self.onReadTimeout,
"AsyncSocket.onReadTimeout")
def do_mythtv_update_autodownload(self, line):
"""Update feeds and auto-download"""
logging.info("Starting auto downloader...")
autodler.start_downloader()
feed.expire_items()
starttime = clock()
logging.timing("Icon clear: %.3f", clock() - starttime)
logging.info("Starting video updates")
moviedata.movieDataUpdater.startThread()
parse_command_line_args()
# autoupdate.check_for_updates()
# Wait a bit before starting the downloader daemon. It can cause a bunch
# of disk/CPU load, so try to avoid it slowing other stuff down.
eventloop.addTimeout(5, downloader.startupDownloader,
"start downloader daemon")
# ditto for feed updates
eventloop.addTimeout(30, feed.start_updates, "start feed updates")
# ditto for clearing stale icon cache files, except it's the very lowest
# priority
eventloop.addTimeout(10, iconcache.clear_orphans, "clear orphans")
def do_mythtv_update_autodownload(self, line):
"""Update feeds and auto-download"""
logging.info("Starting auto downloader...")
autodler.start_downloader()
feed.expire_items()
starttime = clock()
logging.timing("Icon clear: %.3f", clock() - starttime)
logging.info("Starting video updates")
moviedata.movieDataUpdater.startThread()
parse_command_line_args()
# autoupdate.check_for_updates()
# Wait a bit before starting the downloader daemon. It can cause a bunch
# of disk/CPU load, so try to avoid it slowing other stuff down.
eventloop.addTimeout(5, downloader.startupDownloader,
"start downloader daemon")
# ditto for feed updates
eventloop.addTimeout(30, feed.start_updates, "start feed updates")
# ditto for clearing stale icon cache files, except it's the very lowest
# priority
eventloop.addTimeout(10, iconcache.clear_orphans, "clear orphans")
def _start_subprocess(self):
cmd_line, env = utils.miro_helper_program_info()
kwargs = {
"stdout": subprocess.PIPE,
"stdin": subprocess.PIPE,
"stderr": open(os.devnull, 'wb'),
"env": env,
"close_fds": True
}
process = Popen(cmd_line, **kwargs)
self.start_time = clock.clock()
return process
def _start_subprocess(self):
cmd_line, env = utils.miro_helper_program_info()
kwargs = {
"stdout": subprocess.PIPE,
"stdin": subprocess.PIPE,
"stderr": open(os.devnull, 'wb'),
"env": env,
"close_fds": True
}
process = Popen(cmd_line, **kwargs)
self.start_time = clock.clock()
return process
def check_subprocess_hung(self):
task_status = self.responder.movie_data_task_status
if (task_status is not None
and clock.clock() - task_status.start_time > 90):
logging.warn("Worker process is hanging on a movie data task.")
error_result = TaskResult(task_status.task_id,
SubprocessTimeoutError())
self.responder.handle_task_result(error_result)
self.restart()
else:
self.schedule_check_subprocess_hung()
def __init__(self, url, dlid):
self.dlid = dlid
self.url = url
self.startTime = clock()
self.endTime = self.startTime
self.shortFilename = filename_from_url(url)
self.pick_initial_filename()
self.state = u"downloading"
self.currentSize = 0
self.totalSize = -1
self.shortReasonFailed = self.reasonFailed = u"No Error"
self.retryTime = None
self.retryCount = -1
def _on_thread_quit(self, thread):
"""Handle our thread exiting."""
# Ignore this call if it was queued from while we were in the middle
# of shutdown().
if not self.is_running:
return
if thread is not self.thread:
# If we have lost the race between the cleanup on shutdown
# it should be safe to ignore.
#
# This can happen when the process does not immediately shut down
# because the worker process is still processing pending jobs
# and the quit message was not processed in time and so the
# subprocess was forcibly terminated. When that happens
# _cleanup_process() is called which resets the thread attribute
# to None immediately, but _on_thread_quit() is only run some
# time after that (when we notice the pipe to the subprocess's
# close we add _on_thread_quit() to the idle loop).
#
# So if the self.thread attribute is None then it means we are done
# and so things are all good.
if self.thread is not None and thread.quit_type != thread.QUIT_NORMAL:
msg = ('_on_thread_quit called by an old thread '
'self.thread: %s thread: %s quit_type: %s' %
(self.thread.name, thread.name, thread.quit_type))
app.controller.failed_soft('handling subprocess', msg)
return
if (self.thread.quit_type == self.thread.QUIT_NORMAL and
self.sent_quit):
self._cleanup_process()
else:
logging.warn("Subprocess quit unexpectedly (quit_type: %s, "
"sent_quit: %s). Will restart subprocess",
self.thread.quit_type, self.sent_quit)
# NOTE: should we enforce some sort of cool-down time before
# restarting the subprocess?
time_since_start = clock.clock() - self.start_time
delay_time = self.restart_delay - time_since_start
if delay_time <= 0:
logging.warn("Subprocess died after %0.1f seconds. "
"Restarting", time_since_start)
self.restart()
else:
logging.warn("Subprocess died in %0.1f seconds, waiting "
"%0.1f to restart", time_since_start, delay_time)
eventloop.add_timeout(delay_time, self.restart,
'restart failed subprocess')
def _on_thread_quit(self, thread):
"""Handle our thread exiting."""
# Ignore this call if it was queued from while we were in the middle
# of shutdown().
if not self.is_running:
return
if thread is not self.thread:
# If we have lost the race between the cleanup on shutdown
# it should be safe to ignore.
#
# This can happen when the process does not immediately shut down
# because the worker process is still processing pending jobs
# and the quit message was not processed in time and so the
# subprocess was forcibly terminated. When that happens
# _cleanup_process() is called which resets the thread attribute
# to None immediately, but _on_thread_quit() is only run some
# time after that (when we notice the pipe to the subprocess's
# close we add _on_thread_quit() to the idle loop).
#
# So if the self.thread attribute is None then it means we are done
# and so things are all good.
if self.thread is not None and thread.quit_type != thread.QUIT_NORMAL:
msg = ('_on_thread_quit called by an old thread '
'self.thread: %s thread: %s quit_type: %s' %
(self.thread.name, thread.name, thread.quit_type))
app.controller.failed_soft('handling subprocess', msg)
return
if (self.thread.quit_type == self.thread.QUIT_NORMAL and
self.sent_quit):
self._cleanup_process()
else:
logging.warn("Subprocess quit unexpectedly (quit_type: %s, "
"sent_quit: %s). Will restart subprocess",
self.thread.quit_type, self.sent_quit)
# NOTE: should we enforce some sort of cool-down time before
# restarting the subprocess?
time_since_start = clock.clock() - self.start_time
delay_time = self.restart_delay - time_since_start
if delay_time <= 0:
logging.warn("Subprocess died after %0.1f seconds. "
"Restarting", time_since_start)
self.restart()
else:
logging.warn("Subprocess died in %0.1f seconds, waiting "
"%0.1f to restart", time_since_start, delay_time)
eventloop.add_timeout(delay_time, self.restart,
'restart failed subprocess')
def update_status(self):
"""
activity -- string specifying what's currently happening or None for
normal operations.
upRate -- upload rate in B/s
downRate -- download rate in B/s
upTotal -- total MB uploaded
downTotal -- total MB downloaded
fractionDone -- what portion of the download is completed.
timeEst -- estimated completion time, in seconds.
totalSize -- total size of the torrent in bytes
"""
status = self.torrent.status()
self.totalSize = status.total_wanted
self.rate = status.download_payload_rate
self.upRate = status.upload_payload_rate
self.uploaded = status.total_payload_upload + self.uploadedStart
self.seeders = status.num_complete
self.leechers = status.num_incomplete
try:
self.eta = ((status.total_wanted - status.total_wanted_done) /
float(status.download_payload_rate))
except ZeroDivisionError:
self.eta = 0
if status.state == lt.torrent_status.states.queued_for_checking:
self.activity = "waiting to check existing files"
elif status.state == lt.torrent_status.states.checking_files:
self.activity = "checking existing files"
elif status.state == lt.torrent_status.states.allocating:
self.activity = "allocating disk space"
else:
self.activity = None
self.currentSize = status.total_wanted_done
if ((self.state == "downloading"
and status.state == lt.torrent_status.states.seeding)):
self.move_to_movies_directory()
self.state = "uploading"
self.endTime = clock()
self.update_client()
else:
DOWNLOAD_UPDATER.queue_update(self)
if app.config.get(prefs.LIMIT_UPLOAD_RATIO):
if status.state == lt.torrent_status.states.seeding:
if ((float(self.uploaded) / self.totalSize >
app.config.get(prefs.UPLOAD_RATIO))):
self.stop_upload()
if self.should_update_fast_resume_data():
self.update_fast_resume_data()
def time_trap_call(when, function, *args, **kwargs):
global cancel
cancel = False
start = clock()
retval = trap_call(when, function, *args, **kwargs)
end = clock()
if cancel:
return retval
if end-start > 1.0:
logging.timing("WARNING: %s too slow (%.3f secs)", when, end-start)
if TRACK_CUMULATIVE:
try:
total = cumulative[when]
except KeyError:
total = 0
total += end - start
cumulative[when] = total
if total > 5.0:
logging.timing("%s cumulative is too slow (%.3f secs)", when, total)
cumulative[when] = 0
return retval
cancel = True
return retval
def dispatch(self):
success = True
if not self.canceled:
when = "While handling %s" % self.name
start = clock()
success = trapcall.trap_call(when, self.function, *self.args,
**self.kwargs)
end = clock()
if end-start > 0.5:
logging.timing("%s too slow (%.3f secs)",
self.name, end-start)
try:
total = cumulative[self.name]
except (KeyError, AttributeError):
total = 0
total += end - start
cumulative[self.name] = total
if total > 5.0:
logging.timing("%s cumulative is too slow (%.3f secs)",
self.name, total)
cumulative[self.name] = 0
self._unlink()
return success
def time_trap_call(when, function, *args, **kwargs):
global cancel
cancel = False
start = clock()
retval = trap_call(when, function, *args, **kwargs)
end = clock()
if cancel:
return retval
if end - start > 1.0:
logging.timing("WARNING: %s too slow (%.3f secs)", when, end - start)
if TRACK_CUMULATIVE:
try:
total = cumulative[when]
except KeyError:
total = 0
total += end - start
cumulative[when] = total
if total > 5.0:
logging.timing("%s cumulative is too slow (%.3f secs)", when,
total)
cumulative[when] = 0
return retval
cancel = True
return retval
def on_download_finished(self, response):
self.destroy_client()
self.state = "finished"
self.endTime = clock()
# bug 14131 -- if there's nothing here, treat it like a temporary
# error
if self.currentSize == 0:
self.handle_network_error(httpclient.PossiblyTemporaryError(_("no content")))
else:
if self.totalSize == -1:
self.totalSize = self.currentSize
try:
self.move_to_movies_directory()
except IOError, e:
self.handle_write_error(e)
def __init__(self):
self.start_time = self.last_time = clock()
def log_total_time(self):
logging.timing("total time: %0.3f", clock() - self.start_time)
def log_time(self, msg):
current_time = clock()
logging.timing("%s: %0.4f", msg, current_time - self.last_time)
self.last_time = current_time
def __init__(self):
self.start_time = self.last_time = clock()
def handle_movie_data_task_status(self, msg):
if msg.task_id is not None:
self.movie_data_task_status = MovieDataTaskStatusInfo(
msg.task_id, clock.clock())
else:
self.movie_data_task_status = None
def log_time(self, msg):
current_time = clock()
logging.timing("%s: %0.4f", msg, current_time - self.last_time)
self.last_time = current_time
def has_pending_timeout(self):
return len(self.heap) > 0 and self.heap[0][0] < clock()
def next_timeout(self):
if len(self.heap) == 0:
return None
else:
return max(0, self.heap[0][0] - clock())
def log_total_time(self):
logging.timing("total time: %0.3f", clock() - self.start_time)
def should_update_fast_resume_data(self):
return (clock() - self.last_fast_resume_update >
self.FAST_RESUME_UPDATE_INTERVAL)
def update_fast_resume_data(self):
self.last_fast_resume_update = clock()
self.fastResumeData = lt.bencode(self.torrent.write_resume_data())
self.fast_resume_data_updated = True
def handle_movie_data_task_status(self, msg):
if msg.task_id is not None:
self.movie_data_task_status = MovieDataTaskStatusInfo(
msg.task_id, clock.clock())
else:
self.movie_data_task_status = None