def songdbMain(): # Import modules, which will have the side-effect to # init/load the songdb (lazily) and register the GUI. import songdb import Search # This is heavy, ugly, etc... # But it's simple nice hack for now to index everything. import TaskSystem def indexAll(): import appinfo for dir in appinfo.musicdirs: TaskSystem.asyncCall(lambda: songdb.indexSearchDir(dir), name="create search index", mustExec=True) TaskSystem.daemonThreadCall(indexAll, name="create search index") # Reindex played songs. from State import state from Player import PlayerEventCallbacks for ev,args,kwargs in state.updates.read(): try: if ev is PlayerEventCallbacks.onSongChange: newSong = kwargs["newSong"] songdb.insertSearchEntry(newSong) except Exception: import sys sys.excepthook(*sys.exc_info()) songdb.flush()
def main_entry():
import TaskSystem
import Logging
try:
TaskSystem.main_loop()
except KeyboardInterrupt:
Logging.log("KeyboardInterrupt")
def _startSearch(self, txt): def search(): with self._lock: if self._searchText != txt: return res = songdb.search(txt) with self._lock: if self._searchText == txt: self._searchResults = res self.__class__.searchResults.updateEvent(self).push() with self._lock: self._searchText = txt TaskSystem.daemonThreadCall(search, name="Song DB search")
def _startSearch(self, txt):
def search():
with self._lock:
if self._searchText != txt: return
res = songdb.search(txt)
with self._lock:
if self._searchText == txt:
self._searchResults = res
self.__class__.searchResults.updateEvent(self).push()
with self._lock:
self._searchText = txt
TaskSystem.daemonThreadCall(search, name="Song DB search")
def __set__(self, inst, value): if inst is None: # access through class self.value = value return if hasattr(self.value, "__set__"): self.value.__set__(inst, value) else: self.set(inst, value) if self.hasUpdateEvent(): # Do it in a separate thread because we don't expect that some __set__ # could perform badly or even result in some recursive call. import TaskSystem TaskSystem.daemonThreadCall(self.updateEvent, args=(inst,), name="%r update event callback" % self)
def __set__(self, inst, value): if inst is None: # access through class self.value = value return if hasattr(self.value, "__set__"): self.value.__set__(inst, value) else: self.set(inst, value) if self.hasUpdateEvent(): # Do it in a separate thread because we don't expect that some __set__ # could perform badly or even result in some recursive call. import TaskSystem TaskSystem.daemonThreadCall(self.updateEvent, args=(inst,), name="%r update event callback" % self)
def __call__(self):
import main
if not main.allowed_by_blacklist(self.url):
return
try:
self.downloader.run()
except Downloader.DownloadTemporaryError as exc:
print("%s: %s %s" % (self, type(exc).__name__, exc))
# Retry later.
# However, also queue some random action to allow other downloads.
TaskSystem.queue_work(RandomNextFile())
TaskSystem.queue_work(self)
except Downloader.DownloadFatalError as exc:
print("%s: %s %s" % (self, type(exc).__name__, exc))
def __call__(self):
import main
if not main.allowed_by_blacklist(self.url):
return
try:
self.downloader.run()
except Downloader.DownloadTemporaryError:
# Retry later.
# However, also queue some random action to allow other downloads.
TaskSystem.queue_work(RandomNextFile())
TaskSystem.queue_work(self)
except Downloader.DownloadFatalError:
# Cannot handle. Nothing we can do.
pass
def __call__(self):
import main
if not main.DownloadOnly:
# This can happen if we saved this action at an earlier run
# where we had the option enabled.
# Just ignore it now.
return
import TaskSystem
import Threading
# Check if there are no more downloads running.
if len(TaskSystem.currentWorkSet) <= 1: # should only be ourselves
# Exit.
Logging.log("All downloads finished.")
Threading.do_in_main_thread(IssueSystemExit(), wait=False)
else:
# Check again later.
TaskSystem.queue_work(CheckDownloadsFinished())
def setup(*raw_arg_list):
print("RandomFtpGrabber startup.")
import better_exchook
better_exchook.install()
import Logging
better_exchook.output = Logging.log
arg_parser = ArgumentParser()
arg_parser.add_argument("--dir", default=os.getcwd())
arg_parser.add_argument("--numWorkers", type=int)
arg_parser.add_argument("--shell", action="store_true")
arg_parser.add_argument("--skipShodan", action="store_true")
arg_parser.add_argument("--downloadRemaining", action="store_true")
global Args
Args = arg_parser.parse_args(raw_arg_list)
if sys.version_info.major != 3:
Logging.log("Warning: This code was only tested with Python3.")
import time
time.sleep(10) # wait a bit to make sure the user sees this
start_stdin_handler_loop()
import main
main.RootDir = Args.dir
Logging.log("root dir: %s" % RootDir)
main.SkipShodan = Args.skipShodan
if not main.SkipShodan:
prepare_lists()
main.DownloadOnly = Args.downloadRemaining
if not main.DownloadOnly:
setup_lists()
import TaskSystem # important to be initially imported in the main thread
if Args.numWorkers:
TaskSystem.kNumWorkers = Args.numWorkers
TaskSystem.kMinQueuedActions = Args.numWorkers
TaskSystem.kSuggestedMaxQueuedActions = Args.numWorkers * 2
if Args.shell:
TaskSystem.kNumWorkers = 0
TaskSystem.setup()
def __call__(self):
walker = get_random_walker(self.base)
try:
# Either throws TemporaryException or returns None if empty.
url = walker.get_next_file()
except FileSysIntf.TemporaryException as exc:
# Handle another one later.
# Will automatically be added.
Logging.log("%s: TemporaryException:" % self, exc)
return
if not url:
# Can happen if we end up in an empty source or directory.
Logging.log("%s: no file found" % self)
Index.index.remove_source(self.base)
return
if TaskSystem.reached_suggested_max_queue():
# Better go exploring a bit more, and handle the current queue first.
Logging.log("%s: reached suggested max queue, will not queue download" % self)
return
TaskSystem.queue_work(Download(url))
def __init__(
self,
filename,
create_command="create table %s(key blob primary key unique, value blob)"
):
self.rwlock = TaskSystem.ReadWriteLock()
import threading
# We need a workaround wrapper for SQLite connection objects
# because Python might crash in their tp_dealloc.
# See <http://bugs.python.org/issue17263> for details.
class LocalConnection:
refs = set()
lock = threading.RLock()
def __init__(self, conn):
from weakref import ref
self.conn = conn
with self.lock:
self.refs.add(ref(self))
assert ref(self) in self.refs
def get(self):
with self.lock:
return self.conn
def reset(self):
from weakref import ref
with self.lock:
self.refs.discard(ref(self))
self.conn = None
def __del__(self):
self.reset()
@classmethod
def Reset(clazz):
with clazz.lock:
for l in list(clazz.refs):
l = l()
if not l: continue
l.reset()
self.LocalConnection = LocalConnection
self.filename = filename
if filename[0:1] == ":": # e.g. ":memory:"
self.path = filename
else:
self.path = appinfo.userdir + "/" + filename
self.create_command = create_command
self.cache = Cache()
try:
self.sanityCheck()
except Exception as exc:
# Maybe we had an old LevelDB or some other corruption.
# Not much we can do for recovering...
print("DB %s opening error %r, I will reset the DB, sorry..." %
(self.filename, exc))
self._removeOldDb()
self._initNew()
def indexAll(): import appinfo for dir in appinfo.musicdirs: TaskSystem.asyncCall(lambda: songdb.indexSearchDir(dir), name="create search index", mustExec=True)
import TaskSystem as TS
import random as rd
class Task:
def __init__(self):
self.Result = []
self.Arg = []
def DoTask(self, AuxInfo):
pass
class CalTask(Task):
def __init__(self, a, b):
super().__init__()
self.Arg = [a, b]
def DoTask(self, AuxInfo):
self.Result.append(self.Arg[0] + self.Arg[1])
print("Cal:" + str(self.Arg[0]) + "+" + str(self.Arg[1]) + "|" +
AuxInfo)
pairs = [[i, j] for i in range(10) for j in range(10)]
CalTasks = [CalTask(p[0], p[1]) for p in pairs]
for t in CalTasks:
TS.TaskProxy(t).Dispatch(rd.randint(0, 2))
