def main(workers=10):
"""
Executes main function of mini-framework's Control thread.
:param workers: Integer detailing number of worker FIFO threads to employ
"""
start_logging()
log_info("New multiprocessing session with {} workers".format(workers))
# Input JoinableQueue and Output Queue
inq = JoinableQueue(maxsize=int(workers*1.5))
outq = Queue(maxsize=int(workers*1.5))
ot = OutThread(workers, outq)
ot.start()
for _ in range(workers):
w = WorkerThread(inq, outq)
w.start()
# Create a sequence of a 1000 random alphabetic characters
random_chars = (ascii_letters[randint(0, 51)] for _ in range(1000))
# Keep input queue loaded for as long as possible
# Feed the process pool with work units
for work in enumerate(random_chars):
inq.put(work)
# Fill the input queue with Nones to shut the worker threads down
# which terminates the process pool
for _ in range(workers):
inq.put(None)
inq.join()
print("Control process terminating")
def showEvent(self, event: QtGui.QShowEvent) -> None:
if "domains" in self.setting.childGroups():
self.setting.beginGroup("domains")
self.domains = self.setting.childKeys()
self.setting.endGroup()
if not self.domains:
self.domains = configurations.get("domains")
if not self.updating and self.domains:
self.updating = True
self.ui.listWidget.clear()
if hasattr(self.parent(), "resetParsedDomains"):
self.parent().resetParsedDomains()
itemSize = QtCore.QSize(
self.ui.listWidget.width() -
self.ui.listWidget.autoScrollMargin() * 2, 50)
no = 0
for domain in self.domains:
item = QtWidgets.QListWidgetItem()
item.setSizeHint(itemSize)
item.setData(QtWidgets.QListWidgetItem.UserType, domain)
widget = self.makeDomainItemWidget(no, domain)
self.ui.listWidget.addItem(item)
self.ui.listWidget.setItemWidget(item, widget)
no += 1
self.worker = WorkerThread(self.domains)
self.worker.signal.connect(self.process)
self.worker.start()
def startDaemon(self):
"""
Start the thread that takes care of polling and scheduling for all the transports
"""
from worker import WorkerThread
self.thread = WorkerThread(self.name, self.txqueue, self.rxqueue)
self.thread.start()
def main(workers=10):
"""
Executes main function of mini-framework's Control thread.
:param workers: Integer detailing number of worker FIFO threads to employ
"""
start_logging()
log_info("New multiprocessing session with {} workers".format(workers))
# Input JoinableQueue and Output Queue
inq = JoinableQueue(maxsize=int(workers * 1.5))
outq = Queue(maxsize=int(workers * 1.5))
ot = OutThread(workers, outq)
ot.start()
for _ in range(workers):
w = WorkerThread(inq, outq)
w.start()
# Create a sequence of a 1000 random alphabetic characters
random_chars = (ascii_letters[randint(0, 51)] for _ in range(1000))
# Keep input queue loaded for as long as possible
# Feed the process pool with work units
for work in enumerate(random_chars):
inq.put(work)
# Fill the input queue with Nones to shut the worker threads down
# which terminates the process pool
for _ in range(workers):
inq.put(None)
inq.join()
print("Control process terminating")
def __init__(self,*args, **kwargs):
if kwargs is not None:
if 'count' in kwargs:
self._wcnt = kwargs['count']
else:
self._wcnt = WorkerPool.DEFAULT_WORKER_COUNT
if 'all_done' in kwargs:
self._all_done = kwargs['all_done']
else:
def ___pass( *args, **kwargs ):
pass
self._all_done = ___pass
if 'tick' in kwargs:
self._tick = kwargs['tick']
self._wpol = list([]) # all workers
self._wbsy = dict({}) # busy
self._wfre = list([]) # free
self._tque = list([]) # task queue
self._tick = WorkerPool.DEFAULT_TICK # sleep time if all busy
self._wrkr = SchedulerThread(0) # task schedule is worker
self._task = Task() # taks for scheduler
self._task.target = self._task_schedule
self._task.args = tuple()
self._task.kwargs = dict()
self._ball_done = False
def _on_worker_task_done( worker ):
"""
default calback for scheduler
when some worker is done with a task
it is appended to free workers queue
also marked free in busy dict
"""
self._wfre.append(worker)
self._wbsy[worker._id] = False
self.____f = _on_worker_task_done
# pool initialization
for i in xrange(1,self._wcnt+1) :
w = WorkerThread(i)
w._pool = self # back reference
w._worker_task_done = self.____f
self._wpol.append(w)
self._wfre.append(w)
self._wbsy[i] = False
self._wrkr._target = self._task_schedule
self._wrkr._args = tuple()
self._wrkr._kwargs = dict()
def main(workers=WORKERS):
"""
Executes main function of mini-framework's Control thread.
:param workers: Integer detailing number of worker FIFO threads to employ
"""
log_info("New mini-framework session with {} workers".format(workers))
inq = Queue(maxsize=int(workers * 1.5))
outq = Queue(maxsize=int(workers * 1.5))
ot = OutThread(workers, outq)
ot.start()
for _ in range(workers):
w = WorkerThread(inq, outq)
w.start()
# Create a sequence of a 1000 random alphabetic characters
random_chars = (ascii_letters[randint(0, 51)] for _ in range(1000))
# Keep input queue loaded for as long as possible
for work in enumerate(random_chars):
inq.put(work)
# Fill the input queue with Nones to shut the worker threads down
for _ in range(workers):
inq.put(None)
inq.join()
print("Control thread terminating")
log_info("Mini-framework finished. Len: {} chars".format(len(ot.output)))
def __init__(self, *args, **kwargs):
if kwargs is not None:
if 'count' in kwargs:
self._wcnt = kwargs['count']
else:
self._wcnt = WorkerPool.DEFAULT_WORKER_COUNT
if 'all_done' in kwargs:
self._all_done = kwargs['all_done']
else:
def ___pass(*args, **kwargs):
pass
self._all_done = ___pass
if 'tick' in kwargs:
self._tick = kwargs['tick']
self._wpol = list([]) # all workers
self._wbsy = dict({}) # busy
self._wfre = list([]) # free
self._tque = list([]) # task queue
self._tick = WorkerPool.DEFAULT_TICK # sleep time if all busy
self._wrkr = SchedulerThread(0) # task schedule is worker
self._task = Task() # taks for scheduler
self._task.target = self._task_schedule
self._task.args = tuple()
self._task.kwargs = dict()
self._ball_done = False
def _on_worker_task_done(worker):
"""
default calback for scheduler
when some worker is done with a task
it is appended to free workers queue
also marked free in busy dict
"""
self._wfre.append(worker)
self._wbsy[worker._id] = False
self.____f = _on_worker_task_done
# pool initialization
for i in xrange(1, self._wcnt + 1):
w = WorkerThread(i)
w._pool = self # back reference
w._worker_task_done = self.____f
self._wpol.append(w)
self._wfre.append(w)
self._wbsy[i] = False
self._wrkr._target = self._task_schedule
self._wrkr._args = tuple()
self._wrkr._kwargs = dict()
def __init__(self, dispatch, resume=True):
self.__dispatch = dispatch
self.__worker = WorkerThread()
self.hist = [Position(initial, 0, (True, True), (True, True), 0, 0)]
self.board = chess.Board()
self.redo = []
self.searcher = Searcher()
self.store = DictStore('fisher.dat')
if resume:
self.load_game()
def run_worker_job(self, job_func):
if not self.worker_thread:
# Create a thread.
# Must store a reference to the thread in a non-local variable,
# so the thread doesn't get garbage collected after returning
# from this method
# https://stackoverflow.com/a/15702922/
self.worker_thread = WorkerThread()
# This will emit 'started' and start running the thread
self.worker_thread.run_job(job_func)
def start_server(self):
# Check if directory 'www' exists
self.check_directory()
while True:
# Wait for a connection
conn, addr = self.s.accept()
# Create a new thread for each connection
new_thread = WorkerThread(addr, conn, self.dictionary, self.mutex,
self.directory)
new_thread.start()
def startDaemon(self): """ Start the thread that takes care of polling and scheduling for all the transports """ from worker import WorkerThread self.thread = WorkerThread(self.name, self.txqueue, self.rxqueue) self.thread.start()
class FBQController(wx.Frame):
def __init__(self, fbResultViews, facebookService, options, shelfdir=None):
wx.Frame.__init__(self, None, -1, 'title', size = (1, 1),
style=wx.FRAME_NO_TASKBAR|wx.NO_FULL_REPAINT_ON_RESIZE)
self.popUpLinger = None
self.updateInterval = None
self.failedCount = 0
self.loginFirstCount = 0
self.fbResultViews = fbResultViews
self.facebookService = facebookService
self.facebookService.sessionEvtHandler = self
self.options = options
self.shelfdir = shelfdir
self.lastResults = []
self.Bind(wx.EVT_CLOSE, self.onClose)
self.worker = WorkerThread(self.checkWorker, self.checkCallback, self.checkErrback)
self.worker.start()
def onClose(self, evt):
self.Destroy()
def onLoginFirst(self, evt):
if self.loginFirstCount > 0:
wx.PostEvent(wx.GetApp(), FBQApp_StateOffline_Event())
else:
self.loginFirstCount = self.loginFirstCount + 1
db = shelve.open(self.shelfdir and os.path.join(self.shelfdir,'session') or 'session')
if db.has_key('sessionKey') and db.has_key('sessionSecret'):
self.facebookService.sessionKey = db['sessionKey']
self.facebookService.sessionSecret = db['sessionSecret']
db.close()
db = None
if self.facebookService.sessionKey == '':
try:
self.facebookService.auth_createToken()
except FacebookError, e:
self.fbResultViews.result_view(e)
return
self.fbResultViews.login_first_view(self.facebookService.loginUrl)
else:
def run():
WORKERS = 10
inq = Queue(maxsize=int(WORKERS * 1.5))
outq = Queue(maxsize=int(WORKERS * 1.5))
ot = OutThread(WORKERS, outq)
ot.start()
for i in range(WORKERS):
w = WorkerThread(inq, outq)
w.start()
instring = randomstring()
for work in enumerate(instring):
inq.put(work)
for i in range(WORKERS):
inq.put(None)
inq.join()
print("Control thread terminating")
def control():
WORKERS = 10
inq = Queue(maxsize=int(WORKERS*1.5))
outq = Queue(maxsize=int(WORKERS*1.5))
ot = OutThread(WORKERS, outq)
ot.start()
for i in range(WORKERS):
w = WorkerThread(inq, outq)
w.start()
instring = [chr(randint(97, 122)) for _ in range(1000)]
for work in enumerate(instring):
inq.put(work)
for i in range(WORKERS):
inq.put(None)
inq.join()
print("Control thread terminating")
def process(n):
WORKERS = 10
inq = JoinableQueue(maxsize = int(WORKERS * 1.5))
outq = Queue(maxsize = int(WORKERS * 1.5))
ot = OutThread(WORKERS, outq, sorting = True)
ot.start()
for i in range(WORKERS):
w = WorkerThread(inq, outq)
w.start()
instring = "".join(random.choice(string.ascii_letters) for i in range(n))
for work in enumerate(instring):
inq.put(work)
for i in range(WORKERS):
inq.put(None)
inq.join()
print("Control process terminating.")
def run():
WORKERS = 10
inq = Queue(maxsize=int(WORKERS*1.5))
outq = Queue(maxsize=int(WORKERS*1.5))
ot = OutThread(WORKERS, outq)
ot.start()
for i in range(WORKERS):
w = WorkerThread(inq, outq)
w.start()
# generate a random string of alphabetic characters of length one thousand
instring = (''.join(random.choice(string.ascii_letters) for i in range(1000)))
for work in enumerate(instring):
inq.put(work)
for i in range(WORKERS):
inq.put(None)
inq.join()
print("Control thread terminating")
def run():
WORKERS = 10
inq = Queue(maxsize=int(WORKERS * 1.5))
outq = Queue(maxsize=int(WORKERS * 1.5))
ot = OutThread(WORKERS, outq)
ot.start()
for i in range(WORKERS):
w = WorkerThread(inq, outq)
w.start()
# generate a random string of alphabetic characters of length one thousand
instring = (''.join(
random.choice(string.ascii_letters) for i in range(1000)))
for work in enumerate(instring):
inq.put(work)
for i in range(WORKERS):
inq.put(None)
inq.join()
print("Control thread terminating")
def __init__(self, fbResultViews, facebookService, options, shelfdir=None):
wx.Frame.__init__(self, None, -1, 'title', size = (1, 1),
style=wx.FRAME_NO_TASKBAR|wx.NO_FULL_REPAINT_ON_RESIZE)
self.popUpLinger = None
self.updateInterval = None
self.failedCount = 0
self.loginFirstCount = 0
self.fbResultViews = fbResultViews
self.facebookService = facebookService
self.facebookService.sessionEvtHandler = self
self.options = options
self.shelfdir = shelfdir
self.lastResults = []
self.Bind(wx.EVT_CLOSE, self.onClose)
self.worker = WorkerThread(self.checkWorker, self.checkCallback, self.checkErrback)
self.worker.start()
def run_server(addr, port):
global task
log = Log(__name__, level='INFO')
log.info('Run httpd server until ctrl-c input')
def shutdown(task):
task.worker.stop()
task.running = False
def start(httpd, id):
httpd.start()
def signal_handler(signum, stack):
log.info('Sending shutdown to httpd server')
thread.start_new_thread(shutdown, (task, ))
signal.signal(signal.SIGINT, signal_handler)
server = Httpd(port=int(port), address=addr)
task = WorkerThread(server, 'httpd')
worker = WorkerTasks(tasks=[task], func=start)
worker.run()
worker.wait_for_completion(timeout_sec=-1) # run forever
class Engine:
def __init__(self, dispatch, resume=True):
self.__dispatch = dispatch
self.__worker = WorkerThread()
self.hist = [Position(initial, 0, (True, True), (True, True), 0, 0)]
self.board = chess.Board()
self.redo = []
self.searcher = Searcher()
self.store = DictStore('fisher.dat')
if resume:
self.load_game()
@mainthread
def dispatch(self, event, *args):
self.__dispatch(event, *args)
@property
def is_game_over(self):
return self.board.is_game_over()
@property
def moves(self):
return self.board.move_stack
# convert sunfish position into a list of pieces with their file and rank
def position(self, pos=None):
pos = pos or self.hist[-1]
pieces = []
for rank, row in enumerate(pos.board.split()):
for file, p in enumerate(row):
if p == '.':
continue
pieces.append('{}{}{}'.format(p, 'abcdefgh'[file], 8 - rank))
return pieces
def validate(self, _move):
try:
move = chess.Move.from_uci(_move)
except:
return None
while True:
if move in self.board.legal_moves:
return move
if self.humans_turn:
break
# Sunfish move deemed illegal by python-chess? likely a promotion
assert not move.promotion
if not self.humans_turn:
move.promotion = chess.QUEEN
def apply_move(self, _move):
move = self.validate(self.decode(_move))
if move:
self.board.push(move)
self.hist.append(self.hist[-1].move(_move))
# after the machine's move, check if redo list still valid
if self.humans_turn:
self.check_redo()
self.dispatch('on_update', *self.status(), move.uci())
if not self.is_game_over:
return move
def input_move(self, move):
move = self.parse_and_validate(move)
if move and self.apply_move(move):
self.search_move()
def parse_and_validate(self, move):
if not self.is_game_over:
match = re.match('([a-h][1-8])' * 2, move)
if match:
return parse(match.group(1)), parse(match.group(2))
def decode(self, move):
if not self.humans_turn:
move = [119 - m for m in move]
return '{}{}'.format(*(render(m) for m in move))
# Fire up the engine to look for a move -- in the background thread
def search_move(self):
def search():
start = time.time()
for _depth, move, _score in self.searcher.search(
self.hist[-1], self.hist):
if time.time() - start > 1:
break
self.apply_move(move)
self.save_game()
self.__worker.send_message(search)
def status_message(self):
if self.board.is_stalemate():
return 'Stalemate'
if self.board.is_checkmate():
return 'Checkmate!'
if self.board.is_check():
return 'Check!'
return 'Your turn' if self.humans_turn else 'Thinking...'
@property
def humans_turn(self):
return self.board.turn == chess.WHITE
def status(self):
pos = self.hist[-1] if self.hist else None
if pos and not self.humans_turn:
pos = pos.rotate()
return self.position(pos), self.status_message()
def __can_use(self, moves_list):
return len(moves_list) > 0 and (self.humans_turn or self.is_game_over)
def can_undo(self):
return self.__can_use(self.moves)
def can_redo(self):
return self.__can_use(self.redo)
def check_redo(self):
if self.redo and self.last_move != self.redo[-1]:
# history took a different turn; redo list is invalid
self.redo.clear()
def undo_move(self):
assert self.can_undo()
assert len(self.hist) >= 2
# Assuming human plays white -- careful if/when implementing a "switch" feature
# Moves count should be even, unless we lost.
# Length of position history is odd because of initial empty position.
assert len(self.hist) % 2 or self.is_game_over
n = 1 if len(self.moves) % 2 else 2
self.hist = self.hist[:-n]
self.redo.append(self.moves[-n].uci())
while n > 0:
self.board.pop()
n -= 1
self.redo.append(self.last_move)
self.dispatch('on_update', *self.status(), self.last_move)
def redo_move(self):
assert self.redo
assert len(self.redo) % 2 == 0
move = self.redo.pop()
assert move == self.last_move
move = self.redo.pop()
self.input_move(move)
def start(self):
if not self.humans_turn:
self.search_move()
@property
def last_move(self):
return self.moves[-1].uci() if self.moves else None
def load_game(self):
Logger.debug('{}: load'.format(__name__))
if self.store.exists('game'):
data = self.store.get('game')
self.hist = data.get('hist', [])
for move in data.get('moves', []):
self.board.push(move)
def save_game(self):
Logger.debug('{}: save'.format(__name__))
self.store.put('game', hist=self.hist, moves=self.moves)
class MainForm(QtGui.QMainWindow):
def __init__(self, parent=None):
QtGui.QMainWindow.__init__(self, parent)
self.CreateToolBar()
self.CreateStatusBar()
self.CreateToolTip()
self.removeAction.setEnabled(False)
self.editAction.setEnabled(False)
self.itemModel = ItemModel()
self.itemSelectionModel = QtGui.QItemSelectionModel(self.itemModel)
self.listView = QtGui.QTreeView(self)
self.listView.setModel(self.itemModel)
self.listView.setSelectionModel(self.itemSelectionModel)
self.listView.setRootIsDecorated(False)
self.listView.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.listView.setColumnWidth(self.itemModel.countryColumn(), 20)
self.listView.setColumnWidth(self.itemModel.asinColumn(), 100)
self.listView.setColumnWidth(self.itemModel.labelColumn(), 210)
self.listView.setColumnWidth(self.itemModel.priceColumn(), 70)
self.listView.setColumnWidth(self.itemModel.priceLastColumn(), 70)
self.listView.setColumnWidth(self.itemModel.priceMinColumn(), 70)
self.listView.setColumnWidth(self.itemModel.priceMaxColumn(), 70)
self.listView.setColumnWidth(self.itemModel.chartColumn(), 30)
self.itemSelectionModel.selectionChanged.connect(self.OnItemSelectionChanged)
self.listView.viewport().setMouseTracking(True)
self.listView.viewport().installEventFilter(self);
self.listView.installEventFilter(self)
self.timer = QtCore.QTimer(self)
self.timer.timeout.connect(self.OnTimer)
self.timer.start()
self.updateThread = WorkerThread()
self.updateThread.setTask(lambda abort: self.OnUpdateItemsTask(abort))
self.updateThread.resultReady.connect(self.OnUpdateItemsTaskFinished)
self.fetchThread = WorkerThread()
self.fetchThread.resultReady.connect(self.OnFetchImageTaskFinished)
self.CreateTray()
self.tray.show()
self.setCentralWidget(self.listView)
self.resize(640, 200);
self.setWindowTitle(self.tr("Amazon Watch Dog"))
self.setWindowIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "awd.svg"))
self.timer.setInterval(60000 * 20)
self.hideAfterStart = True
self.showNotifications = True
self.settings = QtCore.QSettings(helper.GetConfigDir() + QtCore.QDir.separator() + helper.GetConfigName(), QtCore.QSettings.IniFormat, self)
self.imageCache = ImageCache(helper.GetConfigDir() + QtCore.QDir.separator() + "cache")
self.listView.setItemDelegateForColumn(self.itemModel.chartColumn(), ChartItemDelegate(self.listView))
self.LoadSettings()
self.LoadGeometrySettings()
self.UpdateListView()
if self.hideAfterStart: self.hide()
else: self.show()
self.SetLastUpdateLabel(self.lastUpdate)
def CreateToolBar(self):
self.toolbar = QtGui.QToolBar(self.tr("Main Toolbar"), self)
self.CreateActions()
self.toolbar.addAction(self.addAction)
self.toolbar.addAction(self.editAction)
self.toolbar.addAction(self.removeAction)
self.toolbar.addAction(self.updateAction)
self.addToolBar(self.toolbar)
def CreateStatusBar(self):
statusbar = QtGui.QStatusBar(self)
self.lastUpdateLabel = QtGui.QLabel(statusbar)
self.waitWidget = WaitWidget(statusbar)
self.waitWidget.hide()
statusbar.addWidget(self.lastUpdateLabel)
statusbar.addWidget(self.waitWidget)
self.setStatusBar(statusbar)
def CreateToolTip(self):
self.tooltip = ImageToolTip(self)
self.tooltipItem = None
def CreateTray(self):
self.tray = QtGui.QSystemTrayIcon(self)
self.trayMenu = QtGui.QMenu(self)
self.trayMenu.addAction(self.updateAction)
self.trayMenu.addAction(self.itemsAction)
self.trayMenu.addAction(self.settingsAction)
if helper.debug_mode:
self.trayMenu.addSeparator()
self.trayMenu.addAction(self.builderAction)
self.trayMenu.addAction(self.updateMainTable)
self.trayMenu.addSeparator()
self.trayMenu.addAction(self.aboutAction)
self.trayMenu.addAction(self.quitAction)
self.tray.setContextMenu(self.trayMenu)
self.tray.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + defaults.GetTrayIconName()))
self.tray.activated.connect(self.OnTrayActivated)
def CreateActions(self):
self.addAction = QtGui.QAction(self.tr("Add item..."), self)
self.addAction.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "add.png"))
self.addAction.triggered.connect(self.OnAddItem)
self.editAction = QtGui.QAction(self.tr("Edit item..."), self)
self.editAction.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "edit.png"))
self.editAction.triggered.connect(self.OnEditItem)
self.removeAction = QtGui.QAction(self.tr("Remove item(s)"), self)
self.removeAction.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "remove.png"))
self.removeAction.triggered.connect(self.OnRemoveItem)
self.updateAction = QtGui.QAction(self.tr("Update"), self)
self.updateAction.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "update.png"))
self.updateAction.triggered.connect(self.OnUpdateItems)
self.settingsAction = QtGui.QAction(self.tr("Settings..."), self)
self.settingsAction.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "settings.png"))
self.settingsAction.triggered.connect(self.OnShowSettings)
self.itemsAction = QtGui.QAction(self.tr("Items..."), self)
self.itemsAction.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "list.png"))
self.itemsAction.triggered.connect(self.show)
self.quitAction = QtGui.QAction(self.tr("Quit"), self)
self.quitAction.triggered.connect(self.exit)
self.aboutAction = QtGui.QAction(self.tr("About..."), self)
self.aboutAction.triggered.connect(self.OnAbout)
if helper.debug_mode:
self.builderAction = QtGui.QAction(self.tr("Build request..."), self)
self.builderAction.triggered.connect(self.OnBuildRequest)
self.updateMainTable = QtGui.QAction(self.tr("Update main table"), self)
self.updateMainTable.triggered.connect(self.OnUpdateMaintable)
def OnTimer(self):
self.OnUpdateItems()
def OnTrayActivated(self, reason):
if reason == QtGui.QSystemTrayIcon.DoubleClick:
if self.isHidden(): self.show()
else: self.hide()
def OnAbout(self):
form = AboutForm(self)
form.exec_()
def exit(self):
if self.updateThread.isRunning(): self.updateThread.wait()
if self.fetchThread.isRunning(): self.fetchThread.wait()
self.SaveSettings()
QtGui.qApp.quit()
def closeEvent(self, event):
self.hide()
event.ignore()
def eventFilter(self, obj, event):
if event.type() == QtCore.QEvent.ContextMenu:
self.OnListViewContextMenuEvent(event)
return True
elif event.type() == QtCore.QEvent.MouseMove:
self.OnListViewMouseMoveEvent(event)
return True
elif event.type() == QtCore.QEvent.ToolTip:
self.OnListViewToolTipEvent(event)
return True
return QtGui.QMainWindow.eventFilter(self, obj, event)
def OnListViewContextMenuEvent(self, event):
asins = self.GetSelectedASINs()
if not asins: return
menu = QtGui.QMenu(self)
if len(asins) == 1:
urlAction = menu.addAction(self.tr("Open URL"))
urlAction.triggered.connect(lambda: self.OnOpenURL(asins[0]))
asinAction = menu.addAction(self.tr("Copy ASIN"))
asinAction.triggered.connect(lambda: self.OnCopyASIN(asins[0]))
if helper.debug_mode:
attrsAction = menu.addAction(self.tr("Get attributes..."))
attrsAction.triggered.connect(lambda: self.OnGetAttributes(asins[0]))
imagesAction = menu.addAction(self.tr("Get images..."))
imagesAction.triggered.connect(lambda: self.OnGetImages(asins[0]))
menu.addSeparator()
if len(asins) == 1:
menu.addAction(self.editAction)
menu.addAction(self.removeAction)
if len(asins) > 0:
menu.exec_(event.globalPos())
def OnListViewMouseMoveEvent(self, event):
index = self.listView.indexAt(event.pos())
item = self.itemModel.item(index)
if item is None:
self.tooltipItem = None
self.tooltip.hideTip()
if self.tooltipItem is None or item is self.tooltipItem:
return
if not item is self.tooltipItem:
self.tooltip.showTip(QtGui.QCursor.pos())
self.tooltipItem = item
self.FetchImage()
def OnListViewToolTipEvent(self, event):
if self.tooltip.isVisible(): return
index = self.listView.indexAt(event.pos())
self.tooltipItem = self.itemModel.item(index)
if self.tooltipItem is None:
self.tooltip.hideTip()
else:
self.tooltip.showTip(QtGui.QCursor.pos())
self.FetchImage()
def GetSelectedASINs(self):
selected = self.itemSelectionModel.selectedRows()
asins = map(lambda index: self.itemModel.asin(index), selected)
return asins
def UpdateListView(self):
self.itemModel.update()
def OnItemSelectionChanged(self, selected, deselected):
selectedRows = self.itemSelectionModel.selectedRows()
enable_removing = len(selectedRows) > 0
enable_editing = len(selectedRows) == 1
self.removeAction.setEnabled(enable_removing)
self.editAction.setEnabled(enable_editing)
def OnAddItem(self):
form = ItemForm(self, self.accessKey, self.secretKey, self.associateTag)
if form.exec_() == QtGui.QDialog.Accepted: self.UpdateListView()
def OnEditItem(self):
asins = self.GetSelectedASINs()
if len(asins) > 1: return
form = ItemForm(self, self.accessKey, self.secretKey, self.associateTag, asins[0])
if form.exec_() == QtGui.QDialog.Accepted: self.UpdateListView()
def OnRemoveItem(self):
if QtGui.QMessageBox.warning(self, self.tr("Warning"), self.tr("Delete this item(s) from database?"),
QtGui.QMessageBox.Yes | QtGui.QMessageBox.No, QtGui.QMessageBox.No) == QtGui.QMessageBox.No: return
for asin in self.GetSelectedASINs():
db.DeleteItem(asin)
self.UpdateListView()
def OnUpdateItems(self):
if self.updateThread.isRunning():
print("Worker thread is already running")
return
if self.accessKey == "" or self.secretKey == "" or self.associateTag == "":
QtGui.QMessageBox.warning(self, self.tr("Warning"),
self.tr("Amazon access parameters are not set. Go to \"Settings\" dialog and fill corresponded fields"))
return
self.waitWidget.show()
self.waitWidget.Start()
self.toolbar.setEnabled(False)
self.addAction.setEnabled(False)
self.removeAction.setEnabled(False)
self.editAction.setEnabled(False)
self.updateThread.start()
def OnUpdateItemsTask(self, abort):
result = db.UpdateAllItems(self.accessKey, self.secretKey, self.associateTag)
return TaskResult(result, 0, "")
def OnUpdateItemsTaskFinished(self, result):
self.waitWidget.hide()
self.waitWidget.Stop()
self.toolbar.setEnabled(True)
self.addAction.setEnabled(True)
self.removeAction.setEnabled(True)
self.editAction.setEnabled(True)
self.OnItemSelectionChanged(self.itemSelectionModel.selection(), QtGui.QItemSelection())
if result.error != 0:
QtGui.QMessageBox.information(self, self.tr("Fetching error"), result.message)
return
self.UpdateItems(result.result.total, result.result.changed, result.result.failed, result.result.error)
def UpdateItems(self, total, changed, failed, error):
if error and total == failed:
notify.Notify(self.tr("Updating [Failed]"), error, self, self.sysNotify)
print(error)
return
self.lastUpdate = QtCore.QDateTime.currentDateTime()
self.SetLastUpdateLabel(self.lastUpdate)
self.settings.setValue("last_update", self.lastUpdate)
if not self.showNotifications: return
if failed > 0: text = self.tr("{0} items have been checked\n{1} changed, {2} failed").format(total, changed, failed)
else: text = self.tr("{0} items have been checked\n{1} changed").format(total, changed)
if error: text += "\n" + error
notify.Notify(self.tr("Updating [Finished]"), text, self, self.sysNotify)
self.UpdateListView()
def OnShowSettings(self):
form = SettingsForm(self, self.settings, self.imageCache)
if form.exec_() == QtGui.QDialog.Accepted: self.LoadSettings()
def OnBuildRequest(self):
form = RequestForm(self, self.accessKey, self.secretKey, self.associateTag)
form.exec_()
def OnUpdateMaintable(self):
db.UpdateMainTable()
self.UpdateListView()
def SaveSettings(self):
self.settings.setValue("mainform_size", self.size())
self.settings.sync()
def LoadSettings(self):
self.timer.setInterval(60000 * int(self.settings.value("interval", 20)))
self.hideAfterStart = to_bool(self.settings.value("hide", "true"))
self.showNotifications = to_bool(self.settings.value("notifications", "true"))
self.accessKey = str(self.settings.value("access_key", ""))
self.secretKey = str(self.settings.value("secret_key", ""))
self.associateTag = str(self.settings.value("associate_tag", ""))
self.sysNotify = to_bool(self.settings.value("sys_notify", "false"))
self.lastUpdate = self.settings.value("last_update", QtCore.QDateTime())
self.LoadAppearanceSettings()
def LoadGeometrySettings(self):
self.resize(self.settings.value("mainform_size", QtCore.QSize(640, 200)))
def LoadAppearanceSettings(self):
self.settings.beginGroup("Appearance")
self.itemModel.SetNumSamples(int(self.settings.value("graph_n_samples", defaults.GetNumSamples())))
self.itemModel.SetUpTextForegroundColor(ReadColorValue(self.settings, "text_up_foreground_color", defaults.GetTextUpForegroundColor()))
self.itemModel.SetDownTextForegroundColor(ReadColorValue(self.settings, "text_down_foreground_color", defaults.GetTextDownForegroundColor()))
self.itemModel.update()
delegete = self.listView.itemDelegateForColumn(self.itemModel.chartColumn())
if delegete:
delegete.SetUpLineColor(ReadColorValue(self.settings, "graph_up_line_color", defaults.GetUpLineColor()))
delegete.SetUpFillColor(ReadColorValue(self.settings, "graph_up_fill_color", defaults.GetUpFillColor()))
delegete.SetDownLineColor(ReadColorValue(self.settings, "graph_down_line_color", defaults.GetDownLineColor()))
delegete.SetDownFillColor(ReadColorValue(self.settings, "graph_down_fill_color", defaults.GetDownFillColor()))
delegete.SetNeutralLineColor(ReadColorValue(self.settings, "graph_neutral_line_color", defaults.GetNeutralLineColor()))
delegete.SetNeutralFillColor(ReadColorValue(self.settings, "graph_neutral_fill_color", defaults.GetDefaultNeutralFillColor()))
if self.tray: self.tray.setIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + self.settings.value("tray_icon", defaults.GetTrayIconName())))
self.settings.endGroup()
def OnCopyASIN(self, asin):
clipboard = QtGui.QApplication.clipboard()
clipboard.setText(asin)
def OnOpenURL(self, asin):
country = db.GetItemCountry(asin)
domain = helper.GetDomain(country)
if not domain or not asin: return
url = "http://amzn." + domain + "/" + asin
QtGui.QDesktopServices.openUrl(QtCore.QUrl(url))
def OnGetAttributes(self, asin):
try:
country = db.GetItemCountry(asin)
attrs = GetAttributes(asin, country, self.accessKey, self.secretKey, self.associateTag)
print(attrs)
except AWSError, e:
print(e.GetFullDescription())
class Messenger(object):
"""
The basic messenger interface for clients. It provides the run loop for async transports and their scheduling
as well as routing behaviour if there are multiple transports.
THe user can join/leave groups as well as receive and send messages.
This is the interface used by the caller. Its passes most items as objects in the TxQueue as most operations take
place in another thread.
"""
def __init__(self, name):
""" Creating messaging interface here """
self.name = name
self.rxqueue = Queue.Queue()
self.txqueue = Queue.Queue()
self.thread = None
self.groupMembership = defaultdict(set)
self.startDaemon()
def startDaemon(self):
"""
Start the thread that takes care of polling and scheduling for all the transports
"""
from worker import WorkerThread
self.thread = WorkerThread(self.name, self.txqueue, self.rxqueue)
self.thread.start()
def addTransport(self, transport, keepConnected=False):
"""
Add a new transport to this messages. This can be compared to adding a new ethernet interface on a simple node.
The messenger will take care of routing if there are multiple transports. Examples:
TCPServer('0.0.0.0', 17708)
TCPTransport('192.168.1.50', 17708)
MulticastTransport('192.168.1.40', '255.239.13.4')
"""
self.txqueue.put(TransportRequest(transport, keepConnected))
def join(self, group, caller = "default"):
"""
Join the group 'group' (a string value).
Messages sent to this group by other nodes will now be received by the owner of this messaging object
The call option allows one to attach an 'ID' to the join request so that someone else joining and leaving
the same group doesn't cause the official leave to actually occur.
"""
self.txqueue.put(GroupRequest("join", group, caller))
self.groupMembership[group].add(caller)
def leave(self, group, caller = "default"):
"""
Leave the group 'group' (a string value).
This messaging object will no longer receive messages for this group.
If the caller joined with a caller value, it must use the same value for the leave request
"""
self.txqueue.put(GroupRequest("leave", group, caller))
self.groupMembership[group].discard(caller)
if not self.groupMembership[group]:
del self.groupMembership[group]
def nextMessage(self, block=False, timeout=None):
"""
Called to remove a received message from the queue. block and timeout are as specified in
Queue.Queue.get()
"""
return self.rxqueue.get(block, timeout)
def send(self, msg, **kwargs):
"""
Enqueue a message for transmittal, kwargs is a list of delivery request values to specify desired delivery behavior
"""
self.txqueue.put(TransmitRequest(msg, kwargs))
def trigger(self, **kwargs):
"""
Send a trigger event. Single location for a common action, ick though, this is application level stuff in messaging code
"""
self.send(MAGIMessage(groups="control", docks="daemon", data=yaml.dump(kwargs), contenttype=MAGIMessage.YAML))
def poisinPill(self):
"""
Puts a "PoisinPill" string into the rxqueue so that the first listener on nextMessage will
wake up and get the message.
"""
self.rxqueue.put("PoisinPill")
def stop(self):
self.thread.stop() # stopping the worker thread
class DialogUpdater(QDialog):
def __init__(self, parent=None):
super(DialogUpdater, self).__init__(parent)
# super(DialogUpdater, self).__init__(parent, Qt.CustomizeWindowHint | Qt.WindowTitleHint)
self.ui = dialog_updater.Ui_DialogUpdater()
self.ui.setupUi(self)
self.ui.listWidget.setStyleSheet(
"QListWidget { outline:0; background-color: transparent; }"
"QListWidget::item {border-bottom: 1px solid #999;}"
"QListWidget::item::selected { color: #fff; }")
# self.setWindowFlags(Qt.Window | Qt.CustomizeWindowHint | Qt.WindowMinimizeButtonHint | Qt.WindowMaximizeButtonHint)
self.worker = None
self.cleaner = None
self.setting = QSettings()
self.updating = False
self.domains = []
def process(self, data):
# print(data)
action, host, ip, secs, last = data
if "end" == action and ip and host and secs and secs < 99999:
if hasattr(self.parent(), "addParsedDomain"):
self.parent().addParsedDomain(host, ip)
if self.ui.listWidget.count() > 0:
for index in range(self.ui.listWidget.count()):
item = self.ui.listWidget.item(index)
data = item.data(QtWidgets.QListWidgetItem.UserType)
if data and data == host:
# oldWidget = self.ui.listWidget.itemWidget(item)
widget = self.makeDomainItemWidget(index, host, action, ip,
secs)
self.ui.listWidget.removeItemWidget(item)
self.ui.listWidget.setItemWidget(item, widget)
self.ui.listWidget.scrollToItem(item)
# self.ui.listWidget.setCurrentItem(item)
# itemIndex = self.ui.listWidget.indexFromItem(item)
# self.ui.listWidget.update(itemIndex)
if last:
self.updating = False
self.cleaner = CleanerThread()
self.cleaner.signal.connect(self.close)
self.cleaner.start()
def makeDomainItemWidget(self, idx, domain, status=None, ip=None, ms=None):
widget = QtWidgets.QWidget()
layout = QtWidgets.QHBoxLayout()
layout.setContentsMargins(0, 0, 0, 0)
layout.setSpacing(0)
no = QtWidgets.QLabel()
# no.setStyleSheet('background-color:blue')
no.setText(f"#{idx+1}")
no.setAlignment(QtCore.Qt.AlignCenter)
layout.addWidget(no)
layout.setStretch(0, 1)
name = QtWidgets.QLabel()
name.setText(domain)
# name.setStyleSheet('background-color:gray')
layout.addWidget(name)
layout.setStretch(1, 3)
flag = QtWidgets.QPushButton()
flag.setStyleSheet('background-color: transparent')
flag.setFlat(True)
if "end" == status:
flag.setText('')
if ip and ms < 99999:
flag.setIcon(qta.icon('fa5.check-circle', color='green'))
ms = "{:.3f}s".format(ms / 1000)
else:
flag.setIcon(qta.icon('fa5.times-circle', color='red'))
ms = ""
elif "begin" == status:
flag.setText('')
flag.setIcon(
qta.icon('fa5s.sync', color='gray', animation=qta.Spin(self)))
ip = ""
ms = ""
else:
flag.setText('')
ip = ""
ms = ""
layout.addWidget(flag)
layout.setStretch(2, 1)
ipLabel = QtWidgets.QLabel()
# ipLabel.setStyleSheet('background-color:green')
ipLabel.setText(ip)
layout.addWidget(ipLabel)
layout.setStretch(3, 2)
timer = QtWidgets.QLabel()
# timer.setStyleSheet('background-color:blue')
timer.setText(ms)
layout.addWidget(timer)
layout.setStretch(4, 1)
widget.setLayout(layout)
return widget
def showEvent(self, event: QtGui.QShowEvent) -> None:
if "domains" in self.setting.childGroups():
self.setting.beginGroup("domains")
self.domains = self.setting.childKeys()
self.setting.endGroup()
if not self.domains:
self.domains = configurations.get("domains")
if not self.updating and self.domains:
self.updating = True
self.ui.listWidget.clear()
if hasattr(self.parent(), "resetParsedDomains"):
self.parent().resetParsedDomains()
itemSize = QtCore.QSize(
self.ui.listWidget.width() -
self.ui.listWidget.autoScrollMargin() * 2, 50)
no = 0
for domain in self.domains:
item = QtWidgets.QListWidgetItem()
item.setSizeHint(itemSize)
item.setData(QtWidgets.QListWidgetItem.UserType, domain)
widget = self.makeDomainItemWidget(no, domain)
self.ui.listWidget.addItem(item)
self.ui.listWidget.setItemWidget(item, widget)
no += 1
self.worker = WorkerThread(self.domains)
self.worker.signal.connect(self.process)
self.worker.start()
import os
import urllib
import json
import random
import re
import time
from urllib.parse import urlparse
from config import config
from worker import WorkerThread, WorkerItemStatus
from server import app
from db import db
import statistics
from sample_bots import sample_bots
worker = WorkerThread()
worker.daemon = True
worker.start()
def run_commands(working_dir, commands):
output = []
for command in commands:
print(command)
output.append('-' * 80)
output.append(working_dir + '> ' + ' '.join(command))
result = subprocess.run(command,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
def randstr(length=1000):
result = []
for i in range(length):
result.append(ascii_letters[randrange(len(ascii_letters))])
return "".join(result)
if __name__ == "__main__":
WORKERS = 10
inq = JoinableQueue(maxsize=int(WORKERS * 1.5))
outq = Queue(maxsize=int(WORKERS * 1.5))
ot = OutThread(WORKERS, outq)
ot.start()
for i in range(WORKERS):
w = WorkerThread(inq, outq)
w.start()
instring = randstr()
# put work on the queue
for work in enumerate(instring):
inq.put(work)
# terminate the process pool
for i in range(WORKERS):
inq.put(None)
inq.join()
print("Control process terminating")
def test_simple_creation_equality(self):
wrk1 = WorkerThread('hello')
wrk2 = WorkerThread('hello')
self.assertEqual(wrk1._id, wrk2._id)
class Messenger(object):
"""
The basic messenger interface for clients. It provides the run loop for async transports and their scheduling
as well as routing behaviour if there are multiple transports.
THe user can join/leave groups as well as receive and send messages.
This is the interface used by the caller. Its passes most items as objects in the TxQueue as most operations take
place in another thread.
"""
def __init__(self, name):
""" Creating messaging interface here """
self.name = name
self.rxqueue = Queue.Queue()
self.txqueue = Queue.Queue()
self.thread = None
self.groupMembership = defaultdict(set)
self.startDaemon()
def startDaemon(self):
"""
Start the thread that takes care of polling and scheduling for all the transports
"""
from worker import WorkerThread
self.thread = WorkerThread(self.name, self.txqueue, self.rxqueue)
self.thread.start()
def addTransport(self, transport, keepConnected=False):
"""
Add a new transport to this messages. This can be compared to adding a new ethernet interface on a simple node.
The messenger will take care of routing if there are multiple transports. Examples:
TCPServer('0.0.0.0', 17708)
TCPTransport('192.168.1.50', 17708)
MulticastTransport('192.168.1.40', '255.239.13.4')
"""
self.txqueue.put(TransportRequest(transport, keepConnected))
def join(self, group, caller="default"):
"""
Join the group 'group' (a string value).
Messages sent to this group by other nodes will now be received by the owner of this messaging object
The call option allows one to attach an 'ID' to the join request so that someone else joining and leaving
the same group doesn't cause the official leave to actually occur.
"""
self.txqueue.put(GroupRequest("join", group, caller))
self.groupMembership[group].add(caller)
def leave(self, group, caller="default"):
"""
Leave the group 'group' (a string value).
This messaging object will no longer receive messages for this group.
If the caller joined with a caller value, it must use the same value for the leave request
"""
self.txqueue.put(GroupRequest("leave", group, caller))
self.groupMembership[group].discard(caller)
if not self.groupMembership[group]:
del self.groupMembership[group]
def nextMessage(self, block=False, timeout=None):
"""
Called to remove a received message from the queue. block and timeout are as specified in
Queue.Queue.get()
"""
return self.rxqueue.get(block, timeout)
def send(self, msg, **kwargs):
"""
Enqueue a message for transmittal, kwargs is a list of delivery request values to specify desired delivery behavior
"""
self.txqueue.put(TransmitRequest(msg, kwargs))
def trigger(self, **kwargs):
"""
Send a trigger event. Single location for a common action, ick though, this is application level stuff in messaging code
"""
self.send(
MAGIMessage(groups="control",
docks="daemon",
data=yaml.dump(kwargs),
contenttype=MAGIMessage.YAML))
def poisinPill(self):
"""
Puts a "PoisinPill" string into the rxqueue so that the first listener on nextMessage will
wake up and get the message.
"""
self.rxqueue.put("PoisinPill")
def stop(self):
self.thread.stop() # stopping the worker thread
control.py: Creates queues, starts output and worker threads,
and pushes inputs into the input queue.
"""
from multiprocessing import Queue, JoinableQueue
from output import OutThread
from worker import WorkerThread
from random import choice
from string import ascii_letters
if __name__ == '__main__':
WORKERS = 10
inq = JoinableQueue(maxsize=int(WORKERS * 1.5))
outq = Queue(maxsize=int(WORKERS * 1.5))
ot = OutThread(WORKERS, outq)
ot.start()
for i in range(WORKERS):
w = WorkerThread(inq, outq)
w.start()
instring = ''
for i in range(1000):
instring += choice(ascii_letters)
for work in enumerate(instring):
inq.put(work)
for i in range(WORKERS):
inq.put(None)
inq.join()
print("Control thread terminating")
def __init__(self, parent=None):
QtGui.QMainWindow.__init__(self, parent)
self.CreateToolBar()
self.CreateStatusBar()
self.CreateToolTip()
self.removeAction.setEnabled(False)
self.editAction.setEnabled(False)
self.itemModel = ItemModel()
self.itemSelectionModel = QtGui.QItemSelectionModel(self.itemModel)
self.listView = QtGui.QTreeView(self)
self.listView.setModel(self.itemModel)
self.listView.setSelectionModel(self.itemSelectionModel)
self.listView.setRootIsDecorated(False)
self.listView.setSelectionMode(QtGui.QAbstractItemView.ExtendedSelection)
self.listView.setColumnWidth(self.itemModel.countryColumn(), 20)
self.listView.setColumnWidth(self.itemModel.asinColumn(), 100)
self.listView.setColumnWidth(self.itemModel.labelColumn(), 210)
self.listView.setColumnWidth(self.itemModel.priceColumn(), 70)
self.listView.setColumnWidth(self.itemModel.priceLastColumn(), 70)
self.listView.setColumnWidth(self.itemModel.priceMinColumn(), 70)
self.listView.setColumnWidth(self.itemModel.priceMaxColumn(), 70)
self.listView.setColumnWidth(self.itemModel.chartColumn(), 30)
self.itemSelectionModel.selectionChanged.connect(self.OnItemSelectionChanged)
self.listView.viewport().setMouseTracking(True)
self.listView.viewport().installEventFilter(self);
self.listView.installEventFilter(self)
self.timer = QtCore.QTimer(self)
self.timer.timeout.connect(self.OnTimer)
self.timer.start()
self.updateThread = WorkerThread()
self.updateThread.setTask(lambda abort: self.OnUpdateItemsTask(abort))
self.updateThread.resultReady.connect(self.OnUpdateItemsTaskFinished)
self.fetchThread = WorkerThread()
self.fetchThread.resultReady.connect(self.OnFetchImageTaskFinished)
self.CreateTray()
self.tray.show()
self.setCentralWidget(self.listView)
self.resize(640, 200);
self.setWindowTitle(self.tr("Amazon Watch Dog"))
self.setWindowIcon(QtGui.QIcon("images" + QtCore.QDir.separator() + "awd.svg"))
self.timer.setInterval(60000 * 20)
self.hideAfterStart = True
self.showNotifications = True
self.settings = QtCore.QSettings(helper.GetConfigDir() + QtCore.QDir.separator() + helper.GetConfigName(), QtCore.QSettings.IniFormat, self)
self.imageCache = ImageCache(helper.GetConfigDir() + QtCore.QDir.separator() + "cache")
self.listView.setItemDelegateForColumn(self.itemModel.chartColumn(), ChartItemDelegate(self.listView))
self.LoadSettings()
self.LoadGeometrySettings()
self.UpdateListView()
if self.hideAfterStart: self.hide()
else: self.show()
self.SetLastUpdateLabel(self.lastUpdate)
class MainWidget(QWidget):
after_process_input_gci = pyqtSignal()
def __init__(self):
super().__init__()
self.init_layout()
self.output_filename_defaults = dict(
gci='',
replay_array='',
)
self.worker_thread = None
self.after_process_input_gci.connect(self._after_process_input_gci)
# Accept drag and drop; see event methods for details
self.setAcceptDrops(True)
def init_layout(self):
self.error_label = QLabel("")
self.error_label.setStyleSheet("QLabel { color: red; }")
self.input_gci_label = QLabel("Drag and drop a .gci to get started")
# Given that we have word wrap True, setting a fixed height prevents
# the GUI from resizing (and prevents an associated window size warning
# in the console window) when we switch between long and short
# filepaths.
# To help with reading long filepaths, we'll also set a tooltip each
# time we set the filepath.
self.input_gci_label.setWordWrap(True)
self.input_gci_label.setMinimumWidth(150)
self.input_gci_label.setFixedHeight(35)
self.input_gci_checksum_label = QLabel("")
self.gci_fields_widget = GCIFieldsWidget(self)
self.output_button = QPushButton("Output", self)
self.output_button.clicked.connect(self.on_output_button_click)
self.output_type_combo_box = QComboBox(self)
# The choices here only apply to replays. If we handle other types of
# .gci's in the future, we might change this combo box to add its
# choices dynamically.
self.output_type_combo_box.addItems([".gci", "Replay array .bin"])
self.output_type_combo_box.activated[str].connect(
self.on_output_type_change)
self.output_folder_select_dialog = QFileDialog(
self, "Choose folder to save to", self.output_folder)
self.output_folder_label = QLabel(self.output_folder)
self.output_folder_label.setToolTip(self.output_folder)
self.output_folder_label.setWordWrap(True)
self.output_folder_label.setMinimumWidth(150)
self.output_folder_label.setFixedHeight(50)
self.output_folder_select_button = QPushButton("Choose", self)
self.output_folder_select_button.clicked.connect(
self.show_output_folder_select_dialog)
self.output_filename_line_edit = QLineEdit()
self.output_filename_line_edit.editingFinished.connect(
self.on_output_filename_change)
output_type_hbox = QHBoxLayout()
output_type_hbox.addWidget(self.output_button)
output_type_hbox.addWidget(self.output_type_combo_box)
output_folder_hbox = QHBoxLayout()
output_folder_hbox.addWidget(QLabel("to folder:"))
output_folder_hbox.addWidget(self.output_folder_label)
output_folder_hbox.addWidget(self.output_folder_select_button)
output_filename_hbox = QHBoxLayout()
output_filename_hbox.addWidget(QLabel("with filename:"))
output_filename_hbox.addWidget(self.output_filename_line_edit)
self.output_vbox = QVBoxLayout()
self.output_vbox.addLayout(output_type_hbox)
self.output_vbox.addLayout(output_folder_hbox)
self.output_vbox.addLayout(output_filename_hbox)
self.output_vbox_widget = QWidget()
self.output_vbox_widget.setLayout(self.output_vbox)
self.output_vbox_widget.hide()
vbox = QVBoxLayout()
vbox.addWidget(self.error_label)
vbox.addWidget(self.input_gci_label)
vbox.addWidget(self.input_gci_checksum_label)
vbox.addWidget(self.gci_fields_widget)
vbox.addWidget(self.output_vbox_widget)
self.setLayout(vbox)
self.setWindowTitle("F-Zero GX GCI info")
self.show()
def clear_error_display(self):
self.error_label.setText("")
def display_error(self, message):
self.error_label.setText(message)
def show_output_folder_select_dialog(self):
folder = self.output_folder_select_dialog.getExistingDirectory(
self, 'Choose folder')
if folder:
self.output_folder_label.setText(folder)
self.output_folder_label.setToolTip(folder)
config.set('output_folder', folder)
def run_worker_job(self, job_func):
if not self.worker_thread:
# Create a thread.
# Must store a reference to the thread in a non-local variable,
# so the thread doesn't get garbage collected after returning
# from this method
# https://stackoverflow.com/a/15702922/
self.worker_thread = WorkerThread()
# This will emit 'started' and start running the thread
self.worker_thread.run_job(job_func)
def closeEvent(self, e):
"""Event handler: GUI window is closed"""
config.save()
def dragEnterEvent(self, e):
"""Event handler: Mouse enters the GUI window while dragging
something"""
e.accept()
def dropEvent(self, e):
"""
Event handler: Mouse is released on the GUI window after dragging.
Check that we've dragged a single .gci file. If so, process it.
"""
self.clear_error_display()
mime_data = e.mimeData()
dropped_uris = [uri.toString() for uri in mime_data.urls()]
if len(dropped_uris) > 1:
self.display_error("Please drag and drop a single file.")
return
uri = urllib.parse.urlparse(dropped_uris[0])
uri_scheme = uri.scheme
if uri_scheme != 'file':
self.display_error(
"Please drag and drop a file from your computer.")
return
# url2pathname() will:
# - Replace URL-like escape codes such as %E3%83%BB
# with unescaped Unicode characters.
# - Strip the beginning slash if it's a Windows filepath.
# (/D:/Games/... -> D:\Games\...)
input_gci_filepath = Path(urllib.request.url2pathname(uri.path))
if input_gci_filepath.suffix != '.gci':
self.display_error("The dropped file doesn't seem to be a .gci.")
return
self.input_gci_filepath = input_gci_filepath
self.process_input_gci()
def process_input_gci(self):
self.input_gci_label.setText("Working...")
self.input_gci = gci(self.input_gci_filepath)
input_data = self.input_gci.get_replay_data()
Field.reset_field_structures()
success = self.gci_fields_widget.read_fields_from_spec()
if not success:
self.input_gci_label.setText("Drag and drop a .gci to get started")
return
# Process the replay-specific GCI contents.
# This call can take a while, especially with custom machines.
self.run_worker_job(
functools.partial(self.gci_fields_widget.read_values_from_gci,
input_data))
@pyqtSlot()
def _after_process_input_gci(self):
self.gci_fields_widget.add_field_widgets()
self.input_gci_label.setText(f"Input: {self.input_gci_filepath}")
self.input_gci_label.setToolTip(f"Input: {self.input_gci_filepath}")
self.input_gci_checksum_label.setText(
f"\nChecksum: 0x{self.input_gci.get_checksum().hex()}")
self.output_filename_defaults = dict(
gci='output.gci',
replay_array=f'{self.input_gci_filepath.stem}__replay_array.bin',
)
self.output_vbox_widget.show()
self.on_output_type_change()
@property
def output_type(self):
combo_box_text = self.output_type_combo_box.currentText()
if combo_box_text == ".gci":
return 'gci'
elif combo_box_text == "Replay array .bin":
return 'replay_array'
@property
def output_folder(self):
return config.get('output_folder', '')
@property
def output_filename(self):
fn = config.get(f'output_filename_{self.output_type}')
if not fn:
fn = self.output_filename_defaults[self.output_type]
return fn
@property
def output_filepath(self):
return Path(self.output_folder, self.output_filename)
def on_output_type_change(self):
output_filename = config.get(f'output_filename_{self.output_type}', '')
self.output_filename_line_edit.setText(output_filename)
self.output_filename_line_edit.setPlaceholderText(
self.output_filename_defaults[self.output_type])
def on_output_filename_change(self):
config.set(f'output_filename_{self.output_type}',
self.output_filename_line_edit.text())
def on_output_button_click(self):
self.clear_error_display()
if not self.output_folder:
self.display_error("Please select an output folder.")
return
if self.output_type == 'gci':
self.output_gci()
elif self.output_type == 'replay_array':
self.output_replay_array()
def output_gci(self):
# Re-encode the replay data
new_replay_data = self.gci_fields_widget.output_values_to_gci()
# Zero-pad to make the entire GCI a multiple of 0x2000 bytes + 0x40.
# This replay data does not include the first 0x20A0 bytes of the GCI.
gci_bytes_without_padding = len(new_replay_data) + 0x20A0
gci_target_blocks = math.ceil(
(gci_bytes_without_padding - 0x40) / 0x2000)
gci_target_bytes = (gci_target_blocks * 0x2000) + 0x40
zero_padding = [0] * (gci_target_bytes - gci_bytes_without_padding)
new_replay_data.extend(zero_padding)
# Concatenate new replay data to the rest of the original GCI
self.input_gci.set_replay_data(new_replay_data)
# Recompute the checksum of the whole GCI
self.input_gci.recompute_checksum()
# Write the new GCI to a file
self.output_binary_file(self.input_gci.raw_bytes)
# Success dialog
button_box = QDialogButtonBox(QDialogButtonBox.Ok)
vbox = QVBoxLayout()
vbox.addWidget(
QLabel(f"GCI written to: {self.output_filepath}"
f"\nChecksum: 0x{self.input_gci.get_checksum().hex()}"))
vbox.addWidget(button_box)
confirmed_dialog = QDialog(self)
confirmed_dialog.setWindowTitle("Write success")
confirmed_dialog.setLayout(vbox)
# Make the OK button 'accept' the dialog
button_box.accepted.connect(confirmed_dialog.accept)
# Show the dialog
confirmed_dialog.exec()
def output_replay_array(self):
self.display_error("Replay-array output mode isn't implemented yet.")
def output_binary_file(self, bytes_to_write):
with open(self.output_filepath, "wb") as output_file:
output_file.write(bytes_to_write)
