Exemplo n.º 1
0
    def __init__(self):
        QMainWindow.__init__(self)
        self.setupUi(self)

        #Class Variables
        self.thisNodeName = Utils.myHostName()
        logger.info("This host is %s", self.thisNodeName)
        self.username = Utils.getInfoFromCFG("database", "username")
        self.userFilter = False
        self.showArchivedFilter = False
        self.statusMsg = "ERROR"
        self.currentJobSel = None

        with open(Utils.findResource("styleSheet.css"), "r") as myStyles:
            self.setStyleSheet(myStyles.read())

        #My UI Setup Functions
        self.setupTreeViews()
        self.connectButtons()
        self.setupHotkeys()
        self.setWindowIcon(QIcon(Utils.findResource("Images/FarmView.png")))

        #Make sure this node exists
        self.thisNodeButtonsEnabled = True
        self.thisNodeExists = self.findThisNode()

        #Setup Signals
        SIGNAL("doUpdate")
        QObject.connect(self, SIGNAL("doUpdate"), self.doUpdate)

        #Start autoUpdater and then fetch data from DB
        self.autoUpdateThread = stoppableThread(self.doUpdateSignaler, 10, "AutoUpdate_Thread")
        self.doFetch()
 def __append_to_file(self, root_dir, json, symbol):
     date = self.__format_date_from_timestamp()
     path = os.path.join(root_dir, 'BINANCE', date)
     Utils.create_dir_if_not_exist(path)
     file_path = os.path.join(path, symbol.upper())
     with open(file_path, 'a+') as file:
         file.write(str(json[0]) + '\n')
         self.logger.debug("Appended data for symbol %s at %s" %
                           (symbol, file_path))
Exemplo n.º 3
0
def softwareUpdaterLoop():
    """Checks for a new verison in the HYDRA environ, if one is found it starts
    a batch process to start the new verison and kills the current one running."""
    logger.debug("Checking for updates...")
    updateAnswer = Utils.softwareUpdater()
    if updateAnswer:
        logger.debug("Update found!")
        Utils.launchHydraApp("RenderNodeConsole", 10)
        socketServer.shutdown()
        sys.exit(0)
    else:
        logger.debug("No updates found")
Exemplo n.º 4
0
def get_formatter(name):
    tmp = Utils.formatter(name)
    return FirstFormatter(name) if tmp == '1' else \
        SecondFormatter(name) if tmp == '2' else \
        ThirdFormatter(name) if tmp == '3' else \
        FourthFormatter(name) if tmp == '4' else  \
        FifthFormatter(name) if tmp == '5' else SixthFormatter(name)
Exemplo n.º 5
0
def heartbeat():
    """Updates a column on the node's database with the current time, signifying
    that the node software is still running."""
    host = Utils.myHostName()
    with transaction() as t:
        t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() WHERE host = %s",
                    (host,))
Exemplo n.º 6
0
 def run(self):
     while True:
         try:
             self.__pull_data_from_huobi(self.parent_dir, self.time_window)
         except Exception as e:
             self.logger.error("Unexpected Exception: " + str(e))
         time.sleep(Utils.get_seconds_from_time_window(self.time_window))
Exemplo n.º 7
0
    def __init__(self):
        #Setup Class Variables
        self.renderThread = None
        self.childProcess = None
        self.PSUtilProc = None
        self.statusAfterDeath = None
        self.childKilled = 0
        self.HydraJob = None
        self.HydraTask = None
        self.logPath = None

        #Get this node data from the database and make sure it exists
        self.thisNode = getThisNodeOBJ()
        logger.debug(self.thisNode)
        if not self.thisNode:
            logger.critical(
                "This node does not exist in the database! Please Register this node and try again."
            )
            sys.exit(-1)
            return

        #Detect RedShift GPUs
        self.rsGPUs = Utils.getRedshiftPreference("SelectedCudaDevices")
        if self.rsGPUs:
            self.rsGPUs = self.rsGPUs.split(",")[:-1]
            self.rsGPUids = [x.split(":")[0] for x in self.rsGPUs]
            if len(self.rsGPUs) != len(self.rsGPUids):
                logger.warning("Problems parsing Redshift Preferences")
            logger.info("%s Redshift Enabled GPU(s) found on this node",
                        len(self.rsGPUs))
            logger.debug("GPUs available for rendering are %s", self.rsGPUs)
        else:
            logger.warning("Could not find available Redshift GPUs")

        #Create RenderLog Directory if it doesn't exit
        if not os.path.isdir(Constants.RENDERLOGDIR):
            os.makedirs(Constants.RENDERLOGDIR)

        self.unstickTask()
        self.thisNode.software_version = Constants.VERSION

        with transaction() as t:
            self.thisNode.update(t)

        #Run The Server
        port = int(Utils.getInfoFromCFG("network", "port"))
        self.startServerThread(port)
Exemplo n.º 8
0
 def get_words_num(self):
     my_map = self.script_map
     ret = {}
     for col in my_map:
         x = col['person']
         count = len(Utils.get_token(col['text']))
         ret[x] = count if x not in ret.keys() else ret[x] + count
     return ret
Exemplo n.º 9
0
def qtPrompt():
    app = QApplication(sys.argv)
    loginWin = DatabaseLogin()
    loginWin.show()
    app.exec_()
    username, _password = loginWin.getValues()
    autoLogin = Utils.getInfoFromCFG("database", "autologin")
    autoLogin = True if str(autoLogin).lower().startswith("t") else False
    if username and autoLogin:
        updateAutologinUser(username)
        storeCredentials(username, _password)
    return username, _password
Exemplo n.º 10
0
def getDatabaseInfo():
    logger.debug("Finding login information...")

    #Get databse information
    host = Utils.getInfoFromCFG("database", "host")
    domain = Utils.getInfoFromCFG("network", "dnsDomainExtension").replace(" ", "")
    if domain != "" and host != "localhost":
        host += ".{}".format(domain)
    databaseName = Utils.getInfoFromCFG("database", "db")
    port = int(Utils.getInfoFromCFG("database", "port"))
    db_username = Utils.getInfoFromCFG("database", "username")

    #Get login information
    autoLogin = Utils.getInfoFromCFG("database", "autologin")
    autoLogin = True if str(autoLogin).lower()[0] == "t" else False
    if autoLogin:
        _db_password = PasswordStorage.loadCredentials(db_username)
        if not _db_password:
            autoLogin = False

    if not autoLogin:
        returnValues = PasswordStorage.qtPrompt()
        if not returnValues[0] or not returnValues[1]:
            logger.error("Could not login!")
            sys.exit(1)
        else:
            db_username = returnValues[0]
            _db_password = returnValues[1]

    return host, db_username, _db_password, databaseName, port
Exemplo n.º 11
0
 def getLogPath(self):
     thisHost = Utils.myHostName()
     path = os.path.join(Constants.RENDERLOGDIR, '{:0>10}.log.txt'.format(self.id))
     if thisHost == self.host:
         return path
     else:
         _, shortPath = os.path.splitdrive(path)
         newPath = os.path.join("\\\\{}".format(self.host), shortPath)
         newPath = os.path.normpath(newPath)
         if sys.platform == "win32":
             return "\\{}".format(newPath)
         else:
             return newPath
Exemplo n.º 12
0
def consolePrompt():
    print "\n\nStore Auto Login information?"
    #Get Login Information
    username = raw_input("Username: "******"Password: "******"database", "host")
    domain = Utils.getInfoFromCFG("network", "dnsDomainExtension").replace(" ", "")
    if domain != "" and host != "localhost":
        host += ".{}".format(domain)
    databaseName = Utils.getInfoFromCFG("database", "db")
    port = int(Utils.getInfoFromCFG("database", "port"))

    #Check  if login is valid
    try:
        MySQLdb.connect(host=host, user=username, passwd=_password,
                        db=databaseName, port=port)
        storeCredentials(username, _password)
        updateAutologinUser(username)

    except MySQLdb.Error:
        print "Login information was invalid!"
Exemplo n.º 13
0
    def __init__(self):
        QMainWindow.__init__(self)
        self.setupUi(self)

        with open(Utils.findResource("styleSheet.css"), "r") as myStyles:
            self.setStyleSheet(myStyles.read())

        self.thisNode = NodeUtils.getThisNodeOBJ()
        self.isVisable = True

        self.pulseThreadStatus = False
        self.renderServerStatus = False
        self.schedThreadStatus = False
        self.autoUpdateStatus = False

        if not self.thisNode:
            self.offlineButton.setEnabled(False)
            self.getoffButton.setEnabled(False)
            logger.error("Node does not exist in database!")
            aboutBox(self, "Error",
                "This node was not found in the database! If you wish to render  "
                "on this node it must be registered with the databse. Run "
                "Register.exe or Register.py to regiester this node and "
                " try again.")
            sys.exit(1)

        self.currentSchedule = self.thisNode.weekSchedule
        self.currentScheduleEnabled = self.thisNode.scheduleEnabled

        self.buildUI()
        self.connectButtons()
        self.updateThisNodeInfo()
        self.startupServers()

        logger.info("Render Node Main is live! Waiting for tasks...")

        try:
            autoHide = True if str(sys.argv[1]).lower() == "true" else False
            logger.info(autoHide)
        except IndexError:
            autoHide = False

        if autoHide and self.trayIconBool:
            logger.info("Autohide is enabled!")
            self.sendToTrayHandler()
        else:
            self.show()
Exemplo n.º 14
0
 def get_tokens(self):
     ret = []
     for col in self.script_map:
         ret += Utils.get_token(col['text'])  # col['text'].lower().split()
     return ret
Exemplo n.º 15
0
 def get_char_tokens(self, person):
     ret = []
     for col in self.script_map:
         if col['person'].upper() == person.upper():
             ret += Utils.get_token(col['text'])
     return ret
Exemplo n.º 16
0
 def __init__(self, name):
     self.movie_name = name
     self.soup = bs(Utils.script(name), features='html.parser')
     self.script_map = Formatter.get_script_map(self.soup)
Exemplo n.º 17
0
from Formatter1 import Formatter as FirstFormatter
from Formatter2 import Formatter as SecondFormatter
from Formatter3 import Formatter as ThirdFormatter
from Formatter4 import Formatter as FourthFormatter
from Formatter5 import Formatter as FifthFormatter
from Formatter6 import Formatter as SixthFormatter
from gensim.models import Word2Vec as wv, Word2Vec
from gensim.models import KeyedVectors as kv
import numpy as np
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
from prettytable import PrettyTable

model = kv.load_word2vec_format('GoogleNews-vectors-negative300.bin',
                                binary=True)
utils = Utils()

stop_words = [
    "i", "i'm", "it's", "me", "my", "myself", "we", "our", "ours", "ourselves",
    "you", "your", "yours", "yourself", "yourselves", "he", "him", "his",
    "himself", "she", "her", "hers", "herself", "it", "its", "itself", "they",
    "them", "their", "theirs", "themselves", "what", "which", "who", "whom",
    "this", "that", "these", "those", "am", "is", "are", "was", "were", "be",
    "been", "being", "have", "has", "had", "having", "do", "does", "did",
    "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as",
    "until", "while", "of", "at", "by", "for", "with", "about", "against",
    "between", "into", "through", "during", "before", "after", "above",
    "below", "to", "from", "up", "down", "in", "out", "on", "off", "over",
    "under", "again", "further", "then", "once", "here", "there", "when",
    "where", "why", "how", "all", "any", "both", "each", "few", "more", "most",
    "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so",
                    "translate": True,
                    "has_folder": False,
                    "strings": [],
                    "dicts": [],
                    "dict_arrays": ['history'],
                    "special_characters": None,
                    "v": [],
                    "s": [],
                    "episodeid": 0
                }
            }
        }
    }
}

utils = Utils()
config = utils.create_config(default_config)
utils.format_file('assets.bin', input_path=config['input_path'])
utils.unzip_assets(config)
utils.remove_file(config['input_path'], 'assets.zip')
utils.refresh_output('{0}\\{1}'.format(config['input_path'], 'assets'),'{0}\\{1}'.format(config['output_path'], 'assets'))

for game in config['games'].keys():
    if config['games'][game]['translate']:
        print('Translating: {0}'.format(game))
        gamesManager = GamesManager(config, utils, game)
    else:
        print('{0} was skipped'.format(game))

utils.zip(config['output_path'])
utils.format_file('assets.zip', input_path=config['output_path'], output_path=config['input_path'])
Exemplo n.º 19
0
 def __setattr__(self, k, v):
     self.__dict__[k] = v
     if Utils.nonFlanged(k):
         self.__dirty__.add(k)
Exemplo n.º 20
0
    def buildUI(self):
        def addItem(name, handler, statusTip, menu):
            action = QAction(name, self)
            action.setStatusTip(statusTip)
            action.triggered.connect(handler)
            menu.addAction(action)

        #Add Logging handlers for output field
        emStream = EmittingStream(textWritten=self.normalOutputWritten)
        handler = logging.StreamHandler(emStream)
        handler.setLevel(logging.INFO)
        handler.setFormatter(outputWindowFormatter)
        logger.addHandler(handler)

        sys.stdout = EmittingStream(textWritten=self.normalOutputWritten)
        sys.stderr = EmittingStream(textWritten=self.normalOutputWritten)

        #Get Pixmaps and Icon
        self.donePixmap = QPixmap(Utils.findResource("Images/status/done.png"))
        self.inProgPixmap = QPixmap(Utils.findResource("Images/status/inProgress.png"))
        self.needsAttentionPixmap = QPixmap(Utils.findResource("Images/status/needsAttention.png"))
        self.nonePixmap = QPixmap(Utils.findResource("Images/status/none.png"))
        self.notStartedPixmap = QPixmap(Utils.findResource("Images/status/notStarted.png"))
        self.refreshPixmap = QPixmap(Utils.findResource("Images/refresh.png"))
        self.refreshIcon = QIcon()
        self.refreshIcon.addPixmap(self.refreshPixmap)
        self.RIcon = QIcon(Utils.findResource("Images/RenderNodeMain.png"))

        self.isVisable = True

        self.refreshButton.setIcon(self.refreshIcon)

        self.renderServerPixmap.setPixmap(self.notStartedPixmap)
        self.scheduleThreadPixmap.setPixmap(self.notStartedPixmap)
        self.pulseThreadPixmap.setPixmap(self.notStartedPixmap)
        self.setWindowIcon(self.RIcon)

        #Setup tray icon
        self.trayIcon = QSystemTrayIcon()
        self.trayIconBool = self.trayIcon.isSystemTrayAvailable()
        if self.trayIconBool:
            self.trayIcon.setIcon(self.RIcon)
            self.trayIcon.show()
            self.trayIcon.setVisible(True)
            self.trayIcon.activated.connect(self.activate)
            self.trayIcon.messageClicked.connect(self.activate)

            #Tray Icon Context Menu
            self.taskIconMenu = QMenu(self)

            addItem("Open", self.showWindowHandler,
                    "Show the RenderNodeMain Window", self.taskIconMenu)
            self.taskIconMenu.addSeparator()
            addItem("Update", self.updateThisNodeInfo,
                    "Fetch the latest information from the Database", self.taskIconMenu)
            self.taskIconMenu.addSeparator()
            addItem("Online", self.onlineThisNodeHandler,
                    "Online this node", self.taskIconMenu)
            addItem("Offline", self.offlineThisNodeHandler,
                    "Offline this node", self.taskIconMenu)
            addItem("GetOff!", self.getOffThisNodeHandler,
                    "Kill the current task and offline this node", self.taskIconMenu)

            self.trayIcon.setContextMenu(self.taskIconMenu)
        else:
            logger.error("Tray Icon Error! Could not create tray icon.")
            aboutBox(self, "Tray Icon Error",
                    "Could not create tray icon. Minimizing to tray has been disabled.")
            self.trayButton.setEnabled(False)
Exemplo n.º 21
0
def pulse():
    host = Utils.myHostName()
    with transaction() as t:
        t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() "
                    "WHERE host = '{0}'".format(host))
Exemplo n.º 22
0
"""Registers a node with the database."""
#Standard
import os
import sys

#Third Party
#pylint: disable=E0611
from MySQLdb import IntegrityError

#Hydra
from Setups.LoggingSetup import logger
from Setups.MySQLSetup import hydra_rendernode, OFFLINE, transaction
import Utilities.Utils as Utils

if __name__ == "__main__":
    me = Utils.myHostName()
    hydraPath, execFile = os.path.split(sys.argv[0])
    logger.info(hydraPath)
    response = Utils.changeHydraEnviron(hydraPath)
    if response:
        try:
            with transaction() as t:
                hydra_rendernode(host=me, status=OFFLINE, minPriority=0).insert(t)
        except IntegrityError:
            logger.info("Host %s already exists in the hydra_rendernode table on the databse", me)
    else:
        logger.error("Could not set Hydra Environ! No changes where made. Exiting...")

    raw_input("\nPress enter to exit...")
                    "v": [3, 5, 7, 8, 9, 10, 12, 14],
                    "s": [15, 16, 17],
                    "episodeid": 0
                },
                'SSWeapons': {
                    "translate": True,
                    "has_folder": False,
                    "strings": ['title', 'description'],
                    "dicts": [],
                    "dict_arrays": [],
                    "special_characters": [['<', '>']],
                    "v": [],
                    "s": [],
                    "episodeid": 0
                }
            }
        }
    }
}

utils = Utils()
config = utils.create_config(default_config)
# utils.refresh_output(config['input_path'], config['output_path'])

for game in config['games'].keys():
    if config['games'][game]['translate']:
        print('Translating: {0}'.format(game))
        gamesManager = GamesManager(config, utils, game)
    else:
        print('{0} was skipped'.format(game))
Exemplo n.º 24
0
    }

    keyboard = key.KeyStateHandler()
    oldKeyboard = key.KeyStateHandler()
    gameWindow.push_handlers(keyboard)

    # Load in the resources
    loadResources()

    alienBatch = pyglet.graphics.Batch()

    shipGroup = pyglet.graphics.OrderedGroup(10)
    shieldGroup = pyglet.graphics.OrderedGroup(20)
    overlayGroup = pyglet.graphics.OrderedGroup(30)

    playerShipColored = Utils.colorTexture(playerShipTexture, (120, 120, 120),
                                           (40, 40, 200), (255, 128, 0))
    player = Player(playerShipColored,
                    alienShieldTexture,
                    alienBatch,
                    shipGroup,
                    shieldGroup,
                    x=screenWidth / 2,
                    y=screenHeight / 2)
    player.velocity = np.array((100, 0), dtype=np.float32)
    rotationalSens = 200
    forwardAccel = 300
    reverseAccel = -200
    orthAccel = 150

    swarm1 = []
Exemplo n.º 25
0
def updateAutologinUser(newUsername):
    if Utils.getInfoFromCFG("database", "username") != newUsername:
        return Utils.writeInfoToCFG("database", "username", newUsername)
Exemplo n.º 26
0
    def launchRenderTask(self, HydraJob, HydraTask):
        """Does the actual rendering, then records the results on the database"""
        logger.info("Starting task with id %s on job with id %s", HydraTask.id, HydraJob.id)
        self.HydraJob = HydraJob
        self.HydraTask = HydraTask
        self.childKilled = 0
        self.statusAfterDeath = None
        self.childProcess = None
        self.PSUtilProc = None

        originalCurrentFrame = int(self.HydraTask.currentFrame)
        renderTaskCMD = self.HydraTask.createTaskCMD(self.HydraJob, sys.platform)
        logger.debug(renderTaskCMD)

        self.logPath = self.HydraTask.getLogPath()
        logger.info("Starting render task %s", self.HydraTask.id)
        try:
            log = file(self.logPath, 'w')
        except (IOError, OSError, WindowsError) as e:
            logger.error(e)
            self.thisNode.getOff()
            return
        log.write('Hydra log file {0} on {1}\n'.format(self.logPath, self.HydraTask.host))
        log.write('RenderNode is {0}\n'.format(sys.argv))
        log.write('Command: {0}\n\n'.format(renderTaskCMD))
        Utils.flushOut(log)

        progressUpdateThread = stoppableThread(self.progressUpdate, 300,
                                                "Progress_Update_Thread")

        #Run the job and keep track of the process
        self.childProcess = subprocess.Popen(renderTaskCMD,
                                            stdout=log, stderr=log,
                                            **Utils.buildSubprocessArgs(False))

        logger.info("Started PID %s to do Task %s", self.childProcess.pid, self.HydraTask.id)

        self.PSUtilProc = psutil.Process(self.childProcess.pid)
        #Wait for task to finish
        self.childProcess.communicate()

        #Get Exit Code, Record the results
        self.HydraTask.exitCode = self.childProcess.returncode if self.childProcess else 1234
        logString = "\nProcess exited with code {0} at {1} on {2}\n"
        nowTime = datetime.datetime.now().replace(microsecond=0)
        log.write(logString.format(self.HydraTask.exitCode, nowTime,
                                    self.thisNode.host))

        progressUpdateThread.terminate()

        #Update HydraTask and HydraJob with currentFrame, MPF, and RLTracker
        self.progressUpdate(commit=False)

        #EndTime
        self.HydraTask.endTime = datetime.datetime.now()

        #Work around for batch files
        if self.HydraJob.jobType == "BatchFile" and self.HydraTask.exitCode == 0:
            self.HydraTask.currentFrame = (self.HydraTask.endFrame + 1)
            self.HydraJob.renderLayerTracker = str((self.HydraTask.endFrame + 1))

        #Status, Attempts. Failures
        if self.childKilled == 1:
            self.HydraTask.status = self.statusAfterDeath
            self.HydraTask.exitCode = 1

        else:
            if self.HydraTask.exitCode == 0 and self.HydraTask.currentFrame >= originalCurrentFrame:
                status = FINISHED
            else:
                if self.HydraTask.exitCode == 0:
                    log.write("\n\nERROR: Task returned exit code 0 but it appears to have not actually rendered any frames.")
                status = ERROR
                self.HydraJob.attempts += 1
                if not self.HydraJob.failures or self.HydraJob.failures == "":
                    self.HydraJob.failures = self.thisNode.host
                else:
                    self.HydraJob.failures += ",{0}".format(self.thisNode.host)

            self.HydraTask.status = status

        #Update data on the DB
        with transaction() as t:
            self.HydraTask.update(t)
            self.HydraJob.update(t)

        self.resetThisNode()
        log.close()
        logger.info("Done with render task %s", self.HydraTask.id)
        self.childProcess = None
        self.PSUtilProc = None
        self.HydraJob = None
        self.HydraTask = None
        self.logPath = None