コード例 #1
0
ファイル: FarmView.py プロジェクト: david2777/DPS_Hydra
    def __init__(self):
        QMainWindow.__init__(self)
        self.setupUi(self)

        #Class Variables
        self.thisNodeName = Utils.myHostName()
        logger.info("This host is %s", self.thisNodeName)
        self.username = Utils.getInfoFromCFG("database", "username")
        self.userFilter = False
        self.showArchivedFilter = False
        self.statusMsg = "ERROR"
        self.currentJobSel = None

        with open(Utils.findResource("styleSheet.css"), "r") as myStyles:
            self.setStyleSheet(myStyles.read())

        #My UI Setup Functions
        self.setupTreeViews()
        self.connectButtons()
        self.setupHotkeys()
        self.setWindowIcon(QIcon(Utils.findResource("Images/FarmView.png")))

        #Make sure this node exists
        self.thisNodeButtonsEnabled = True
        self.thisNodeExists = self.findThisNode()

        #Setup Signals
        SIGNAL("doUpdate")
        QObject.connect(self, SIGNAL("doUpdate"), self.doUpdate)

        #Start autoUpdater and then fetch data from DB
        self.autoUpdateThread = stoppableThread(self.doUpdateSignaler, 10, "AutoUpdate_Thread")
        self.doFetch()
コード例 #2
0
 def __append_to_file(self, root_dir, json, symbol):
     date = self.__format_date_from_timestamp()
     path = os.path.join(root_dir, 'BINANCE', date)
     Utils.create_dir_if_not_exist(path)
     file_path = os.path.join(path, symbol.upper())
     with open(file_path, 'a+') as file:
         file.write(str(json[0]) + '\n')
         self.logger.debug("Appended data for symbol %s at %s" %
                           (symbol, file_path))
コード例 #3
0
ファイル: RenderNode.py プロジェクト: david2777/DPS_Hydra
def softwareUpdaterLoop():
    """Checks for a new verison in the HYDRA environ, if one is found it starts
    a batch process to start the new verison and kills the current one running."""
    logger.debug("Checking for updates...")
    updateAnswer = Utils.softwareUpdater()
    if updateAnswer:
        logger.debug("Update found!")
        Utils.launchHydraApp("RenderNodeConsole", 10)
        socketServer.shutdown()
        sys.exit(0)
    else:
        logger.debug("No updates found")
コード例 #4
0
ファイル: Test.py プロジェクト: Shlomi643/NLP
def get_formatter(name):
    tmp = Utils.formatter(name)
    return FirstFormatter(name) if tmp == '1' else \
        SecondFormatter(name) if tmp == '2' else \
        ThirdFormatter(name) if tmp == '3' else \
        FourthFormatter(name) if tmp == '4' else  \
        FifthFormatter(name) if tmp == '5' else SixthFormatter(name)
コード例 #5
0
ファイル: RenderNode.py プロジェクト: david2777/DPS_Hydra
def heartbeat():
    """Updates a column on the node's database with the current time, signifying
    that the node software is still running."""
    host = Utils.myHostName()
    with transaction() as t:
        t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() WHERE host = %s",
                    (host,))
コード例 #6
0
 def run(self):
     while True:
         try:
             self.__pull_data_from_huobi(self.parent_dir, self.time_window)
         except Exception as e:
             self.logger.error("Unexpected Exception: " + str(e))
         time.sleep(Utils.get_seconds_from_time_window(self.time_window))
コード例 #7
0
ファイル: RenderNode.py プロジェクト: david2777/DPS_Hydra
    def __init__(self):
        #Setup Class Variables
        self.renderThread = None
        self.childProcess = None
        self.PSUtilProc = None
        self.statusAfterDeath = None
        self.childKilled = 0
        self.HydraJob = None
        self.HydraTask = None
        self.logPath = None

        #Get this node data from the database and make sure it exists
        self.thisNode = getThisNodeOBJ()
        logger.debug(self.thisNode)
        if not self.thisNode:
            logger.critical(
                "This node does not exist in the database! Please Register this node and try again."
            )
            sys.exit(-1)
            return

        #Detect RedShift GPUs
        self.rsGPUs = Utils.getRedshiftPreference("SelectedCudaDevices")
        if self.rsGPUs:
            self.rsGPUs = self.rsGPUs.split(",")[:-1]
            self.rsGPUids = [x.split(":")[0] for x in self.rsGPUs]
            if len(self.rsGPUs) != len(self.rsGPUids):
                logger.warning("Problems parsing Redshift Preferences")
            logger.info("%s Redshift Enabled GPU(s) found on this node",
                        len(self.rsGPUs))
            logger.debug("GPUs available for rendering are %s", self.rsGPUs)
        else:
            logger.warning("Could not find available Redshift GPUs")

        #Create RenderLog Directory if it doesn't exit
        if not os.path.isdir(Constants.RENDERLOGDIR):
            os.makedirs(Constants.RENDERLOGDIR)

        self.unstickTask()
        self.thisNode.software_version = Constants.VERSION

        with transaction() as t:
            self.thisNode.update(t)

        #Run The Server
        port = int(Utils.getInfoFromCFG("network", "port"))
        self.startServerThread(port)
コード例 #8
0
ファイル: Formatter2.py プロジェクト: Shlomi643/NLP
 def get_words_num(self):
     my_map = self.script_map
     ret = {}
     for col in my_map:
         x = col['person']
         count = len(Utils.get_token(col['text']))
         ret[x] = count if x not in ret.keys() else ret[x] + count
     return ret
コード例 #9
0
def qtPrompt():
    app = QApplication(sys.argv)
    loginWin = DatabaseLogin()
    loginWin.show()
    app.exec_()
    username, _password = loginWin.getValues()
    autoLogin = Utils.getInfoFromCFG("database", "autologin")
    autoLogin = True if str(autoLogin).lower().startswith("t") else False
    if username and autoLogin:
        updateAutologinUser(username)
        storeCredentials(username, _password)
    return username, _password
コード例 #10
0
ファイル: MySQLSetup.py プロジェクト: david2777/DPS_Hydra
def getDatabaseInfo():
    logger.debug("Finding login information...")

    #Get databse information
    host = Utils.getInfoFromCFG("database", "host")
    domain = Utils.getInfoFromCFG("network", "dnsDomainExtension").replace(" ", "")
    if domain != "" and host != "localhost":
        host += ".{}".format(domain)
    databaseName = Utils.getInfoFromCFG("database", "db")
    port = int(Utils.getInfoFromCFG("database", "port"))
    db_username = Utils.getInfoFromCFG("database", "username")

    #Get login information
    autoLogin = Utils.getInfoFromCFG("database", "autologin")
    autoLogin = True if str(autoLogin).lower()[0] == "t" else False
    if autoLogin:
        _db_password = PasswordStorage.loadCredentials(db_username)
        if not _db_password:
            autoLogin = False

    if not autoLogin:
        returnValues = PasswordStorage.qtPrompt()
        if not returnValues[0] or not returnValues[1]:
            logger.error("Could not login!")
            sys.exit(1)
        else:
            db_username = returnValues[0]
            _db_password = returnValues[1]

    return host, db_username, _db_password, databaseName, port
コード例 #11
0
ファイル: MySQLSetup.py プロジェクト: david2777/DPS_Hydra
 def getLogPath(self):
     thisHost = Utils.myHostName()
     path = os.path.join(Constants.RENDERLOGDIR, '{:0>10}.log.txt'.format(self.id))
     if thisHost == self.host:
         return path
     else:
         _, shortPath = os.path.splitdrive(path)
         newPath = os.path.join("\\\\{}".format(self.host), shortPath)
         newPath = os.path.normpath(newPath)
         if sys.platform == "win32":
             return "\\{}".format(newPath)
         else:
             return newPath
コード例 #12
0
def consolePrompt():
    print "\n\nStore Auto Login information?"
    #Get Login Information
    username = raw_input("Username: "******"Password: "******"database", "host")
    domain = Utils.getInfoFromCFG("network", "dnsDomainExtension").replace(" ", "")
    if domain != "" and host != "localhost":
        host += ".{}".format(domain)
    databaseName = Utils.getInfoFromCFG("database", "db")
    port = int(Utils.getInfoFromCFG("database", "port"))

    #Check  if login is valid
    try:
        MySQLdb.connect(host=host, user=username, passwd=_password,
                        db=databaseName, port=port)
        storeCredentials(username, _password)
        updateAutologinUser(username)

    except MySQLdb.Error:
        print "Login information was invalid!"
コード例 #13
0
ファイル: RenderNodeMain.py プロジェクト: david2777/DPS_Hydra
    def __init__(self):
        QMainWindow.__init__(self)
        self.setupUi(self)

        with open(Utils.findResource("styleSheet.css"), "r") as myStyles:
            self.setStyleSheet(myStyles.read())

        self.thisNode = NodeUtils.getThisNodeOBJ()
        self.isVisable = True

        self.pulseThreadStatus = False
        self.renderServerStatus = False
        self.schedThreadStatus = False
        self.autoUpdateStatus = False

        if not self.thisNode:
            self.offlineButton.setEnabled(False)
            self.getoffButton.setEnabled(False)
            logger.error("Node does not exist in database!")
            aboutBox(self, "Error",
                "This node was not found in the database! If you wish to render  "
                "on this node it must be registered with the databse. Run "
                "Register.exe or Register.py to regiester this node and "
                " try again.")
            sys.exit(1)

        self.currentSchedule = self.thisNode.weekSchedule
        self.currentScheduleEnabled = self.thisNode.scheduleEnabled

        self.buildUI()
        self.connectButtons()
        self.updateThisNodeInfo()
        self.startupServers()

        logger.info("Render Node Main is live! Waiting for tasks...")

        try:
            autoHide = True if str(sys.argv[1]).lower() == "true" else False
            logger.info(autoHide)
        except IndexError:
            autoHide = False

        if autoHide and self.trayIconBool:
            logger.info("Autohide is enabled!")
            self.sendToTrayHandler()
        else:
            self.show()
コード例 #14
0
ファイル: Formatter2.py プロジェクト: Shlomi643/NLP
 def get_tokens(self):
     ret = []
     for col in self.script_map:
         ret += Utils.get_token(col['text'])  # col['text'].lower().split()
     return ret
コード例 #15
0
ファイル: Formatter2.py プロジェクト: Shlomi643/NLP
 def get_char_tokens(self, person):
     ret = []
     for col in self.script_map:
         if col['person'].upper() == person.upper():
             ret += Utils.get_token(col['text'])
     return ret
コード例 #16
0
ファイル: Formatter2.py プロジェクト: Shlomi643/NLP
 def __init__(self, name):
     self.movie_name = name
     self.soup = bs(Utils.script(name), features='html.parser')
     self.script_map = Formatter.get_script_map(self.soup)
コード例 #17
0
ファイル: Test.py プロジェクト: Shlomi643/NLP
from Formatter1 import Formatter as FirstFormatter
from Formatter2 import Formatter as SecondFormatter
from Formatter3 import Formatter as ThirdFormatter
from Formatter4 import Formatter as FourthFormatter
from Formatter5 import Formatter as FifthFormatter
from Formatter6 import Formatter as SixthFormatter
from gensim.models import Word2Vec as wv, Word2Vec
from gensim.models import KeyedVectors as kv
import numpy as np
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
from prettytable import PrettyTable

model = kv.load_word2vec_format('GoogleNews-vectors-negative300.bin',
                                binary=True)
utils = Utils()

stop_words = [
    "i", "i'm", "it's", "me", "my", "myself", "we", "our", "ours", "ourselves",
    "you", "your", "yours", "yourself", "yourselves", "he", "him", "his",
    "himself", "she", "her", "hers", "herself", "it", "its", "itself", "they",
    "them", "their", "theirs", "themselves", "what", "which", "who", "whom",
    "this", "that", "these", "those", "am", "is", "are", "was", "were", "be",
    "been", "being", "have", "has", "had", "having", "do", "does", "did",
    "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as",
    "until", "while", "of", "at", "by", "for", "with", "about", "against",
    "between", "into", "through", "during", "before", "after", "above",
    "below", "to", "from", "up", "down", "in", "out", "on", "off", "over",
    "under", "again", "further", "then", "once", "here", "there", "when",
    "where", "why", "how", "all", "any", "both", "each", "few", "more", "most",
    "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so",
コード例 #18
0
                    "translate": True,
                    "has_folder": False,
                    "strings": [],
                    "dicts": [],
                    "dict_arrays": ['history'],
                    "special_characters": None,
                    "v": [],
                    "s": [],
                    "episodeid": 0
                }
            }
        }
    }
}

utils = Utils()
config = utils.create_config(default_config)
utils.format_file('assets.bin', input_path=config['input_path'])
utils.unzip_assets(config)
utils.remove_file(config['input_path'], 'assets.zip')
utils.refresh_output('{0}\\{1}'.format(config['input_path'], 'assets'),'{0}\\{1}'.format(config['output_path'], 'assets'))

for game in config['games'].keys():
    if config['games'][game]['translate']:
        print('Translating: {0}'.format(game))
        gamesManager = GamesManager(config, utils, game)
    else:
        print('{0} was skipped'.format(game))

utils.zip(config['output_path'])
utils.format_file('assets.zip', input_path=config['output_path'], output_path=config['input_path'])
コード例 #19
0
ファイル: MySQLSetup.py プロジェクト: david2777/DPS_Hydra
 def __setattr__(self, k, v):
     self.__dict__[k] = v
     if Utils.nonFlanged(k):
         self.__dirty__.add(k)
コード例 #20
0
ファイル: RenderNodeMain.py プロジェクト: david2777/DPS_Hydra
    def buildUI(self):
        def addItem(name, handler, statusTip, menu):
            action = QAction(name, self)
            action.setStatusTip(statusTip)
            action.triggered.connect(handler)
            menu.addAction(action)

        #Add Logging handlers for output field
        emStream = EmittingStream(textWritten=self.normalOutputWritten)
        handler = logging.StreamHandler(emStream)
        handler.setLevel(logging.INFO)
        handler.setFormatter(outputWindowFormatter)
        logger.addHandler(handler)

        sys.stdout = EmittingStream(textWritten=self.normalOutputWritten)
        sys.stderr = EmittingStream(textWritten=self.normalOutputWritten)

        #Get Pixmaps and Icon
        self.donePixmap = QPixmap(Utils.findResource("Images/status/done.png"))
        self.inProgPixmap = QPixmap(Utils.findResource("Images/status/inProgress.png"))
        self.needsAttentionPixmap = QPixmap(Utils.findResource("Images/status/needsAttention.png"))
        self.nonePixmap = QPixmap(Utils.findResource("Images/status/none.png"))
        self.notStartedPixmap = QPixmap(Utils.findResource("Images/status/notStarted.png"))
        self.refreshPixmap = QPixmap(Utils.findResource("Images/refresh.png"))
        self.refreshIcon = QIcon()
        self.refreshIcon.addPixmap(self.refreshPixmap)
        self.RIcon = QIcon(Utils.findResource("Images/RenderNodeMain.png"))

        self.isVisable = True

        self.refreshButton.setIcon(self.refreshIcon)

        self.renderServerPixmap.setPixmap(self.notStartedPixmap)
        self.scheduleThreadPixmap.setPixmap(self.notStartedPixmap)
        self.pulseThreadPixmap.setPixmap(self.notStartedPixmap)
        self.setWindowIcon(self.RIcon)

        #Setup tray icon
        self.trayIcon = QSystemTrayIcon()
        self.trayIconBool = self.trayIcon.isSystemTrayAvailable()
        if self.trayIconBool:
            self.trayIcon.setIcon(self.RIcon)
            self.trayIcon.show()
            self.trayIcon.setVisible(True)
            self.trayIcon.activated.connect(self.activate)
            self.trayIcon.messageClicked.connect(self.activate)

            #Tray Icon Context Menu
            self.taskIconMenu = QMenu(self)

            addItem("Open", self.showWindowHandler,
                    "Show the RenderNodeMain Window", self.taskIconMenu)
            self.taskIconMenu.addSeparator()
            addItem("Update", self.updateThisNodeInfo,
                    "Fetch the latest information from the Database", self.taskIconMenu)
            self.taskIconMenu.addSeparator()
            addItem("Online", self.onlineThisNodeHandler,
                    "Online this node", self.taskIconMenu)
            addItem("Offline", self.offlineThisNodeHandler,
                    "Offline this node", self.taskIconMenu)
            addItem("GetOff!", self.getOffThisNodeHandler,
                    "Kill the current task and offline this node", self.taskIconMenu)

            self.trayIcon.setContextMenu(self.taskIconMenu)
        else:
            logger.error("Tray Icon Error! Could not create tray icon.")
            aboutBox(self, "Tray Icon Error",
                    "Could not create tray icon. Minimizing to tray has been disabled.")
            self.trayButton.setEnabled(False)
コード例 #21
0
ファイル: RenderNodeMain.py プロジェクト: david2777/DPS_Hydra
def pulse():
    host = Utils.myHostName()
    with transaction() as t:
        t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() "
                    "WHERE host = '{0}'".format(host))
コード例 #22
0
ファイル: Register.py プロジェクト: david2777/DPS_Hydra
"""Registers a node with the database."""
#Standard
import os
import sys

#Third Party
#pylint: disable=E0611
from MySQLdb import IntegrityError

#Hydra
from Setups.LoggingSetup import logger
from Setups.MySQLSetup import hydra_rendernode, OFFLINE, transaction
import Utilities.Utils as Utils

if __name__ == "__main__":
    me = Utils.myHostName()
    hydraPath, execFile = os.path.split(sys.argv[0])
    logger.info(hydraPath)
    response = Utils.changeHydraEnviron(hydraPath)
    if response:
        try:
            with transaction() as t:
                hydra_rendernode(host=me, status=OFFLINE, minPriority=0).insert(t)
        except IntegrityError:
            logger.info("Host %s already exists in the hydra_rendernode table on the databse", me)
    else:
        logger.error("Could not set Hydra Environ! No changes where made. Exiting...")

    raw_input("\nPress enter to exit...")
コード例 #23
0
                    "v": [3, 5, 7, 8, 9, 10, 12, 14],
                    "s": [15, 16, 17],
                    "episodeid": 0
                },
                'SSWeapons': {
                    "translate": True,
                    "has_folder": False,
                    "strings": ['title', 'description'],
                    "dicts": [],
                    "dict_arrays": [],
                    "special_characters": [['<', '>']],
                    "v": [],
                    "s": [],
                    "episodeid": 0
                }
            }
        }
    }
}

utils = Utils()
config = utils.create_config(default_config)
# utils.refresh_output(config['input_path'], config['output_path'])

for game in config['games'].keys():
    if config['games'][game]['translate']:
        print('Translating: {0}'.format(game))
        gamesManager = GamesManager(config, utils, game)
    else:
        print('{0} was skipped'.format(game))
コード例 #24
0
ファイル: main.py プロジェクト: tux2603/SpaceNEAT
    }

    keyboard = key.KeyStateHandler()
    oldKeyboard = key.KeyStateHandler()
    gameWindow.push_handlers(keyboard)

    # Load in the resources
    loadResources()

    alienBatch = pyglet.graphics.Batch()

    shipGroup = pyglet.graphics.OrderedGroup(10)
    shieldGroup = pyglet.graphics.OrderedGroup(20)
    overlayGroup = pyglet.graphics.OrderedGroup(30)

    playerShipColored = Utils.colorTexture(playerShipTexture, (120, 120, 120),
                                           (40, 40, 200), (255, 128, 0))
    player = Player(playerShipColored,
                    alienShieldTexture,
                    alienBatch,
                    shipGroup,
                    shieldGroup,
                    x=screenWidth / 2,
                    y=screenHeight / 2)
    player.velocity = np.array((100, 0), dtype=np.float32)
    rotationalSens = 200
    forwardAccel = 300
    reverseAccel = -200
    orthAccel = 150

    swarm1 = []
コード例 #25
0
def updateAutologinUser(newUsername):
    if Utils.getInfoFromCFG("database", "username") != newUsername:
        return Utils.writeInfoToCFG("database", "username", newUsername)
コード例 #26
0
ファイル: RenderNode.py プロジェクト: david2777/DPS_Hydra
    def launchRenderTask(self, HydraJob, HydraTask):
        """Does the actual rendering, then records the results on the database"""
        logger.info("Starting task with id %s on job with id %s", HydraTask.id, HydraJob.id)
        self.HydraJob = HydraJob
        self.HydraTask = HydraTask
        self.childKilled = 0
        self.statusAfterDeath = None
        self.childProcess = None
        self.PSUtilProc = None

        originalCurrentFrame = int(self.HydraTask.currentFrame)
        renderTaskCMD = self.HydraTask.createTaskCMD(self.HydraJob, sys.platform)
        logger.debug(renderTaskCMD)

        self.logPath = self.HydraTask.getLogPath()
        logger.info("Starting render task %s", self.HydraTask.id)
        try:
            log = file(self.logPath, 'w')
        except (IOError, OSError, WindowsError) as e:
            logger.error(e)
            self.thisNode.getOff()
            return
        log.write('Hydra log file {0} on {1}\n'.format(self.logPath, self.HydraTask.host))
        log.write('RenderNode is {0}\n'.format(sys.argv))
        log.write('Command: {0}\n\n'.format(renderTaskCMD))
        Utils.flushOut(log)

        progressUpdateThread = stoppableThread(self.progressUpdate, 300,
                                                "Progress_Update_Thread")

        #Run the job and keep track of the process
        self.childProcess = subprocess.Popen(renderTaskCMD,
                                            stdout=log, stderr=log,
                                            **Utils.buildSubprocessArgs(False))

        logger.info("Started PID %s to do Task %s", self.childProcess.pid, self.HydraTask.id)

        self.PSUtilProc = psutil.Process(self.childProcess.pid)
        #Wait for task to finish
        self.childProcess.communicate()

        #Get Exit Code, Record the results
        self.HydraTask.exitCode = self.childProcess.returncode if self.childProcess else 1234
        logString = "\nProcess exited with code {0} at {1} on {2}\n"
        nowTime = datetime.datetime.now().replace(microsecond=0)
        log.write(logString.format(self.HydraTask.exitCode, nowTime,
                                    self.thisNode.host))

        progressUpdateThread.terminate()

        #Update HydraTask and HydraJob with currentFrame, MPF, and RLTracker
        self.progressUpdate(commit=False)

        #EndTime
        self.HydraTask.endTime = datetime.datetime.now()

        #Work around for batch files
        if self.HydraJob.jobType == "BatchFile" and self.HydraTask.exitCode == 0:
            self.HydraTask.currentFrame = (self.HydraTask.endFrame + 1)
            self.HydraJob.renderLayerTracker = str((self.HydraTask.endFrame + 1))

        #Status, Attempts. Failures
        if self.childKilled == 1:
            self.HydraTask.status = self.statusAfterDeath
            self.HydraTask.exitCode = 1

        else:
            if self.HydraTask.exitCode == 0 and self.HydraTask.currentFrame >= originalCurrentFrame:
                status = FINISHED
            else:
                if self.HydraTask.exitCode == 0:
                    log.write("\n\nERROR: Task returned exit code 0 but it appears to have not actually rendered any frames.")
                status = ERROR
                self.HydraJob.attempts += 1
                if not self.HydraJob.failures or self.HydraJob.failures == "":
                    self.HydraJob.failures = self.thisNode.host
                else:
                    self.HydraJob.failures += ",{0}".format(self.thisNode.host)

            self.HydraTask.status = status

        #Update data on the DB
        with transaction() as t:
            self.HydraTask.update(t)
            self.HydraJob.update(t)

        self.resetThisNode()
        log.close()
        logger.info("Done with render task %s", self.HydraTask.id)
        self.childProcess = None
        self.PSUtilProc = None
        self.HydraJob = None
        self.HydraTask = None
        self.logPath = None