def __init__(self):
QMainWindow.__init__(self)
self.setupUi(self)
#Class Variables
self.thisNodeName = Utils.myHostName()
logger.info("This host is %s", self.thisNodeName)
self.username = Utils.getInfoFromCFG("database", "username")
self.userFilter = False
self.showArchivedFilter = False
self.statusMsg = "ERROR"
self.currentJobSel = None
with open(Utils.findResource("styleSheet.css"), "r") as myStyles:
self.setStyleSheet(myStyles.read())
#My UI Setup Functions
self.setupTreeViews()
self.connectButtons()
self.setupHotkeys()
self.setWindowIcon(QIcon(Utils.findResource("Images/FarmView.png")))
#Make sure this node exists
self.thisNodeButtonsEnabled = True
self.thisNodeExists = self.findThisNode()
#Setup Signals
SIGNAL("doUpdate")
QObject.connect(self, SIGNAL("doUpdate"), self.doUpdate)
#Start autoUpdater and then fetch data from DB
self.autoUpdateThread = stoppableThread(self.doUpdateSignaler, 10, "AutoUpdate_Thread")
self.doFetch()
def __append_to_file(self, root_dir, json, symbol):
date = self.__format_date_from_timestamp()
path = os.path.join(root_dir, 'BINANCE', date)
Utils.create_dir_if_not_exist(path)
file_path = os.path.join(path, symbol.upper())
with open(file_path, 'a+') as file:
file.write(str(json[0]) + '\n')
self.logger.debug("Appended data for symbol %s at %s" %
(symbol, file_path))
def softwareUpdaterLoop():
"""Checks for a new verison in the HYDRA environ, if one is found it starts
a batch process to start the new verison and kills the current one running."""
logger.debug("Checking for updates...")
updateAnswer = Utils.softwareUpdater()
if updateAnswer:
logger.debug("Update found!")
Utils.launchHydraApp("RenderNodeConsole", 10)
socketServer.shutdown()
sys.exit(0)
else:
logger.debug("No updates found")
def get_formatter(name):
tmp = Utils.formatter(name)
return FirstFormatter(name) if tmp == '1' else \
SecondFormatter(name) if tmp == '2' else \
ThirdFormatter(name) if tmp == '3' else \
FourthFormatter(name) if tmp == '4' else \
FifthFormatter(name) if tmp == '5' else SixthFormatter(name)
def heartbeat():
"""Updates a column on the node's database with the current time, signifying
that the node software is still running."""
host = Utils.myHostName()
with transaction() as t:
t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() WHERE host = %s",
(host,))
def run(self):
while True:
try:
self.__pull_data_from_huobi(self.parent_dir, self.time_window)
except Exception as e:
self.logger.error("Unexpected Exception: " + str(e))
time.sleep(Utils.get_seconds_from_time_window(self.time_window))
def __init__(self):
#Setup Class Variables
self.renderThread = None
self.childProcess = None
self.PSUtilProc = None
self.statusAfterDeath = None
self.childKilled = 0
self.HydraJob = None
self.HydraTask = None
self.logPath = None
#Get this node data from the database and make sure it exists
self.thisNode = getThisNodeOBJ()
logger.debug(self.thisNode)
if not self.thisNode:
logger.critical(
"This node does not exist in the database! Please Register this node and try again."
)
sys.exit(-1)
return
#Detect RedShift GPUs
self.rsGPUs = Utils.getRedshiftPreference("SelectedCudaDevices")
if self.rsGPUs:
self.rsGPUs = self.rsGPUs.split(",")[:-1]
self.rsGPUids = [x.split(":")[0] for x in self.rsGPUs]
if len(self.rsGPUs) != len(self.rsGPUids):
logger.warning("Problems parsing Redshift Preferences")
logger.info("%s Redshift Enabled GPU(s) found on this node",
len(self.rsGPUs))
logger.debug("GPUs available for rendering are %s", self.rsGPUs)
else:
logger.warning("Could not find available Redshift GPUs")
#Create RenderLog Directory if it doesn't exit
if not os.path.isdir(Constants.RENDERLOGDIR):
os.makedirs(Constants.RENDERLOGDIR)
self.unstickTask()
self.thisNode.software_version = Constants.VERSION
with transaction() as t:
self.thisNode.update(t)
#Run The Server
port = int(Utils.getInfoFromCFG("network", "port"))
self.startServerThread(port)
def get_words_num(self):
my_map = self.script_map
ret = {}
for col in my_map:
x = col['person']
count = len(Utils.get_token(col['text']))
ret[x] = count if x not in ret.keys() else ret[x] + count
return ret
def qtPrompt():
app = QApplication(sys.argv)
loginWin = DatabaseLogin()
loginWin.show()
app.exec_()
username, _password = loginWin.getValues()
autoLogin = Utils.getInfoFromCFG("database", "autologin")
autoLogin = True if str(autoLogin).lower().startswith("t") else False
if username and autoLogin:
updateAutologinUser(username)
storeCredentials(username, _password)
return username, _password
def getDatabaseInfo():
logger.debug("Finding login information...")
#Get databse information
host = Utils.getInfoFromCFG("database", "host")
domain = Utils.getInfoFromCFG("network", "dnsDomainExtension").replace(" ", "")
if domain != "" and host != "localhost":
host += ".{}".format(domain)
databaseName = Utils.getInfoFromCFG("database", "db")
port = int(Utils.getInfoFromCFG("database", "port"))
db_username = Utils.getInfoFromCFG("database", "username")
#Get login information
autoLogin = Utils.getInfoFromCFG("database", "autologin")
autoLogin = True if str(autoLogin).lower()[0] == "t" else False
if autoLogin:
_db_password = PasswordStorage.loadCredentials(db_username)
if not _db_password:
autoLogin = False
if not autoLogin:
returnValues = PasswordStorage.qtPrompt()
if not returnValues[0] or not returnValues[1]:
logger.error("Could not login!")
sys.exit(1)
else:
db_username = returnValues[0]
_db_password = returnValues[1]
return host, db_username, _db_password, databaseName, port
def getLogPath(self):
thisHost = Utils.myHostName()
path = os.path.join(Constants.RENDERLOGDIR, '{:0>10}.log.txt'.format(self.id))
if thisHost == self.host:
return path
else:
_, shortPath = os.path.splitdrive(path)
newPath = os.path.join("\\\\{}".format(self.host), shortPath)
newPath = os.path.normpath(newPath)
if sys.platform == "win32":
return "\\{}".format(newPath)
else:
return newPath
def consolePrompt():
print "\n\nStore Auto Login information?"
#Get Login Information
username = raw_input("Username: "******"Password: "******"database", "host")
domain = Utils.getInfoFromCFG("network", "dnsDomainExtension").replace(" ", "")
if domain != "" and host != "localhost":
host += ".{}".format(domain)
databaseName = Utils.getInfoFromCFG("database", "db")
port = int(Utils.getInfoFromCFG("database", "port"))
#Check if login is valid
try:
MySQLdb.connect(host=host, user=username, passwd=_password,
db=databaseName, port=port)
storeCredentials(username, _password)
updateAutologinUser(username)
except MySQLdb.Error:
print "Login information was invalid!"
def __init__(self):
QMainWindow.__init__(self)
self.setupUi(self)
with open(Utils.findResource("styleSheet.css"), "r") as myStyles:
self.setStyleSheet(myStyles.read())
self.thisNode = NodeUtils.getThisNodeOBJ()
self.isVisable = True
self.pulseThreadStatus = False
self.renderServerStatus = False
self.schedThreadStatus = False
self.autoUpdateStatus = False
if not self.thisNode:
self.offlineButton.setEnabled(False)
self.getoffButton.setEnabled(False)
logger.error("Node does not exist in database!")
aboutBox(self, "Error",
"This node was not found in the database! If you wish to render "
"on this node it must be registered with the databse. Run "
"Register.exe or Register.py to regiester this node and "
" try again.")
sys.exit(1)
self.currentSchedule = self.thisNode.weekSchedule
self.currentScheduleEnabled = self.thisNode.scheduleEnabled
self.buildUI()
self.connectButtons()
self.updateThisNodeInfo()
self.startupServers()
logger.info("Render Node Main is live! Waiting for tasks...")
try:
autoHide = True if str(sys.argv[1]).lower() == "true" else False
logger.info(autoHide)
except IndexError:
autoHide = False
if autoHide and self.trayIconBool:
logger.info("Autohide is enabled!")
self.sendToTrayHandler()
else:
self.show()
def get_tokens(self):
ret = []
for col in self.script_map:
ret += Utils.get_token(col['text']) # col['text'].lower().split()
return ret
def get_char_tokens(self, person):
ret = []
for col in self.script_map:
if col['person'].upper() == person.upper():
ret += Utils.get_token(col['text'])
return ret
def __init__(self, name):
self.movie_name = name
self.soup = bs(Utils.script(name), features='html.parser')
self.script_map = Formatter.get_script_map(self.soup)
from Formatter1 import Formatter as FirstFormatter
from Formatter2 import Formatter as SecondFormatter
from Formatter3 import Formatter as ThirdFormatter
from Formatter4 import Formatter as FourthFormatter
from Formatter5 import Formatter as FifthFormatter
from Formatter6 import Formatter as SixthFormatter
from gensim.models import Word2Vec as wv, Word2Vec
from gensim.models import KeyedVectors as kv
import numpy as np
from sklearn.manifold import TSNE
import matplotlib.pyplot as plt
from prettytable import PrettyTable
model = kv.load_word2vec_format('GoogleNews-vectors-negative300.bin',
binary=True)
utils = Utils()
stop_words = [
"i", "i'm", "it's", "me", "my", "myself", "we", "our", "ours", "ourselves",
"you", "your", "yours", "yourself", "yourselves", "he", "him", "his",
"himself", "she", "her", "hers", "herself", "it", "its", "itself", "they",
"them", "their", "theirs", "themselves", "what", "which", "who", "whom",
"this", "that", "these", "those", "am", "is", "are", "was", "were", "be",
"been", "being", "have", "has", "had", "having", "do", "does", "did",
"doing", "a", "an", "the", "and", "but", "if", "or", "because", "as",
"until", "while", "of", "at", "by", "for", "with", "about", "against",
"between", "into", "through", "during", "before", "after", "above",
"below", "to", "from", "up", "down", "in", "out", "on", "off", "over",
"under", "again", "further", "then", "once", "here", "there", "when",
"where", "why", "how", "all", "any", "both", "each", "few", "more", "most",
"other", "some", "such", "no", "nor", "not", "only", "own", "same", "so",
"translate": True,
"has_folder": False,
"strings": [],
"dicts": [],
"dict_arrays": ['history'],
"special_characters": None,
"v": [],
"s": [],
"episodeid": 0
}
}
}
}
}
utils = Utils()
config = utils.create_config(default_config)
utils.format_file('assets.bin', input_path=config['input_path'])
utils.unzip_assets(config)
utils.remove_file(config['input_path'], 'assets.zip')
utils.refresh_output('{0}\\{1}'.format(config['input_path'], 'assets'),'{0}\\{1}'.format(config['output_path'], 'assets'))
for game in config['games'].keys():
if config['games'][game]['translate']:
print('Translating: {0}'.format(game))
gamesManager = GamesManager(config, utils, game)
else:
print('{0} was skipped'.format(game))
utils.zip(config['output_path'])
utils.format_file('assets.zip', input_path=config['output_path'], output_path=config['input_path'])
def __setattr__(self, k, v):
self.__dict__[k] = v
if Utils.nonFlanged(k):
self.__dirty__.add(k)
def buildUI(self):
def addItem(name, handler, statusTip, menu):
action = QAction(name, self)
action.setStatusTip(statusTip)
action.triggered.connect(handler)
menu.addAction(action)
#Add Logging handlers for output field
emStream = EmittingStream(textWritten=self.normalOutputWritten)
handler = logging.StreamHandler(emStream)
handler.setLevel(logging.INFO)
handler.setFormatter(outputWindowFormatter)
logger.addHandler(handler)
sys.stdout = EmittingStream(textWritten=self.normalOutputWritten)
sys.stderr = EmittingStream(textWritten=self.normalOutputWritten)
#Get Pixmaps and Icon
self.donePixmap = QPixmap(Utils.findResource("Images/status/done.png"))
self.inProgPixmap = QPixmap(Utils.findResource("Images/status/inProgress.png"))
self.needsAttentionPixmap = QPixmap(Utils.findResource("Images/status/needsAttention.png"))
self.nonePixmap = QPixmap(Utils.findResource("Images/status/none.png"))
self.notStartedPixmap = QPixmap(Utils.findResource("Images/status/notStarted.png"))
self.refreshPixmap = QPixmap(Utils.findResource("Images/refresh.png"))
self.refreshIcon = QIcon()
self.refreshIcon.addPixmap(self.refreshPixmap)
self.RIcon = QIcon(Utils.findResource("Images/RenderNodeMain.png"))
self.isVisable = True
self.refreshButton.setIcon(self.refreshIcon)
self.renderServerPixmap.setPixmap(self.notStartedPixmap)
self.scheduleThreadPixmap.setPixmap(self.notStartedPixmap)
self.pulseThreadPixmap.setPixmap(self.notStartedPixmap)
self.setWindowIcon(self.RIcon)
#Setup tray icon
self.trayIcon = QSystemTrayIcon()
self.trayIconBool = self.trayIcon.isSystemTrayAvailable()
if self.trayIconBool:
self.trayIcon.setIcon(self.RIcon)
self.trayIcon.show()
self.trayIcon.setVisible(True)
self.trayIcon.activated.connect(self.activate)
self.trayIcon.messageClicked.connect(self.activate)
#Tray Icon Context Menu
self.taskIconMenu = QMenu(self)
addItem("Open", self.showWindowHandler,
"Show the RenderNodeMain Window", self.taskIconMenu)
self.taskIconMenu.addSeparator()
addItem("Update", self.updateThisNodeInfo,
"Fetch the latest information from the Database", self.taskIconMenu)
self.taskIconMenu.addSeparator()
addItem("Online", self.onlineThisNodeHandler,
"Online this node", self.taskIconMenu)
addItem("Offline", self.offlineThisNodeHandler,
"Offline this node", self.taskIconMenu)
addItem("GetOff!", self.getOffThisNodeHandler,
"Kill the current task and offline this node", self.taskIconMenu)
self.trayIcon.setContextMenu(self.taskIconMenu)
else:
logger.error("Tray Icon Error! Could not create tray icon.")
aboutBox(self, "Tray Icon Error",
"Could not create tray icon. Minimizing to tray has been disabled.")
self.trayButton.setEnabled(False)
def pulse():
host = Utils.myHostName()
with transaction() as t:
t.cur.execute("UPDATE hydra_rendernode SET pulse = NOW() "
"WHERE host = '{0}'".format(host))
"""Registers a node with the database."""
#Standard
import os
import sys
#Third Party
#pylint: disable=E0611
from MySQLdb import IntegrityError
#Hydra
from Setups.LoggingSetup import logger
from Setups.MySQLSetup import hydra_rendernode, OFFLINE, transaction
import Utilities.Utils as Utils
if __name__ == "__main__":
me = Utils.myHostName()
hydraPath, execFile = os.path.split(sys.argv[0])
logger.info(hydraPath)
response = Utils.changeHydraEnviron(hydraPath)
if response:
try:
with transaction() as t:
hydra_rendernode(host=me, status=OFFLINE, minPriority=0).insert(t)
except IntegrityError:
logger.info("Host %s already exists in the hydra_rendernode table on the databse", me)
else:
logger.error("Could not set Hydra Environ! No changes where made. Exiting...")
raw_input("\nPress enter to exit...")
"v": [3, 5, 7, 8, 9, 10, 12, 14],
"s": [15, 16, 17],
"episodeid": 0
},
'SSWeapons': {
"translate": True,
"has_folder": False,
"strings": ['title', 'description'],
"dicts": [],
"dict_arrays": [],
"special_characters": [['<', '>']],
"v": [],
"s": [],
"episodeid": 0
}
}
}
}
}
utils = Utils()
config = utils.create_config(default_config)
# utils.refresh_output(config['input_path'], config['output_path'])
for game in config['games'].keys():
if config['games'][game]['translate']:
print('Translating: {0}'.format(game))
gamesManager = GamesManager(config, utils, game)
else:
print('{0} was skipped'.format(game))
}
keyboard = key.KeyStateHandler()
oldKeyboard = key.KeyStateHandler()
gameWindow.push_handlers(keyboard)
# Load in the resources
loadResources()
alienBatch = pyglet.graphics.Batch()
shipGroup = pyglet.graphics.OrderedGroup(10)
shieldGroup = pyglet.graphics.OrderedGroup(20)
overlayGroup = pyglet.graphics.OrderedGroup(30)
playerShipColored = Utils.colorTexture(playerShipTexture, (120, 120, 120),
(40, 40, 200), (255, 128, 0))
player = Player(playerShipColored,
alienShieldTexture,
alienBatch,
shipGroup,
shieldGroup,
x=screenWidth / 2,
y=screenHeight / 2)
player.velocity = np.array((100, 0), dtype=np.float32)
rotationalSens = 200
forwardAccel = 300
reverseAccel = -200
orthAccel = 150
swarm1 = []
def updateAutologinUser(newUsername):
if Utils.getInfoFromCFG("database", "username") != newUsername:
return Utils.writeInfoToCFG("database", "username", newUsername)
def launchRenderTask(self, HydraJob, HydraTask):
"""Does the actual rendering, then records the results on the database"""
logger.info("Starting task with id %s on job with id %s", HydraTask.id, HydraJob.id)
self.HydraJob = HydraJob
self.HydraTask = HydraTask
self.childKilled = 0
self.statusAfterDeath = None
self.childProcess = None
self.PSUtilProc = None
originalCurrentFrame = int(self.HydraTask.currentFrame)
renderTaskCMD = self.HydraTask.createTaskCMD(self.HydraJob, sys.platform)
logger.debug(renderTaskCMD)
self.logPath = self.HydraTask.getLogPath()
logger.info("Starting render task %s", self.HydraTask.id)
try:
log = file(self.logPath, 'w')
except (IOError, OSError, WindowsError) as e:
logger.error(e)
self.thisNode.getOff()
return
log.write('Hydra log file {0} on {1}\n'.format(self.logPath, self.HydraTask.host))
log.write('RenderNode is {0}\n'.format(sys.argv))
log.write('Command: {0}\n\n'.format(renderTaskCMD))
Utils.flushOut(log)
progressUpdateThread = stoppableThread(self.progressUpdate, 300,
"Progress_Update_Thread")
#Run the job and keep track of the process
self.childProcess = subprocess.Popen(renderTaskCMD,
stdout=log, stderr=log,
**Utils.buildSubprocessArgs(False))
logger.info("Started PID %s to do Task %s", self.childProcess.pid, self.HydraTask.id)
self.PSUtilProc = psutil.Process(self.childProcess.pid)
#Wait for task to finish
self.childProcess.communicate()
#Get Exit Code, Record the results
self.HydraTask.exitCode = self.childProcess.returncode if self.childProcess else 1234
logString = "\nProcess exited with code {0} at {1} on {2}\n"
nowTime = datetime.datetime.now().replace(microsecond=0)
log.write(logString.format(self.HydraTask.exitCode, nowTime,
self.thisNode.host))
progressUpdateThread.terminate()
#Update HydraTask and HydraJob with currentFrame, MPF, and RLTracker
self.progressUpdate(commit=False)
#EndTime
self.HydraTask.endTime = datetime.datetime.now()
#Work around for batch files
if self.HydraJob.jobType == "BatchFile" and self.HydraTask.exitCode == 0:
self.HydraTask.currentFrame = (self.HydraTask.endFrame + 1)
self.HydraJob.renderLayerTracker = str((self.HydraTask.endFrame + 1))
#Status, Attempts. Failures
if self.childKilled == 1:
self.HydraTask.status = self.statusAfterDeath
self.HydraTask.exitCode = 1
else:
if self.HydraTask.exitCode == 0 and self.HydraTask.currentFrame >= originalCurrentFrame:
status = FINISHED
else:
if self.HydraTask.exitCode == 0:
log.write("\n\nERROR: Task returned exit code 0 but it appears to have not actually rendered any frames.")
status = ERROR
self.HydraJob.attempts += 1
if not self.HydraJob.failures or self.HydraJob.failures == "":
self.HydraJob.failures = self.thisNode.host
else:
self.HydraJob.failures += ",{0}".format(self.thisNode.host)
self.HydraTask.status = status
#Update data on the DB
with transaction() as t:
self.HydraTask.update(t)
self.HydraJob.update(t)
self.resetThisNode()
log.close()
logger.info("Done with render task %s", self.HydraTask.id)
self.childProcess = None
self.PSUtilProc = None
self.HydraJob = None
self.HydraTask = None
self.logPath = None
