def _syncGitHubMasterRepository(self):
reposPath = self.procedureItems[bi.InstallationSetupItem.NAME].installTypeItem.REPOS_PATH
projectName = self.procedureItems[bi.InstallationSetupItem.NAME].projectName
masterReposPath = os.path.join(reposPath, projectName)
if self.getCurrentInstallType() == bi.BaseInstallType.TYPE_ANSA_CHECK:
projectName = 'checks'
masterReposPath = reposPath
# synchronise with github if not there already
if self.projectSourceType is not bi.MasterReposProjectSourceType:
if projectName in self.availableTools:
print('Project exists in master repository')
githubio.Githubio.pushProject(projectName, masterReposPath)
else:
print('Creating a new project in master repository')
githubio.Githubio.createProject(projectName, masterReposPath)
# add files to doc repository
docSourceDir = os.path.join(di.SPHINX_SOURCE,
utils.getInstallTypePaths()['GENERAL']['LOCAL_DOCUMENTATION'],
self.procedureItems[bi.DocumetationItem.NAME].docuGroup.replace(' ', '_'),
projectName) + os.path.sep + '*'
print('Adding new documentation files to doc repository: "%s"' % docSourceDir)
utils.runSubprocess('git add %s' % docSourceDir, cwd=di.SPHINX_SOURCE)
# synchronise documentation
di.ToolDocumentation.commitToolAdded('Tool: %s (%s) installed.' % (
projectName, self.procedureItems[bi.VersionItem.NAME].tagName))
def commitToolAdded(cls, commitMessage):
utils.runSubprocess('git commit -a -m "%s"' % commitMessage,
DOCUMENTATON_PATH)
githubio.Githubio.pushProject(os.path.basename(DOCUMENTATON_PATH),
DOCUMENTATON_PATH)
def initiateFromGithub(cls):
if os.path.exists(DOCUMENTATON_PATH):
utils.runSubprocess('git pull origin master', DOCUMENTATON_PATH)
else:
githubio.Githubio.cloneProject('idiada_tools_doc',
os.path.dirname(DOCUMENTATON_PATH))
def cloneProject(cls, projectName, targetPath=None):
if targetPath is None:
targetPath = tempfile.TemporaryDirectory().name
templateString = Template(CONTENT_CLONE)
content = templateString.safe_substitute({
'userName':
cls.githubSettings['GITHUB_USER'],
'password':
cls.githubSettings['GITHUB_PASSWORD'],
'projectName':
projectName
})
if not os.path.isdir(targetPath):
os.makedirs(targetPath)
# process = subprocess.Popen(
# '', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
# cwd=targetPath)
# print(content.split('\n'))
# for command in content.split('\n'):
# command += '\n'
# print(command)
# try:
# process.stdin.write(command.encode('utf-8'))
# except IOError as e:
# print(str(e))
# if e.errno == errno.EPIPE or e.errno == errno.EINVAL:
# # Stop loop on "Invalid pipe" or "Invalid argument".
# # No sense in continuing with broken pipe.
# break
# else:
# # Raise any other error.
# raise
#
# process.stdin.close()
# process.wait()
cloneFileName = os.path.join(targetPath, 'cloneScript.script')
fo = open(cloneFileName, 'wt')
fo.write(content)
fo.close()
os.chmod(cloneFileName, 0o775)
utils.runSubprocess('%s' % cloneFileName, targetPath)
if os.path.isfile(cloneFileName):
os.remove(cloneFileName)
return targetPath
def getQueueStat():
stdout, _ = utils.runSubprocess('qstat -u \* -xml -r')
rootElement = ETree.fromstring(stdout)
jobElements = list()
for elem in rootElement.iter():
if elem.tag == 'job_list':
jobElements.append(elem)
jobs = list()
for jobElement in jobElements:
# jobState = jobElement.get('state')
# if jobState not in jobs:
# jobs[jobState] = list()
elemAttributes = dict()
for eNo, attributeElement in enumerate(jobElement.getchildren()):
if attributeElement.tag in elemAttributes:
attrName = attributeElement.tag + '_%s' % eNo
else:
attrName = attributeElement.tag
elemAttributes[attrName] = attributeElement.text
# jobs[jobState].append(elemAttributes)
jobs.append(elemAttributes)
return jobs
def _setupProjectStatus(self):
stdout, stderr = utils.runSubprocess(
'git status',
cwd=os.path.dirname(
os.path.dirname(self.parentApplication.sourceMainPath)))
untrackedFiles = list()
untrackedBlock = False
for line in stdout.splitlines():
parts = line.split()
if line.startswith('# Untracked files:'):
untrackedBlock = True
continue
elif not line.startswith('#'):
continue
# skip empty line
if len(parts) <= 1:
continue
elif line.startswith('# (use "git add <file>'):
continue
if untrackedBlock:
untrackedFiles.append(line.replace('#', '').strip())
for untrackedFile in untrackedFiles:
self.utrackedFilesListWidget.addItem(
bw.BaseListWidgetItem(self, untrackedFile))
def getHostsStat():
''' Returns a list of all hosts - execution servers and their running
jobs.
'''
stdout, _ = utils.runSubprocess('qhost -j -xml')
rootElement = ETree.fromstring(stdout)
hosts = dict()
for hostElement in rootElement.iter('host'):
hostName = hostElement.attrib['name']
# skip global host
if hostName == 'global' or 'test' in hostName:
continue
hosts[hostName] = dict()
for hostValueElement in hostElement.iter('hostvalue'):
hostAttributeName = hostValueElement.attrib['name']
hosts[hostName][hostAttributeName] = hostValueElement.text
# add running jobs
hosts[hostName]['jobs'] = dict()
for jobElement in hostElement.iter('job'):
jobId = jobElement.attrib['name']
hosts[hostName]['jobs'][jobId] = dict()
for jobValueElement in jobElement:
jobAttributeName = jobValueElement.attrib['name']
hosts[hostName]['jobs'][jobId][
jobAttributeName] = jobValueElement.text
return hosts
def checkUpdated(cls):
stdout, _ = utils.runSubprocess('git log --oneline', DOCUMENTATON_PATH)
commitCount = len(stdout.strip().split('\n'))
cls.synchronise()
stdout, _ = utils.runSubprocess('git log --oneline', DOCUMENTATON_PATH)
newCommitCount = len(stdout.strip().split('\n'))
if commitCount != newCommitCount:
print('Documentation update found in the master repository.')
return True
else:
print('Up-to-date')
return False
def pushProject(cls, projectName, projectPath):
templateString = Template(CONTENT_SYNC)
content = templateString.safe_substitute({
'userName':
cls.githubSettings['GITHUB_USER'],
'password':
cls.githubSettings['GITHUB_PASSWORD'],
'projectName':
projectName
})
createFileName = os.path.join(projectPath, 'syncScript.script')
fo = open(createFileName, 'wt')
fo.write(content)
fo.close()
os.chmod(createFileName, 0o775)
utils.runSubprocess('%s' % createFileName, projectPath)
os.remove(createFileName)
def accept(self):
message = str(self.commitMessageTextEdit.toPlainText())
if len(message) == 0:
QtGui.QMessageBox.critical(self, '%s' % self.TITLE,
'Please add commit description.')
return
for i in range(self.utrackedFilesListWidget.count()):
item = self.utrackedFilesListWidget.item(i)
if item.checkState() == QtCore.Qt.Checked:
utils.runSubprocess(
'git add %s' % item.text(),
cwd=os.path.dirname(
os.path.dirname(
self.parentApplication.sourceMainPath)))
# commit message
utils.runSubprocess('git commit -a -m "%s"' % message,
cwd=os.path.dirname(
os.path.dirname(
self.parentApplication.sourceMainPath)))
# add tag
utils.runSubprocess('git tag %s' % self.tagName,
cwd=os.path.dirname(
os.path.dirname(
self.parentApplication.sourceMainPath)))
super(NewRevisionDialog, self).accept()
def getAvailableHost(licenseServerName):
stdout, _ = utils.runSubprocess('qstat -f -q %s -xml' %
licenseServerName)
rootElement = ETree.fromstring(stdout)
hosts = list()
for hostElement in rootElement.iter('Queue-List'):
hostName = hostElement.find('name').text
hosts.append(hostName.split('@')[-1])
return hosts
def __init__(self, args):
self.args = args
self.jobs = list()
if DEBUG:
level = logging.DEBUG
else:
level = logging.INFO
utils.initiateLogging(self, level)
self.setWorkDir(os.getcwd())
# initiate resource status
ci.Resources.initialise()
# check input interface
paramProvided = self._checkInputArgs()
if paramProvided:
self.setupFromParameters()
else:
self.profile = self.setProfile()
self.profile.runDataSelectionSequence()
for inpFileName in self.profile.inpFileNames:
newJob = self.profile.job.getCopy()
newJob.setInpFile(inpFileName, self.profile)
newJob.setExecutableFile(newJob.EXECUTABLE_FILE_TYPE(self, newJob))
newJob.executableFile.save()
self.jobs.append(object)
logging.debug(newJob.executableFile.getContent())
logging.debug('qsub %s' % newJob.executableFile.outputFileName)
if not DEBUG:
utils.runSubprocess('qsub %s' %
newJob.executableFile.outputFileName,
cwd=newJob.inpFile.dirName)
def _createMasterRepository(self):
print('Master repository synchronisation')
reposPath = self.procedureItems[bi.InstallationSetupItem.NAME].installTypeItem.REPOS_PATH
projectName = self.procedureItems[bi.InstallationSetupItem.NAME].projectName
masterReposPath = os.path.join(reposPath, projectName)
# check if repository exists
if not os.path.isdir(masterReposPath):
os.makedirs(masterReposPath)
utils.runSubprocess('git init', cwd=masterReposPath)
localReposPath = os.path.dirname(os.path.dirname(self.mainModulePath))
# handle ANSA check type special behavior
if not self.getCurrentInstallType() == bi.BaseInstallType.TYPE_ANSA_CHECK:
utils.runSubprocess('git pull %s' % localReposPath, cwd=masterReposPath)
utils.runSubprocess('git checkout master', cwd=masterReposPath)
utils.runSubprocess('git fetch --tag %s' % localReposPath, cwd=masterReposPath)
def _createNewRevision(self, tagName, commitMessage, newFileList):
# add files
for newFileName in newFileList:
print('Adding file to repository: %s' % newFileName)
utils.runSubprocess('git add %s' % newFileName, cwd=os.path.dirname(
os.path.dirname(self.mainModulePath)))
# commit changes
utils.runSubprocess('git commit -a -m "%s"' % commitMessage, cwd=os.path.dirname(
os.path.dirname(self.mainModulePath)))
# add tag
utils.runSubprocess('git tag %s' % tagName, cwd=os.path.dirname(
os.path.dirname(self.mainModulePath)))
def createPlist(cls):
# create plist for available checks and the documentation string
command = '%s -nogui -execscript %s' % (utils.getPathAnsaExecutable(),
os.path.join(utils.PATH_BIN, 'interfaces', 'ansaChecksPlistUpdater',
'bin', cls.PLIST_GENERATOR_NAME))
stdout, stderr = utils.runSubprocess(command)
for line in stdout.splitlines():
print(line)
if stderr is not None:
for line in stderr.splitlines():
print(line)
#==============================================================================
# if __name__ == '__main__':
#
# AnsaChecksPlistUpdater.createPlist()
def show(self):
docString = self.parentApplication.installer.procedureItems[
bi.DocumetationItem.NAME].docString
TEMPLATE = '''documentation preview
==========================
.. toctree::
:maxdepth: 2
%s
'''
srcConf = os.path.join(utils.PATH_RES, 'doc_preview', 'source',
'conf.py')
dst = os.path.join(
utils.getInstallTypePaths()['INSTALLATION_PATHS_BASE']
['DOCUMENTATON_PATH_TEMP'], 'sphinx')
sourceSphinxPath = os.path.join(dst, 'source')
# initiate
if os.path.isdir(dst):
shutil.rmtree(dst)
# copy source files
try:
projectDocSouce = os.path.join(
self.parentApplication.installer.pyProjectPath, 'doc',
'sphinx', 'source')
shutil.copytree(projectDocSouce, sourceSphinxPath)
# delete file which should be updated
if os.path.isfile(os.path.join(sourceSphinxPath, 'conf.py')):
os.remove(os.path.join(sourceSphinxPath, 'conf.py'))
# if os.path.isfile(os.path.join(sourceSphinxPath, 'index.rst')):
# os.remove(os.path.join(sourceSphinxPath, 'index.rst'))
except Exception as e:
print('Failed to copy documentation source files! (%s)' % str(e))
# if not os.path.isfile(os.path.join(sourceSphinxPath, 'conf.py')):
shutil.copy(srcConf, os.path.join(sourceSphinxPath, 'conf.py'))
if os.path.isfile(os.path.join(sourceSphinxPath, 'index.rst')):
fi = open(os.path.join(sourceSphinxPath, 'index.rst'))
TEMPLATE = fi.read().replace('.. automodule:: main', '%s')
fi.close()
fo = open(os.path.join(sourceSphinxPath, 'index.rst'), 'wt')
fo.write(TEMPLATE % docString)
fo.close()
# create local documentation using installer environment
envExecutable = utils.getEnvironmentExecutable(utils.PATH_INI)
utils.runSubprocess(
'%s -b html -d %s %s %s' %
(os.path.join(os.path.dirname(envExecutable), 'sphinx-build'),
os.path.join(dst, 'build', 'doctrees'), sourceSphinxPath,
os.path.join(dst, 'build', 'html')))
address = os.path.join(dst, 'build', 'html', 'index.html')
os.system('%s %s &' % (utils.getDocumentationBrowser(), address))
def syncProject(cls, projectPath):
utils.runSubprocess('%s sync' % HUB_EXECUTABLE, projectPath)
def submit(self):
if len(self.profile.inpFileNames) == 0:
raise si.DataSelectorException('No files selected!')
# check license restriction
licenseServer = self.profile.jobSettings.licenseServer
if licenseServer.restriction.isActivated():
message = licenseServer.restriction.getOverviewMessage() + '\nDo you want to ignore restrictions and submit the job anyway?'
self.parentApplication.restoreOverrideCursor()
reply = QtGui.QMessageBox.question(self.parentApplication.mainWindow, 'License restriction',
message, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.No:
return
self.parentApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
# check max number of tokens
if self.profile.job.getTokensRequired() > self.profile.jobSettings.licenseServer.getMaxTokens():
self.parentApplication.restoreOverrideCursor()
message = 'It seems that current settings (tokens required: %s) exceeds the maximum available tokens (tokens available: %s).\n\
Are you sure to submit the job with the current settings?' % (
self.profile.job.getTokensRequired(),
self.profile.jobSettings.licenseServer.getMaxTokens())
reply = QtGui.QMessageBox.question(self.parentApplication.mainWindow, 'Token restriction',
message, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No)
if reply == QtGui.QMessageBox.No:
return
self.parentApplication.setOverrideCursor(QtCore.Qt.WaitCursor)
message = ''
for inpFileName in self.profile.inpFileNames:
newJob = self.profile.job.getCopy()
newJob.setInpFile(inpFileName, self.profile)
newJob.setExecutableFile(newJob.EXECUTABLE_FILE_TYPE(self, newJob))
# in case of restart read
if newJob.inpFile.subAllFiles:
self.parentApplication.restoreOverrideCursor()
restartInpFileName = QtGui.QFileDialog.getOpenFileName(self,
'Select restart input file', newJob.inpFile.dirName,
filter = "*%s file for analysis: '%s' (*%s)" % (
ei.FileExtensions.ABAQUS_RESTART, newJob.inpFile.baseName,
ei.FileExtensions.ABAQUS_RESTART))
if not restartInpFileName:
raise si.DataSelectorException('No restart file selected!')
logging.debug('Selected restart file: %s' % restartInpFileName)
info = newJob.setRestartInpFile(str(restartInpFileName))
logging.debug(info)
message = 'Restart files status:'
for baseName, status in info.iteritems():
message += '\n%s: %s' % (baseName, status)
self.parentApplication.showInfoMessage(message)
newJob.executableFile.save()
message += 'Submitting job: %s\n' % newJob.inpFile.baseName
logging.debug(message)
logging.debug(newJob.executableFile.getContent())
if not self.parentApplication.DEBUG:
utils.runSubprocess(
'qsub %s' % newJob.executableFile.outputFileName,
cwd = newJob.inpFile.dirName)
self.parentApplication.showInfoMessage(message)