def _syncGitHubMasterRepository(self): reposPath = self.procedureItems[bi.InstallationSetupItem.NAME].installTypeItem.REPOS_PATH projectName = self.procedureItems[bi.InstallationSetupItem.NAME].projectName masterReposPath = os.path.join(reposPath, projectName) if self.getCurrentInstallType() == bi.BaseInstallType.TYPE_ANSA_CHECK: projectName = 'checks' masterReposPath = reposPath # synchronise with github if not there already if self.projectSourceType is not bi.MasterReposProjectSourceType: if projectName in self.availableTools: print('Project exists in master repository') githubio.Githubio.pushProject(projectName, masterReposPath) else: print('Creating a new project in master repository') githubio.Githubio.createProject(projectName, masterReposPath) # add files to doc repository docSourceDir = os.path.join(di.SPHINX_SOURCE, utils.getInstallTypePaths()['GENERAL']['LOCAL_DOCUMENTATION'], self.procedureItems[bi.DocumetationItem.NAME].docuGroup.replace(' ', '_'), projectName) + os.path.sep + '*' print('Adding new documentation files to doc repository: "%s"' % docSourceDir) utils.runSubprocess('git add %s' % docSourceDir, cwd=di.SPHINX_SOURCE) # synchronise documentation di.ToolDocumentation.commitToolAdded('Tool: %s (%s) installed.' % ( projectName, self.procedureItems[bi.VersionItem.NAME].tagName))
def commitToolAdded(cls, commitMessage): utils.runSubprocess('git commit -a -m "%s"' % commitMessage, DOCUMENTATON_PATH) githubio.Githubio.pushProject(os.path.basename(DOCUMENTATON_PATH), DOCUMENTATON_PATH)
def initiateFromGithub(cls): if os.path.exists(DOCUMENTATON_PATH): utils.runSubprocess('git pull origin master', DOCUMENTATON_PATH) else: githubio.Githubio.cloneProject('idiada_tools_doc', os.path.dirname(DOCUMENTATON_PATH))
def cloneProject(cls, projectName, targetPath=None): if targetPath is None: targetPath = tempfile.TemporaryDirectory().name templateString = Template(CONTENT_CLONE) content = templateString.safe_substitute({ 'userName': cls.githubSettings['GITHUB_USER'], 'password': cls.githubSettings['GITHUB_PASSWORD'], 'projectName': projectName }) if not os.path.isdir(targetPath): os.makedirs(targetPath) # process = subprocess.Popen( # '', shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, # cwd=targetPath) # print(content.split('\n')) # for command in content.split('\n'): # command += '\n' # print(command) # try: # process.stdin.write(command.encode('utf-8')) # except IOError as e: # print(str(e)) # if e.errno == errno.EPIPE or e.errno == errno.EINVAL: # # Stop loop on "Invalid pipe" or "Invalid argument". # # No sense in continuing with broken pipe. # break # else: # # Raise any other error. # raise # # process.stdin.close() # process.wait() cloneFileName = os.path.join(targetPath, 'cloneScript.script') fo = open(cloneFileName, 'wt') fo.write(content) fo.close() os.chmod(cloneFileName, 0o775) utils.runSubprocess('%s' % cloneFileName, targetPath) if os.path.isfile(cloneFileName): os.remove(cloneFileName) return targetPath
def getQueueStat(): stdout, _ = utils.runSubprocess('qstat -u \* -xml -r') rootElement = ETree.fromstring(stdout) jobElements = list() for elem in rootElement.iter(): if elem.tag == 'job_list': jobElements.append(elem) jobs = list() for jobElement in jobElements: # jobState = jobElement.get('state') # if jobState not in jobs: # jobs[jobState] = list() elemAttributes = dict() for eNo, attributeElement in enumerate(jobElement.getchildren()): if attributeElement.tag in elemAttributes: attrName = attributeElement.tag + '_%s' % eNo else: attrName = attributeElement.tag elemAttributes[attrName] = attributeElement.text # jobs[jobState].append(elemAttributes) jobs.append(elemAttributes) return jobs
def _setupProjectStatus(self): stdout, stderr = utils.runSubprocess( 'git status', cwd=os.path.dirname( os.path.dirname(self.parentApplication.sourceMainPath))) untrackedFiles = list() untrackedBlock = False for line in stdout.splitlines(): parts = line.split() if line.startswith('# Untracked files:'): untrackedBlock = True continue elif not line.startswith('#'): continue # skip empty line if len(parts) <= 1: continue elif line.startswith('# (use "git add <file>'): continue if untrackedBlock: untrackedFiles.append(line.replace('#', '').strip()) for untrackedFile in untrackedFiles: self.utrackedFilesListWidget.addItem( bw.BaseListWidgetItem(self, untrackedFile))
def getHostsStat(): ''' Returns a list of all hosts - execution servers and their running jobs. ''' stdout, _ = utils.runSubprocess('qhost -j -xml') rootElement = ETree.fromstring(stdout) hosts = dict() for hostElement in rootElement.iter('host'): hostName = hostElement.attrib['name'] # skip global host if hostName == 'global' or 'test' in hostName: continue hosts[hostName] = dict() for hostValueElement in hostElement.iter('hostvalue'): hostAttributeName = hostValueElement.attrib['name'] hosts[hostName][hostAttributeName] = hostValueElement.text # add running jobs hosts[hostName]['jobs'] = dict() for jobElement in hostElement.iter('job'): jobId = jobElement.attrib['name'] hosts[hostName]['jobs'][jobId] = dict() for jobValueElement in jobElement: jobAttributeName = jobValueElement.attrib['name'] hosts[hostName]['jobs'][jobId][ jobAttributeName] = jobValueElement.text return hosts
def checkUpdated(cls): stdout, _ = utils.runSubprocess('git log --oneline', DOCUMENTATON_PATH) commitCount = len(stdout.strip().split('\n')) cls.synchronise() stdout, _ = utils.runSubprocess('git log --oneline', DOCUMENTATON_PATH) newCommitCount = len(stdout.strip().split('\n')) if commitCount != newCommitCount: print('Documentation update found in the master repository.') return True else: print('Up-to-date') return False
def pushProject(cls, projectName, projectPath): templateString = Template(CONTENT_SYNC) content = templateString.safe_substitute({ 'userName': cls.githubSettings['GITHUB_USER'], 'password': cls.githubSettings['GITHUB_PASSWORD'], 'projectName': projectName }) createFileName = os.path.join(projectPath, 'syncScript.script') fo = open(createFileName, 'wt') fo.write(content) fo.close() os.chmod(createFileName, 0o775) utils.runSubprocess('%s' % createFileName, projectPath) os.remove(createFileName)
def accept(self): message = str(self.commitMessageTextEdit.toPlainText()) if len(message) == 0: QtGui.QMessageBox.critical(self, '%s' % self.TITLE, 'Please add commit description.') return for i in range(self.utrackedFilesListWidget.count()): item = self.utrackedFilesListWidget.item(i) if item.checkState() == QtCore.Qt.Checked: utils.runSubprocess( 'git add %s' % item.text(), cwd=os.path.dirname( os.path.dirname( self.parentApplication.sourceMainPath))) # commit message utils.runSubprocess('git commit -a -m "%s"' % message, cwd=os.path.dirname( os.path.dirname( self.parentApplication.sourceMainPath))) # add tag utils.runSubprocess('git tag %s' % self.tagName, cwd=os.path.dirname( os.path.dirname( self.parentApplication.sourceMainPath))) super(NewRevisionDialog, self).accept()
def getAvailableHost(licenseServerName): stdout, _ = utils.runSubprocess('qstat -f -q %s -xml' % licenseServerName) rootElement = ETree.fromstring(stdout) hosts = list() for hostElement in rootElement.iter('Queue-List'): hostName = hostElement.find('name').text hosts.append(hostName.split('@')[-1]) return hosts
def __init__(self, args): self.args = args self.jobs = list() if DEBUG: level = logging.DEBUG else: level = logging.INFO utils.initiateLogging(self, level) self.setWorkDir(os.getcwd()) # initiate resource status ci.Resources.initialise() # check input interface paramProvided = self._checkInputArgs() if paramProvided: self.setupFromParameters() else: self.profile = self.setProfile() self.profile.runDataSelectionSequence() for inpFileName in self.profile.inpFileNames: newJob = self.profile.job.getCopy() newJob.setInpFile(inpFileName, self.profile) newJob.setExecutableFile(newJob.EXECUTABLE_FILE_TYPE(self, newJob)) newJob.executableFile.save() self.jobs.append(object) logging.debug(newJob.executableFile.getContent()) logging.debug('qsub %s' % newJob.executableFile.outputFileName) if not DEBUG: utils.runSubprocess('qsub %s' % newJob.executableFile.outputFileName, cwd=newJob.inpFile.dirName)
def _createMasterRepository(self): print('Master repository synchronisation') reposPath = self.procedureItems[bi.InstallationSetupItem.NAME].installTypeItem.REPOS_PATH projectName = self.procedureItems[bi.InstallationSetupItem.NAME].projectName masterReposPath = os.path.join(reposPath, projectName) # check if repository exists if not os.path.isdir(masterReposPath): os.makedirs(masterReposPath) utils.runSubprocess('git init', cwd=masterReposPath) localReposPath = os.path.dirname(os.path.dirname(self.mainModulePath)) # handle ANSA check type special behavior if not self.getCurrentInstallType() == bi.BaseInstallType.TYPE_ANSA_CHECK: utils.runSubprocess('git pull %s' % localReposPath, cwd=masterReposPath) utils.runSubprocess('git checkout master', cwd=masterReposPath) utils.runSubprocess('git fetch --tag %s' % localReposPath, cwd=masterReposPath)
def _createNewRevision(self, tagName, commitMessage, newFileList): # add files for newFileName in newFileList: print('Adding file to repository: %s' % newFileName) utils.runSubprocess('git add %s' % newFileName, cwd=os.path.dirname( os.path.dirname(self.mainModulePath))) # commit changes utils.runSubprocess('git commit -a -m "%s"' % commitMessage, cwd=os.path.dirname( os.path.dirname(self.mainModulePath))) # add tag utils.runSubprocess('git tag %s' % tagName, cwd=os.path.dirname( os.path.dirname(self.mainModulePath)))
def createPlist(cls): # create plist for available checks and the documentation string command = '%s -nogui -execscript %s' % (utils.getPathAnsaExecutable(), os.path.join(utils.PATH_BIN, 'interfaces', 'ansaChecksPlistUpdater', 'bin', cls.PLIST_GENERATOR_NAME)) stdout, stderr = utils.runSubprocess(command) for line in stdout.splitlines(): print(line) if stderr is not None: for line in stderr.splitlines(): print(line) #============================================================================== # if __name__ == '__main__': # # AnsaChecksPlistUpdater.createPlist()
def show(self): docString = self.parentApplication.installer.procedureItems[ bi.DocumetationItem.NAME].docString TEMPLATE = '''documentation preview ========================== .. toctree:: :maxdepth: 2 %s ''' srcConf = os.path.join(utils.PATH_RES, 'doc_preview', 'source', 'conf.py') dst = os.path.join( utils.getInstallTypePaths()['INSTALLATION_PATHS_BASE'] ['DOCUMENTATON_PATH_TEMP'], 'sphinx') sourceSphinxPath = os.path.join(dst, 'source') # initiate if os.path.isdir(dst): shutil.rmtree(dst) # copy source files try: projectDocSouce = os.path.join( self.parentApplication.installer.pyProjectPath, 'doc', 'sphinx', 'source') shutil.copytree(projectDocSouce, sourceSphinxPath) # delete file which should be updated if os.path.isfile(os.path.join(sourceSphinxPath, 'conf.py')): os.remove(os.path.join(sourceSphinxPath, 'conf.py')) # if os.path.isfile(os.path.join(sourceSphinxPath, 'index.rst')): # os.remove(os.path.join(sourceSphinxPath, 'index.rst')) except Exception as e: print('Failed to copy documentation source files! (%s)' % str(e)) # if not os.path.isfile(os.path.join(sourceSphinxPath, 'conf.py')): shutil.copy(srcConf, os.path.join(sourceSphinxPath, 'conf.py')) if os.path.isfile(os.path.join(sourceSphinxPath, 'index.rst')): fi = open(os.path.join(sourceSphinxPath, 'index.rst')) TEMPLATE = fi.read().replace('.. automodule:: main', '%s') fi.close() fo = open(os.path.join(sourceSphinxPath, 'index.rst'), 'wt') fo.write(TEMPLATE % docString) fo.close() # create local documentation using installer environment envExecutable = utils.getEnvironmentExecutable(utils.PATH_INI) utils.runSubprocess( '%s -b html -d %s %s %s' % (os.path.join(os.path.dirname(envExecutable), 'sphinx-build'), os.path.join(dst, 'build', 'doctrees'), sourceSphinxPath, os.path.join(dst, 'build', 'html'))) address = os.path.join(dst, 'build', 'html', 'index.html') os.system('%s %s &' % (utils.getDocumentationBrowser(), address))
def syncProject(cls, projectPath): utils.runSubprocess('%s sync' % HUB_EXECUTABLE, projectPath)
def submit(self): if len(self.profile.inpFileNames) == 0: raise si.DataSelectorException('No files selected!') # check license restriction licenseServer = self.profile.jobSettings.licenseServer if licenseServer.restriction.isActivated(): message = licenseServer.restriction.getOverviewMessage() + '\nDo you want to ignore restrictions and submit the job anyway?' self.parentApplication.restoreOverrideCursor() reply = QtGui.QMessageBox.question(self.parentApplication.mainWindow, 'License restriction', message, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No) if reply == QtGui.QMessageBox.No: return self.parentApplication.setOverrideCursor(QtCore.Qt.WaitCursor) # check max number of tokens if self.profile.job.getTokensRequired() > self.profile.jobSettings.licenseServer.getMaxTokens(): self.parentApplication.restoreOverrideCursor() message = 'It seems that current settings (tokens required: %s) exceeds the maximum available tokens (tokens available: %s).\n\ Are you sure to submit the job with the current settings?' % ( self.profile.job.getTokensRequired(), self.profile.jobSettings.licenseServer.getMaxTokens()) reply = QtGui.QMessageBox.question(self.parentApplication.mainWindow, 'Token restriction', message, QtGui.QMessageBox.Yes, QtGui.QMessageBox.No) if reply == QtGui.QMessageBox.No: return self.parentApplication.setOverrideCursor(QtCore.Qt.WaitCursor) message = '' for inpFileName in self.profile.inpFileNames: newJob = self.profile.job.getCopy() newJob.setInpFile(inpFileName, self.profile) newJob.setExecutableFile(newJob.EXECUTABLE_FILE_TYPE(self, newJob)) # in case of restart read if newJob.inpFile.subAllFiles: self.parentApplication.restoreOverrideCursor() restartInpFileName = QtGui.QFileDialog.getOpenFileName(self, 'Select restart input file', newJob.inpFile.dirName, filter = "*%s file for analysis: '%s' (*%s)" % ( ei.FileExtensions.ABAQUS_RESTART, newJob.inpFile.baseName, ei.FileExtensions.ABAQUS_RESTART)) if not restartInpFileName: raise si.DataSelectorException('No restart file selected!') logging.debug('Selected restart file: %s' % restartInpFileName) info = newJob.setRestartInpFile(str(restartInpFileName)) logging.debug(info) message = 'Restart files status:' for baseName, status in info.iteritems(): message += '\n%s: %s' % (baseName, status) self.parentApplication.showInfoMessage(message) newJob.executableFile.save() message += 'Submitting job: %s\n' % newJob.inpFile.baseName logging.debug(message) logging.debug(newJob.executableFile.getContent()) if not self.parentApplication.DEBUG: utils.runSubprocess( 'qsub %s' % newJob.executableFile.outputFileName, cwd = newJob.inpFile.dirName) self.parentApplication.showInfoMessage(message)