def buildBin(testname, cmdToBuild = None, subdir = None, cmdToInstall = 'default', pkg2build = None): ''' Build binary if necessary ''' if pkg2build is not None: # To double ensure in the right place os.chdir(u.codeIn(srcDir(testname), pkg2build)) Makefile = 'Makefile' makefile = 'makefile' configure = 'configure' if testcaseAttr(testname, 'binary2build') == 'Yes': u.require('make') # FIXME - CLR doesn't search sub-folders in env PATH #installTo = binDir() + '/' + testcaseAttr(testname, 'name') installTo = binDir() if cmdToBuild is not None: MAKE = cmdToBuild else: MAKE = 'make all' if cmdToInstall == 'default' or cmdToInstall == None: INSTALL = "make install --eval='DESTDIR=%s'" % installTo #u.mkdir(installTo) elif cmdToInstall == 'manual': INSTALL = None else: INSTALL = cmdToInstall if subdir is not None: MAKE = "%s -C %s" % (MAKE,subdir) if INSTALL is not None: INSTALL = "%s -C %s" % (INSTALL,subdir) if subdir[-1] is '/': Makefile = subdir + 'Makefile' makefile = subdir + 'makefile' configure = subdir + 'configure' else: Makefile = '/'.join([subdir,'Makefile']) makefile = '/'.join([subdir,'makefile']) configure = '/'.join([subdir, 'configure']) # Make sure there is make file before it really starts if u.isfile(Makefile) or u.isfile(makefile) or u.isfile(configure): os.system(MAKE) if INSTALL is not None: u.RunCommand(INSTALL).run() else: u.manualInstall(srcDir(testname), binDir()) u.manualInstall(subdir, binDir()) else: raise Exception("No necessary Makefile found!") else: u.logger.warning("%s -- Not necessary to build running binaries."% testname)
def install(should_identify=True): # Based on ideas from https://github.com/harvimt/quamash/blob/master/.travis.yml if should_identify: system_identify() td = Travis_Dispatcher() xqt( # Cached Downloads 'sudo mkdir -p /downloads', 'sudo chmod a+rw /downloads') sip_ver = 'sip-4.17' if not isfile('/downloads/sip.tar.gz'): wget('http://downloads.sourceforge.net/project/pyqt/sip/{}/{}'. format(sip_ver, _gz(sip_ver)), '/downloads/sip.tar.gz') # _`pyqt_ver`: Select a PyQt version. See also qt5_Linux_ and qt5_OS_X_. pyqt_ver = '5.5.1' pyqt_gpl_ver = 'PyQt-gpl-' + pyqt_ver if not isfile('/downloads/pyqt5.tar.gz'): wget('http://downloads.sourceforge.net/project/pyqt/PyQt5/PyQt-{}/{}'. format(pyqt_ver, _gz(pyqt_gpl_ver)), '/downloads/pyqt5.tar.gz') # Builds xqt('sudo mkdir -p /builds', 'sudo chmod a+rw /builds') # Qt5 td.qt5() # SIP. With Linux or OS_X, don't use the package manager to install these, # since they're installed for the system python, not the pyenv version # we're testing with. with pushd('/builds'): xqt('tar xzf /downloads/sip.tar.gz --keep-newer-files') chdir(sip_ver) xqt('python configure.py', 'make', 'sudo make install') # PyQt5 with pushd('/builds'): xqt('tar xzf /downloads/pyqt5.tar.gz --keep-newer-files') chdir(pyqt_gpl_ver) td.pyqt5_configure() xqt('make', 'sudo make install') # PCRE td.pcre() # Qutepart if build_os == 'Linux': set_display() xqt('sh -e /etc/init.d/xvfb start') # Install, which also builds Python C extensions. Use this instead of # ``build_ext`` so that Enki will have an already-installed qutepart, # rather than needing to regenrate the command below. xqt('python setup.py install')
def install_Windows(self): # ctags ctags_zip = os.path.join(DOWNLOADS, CTAGS_VER + '.zip') if not isfile(ctags_zip): wget('http://sourceforge.net/projects/ctags/files/ctags/5.8/{}.zip'. format(CTAGS_VER), ctags_zip) unzip(ctags_zip, CTAGS_VER + '/ctags.exe')
def load_hosts(self): """Load hosts from file""" if not isfile(self.hostfile): self.active_host = [] self.available_hosts = [] else: with open(self.hostfile, 'r') as f: hosts = json.load(f) self.active_host = hosts['active'] self.available_hosts = hosts['available'] if any(isinstance(i, list) for i in self.active_host): print( "Error: more than one active host. Please check config file ~/.config/loon/host.json and modify or remove it if necessary." ) # Python code to remove duplicate elements def RemoveDups(duplicate): final_list = [] flag = False for num in duplicate: if num not in final_list: final_list.append(num) else: flag = True return final_list, flag self.available_hosts, flag = RemoveDups(self.available_hosts) if flag: # Save unique hosts immediately self.save_hosts() return
def run(self, inpath, outpath, n=sys.maxsize): if exists(outpath): exit("Warning: output already exists") elif isdir(inpath): self.classify_directory(inpath, outpath) elif isfile(inpath): self.classify_file(inpath, outpath)
def install(should_identify=True): if should_identify: system_identify() # Create a place to store downloads. if not isdir(DOWNLOADS): mkdir(DOWNLOADS) # Download and install PyQt5. Only download if we don't have a cached copy # available. install_PyQt5 = os.path.join(DOWNLOADS, 'install-PyQt5.exe') if not isfile(install_PyQt5): wget( 'http://downloads.sourceforge.net/project/pyqt/PyQt5/PyQt-5.5.1/' 'PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe', install_PyQt5) # See https://github.com/appveyor/ci/issues/363#issuecomment-148915001. xqt( 'REG ADD HKCU\\Software\\Python\\PythonCore\\3.4\\InstallPath /f /ve ' '/t REG_SZ /d C:\\Python34', install_PyQt5 + ' /S') # Download and compile PCRE. pcre_ver = 'pcre-8.38' pcre_zip = pcre_ver + '.zip' pcre_zip_path = os.path.join(DOWNLOADS, pcre_zip) if not isfile(pcre_zip_path): # Note: Don't use ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/, # because this sometimes hangs during download, causing the build to # fail. Instead, use the more reliable SourceForge mirror. wget( 'http://downloads.sourceforge.net/project/pcre/pcre/8.38/' + pcre_zip, pcre_zip_path) # See https://sevenzip.osdn.jp/chm/cmdline/commands/extract_full.htm. xqt('7z x {} > nul'.format(pcre_zip_path)) with pushd(pcre_ver): mkdir('build') chdir('build') xqt( 'cmake .. -DBUILD_SHARED_LIBS:BOOL=OFF -DPCRE_SUPPORT_UTF:BOOL=ON ' '-DPCRE_SUPPORT_JIT:BOOL=ON -G "Visual Studio 10 2010"', 'cmake --build . --config Release') # Install, which also builds Python C extensions. Use this instead of # ``build_ext`` so that Enki will have an already-installed qutepart, # rather than needing to regenrate the command below. xqt('python setup.py install --include-dir={}/build ' '--lib-dir={}/build/Release --force'.format(pcre_ver, pcre_ver))
def install_Windows(self): # ctags ctags_zip = os.path.join(DOWNLOADS, CTAGS_VER + '.zip') if not isfile(ctags_zip): wget( 'http://sourceforge.net/projects/ctags/files/ctags/5.8/{}.zip'. format(CTAGS_VER), ctags_zip) unzip(ctags_zip, CTAGS_VER + '/ctags.exe')
def install(should_identify=True): if should_identify: system_identify() # Create a place to store downloads. if not isdir(DOWNLOADS): mkdir(DOWNLOADS) # Download and install PyQt5. Only download if we don't have a cached copy # available. install_PyQt5 = os.path.join(DOWNLOADS, 'install-PyQt5.exe') if not isfile(install_PyQt5): wget('http://downloads.sourceforge.net/project/pyqt/PyQt5/PyQt-5.5.1/' 'PyQt5-5.5.1-gpl-Py3.4-Qt5.5.1-x32.exe', install_PyQt5) # See https://github.com/appveyor/ci/issues/363#issuecomment-148915001. xqt('REG ADD HKCU\\Software\\Python\\PythonCore\\3.4\\InstallPath /f /ve ' '/t REG_SZ /d C:\\Python34', install_PyQt5 + ' /S') # Download and compile PCRE. pcre_ver = 'pcre-8.38' pcre_zip = pcre_ver + '.zip' pcre_zip_path = os.path.join(DOWNLOADS, pcre_zip) if not isfile(pcre_zip_path): # Note: Don't use ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/, # because this sometimes hangs during download, causing the build to # fail. Instead, use the more reliable SourceForge mirror. wget('http://downloads.sourceforge.net/project/pcre/pcre/8.38/' + pcre_zip, pcre_zip_path) # See https://sevenzip.osdn.jp/chm/cmdline/commands/extract_full.htm. xqt('7z x {} > nul'.format(pcre_zip_path)) with pushd(pcre_ver): mkdir('build') chdir('build') xqt('cmake .. -DBUILD_SHARED_LIBS:BOOL=OFF -DPCRE_SUPPORT_UTF:BOOL=ON ' '-DPCRE_SUPPORT_JIT:BOOL=ON -G "Visual Studio 10 2010"', 'cmake --build . --config Release') # Install, which also builds Python C extensions. Use this instead of # ``build_ext`` so that Enki will have an already-installed qutepart, # rather than needing to regenrate the command below. xqt('python setup.py install --include-dir={}/build ' '--lib-dir={}/build/Release --force'.format(pcre_ver, pcre_ver))
def save_hosts(self): """Save hosts to file""" # if len(self.active_host)==0 or len(self.available_hosts)==0: # raise ValueError("Cannot save to file due to null host.") hosts = {'active': self.active_host, 'available': self.available_hosts} if not isfile(self.hostfile): # Create parent dir if hostfile does not exist create_parentdir(self.hostfile) with open(self.hostfile, 'w') as f: json.dump(hosts, f) return
def gen_template(self, input, output, dry_run=False): """Generate a PBS template Args: input: a string representing the path to template file output: a string representing the path to output file dyr_run: if `True`, dry run the code Returns: None """ if output is None: output = os.path.join(os.getcwd(), 'work.pbs') print("=> Generating %s" % output) if dry_run: sys.exit(0) if isfile(output): print("Warning: the output file exists, it will be overwritten.") if input is None: with io.open(output, 'w', encoding='utf-8', newline='\n') as f: with open(self.tmp_header, 'r') as header: for i in header: print(i, file=f, sep='', end="") with io.open(output, 'a', encoding='utf-8', newline='\n') as f: with open(self.tmp_cmds, 'r') as cmds: for i in cmds: print(i, file=f, sep='', end="") else: if not isfile(input): print("Error: cannot find the template file.") sys.exit(1) with io.open(output, 'w', encoding='utf-8', newline='\n') as f: with open(input, 'r') as inf: for i in inf: print(i, file=f, sep='', end="") print("=> Done.") return
def iterate_xz_files(self,path = None): if path is None: path = os.getcwd() for f in os.listdir(path): if utils.isfile(f): s.check_call('xz -zkv %s' % f, shell=True) elif utils.isDir(f): subdir = path + '/' + f os.chdir(f) self.iterate_xz_files(subdir) os.chdir(path) else: utils.logger.error("Error: Unknown file or directory type") return False return True
def install(should_identify=True): if should_identify: system_identify() # Create a place to store downloads. if not isdir(DOWNLOADS): mkdir(DOWNLOADS) # Download and compile PCRE. pcre_raw_ver = '8.42' pcre_ver = 'pcre-' + pcre_raw_ver pcre_zip = pcre_ver + '.zip' pcre_zip_path = os.path.join(DOWNLOADS, pcre_zip) if not isfile(pcre_zip_path): # Note: Don't use ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/, # because this sometimes hangs during download, causing the build to # fail. Instead, use the more reliable SourceForge mirror. wget( 'http://downloads.sourceforge.net/project/pcre/pcre/{}/{}'.format( pcre_raw_ver, pcre_zip), pcre_zip_path) # See https://sevenzip.osdn.jp/chm/cmdline/commands/extract_full.htm. xqt('7z x {} > nul'.format(pcre_zip_path)) with pushd(pcre_ver): mkdir('build') chdir('build') # Per https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2014%202015.html, # add the Win64 string for 64-bit Python. use_Win64 = ' Win64' if is_64bits else '' xqt( 'cmake .. -DBUILD_SHARED_LIBS:BOOL=OFF -DPCRE_SUPPORT_UTF:BOOL=ON ' '-DPCRE_SUPPORT_JIT:BOOL=ON -G "Visual Studio 14 2015{}"'.format( use_Win64), 'cmake --build . --config Release') # First, build Python C extensions. Use this instead of # ``build_ext`` so that Enki will have an already-installed qutepart, # rather than needing to regenrate the command below. xqt('python setup.py build_ext --include-dir={}/build ' '--lib-dir={}/build/Release --force'.format(pcre_ver, pcre_ver)) # Next, install it along with its dependencies. See comments at # ``install_requires`` on why this is necessary. xqt('python -m pip install -e .')
def install(should_identify=True): if should_identify: system_identify() # Create a place to store downloads. if not isdir(DOWNLOADS): mkdir(DOWNLOADS) # Download and compile PCRE. pcre_raw_ver = '8.39' pcre_ver = 'pcre-' + pcre_raw_ver pcre_zip = pcre_ver + '.zip' pcre_zip_path = os.path.join(DOWNLOADS, pcre_zip) if not isfile(pcre_zip_path): # Note: Don't use ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/, # because this sometimes hangs during download, causing the build to # fail. Instead, use the more reliable SourceForge mirror. wget('http://downloads.sourceforge.net/project/pcre/pcre/{}/{}'. format(pcre_raw_ver, pcre_zip), pcre_zip_path) # See https://sevenzip.osdn.jp/chm/cmdline/commands/extract_full.htm. xqt('7z x {} > nul'.format(pcre_zip_path)) with pushd(pcre_ver): mkdir('build') chdir('build') # Per https://cmake.org/cmake/help/latest/generator/Visual%20Studio%2014%202015.html, # add the Win64 string for 64-bit Python. use_Win64 = ' Win64' if is_64bits else '' xqt('cmake .. -DBUILD_SHARED_LIBS:BOOL=OFF -DPCRE_SUPPORT_UTF:BOOL=ON ' '-DPCRE_SUPPORT_JIT:BOOL=ON -G "Visual Studio 14 2015{}"'. format(use_Win64), 'cmake --build . --config Release') # First, build Python C extensions. Use this instead of # ``build_ext`` so that Enki will have an already-installed qutepart, # rather than needing to regenrate the command below. xqt('python setup.py build_ext --include-dir={}/build ' '--lib-dir={}/build/Release --force'.format(pcre_ver, pcre_ver)) # Next, install it along with its dependencies. See comments at # ``install_requires`` on why this is necessary. xqt('python -m pip install -e .')
def add_local_nzb(): log("add_local_nzb:") if not utils.exists(NZB_FOLDER): __settings__.openSettings() return None dialog = xbmcgui.Dialog() nzb_file = dialog.browse(1, 'Pick a NZB', 'files', '.nzb', False, False, NZB_FOLDER) # XBMC outputs utf-8 path = unicode(nzb_file, 'utf-8') log("add_local_nzb: path: %s" % path) if not utils.isfile(path): return None else: params = dict() # Fixing the naming of nzb according to SAB rules params['nzbname'] = m_nzb.Nzbname(os.path.basename(path)).final_name params['nzb'] = path if IS_SAB_LOCAL: params['type'] = 'add_local' else: params['type'] = 'add_file' return params
def generate(fname, outdir='.'): if not utils.isfile(fname): print( "error: '{}' does not exist or could not be opened".format(fname)) return else: prefix = utils.get_file_prefix(fname) # save prefix for naming things if not prefix: print("error: '{}' must be named '<prefix>.idl'".format(fname)) return # parse idl declarations into python dictionary decls = json.loads( subprocess.check_output(["./idl_to_json", fname]).decode('utf-8')) funcsdict = decls['functions'] typesdict = decls['types'] # generate files with open('{}/{}.proxy.cpp'.format(outdir.rstrip('/'), prefix), 'w+') as f: f.write(generate_proxy(funcsdict, typesdict, prefix)) with open('{}/{}.stub.cpp'.format(outdir.rstrip('/'), prefix), 'w+') as f: f.write(generate_stub(funcsdict, typesdict, prefix))
def gen_pbs(self, template, samplefile, mapfile, outdir, _logger, pbs_mode=True, dry_run=False): """Generate a batch of (script) files (PBS tasks) based on template and mapping file Args: template: a string representing the path to the template file samplefile: a string representing the path to the sample file mapfile: a string representing the path to the mapping file outdir: a string representing the path to output directory _logger: the logging logger pbs_mode: if `True`, use PBS mode dry_run: if `True`, dry run the code Returns: None """ if not isdir(outdir): print("Directory %s does not exist, creating it" % outdir) os.makedirs(outdir) if not isfile(template): print("Error: file %s does not exist" % template) if not isfile(samplefile): print("Error: file %s does not exist" % samplefile) if not isfile(mapfile): print("Error: file %s does not exist" % mapfile) print("=====================") print("Output path : " + outdir) if pbs_mode: print("PBS Template: " + template) else: print("Template: " + template) print("Sample file : " + samplefile) print("Mapping file: " + mapfile) print("=====================") if dry_run: sys.exit(0) print("=> Reading %s ..." % samplefile) sample_data = read_csv(samplefile) print("=> Reading %s ..." % mapfile) map_data = read_csv(mapfile) # Check if input files are valid check_list = [i[0] for i in sample_data] check_list = set(check_list) if len(sample_data) != len(check_list): print("Error: the first column is not unique!") sys.exit(1) for row in map_data: if len(row) != 2: print("Error: only two columns are quired in mapfile!") try: _ = int(row[1]) except Exception: print( "Error: the second column must be (or can be transformed to) an integer!" ) sys.exit(1) print("=> Reading %s ..." % template) with open(template, 'r') as f: temp_data = f.read() print("Generating...") for row in sample_data: if pbs_mode: pbsfile = os.path.join(outdir, row[0] + '.pbs') else: pbsfile = os.path.join(outdir, row[0]) _logger.info("Generating %s" % pbsfile) content = temp_data for i in map_data: try: _logger.info("Replacing %s with %s" % (i[0], row[int(i[1])])) content = content.replace(i[0], row[int(i[1])]) except Exception: print( "Error: the second column out of range for label %s!" % i[0]) with io.open(pbsfile, 'w', encoding='utf-8', newline='\n') as f: f.write(content) print("Done.") return
def _load_latest_checkpoint(cls, dir): last_chkpt_path = join_path(dir, cls.CHECKPOINT_LAST) if isfile(last_chkpt_path): return th.load(last_chkpt_path) return None
def cmd(self, commands, _logger=None, run_file=False, data_dir=None, remote_file=False, dir='/tmp', prog=None, dry_run=False): """Run command(s) in active remote host using channel session Therefore, `open_channel` in `connect` method must be `True` before using it. Args: commands: commands/scripts run on active remote host _logger: the logging logger run_file: if `True`, run scripts instead of commands data_dir: a path representing data directory remote_file: if `True`, collect input from remote host instead of local machine dir: Remote directory for storing local scripts prog: a string representing the program to run the commands dry_run: if `True`, dry run the code Returns: A string containing result information """ if dry_run: print("Running", "files:" if run_file else "commands:", commands) sys.exit(0) if not run_file: self.connect() self.channel.execute(commands) else: # Run scripts _logger.info(commands) scripts = commands # commands are scripts here if remote_file: # Run remote scripts # Support some wildcards # *,?,{} wildcards = r'\*|\?|\{\}' matches = [ re.compile(wildcards).search(i) is not None for i in scripts ] if any(matches): commands_1 = list(map(lambda x: 'ls ' + x, scripts)) commands_1 = ';'.join(commands_1) self.connect() self.channel.execute(commands_1) scripts = self.get_result(print_info=False).split('\n') if '' in scripts: scripts.remove('') if prog is None: commands_1 = list(map(lambda x: 'chmod u+x ' + x, scripts)) commands_1 = ';'.join(commands_1) commands_2 = ';'.join(scripts) commands = commands_1 + ';' + commands_2 else: commands = list( map(lambda x: '{} '.format(prog) + x, scripts)) commands = ';'.join(commands) _logger.info(commands) self.connect() print("=> Getting results:") self.channel.execute(commands) else: # Run local scripts # # 1) upload self.upload(scripts, dir, _logger) if data_dir is not None: self.upload(data_dir, dir, _logger) # 2) get all file names if len(scripts) == 1: if isdir(scripts[0]): if list(scripts[0])[-1] == '/': dir = os.path.join( dir, os.path.basename(os.path.dirname(scripts[0]))) else: dir = os.path.join(dir, os.path.basename(scripts[0])) scripts = glob.glob(scripts[0] + '/*') filelist = [] for fp in scripts: _logger.info("fp:%s" % fp) fs = glob.glob(fp) _logger.info("fs:%s" % fs) for f in fs: _logger.info("f:%s" % f) if isdir(f): print( "Warning: directory %s is detected, note anything in it will be ignored to execute." % f) elif isfile(f): filelist.append(f) else: print('Error: file %s does not exist.' % f) sys.exit(1) filelist = list(map(os.path.basename, filelist)) _logger.info(filelist) # 3) run them one by one scripts = list(map(lambda x: '/'.join([dir, x]), filelist)) if prog is None: commands_1 = list(map(lambda x: 'chmod u+x ' + x, scripts)) commands_1 = ';'.join(commands_1) commands_2 = ';'.join(scripts) commands = commands_1 + ';' + commands_2 else: commands = list( map(lambda x: '{} '.format(prog) + x, scripts)) commands = ';'.join(commands) _logger.info(commands) self.connect() print("=> Getting results:") self.channel.execute(commands) datalist = self.get_result() # Return a string containing output return "".join(datalist)
def sub(self, host, tasks, remote, workdir, _logger, dry_run=False): """Submit pbs tasks Args: host: a host object tasks: a list of PBS files, glob pattern is supported remote: if `True`, means that PBS task files are located at the active remote host workdir: a directory representing the working directory _logger: the logging logger dry_run: if `True`, dry run the code Returns: A list of files """ print('NOTE: PBS file must be LF mode (Unix), not CRLF mode (Windows)') print('====================================================') filelist = [] if remote: tasks = ' '.join(tasks) host.connect() _logger.info('ls -p ' + tasks) host.channel.execute('ls -p ' + tasks) filelist = host.get_result(print_info=False).split('\n') if '' in filelist: filelist.remove('') fl_bk = filelist.copy() for f in fl_bk: if len(f) > 1 and (f[-1] == '/' or f[-1] == ':'): filelist.remove(f) if f == '' or f == ' ': filelist.remove(f) _logger.info(filelist) if workdir is None: workdir = '/tmp' cmds = 'cd {}; for i in {}; do qsub $i; done'.format( workdir, ' '.join(filelist)) if dry_run: print(cmds) sys.exit(0) _logger.info(cmds) host.cmd(cmds, _logger=_logger) else: if workdir is None: workdir = os.getcwd() for fp in tasks: fs = glob.glob(fp) for f in fs: if isdir(f): print( "Warning: directory %s is detected, note anything in it will be ignored to execute." % f) elif isfile(f): filelist.append(f) cmds = 'cd ' + workdir + ';qsub ' + f if dry_run: print(cmds) else: _logger.info(cmds) run(cmds, shell=True) else: print('Error: file %s does not exist.' % f) sys.exit(1) return filelist