def CreateFulltextIdsFile(ids_output_file, log_file_name): elasticsearch_access_conf = "/usr/local/var/lib/tuelib/Elasticsearch.conf" if os.access(elasticsearch_access_conf, os.F_OK): util.ExecOrDie("/usr/local/bin/extract_existing_fulltext_ids.sh", [ids_output_file], log_file_name) else: # Skip if configuration is not present util.ExecOrDie(util.Which("truncate"), ["-s", "0", log_file_name]) util.ExecOrDie(util.Which("echo"), [ "Skip extraction since " + elasticsearch_access_conf + " not present" ], log_file_name)
def CheckGdbVersion(): """ Get the version of GDB and check to make sure it's a version which can be used for DrDebug. Exit if the version of GDB is too old to run with DrDebug. @return no return @return NOTE: side effect sets global gdb_path """ global gdb_path # Get GDB path # # import pdb; pdb.set_trace() gdb_path = util.Which('gdb') if not gdb_path: parser.error('gdb not found in PATH') # Path to GDB and version info # cmd = gdb_path + ' --version' p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (stdout, stderr) = p.communicate() # Parse the version string to get the actual version number. # Here's an example of the entire version string: # # $ gdb --version # GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6) # Copyright (C) 2010 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. Type "show copying" # and "show warranty" for details. # This GDB was configured as "x86_64-redhat-linux-gnu". # For bug reporting instructions, please see: # <http://www.gnu.org/software/gdb/bugs/>. # line = stdout.split('\n', 1)[0] last = line.split()[-1] float_pattern = '[1-9][0-9]*\.?[0-9]*' f = re.search(float_pattern, last) version = f.group(0) # Check to make sure it's at least the base version required for DrDebug. # if float(version) < float(config.gdb_base_version): msg.PrintAndExit('gdb must be at least version: ' + config.gdb_base_version + '\nVersion found was: ' + version + '\nPath to gdb binary: ' + gdb_path)
def DumpMongoDB(config, log_file_name="/dev/stderr"): # Backup to intermediate hidden directory that is exluded from backup # to prevent inconsistent saving dump_base_path = config.get("LocalConfig", "dump_base_path") dump_root = config.get("LocalConfig", "dump_root") intermediate_dump_dir = dump_base_path + '/.' + dump_root util.ExecOrDie(util.Which("mongodump"), ["--out=" + intermediate_dump_dir, "--gzip"], log_file_name) final_dump_dir = dump_base_path + '/' + dump_root if os.path.exists(final_dump_dir) and os.path.isdir(final_dump_dir): rmtree(final_dump_dir) move(intermediate_dump_dir, final_dump_dir)
def ImportOADOIsToMongo(update_list, source_directory=None, log_file_name="/dev/stderr"): if not source_directory is None: os.chdir(source_directory) imported_symlinks_directory = os.getcwd() + "/imported" for filename in update_list: imported_symlink_full_path = imported_symlinks_directory + "/" + filename if os.path.islink(imported_symlink_full_path): print("Skipping " + filename + " since apparently already imported") continue print("Importing \"" + filename + "\"") util.ExecOrDie(util.Which("import_oadois_to_mongo.sh"), [filename], log_file_name) CreateImportedSymlink(filename, imported_symlink_full_path)
def ImportIntoVuFind(title_pattern, authority_pattern, log_file_name): vufind_dir = os.getenv("VUFIND_HOME") if vufind_dir == None: util.Error("VUFIND_HOME not set, cannot start solr import!") # import title data title_index = 'biblio' title_args = [sorted(glob.glob(title_pattern), reverse=True)[0]] if len(title_args) != 1: util.Error("\"" + title_pattern + "\" matched " + str(len(title_args)) + " files! (Should have matched exactly 1 file!)") ClearSolrIndex(title_index) util.ExecOrDie(vufind_dir + "/import-marc.sh", title_args, log_file_name) OptimizeSolrIndex(title_index) # import authority data authority_index = 'authority' authority_args = [sorted(glob.glob(authority_pattern), reverse=True)[0]] if len(authority_args) != 1: util.Error("\"" + authority_pattern + "\" matched " + str(len(authority_args)) + " files! (Should have matched exactly 1 file!)") ClearSolrIndex(authority_index) util.ExecOrDie(vufind_dir + "/import-marc-auth.sh", authority_args, log_file_name) OptimizeSolrIndex(authority_index) util.ExecOrDie( util.Which("sudo"), ["-u", "solr", "-E", vufind_dir + "/index-alphabetic-browse.sh"], log_file_name) # cleanup logs util.ExecOrDie("/usr/local/bin/summarize_logs", [vufind_dir + "/import/solrmarc.log", solrmarc_log_summary]) util.ExecOrDie("/usr/local/bin/log_rotate", [vufind_dir + "/import/", "solrmarc\\.log"]) util.ExecOrDie("/usr/local/bin/summarize_logs", [log_file_name, import_log_summary]) util.ExecOrDie( "/usr/local/bin/log_rotate", [os.path.dirname(log_file_name), os.path.basename(log_file_name)])
def GetArgs(self, args, parser, options): """ Get binary and arguments passed to the script. If using option '--pid' then can only have the binary. Otherwise can give command line with binary/options. NOTE: Side effect adds command line to 'options'. @param args List of arguments from cmd line @param parser Command line parser @param options Options given on cmd line @return no return value """ num_args = len(args) if (hasattr(options, 'pid') and options.pid): # If user gave --pid, then command line must be just the binary. # if num_args > 1: parser.error('Only binary, with no arguments, allowed on command\n'\ 'line with \'--pid\'.') elif num_args == 0: parser.error('Must give a binary with \'--pid PID\'.') elif num_args == 0: # User has not given a command line. # parser.error('Must either: Include binary and arguments after\n'\ 'string \'--\' or use option \'--pid PID\', the string \'--\' and binary.') # Check to make sure the binary exists and is executable. # cmd = args[0] if not util.Which(cmd): parser.error('Binary \'%s\' either not found, or not executable' % cmd) # Add the command line to the options # cmd_line = " ".join(args) setattr(options, 'command', cmd_line)
def GetArgs(self, args, parser, options): """ Get pinball to replay and corresponding binary needed to replay pinball with GDB. NOTE: Side effect adds command line to 'options'. @param args List of arguments from cmd line @param parser Command line parser @param options Options given on cmd line @return tuple with: @return - old pinball to replay @return - new pinball to generate """ num_args = len(args) if num_args != 2: parser.error( 'Must give \'--\' followed by pinball_basename and program_binary') # Check to make sure the binary exists, is executable and the pinball files exists. # replay_pb = args[0] cmd = args[1] if len(glob.glob(replay_pb + '.*')) < 7: parser.error('Invalid pinball name: %s' % replay_pb) if not util.Which(cmd): parser.error( 'Binary \'%s\' either not found, or not executable' % cmd) # Add the command to the options # setattr(options, 'command', cmd) # For GDB, there is no name for the new pinball which will be created, hence return 'none'. # log_pb = None return replay_pb, log_pb
def CheckGdb(options, check_version=True): """ Check GDB to see if it's compatible with the DrDebug environment. Make sure GDB is built with Python support and, if required, GDB version is new enough. A base version of GDB is required because PinADX uses a new shared-library related remote command (only available in/after version 7.4) to convey shared library load locations etc to GDB. Prints a warning message if either check fails, but continues to run. NOTE: Sets global attribute gdb_path @return no return """ def RunCmd(cmd): """ Run a command and return stdout/stderrr. """ p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) stdout, stderr = p.communicate() return stdout, stderr global gdb_path # Get GDB from either the user option or the current PATH. # # import pdb; pdb.set_trace() if options.gdb: gdb_path = util.Which(options.gdb) else: gdb_path = util.Which('gdb') if not gdb_path: if options.gdb: msg.PrintAndExit('gdb binary not found: ' + options.gdb) else: msg.PrintAndExit('gdb not found in PATH') # Is there Python support in GDB? # err_str = 'Python scripting is not supported in this copy of GDB' cmd = gdb_path + ' --batch -ex \'python print "hello"\'' stdout, stderr = RunCmd(cmd) if err_str in stderr: python_support = False msg.PrintMsg( '\nWARNING: This version of gdb (%s) does not support Python.\n' 'As a result, \'pin\' commands will not work. Try \'monitor\' versions\n' 'of the commands instead.\n' % (gdb_path)) else: python_support = True if not check_version: return # Get version info from gdb # cmd = gdb_path + ' --version' stdout, stderr = RunCmd(cmd) # Parse the version string to get the actual version number. # Here's an example of the entire version string: # # $ gdb --version # GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6) # Copyright (C) 2010 Free Software Foundation, Inc. # License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html> # This is free software: you are free to change and redistribute it. # There is NO WARRANTY, to the extent permitted by law. Type "show copying" # and "show warranty" for details. # This GDB was configured as "x86_64-redhat-linux-gnu". # For bug reporting instructions, please see: # <http://www.gnu.org/software/gdb/bugs/>. # line = stdout.split('\n', 1)[0] last = line.split()[-1] float_pattern = '[1-9][0-9]*\.?[0-9]*' f = re.search(float_pattern, last) version = f.group(0) major = int(version.split('.')[0]) minor = int(version.split('.')[1]) cmajor = int(config.gdb_base_version.split('.')[0]) cminor = int(config.gdb_base_version.split('.')[1]) # Check to make sure it's at least the base version required for DrDebug. # if (major < cmajor) or ((major == cmajor) and (minor < cminor)): if python_support: msg.PrintMsg('\n') msg.PrintMsg( 'WARNING: This version of gdb is: %s When using versions of gdb < %s' '\nthe script will run, but with reduced functionality.\n' % (version, config.gdb_base_version))
def Run(self): """Run the scripts required to run simpoint and generate a region CSV file with the results.""" msg.PrintMsg('') # Get the command line options # (options, args) = self.ParseCommandLine() # Make sure required utilities exist and are executable. # # import pdb; pdb.set_trace() if util.Which(self.simpoint_bin) == None: msg.PrintAndExit('simpoint binary not in path.\n' 'Add directory where it exists to your path.') if util.Which(self.csv_bin) == None: msg.PrintAndExit( 'script to generate the region CSV file not in path.\n' 'Add directory where it exists to your path.') # Go to the data directory. Both utilities should be run in this directory. # os.chdir(options.data_dir) # Always copy the specific BBV to the generic name used by simpoint. # If LDV file exists, then copy it to generic name. # if os.path.isfile(self.generic_bbv_name): os.remove(self.generic_bbv_name) shutil.copy(options.bbv_file, self.generic_bbv_name) ldv_file = options.bbv_file.replace('.bb', '.ldv') if os.path.isfile(ldv_file): if os.path.isfile(self.generic_ldv_name): os.remove(self.generic_ldv_name) shutil.copy(options.bbv_file.replace('.bb', '.ldv'), self.generic_ldv_name) # Get the instruction count and slice_size from the BBV file. # try: f = open(self.generic_bbv_name) except IOError: msg.PrintAndExit('problem opening BBV file: ' + self.generic_bbv_name) instr_count = 0 slice_size = 0 for line in f.readlines(): if line.find('Dynamic') != -1: tmp = line.split() count = tmp[len(tmp) - 1] if count > instr_count: instr_count = int(count) if line.find('SliceSize') != -1: tmp = line.split() size = tmp[len(tmp) - 1] if size > slice_size: slice_size = int(size) # Check to make sure instruction count > slice_size. # if slice_size > instr_count: import locale locale.setlocale(locale.LC_ALL, "") msg.PrintAndExit('Slice size is greater than the number of instructions. Reduce parameter \'slice_size\'.' + \ '\nInstruction count: ' + locale.format('%14d', int(instr_count), True) + \ '\nSlice size: ' + locale.format('%14d', int(slice_size), True)) if options.ldv: # Run to generate regions CSV file using both BBV and LDV files. # result = self.NormProjectBBV(options) util.CheckResult( result, options, 'normalizing and projecting BBV with: ' + self.csv_bin) result = self.NormWeightLDV(options) util.CheckResult( result, options, 'normalizing and applying weights to LDV with: ' + self.csv_bin) result = self.CombineFreqVectFiles(options) util.CheckResult( result, options, 'scaling and combining BBV and LDV with: ' + self.csv_bin) result = self.RunSimpoint(options) util.CheckResult( result, options, 'generating clusters (regions) with: ' + self.simpoint_bin) result, regions_csv_file = self.GenRegionCSVFile(options) util.CheckResult(result, options, 'creating regions CSV file with: ' + self.csv_bin) msg.PrintMsg('\nRegions CSV file: ' + os.path.join(options.data_dir, regions_csv_file)) else: # Run scripts to generate regions CSV file using the new method with just the BBV file. # result = self.NormProjectBBV(options) util.CheckResult( result, options, 'normalizing and projecting BBV with: ' + self.csv_bin) result = self.RunSimpoint(options) util.CheckResult( result, options, 'generating clusters (regions) with: ' + self.simpoint_bin) result, regions_csv_file = self.GenRegionCSVFile(options) util.CheckResult(result, options, 'creating regions CSF file with: ' + self.csv_bin) msg.PrintMsg('\nRegions CSV file: ' + os.path.join(options.data_dir, regions_csv_file)) return result
def _GetGomaFromPath(): """Returns goma directory from PATH, otherwise is None.""" gomacc_path = util.Which('gomacc') if gomacc_path is not None: return os.path.dirname(gomacc_path) return None
def GetPathOrDie(executable): full_path = util.Which(executable) if full_path is None: util.Error("Can't find \"" + executable + "\" in our PATH!") return full_path
def ExtractOADOIURLs(share_directory, all_dois_file, urls_file, log_file_name): print("Extract URLs for DOI's in " + all_dois_file) util.ExecOrDie(util.Which("extract_oadoi_urls.sh"), [share_directory + '/' + all_dois_file, urls_file], log_file_name)