def CreateFulltextIdsFile(ids_output_file, log_file_name):
elasticsearch_access_conf = "/usr/local/var/lib/tuelib/Elasticsearch.conf"
if os.access(elasticsearch_access_conf, os.F_OK):
util.ExecOrDie("/usr/local/bin/extract_existing_fulltext_ids.sh",
[ids_output_file], log_file_name)
else: # Skip if configuration is not present
util.ExecOrDie(util.Which("truncate"), ["-s", "0", log_file_name])
util.ExecOrDie(util.Which("echo"), [
"Skip extraction since " + elasticsearch_access_conf +
" not present"
], log_file_name)
def CheckGdbVersion():
"""
Get the version of GDB and check to make sure it's a version which can
be used for DrDebug. Exit if the version of GDB is too old to run
with DrDebug.
@return no return
@return NOTE: side effect sets global gdb_path
"""
global gdb_path
# Get GDB path
#
# import pdb; pdb.set_trace()
gdb_path = util.Which('gdb')
if not gdb_path:
parser.error('gdb not found in PATH')
# Path to GDB and version info
#
cmd = gdb_path + ' --version'
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(stdout, stderr) = p.communicate()
# Parse the version string to get the actual version number.
# Here's an example of the entire version string:
#
# $ gdb --version
# GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6)
# Copyright (C) 2010 Free Software Foundation, Inc.
# License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
# This is free software: you are free to change and redistribute it.
# There is NO WARRANTY, to the extent permitted by law. Type "show copying"
# and "show warranty" for details.
# This GDB was configured as "x86_64-redhat-linux-gnu".
# For bug reporting instructions, please see:
# <http://www.gnu.org/software/gdb/bugs/>.
#
line = stdout.split('\n', 1)[0]
last = line.split()[-1]
float_pattern = '[1-9][0-9]*\.?[0-9]*'
f = re.search(float_pattern, last)
version = f.group(0)
# Check to make sure it's at least the base version required for DrDebug.
#
if float(version) < float(config.gdb_base_version):
msg.PrintAndExit('gdb must be at least version: ' +
config.gdb_base_version + '\nVersion found was: ' +
version + '\nPath to gdb binary: ' + gdb_path)
def DumpMongoDB(config, log_file_name="/dev/stderr"):
# Backup to intermediate hidden directory that is exluded from backup
# to prevent inconsistent saving
dump_base_path = config.get("LocalConfig", "dump_base_path")
dump_root = config.get("LocalConfig", "dump_root")
intermediate_dump_dir = dump_base_path + '/.' + dump_root
util.ExecOrDie(util.Which("mongodump"),
["--out=" + intermediate_dump_dir, "--gzip"], log_file_name)
final_dump_dir = dump_base_path + '/' + dump_root
if os.path.exists(final_dump_dir) and os.path.isdir(final_dump_dir):
rmtree(final_dump_dir)
move(intermediate_dump_dir, final_dump_dir)
def ImportOADOIsToMongo(update_list,
source_directory=None,
log_file_name="/dev/stderr"):
if not source_directory is None:
os.chdir(source_directory)
imported_symlinks_directory = os.getcwd() + "/imported"
for filename in update_list:
imported_symlink_full_path = imported_symlinks_directory + "/" + filename
if os.path.islink(imported_symlink_full_path):
print("Skipping " + filename +
" since apparently already imported")
continue
print("Importing \"" + filename + "\"")
util.ExecOrDie(util.Which("import_oadois_to_mongo.sh"), [filename],
log_file_name)
CreateImportedSymlink(filename, imported_symlink_full_path)
def ImportIntoVuFind(title_pattern, authority_pattern, log_file_name):
vufind_dir = os.getenv("VUFIND_HOME")
if vufind_dir == None:
util.Error("VUFIND_HOME not set, cannot start solr import!")
# import title data
title_index = 'biblio'
title_args = [sorted(glob.glob(title_pattern), reverse=True)[0]]
if len(title_args) != 1:
util.Error("\"" + title_pattern + "\" matched " +
str(len(title_args)) +
" files! (Should have matched exactly 1 file!)")
ClearSolrIndex(title_index)
util.ExecOrDie(vufind_dir + "/import-marc.sh", title_args, log_file_name)
OptimizeSolrIndex(title_index)
# import authority data
authority_index = 'authority'
authority_args = [sorted(glob.glob(authority_pattern), reverse=True)[0]]
if len(authority_args) != 1:
util.Error("\"" + authority_pattern + "\" matched " +
str(len(authority_args)) +
" files! (Should have matched exactly 1 file!)")
ClearSolrIndex(authority_index)
util.ExecOrDie(vufind_dir + "/import-marc-auth.sh", authority_args,
log_file_name)
OptimizeSolrIndex(authority_index)
util.ExecOrDie(
util.Which("sudo"),
["-u", "solr", "-E", vufind_dir + "/index-alphabetic-browse.sh"],
log_file_name)
# cleanup logs
util.ExecOrDie("/usr/local/bin/summarize_logs",
[vufind_dir + "/import/solrmarc.log", solrmarc_log_summary])
util.ExecOrDie("/usr/local/bin/log_rotate",
[vufind_dir + "/import/", "solrmarc\\.log"])
util.ExecOrDie("/usr/local/bin/summarize_logs",
[log_file_name, import_log_summary])
util.ExecOrDie(
"/usr/local/bin/log_rotate",
[os.path.dirname(log_file_name),
os.path.basename(log_file_name)])
def GetArgs(self, args, parser, options):
"""
Get binary and arguments passed to the script. If using option
'--pid' then can only have the binary. Otherwise can give command line
with binary/options.
NOTE: Side effect adds command line to 'options'.
@param args List of arguments from cmd line
@param parser Command line parser
@param options Options given on cmd line
@return no return value
"""
num_args = len(args)
if (hasattr(options, 'pid') and options.pid):
# If user gave --pid, then command line must be just the binary.
#
if num_args > 1:
parser.error('Only binary, with no arguments, allowed on command\n'\
'line with \'--pid\'.')
elif num_args == 0:
parser.error('Must give a binary with \'--pid PID\'.')
elif num_args == 0:
# User has not given a command line.
#
parser.error('Must either: Include binary and arguments after\n'\
'string \'--\' or use option \'--pid PID\', the string \'--\' and binary.')
# Check to make sure the binary exists and is executable.
#
cmd = args[0]
if not util.Which(cmd):
parser.error('Binary \'%s\' either not found, or not executable' %
cmd)
# Add the command line to the options
#
cmd_line = " ".join(args)
setattr(options, 'command', cmd_line)
def GetArgs(self, args, parser, options):
"""
Get pinball to replay and corresponding binary needed to replay pinball
with GDB.
NOTE: Side effect adds command line to 'options'.
@param args List of arguments from cmd line
@param parser Command line parser
@param options Options given on cmd line
@return tuple with:
@return - old pinball to replay
@return - new pinball to generate
"""
num_args = len(args)
if num_args != 2:
parser.error(
'Must give \'--\' followed by pinball_basename and program_binary')
# Check to make sure the binary exists, is executable and the pinball files exists.
#
replay_pb = args[0]
cmd = args[1]
if len(glob.glob(replay_pb + '.*')) < 7:
parser.error('Invalid pinball name: %s' % replay_pb)
if not util.Which(cmd):
parser.error(
'Binary \'%s\' either not found, or not executable' % cmd)
# Add the command to the options
#
setattr(options, 'command', cmd)
# For GDB, there is no name for the new pinball which will be created, hence return 'none'.
#
log_pb = None
return replay_pb, log_pb
def CheckGdb(options, check_version=True):
"""
Check GDB to see if it's compatible with the DrDebug environment.
Make sure GDB is built with Python support and, if required, GDB version is
new enough. A base version of GDB is required because PinADX uses a new
shared-library related remote command (only available in/after version 7.4)
to convey shared library load locations etc to GDB.
Prints a warning message if either check fails, but continues to run.
NOTE: Sets global attribute gdb_path
@return no return
"""
def RunCmd(cmd):
"""
Run a command and return stdout/stderrr.
"""
p = subprocess.Popen(cmd,
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
return stdout, stderr
global gdb_path
# Get GDB from either the user option or the current PATH.
#
# import pdb; pdb.set_trace()
if options.gdb:
gdb_path = util.Which(options.gdb)
else:
gdb_path = util.Which('gdb')
if not gdb_path:
if options.gdb:
msg.PrintAndExit('gdb binary not found: ' + options.gdb)
else:
msg.PrintAndExit('gdb not found in PATH')
# Is there Python support in GDB?
#
err_str = 'Python scripting is not supported in this copy of GDB'
cmd = gdb_path + ' --batch -ex \'python print "hello"\''
stdout, stderr = RunCmd(cmd)
if err_str in stderr:
python_support = False
msg.PrintMsg(
'\nWARNING: This version of gdb (%s) does not support Python.\n'
'As a result, \'pin\' commands will not work. Try \'monitor\' versions\n'
'of the commands instead.\n' % (gdb_path))
else:
python_support = True
if not check_version:
return
# Get version info from gdb
#
cmd = gdb_path + ' --version'
stdout, stderr = RunCmd(cmd)
# Parse the version string to get the actual version number.
# Here's an example of the entire version string:
#
# $ gdb --version
# GNU gdb (GDB) Red Hat Enterprise Linux (7.2-60.el6)
# Copyright (C) 2010 Free Software Foundation, Inc.
# License GPLv3+: GNU GPL version 3 or later <http://gnu.org/licenses/gpl.html>
# This is free software: you are free to change and redistribute it.
# There is NO WARRANTY, to the extent permitted by law. Type "show copying"
# and "show warranty" for details.
# This GDB was configured as "x86_64-redhat-linux-gnu".
# For bug reporting instructions, please see:
# <http://www.gnu.org/software/gdb/bugs/>.
#
line = stdout.split('\n', 1)[0]
last = line.split()[-1]
float_pattern = '[1-9][0-9]*\.?[0-9]*'
f = re.search(float_pattern, last)
version = f.group(0)
major = int(version.split('.')[0])
minor = int(version.split('.')[1])
cmajor = int(config.gdb_base_version.split('.')[0])
cminor = int(config.gdb_base_version.split('.')[1])
# Check to make sure it's at least the base version required for DrDebug.
#
if (major < cmajor) or ((major == cmajor) and (minor < cminor)):
if python_support:
msg.PrintMsg('\n')
msg.PrintMsg(
'WARNING: This version of gdb is: %s When using versions of gdb < %s'
'\nthe script will run, but with reduced functionality.\n' %
(version, config.gdb_base_version))
def Run(self):
"""Run the scripts required to run simpoint and generate a region CSV file with the results."""
msg.PrintMsg('')
# Get the command line options
#
(options, args) = self.ParseCommandLine()
# Make sure required utilities exist and are executable.
#
# import pdb; pdb.set_trace()
if util.Which(self.simpoint_bin) == None:
msg.PrintAndExit('simpoint binary not in path.\n'
'Add directory where it exists to your path.')
if util.Which(self.csv_bin) == None:
msg.PrintAndExit(
'script to generate the region CSV file not in path.\n'
'Add directory where it exists to your path.')
# Go to the data directory. Both utilities should be run in this directory.
#
os.chdir(options.data_dir)
# Always copy the specific BBV to the generic name used by simpoint.
# If LDV file exists, then copy it to generic name.
#
if os.path.isfile(self.generic_bbv_name):
os.remove(self.generic_bbv_name)
shutil.copy(options.bbv_file, self.generic_bbv_name)
ldv_file = options.bbv_file.replace('.bb', '.ldv')
if os.path.isfile(ldv_file):
if os.path.isfile(self.generic_ldv_name):
os.remove(self.generic_ldv_name)
shutil.copy(options.bbv_file.replace('.bb', '.ldv'),
self.generic_ldv_name)
# Get the instruction count and slice_size from the BBV file.
#
try:
f = open(self.generic_bbv_name)
except IOError:
msg.PrintAndExit('problem opening BBV file: ' +
self.generic_bbv_name)
instr_count = 0
slice_size = 0
for line in f.readlines():
if line.find('Dynamic') != -1:
tmp = line.split()
count = tmp[len(tmp) - 1]
if count > instr_count:
instr_count = int(count)
if line.find('SliceSize') != -1:
tmp = line.split()
size = tmp[len(tmp) - 1]
if size > slice_size:
slice_size = int(size)
# Check to make sure instruction count > slice_size.
#
if slice_size > instr_count:
import locale
locale.setlocale(locale.LC_ALL, "")
msg.PrintAndExit('Slice size is greater than the number of instructions. Reduce parameter \'slice_size\'.' + \
'\nInstruction count: ' + locale.format('%14d', int(instr_count), True) + \
'\nSlice size: ' + locale.format('%14d', int(slice_size), True))
if options.ldv:
# Run to generate regions CSV file using both BBV and LDV files.
#
result = self.NormProjectBBV(options)
util.CheckResult(
result, options,
'normalizing and projecting BBV with: ' + self.csv_bin)
result = self.NormWeightLDV(options)
util.CheckResult(
result, options,
'normalizing and applying weights to LDV with: ' +
self.csv_bin)
result = self.CombineFreqVectFiles(options)
util.CheckResult(
result, options,
'scaling and combining BBV and LDV with: ' + self.csv_bin)
result = self.RunSimpoint(options)
util.CheckResult(
result, options,
'generating clusters (regions) with: ' + self.simpoint_bin)
result, regions_csv_file = self.GenRegionCSVFile(options)
util.CheckResult(result, options,
'creating regions CSV file with: ' + self.csv_bin)
msg.PrintMsg('\nRegions CSV file: ' +
os.path.join(options.data_dir, regions_csv_file))
else:
# Run scripts to generate regions CSV file using the new method with just the BBV file.
#
result = self.NormProjectBBV(options)
util.CheckResult(
result, options,
'normalizing and projecting BBV with: ' + self.csv_bin)
result = self.RunSimpoint(options)
util.CheckResult(
result, options,
'generating clusters (regions) with: ' + self.simpoint_bin)
result, regions_csv_file = self.GenRegionCSVFile(options)
util.CheckResult(result, options,
'creating regions CSF file with: ' + self.csv_bin)
msg.PrintMsg('\nRegions CSV file: ' +
os.path.join(options.data_dir, regions_csv_file))
return result
def _GetGomaFromPath():
"""Returns goma directory from PATH, otherwise is None."""
gomacc_path = util.Which('gomacc')
if gomacc_path is not None:
return os.path.dirname(gomacc_path)
return None
def GetPathOrDie(executable):
full_path = util.Which(executable)
if full_path is None:
util.Error("Can't find \"" + executable + "\" in our PATH!")
return full_path
def ExtractOADOIURLs(share_directory, all_dois_file, urls_file, log_file_name):
print("Extract URLs for DOI's in " + all_dois_file)
util.ExecOrDie(util.Which("extract_oadoi_urls.sh"),
[share_directory + '/' + all_dois_file, urls_file],
log_file_name)
