def find_matching_files(basedir, filetypes):
"""Try to find all files matching given filetypes.
By looking at all the files and filenames in the given directory,
including subdirectories.
:param basedir: full path to base directory to search in
:type basedir: string
:param filetypes: list of filetypes, extensions
:type filetypes: list
:return: exitcode and any error messages as: (exitcode, err_msg)
:rtype: tuple
"""
files_list = []
for dirpath, dummy0, filenames in os.walk(basedir):
for filename in filenames:
full_path = os.path.join(dirpath, filename)
dummy1, cmd_out, dummy2 = run_shell_command(
'file %s', (full_path,)
)
for filetype in filetypes:
if cmd_out.lower().find(filetype) > -1:
files_list.append(full_path)
elif filename.split('.')[-1].lower() == filetype:
files_list.append(full_path)
return files_list
def _grep_version_from_executable(path_to_exec, version_regexp):
"""Try to detect a program version.
Grep in its binary PATH_TO_EXEC and looking for VERSION_REGEXP. Return
program version as a string. Return empty string if not succeeded.
"""
from invenio_utils.shell import run_shell_command
exec_version = ""
if os.path.exists(path_to_exec):
dummy1, cmd2_out, dummy2 = run_shell_command("strings %s | grep %s", (path_to_exec, version_regexp))
if cmd2_out:
for cmd2_out_line in cmd2_out.split("\n"):
if len(cmd2_out_line) > len(exec_version):
# the longest the better
exec_version = cmd2_out_line
return exec_version
def _grep_version_from_executable(path_to_exec, version_regexp):
"""Try to detect a program version.
Grep in its binary PATH_TO_EXEC and looking for VERSION_REGEXP. Return
program version as a string. Return empty string if not succeeded.
"""
from invenio_utils.shell import run_shell_command
exec_version = ""
if os.path.exists(path_to_exec):
dummy1, cmd2_out, dummy2 = run_shell_command(
"strings %s | grep %s", (path_to_exec, version_regexp))
if cmd2_out:
for cmd2_out_line in cmd2_out.split("\n"):
if len(cmd2_out_line) > len(exec_version):
# the longest the better
exec_version = cmd2_out_line
return exec_version
def version(separator="\n", formatting="{version} [{executable}]"):
"""
Try to detect Apache version by localizing httpd or apache
executables and grepping inside binaries. Return list of all
found Apache versions and paths. (For a given executable, the
returned format is 'apache_version [apache_path]'.) Return empty
list if no success.
"""
from invenio_utils.shell import run_shell_command
out = []
dummy1, cmd_out, dummy2 = run_shell_command("locate bin/httpd bin/apache")
for apache in cmd_out.split("\n"):
apache_version = _grep_version_from_executable(apache, "^Apache\/")
if apache_version:
out.append(formatting.format(version=apache_version, executable=apache))
if separator is None:
return out
return separator.join(out)
def version(separator='\n', formatting='{version} [{executable}]'):
"""
Try to detect Apache version by localizing httpd or apache
executables and grepping inside binaries. Return list of all
found Apache versions and paths. (For a given executable, the
returned format is 'apache_version [apache_path]'.) Return empty
list if no success.
"""
from invenio_utils.shell import run_shell_command
out = []
dummy1, cmd_out, dummy2 = run_shell_command("locate bin/httpd bin/apache")
for apache in cmd_out.split("\n"):
apache_version = _grep_version_from_executable(apache, '^Apache\/')
if apache_version:
out.append(formatting.format(version=apache_version,
executable=apache))
if separator is None:
return out
return separator.join(out)
def test_run_cmd_hello(self):
"""shellutils - running simple command"""
self.assertEqual((0, "hello world\n", ''),
run_shell_command("echo 'hello world'"))
def test_run_cmd_timeout_no_zombie(self):
"""shellutils - running simple command no zombie"""
self.assertRaises(Timeout, run_process_with_timeout, (self.script_path, '15', "THISISATEST"), timeout=5)
ps_output = run_shell_command('ps aux')[1]
self.failIf('THISISATEST' in ps_output, '"THISISATEST" was found in %s' % ps_output)
self.failIf('sleep 15' in ps_output, '"sleep 15" was found in %s' % ps_output)
def test_run_cmd_hello_quote(self):
"""shellutils - running simple command with an argument with quote"""
self.assertEqual((0, "hel'lo world\n", ''),
run_shell_command("echo %s %s", ("hel'lo", "world",)))
def test_run_cmd_hello_args(self):
"""shellutils - running simple command with an argument"""
self.assertEqual((0, "hello world\n", ''),
run_shell_command("echo 'hello %s'", ("world",)))
def dump_database(dump_path, host=CFG_DATABASE_HOST, port=CFG_DATABASE_PORT, \
user=CFG_DATABASE_USER, passw=CFG_DATABASE_PASS, \
name=CFG_DATABASE_NAME, params=None, compress=False, \
ignore_tables=None):
"""
Dump Invenio database into SQL file located at DUMP_PATH.
Will perform the command to mysqldump with the given host configuration
and user credentials.
Optional mysqldump parameters can also be passed. Otherwise, a default
set of parameters will be used.
@param dump_path: path on the filesystem to save the dump to.
@type dump_path: string
@param host: hostname of mysql database node to connect to.
@type host: string
@param port: port of mysql database node to connect to
@type port: string
@param user: username to connect with
@type user: string
@param passw: password to connect to with
@type passw: string
@param name: name of mysql database node to dump
@type name: string
@param params: command line parameters to pass to mysqldump. Optional.
@type params: string
@param compress: should the dump be compressed through gzip?
@type compress: bool
@param ignore_tables: list of tables to ignore in the dump
@type ignore: list of string
"""
write_message("... writing %s" % (dump_path,))
partial_dump_path = dump_path + ".part"
# Is mysqldump installed or in the right path?
cmd_prefix = CFG_PATH_MYSQL + 'dump'
if not os.path.exists(cmd_prefix):
raise StandardError("%s is not installed." % (cmd_prefix))
if not params:
# No parameters set, lets use the default ones.
params = " --skip-opt --add-drop-table --add-locks --create-options" \
" --quick --extended-insert --set-charset --disable-keys" \
" --lock-tables=false --max_allowed_packet=2G "
if ignore_tables:
params += " ".join([escape_shell_arg("--ignore-table=%s.%s" % (CFG_DATABASE_NAME, table)) for table in ignore_tables])
dump_cmd = "%s %s " \
" --host=%s --port=%s --user=%s --password=%s %s" % \
(cmd_prefix, \
params, \
escape_shell_arg(host), \
escape_shell_arg(str(port)), \
escape_shell_arg(user), \
escape_shell_arg(passw), \
escape_shell_arg(name))
if compress:
dump_cmd = "%s | %s -cf; exit ${PIPESTATUS[0]}" % \
(dump_cmd, \
CFG_PATH_GZIP)
dump_cmd = "bash -c %s" % (escape_shell_arg(dump_cmd),)
write_message(dump_cmd, verbose=2)
exit_code, stdout, stderr = run_shell_command(dump_cmd, None, partial_dump_path)
if exit_code:
raise StandardError("ERROR: mysqldump exit code is %s. stderr: %s stdout: %s" % \
(repr(exit_code), \
repr(stderr), \
repr(stdout)))
else:
os.rename(partial_dump_path, dump_path)
write_message("... completed writing %s" % (dump_path,))
def dump_database(dump_path, host=CFG_DATABASE_HOST, port=CFG_DATABASE_PORT, \
user=CFG_DATABASE_USER, passw=CFG_DATABASE_PASS, \
name=CFG_DATABASE_NAME, params=None, compress=False, \
ignore_tables=None):
"""
Dump Invenio database into SQL file located at DUMP_PATH.
Will perform the command to mysqldump with the given host configuration
and user credentials.
Optional mysqldump parameters can also be passed. Otherwise, a default
set of parameters will be used.
@param dump_path: path on the filesystem to save the dump to.
@type dump_path: string
@param host: hostname of mysql database node to connect to.
@type host: string
@param port: port of mysql database node to connect to
@type port: string
@param user: username to connect with
@type user: string
@param passw: password to connect to with
@type passw: string
@param name: name of mysql database node to dump
@type name: string
@param params: command line parameters to pass to mysqldump. Optional.
@type params: string
@param compress: should the dump be compressed through gzip?
@type compress: bool
@param ignore_tables: list of tables to ignore in the dump
@type ignore: list of string
"""
write_message("... writing %s" % (dump_path, ))
partial_dump_path = dump_path + ".part"
# Is mysqldump installed or in the right path?
cmd_prefix = CFG_PATH_MYSQL + 'dump'
if not os.path.exists(cmd_prefix):
raise StandardError("%s is not installed." % (cmd_prefix))
if not params:
# No parameters set, lets use the default ones.
params = " --skip-opt --add-drop-table --add-locks --create-options" \
" --quick --extended-insert --set-charset --disable-keys" \
" --lock-tables=false --max_allowed_packet=2G "
if ignore_tables:
params += " ".join([
escape_shell_arg("--ignore-table=%s.%s" %
(CFG_DATABASE_NAME, table))
for table in ignore_tables
])
dump_cmd = "%s %s " \
" --host=%s --port=%s --user=%s --password=%s %s" % \
(cmd_prefix, \
params, \
escape_shell_arg(host), \
escape_shell_arg(str(port)), \
escape_shell_arg(user), \
escape_shell_arg(passw), \
escape_shell_arg(name))
if compress:
dump_cmd = "%s | %s -cf; exit ${PIPESTATUS[0]}" % \
(dump_cmd, \
CFG_PATH_GZIP)
dump_cmd = "bash -c %s" % (escape_shell_arg(dump_cmd), )
write_message(dump_cmd, verbose=2)
exit_code, stdout, stderr = run_shell_command(dump_cmd, None,
partial_dump_path)
if exit_code:
raise StandardError("ERROR: mysqldump exit code is %s. stderr: %s stdout: %s" % \
(repr(exit_code), \
repr(stderr), \
repr(stdout)))
else:
os.rename(partial_dump_path, dump_path)
write_message("... completed writing %s" % (dump_path, ))
