def __import_workspace(config_file_handler, args):
'''Imports a workspace (project dir, SQL database)'''
tgz_file = getattr(args, "import")
if not os.path.exists(tgz_file):
sys.exit(tgz_file + " does not exists!")
#--- Get workspace name from tgz ---#
with tarfile.open(tgz_file) as tar:
files = tar.getmembers()
if len(files) == 0:
sys.exit(tgz_file + " is empty!")
project_name = files[0].name
projectdir = join(args.workdir, project_name)
if config_file_handler.exists(project_name):
sys.exit("Workspace '"+ project_name +"' already exists! Use --delete to " \
"remove and then try again.")
#--- Unpack tar.gz file ---#
try:
print "Unpacking project '" + project_name + "' from " + tgz_file
proc = subprocess.Popen(['tar', '-C', args.workdir, '-xzf', tgz_file])
proc.wait()
except Exception as e:
sys.exit("Failed to unpack " + tgz_file)
#--- Read CodeCompass.export ---#
with open(join(projectdir, "CodeCompass.export"), 'r') as exp:
exportData = json.load(exp)
#--- Restore database ---#
print "Restoring database"
dump_filename = join(projectdir, 'database.dump')
util.pgsql_cmd(projectdir, args, 'dropdb', project_name)
restoreRetCode = util.pgsql_cmd(
projectdir,
args,
'pg_restore',
'-d',
'template1',
'-F',
'tar',
'-C', # Add CREATE DATABASE to the dump
'-O', # Do not set ownership in the dump
dump_filename)
if restoreRetCode != 0:
sys.exit("Failed! Please check " + join(projectdir, "dblog") +
" for details.")
else:
#--- Delete dumps ---#
if os.path.exists(dump_filename):
os.remove(dump_filename)
#--- Add to workspace ---#
description = project_name
if 'config' in exportData and 'description' in exportData['config']:
description = exportData['config']['description']
connArgs = copy.copy(args)
connArgs.name = project_name
connection_string = util.create_connection_string(connArgs)
config_file_handler.add(project_name)
config_file_handler.set_attribute(project_name, 'connection',
connection_string)
config_file_handler.set_attribute(project_name, 'description', description)
config_file_handler.save()
print "Done"
def __run_parser(args):
projectdir = join(args.workdir, args.name)
print __current_time(), 'Parsing...'
#--- Run parser ---#
while True:
with open(join(projectdir, 'parselog'), 'a') as f:
f.write('Parser started at ' + __current_time() + '\n')
parse_command = [
join(os.environ['CODECOMPASS_HOME'], 'bin', 'parse')
]
parse_command.extend(['-d', util.create_connection_string(args)])
parse_command.extend(['-p', args.logfile])
parse_command.extend(['-a', 'data'])
parse_command.extend(['-k', 'skippedactions'])
parse_command.extend(['-n', args.name])
parse_command.extend(['--max-parser-threads', str(args.thread)])
parse_command.extend(['--roots', args.labels])
if (args.extra):
parse_command.extend((args.extra).split())
old = util.set_environ()
proc = subprocess.Popen(parse_command,
stdout=f,
stderr=f,
cwd=projectdir,
env=os.environ)
proc.wait()
os.environ = old
f.write('Parse ended with code ' + str(proc.returncode) + ' at ' +
__current_time() + '\n')
has_crashes = False
if glob.glob(join(projectdir, '*core*')):
print __current_time(), 'Parser crashed! :-('
if not args.crash:
__handle_parser_crash(projectdir)
has_crashes = True
print __current_time(), 'Crash handled, continuing...'
else:
break
else:
break
#--- Last messages after finishing parsing ---#
print __current_time(), 'Done',
if has_crashes:
print 'with crashes :\'('
elif proc.returncode == 0:
print 'successfully! :-D'
else:
print 'with error! :-('
return proc.returncode
def server_handler(args):
config_file = join(args.workdir, 'workspace.cfg')
pid_file = join(args.workdir, 'pids')
#--- Stop the webserver ---#
if args.stop:
if not os.path.isfile(pid_file):
return
with open(pid_file) as f:
for line in f:
try:
os.kill(int(line), signal.SIGTERM)
except:
print 'No process number ' + line
os.remove(pid_file)
return
#--- Start codechecker server ---#
checker_proc = None
checker_sourced = False
wait_for_checker = False
try:
subprocess.call(
join(os.environ['CODECHECKER_HOME'], 'bin', 'CodeChecker'),
stdout = subprocess.PIPE, stderr = subprocess.PIPE)
except (OSError, KeyError):
sys.stderr.write('Set CODECHECKER_HOME environment variable!\n')
else:
checker_sourced = True
if checker_sourced:
print 'Start CodeChecker server...'
checker_proc, checker_port = util.start_codechecker_server(args)
configfilehandler = ConfigFileHandler(config_file)
for name in configfilehandler.names():
if configfilehandler.get_attribute(name, 'codecheckerrunname'):
configfilehandler.set_attribute(
name, 'codecheckerurl', '{}:{}/CodeCheckerService'.format(
'http://localhost', str(checker_port)))
configfilehandler.save()
wait_for_checker = True
while wait_for_checker:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
result = sock.connect_ex((args.dbhost, checker_port))
wait_for_checker = result != 0
time.sleep(1)
#--- Start the webserver ---#
with open(pid_file, 'w') as f:
# The parent's PID is stored because the CodeCompass.py is wrapped by a
# bash script.
f.write(str(os.getppid()) + '\n')
configfilehandler = ConfigFileHandler(config_file)
for name in configfilehandler.names():
connArgs = copy.copy(args)
connArgs.name = name
connection = util.create_connection_string(connArgs)
configfilehandler.set_attribute(name, 'connection', connection)
configfilehandler.save()
command = [join(os.environ['CODECOMPASS_HOME'], 'bin', 'pluginablewebserver')]
command.extend(
['-P', join(os.environ['CODECOMPASS_HOME'], 'lib', 'codecompass', 'webplugin')])
command.extend(
['-d', join(os.environ['CODECOMPASS_HOME'], 'share', 'codecompass', 'html')])
command.extend(['-p', str(args.port)])
command.extend(['--workspaces', config_file])
if args.extra:
command.extend(args.extra.split(" "))
wslog = join(args.workdir, 'webserver.log')
while True:
print '(Re)Starting web server with log file: {}'.format(config_file)
print 'Web server log files can be found at: ' + wslog
with open(wslog, 'a') as f:
old = util.set_environ()
proc = subprocess.Popen(command, stdout = f, stderr = f, env = os.environ)
with open(pid_file, 'w') as f:
f.write(str(proc.pid) + '\n')
global SERVER_PID
SERVER_PID = proc.pid
signal.signal(signal.SIGTERM, handler)
proc.wait()
if proc.returncode & 126 == 0:
print 'Web server exited with code: ' + str(proc.returncode)
if checker_proc:
os.killpg(os.getpgid(checker_proc.pid), signal.SIGTERM)
sys.exit(proc.returncode)
os.environ = old
def parse_handler(args):
signal.signal(signal.SIGINT, interrupt_parse_handler)
#--- Print database connection data ---#
print '--- Database connection ---'
print 'Database host: ' + args.dbhost
print 'Database port: ' + str(args.dbport)
print 'Database user: '******'Database pass: '******'---------------------------'
#--- Create workdir and project directory ---#
projectdir = join(args.workdir, args.name)
if not os.path.exists(projectdir):
try:
os.makedirs(projectdir)
except OSError:
sys.stderr.write('Permission denied to create ' + projectdir +
'\n')
sys.exit(1)
elif not args.force:
sys.stderr.write(projectdir +
' already exists. Use -f to force reparsing!\n')
sys.exit(1)
else:
if args.logfile:
for entry in os.listdir(projectdir):
fullentry = os.path.join(projectdir, entry)
if (os.path.isdir(fullentry)):
shutil.rmtree(fullentry)
elif (entry != "build.json"):
os.remove(fullentry)
else:
shutil.rmtree(projectdir)
os.makedirs(projectdir)
#--- Create build log ---#
print __current_time(), 'Logger started...'
if args.build or args.logonly:
param = args.build if args.build else args.logonly
try:
args.logfile = join(projectdir, 'build.json')
if not args.force and os.path.exists(args.logfile):
sys.stderr.write('The project has already been logged\n')
sys.exit(1)
cmd = 'source {} {}; bash -c "{}"'.format(
join(os.environ['CODECOMPASS_HOME'], 'bin', 'setldlogenv.sh'),
args.logfile, param)
with open(join(projectdir, 'buildlog'), 'a') as f:
os.environ['LDLOGGER_HOME'] = join(
os.environ['CODECOMPASS_HOME'], 'logger')
proc = subprocess.Popen(['/bin/bash', '-c', cmd],
stderr=f,
stdout=f,
env=os.environ)
proc.wait()
if not os.path.exists(args.logfile):
sys.exit("Failed to log the build commands!")
elif os.path.getsize(args.logfile) <= 5:
sys.exit(
"Failed to log the build commands: the build log is empty!"
)
except RuntimeError as err:
sys.stderr.write(str(err) + '\n')
sys.exit(1)
if not args.build and not args.logfile:
args.logfile = join(projectdir, 'build.json')
with open(args.logfile, 'w') as logfile:
logfile.write('[]')
print __current_time(), 'Done'
#--- Parse the project ---#
if not args.logonly:
if not os.path.isfile(args.logfile):
sys.exit('The given build log file not found!')
__create_database_and_clean(args)
returncode = __run_parser(args)
if not args.logfile.startswith(projectdir):
shutil.copy(args.logfile, projectdir)
if not args.noindex:
__run_after_sql_script(args)
#--- Add workspace to config file ---#
if returncode == 0:
desc = args.description if hasattr(
args, 'description') and args.description <> "" else args.name
config_file_handler = ConfigFileHandler(
join(args.workdir, 'workspace.cfg'))
connection_string = util.create_connection_string(args)
config_file_handler.add(args.name)
config_file_handler.set_attribute(args.name, 'connection',
connection_string)
config_file_handler.set_attribute(args.name, 'description', desc)
config_file_handler.save()
print 'Parse log files can be found at: ' + join(
projectdir, 'parselog')
print 'Database log files can be found at: ' + join(
projectdir, 'dblog')