class Transcript(Resource):
def __init__(self):
self.debug_logger = SetupLogger('transcript.log', False)
def post(self):
f = request.files['transcript']
filename = secure_filename(f.filename)
unique_pdf_name = str(uuid.uuid4()) + '.pdf'
# f.save(unique_pdf_name)
file_data = f.read()
try:
transcript_parser = TranscriptParser(f)
student_id = transcript_parser.get_student_id()
transcript = TranscriptModel(student_id, file_data)
transcript.save_to_db()
except Exception as e:
self.debug_logger.error(
'Transcript post failed with error: {}'.format(e))
# TODO: provide reason for failure.
return redirect('/', code=303)
# TODO: need to load correct checklist
return redirect(url_for('transcript'), code=303)
def get(self):
headers = {'Content-Type': 'text/html'}
return make_response(render_template('2018-2019_BCS_AI.j2'), 200,
headers)
class SoapUIRunner:
def __init__(self, configs):
self.timeout = 30 * 60 # length of timeout (in seconds)
self.kill_delay = 2
self.returncode = 1
self.result = 1 # 1 = there were problems, 0 = everything was fine
self.summary = ''
self.configs = configs
self.configure()
self.logger = SetupLogger().logger("SoapUIRunner")
def configure(self):
self.runner_location = self.configs.get('soapui_runner_dir')
self.execution_root = self.configs.get('execution_root')
def command(self, service_id, test_id, project_file):
self.service_id = service_id
self.test_id = test_id
self.project_file = project_file
self.test_dir = os.path.join(self.execution_root, str(service_id), str(test_id))
self.work_dir = os.path.join(self.test_dir, 'package')
self.log = os.path.join(self.test_dir, os.path.join("logs", ("%s.tmp.log" % datetime.datetime.now())))
options = ' -a -r -f %s '%self.work_dir
cmd = os.path.join(self.runner_location, 'testrunner.sh' )
cmd +=' %s ' %options
cmd +=' %s ' %os.path.join(self.work_dir, self.project_file)
self.cmd = cmd
self.logger.debug( cmd)
return cmd
def parse_log(self, log=''):
if log=='' : log = self.log
try:
lines = open(log, 'r').readlines()
lines = [line for line in lines if (line.startswith('SoapUI')
or line.startswith('Time')
or line.startswith('Total') )]
result_line = [line for line in lines if line.startswith('Total TestCases:')]
for line in lines:
self.summary += line
if len(result_line) == 1:
parts = result_line[0].split()
total = parts[2]
failed = parts[3][1:]
if int(failed) == 0 :
self.result = 0
else:
self.result = 1
print "Overall result %s "% self.result
print "Summary Report"
print self.summary
except Exception, e:
print "There were problems parsing the log"
print str(e)
def __init__(self, configs):
self.configs = configs
self.timeout = 30 * 60 # length of timeout (in seconds)
self.kill_delay = 2
self.returncode = 1
self.result = 1 # 1 = there were problems, 0 = everything was fine
self.summary = ''
self.configure()
self.logger = SetupLogger().logger("TestScriptRunner")
class TestScriptRunner:
def __init__(self, configs):
self.configs = configs
self.timeout = 30 * 60 # length of timeout (in seconds)
self.kill_delay = 2
self.returncode = 1
self.result = 1 # 1 = there were problems, 0 = everything was fine
self.summary = ''
self.configure()
self.logger = SetupLogger().logger("TestScriptRunner")
def configure(self):
self.runner_location = self.configs.get('script_runner_dir')
self.execution_root = self.configs.get('execution_root')
def command(self, service_id, test_id, script_name, conf_file):
self.service_id = service_id
self.test_id = test_id
self.script_name = script_name
self.test_dir = os.path.join(self.execution_root, str(service_id),
str(test_id))
self.work_dir = os.path.join(self.test_dir, 'package')
self.log = os.path.join(
self.test_dir,
os.path.join("logs", ("%s.tmp.log" % datetime.datetime.now())))
self.cmd = os.path.join(self.work_dir, self.script_name)
self.logger.debug(self.cmd + ' %s' % conf_file)
return self.cmd + ' %s' % conf_file
def post_result(self, url, user, psswd):
self.logger.debug("Posting result to %s" % url)
conf = {
'result': int(self.result),
'message': self.summary,
'action': self.script_name,
'test_id': int(self.test_id)
}
tr = TestResult(conf)
return tr.post(url, user, psswd)
def signal_handler(self, signum, child):
os.kill(child.pid, signal.SIGKILL)
def status_message(self, exit_code):
if exit_code == 0:
return "Success: Script ran successfully"
elif (exit_code in range(255)):
return "Failure: Script failed with exit code : %s" % str(
exit_code)
else:
return "Warning: Script exited with code : %s " % str(exit_code)
class ConfigReader:
def __init__(self, config_file):
self.config = config_file
self.logger = SetupLogger().logger("ConfigReader")
def read(self):
self.logger.debug("Reading the config file")
params = {}
lines = open(self.config,'r').readlines()
lines = [line.strip() for line in lines]
for line in lines:
#skip empty lines and comment lines
if not line.startswith('#') and line.strip():
k,v = line.split('=')
params[k.strip()] = v.strip()
#print "Done"
return params
class ConfigReader:
def __init__(self, config_file):
self.config = config_file
self.logger = SetupLogger().logger("ConfigReader")
def read(self):
self.logger.debug("Reading the config file")
params = {}
lines = open(self.config, 'r').readlines()
lines = [line.strip() for line in lines]
for line in lines:
#skip empty lines and comment lines
if not line.startswith('#') and line.strip():
k, v = line.split('=')
params[k.strip()] = v.strip()
#print "Done"
return params
class TestScriptRunner:
def __init__(self, configs):
self.configs = configs
self.timeout = 30 * 60 # length of timeout (in seconds)
self.kill_delay = 2
self.returncode = 1
self.result = 1 # 1 = there were problems, 0 = everything was fine
self.summary = ''
self.configure()
self.logger = SetupLogger().logger("TestScriptRunner")
def configure(self):
self.runner_location = self.configs.get('script_runner_dir')
self.execution_root = self.configs.get('execution_root')
def command(self, service_id, test_id, script_name, conf_file):
self.service_id = service_id
self.test_id = test_id
self.script_name = script_name
self.test_dir = os.path.join(self.execution_root, str(service_id), str(test_id))
self.work_dir = os.path.join(self.test_dir, 'package')
self.log = os.path.join(self.test_dir, os.path.join("logs", ("%s.tmp.log" % datetime.datetime.now())))
self.cmd = os.path.join(self.work_dir, self.script_name)
self.logger.debug( self.cmd +' %s'%conf_file )
return self.cmd +' %s'%conf_file
def post_result(self, url, user, psswd):
self.logger.debug( "Posting result to %s"%url)
conf = {'result' : int(self.result),
'message' : self.summary,
'action' : self.script_name,
'test_id' : int(self.test_id) }
tr = TestResult(conf)
return tr.post(url, user, psswd)
def signal_handler(self, signum, child):
os.kill(child.pid, signal.SIGKILL)
def status_message(self, exit_code):
if exit_code == 0:
return "Success: Script ran successfully"
elif (exit_code in range(255)):
return "Failure: Script failed with exit code : %s"%str(exit_code)
else:
return "Warning: Script exited with code : %s "%str(exit_code)
def __init__(self, configs):
self.configs = configs
self.timeout = 30 * 60 # length of timeout (in seconds)
self.kill_delay = 2
self.returncode = 1
self.result = 1 # 1 = there were problems, 0 = everything was fine
self.summary = ''
self.configure()
self.logger = SetupLogger().logger("TestScriptRunner")
class TestResult:
def __init__(self, configs):
self.configs = configs
self.logger = SetupLogger().logger("TestResult")
def post(self, url, user, password):
self.logger.info("posting result using curl")
test_type = "TestScript"
test_id = self.configs['test_id']
result = self.configs['result']
action = self.configs['action']
message = self.configs['message']
user = user
password = password
data = '<?xml version="1.0"?>'
data += '<test_result> '
data += '<result>%d</result>' % (result)
data += '<action>%s</action>' % (action)
data += '<message>%s</message>' % (message)
data += '<service_test_id>%d</service_test_id>' % (test_id)
data += '</test_result>'
curl = "curl -X POST -u %s:%s -d '%s' -H 'Content-Type:application/xml' %s" % (
user, password, data, url)
#self.logger.debug(curl)
#os.system(curl)
try:
retcode = subprocess.call(curl, shell=True)
if retcode < 0:
print >> sys.stderr, "Child was terminated by signal", -retcode
return False
elif retcode > 0:
print >> sys.stderr, "There were problems with posting the result ", retcode
#print >>sys.stderr, "Child returned", retcode
return False
else:
print >> sys.stdout, "Result was posted successfully "
return True
except OSError, e:
print >> sys.stderr, "Exception while posting result:", e
return False
return True
class TestResult:
def __init__(self, configs):
self.configs = configs
self.logger = SetupLogger().logger("TestResult")
def post(self, url, user, password):
self.logger.info("posting result using curl")
test_type ="TestScript"
test_id = self.configs['test_id']
result = self.configs['result']
action = self.configs['action']
message = self.configs['message']
user = user
password = password
data = '<?xml version="1.0"?>'
data +='<test_result> '
data +='<result>%d</result>'%(result)
data +='<action>%s</action>'%(action)
data +='<message>%s</message>'%(message)
data +='<service_test_id>%d</service_test_id>'%(test_id)
data +='</test_result>'
curl = "curl -X POST -u %s:%s -d '%s' -H 'Content-Type:application/xml' %s" %(user, password,data, url)
#self.logger.debug(curl)
#os.system(curl)
try:
retcode = subprocess.call(curl, shell=True)
if retcode < 0:
print >>sys.stderr, "Child was terminated by signal", -retcode
return False
elif retcode > 0:
print >>sys.stderr, "There were problems with posting the result ", retcode
#print >>sys.stderr, "Child returned", retcode
return False
else:
print >> sys.stdout, "Result was posted successfully "
return True
except OSError, e:
print >>sys.stderr, "Exception while posting result:", e
return False
return True
def __init__(self, fn):
self.configs = ConfigReader(fn).read()
self.logger = SetupLogger().logger("DirectoryMaker")
class DirectoryMaker:
def __init__(self, fn):
self.configs = ConfigReader(fn).read()
self.logger = SetupLogger().logger("DirectoryMaker")
def get_script_references(self, db):
#stmt = "SELECT service_tests.service_id, service_tests.test_id, test_scripts.filename, content_blobs.data "
stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.filename, content_blobs.data "
stmt += "FROM service_tests,test_scripts,content_blobs "
stmt += "WHERE service_tests.test_id = test_scripts.id AND service_tests.test_type ='TestScript' "
stmt += "AND test_scripts.content_blob_id = content_blobs.id AND service_tests.activated_at IS NOT NULL; "
self.logger.debug(stmt)
results = db.execute(stmt)
references = db.fetchall()
return references
#make dirs of pattern <base_dir>/<service id>/<test_id>/package/
def setup_run_dirs(self, base_dir, results =[]):
if not os.path.exists(base_dir): return "ERROR! test base dir %s does not exist"%base_dir
for result in results:
service_id = result[0]
test_id = result[1]
run_dir = os.path.join(base_dir, str(service_id), str(test_id), 'package')
if not os.path.exists(run_dir):
self.create_run_dir(run_dir, result[2], result[3])
self.logger.debug("Created test script run directory %s"%run_dir)
else:
self.logger.debug("Test script run directory %s already exists "%run_dir)
# create the directory to run the jobs
def create_run_dir(self, dir, fname, data):
cwd = os.getcwd()
base, package = os.path.split(dir)
base, test_id = os.path.split(base)
base, service_id = os.path.split(base)
try:
if not os.path.exists(os.path.join(base, service_id)):
os.mkdir(os.path.join(base, service_id))
if not os.path.exists(os.path.join(base, service_id, test_id)):
os.mkdir(os.path.join(base, service_id, test_id))
os.mkdir(dir)
os.mkdir(os.path.join(os.path.split(dir)[0], "logs"))
except Exception, e:
e.stacktrace()
if not os.path.exists(os.path.join(dir, fname)):
if data != None:
try:
fh = open(os.path.join(dir, fname), 'wb')
fh.write(data)
fh.close()
if self.is_zip(fname):
os.chdir(dir)
os.system("unzip %s"%fname )
os.chdir(cwd)
cmd = "chmod -R u=rxw %s"%dir
self.logger.debug(cmd)
os.system(cmd)
except Exception, e:
self.logger.debug(str(e))
def __init__(self, configs):
self.configs = configs
self.logger = SetupLogger().logger("TestResult")
def __init__(self, configs):
self.configs = configs
self.logger = SetupLogger().logger("TestResult")
def __init__(self, config_file):
self.config = config_file
self.logger = SetupLogger().logger("ConfigReader")
class TranscriptParser:
def __init__(self, file_data):
self.file_data = file_data
self.parser_logger = SetupLogger('transcript_parser.log', False)
self.textstr = self.read_pdf()
def get_student_id(self):
keyword = "Student ID:"
before_kew, kw, after_kw = self.textstr.partition(keyword)
return int(after_kw.split()[0].strip())
def read_pdf(self):
rsrcmgr = PDFResourceManager()
retstr = StringIO()
codec = 'utf-8'
laparams = LAParams(char_margin=200)
device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams)
interpreter = PDFPageInterpreter(rsrcmgr, device)
password = ""
maxpages = 0
caching = True
pagenos = set()
for page in PDFPage.get_pages(self.file_data,
pagenos,
maxpages=maxpages,
password=password,
caching=caching,
check_extractable=False):
interpreter.process_page(page)
textstr = retstr.getvalue()
device.close()
retstr.close()
return textstr
def parse1(self):
file_string = self.read_pdf()
with open("delete1.txt", "a") as f:
f.write(file_string)
def parse(self):
pdf_binary = open(self.filename, 'rb')
pdf_reader = p2.PdfFileReader(pdf_binary)
if pdf_reader.isEncrypted:
'''
pyPDF2 requires the pdf be decrypted.
<Encryption Level: 128-bit AES> raises NotImplementedError
<Encryption level: 128-bit RC4> can be decrypted with empty string
'''
try:
pdf_reader.decrypt('')
self.parser_logger.info('File Decrypted (PyPDF2)')
except NotImplementedError:
try:
# prevent race conditions when creating tmp files
temp_dir = tempfile.mkdtemp(
dir=os.path.dirname(self.filename))
temp_out = os.path.join(temp_dir, 'qpdf_out.pdf')
# Will raise if this call is unsuccessful
check_call([
'qpdf', "--password="******""
num_pages = pdf_reader.getNumPages()
for i in range(num_pages):
content += pdf_reader.getPage(i).extractText() + "\n"
content = " ".join(content.replace(u"\xa0", " ").strip().split())
with open("delete1.txt", "a") as f:
f.write(content)
def __init__(self, fn):
self.configs = ConfigReader(fn).read()
self.outfile = '../config/script_listing.xml'
self.logger = SetupLogger().logger("ScriptListingWriter:")
def __init__(self, fn):
self.configs = ConfigReader(fn).read()
self.outfile = "../config/script_listing.xml"
self.logger = SetupLogger().logger("ScriptListingWriter:")
class ScriptListingWriter:
def __init__(self, fn):
self.configs = ConfigReader(fn).read()
self.outfile = "../config/script_listing.xml"
self.logger = SetupLogger().logger("ScriptListingWriter:")
def get_script_references(self, db):
self.logger.info("getting script references from db : %s" % db)
stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name "
stmt += "FROM service_tests,test_scripts "
stmt += "WHERE service_tests.test_id =test_scripts.id "
stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language <> 'soapui' "
stmt += "AND service_tests.activated_at IS NOT NULL ; "
self.logger.info(stmt)
results = db.execute(stmt)
references = db.fetchall()
return references
def get_soapui_references(self, db):
self.logger.info("getting soapui project references from db : %s" % db)
stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name "
stmt += "FROM service_tests,test_scripts "
stmt += "WHERE service_tests.test_id =test_scripts.id "
stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language = 'soapui' "
stmt += "AND service_tests.activated_at IS NOT NULL ; "
self.logger.info(stmt)
results = db.execute(stmt)
references = db.fetchall()
return references
def write_listing(self, scripts, soapui):
self.logger.info("writing script listing file : %s" % self.outfile)
fh = open(self.outfile, "w")
fh.write("<test_harness> \n")
fh.write("<scripts> \n")
for script in scripts:
fh.write(" <script> \n")
fh.write("\t<service_id>%s</service_id> \n" % str(script[0]))
fh.write("\t<test_id>%s</test_id> \n" % str(script[1]))
fh.write("\t<executable>%s</executable> \n" % str(script[2]))
fh.write(" </script> \n\n")
fh.write("</scripts> \n")
fh.write("<soapui> \n")
for project in soapui:
fh.write(" <project> \n")
fh.write("\t<service_id>%s</service_id> \n" % str(project[0]))
fh.write("\t<test_id>%s</test_id> \n" % str(project[1]))
fh.write("\t<executable>%s</executable> \n" % str(project[2]))
fh.write(" </project> \n\n")
fh.write("</soapui> \n")
fh.write("</test_harness> \n")
fh.close()
# connect to db
def connect_to_db(self, host, port, user, passwd, db):
conn = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=db)
cursor = conn.cursor()
self.logger.info("Connected to db : %s" % db)
return cursor
from setup_logger import SetupLogger
MAX_PROCESS_TIME = 60 * 5 # Scripts should not run run for more than set no of seconds
pool_size = 10 # process pool size
pool = {} # process pool
poll_time = 30 # length of wait between polls (in seconds)
queue = [] # Queue of processes to execute
soapui_jobs = []
cursor = None # DB connection handle
completed = [] # completed processes
passed = [] # sucessful processes
failed = [] # failed processes
total_scripts = 0
total_soapui = 0
logger = SetupLogger().logger("biocat_harness")
# command line options
usage = "usage: %prog [options] configuration_file"
parser = OptionParser(usage=usage)
parser.add_option("-l",
"--log",
dest="logfile",
help="send output to logfile",
metavar="FILE",
default="harness-status-%s.log" %
(time.strftime("%m%d-%Y-%H%M%S", time.gmtime())))
parser.add_option("-f",
"--fromFile",
dest="fromFile",
help="XML file containing the scripts to be run",
def __init__(self, config_file):
self.config = config_file
self.logger = SetupLogger().logger("ConfigReader")
def __init__(self, filename):
self.filename = filename
self.logger = SetupLogger().logger("ScriptListingReader")
class ScriptListingWriter:
def __init__(self, fn):
self.configs = ConfigReader(fn).read()
self.outfile = '../config/script_listing.xml'
self.logger = SetupLogger().logger("ScriptListingWriter:")
def get_script_references(self, db):
self.logger.info("getting script references from db : %s" %db)
stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name "
stmt += "FROM service_tests,test_scripts "
stmt += "WHERE service_tests.test_id =test_scripts.id "
stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language <> 'soapui' "
stmt += "AND service_tests.activated_at IS NOT NULL ; "
self.logger.info(stmt)
results = db.execute(stmt)
references = db.fetchall()
return references
def get_soapui_references(self, db):
self.logger.info("getting soapui project references from db : %s" %db)
stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name "
stmt += "FROM service_tests,test_scripts "
stmt += "WHERE service_tests.test_id =test_scripts.id "
stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language = 'soapui' "
stmt += "AND service_tests.activated_at IS NOT NULL ; "
self.logger.info(stmt)
results = db.execute(stmt)
references = db.fetchall()
return references
def write_listing(self, scripts, soapui ):
self.logger.info("writing script listing file : %s" %self.outfile)
fh = open(self.outfile, 'w')
fh.write('<test_harness> \n' )
fh.write('<scripts> \n' )
for script in scripts:
fh.write(' <script> \n' )
fh.write('\t<service_id>%s</service_id> \n'%str(script[0]) )
fh.write('\t<test_id>%s</test_id> \n'%str(script[1]) )
fh.write('\t<executable>%s</executable> \n'%str(script[2]) )
fh.write(' </script> \n\n' )
fh.write('</scripts> \n' )
fh.write('<soapui> \n' )
for project in soapui:
fh.write(' <project> \n' )
fh.write('\t<service_id>%s</service_id> \n'%str(project[0]) )
fh.write('\t<test_id>%s</test_id> \n'%str(project[1]) )
fh.write('\t<executable>%s</executable> \n'%str(project[2]) )
fh.write(' </project> \n\n' )
fh.write('</soapui> \n' )
fh.write('</test_harness> \n' )
fh.close()
#connect to db
def connect_to_db(self, host, port, user, passwd, db):
conn = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=db)
cursor = conn.cursor()
self.logger.info("Connected to db : %s" %db )
return cursor
def __init__(self, file_data):
self.file_data = file_data
self.parser_logger = SetupLogger('transcript_parser.log', False)
self.textstr = self.read_pdf()
def __init__(self):
self.debug_logger = SetupLogger('transcript.log', False)
MAX_PROCESS_TIME = 60*5 # Scripts should not run run for more than set no of seconds
pool_size = 10 # process pool size
pool = {} # process pool
poll_time = 30 # length of wait between polls (in seconds)
queue = [] # Queue of processes to execute
soapui_jobs = []
cursor = None # DB connection handle
completed = [] # completed processes
passed = [] # sucessful processes
failed = [] # failed processes
total_scripts = 0
total_soapui = 0
logger = SetupLogger().logger("biocat_harness")
# command line options
usage = "usage: %prog [options] configuration_file"
parser = OptionParser(usage=usage)
parser.add_option("-l", "--log", dest="logfile",
help="send output to logfile", metavar="FILE", default="harness-status-%s.log" %(time.strftime("%m%d-%Y-%H%M%S", time.gmtime())))
parser.add_option("-f", "--fromFile", dest="fromFile",
help="XML file containing the scripts to be run", metavar="FILE")
parser.add_option("-d", "--database", dest="db", default=False,
help="get the scripts to run from a Biocatalogue database")
(options, args) = parser.parse_args()
save_stdout = sys.stdout