class Transcript(Resource): def __init__(self): self.debug_logger = SetupLogger('transcript.log', False) def post(self): f = request.files['transcript'] filename = secure_filename(f.filename) unique_pdf_name = str(uuid.uuid4()) + '.pdf' # f.save(unique_pdf_name) file_data = f.read() try: transcript_parser = TranscriptParser(f) student_id = transcript_parser.get_student_id() transcript = TranscriptModel(student_id, file_data) transcript.save_to_db() except Exception as e: self.debug_logger.error( 'Transcript post failed with error: {}'.format(e)) # TODO: provide reason for failure. return redirect('/', code=303) # TODO: need to load correct checklist return redirect(url_for('transcript'), code=303) def get(self): headers = {'Content-Type': 'text/html'} return make_response(render_template('2018-2019_BCS_AI.j2'), 200, headers)
class SoapUIRunner: def __init__(self, configs): self.timeout = 30 * 60 # length of timeout (in seconds) self.kill_delay = 2 self.returncode = 1 self.result = 1 # 1 = there were problems, 0 = everything was fine self.summary = '' self.configs = configs self.configure() self.logger = SetupLogger().logger("SoapUIRunner") def configure(self): self.runner_location = self.configs.get('soapui_runner_dir') self.execution_root = self.configs.get('execution_root') def command(self, service_id, test_id, project_file): self.service_id = service_id self.test_id = test_id self.project_file = project_file self.test_dir = os.path.join(self.execution_root, str(service_id), str(test_id)) self.work_dir = os.path.join(self.test_dir, 'package') self.log = os.path.join(self.test_dir, os.path.join("logs", ("%s.tmp.log" % datetime.datetime.now()))) options = ' -a -r -f %s '%self.work_dir cmd = os.path.join(self.runner_location, 'testrunner.sh' ) cmd +=' %s ' %options cmd +=' %s ' %os.path.join(self.work_dir, self.project_file) self.cmd = cmd self.logger.debug( cmd) return cmd def parse_log(self, log=''): if log=='' : log = self.log try: lines = open(log, 'r').readlines() lines = [line for line in lines if (line.startswith('SoapUI') or line.startswith('Time') or line.startswith('Total') )] result_line = [line for line in lines if line.startswith('Total TestCases:')] for line in lines: self.summary += line if len(result_line) == 1: parts = result_line[0].split() total = parts[2] failed = parts[3][1:] if int(failed) == 0 : self.result = 0 else: self.result = 1 print "Overall result %s "% self.result print "Summary Report" print self.summary except Exception, e: print "There were problems parsing the log" print str(e)
def __init__(self, configs): self.configs = configs self.timeout = 30 * 60 # length of timeout (in seconds) self.kill_delay = 2 self.returncode = 1 self.result = 1 # 1 = there were problems, 0 = everything was fine self.summary = '' self.configure() self.logger = SetupLogger().logger("TestScriptRunner")
class TestScriptRunner: def __init__(self, configs): self.configs = configs self.timeout = 30 * 60 # length of timeout (in seconds) self.kill_delay = 2 self.returncode = 1 self.result = 1 # 1 = there were problems, 0 = everything was fine self.summary = '' self.configure() self.logger = SetupLogger().logger("TestScriptRunner") def configure(self): self.runner_location = self.configs.get('script_runner_dir') self.execution_root = self.configs.get('execution_root') def command(self, service_id, test_id, script_name, conf_file): self.service_id = service_id self.test_id = test_id self.script_name = script_name self.test_dir = os.path.join(self.execution_root, str(service_id), str(test_id)) self.work_dir = os.path.join(self.test_dir, 'package') self.log = os.path.join( self.test_dir, os.path.join("logs", ("%s.tmp.log" % datetime.datetime.now()))) self.cmd = os.path.join(self.work_dir, self.script_name) self.logger.debug(self.cmd + ' %s' % conf_file) return self.cmd + ' %s' % conf_file def post_result(self, url, user, psswd): self.logger.debug("Posting result to %s" % url) conf = { 'result': int(self.result), 'message': self.summary, 'action': self.script_name, 'test_id': int(self.test_id) } tr = TestResult(conf) return tr.post(url, user, psswd) def signal_handler(self, signum, child): os.kill(child.pid, signal.SIGKILL) def status_message(self, exit_code): if exit_code == 0: return "Success: Script ran successfully" elif (exit_code in range(255)): return "Failure: Script failed with exit code : %s" % str( exit_code) else: return "Warning: Script exited with code : %s " % str(exit_code)
class ConfigReader: def __init__(self, config_file): self.config = config_file self.logger = SetupLogger().logger("ConfigReader") def read(self): self.logger.debug("Reading the config file") params = {} lines = open(self.config,'r').readlines() lines = [line.strip() for line in lines] for line in lines: #skip empty lines and comment lines if not line.startswith('#') and line.strip(): k,v = line.split('=') params[k.strip()] = v.strip() #print "Done" return params
class ConfigReader: def __init__(self, config_file): self.config = config_file self.logger = SetupLogger().logger("ConfigReader") def read(self): self.logger.debug("Reading the config file") params = {} lines = open(self.config, 'r').readlines() lines = [line.strip() for line in lines] for line in lines: #skip empty lines and comment lines if not line.startswith('#') and line.strip(): k, v = line.split('=') params[k.strip()] = v.strip() #print "Done" return params
class TestScriptRunner: def __init__(self, configs): self.configs = configs self.timeout = 30 * 60 # length of timeout (in seconds) self.kill_delay = 2 self.returncode = 1 self.result = 1 # 1 = there were problems, 0 = everything was fine self.summary = '' self.configure() self.logger = SetupLogger().logger("TestScriptRunner") def configure(self): self.runner_location = self.configs.get('script_runner_dir') self.execution_root = self.configs.get('execution_root') def command(self, service_id, test_id, script_name, conf_file): self.service_id = service_id self.test_id = test_id self.script_name = script_name self.test_dir = os.path.join(self.execution_root, str(service_id), str(test_id)) self.work_dir = os.path.join(self.test_dir, 'package') self.log = os.path.join(self.test_dir, os.path.join("logs", ("%s.tmp.log" % datetime.datetime.now()))) self.cmd = os.path.join(self.work_dir, self.script_name) self.logger.debug( self.cmd +' %s'%conf_file ) return self.cmd +' %s'%conf_file def post_result(self, url, user, psswd): self.logger.debug( "Posting result to %s"%url) conf = {'result' : int(self.result), 'message' : self.summary, 'action' : self.script_name, 'test_id' : int(self.test_id) } tr = TestResult(conf) return tr.post(url, user, psswd) def signal_handler(self, signum, child): os.kill(child.pid, signal.SIGKILL) def status_message(self, exit_code): if exit_code == 0: return "Success: Script ran successfully" elif (exit_code in range(255)): return "Failure: Script failed with exit code : %s"%str(exit_code) else: return "Warning: Script exited with code : %s "%str(exit_code)
class TestResult: def __init__(self, configs): self.configs = configs self.logger = SetupLogger().logger("TestResult") def post(self, url, user, password): self.logger.info("posting result using curl") test_type = "TestScript" test_id = self.configs['test_id'] result = self.configs['result'] action = self.configs['action'] message = self.configs['message'] user = user password = password data = '<?xml version="1.0"?>' data += '<test_result> ' data += '<result>%d</result>' % (result) data += '<action>%s</action>' % (action) data += '<message>%s</message>' % (message) data += '<service_test_id>%d</service_test_id>' % (test_id) data += '</test_result>' curl = "curl -X POST -u %s:%s -d '%s' -H 'Content-Type:application/xml' %s" % ( user, password, data, url) #self.logger.debug(curl) #os.system(curl) try: retcode = subprocess.call(curl, shell=True) if retcode < 0: print >> sys.stderr, "Child was terminated by signal", -retcode return False elif retcode > 0: print >> sys.stderr, "There were problems with posting the result ", retcode #print >>sys.stderr, "Child returned", retcode return False else: print >> sys.stdout, "Result was posted successfully " return True except OSError, e: print >> sys.stderr, "Exception while posting result:", e return False return True
class TestResult: def __init__(self, configs): self.configs = configs self.logger = SetupLogger().logger("TestResult") def post(self, url, user, password): self.logger.info("posting result using curl") test_type ="TestScript" test_id = self.configs['test_id'] result = self.configs['result'] action = self.configs['action'] message = self.configs['message'] user = user password = password data = '<?xml version="1.0"?>' data +='<test_result> ' data +='<result>%d</result>'%(result) data +='<action>%s</action>'%(action) data +='<message>%s</message>'%(message) data +='<service_test_id>%d</service_test_id>'%(test_id) data +='</test_result>' curl = "curl -X POST -u %s:%s -d '%s' -H 'Content-Type:application/xml' %s" %(user, password,data, url) #self.logger.debug(curl) #os.system(curl) try: retcode = subprocess.call(curl, shell=True) if retcode < 0: print >>sys.stderr, "Child was terminated by signal", -retcode return False elif retcode > 0: print >>sys.stderr, "There were problems with posting the result ", retcode #print >>sys.stderr, "Child returned", retcode return False else: print >> sys.stdout, "Result was posted successfully " return True except OSError, e: print >>sys.stderr, "Exception while posting result:", e return False return True
def __init__(self, fn): self.configs = ConfigReader(fn).read() self.logger = SetupLogger().logger("DirectoryMaker")
class DirectoryMaker: def __init__(self, fn): self.configs = ConfigReader(fn).read() self.logger = SetupLogger().logger("DirectoryMaker") def get_script_references(self, db): #stmt = "SELECT service_tests.service_id, service_tests.test_id, test_scripts.filename, content_blobs.data " stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.filename, content_blobs.data " stmt += "FROM service_tests,test_scripts,content_blobs " stmt += "WHERE service_tests.test_id = test_scripts.id AND service_tests.test_type ='TestScript' " stmt += "AND test_scripts.content_blob_id = content_blobs.id AND service_tests.activated_at IS NOT NULL; " self.logger.debug(stmt) results = db.execute(stmt) references = db.fetchall() return references #make dirs of pattern <base_dir>/<service id>/<test_id>/package/ def setup_run_dirs(self, base_dir, results =[]): if not os.path.exists(base_dir): return "ERROR! test base dir %s does not exist"%base_dir for result in results: service_id = result[0] test_id = result[1] run_dir = os.path.join(base_dir, str(service_id), str(test_id), 'package') if not os.path.exists(run_dir): self.create_run_dir(run_dir, result[2], result[3]) self.logger.debug("Created test script run directory %s"%run_dir) else: self.logger.debug("Test script run directory %s already exists "%run_dir) # create the directory to run the jobs def create_run_dir(self, dir, fname, data): cwd = os.getcwd() base, package = os.path.split(dir) base, test_id = os.path.split(base) base, service_id = os.path.split(base) try: if not os.path.exists(os.path.join(base, service_id)): os.mkdir(os.path.join(base, service_id)) if not os.path.exists(os.path.join(base, service_id, test_id)): os.mkdir(os.path.join(base, service_id, test_id)) os.mkdir(dir) os.mkdir(os.path.join(os.path.split(dir)[0], "logs")) except Exception, e: e.stacktrace() if not os.path.exists(os.path.join(dir, fname)): if data != None: try: fh = open(os.path.join(dir, fname), 'wb') fh.write(data) fh.close() if self.is_zip(fname): os.chdir(dir) os.system("unzip %s"%fname ) os.chdir(cwd) cmd = "chmod -R u=rxw %s"%dir self.logger.debug(cmd) os.system(cmd) except Exception, e: self.logger.debug(str(e))
def __init__(self, configs): self.configs = configs self.logger = SetupLogger().logger("TestResult")
def __init__(self, config_file): self.config = config_file self.logger = SetupLogger().logger("ConfigReader")
class TranscriptParser: def __init__(self, file_data): self.file_data = file_data self.parser_logger = SetupLogger('transcript_parser.log', False) self.textstr = self.read_pdf() def get_student_id(self): keyword = "Student ID:" before_kew, kw, after_kw = self.textstr.partition(keyword) return int(after_kw.split()[0].strip()) def read_pdf(self): rsrcmgr = PDFResourceManager() retstr = StringIO() codec = 'utf-8' laparams = LAParams(char_margin=200) device = TextConverter(rsrcmgr, retstr, codec=codec, laparams=laparams) interpreter = PDFPageInterpreter(rsrcmgr, device) password = "" maxpages = 0 caching = True pagenos = set() for page in PDFPage.get_pages(self.file_data, pagenos, maxpages=maxpages, password=password, caching=caching, check_extractable=False): interpreter.process_page(page) textstr = retstr.getvalue() device.close() retstr.close() return textstr def parse1(self): file_string = self.read_pdf() with open("delete1.txt", "a") as f: f.write(file_string) def parse(self): pdf_binary = open(self.filename, 'rb') pdf_reader = p2.PdfFileReader(pdf_binary) if pdf_reader.isEncrypted: ''' pyPDF2 requires the pdf be decrypted. <Encryption Level: 128-bit AES> raises NotImplementedError <Encryption level: 128-bit RC4> can be decrypted with empty string ''' try: pdf_reader.decrypt('') self.parser_logger.info('File Decrypted (PyPDF2)') except NotImplementedError: try: # prevent race conditions when creating tmp files temp_dir = tempfile.mkdtemp( dir=os.path.dirname(self.filename)) temp_out = os.path.join(temp_dir, 'qpdf_out.pdf') # Will raise if this call is unsuccessful check_call([ 'qpdf', "--password="******"" num_pages = pdf_reader.getNumPages() for i in range(num_pages): content += pdf_reader.getPage(i).extractText() + "\n" content = " ".join(content.replace(u"\xa0", " ").strip().split()) with open("delete1.txt", "a") as f: f.write(content)
def __init__(self, fn): self.configs = ConfigReader(fn).read() self.outfile = '../config/script_listing.xml' self.logger = SetupLogger().logger("ScriptListingWriter:")
def __init__(self, fn): self.configs = ConfigReader(fn).read() self.outfile = "../config/script_listing.xml" self.logger = SetupLogger().logger("ScriptListingWriter:")
class ScriptListingWriter: def __init__(self, fn): self.configs = ConfigReader(fn).read() self.outfile = "../config/script_listing.xml" self.logger = SetupLogger().logger("ScriptListingWriter:") def get_script_references(self, db): self.logger.info("getting script references from db : %s" % db) stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name " stmt += "FROM service_tests,test_scripts " stmt += "WHERE service_tests.test_id =test_scripts.id " stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language <> 'soapui' " stmt += "AND service_tests.activated_at IS NOT NULL ; " self.logger.info(stmt) results = db.execute(stmt) references = db.fetchall() return references def get_soapui_references(self, db): self.logger.info("getting soapui project references from db : %s" % db) stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name " stmt += "FROM service_tests,test_scripts " stmt += "WHERE service_tests.test_id =test_scripts.id " stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language = 'soapui' " stmt += "AND service_tests.activated_at IS NOT NULL ; " self.logger.info(stmt) results = db.execute(stmt) references = db.fetchall() return references def write_listing(self, scripts, soapui): self.logger.info("writing script listing file : %s" % self.outfile) fh = open(self.outfile, "w") fh.write("<test_harness> \n") fh.write("<scripts> \n") for script in scripts: fh.write(" <script> \n") fh.write("\t<service_id>%s</service_id> \n" % str(script[0])) fh.write("\t<test_id>%s</test_id> \n" % str(script[1])) fh.write("\t<executable>%s</executable> \n" % str(script[2])) fh.write(" </script> \n\n") fh.write("</scripts> \n") fh.write("<soapui> \n") for project in soapui: fh.write(" <project> \n") fh.write("\t<service_id>%s</service_id> \n" % str(project[0])) fh.write("\t<test_id>%s</test_id> \n" % str(project[1])) fh.write("\t<executable>%s</executable> \n" % str(project[2])) fh.write(" </project> \n\n") fh.write("</soapui> \n") fh.write("</test_harness> \n") fh.close() # connect to db def connect_to_db(self, host, port, user, passwd, db): conn = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=db) cursor = conn.cursor() self.logger.info("Connected to db : %s" % db) return cursor
from setup_logger import SetupLogger MAX_PROCESS_TIME = 60 * 5 # Scripts should not run run for more than set no of seconds pool_size = 10 # process pool size pool = {} # process pool poll_time = 30 # length of wait between polls (in seconds) queue = [] # Queue of processes to execute soapui_jobs = [] cursor = None # DB connection handle completed = [] # completed processes passed = [] # sucessful processes failed = [] # failed processes total_scripts = 0 total_soapui = 0 logger = SetupLogger().logger("biocat_harness") # command line options usage = "usage: %prog [options] configuration_file" parser = OptionParser(usage=usage) parser.add_option("-l", "--log", dest="logfile", help="send output to logfile", metavar="FILE", default="harness-status-%s.log" % (time.strftime("%m%d-%Y-%H%M%S", time.gmtime()))) parser.add_option("-f", "--fromFile", dest="fromFile", help="XML file containing the scripts to be run",
def __init__(self, filename): self.filename = filename self.logger = SetupLogger().logger("ScriptListingReader")
class ScriptListingWriter: def __init__(self, fn): self.configs = ConfigReader(fn).read() self.outfile = '../config/script_listing.xml' self.logger = SetupLogger().logger("ScriptListingWriter:") def get_script_references(self, db): self.logger.info("getting script references from db : %s" %db) stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name " stmt += "FROM service_tests,test_scripts " stmt += "WHERE service_tests.test_id =test_scripts.id " stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language <> 'soapui' " stmt += "AND service_tests.activated_at IS NOT NULL ; " self.logger.info(stmt) results = db.execute(stmt) references = db.fetchall() return references def get_soapui_references(self, db): self.logger.info("getting soapui project references from db : %s" %db) stmt = "SELECT service_tests.service_id, service_tests.id, test_scripts.exec_name " stmt += "FROM service_tests,test_scripts " stmt += "WHERE service_tests.test_id =test_scripts.id " stmt += "AND service_tests.test_type ='TestScript' AND test_scripts.prog_language = 'soapui' " stmt += "AND service_tests.activated_at IS NOT NULL ; " self.logger.info(stmt) results = db.execute(stmt) references = db.fetchall() return references def write_listing(self, scripts, soapui ): self.logger.info("writing script listing file : %s" %self.outfile) fh = open(self.outfile, 'w') fh.write('<test_harness> \n' ) fh.write('<scripts> \n' ) for script in scripts: fh.write(' <script> \n' ) fh.write('\t<service_id>%s</service_id> \n'%str(script[0]) ) fh.write('\t<test_id>%s</test_id> \n'%str(script[1]) ) fh.write('\t<executable>%s</executable> \n'%str(script[2]) ) fh.write(' </script> \n\n' ) fh.write('</scripts> \n' ) fh.write('<soapui> \n' ) for project in soapui: fh.write(' <project> \n' ) fh.write('\t<service_id>%s</service_id> \n'%str(project[0]) ) fh.write('\t<test_id>%s</test_id> \n'%str(project[1]) ) fh.write('\t<executable>%s</executable> \n'%str(project[2]) ) fh.write(' </project> \n\n' ) fh.write('</soapui> \n' ) fh.write('</test_harness> \n' ) fh.close() #connect to db def connect_to_db(self, host, port, user, passwd, db): conn = MySQLdb.connect(host=host, port=port, user=user, passwd=passwd, db=db) cursor = conn.cursor() self.logger.info("Connected to db : %s" %db ) return cursor
def __init__(self, file_data): self.file_data = file_data self.parser_logger = SetupLogger('transcript_parser.log', False) self.textstr = self.read_pdf()
def __init__(self): self.debug_logger = SetupLogger('transcript.log', False)
MAX_PROCESS_TIME = 60*5 # Scripts should not run run for more than set no of seconds pool_size = 10 # process pool size pool = {} # process pool poll_time = 30 # length of wait between polls (in seconds) queue = [] # Queue of processes to execute soapui_jobs = [] cursor = None # DB connection handle completed = [] # completed processes passed = [] # sucessful processes failed = [] # failed processes total_scripts = 0 total_soapui = 0 logger = SetupLogger().logger("biocat_harness") # command line options usage = "usage: %prog [options] configuration_file" parser = OptionParser(usage=usage) parser.add_option("-l", "--log", dest="logfile", help="send output to logfile", metavar="FILE", default="harness-status-%s.log" %(time.strftime("%m%d-%Y-%H%M%S", time.gmtime()))) parser.add_option("-f", "--fromFile", dest="fromFile", help="XML file containing the scripts to be run", metavar="FILE") parser.add_option("-d", "--database", dest="db", default=False, help="get the scripts to run from a Biocatalogue database") (options, args) = parser.parse_args() save_stdout = sys.stdout