def do_child(runname, debug=False): """ Run a test inside the child process. @param runname: Name of muFAT run @param debug: Whether to actually store results """ # load and execute tests from .mvrt import Core from runpy import run_path # pre-clean data folders if os.path.isdir(Core.UserDataFolder): print "Cleaning folder:", Core.UserDataFolder shutil.rmtree(Core.UserDataFolder) path = normalize( os.path.join(r"Y:\mufat\testruns\regressionpaths", runname)) sys.path.append(os.path.dirname(path)) results = run_path(path, run_name="__main__") # return results to parent for processing if not debug: q = RedisQueue("_".join(["Q", DBKEY, HOST])) q.put({ 'pass': results["passed"], 'fail': results["failed"], 'untested': results["skipped"], 'summary': results["logfile"], 'shutdown': True, 'crash': results["skipped"], 'retained_samples': [], 'return_code': 0, 'timeout': False, 'svn_rev': Core.GetRuntimeSpecialBuild() }) Core.Release() sys.exit(0)
def do_child(runname, debug=False): """ Run a test inside the child process. @param runname: Name of muFAT run @param debug: Whether to actually store results """ # load and execute tests from .mvrt import Core from runpy import run_path # pre-clean data folders if os.path.isdir(Core.UserDataFolder): print "Cleaning folder:", Core.UserDataFolder shutil.rmtree(Core.UserDataFolder) path = normalize(os.path.join(r"Y:\mufat\testruns\regressionpaths", runname)) sys.path.append(os.path.dirname(path)) results = run_path(path, run_name="__main__") # return results to parent for processing if not debug: q = RedisQueue("_".join(["Q", DBKEY, HOST])) q.put({ 'pass': results["passed"], 'fail': results["failed"], 'untested': results["skipped"], 'summary': results["logfile"], 'shutdown': True, 'crash': results["skipped"], 'retained_samples': [], 'return_code': 0, 'timeout': False, 'svn_rev': Core.GetRuntimeSpecialBuild() }) Core.Release() sys.exit(0)
continue return result if (__name__ == '__main__'): strNum = '' RESERVER_NUM = 1 ROBOT_NUM = 2 robotNumber = [] luckNumber = [] norNumber = [] result = False lucknum = RedisQueue("HeyDo_LuckNum") robotnum = RedisQueue("HeyDo_RobotNumBlock") nornum = RedisQueue("HeyDo_NorNumBlock") if not result: lucknum.del_key() robotnum.del_key() nornum.del_key() f = open("./userid.txt", "w+") for n in range(2000404, 2030000): strNum = str(n) result = checkLuckNumber(strNum) if result: luckNumber.append(strNum) lucknum.push(strNum)
import json from flask import Flask from flask import request from index.SelasticSearch import SelasticSeacrh from index.settings import DOCMAP_FILE from queue import RedisQueue app = Flask(__name__) from index.settings import INDEX_LOCK_QUEUE queue = RedisQueue(INDEX_LOCK_QUEUE) CACHE = {} REVERSE_INDEX = SelasticSeacrh.MERGE_SEGMENTS() doc_memorymap = json.load(open(DOCMAP_FILE, 'r')) @app.route('/') def hello_world(): return 'Selasticsearch is up now' @app.route('/search') def search(): q = request.args['q'] cache_q = q field = request.args.get('field', '') q = map(lambda x: x.lower(), q.split()) search = SelasticSeacrh([], REVERSE_INDEX, doc_memorymap) if len(q) > 1: if cache_q in CACHE:
import redis import os import time import run import json from queue import RedisQueue from config import r trend_q = RedisQueue('trending', r) trend_store = 'trending_store' while True: print 'checking work queue' message = trend_q.dequeue() print message # get trending players if message[1] == 'GET TRENDING PLAYERS': results = run.trending_players() print results r.set(trend_store, json.dumps(results)) time.sleep(2)
import redis import run import os import time from queue import RedisQueue redis_url = os.getenv('REDISTOGO_URL') scraper_q = RedisQueue('scrapers', redis_url) # Listens for message 'CRAWL ALL SPIDERS' on Redis queue at key 'scrapers' while True: print 'checking work queue' message = scraper_q.dequeue() print message if message[1] == 'CRAWL ALL SPIDERS': run.crawl_all() print 'DONE CRAWLING' if message[1] == 'PRINT DIRS': run.print_dirs() time.sleep(2)
def main(suites_or_runs, debug=False): """ Runs a list of suites of runs inside the parent process. :param suites_or_runs: A list of suite names or run names to be executed :param debug: Whether to actually store results """ suites = {} # argument is just a single string if isinstance(suites_or_runs, basestring): suites_or_runs = [suites_or_runs] for arg in suites_or_runs: if os.path.splitext(arg)[1] != ".py": if not suites.has_key(arg): suites[arg] = set() suites[arg].update(load_suite(arg)) else: if not suites.has_key("mac"): suites["mac"] = set() suites["mac"].add(arg) global DBKEY DBKEY = DBKEY or time.strftime("%Y-%m-%d,%H-%M-%S") # cleanup/create necessary folders if not os.path.exists(MUVEEDEBUG): os.makedirs(MUVEEDEBUG) cachedir = os.path.expanduser("~/Library/Application\ Support/muvee\ Technologies/071203") if os.path.exists(cachedir): shutil.rmtree(cachedir) # python command to launch the child process svn_rev = 0 cmd = [sys.executable, "-u", "-m", "muvee.runner"] if sys.platform == "darwin": cmd = ["arch -i386"] + cmd # prepare to upload logfiles to Amazon S3 new_key = boto.connect_s3(AWS_ACCESS_KEY, AWS_SECRET_KEY) \ .get_bucket("mufat").new_key subkey = time.strftime("%Y-%m-%d_%H_%M", time.strptime(DBKEY, "%Y-%m-%d,%H-%M-%S")) global PRINT_OUTPUT for suite, runs in suites.iteritems(): for run in runs: shortname = os.path.splitext(os.path.basename(run))[0] start = time.time() logfile = os.path.join(MUVEEDEBUG, "(%s)%s_Log.txt" % \ (time.strftime("%Y%m%d%H%M%S", time.localtime(start)), shortname)) # run child process print "Starting muFAT process for %s (%s)." % (run, suite) p = subprocess.Popen(" ".join(cmd + ['"' + run + '"', "--child", "--key", DBKEY]), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # start a watchdog to kill the child if it takes too long Watchdog(p, 3600).start() # collect output text for processing later with codecs.open(logfile, "w+", "utf-8") as f: while True: line = unicode(p.stdout.readline(), errors="replace") if not line: break f.write(line) if PRINT_OUTPUT: print line.encode("ascii", errors="replace"), # block until process completes and record running time p.communicate() minutes, seconds = divmod(time.time() - start, 60) hours, minutes = divmod(minutes, 60) # read results from child try: q = RedisQueue("_".join(["Q", DBKEY, HOST])) result = q.get_nowait() except RedisQueue.Empty: # no results - child probably died? result = { 'pass': 0, 'fail': 0, 'untested': 0, 'summary': '', 'shutdown': False, 'crash': True, 'retained_samples': [], 'return_code':-1, 'timeout': False } # parse log file for assertions and upload to Amazon S3 try: with open(logfile, "r") as f: asserts, assertdict = get_asserts(f.read()) result.update({ 'assert': asserts, 'unique_asserts': assertdict, 'time': (hours, minutes, seconds) }) key = new_key("%s/%s/(%s)%s_Log.txt" % (HOST.upper(), subkey, \ time.strftime("%Y%m%d%H%M%S", time.localtime(start)), shortname)) key.set_contents_from_file(f, reduced_redundancy=True, rewind=True) key.make_public() result["log"] = "https://mufat.s3.amazonaws.com/" + key.key finally: os.remove(logfile) # upload summary file to Amazon S3 if result.get("summary"): summary = result["summary"] try: key = new_key("%s/%s/%s" % (HOST.upper(), subkey, shortname + ".txt")) key.set_contents_from_filename(summary, reduced_redundancy=True) key.make_public() result["summary"] = "https://mufat.s3.amazonaws.com/" + key.key finally: os.remove(summary) if result.get("svn_rev"): svn_rev = result["svn_rev"] if not debug: print "Uploading intermediate results..." r = requests.post(SERVER_URL + "%s/%s/%s/submit" % (DB, DBKEY, HOST), { 'suite': suite, 'runname': run, 'results': json.dumps(result) }, headers={ "X-NO-LOGIN": "******" }) r.raise_for_status()
def main(suites_or_runs, debug=False): """ Runs a list of suites of runs inside the parent process. :param suites_or_runs: A list of suite names or run names to be executed :param debug: Whether to actually store results """ suites = {} # argument is just a single string if isinstance(suites_or_runs, basestring): suites_or_runs = [suites_or_runs] for arg in suites_or_runs: if os.path.splitext(arg)[1] != ".py": if not suites.has_key(arg): suites[arg] = set() suites[arg].update(load_suite(arg)) else: if not suites.has_key("mac"): suites["mac"] = set() suites["mac"].add(arg) global DBKEY DBKEY = DBKEY or time.strftime("%Y-%m-%d,%H-%M-%S") # cleanup/create necessary folders if not os.path.exists(MUVEEDEBUG): os.makedirs(MUVEEDEBUG) cachedir = os.path.expanduser( "~/Library/Application\ Support/muvee\ Technologies/071203") if os.path.exists(cachedir): shutil.rmtree(cachedir) # python command to launch the child process svn_rev = 0 cmd = [sys.executable, "-u", "-m", "muvee.runner"] if sys.platform == "darwin": cmd = ["arch -i386"] + cmd # prepare to upload logfiles to Amazon S3 new_key = boto.connect_s3(AWS_ACCESS_KEY, AWS_SECRET_KEY) \ .get_bucket("mufat").new_key subkey = time.strftime("%Y-%m-%d_%H_%M", time.strptime(DBKEY, "%Y-%m-%d,%H-%M-%S")) global PRINT_OUTPUT for suite, runs in suites.iteritems(): for run in runs: shortname = os.path.splitext(os.path.basename(run))[0] start = time.time() logfile = os.path.join(MUVEEDEBUG, "(%s)%s_Log.txt" % \ (time.strftime("%Y%m%d%H%M%S", time.localtime(start)), shortname)) # run child process print "Starting muFAT process for %s (%s)." % (run, suite) p = subprocess.Popen( " ".join(cmd + ['"' + run + '"', "--child", "--key", DBKEY]), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) # start a watchdog to kill the child if it takes too long Watchdog(p, 3600).start() # collect output text for processing later with codecs.open(logfile, "w+", "utf-8") as f: while True: line = unicode(p.stdout.readline(), errors="replace") if not line: break f.write(line) if PRINT_OUTPUT: print line.encode("ascii", errors="replace"), # block until process completes and record running time p.communicate() minutes, seconds = divmod(time.time() - start, 60) hours, minutes = divmod(minutes, 60) # read results from child try: q = RedisQueue("_".join(["Q", DBKEY, HOST])) result = q.get_nowait() except RedisQueue.Empty: # no results - child probably died? result = { 'pass': 0, 'fail': 0, 'untested': 0, 'summary': '', 'shutdown': False, 'crash': True, 'retained_samples': [], 'return_code': -1, 'timeout': False } # parse log file for assertions and upload to Amazon S3 try: with open(logfile, "r") as f: asserts, assertdict = get_asserts(f.read()) result.update({ 'assert': asserts, 'unique_asserts': assertdict, 'time': (hours, minutes, seconds) }) key = new_key("%s/%s/(%s)%s_Log.txt" % (HOST.upper(), subkey, \ time.strftime("%Y%m%d%H%M%S", time.localtime(start)), shortname)) key.set_contents_from_file(f, reduced_redundancy=True, rewind=True) key.make_public() result["log"] = "https://mufat.s3.amazonaws.com/" + key.key finally: os.remove(logfile) # upload summary file to Amazon S3 if result.get("summary"): summary = result["summary"] try: key = new_key("%s/%s/%s" % (HOST.upper(), subkey, shortname + ".txt")) key.set_contents_from_filename(summary, reduced_redundancy=True) key.make_public() result[ "summary"] = "https://mufat.s3.amazonaws.com/" + key.key finally: os.remove(summary) if result.get("svn_rev"): svn_rev = result["svn_rev"] if not debug: print "Uploading intermediate results..." r = requests.post(SERVER_URL + "%s/%s/%s/submit" % (DB, DBKEY, HOST), { 'suite': suite, 'runname': run, 'results': json.dumps(result) }, headers={"X-NO-LOGIN": "******"}) r.raise_for_status()