def runSimsForApp(self, app, appConf): testReportFile = self.prepareTestReport(self.getConfigSetting("base/reportDirectory", ""), app) logDirectory = self.getConfigSetting("base/logDirectory", False) if logDirectory: testLogFile = "%s/%s/%s.log" %(logDirectory,app,util.getTimestamp()) seleniumConfig = self.getConfigSetting("selenium") manageSeleniumServer = self.getConfigSetting("selenium/seleniumServerJar", False) individualServer = self.getConfigSetting("individualServer", True, appConf) if manageSeleniumServer and not individualServer: self.log.info("Using one Selenium server instance for all %s simulations." %app) seleniumOptions = self.getConfigSetting("seleniumServerOptions", None, appConf) if seleniumOptions: seleniumConfig["options"] = seleniumOptions seleniumServer = SeleniumServer(seleniumConfig) seleniumServer.start() if app in self.buildStatus: if self.buildStatus[app]["BuildError"]: self.sendDummyReport(app, appConf, testReportFile) return self.log.info("Running simulations for %s" %app) for browser in appConf["browsers"]: if manageSeleniumServer and individualServer: seleniumServer = SeleniumServer(seleniumConfig) seleniumServer.start() simConf = self.getSimulationConfig(app, "applications", browser) simConf["testLogFile"] = testLogFile sim = Simulation(simConf, logger=self.log) sim.run() if manageSeleniumServer and individualServer: seleniumServer.stop() if manageSeleniumServer and not individualServer: seleniumServer = SeleniumServer(seleniumConfig) seleniumServer.stop() if self.getConfigSetting("reporting/mailTo", False): self.formatLog(simConf["testLogFile"], testReportFile, None) self.sendReport(app, testReportFile, self.getConfigSetting("reporting")) reportServer = self.getConfigSetting("reporting/reportServer", False) if reportServer: reportConf = { "runType" : self.getConfigSetting("testRun/runType", ""), "autName" : app, "autHost" : self.getConfigSetting("testRun/host", ""), "autQxPath" : self.getConfigSetting("testRun/qxPath", ""), "autPath" : appConf["path"], "startDate" : util.getTimestamp(), "testHostName" : self.getConfigSetting("base/testHostName", ""), "testHostId" : self.getConfigSetting("base/testHostId", ""), } try: self.reportResults(reportServer, simConf["testLogFile"], reportConf, simConf["ignoreLogEntries"]) except Exception, e: self.log.error("Error sending dummy report: " + repr(e))
def getUserLoginsAndContribOvertime(self, date, org=None, prev_days=6): dico_hours_contrib = {} dico_hours = {} for curDate in util.getXPrevHoursSpan(date, prev_days * 24): dico_hours[util.getTimestamp( curDate)] = 0 # populate with empty data dico_hours_contrib[util.getTimestamp( curDate)] = 0 # populate with empty data for curDate in util.getXPrevDaysSpan(date, prev_days): if org is None: dates = self.getUserLogins(curDate) else: dates = self.getDates(org, date=curDate) keyname = "{}:{}".format(self.keyContribDay, util.getDateStrFormat(curDate)) if org is None: orgs_contri = self.serv_redis_db.zrange(keyname, 0, -1, desc=True, withscores=True) orgs_contri_num = 0 for _, count in orgs_contri: orgs_contri_num += count else: orgs_contri_num = self.serv_redis_db.zscore(keyname, org) orgs_contri_num = orgs_contri_num if orgs_contri_num is not None else 0 for curDate in util.getHoursSpanOfDate( curDate, adaptToFitCurrentTime=True): #fill hole day dico_hours_contrib[util.getTimestamp( curDate)] = orgs_contri_num for d in dates: # sum occurence during the current hour dateTimestamp = d.replace(minute=0, second=0, microsecond=0) try: dico_hours[util.getTimestamp(dateTimestamp)] += 1 except KeyError: # timestamp out of bound (greater than 1 week) pass # Format data # login to_ret = {} data = [] for curDate, occ in dico_hours.items(): data.append([curDate, occ]) data.sort(key=lambda x: x[0]) to_ret['login'] = data # contrib data = [] for curDate, occ in dico_hours_contrib.items(): data.append([curDate, occ]) data.sort(key=lambda x: x[0]) to_ret['contrib'] = data return to_ret
def testGenerator(cls, config, logger): initialPath = os.getcwd() results = [] for test in config: logger.debug("Changing working dir to " + test["path"]) os.chdir(test["path"]) logger.info("Getting list of Generator jobs") jobList = Builder.__getGeneratorJobs(test) configOption = "" if "config" in test: configOption = " -c %s" %test["config"] for job in jobList: cmd = "%s generate.py -s %s %s" %(sys.executable, configOption, job) result = Builder.__getGeneratorResultDict(job, test["path"]) logger.info("Running job: " + cmd) (ret,out,err) = util.invokePiped(cmd) result["stopDate"] = util.getTimestamp() result["command"] = cmd result["returncode"] = ret result["stdout"] = out result["stderr"] = err results.append(result) if ret == 0: logger.info("Result: OK") else: logger.warn("Result: Error") logger.warn(err) os.chdir(initialPath) return results
def sendDummyReport(self, app, appConf, testReportFile): self.log.debug("%s built with errors" %app) browserList = [] for browser in appConf["browsers"]: browserList.append(browser["browserId"]) startDate = util.getTimestamp() dummyLogFile = self.getDummyLog(self.getConfigSetting("base/logDirectory", None), app, browserList, self.buildStatus[app]["BuildError"]) #TODO: ignore ignore = None if self.getConfigSetting("reporting/mailTo", False): logFormatted = self.formatLog(dummyLogFile, testReportFile, ignore) if logFormatted: self.sendReport(app, testReportFile, self.configuration["reporting"]) else: self.log.warn("No report HTML to send.") reportServer = self.getConfigSetting("reporting/reportServer", False) if reportServer: reportConf = { "runType" : self.getConfigSetting("testRun/runType", ""), "autName" : app, "autHost" : self.getConfigSetting("testRun/host"), "autQxPath" : self.getConfigSetting("testRun/qxPath", ""), "autPath" : appConf["path"], "startDate" : startDate, "testHostName" : self.getConfigSetting("base/testHostName", ""), "testHostId" : self.getConfigSetting("base/testHostId", ""), } try: self.reportResults(reportServer, dummyLogFile, reportConf) except Exception, e: self.log.error("Error sending dummy report: " + str(e))
def testGenerator(cls, config, logger): initialPath = os.getcwd() results = [] for test in config: logger.debug("Changing working dir to " + test["path"]) os.chdir(test["path"]) logger.info("Getting list of Generator jobs") jobList = Builder.__getGeneratorJobs(test) configOption = "" if "config" in test: configOption = " -c %s" % test["config"] for job in jobList: cmd = "%s generate.py -s %s %s" % (sys.executable, configOption, job) result = Builder.__getGeneratorResultDict(job, test["path"]) logger.info("Running job: " + cmd) (ret, out, err) = util.invokePiped(cmd) result["stopDate"] = util.getTimestamp() result["command"] = cmd result["returncode"] = ret result["stdout"] = out result["stderr"] = err results.append(result) if ret == 0: logger.info("Result: OK") else: logger.warn("Result: Error") logger.warn(err) os.chdir(initialPath) return results
def __init__(self, config, simulate=False, logger=log): self.configuration = config if self.getConfigSetting("base/type", "") == "standalone-generator": self.configuration = self.getConfigFromGenerator() self.log = logger self.simulate = simulate self.startDate = util.getTimestamp() self.buildStatus = {} if "base" in self.configuration: self.configuration["base"]["testHostName"] = util.getHostName() self.log.info("New test run started on %s" % self.startDate)
def __init__(self, config, simulate=False, logger=log): self.configuration = config if self.getConfigSetting("base/type", "") == "standalone-generator": self.configuration = self.getConfigFromGenerator() self.log = logger self.simulate = simulate self.startDate = util.getTimestamp() self.buildStatus = {} if "base" in self.configuration: self.configuration["base"]["testHostName"] = util.getHostName() self.log.info("New test run started on %s" %self.startDate)
def getOrgOvertime(self, org): overtime = [] today = datetime.datetime.today() today = today.replace(hour=0, minute=0, second=0, microsecond=0) for curDate in util.getXPrevDaysSpan(today, 7): timestamp = util.getTimestamp(curDate) keyname = "{}:{}".format(self.keyDay, util.getDateStrFormat(curDate)) org_score = self.serv_redis_db.zscore(keyname, org) if org_score is None: org_score = 0 overtime.append([timestamp, org_score]) to_return = {'label': org, 'data': overtime} return to_return
def getGenericTrendingOvertime(self, dateS, dateE, choice=None, topNum=0): if choice == 'categs': trendingType = self.keyCateg elif choice == 'tags': trendingType = self.keyTag else: trendingType = self.keyEvent dico_items = {} to_format = [] prev_days = (dateE - dateS).days # get data for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum - 1, desc=True, withscores=True) data = [[record[0].decode('utf8'), record[1]] for record in data] data = data if data is not None else [] to_format.append([util.getTimestamp(curDate), data]) for timestamp, array in to_format: for item, _ in array: if item not in dico_items: dico_items[item] = [] dico_items[item].append(timestamp) # sort timestamps in correct order for item in dico_items.keys(): dico_items[item].sort() # dico_items have the form: {item: [t1,t2,t4], ...} to_ret = [] ONEDAY = 60 * 60 * 24 for item, timestamps in dico_items.items(): obj = { 'name': item, 'start': timestamps[0], 'end': timestamps[0] + ONEDAY } for t in timestamps: if t - obj['end'] > ONEDAY: #new entry to_ret.append(copy.deepcopy(obj)) obj['start'] = t obj['end'] = t + ONEDAY else: # contrinue entry obj['end'] = t + ONEDAY to_ret.append(obj) return to_ret
def __getGeneratorResultDict(cls, job, path): result = { "job" : job, "path" : path, "command" : False, "startDate" : util.getTimestamp(), "revision" : util.getSvnVersion(path), "host" : util.getHostName(), "stopDate" : False, "returncode" : False, "stdout" : False, "stderr" : False } return result
def __getGeneratorResultDict(cls, job, path): result = { "job": job, "path": path, "command": False, "startDate": util.getTimestamp(), "revision": util.getSvnVersion(path), "host": util.getHostName(), "stopDate": False, "returncode": False, "stdout": False, "stderr": False } return result
def getSpecificTrending(self, trendingType, dateS, dateE, specificLabel=''): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zscore(keyname, specificLabel) data = [[specificLabel, data]] if data is not None else [] to_ret.append([util.getTimestamp(curDate), data]) return to_ret
def __init__(self, config, simulate=False, logger=None): self.configuration = config if self.getConfigSetting("type", "") == "standalone-generator": self.configuration = self.getConfigFromGenerator() if not logger: self.log = self.getLogger() else: self.log = logger self.simulate = simulate self.startDate = util.getTimestamp() self.configuration["base"]["testHostName"] = util.getHostName() self.log.info("New test run started on %s" %self.startDate)
def getGenericTrending(self, trendingType, dateS, dateE, topNum=0): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(trendingType, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum - 1, desc=True, withscores=True) data = [[record[0].decode('utf8'), record[1]] for record in data] data = data if data is not None else [] to_ret.append([util.getTimestamp(curDate), data]) return to_ret
def sendResultToReportServer(reportServerUrl, data, resultType = "testRun" ): if type(data).__name__ != "str": data = util.getJson(data) postdata = urllib.urlencode({resultType : data}) req = urllib2.Request(reportServerUrl, postdata) try: response = urllib2.urlopen(req) except urllib2.URLError, e: try: fileName = "reportservererror_%s.html" %util.getTimestamp() errorFile = open(fileName, "w") errorFile.write(e.read()) errorFile.close() except Exception: pass raise RuntimeError, e
def sendResultToReportServer(reportServerUrl, data, resultType="testRun"): if type(data).__name__ != "str": data = util.getJson(data) postdata = urllib.urlencode({resultType: data}) req = urllib2.Request(reportServerUrl, postdata) try: response = urllib2.urlopen(req) except urllib2.URLError, e: try: fileName = "reportservererror_%s.html" % util.getTimestamp() errorFile = open(fileName, "w") errorFile.write(e.read()) errorFile.close() except Exception: pass raise RuntimeError, e
def getTrendingSightings(self, dateS, dateE): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(self.keySigh, util.getDateStrFormat(curDate)) sight = self.serv_redis_db.get(keyname) sight = 0 if sight is None else int(sight.decode('utf8')) keyname = "{}:{}".format(self.keyFalse, util.getDateStrFormat(curDate)) fp = self.serv_redis_db.get(keyname) fp = 0 if fp is None else int(fp.decode('utf8')) to_ret.append([ util.getTimestamp(curDate), { 'sightings': sight, 'false_positive': fp } ]) return to_ret
def getTrendingTags(self, dateS, dateE, topNum=12): to_ret = [] prev_days = (dateE - dateS).days for curDate in util.getXPrevDaysSpan(dateE, prev_days): keyname = "{}:{}".format(self.keyTag, util.getDateStrFormat(curDate)) data = self.serv_redis_db.zrange(keyname, 0, topNum - 1, desc=True, withscores=True) data = [[record[0].decode('utf8'), record[1]] for record in data] data = data if data is not None else [] temp = [] for jText, score in data: temp.append([json.loads(jText), score]) data = temp to_ret.append([util.getTimestamp(curDate), data]) return to_ret
def testGenerator(cls, config): initialPath = os.getcwd() results = [] for test in config: os.chdir(test["path"]) jobList = Builder.__getGeneratorJobs(test) configOption = "" if "config" in test: configOption = " -c %s" %test["config"] for job in jobList: cmd = "python generate.py -s %s %s" %(configOption, job) result = Builder.__getGeneratorResultDict(job, test["path"]) (ret,out,err) = util.invokePiped(cmd) result["stopDate"] = util.getTimestamp() result["command"] = cmd result["returncode"] = ret result["stdout"] = out result["stderr"] = err results.append(result) os.chdir(initialPath) return results
def sendDummyReport(self, app, appConf, testReportFile): self.log.debug("%s built with errors" % app) browserList = [] for browser in appConf["browsers"]: browserList.append(browser["browserId"]) startDate = util.getTimestamp() dummyLogFile = self.getDummyLog( self.getConfigSetting("base/logDirectory", None), app, browserList, self.buildStatus[app]["BuildError"]) #TODO: ignore ignore = None if self.getConfigSetting("reporting/mailTo", False): logFormatted = self.formatLog(dummyLogFile, testReportFile, ignore) if logFormatted: self.sendReport(app, testReportFile, self.configuration["reporting"]) else: self.log.warn("No report HTML to send.") reportServer = self.getConfigSetting("reporting/reportServer", False) if reportServer: reportConf = { "runType": self.getConfigSetting("testRun/runType", ""), "autName": app, "autHost": self.getConfigSetting("testRun/host"), "autQxPath": self.getConfigSetting("testRun/qxPath", ""), "autPath": appConf["path"], "startDate": startDate, "testHostName": self.getConfigSetting("base/testHostName", ""), "testHostId": self.getConfigSetting("base/testHostId", ""), } try: self.reportResults(reportServer, dummyLogFile, reportConf) except Exception, e: self.log.error("Error sending dummy report: " + str(e))
def getTestRunDict(self, config): autName = config["autName"] if "Source" in config["autName"]: autName = config["autName"][0:config["autName"].find("Source")] testRunDict = { "test_type" : config["runType"], "revision" : "", "aut_name" : autName, "aut_host" : config["autHost"], "aut_qxpath" : "", "aut_path" : config["autPath"], "test_host" : config["testHostName"], "test_hostos" : util.getOperatingSystemName(), "test_hostid" : "", "start_date" : config["startDate"], "end_date" : util.getTimestamp(), "simulations": [], "dev_run" : False } if config["autName"] in self.buildStatus: if "SVNRevision" in self.buildStatus[config["autName"]]: revision = self.buildStatus[config["autName"]]["SVNRevision"] testRunDict["revision"] = revision if "autQxPath" in config: testRunDict["aut_qxpath"] = config["autQxPath"] if "testHostId" in config: testRunDict["test_hostid"] = config["testHostId"] if "devRun" in config: testRunDict["dev_run"] = config["devRun"] return testRunDict
def getTestRunDict(self, config): autName = config["autName"] if "Source" in config["autName"]: autName = config["autName"][0:config["autName"].find("Source")] testRunDict = { "test_type": config["runType"], "revision": "", "aut_name": autName, "aut_host": config["autHost"], "aut_qxpath": "", "aut_path": config["autPath"], "test_host": config["testHostName"], "test_hostos": util.getOperatingSystemName(), "test_hostid": "", "start_date": config["startDate"], "end_date": util.getTimestamp(), "simulations": [], "dev_run": False } if config["autName"] in self.buildStatus: if "SVNRevision" in self.buildStatus[config["autName"]]: revision = self.buildStatus[config["autName"]]["SVNRevision"] testRunDict["revision"] = revision if "autQxPath" in config: testRunDict["aut_qxpath"] = config["autQxPath"] if "testHostId" in config: testRunDict["test_hostid"] = config["testHostId"] if "devRun" in config: testRunDict["dev_run"] = config["devRun"] return testRunDict
if(detectedMotion) or ((force_capture_seconds != None and \ (datetime.datetime.now() - last_capture).total_seconds() > force_capture_seconds)): subFolder = '/forced_capture/' if detectedMotion: log.info('Sensed motion, capturing image...') subFolder = '/motion/' else: log.info('Forcing capture after %d seconds idle' % force_capture_seconds) util.takeFullPhoto(filename) last_capture = datetime.datetime.now() destination = base_folder + '/' + util.getDate( ) + subFolder + util.getTimestamp() + '.jpg' dropbox_result = util.backup_file(filename, destination) if (dropbox_result == 'OK'): log.info('Image sent to dropbox location ' + destination) else: log.error('Error uploading file to Dropbox: ' + dropbox_result) image1 = image2 #ready for next iteration! except BaseException as err: log.error('Unxepected exception, process dies here. Message: ' + str(err)) exit(1) exit(0)
"qooxdoo", "buildStatus.json") self.buildStatus = self.getBuildStatusFromFile(buildStatusFile) except Exception, e: self.buildStatus = {} def prepareBuildLog(self, buildLogDir, target): try: if not os.path.isdir(buildLogDir): os.mkdir(buildLogDir) except Exception, e: self.log.error("Failed to create build log directory %s: " % (buildLogDir, str(e))) return False buildLog = os.path.join(buildLogDir, target + '_' + util.getTimestamp() + '.log') self.log.info("Opening build log file %s" % buildLog) try: buildLogFile = codecs.open(buildLog, 'a', 'utf-8') except Exception, e: self.log.error("Error while opening build log file: %s" % str(e)) return buildLogFile def buildApp(self, target): buildLog = None if "buildLogDir" in self.config: buildLog = self.prepareBuildLog(self.config["buildLogDir"], target) cmd = "%s -w %s %s" % (self.config["batbuild"], self.config["stageDir"],
glob.glob(os.path.join(UNET_TRAIN_SPLIT_FOLDER, '*', UNET_IMAGE_FORMAT))) nbr_validation_samples = len( glob.glob(os.path.join(UNET_VAL_SPLIT_FOLDER, '*', UNET_IMAGE_FORMAT))) # autosave best Model best_model_file = os.path.join(info, 'weights.h5') best_model = ModelCheckpoint(best_model_file, monitor='val_loss', verbose=1, save_best_only=True) if os.path.exists(best_model_file): print('WARNING: Resume model and weights from previous training ...') # Backup previous model file copyfile(best_model_file, best_model_file + '.' + getTimestamp()) model = load_model(img_height, img_width, nb_channels, learning_rate, best_model_file) model.summary() else: print('Using UNET impls ... save best model to:{}'.format( best_model_file)) model = create_model(img_height, img_width, nb_channels, learning_rate) model.summary() steps_per_epoch = math.ceil(1. * nbr_train_samples / batch_size) validation_steps = math.ceil(1. * nbr_validation_samples / batch_size) print('steps_per_epoch={} , validation_steps={} epochs={}'.format( steps_per_epoch, validation_steps, nbr_epochs)) if steps_per_epoch <= 0: raise AssertionError("Found 0 train samples")
def runSimsForApp(self, app, appConf): testReportFile = self.prepareTestReport( self.getConfigSetting("base/reportDirectory", ""), app) logDirectory = self.getConfigSetting("base/logDirectory", False) if logDirectory: testLogFile = "%s/%s/%s.log" % (logDirectory, app, util.getTimestamp()) seleniumConfig = self.getConfigSetting("selenium") manageSeleniumServer = self.getConfigSetting( "selenium/seleniumServerJar", False) individualServer = self.getConfigSetting("individualServer", True, appConf) if manageSeleniumServer and not individualServer: self.log.info( "Using one Selenium server instance for all %s simulations." % app) seleniumOptions = self.getConfigSetting("seleniumServerOptions", None, appConf) if seleniumOptions: seleniumConfig["options"] = seleniumOptions seleniumServer = SeleniumServer(seleniumConfig, logger=self.log) seleniumServer.start() if app in self.buildStatus: if self.buildStatus[app]["BuildError"]: self.sendDummyReport(app, appConf, testReportFile) return self.log.info("Running simulations for %s" % app) for browser in appConf["browsers"]: if manageSeleniumServer and individualServer: seleniumServer = SeleniumServer(seleniumConfig, logger=self.log) seleniumServer.start() simConf = self.getSimulationConfig(app, "applications", browser) if testLogFile: simConf["testLogFile"] = testLogFile sim = Simulation(simConf, logger=self.log) sim.run() if manageSeleniumServer and individualServer: seleniumServer.stop() if manageSeleniumServer and not individualServer: seleniumServer = SeleniumServer(seleniumConfig, logger=self.log) seleniumServer.stop() if self.getConfigSetting("reporting/mailTo", False): self.formatLog(simConf["testLogFile"], testReportFile, None) self.sendReport(app, testReportFile, self.getConfigSetting("reporting")) reportServer = self.getConfigSetting("reporting/reportServer", False) if reportServer: reportConf = { "runType": self.getConfigSetting("testRun/runType", ""), "autName": app, "autHost": self.getConfigSetting("testRun/host", ""), "autQxPath": self.getConfigSetting("testRun/qxPath", ""), "autPath": appConf["path"], "startDate": self.startDate, "testHostName": self.getConfigSetting("base/testHostName", ""), "testHostId": self.getConfigSetting("base/testHostId", ""), } try: self.reportResults(reportServer, simConf["testLogFile"], reportConf, simConf["ignoreLogEntries"]) except Exception, e: self.log.error("Error sending report: " + str(e))
try: buildStatusFile = os.path.join(self.config["stageDir"], "trunk", "qooxdoo", "buildStatus.json") self.buildStatus = self.getBuildStatusFromFile(buildStatusFile) except Exception, e: self.buildStatus = {} def prepareBuildLog(self, buildLogDir, target): try: if not os.path.isdir(buildLogDir): os.mkdir(buildLogDir) except Exception, e: self.log.error("Failed to create build log directory %s: " %(buildLogDir, str(e))) return False buildLog = os.path.join(buildLogDir, target + '_' + util.getTimestamp() + '.log') self.log.info("Opening build log file %s" %buildLog) try: buildLogFile = codecs.open(buildLog, 'a', 'utf-8') except Exception, e: self.log.error("Error while opening build log file: %s" %str(e)) return buildLogFile def buildApp(self, target): buildLog = None if "buildLogDir" in self.config: buildLog = self.prepareBuildLog(self.config["buildLogDir"], target) cmd = "%s -w %s %s" %(self.config["batbuild"], self.config["stageDir"], self.config["targets"][target])