def fetch_all(self,deploy=True): juche.dictate(task="fetch-all") projects = get_config()["Projects"] #print projects for project in projects: juche.dictate(fetch=project["url"]) git = self.clone_or_get(project["url"],WORK_DIR+project["name"]) git.fetch() #Iterate over milestones for the current project for sFixFor in self.f.listFixFors(sProject=project["name"],onlyProjectMilestones=True): sFixFor = sFixFor.sfixfor.contents[0].encode('utf-8') if sFixFor.endswith("-test"): continue try: git.resetHard_INCREDIBLY_DESTRUCTIVE_COMMAND() # I'm still against this, but it might make things work for this week git.checkoutExistingBranchRaw(sFixFor) except: juche.warning("Can't check out branch %s in project %s, perhaps it doesn't exist?" % (sFixFor,project["name"])) continue if git.needsPull(): juche.info("Pulling %s in project %s" % (sFixFor,project["name"])) git.pull() self.integrate_changed(git,git.getBranch(),project["name"]) if deploy: try: self.deploy(git,sFixFor,project) except Exception as e: juche.error("Can't deploy") juche.exception(e)
def annoyableIxPeople(self): from dateutil import tz annoyable = [] people = self.fbConnection.listPeople() import datetime for person in people.people: ix = int(person.ixperson.contents[0]) referencetime = datetime.datetime.now().replace(hour=12,minute=0,second=0,tzinfo=tz.gettz(name="CST")) nowtime = datetime.datetime.now().replace(tzinfo=tz.tzlocal()) #print referencetime #print nowtime if nowtime > referencetime: delta = (nowtime - referencetime).total_seconds() else: delta = (referencetime-nowtime).total_seconds() if delta / 60.0 / 60.0 < 4.0: #within four hours of the prescribed time annoyable.append(ix) else: lastActive = self.userLastActive(ix) if not lastActive: juche.info("User %s does not appear to be active recently" % ix) continue delta = (nowtime - lastActive).total_seconds() if delta / 60.0 / 60.0 < 1.0: #within an hour of the prescribed time annoyable.append(ix) return annoyable
def test_active_tickets(self): projects = get_config()["Projects"] from work.fogbugzConnect import TEST_IXCATEGORY cases = self.f.fbConnection.search(q="assignedTo:=%d" % self.f.ixPerson,cols="ixCategory,sProject,hrsOrigEst,hrsElapsed,sStatus") #clean up any cases erroneously assigned to glados. for case in cases.cases: caseno = int(case["ixbug"]) if int(case.ixcategory.contents[0])==TEST_IXCATEGORY: self.glados_reassign(caseno,reactivate=False) continue proj = project_with_name(case.sproject.contents[0]) if not proj: self.glados_reassign(caseno,reactivate=False) continue if "review-workflow" in proj and proj["review-workflow"] == "no": self.glados_reassign(caseno, reactivate=False) continue if not self.f.isReadyForTest(caseno): self.glados_reassign(caseno,reactivate=False,why="Not testing this case because it is not marked as resolved/implemented.") continue (parent,test) = self.f.getCaseTuple(caseno,oldTestCasesOK=True,exceptOnFailure=False) if not test: juche.info("Ignoring case %d because it has no test case assigned." % caseno) continue if self.be_in_purgatory(parent,test,case,proj): juche.info("Ignoring case %d because of purgatory." % caseno) continue if self.test(case,proj): #returns true if we integrate something return self.test_active_tickets() #break out of this loop, because who knows what is happening with the list of active cases now, and re-test everything.
def chargeback(case): import re fbConnection = FogBugzConnect() events = fbConnection.allEvents(case) total_time = 0 def parsecase(match): if match: (fromt,tot) = match.groups(0) import dateutil.parser fromd = dateutil.parser.parse(fromt) tod = dateutil.parser.parse(tot) return (tod - fromd).total_seconds() return 0 for event in events: match = re.match("recharge: A record was removed from this ticket: From (.*) to (.*)(?=ixPerson)",event) total_time += parsecase(match) match = re.match("recharge: A record was added to this ticket: From (.*) to (.*)(?=ixPerson)",event) total_time -= parsecase(match) total_time += fbConnection.getElapsed(case) * 60.0 * 60.0 (pcase,test) = fbConnection.getCaseTuple(case,oldTestCasesOK=True,exceptOnFailure=False) if test: total_time += fbConnection.getElapsed(test) * 60.0 * 60.0 juche.info(" %d hours" % (total_time / 60.0 / 60.0) ) return total_time / 60.0 / 60.0
def projectStart(CASE_NO, fromSpec): """ print "Case no: " + str(CASE_NO) print "from: " + fromSpec """ #create new gitConnect object to talk to git gitConnection = GitConnect() #check for unsaved changes to source code gitConnection.checkForUnsavedChanges() #create new FogBugzConnect object to talk to FBAPI fbConnection = FogBugzConnect() #check for FogBugz case and clock in fbConnection.startCase(CASE_NO) #checkout or create branch with CASE_NO gitConnection.checkoutBranch(CASE_NO,fromSpec,fbConnection) settings = get_setting_dict() if not "viewOnStart" in settings or settings["viewOnStart"] == 1: fbConnection.view(CASE_NO) juche.info("Use work ship to commit your changes")
def reactivate(self,CASE_NO,assignTo,msg): try: response = self.fbConnection.reactivate(ixBug=CASE_NO,sEvent=msg,ixPersonAssignedTo=assignTo) except FogBugzAPIError as e: juche.error("Unexpected condition [%s] Is case closed? Attempting to recover..." % e) response = self.fbConnection.reopen(ixBug=CASE_NO,sEvent=msg,ixPersonAssignedTo=assignTo) juche.info("Recovery was successful.")
def createTestCase(self,PARENT_CASE,estimate="0 hours",ixTester=None): if not ixTester: ixTester = self.ixPerson #extract parent info resp = self.fbConnection.search(q=PARENT_CASE,cols="ixProject,ixArea,ixFixFor,sFixFor,ixPriority") #look for a test milestone milestones = self.fbConnection.listFixFors(ixProject=resp.case.ixproject.contents[0]) ixTestMilestone = 0 foundTestMilestone = False for aMilestone in milestones.fixfors: #print aMilestone.sfixfor.contents[0], resp.case.sfixfor.contents[0] + "-test" if(aMilestone.sfixfor.contents[0].find(resp.case.sfixfor.contents[0] + "-test") != -1): foundTestMilestone = True ixTestMilestone = aMilestone.ixfixfor.contents[0] testMilestone = resp.case.sfixfor.contents[0] + "-test" #print "testMilestone: ", testMilestone #print "\nfoundTestMilestone: ", foundTestMilestone if not foundTestMilestone: ixTestMilestone = self.fbConnection.newFixFor(ixProject=resp.case.ixproject.contents[0],sFixFor=testMilestone, fAssignable="1") self.fbConnection.addFixForDependency(ixFixFor=ixTestMilestone, ixFixForDependsOn=resp.case.ixproject.contents[0]) #print "creating new milestone and setting dependencies! New Milestone: ", ixTestMilestone.ixfixfor.contents[0] ixTestMilestone = ixTestMilestone.ixfixfor.contents[0] #print resp.case response = self.fbConnection.new(ixBugParent=PARENT_CASE,sTitle="Review",ixPersonAssignedTo=ixTester,hrsCurrEst=estimate,ixPriority=resp.case.ixpriority.contents[0],sEvent="Cake and grief counseling will be available at the conclusion of the test.",ixCategory=6, ixProject=resp.case.ixproject.contents[0],ixArea=resp.case.ixarea.contents[0],ixFixFor=ixTestMilestone,sTags=TEST_TAG) juche.info("Created case %s" % response.case['ixbug'])
def create_tests(): from work.fogbugzConnect import FogBugzConnect from config import project_with_name f = FogBugzConnect() #we create test cases if and only if all the following conditions are met: #1. Bugs and features #2. Open cases #3. Cases with an estimate (otherwise, the person assigned might just be a placeholder person...) #4. Cases that are decided (not Undecided) #5. Cases that are in projects for which we have review-workflow: yes cases = f.fbConnection.search(q='(category:"bug" OR category:"feature") status:"open" estimatecurrent:"1m.." -milestone:"Undecided"', cols="sProject") cache = buildbot_cache_get() CACHE_KEY = "autoTestMake-cache" if not cache.has_key(CACHE_KEY): cache[CACHE_KEY] = [] casesList = map(lambda x: int(x["ixbug"]),cases.cases) casesList = filter(lambda x: x not in cache[CACHE_KEY],casesList) juche.info(casesList) from work.work import autoTestMake for case in cases.cases.contents: project = project_with_name(case.sproject.contents[0]) if not project: juche.info("No project named %s in buildbot.yaml" % case.sproject.contents[0]) continue if not ("review-workflow" in project.keys() and project["review-workflow"] == "yes"): continue result = autoTestMake(int(case["ixbug"])) if not result: cache[CACHE_KEY].append(case["ixbug"]) buildbot_cache_set(cache)
def setEstimate(self, CASE_NO,timespan=None): if not timespan: juche.info("Please provide an estimate for this case: ") timespan = raw_input() self.fbConnection.edit(ixBug=CASE_NO, hrsCurrEst=timespan) return timespan;
def autoTestMake(CASE_NO,fbConnection=None): juche.info("autotestmake %d" % CASE_NO) if not fbConnection: fbConnection = FogBugzConnect() (implement,test) = fbConnection.getCaseTuple(CASE_NO,oldTestCasesOK=True,exceptOnFailure=False) if not test: ixTester = fbConnection.optimalIxTester(CASE_NO) fbConnection.createTestCase(CASE_NO,ixTester=ixTester) return True return False
def priority_fix(): from work.fogbugzConnect import FogBugzConnect f = FogBugzConnect() for case in f.listTestCases().cases: ixBug = case["ixbug"] (parent,child) = f.getCaseTuple(ixBug) parent_priority = f.fbConnection.search(q=parent,cols="ixPriority").ixpriority.contents[0] child_priority = f.fbConnection.search(q=child,cols="ixPriority").ixpriority.contents[0] if parent_priority != child_priority: juche.info("Fixing priority of case %s to %s" % (child,parent_priority)) f.setPriority(child,parent_priority)
def __checkoutExistingBranchRaw(self,arg): juche.dictate(double_checkout_existing_branch_raw=arg) (checkoutNewBranchStatus, output) = self.statusOutput("git checkout {0}".format(arg)) if(checkoutNewBranchStatus): juche.info(output) return False (status,output) = self.statusOutput("git submodule init") if status: juche.error("could not init submodule: %s",output) (status,output) = self.statusOutput("git submodule update") if status: juche.error("Error updating a submodule %s",output) return True
def exec_tests(self,proj): import subprocess import shlex shortdesc = "" files = {} passed = True #WARNING: YOU MUST PATCH $DEVELOPER/Platforms/iPhoneSimulator.platform/Developer/Tools/RunPlatformUnitTests for this to work. #See http://longweekendmobile.com/2011/04/17/xcode4-running-application-tests-from-the-command-line-in-ios/ if proj["tests"]: juche.dictate(project=proj) for test in proj["tests"]: juche.dictate(running_test=test) juche.info("running test") r = subprocess.Popen(test["cmd"],shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=WORK_DIR+proj["name"]) (status,output) = self.wait_for(r) if status and test["type"]!="kif": #kif tests always return a status code of 1 shortdesc += "Failing in part because test %s returned a non-zero return code %d\n" % (test,status) passed = False if test["type"]=="xcode": (passed,shortdesc,files) = self.parse_xcodelike_response(passed,shortdesc,files,output,test["name"]+".log") elif test["type"]=="python": (passed,shortdesc,files) = self.parse_python_response(passed,shortdesc,files,output,test["name"]+".log",status) elif test["type"]=="kif": (passed,shortdesc,files) = self.parse_kif_response(passed,shortdesc,files,output,test["name"]+".log") else: raise Exception("Unknown test type.") if passed: if "commit-files" in test: upload_files = test["commit-files"] juche.dictate(upload_files=upload_files) juche.info("commit-files") git = GitConnect(wd=WORK_DIR+proj["name"]) git.add(upload_files) try: git.commit("If the subject survived the test, we let them purchase the pictures for $5. If the subject died, we gave the photo to their next of kin free of charge") git.pushChangesToOriginBranch() except Exception as e: juche.exception(e) juche.warn("Was not able to upload the files successfully. Perhaps they have not changed?") return (passed,shortdesc,files)
def _fixFors_to_EBS_dates(abbreviatedTest=False): juche.dictate(fixing_dates_per_ebs=1) fbConnection = FogBugzConnect() fixfors = fbConnection.dependencyOrder(fbConnection.listFixFors()) if abbreviatedTest: fixfors = fixfors[:5] for item in fixfors: name = fbConnection.nameForFixFor(fbConnection.fixForDetail(item)) if name.startswith("Never"): continue juche.info("processing %s %s" % (name,item)) date = fbConnection.getShipDate(item) from dateutil.parser import parse if not abbreviatedTest: fbConnection.editFixForShipDate(item,parse(date)) #It's bad to leave EBS in a partially-edited state. Therefore we simply log the output and don't actually write any changes when in abbreviated test mode. else: juche.warning("Not editing the ship date in %s to %s because this is an abbreviated test." % (item,date))
def mergeIn(self,BRANCH_NAME,pretend=False): if pretend: return self.__mergeInPretend(BRANCH_NAME) juche.dictate(merging_in=BRANCH_NAME) (status,output) = self.statusOutput("git merge --no-ff %s" % BRANCH_NAME) juche.info(output) if status: juche.error("merge was unsuccessful.") # play sounds! self.statusOutput ("afplay -v 7 %s/media/ohno.aiff" % sys.prefix) raise Exception("stacktraceplease") else: # play sounds! from work import get_setting_dict if get_setting_dict().has_key("disablesounds") and get_setting_dict()["disablesounds"]=="YES": pass else: self.statusOutput ("afplay -v 7 %s/media/hooray.aiff" % sys.prefix) juche.info("Use 'git push' to ship.")
def editFixForShipDate(self,ixFixFor,shipDate,depCheck=True): if depCheck: override = False deptree = self._deptree(self.listFixFors()) from dateutil.parser import parse import datetime for dep in deptree[ixFixFor]: if dep not in deptree.keys(): continue depdetail = self.fixForDetail(dep) depdate = parse(depdetail.dt.contents[0]) if depdate > shipDate: override = True shipDate = depdate + datetime.timedelta(hours=1) if override: juche.info("Ship date overridden to %s" % shipDate) detail = self.fixForDetail(ixFixFor) name = self.nameForFixFor(detail) juche.info("editing ship date of %s" % name) self.fbConnection.editFixFor(ixFixFor=ixFixFor,sFixFor=name,dtRelease=shipDate,dtStart=self.getFixForStartDate(detail),fAssignable=self.getFixForDeleted(detail) and "0" or "1")
def integrate_changed(self,gitConnection,integration_branch,sProject): juche.dictate(integrate_changed_hook=integration_branch) gitConnection.checkoutExistingBranchRaw(integration_branch) self.deploy(gitConnection,integration_branch,project_with_name(sProject)) #Here's the problem: A and B are in the review queue. They are mutually exclusive (can't merge them both in.) #You pass patch A. It's merged in. We need to re-test B. Perhaps the merge will fail now, or perhaps the unit test will fail. #We "invalidate" (put back in the glados queue; snatch from the reviewer) all cases in the review queue and they will be re-tested on the next pass. cases = self.f.fbConnection.search(q="project:'%s' status:'Resolved' (category:'Bug' OR category:'Feature')" % (sProject),cols="sStatus") for case in cases.cases: caseno = int(case["ixbug"]) if self.f.getIntegrationBranch(caseno)!=integration_branch: continue if not self.f.isReadyForTest(caseno): continue juche.info( "Invalidating "+case["ixbug"]) self.f.fbConnection.assign(ixBug=case["ixbug"],ixPersonAssignedTo=self.f.ixPerson,sEvent="Invalidation. Are you getting unexpected e-mails surrounding this case event? File a bug against buildbot.") self.test_active_tickets()
def pull(self): self.checkForRepository() import ConfigParser c = ConfigParser.ConfigParser() if self.wd: path = self.wd + "/.git/config" else: path =".git/config" import os for i in range(0,30): if os.path.exists(path): break path = "../" + path if i==30: raise Exception("Not a git repository?") file = open(path) str = file.read() file.close() juche.dictate(pull=1) if self.getBranch() not in str: juche.warn( "%s is not a tracking branch. Attempting to fix..." % self.getBranch()) try: self.setUpstream(self.getBranch(),"remotes/origin/{0}".format(self.getBranch())) juche.info("Success!") except: juche.error("DID NOT AUTOMATICALLY FIX BRANCH UPSTREAM / TRACKING. PLEASE FILE A BUG.") (status,output) = self.statusOutput("git pull origin %s" % self.getBranch()) if status: juche.error("Cannot pull! %s" % output) else: (status,output) = self.statusOutput("git pull") if status: juche.error("ERROR: Cannot pull! %s" % output) raise Exception("stacktraceplease") self.submoduleUpdate() juche.info("Success!")
def recharge(fr,to): import dateutil.parser fbConnection = FogBugzConnect() fbConnection.setParentIfUnset(fr,to) #cannot create a time record for a closed case... mustOpen = not fbConnection.isOpen(to) if mustOpen: fbConnection.reopen(to,"work.py recharge") results = fbConnection.listTimeRecords(fr) time_interval = 0 my_records = [] for record in results: #print record if record.fdeleted.contents[0]!="false": juche.warn("Skipping deleted record %s" % record) continue if len(record.dtend)==0: juche.warn("Skipping open time record %s" % record) continue my_records.append(record) r = 0 for record in my_records: juche.info("%d: %s-%s" % (r,record.dtstart.contents[0],record.dtend.contents[0])) r += 1 def parse_range(astr): # http://stackoverflow.com/questions/4726168/parsing-command-line-input-for-numbers result=set() for part in astr.split(','): x=part.split('-') result.update(range(int(x[0]),int(x[-1])+1)) return sorted(result) strl = raw_input("records (syntax like 22-27,51-64): ") results = parse_range(strl) for result in results: record = my_records[result] record_desc = "From %s to %s ixPerson %s ixBug %s" % (record.dtstart.contents[0],record.dtend.contents[0],record.ixperson.contents[0],record.ixbug.contents[0]) from_time = dateutil.parser.parse(record.dtstart.contents[0]) to_time = dateutil.parser.parse(record.dtend.contents[0]) time_interval += (to_time-from_time).total_seconds() juche.info("from_time %s to_time %s time_interval %s" % (from_time,to_time,time_interval)) fbConnection.commentOn(fr,"recharge: A record was removed from this ticket: %s, see case %d" % (record_desc,to)) fbConnection.commentOn(to,"recharge: A record was added to this ticket: %s, see case %d" % (record_desc, fr)) fbConnection.createTimeRecord(to,str(record.dtstart.contents[0]),str(record.dtend.contents[0])) oldEst = fbConnection.getEstimate(fr) * 60.0 * 60.0 newEst = (oldEst - time_interval) / 60.0 / 60.0 if newEst <= 0: newEst = 1/60.0 juche.info("Setting estimate to",newEst) fbConnection.setEstimate(fr,timespan="%f hours" % newEst) #fbConnection.deleteTimeRecord(record.ixinterval.contents[0]) if mustOpen: fbConnection.closeCase(to)
def complain(ixComplainAboutPerson,sComplainAboutProject): fbConnection = FogBugzConnect() response = fbConnection.fbConnection.search(q="status:active assignedto:=%d project:%s" % (ixComplainAboutPerson,sComplainAboutProject),cols="hrsCurrEst,hrsElapsed,sPersonAssignedTo,sFixFor,sCategory") for case in response.cases: #print case if case.hrscurrest.contents[0]=="0": juche.info("%s's case %s has no estimate" % (case.spersonassignedto.contents[0], case["ixbug"])) fbConnection.commentOn(case["ixbug"],"This next test could take a very, VERY long time.") if case.sfixfor.contents[0]=="Undecided" and (case.scategory.contents[0] == "bug" or case.scategory.contents[0] == "feature"): juche.info("%s needs a milestone" % case["ixbug"]) fbConnection.commentOn(case["ixbug"],"If you choose not to decide, you still have made a choice. (Don't think about it, don't think about it...) It's a paradox! There IS no answer.") est = float(case.hrscurrest.contents[0]) act = float(case.hrselapsed.contents[0]) if est - act < 0: juche.info("%s's case %s requires updated estimate" % (case.spersonassignedto.contents[0], case["ixbug"])) fbConnection.commentOn(case["ixbug"],"I'll give you credit: I guess you ARE listening to me. But for the record: You don't have to go THAT slowly.")
def _fixFors_test_quickly_dates(abbreviatedTest=False): juche.dictate(fixing_test_milestones=1) fbConnection = FogBugzConnect() fixfors_raw = fbConnection.listFixFors() fixfors = fbConnection.dependencyOrder(fixfors_raw) from dateutil.parser import parse import datetime if abbreviatedTest: fixfors = fixfors[:5] for testMilestone in fixfors: testMilestone_raw = fbConnection.fixForDetail(testMilestone) testName = fbConnection.nameForFixFor(testMilestone_raw) if testName.startswith("Never"): continue juche.dictate(testMilestone=testMilestone,testName=testName) if not testName.endswith("-test"): continue if testName=="Undecided-test": continue matched = False if testMilestone_raw.ixproject.contents==[]: continue for item in fbConnection.listFixFors(ixProject=int(testMilestone_raw.ixproject.contents[0])): #print testName[:-5],fbConnection.nameForFixFor(item) if item.sfixfor.contents[0]==testName[:-5]: #print "matching",testName,fbConnection.nameForFixFor(item) matched = True break if not matched: juche.info(testMilestone_raw) raise Exception("Cannot match "+testName) if item.dt.contents==[]: juche.info("Can't set %s because the non-test milestone has no completion date." % testName) continue date = item.dt.contents[0] newDate = parse(date)+datetime.timedelta(hours=6) #turns out that using 1 day produces weird results. If the next implementation milestone is completed within 24 hours, lots of weird things can happen juche.info("setting %s to %s"% (testName,newDate)) if not abbreviatedTest: fbConnection.editFixForShipDate(testMilestone,newDate) #It's bad to leave EBS in a partially-edited state. Therefore we simply log the output and don't actually write any changes when in abbreviated test mode. else: juche.warn("Not editing the ship date in %s to %s because this is an abbreviated test." % (testMilestone,newDate))
def test_workingschedule(self): import datetime juche.info("Drew works %f hours" % self.f.expectedWorkHours(ixPerson=2,date=datetime.datetime.now()))
def test_admin(self): juche.info("I am an administrator: %s" % self.f.amIAdministrator())
def test_getship(self): juche.info(self.f.getShipDate(ixFixFor=43))
def test_deptree(self): juche.info(self.f.dependencyOrder(self.f.listFixFors()))
def test_lastactive(self): juche.info(self.f.userLastActive(self.f.ixPerson))
def test_optimaltester(self): juche.info(self.f.optimalIxTester(3028))
def test_listfixfors(self): semaps_fixfors = self.f.listFixFors(sProject="semaps") juche.info(semaps_fixfors) self.assertTrue(len(semaps_fixfors) > 0)
def test_annoyables(self): if not self.f.amIAdministrator(): juche.warning("NOT RUNNING test_annoyables BECAUSE YOU ARE NOT AN ADMINISTRATOR") return juche.info(self.f.annoyableIxPeople())
def still_alive(): juche.info("still alive")