def fetch_all(self,deploy=True):
juche.dictate(task="fetch-all")
projects = get_config()["Projects"]
#print projects
for project in projects:
juche.dictate(fetch=project["url"])
git = self.clone_or_get(project["url"],WORK_DIR+project["name"])
git.fetch()
#Iterate over milestones for the current project
for sFixFor in self.f.listFixFors(sProject=project["name"],onlyProjectMilestones=True):
sFixFor = sFixFor.sfixfor.contents[0].encode('utf-8')
if sFixFor.endswith("-test"): continue
try:
git.resetHard_INCREDIBLY_DESTRUCTIVE_COMMAND() # I'm still against this, but it might make things work for this week
git.checkoutExistingBranchRaw(sFixFor)
except:
juche.warning("Can't check out branch %s in project %s, perhaps it doesn't exist?" % (sFixFor,project["name"]))
continue
if git.needsPull():
juche.info("Pulling %s in project %s" % (sFixFor,project["name"]))
git.pull()
self.integrate_changed(git,git.getBranch(),project["name"])
if deploy:
try:
self.deploy(git,sFixFor,project)
except Exception as e:
juche.error("Can't deploy")
juche.exception(e)
def annoyableIxPeople(self):
from dateutil import tz
annoyable = []
people = self.fbConnection.listPeople()
import datetime
for person in people.people:
ix = int(person.ixperson.contents[0])
referencetime = datetime.datetime.now().replace(hour=12,minute=0,second=0,tzinfo=tz.gettz(name="CST"))
nowtime = datetime.datetime.now().replace(tzinfo=tz.tzlocal())
#print referencetime
#print nowtime
if nowtime > referencetime:
delta = (nowtime - referencetime).total_seconds()
else:
delta = (referencetime-nowtime).total_seconds()
if delta / 60.0 / 60.0 < 4.0: #within four hours of the prescribed time
annoyable.append(ix)
else:
lastActive = self.userLastActive(ix)
if not lastActive:
juche.info("User %s does not appear to be active recently" % ix)
continue
delta = (nowtime - lastActive).total_seconds()
if delta / 60.0 / 60.0 < 1.0: #within an hour of the prescribed time
annoyable.append(ix)
return annoyable
def test_active_tickets(self):
projects = get_config()["Projects"]
from work.fogbugzConnect import TEST_IXCATEGORY
cases = self.f.fbConnection.search(q="assignedTo:=%d" % self.f.ixPerson,cols="ixCategory,sProject,hrsOrigEst,hrsElapsed,sStatus")
#clean up any cases erroneously assigned to glados.
for case in cases.cases:
caseno = int(case["ixbug"])
if int(case.ixcategory.contents[0])==TEST_IXCATEGORY:
self.glados_reassign(caseno,reactivate=False)
continue
proj = project_with_name(case.sproject.contents[0])
if not proj:
self.glados_reassign(caseno,reactivate=False)
continue
if "review-workflow" in proj and proj["review-workflow"] == "no":
self.glados_reassign(caseno, reactivate=False)
continue
if not self.f.isReadyForTest(caseno):
self.glados_reassign(caseno,reactivate=False,why="Not testing this case because it is not marked as resolved/implemented.")
continue
(parent,test) = self.f.getCaseTuple(caseno,oldTestCasesOK=True,exceptOnFailure=False)
if not test:
juche.info("Ignoring case %d because it has no test case assigned." % caseno)
continue
if self.be_in_purgatory(parent,test,case,proj):
juche.info("Ignoring case %d because of purgatory." % caseno)
continue
if self.test(case,proj): #returns true if we integrate something
return self.test_active_tickets() #break out of this loop, because who knows what is happening with the list of active cases now, and re-test everything.
def chargeback(case):
import re
fbConnection = FogBugzConnect()
events = fbConnection.allEvents(case)
total_time = 0
def parsecase(match):
if match:
(fromt,tot) = match.groups(0)
import dateutil.parser
fromd = dateutil.parser.parse(fromt)
tod = dateutil.parser.parse(tot)
return (tod - fromd).total_seconds()
return 0
for event in events:
match = re.match("recharge: A record was removed from this ticket: From (.*) to (.*)(?=ixPerson)",event)
total_time += parsecase(match)
match = re.match("recharge: A record was added to this ticket: From (.*) to (.*)(?=ixPerson)",event)
total_time -= parsecase(match)
total_time += fbConnection.getElapsed(case) * 60.0 * 60.0
(pcase,test) = fbConnection.getCaseTuple(case,oldTestCasesOK=True,exceptOnFailure=False)
if test:
total_time += fbConnection.getElapsed(test) * 60.0 * 60.0
juche.info(" %d hours" % (total_time / 60.0 / 60.0) )
return total_time / 60.0 / 60.0
def projectStart(CASE_NO, fromSpec):
"""
print "Case no: " + str(CASE_NO)
print "from: " + fromSpec
"""
#create new gitConnect object to talk to git
gitConnection = GitConnect()
#check for unsaved changes to source code
gitConnection.checkForUnsavedChanges()
#create new FogBugzConnect object to talk to FBAPI
fbConnection = FogBugzConnect()
#check for FogBugz case and clock in
fbConnection.startCase(CASE_NO)
#checkout or create branch with CASE_NO
gitConnection.checkoutBranch(CASE_NO,fromSpec,fbConnection)
settings = get_setting_dict()
if not "viewOnStart" in settings or settings["viewOnStart"] == 1:
fbConnection.view(CASE_NO)
juche.info("Use work ship to commit your changes")
def reactivate(self,CASE_NO,assignTo,msg):
try:
response = self.fbConnection.reactivate(ixBug=CASE_NO,sEvent=msg,ixPersonAssignedTo=assignTo)
except FogBugzAPIError as e:
juche.error("Unexpected condition [%s] Is case closed? Attempting to recover..." % e)
response = self.fbConnection.reopen(ixBug=CASE_NO,sEvent=msg,ixPersonAssignedTo=assignTo)
juche.info("Recovery was successful.")
def createTestCase(self,PARENT_CASE,estimate="0 hours",ixTester=None):
if not ixTester:
ixTester = self.ixPerson
#extract parent info
resp = self.fbConnection.search(q=PARENT_CASE,cols="ixProject,ixArea,ixFixFor,sFixFor,ixPriority")
#look for a test milestone
milestones = self.fbConnection.listFixFors(ixProject=resp.case.ixproject.contents[0])
ixTestMilestone = 0
foundTestMilestone = False
for aMilestone in milestones.fixfors:
#print aMilestone.sfixfor.contents[0], resp.case.sfixfor.contents[0] + "-test"
if(aMilestone.sfixfor.contents[0].find(resp.case.sfixfor.contents[0] + "-test") != -1):
foundTestMilestone = True
ixTestMilestone = aMilestone.ixfixfor.contents[0]
testMilestone = resp.case.sfixfor.contents[0] + "-test"
#print "testMilestone: ", testMilestone
#print "\nfoundTestMilestone: ", foundTestMilestone
if not foundTestMilestone:
ixTestMilestone = self.fbConnection.newFixFor(ixProject=resp.case.ixproject.contents[0],sFixFor=testMilestone, fAssignable="1")
self.fbConnection.addFixForDependency(ixFixFor=ixTestMilestone, ixFixForDependsOn=resp.case.ixproject.contents[0])
#print "creating new milestone and setting dependencies! New Milestone: ", ixTestMilestone.ixfixfor.contents[0]
ixTestMilestone = ixTestMilestone.ixfixfor.contents[0]
#print resp.case
response = self.fbConnection.new(ixBugParent=PARENT_CASE,sTitle="Review",ixPersonAssignedTo=ixTester,hrsCurrEst=estimate,ixPriority=resp.case.ixpriority.contents[0],sEvent="Cake and grief counseling will be available at the conclusion of the test.",ixCategory=6,
ixProject=resp.case.ixproject.contents[0],ixArea=resp.case.ixarea.contents[0],ixFixFor=ixTestMilestone,sTags=TEST_TAG)
juche.info("Created case %s" % response.case['ixbug'])
def create_tests():
from work.fogbugzConnect import FogBugzConnect
from config import project_with_name
f = FogBugzConnect()
#we create test cases if and only if all the following conditions are met:
#1. Bugs and features
#2. Open cases
#3. Cases with an estimate (otherwise, the person assigned might just be a placeholder person...)
#4. Cases that are decided (not Undecided)
#5. Cases that are in projects for which we have review-workflow: yes
cases = f.fbConnection.search(q='(category:"bug" OR category:"feature") status:"open" estimatecurrent:"1m.." -milestone:"Undecided"', cols="sProject")
cache = buildbot_cache_get()
CACHE_KEY = "autoTestMake-cache"
if not cache.has_key(CACHE_KEY):
cache[CACHE_KEY] = []
casesList = map(lambda x: int(x["ixbug"]),cases.cases)
casesList = filter(lambda x: x not in cache[CACHE_KEY],casesList)
juche.info(casesList)
from work.work import autoTestMake
for case in cases.cases.contents:
project = project_with_name(case.sproject.contents[0])
if not project:
juche.info("No project named %s in buildbot.yaml" % case.sproject.contents[0])
continue
if not ("review-workflow" in project.keys() and project["review-workflow"] == "yes"):
continue
result = autoTestMake(int(case["ixbug"]))
if not result: cache[CACHE_KEY].append(case["ixbug"])
buildbot_cache_set(cache)
def setEstimate(self, CASE_NO,timespan=None):
if not timespan:
juche.info("Please provide an estimate for this case: ")
timespan = raw_input()
self.fbConnection.edit(ixBug=CASE_NO, hrsCurrEst=timespan)
return timespan;
def autoTestMake(CASE_NO,fbConnection=None):
juche.info("autotestmake %d" % CASE_NO)
if not fbConnection: fbConnection = FogBugzConnect()
(implement,test) = fbConnection.getCaseTuple(CASE_NO,oldTestCasesOK=True,exceptOnFailure=False)
if not test:
ixTester = fbConnection.optimalIxTester(CASE_NO)
fbConnection.createTestCase(CASE_NO,ixTester=ixTester)
return True
return False
def priority_fix():
from work.fogbugzConnect import FogBugzConnect
f = FogBugzConnect()
for case in f.listTestCases().cases:
ixBug = case["ixbug"]
(parent,child) = f.getCaseTuple(ixBug)
parent_priority = f.fbConnection.search(q=parent,cols="ixPriority").ixpriority.contents[0]
child_priority = f.fbConnection.search(q=child,cols="ixPriority").ixpriority.contents[0]
if parent_priority != child_priority:
juche.info("Fixing priority of case %s to %s" % (child,parent_priority))
f.setPriority(child,parent_priority)
def __checkoutExistingBranchRaw(self,arg):
juche.dictate(double_checkout_existing_branch_raw=arg)
(checkoutNewBranchStatus, output) = self.statusOutput("git checkout {0}".format(arg))
if(checkoutNewBranchStatus):
juche.info(output)
return False
(status,output) = self.statusOutput("git submodule init")
if status:
juche.error("could not init submodule: %s",output)
(status,output) = self.statusOutput("git submodule update")
if status:
juche.error("Error updating a submodule %s",output)
return True
def exec_tests(self,proj):
import subprocess
import shlex
shortdesc = ""
files = {}
passed = True
#WARNING: YOU MUST PATCH $DEVELOPER/Platforms/iPhoneSimulator.platform/Developer/Tools/RunPlatformUnitTests for this to work.
#See http://longweekendmobile.com/2011/04/17/xcode4-running-application-tests-from-the-command-line-in-ios/
if proj["tests"]:
juche.dictate(project=proj)
for test in proj["tests"]:
juche.dictate(running_test=test)
juche.info("running test")
r = subprocess.Popen(test["cmd"],shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE,cwd=WORK_DIR+proj["name"])
(status,output) = self.wait_for(r)
if status and test["type"]!="kif": #kif tests always return a status code of 1
shortdesc += "Failing in part because test %s returned a non-zero return code %d\n" % (test,status)
passed = False
if test["type"]=="xcode":
(passed,shortdesc,files) = self.parse_xcodelike_response(passed,shortdesc,files,output,test["name"]+".log")
elif test["type"]=="python":
(passed,shortdesc,files) = self.parse_python_response(passed,shortdesc,files,output,test["name"]+".log",status)
elif test["type"]=="kif":
(passed,shortdesc,files) = self.parse_kif_response(passed,shortdesc,files,output,test["name"]+".log")
else:
raise Exception("Unknown test type.")
if passed:
if "commit-files" in test:
upload_files = test["commit-files"]
juche.dictate(upload_files=upload_files)
juche.info("commit-files")
git = GitConnect(wd=WORK_DIR+proj["name"])
git.add(upload_files)
try:
git.commit("If the subject survived the test, we let them purchase the pictures for $5. If the subject died, we gave the photo to their next of kin free of charge")
git.pushChangesToOriginBranch()
except Exception as e:
juche.exception(e)
juche.warn("Was not able to upload the files successfully. Perhaps they have not changed?")
return (passed,shortdesc,files)
def _fixFors_to_EBS_dates(abbreviatedTest=False):
juche.dictate(fixing_dates_per_ebs=1)
fbConnection = FogBugzConnect()
fixfors = fbConnection.dependencyOrder(fbConnection.listFixFors())
if abbreviatedTest:
fixfors = fixfors[:5]
for item in fixfors:
name = fbConnection.nameForFixFor(fbConnection.fixForDetail(item))
if name.startswith("Never"): continue
juche.info("processing %s %s" % (name,item))
date = fbConnection.getShipDate(item)
from dateutil.parser import parse
if not abbreviatedTest:
fbConnection.editFixForShipDate(item,parse(date))
#It's bad to leave EBS in a partially-edited state. Therefore we simply log the output and don't actually write any changes when in abbreviated test mode.
else:
juche.warning("Not editing the ship date in %s to %s because this is an abbreviated test." % (item,date))
def mergeIn(self,BRANCH_NAME,pretend=False):
if pretend:
return self.__mergeInPretend(BRANCH_NAME)
juche.dictate(merging_in=BRANCH_NAME)
(status,output) = self.statusOutput("git merge --no-ff %s" % BRANCH_NAME)
juche.info(output)
if status:
juche.error("merge was unsuccessful.")
# play sounds!
self.statusOutput ("afplay -v 7 %s/media/ohno.aiff" % sys.prefix)
raise Exception("stacktraceplease")
else:
# play sounds!
from work import get_setting_dict
if get_setting_dict().has_key("disablesounds") and get_setting_dict()["disablesounds"]=="YES":
pass
else:
self.statusOutput ("afplay -v 7 %s/media/hooray.aiff" % sys.prefix)
juche.info("Use 'git push' to ship.")
def editFixForShipDate(self,ixFixFor,shipDate,depCheck=True):
if depCheck:
override = False
deptree = self._deptree(self.listFixFors())
from dateutil.parser import parse
import datetime
for dep in deptree[ixFixFor]:
if dep not in deptree.keys(): continue
depdetail = self.fixForDetail(dep)
depdate = parse(depdetail.dt.contents[0])
if depdate > shipDate:
override = True
shipDate = depdate + datetime.timedelta(hours=1)
if override:
juche.info("Ship date overridden to %s" % shipDate)
detail = self.fixForDetail(ixFixFor)
name = self.nameForFixFor(detail)
juche.info("editing ship date of %s" % name)
self.fbConnection.editFixFor(ixFixFor=ixFixFor,sFixFor=name,dtRelease=shipDate,dtStart=self.getFixForStartDate(detail),fAssignable=self.getFixForDeleted(detail) and "0" or "1")
def integrate_changed(self,gitConnection,integration_branch,sProject):
juche.dictate(integrate_changed_hook=integration_branch)
gitConnection.checkoutExistingBranchRaw(integration_branch)
self.deploy(gitConnection,integration_branch,project_with_name(sProject))
#Here's the problem: A and B are in the review queue. They are mutually exclusive (can't merge them both in.)
#You pass patch A. It's merged in. We need to re-test B. Perhaps the merge will fail now, or perhaps the unit test will fail.
#We "invalidate" (put back in the glados queue; snatch from the reviewer) all cases in the review queue and they will be re-tested on the next pass.
cases = self.f.fbConnection.search(q="project:'%s' status:'Resolved' (category:'Bug' OR category:'Feature')" % (sProject),cols="sStatus")
for case in cases.cases:
caseno = int(case["ixbug"])
if self.f.getIntegrationBranch(caseno)!=integration_branch: continue
if not self.f.isReadyForTest(caseno): continue
juche.info( "Invalidating "+case["ixbug"])
self.f.fbConnection.assign(ixBug=case["ixbug"],ixPersonAssignedTo=self.f.ixPerson,sEvent="Invalidation. Are you getting unexpected e-mails surrounding this case event? File a bug against buildbot.")
self.test_active_tickets()
def pull(self):
self.checkForRepository()
import ConfigParser
c = ConfigParser.ConfigParser()
if self.wd:
path = self.wd + "/.git/config"
else:
path =".git/config"
import os
for i in range(0,30):
if os.path.exists(path): break
path = "../" + path
if i==30:
raise Exception("Not a git repository?")
file = open(path)
str = file.read()
file.close()
juche.dictate(pull=1)
if self.getBranch() not in str:
juche.warn( "%s is not a tracking branch. Attempting to fix..." % self.getBranch())
try:
self.setUpstream(self.getBranch(),"remotes/origin/{0}".format(self.getBranch()))
juche.info("Success!")
except:
juche.error("DID NOT AUTOMATICALLY FIX BRANCH UPSTREAM / TRACKING. PLEASE FILE A BUG.")
(status,output) = self.statusOutput("git pull origin %s" % self.getBranch())
if status:
juche.error("Cannot pull! %s" % output)
else:
(status,output) = self.statusOutput("git pull")
if status:
juche.error("ERROR: Cannot pull! %s" % output)
raise Exception("stacktraceplease")
self.submoduleUpdate()
juche.info("Success!")
def recharge(fr,to):
import dateutil.parser
fbConnection = FogBugzConnect()
fbConnection.setParentIfUnset(fr,to)
#cannot create a time record for a closed case...
mustOpen = not fbConnection.isOpen(to)
if mustOpen:
fbConnection.reopen(to,"work.py recharge")
results = fbConnection.listTimeRecords(fr)
time_interval = 0
my_records = []
for record in results:
#print record
if record.fdeleted.contents[0]!="false":
juche.warn("Skipping deleted record %s" % record)
continue
if len(record.dtend)==0:
juche.warn("Skipping open time record %s" % record)
continue
my_records.append(record)
r = 0
for record in my_records:
juche.info("%d: %s-%s" % (r,record.dtstart.contents[0],record.dtend.contents[0]))
r += 1
def parse_range(astr): # http://stackoverflow.com/questions/4726168/parsing-command-line-input-for-numbers
result=set()
for part in astr.split(','):
x=part.split('-')
result.update(range(int(x[0]),int(x[-1])+1))
return sorted(result)
strl = raw_input("records (syntax like 22-27,51-64): ")
results = parse_range(strl)
for result in results:
record = my_records[result]
record_desc = "From %s to %s ixPerson %s ixBug %s" % (record.dtstart.contents[0],record.dtend.contents[0],record.ixperson.contents[0],record.ixbug.contents[0])
from_time = dateutil.parser.parse(record.dtstart.contents[0])
to_time = dateutil.parser.parse(record.dtend.contents[0])
time_interval += (to_time-from_time).total_seconds()
juche.info("from_time %s to_time %s time_interval %s" % (from_time,to_time,time_interval))
fbConnection.commentOn(fr,"recharge: A record was removed from this ticket: %s, see case %d" % (record_desc,to))
fbConnection.commentOn(to,"recharge: A record was added to this ticket: %s, see case %d" % (record_desc, fr))
fbConnection.createTimeRecord(to,str(record.dtstart.contents[0]),str(record.dtend.contents[0]))
oldEst = fbConnection.getEstimate(fr) * 60.0 * 60.0
newEst = (oldEst - time_interval) / 60.0 / 60.0
if newEst <= 0: newEst = 1/60.0
juche.info("Setting estimate to",newEst)
fbConnection.setEstimate(fr,timespan="%f hours" % newEst)
#fbConnection.deleteTimeRecord(record.ixinterval.contents[0])
if mustOpen: fbConnection.closeCase(to)
def complain(ixComplainAboutPerson,sComplainAboutProject):
fbConnection = FogBugzConnect()
response = fbConnection.fbConnection.search(q="status:active assignedto:=%d project:%s" % (ixComplainAboutPerson,sComplainAboutProject),cols="hrsCurrEst,hrsElapsed,sPersonAssignedTo,sFixFor,sCategory")
for case in response.cases:
#print case
if case.hrscurrest.contents[0]=="0":
juche.info("%s's case %s has no estimate" % (case.spersonassignedto.contents[0], case["ixbug"]))
fbConnection.commentOn(case["ixbug"],"This next test could take a very, VERY long time.")
if case.sfixfor.contents[0]=="Undecided" and (case.scategory.contents[0] == "bug" or case.scategory.contents[0] == "feature"):
juche.info("%s needs a milestone" % case["ixbug"])
fbConnection.commentOn(case["ixbug"],"If you choose not to decide, you still have made a choice. (Don't think about it, don't think about it...) It's a paradox! There IS no answer.")
est = float(case.hrscurrest.contents[0])
act = float(case.hrselapsed.contents[0])
if est - act < 0:
juche.info("%s's case %s requires updated estimate" % (case.spersonassignedto.contents[0], case["ixbug"]))
fbConnection.commentOn(case["ixbug"],"I'll give you credit: I guess you ARE listening to me. But for the record: You don't have to go THAT slowly.")
def _fixFors_test_quickly_dates(abbreviatedTest=False):
juche.dictate(fixing_test_milestones=1)
fbConnection = FogBugzConnect()
fixfors_raw = fbConnection.listFixFors()
fixfors = fbConnection.dependencyOrder(fixfors_raw)
from dateutil.parser import parse
import datetime
if abbreviatedTest:
fixfors = fixfors[:5]
for testMilestone in fixfors:
testMilestone_raw = fbConnection.fixForDetail(testMilestone)
testName = fbConnection.nameForFixFor(testMilestone_raw)
if testName.startswith("Never"): continue
juche.dictate(testMilestone=testMilestone,testName=testName)
if not testName.endswith("-test"): continue
if testName=="Undecided-test": continue
matched = False
if testMilestone_raw.ixproject.contents==[]: continue
for item in fbConnection.listFixFors(ixProject=int(testMilestone_raw.ixproject.contents[0])):
#print testName[:-5],fbConnection.nameForFixFor(item)
if item.sfixfor.contents[0]==testName[:-5]:
#print "matching",testName,fbConnection.nameForFixFor(item)
matched = True
break
if not matched:
juche.info(testMilestone_raw)
raise Exception("Cannot match "+testName)
if item.dt.contents==[]:
juche.info("Can't set %s because the non-test milestone has no completion date." % testName)
continue
date = item.dt.contents[0]
newDate = parse(date)+datetime.timedelta(hours=6) #turns out that using 1 day produces weird results. If the next implementation milestone is completed within 24 hours, lots of weird things can happen
juche.info("setting %s to %s"% (testName,newDate))
if not abbreviatedTest:
fbConnection.editFixForShipDate(testMilestone,newDate)
#It's bad to leave EBS in a partially-edited state. Therefore we simply log the output and don't actually write any changes when in abbreviated test mode.
else:
juche.warn("Not editing the ship date in %s to %s because this is an abbreviated test." % (testMilestone,newDate))
def test_workingschedule(self):
import datetime
juche.info("Drew works %f hours" % self.f.expectedWorkHours(ixPerson=2,date=datetime.datetime.now()))
def test_admin(self):
juche.info("I am an administrator: %s" % self.f.amIAdministrator())
def test_getship(self):
juche.info(self.f.getShipDate(ixFixFor=43))
def test_deptree(self):
juche.info(self.f.dependencyOrder(self.f.listFixFors()))
def test_lastactive(self):
juche.info(self.f.userLastActive(self.f.ixPerson))
def test_optimaltester(self):
juche.info(self.f.optimalIxTester(3028))
def test_listfixfors(self):
semaps_fixfors = self.f.listFixFors(sProject="semaps")
juche.info(semaps_fixfors)
self.assertTrue(len(semaps_fixfors) > 0)
def test_annoyables(self):
if not self.f.amIAdministrator():
juche.warning("NOT RUNNING test_annoyables BECAUSE YOU ARE NOT AN ADMINISTRATOR")
return
juche.info(self.f.annoyableIxPeople())
def still_alive():
juche.info("still alive")