def backup_database(url, username, password, database, destination):
if not url.endswith("/"):
url+="/"
br = Browser()
br.open(url)
br.select_form(name="login_form")
br["pma_username"] = username
br["pma_password"] = password
login_response = br.submit()
resp = br.follow_link(url_regex=re.compile("^main\.php.*"))
resp = br.follow_link(url_regex=re.compile("\./server_export\.php.*"))
br.select_form(name="dump")
# Select SQL export
br.find_control(name="what").get(id="radio_plugin_sql").selected = True
# Select database to export
br.find_control(name="db_select[]").get(name=database).selected = True
# Add SQL DROP statements to export
br.find_control(name="sql_drop").get(id="checkbox_sql_drop").selected = True
# Send as file
br.find_control(name="asfile").get("sendit").selected = True
# Compress file with bzip
br.find_control(name="compression").get(id="radio_compression_bzip").selected = True
ret = br.submit()
open(destination, 'w').write(ret.read())
def do_processing(street1, street2, locationdetail, email, phone):
br = Browser()
#br.set_handle_robots(False)
#br.set_handle_equiv(False)
#br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
br.open("https://secure.toronto.ca/webwizard/html/litter_bin_overflow.htm")
br.select_form(nr=0)
for i in range(0, len(br.find_control(type="checkbox").items)):
br.find_control(type="checkbox").items[i].selected = True
#checkbox_select(br)
br.submit()
#br.select_form(nr=0)
#for i in range(0, len(br.find_control(type="checkbox").items)):
# br.find_control(type="checkbox").items[i].selected =True
checkbox_select(br)
br.submit()
#br.select_form(nr=0)
#for i in range(0, len(br.find_control(type="checkbox").items)):
# br.find_control(type="checkbox").items[i].selected =True
checkbox_select(br)
br["probCrossStreet1"] = street1
br["probCrossStreet2"] = street2
br["probLocationDetails"] = locationdetail
br["ctctEmail"] = email
br["ctctPhoneNumb"] = str(phone)
br.submit()
br.select_form(nr=0)
br["additional_information"] = "Reported by the waterfront BIA"
br.submit()
br.select_form(nr=0)
response = br.submit()
return (response.read())
def download():
b=Browser()
b.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
r1=b.open(URL)
b.select_form(nr=0)
b.set_all_readonly(False)
b["__EVENTTARGET"] = "ctl00$CPH1$dgMain"
b["__EVENTARGUMENT"] = "Page$2"
b.find_control("ctl00$CPH1$tbSearch").disabled=True
r2=b.submit()
return [r1,r2]
def test_login(login_url, studentid, username, password):
#create a browser object
br = Browser()
#br.set_handle_redirect(False)
br.set_handle_robots(False)
br.addheaders = [(
'User-agent', 'Mozilla/5.0 (Windows NT 5.2; '
'WOW64) AppleWebKit/536.11 (KHTML, like Gecko) '
' Chrome/20.0.1132.47 Safari/536.11')]
#open the login page
br.open(login_url)
br.select_form(nr=0)
#read only? try me.
br.find_control("credentialsForm2_hf_0").readonly = False
#whoa. via http://stackoverflow.com/a/1549036/561698
br.form.new_control('text', 'username', {'value': ''})
br.form.new_control('text', 'password', {'value': ''})
br.form.fixup()
br['username'] = username
br['password'] = password
outcome = None
while True:
try:
br.submit()
#print br.title()
if br.title() == 'Fastt Math NG Stretch':
outcome = 'PASSED'
else:
outcome = 'FAILED (no error)'
break
except HTTPError:
#print "Got error code", e.code
outcome = 'FAILED' + ' (u:' + username + '/p: ' + password + ')'
break
#return a dictionary with test results
resp_dict = {
'product': 'FASTT Math',
'studentid': studentid,
'tested_on': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'),
'outcome': outcome
}
return resp_dict
def yapistir():
br = Browser()
br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0')]
br.set_handle_robots(False)
br.open("http://paste.ubuntu.com")
br.select_form("pasteform")
br['poster'] = coder
br.find_control(name="syntax").value = ["python"]
dosya_ac = open(dosya)
kodlar = dosya_ac.read()
br['content'] = kodlar
br.submit()
for link in br.links():
k_link.append(link.url)
def yapistir():
br = Browser()
br.addheaders = [(
'User-agent',
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0'
)]
br.set_handle_robots(False)
br.open("http://paste.ubuntu.com")
br.select_form("pasteform")
br['poster'] = coder
br.find_control(name="syntax").value = ["python"]
dosya_ac = open(dosya)
kodlar = dosya_ac.read()
br['content'] = kodlar
br.submit()
for link in br.links():
k_link.append(link.url)
'Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1')
]
# define target URL
url = "http://www.bseindia.com/getquote.htm"
br = Browser()
# browser parameters
br.set_handle_equiv(True)
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
br.addheaders = headers
# make request
main_page = br.open(url)
# select the default form
br.select_form(nr=0)
br.find_control(id="suggestBoxEQ").value = "CAREERP"
# submit form
br.submit()
page_response = bs(br.response().read())
quote = page_response.find_all('div', {'class': 'newsrheadingdiv'})
print quote
def getSolutions (path_prefix, path_proxy):
global br, username, password
# create a browser object
br = Browser()
# add proxy support to browser
if len(path_proxy) != 0:
protocol,proxy = options.proxy.split("://")
br.set_proxies({protocol:proxy})
# let browser fool robots.txt
br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; \
rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
br.set_handle_robots(False)
print "Enter yout SPOJ username :"******"Authenticating " + username
br.open ("http://spoj.pl")
br.select_form (name="login")
br["login_user"] = username
br["password"] = password
# sign in for a day to avoid timeouts
br.find_control(name="autologin").items[0].selected = True
br.form.action = "http://www.spoj.pl"
response = br.submit()
verify = response.read()
if (verify.find("Authentication failed!") != -1):
print "Error authenticating - " + username
exit(0)
# grab the signed submissions list
print "Grabbing siglist for " + username
siglist = br.open("http://www.spoj.pl/status/" + username + "/signedlist")
# dump first nine useless lines in signed list for formatting
for i in xrange(9):
siglist.readline()
# make a list of all AC's and challenges
print "Filtering siglist for AC/Challenge solutions..."
mysublist = list()
while True:
temp = siglist.readline()
if temp=='\------------------------------------------------------------------------------/\n':
# reached end of siglist
break
if not len(temp) :
print "Reached EOF, siglist format has probably changed," + \
" contact author."
exit(1)
entry = [x.strip() for x in temp.split('|')]
if entry[4] == 'AC' or entry[4].isdigit():
mysublist.append (entry)
print "Done !!!"
return mysublist
class CoreEmulator(Emulator):
def __init__(self, username, password):
super(CoreEmulator, self).__init__(username, password)
self.setup_emulator()
def setup_emulator(self):
self.browser = Browser()
self.browser.set_handle_robots(False)
self.browser.addheaders = moodle.USER_AGENT
self.cookiejar = CookieJar()
self.browser.set_cookiejar(self.cookiejar)
def session_expired(self):
return self.browser.geturl().endswith(moodle.LOGIN_LOCATION)
@throws_moodlefuse_error(exception.LoginException)
def login(self):
self.open_login_page(self.browser.open)
self.browser.select_form(
predicate=lambda form: form.attrs.get('id') == attributes.LOGIN
)
self.browser.form.set_value(self.username, name='username')
self.browser.form.set_value(self.password, name='password')
resp = self.browser.submit()
if resp.geturl().endswith(moodle.LOGIN_LOCATION):
raise Exception
@throws_moodlefuse_error(resource_errors.UnableToDownloadResource)
def download(self, destination, source):
source = str(source)
if not source.startswith('http://') and not source.startswith('file://'):
source = config['TEST_DATA'] + '/' + source
self.browser.retrieve(source, destination)
def open_link(self, url):
response = self.browser.open(url)
return BeautifulSoup(response.read())
def check_form_checkbox(self, checkboxname):
self.browser.find_control(checkboxname).items[0].selected = True
def uncheck_form_checkbox(self, checkboxname):
self.browser.find_control(checkboxname).items[0].selected = False
def add_form_content(self, inputname, content):
self.browser.form.set_value(content, name=inputname)
def close_form(self):
self.browser.submit()
def set_form_to_first_form(self):
self.browser.select_form(nr=0)
def set_form_to_form_with_control_value(self, value):
for form in self.browser.forms():
for control in form.controls:
if control.value == value:
self.browser.form = form
@throws_moodlefuse_error(exception.UnableToToggleEditing)
def turn_course_editing_on(self):
self.set_form_to_form_with_control_value(moodle.EDIT_ON_MOODLE_BUTTON_TEXT)
response = self.browser.submit()
return BeautifulSoup(response.read())
def _setup_assignments_for_parsing(self, submission_filter):
self.set_form_to_form_with_control_value('Save and update table')
self.browser.form["filter"] = [submission_filter]
self.browser.form["perpage"] = ["100"]
self.uncheck_form_checkbox('quickgrading')
response = self.browser.submit()
return BeautifulSoup(response.read())
def filter_assignment_submissions(self):
return self._setup_assignments_for_parsing("submitted")
def unfilter_assignment_submissions(self):
return self._setup_assignments_for_parsing("")
@throws_moodlefuse_error(exception.UnableToToggleEditing)
def turn_course_editing_off(self):
self.set_form_to_form_with_control_value(moodle.EDIT_OFF_MOODLE_BUTTON_TEXT)
response = self.browser.submit()
return BeautifulSoup(response.read())
@throws_moodlefuse_error(course_errors.InvalidMoodleIndex)
def get_courses(self):
return self.open_link(config['MOODLE_INDEX_ADDRESS'])
@throws_moodlefuse_error(course_errors.UnableToObtainCategoryList)
def get_course_categories(self, url):
return self.open_link(url)
@throws_moodlefuse_error(resource_errors.UnableToObtainResourceList)
def get_course_resource_names(self, url):
return self.open_link(url)
def close(self):
self.browser.close()
br.form.find_control("dummy3").readonly=False #så att vi kan manipulera värdena
br.form.find_control("dummy2").readonly=False
br.form.find_control("dummy1").readonly=False
br.form["dummy1"]="3"
br.form["dummy2"]="filter"
br.form["dummy3"]= "akt" #detta är för att bara få aktiva AB. De övriga två kontrollerna vet jag inte exakt vad de gör...
print br.form
print "nu är vi här"
response7 = br.submit() # 26 april: skumt, detta fungerar om jag inte tidigare preciserat vilket län jag vill söka i. Med länet utvalt får jag däremot noll träffar. Torde ha något att göra med vad br.form har för värden på sina olika attribut, men vad?
br.select_form(name="mainform")
dagens_org_nr = str(br.find_control("uss")) #här går vi "bakvägen" genom att extrahera strängen som innehåller samtliga org-nr för bolag med kunggörelser i dag.
#print br.form
dagens_org_nr=dagens_org_nr[30:-15] #en enda lång sträng med de orgnr som har kunggörelser i dag. Denna sträng stoppar vi tillbaka i formuläret lite längre ner.
print "dagens org_nr:"+dagens_org_nr
print "längd dagens org nr:" + str(len(dagens_org_nr))
print br
#print response7.read()
response8 = br.follow_link(text="F\xf6retag")
#print br
br.select_form(name="mainform")
print "mainform, rad 129"
print br.form
#br.form.find_control("limit").readonly = False #sätt till False så att vi kan ändra värdet
br.open(url)
br.select_form(nr=0)
br["login"] = args.username
br["password"] = args.password
if args.debug: print("Logging in . . .")
br.submit()
writeout("past_login", br)
if args.debug: print("Getting search page . . .")
br.open("https://hrsa.cunyfirst.cuny.edu/psc/cnyhcprd/EMPLOYEE/HRMS/c/SA_LEARNER_SERVICES.CLASS_SEARCH.GBL")
writeout("on_search_form", br)
br.select_form(nr=0)
# How do I change this next semester?
mods = br.find_control('CLASS_SRCH_WRK2_STRM$45$')
item1 = Item(mods, {"contents": "1162", "value": "1162"})
item1.selected=True
mods = br.find_control('SSR_CLSRCH_WRK_SUBJECT_SRCH$0')
item2 = Item(mods, {"contents": args.subject, "value": args.subject})
item2.selected=True
# Next semester?
br['CLASS_SRCH_WRK2_STRM$45$']=['1162']
br['DERIVED_SSTSNAV_SSTS_MAIN_GOTO$155$']=['9999']
br['CLASS_SRCH_WRK2_INSTITUTION$42$']=[args.college + '01']
br['SSR_CLSRCH_WRK_SSR_EXACT_MATCH1$1']=['E']
br.form['SSR_CLSRCH_WRK_CATALOG_NBR$1']=args.klass
br['SSR_CLSRCH_WRK_SUBJECT_SRCH$0']=[args.subject]
writeout("without_ICAction", br)
class RequestQuery:
def __init__(self,config):
self.br=Browser()
self.config = config
self.isLoggedIn = self.login2Savannah()
def __del__(self):
self.br.close()
def closeRequest(self,task,msg):
if self.isLoggedIn:
self.createValueDicts()
response = self.br.open('https://savannah.cern.ch/task/?'+str(task))
html = response.read()
self.br.select_form(name="item_form")
control = self.br.find_control("status_id",type="select")
control.value = [self.TicketStatusByLabelDict["Closed"]]
#Put reason to the comment field
control = self.br.find_control("comment",type="textarea")
control.value = msg
#DBS Drop Down is a mandatory field, if set to None (for old requests), it is not possible to close the request
self.setDBSDropDown()
self.br.submit()
#remove JSON ticket
self.removeJSONFile(task)
return
def createValueDicts(self):
if self.isLoggedIn:
self.br.select_form(name="bug_form")
control = self.br.find_control("custom_sb2",type="select")
self.ReleaseByValueDict = self.getLabelByValueDict(control)
control = self.br.find_control("custom_sb3",type="select")
self.GroupByValueDict = self.getLabelByValueDict(control)
control = self.br.find_control("custom_sb4",type="select")
self.DBSByValueDict = self.getLabelByValueDict(control)
self.DBSByLabelDict = self.getValueByLabelDict(control)
control = self.br.find_control("resolution_id",type="select")
self.StatusByValueDict = self.getLabelByValueDict(control)
control = self.br.find_control("status_id",type="select")
self.TicketStatusByLabelDict = self.getValueByLabelDict(control)
return
def setDBSDropDown(self):
## Get DBS URL by Drop Down
control = self.br.find_control("custom_sb4",type="select")
dbs_url = self.DBSByValueDict[control.value[0]]
## Get DBS URL by text field (for old entries)
if dbs_url=='None':
tmp = self.br.find_control("custom_tf4",type="text")
dbs_url = tmp.value.replace(' ','')
if dbs_url.find("analysis_02")!=-1:
control.value = [self.DBSByLabelDict["cms_dbs_ph_analysis_02"]]
elif dbs_url.find("analysis_01")!=-1:
control.value = [self.DBSByLabelDict["cms_dbs_ph_analysis_01"]]
elif dbs_url.find("local_09")!=-1:
control.value = [self.DBSByLabelDict["cms_dbs_ph_prod_local_09"]]
else:
msg = 'DBS URL of the old request is neither analysis_01, analysis_02 nor local_09. Please, check!'
logging.error(msg)
raise RuntimeError, msg
return
def getLabelByValueDict(self, control):
d = {}
for item in control.items:
value = item.attrs['value']
label = item.attrs['label']
d[value] = label
return d
def getRequests(self,**kargs):
requests = []
if self.isLoggedIn:
self.selectQueryForm(**kargs)
self.createValueDicts()
self.br.select_form(name="bug_form")
response = self.br.submit()
html_ouput = response.read()
for link in self.br.links(text_regex="#[0-9]+"):
response = self.br.follow_link(link)
## Get Information
self.br.select_form(name="item_form")
## Get input dataset name
control = self.br.find_control("custom_tf1",type="text")
input_dataset = control.value.split('/')
input_primary_dataset = input_dataset[1].replace(' ','')
input_processed_dataset = input_dataset[2].replace(' ','')
## Get DBS URL by Drop Down
control = self.br.find_control("custom_sb4",type="select")
dbs_url = self.DBSByValueDict[control.value[0]]
## Get DBS URL by text field (for old entries)
if dbs_url=='None':
control = self.br.find_control("custom_tf4",type="text")
dbs_url = control.value.replace(' ','')
else: # Transform input value to a valid DBS url
dbs_url = "http://cmsdbsprod.cern.ch/"+dbs_url+"/servlet/DBSServlet"
## Get Release
control = self.br.find_control("custom_sb2",type="select")
release_id = control.value
## Get Physics Group
control = self.br.find_control("custom_sb3",type="select")
group_id = control.value[0]
group_squad = 'cms-storeresults-'+self.GroupByValueDict[group_id].replace("-","_").lower()
## Get Dataset Version
control = self.br.find_control("custom_tf3",type="text")
dataset_version = control.value.replace(' ','')
## Get current status
control = self.br.find_control("resolution_id",type="select")
status_id = control.value
## Get current request status
control = self.br.find_control("status_id",type="select")
request_status_id = control.value
RequestStatusByValueDict = self.getLabelByValueDict(control)
## Get assigned to
control = self.br.find_control("assigned_to",type="select")
AssignedToByValueDict = self.getLabelByValueDict(control)
assignedTo_id = control.value
##Assign task to the physics group squad
if AssignedToByValueDict[assignedTo_id[0]]!=group_squad:
control.value = [self.getValueByLabelDict(control)[group_squad]]
self.br.submit()
## Construction of the new dataset name
## remove leading hypernews or physics group name and StoreResults+Version
if len(dataset_version)>0:
dataset_prefix = "StoreResults-"+dataset_version
else:
dataset_prefix = "StoreResults"
if input_processed_dataset.find(self.GroupByValueDict[group_id])==0:
new_dataset = input_processed_dataset.replace(self.GroupByValueDict[group_id],dataset_prefix,1)
else:
stripped_dataset = input_processed_dataset.split("-")[1:]
new_dataset = dataset_prefix+'-'+'-'.join(stripped_dataset)
self.br.back()
## remove leading   and # from task
task = link.text.replace('#','').decode('utf-8').strip()
infoDict = {}
infoDict["primaryDataset"] = input_primary_dataset
infoDict["processedDataset"] = input_processed_dataset
infoDict["outputDataset"] = new_dataset
infoDict["physicsGroup"] = self.GroupByValueDict[group_id]
infoDict["inputDBSURL"] = dbs_url
# close the request if deprecated release was used
try:
infoDict["cmsswRelease"] = self.ReleaseByValueDict[release_id[0]]
except:
if len(self.ReleaseByValueDict)>0 and RequestStatusByValueDict[request_status_id[0]] != "Closed":
msg = "Your request is not valid anymore, since the given CMSSW release is deprecated. If your request should be still processed, please reopen the request and update the CMSSW release to a more recent *working* release.\n"
msg+= "\n"
msg+= "Thanks,\n"
msg+= "Your StoreResults team"
self.closeRequest(task,msg)
#Fill json file, if status is done
if self.StatusByValueDict[status_id[0]]=='Done' and RequestStatusByValueDict[request_status_id[0]] != "Closed":
self.writeJSONFile(task, infoDict)
infoDict["task"] = int(task)
infoDict["ticketStatus"] = self.StatusByValueDict[status_id[0]]
infoDict["assignedTo"] = AssignedToByValueDict[assignedTo_id[0]]
if infoDict["ticketStatus"] == "Done" and RequestStatusByValueDict[request_status_id[0]] == "Closed":
infoDict["ticketStatus"] = "Closed"
requests.append(infoDict)
return requests
def getValueByLabelDict(self, control):
d = {}
for item in control.items:
value = item.attrs['value']
label = item.attrs['label']
d[label] = value
return d
def login2Savannah(self):
login_page='https://savannah.cern.ch/account/login.php?uri=%2F'
savannah_page='https://savannah.cern.ch/task/?group=cms-storeresults'
self.br.open(login_page)
## 'Search' form is form 0
## login form is form 1
self.br.select_form(nr=1)
username = self.config["SavannahUser"]
self.br['form_loginname']=username
self.br['form_pw']=self.config["SavannahPasswd"]
self.br.submit()
response = self.br.open(savannah_page)
# Check to see if login was successful
if not re.search('Logged in as ' + username, response.read()):
logging.error('login unsuccessful, please check your username and password')
return False
else:
return True
def selectQueryForm(self,**kargs):
if self.isLoggedIn:
self.br.select_form(name="bug_form")
## Use right query form labelled Test
control = self.br.find_control("report_id",type="select")
for item in control.items:
if item.attrs['label'] == "Test":
control.value = [item.attrs['value']]
##select number of entries displayed per page
control = self.br.find_control("chunksz",type="text")
control.value = "150"
##check additional searching parameter
for arg in kargs:
if arg == "approval_status":
control = self.br.find_control("resolution_id",type="select")
for item in control.items:
if item.attrs['label'] == kargs[arg].strip():
control.value = [item.attrs['value']]
elif arg == "task_status":
control = self.br.find_control("status_id",type="select")
for item in control.items:
if item.attrs['label'] == kargs[arg].strip():
control.value = [item.attrs['value']]
elif arg == "team":
control = self.br.find_control("custom_sb5",type="select")
for item in control.items:
if item.attrs['label'] == kargs[arg].strip():
control.value = [item.attrs['value']]
response = self.br.submit()
response.read()
return
def removeJSONFile(self,task):
filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json'
if os.access(filename,os.F_OK):
os.remove(filename)
return
def writeJSONFile(self, task, infoDict):
##check if file already exists
filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json'
if not os.access(filename,os.F_OK):
jsonfile = open(filename,'w')
jsonfile.write(json.dumps(infoDict,sort_keys=True, indent=4))
jsonfile.close
return
class Bugzilla:
def __init__(self, dryrun=False):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
# This could eventually be a text file
reviewer_usernames_to_full_names = {
"abarth": "Adam Barth",
"adele": "Adele Peterson",
"aroben": "Adam Roben",
"ap": "Alexey Proskuryakov",
"ariya.hidayat": "Ariya Hidayat",
"barraclough": "Gavin Barraclough",
"beidson": "Brady Eidson",
"darin": "Darin Adler",
"ddkilzer": "David Kilzer",
"dglazkov": "Dimitri Glazkov",
"eric": "Eric Seidel",
"fishd": "Darin Fisher",
"gns": "Gustavo Noronha",
"hausmann": "Simon Hausmann",
"hyatt": "David Hyatt",
"jmalonzo": "Jan Alonzo",
"justin.garcia": "Justin Garcia",
"kevino": "Kevin Ollivier",
"koivisto": "Antti Koivisto",
"levin": "David Levin",
"mitz": "Dan Bernstein",
"mjs": "Maciej Stachowiak",
"mrowe": "Mark Rowe",
"oliver": "Oliver Hunt",
"sam": "Sam Weinig",
"simon.fraser": "Simon Fraser",
"staikos": "George Staikos",
"timothy": "Timothy Hatcher",
"treat": "Adam Treat",
"vestbo": u'Tor Arne Vestb\xf8',
"xan.lopez": "Xan Lopez",
"zecke": "Holger Freyther",
"zimmermann": "Nikolas Zimmermann",
}
def full_name_from_bugzilla_name(self, bugzilla_name):
if not bugzilla_name in self.reviewer_usernames_to_full_names:
raise Exception("ERROR: Unknown reviewer! " + bugzilla_name)
return self.reviewer_usernames_to_full_names[bugzilla_name]
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id,
content_type)
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=" + action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id, action_param)
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: " + bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll('attachment'):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete')
and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch')
and element['ispatch'] == "1")
attachment['id'] = str(element.find('attachid').string)
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['type'] = str(element.find('type').string)
review_flag = element.find('flag', attrs={"name": "review"})
if review_flag and review_flag['status'] == '+':
reviewer_email = review_flag['setter']
# We could lookup the full email address instead once we update full_name_from_bugzilla_name
bugzilla_name = reviewer_email.split('@')[0]
attachment['reviewer'] = self.full_name_from_bugzilla_name(
bugzilla_name)
attachments.append(attachment)
return attachments
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment['is_patch'] and not attachment['is_obsolete']:
patches.append(attachment)
return patches
def fetch_reviewed_patches_from_bug(self, bug_id):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if 'reviewer' in attachment and not attachment['is_obsolete']:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_bug_ids_from_commit_queue(self):
# FIXME: We should have an option for restricting the search by email. Example:
# unassigned_only = "&emailassigned_to1=1&emailtype1=substring&email1=unassigned"
commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
log("Loading commit queue")
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup(
'td', "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(
bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_reviewed_patches_from_bug(bug_id)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
error("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self,
bug_id,
patch_file_object,
description,
comment_text=None,
mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_server_url +
"attachment.cgi?action=enter&bugid=" + bug_id)
self.browser.select_form(name="entryform")
self.browser['description'] = description
self.browser['ispatch'] = ("1", )
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['flag_type-1'] = ('?', ) if mark_for_review else ('X', )
self.browser.add_file(patch_file_object, "text/plain",
"bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self.browser.find_control(type='select', nr=0).value = ("X", )
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['comment'] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
cmssw2rank = {}
rank = 10
for release in release_list:
cmssw2rank[release] = rank
rank += 10
## Loop overall releases in Savannah and check if there are valid
## Hide deprecated versions
for links in br.links(text_regex="CMSSW_*_*"):
if links.text not in release_list:
br.follow_link(links)
print "Hide deprecated version %s" % links.text
br.select_form(nr=1)
control = br.find_control("status")
control.value = ['H']
br.submit()
## Doing a loop in all releases
## Create entry in Savannah for releases. If it already exist, update the rank
for release in release_list:
try:
#First, look if this CMSSW release is already there
br.follow_link(text=release, nr=0)
rank = cmssw2rank[release]
print("Release %s already exists. Changing rank to %s" %(release,rank))
br.select_form(nr=1)
br['order_id']= str(rank)
br['status']=['A']
br.submit()
br.set_handle_robots(False)
br.open('https://m.facebook.com/login.php?next=https://m.facebook.com/settings/security/?active_sessions')
br._factory.is_html = True
br.select_form(nr=0)
br.form["email"] = USER
br.form["pass"] = PASSWD
br.submit()
# Infinite loop checking for unauthorized sessions
while True:
time.sleep(TIME)
br.open("https://m.facebook.com/settings/security/?active_sessions")
br._factory.is_html = True
br.select_form(nr=1)
try:
for i in range(0, len(br.find_control(type="checkbox").items)):
br.find_control(type="checkbox").items[i].selected = True
sessions_detected = True
except:
print '[+] No active sessions'
sessions_detected = False
if sessions_detected == True:
print '[+] Active sessions detected. Checking Latch status...'
status = api.status(accountId)
statusData = status.get_data()
try:
if(statusData['operations'][APP_ID]['status'] == 'off'):
print '[!] INTRUDER'
br.submit()
else:
import pandas as pd
a=pd.read_csv('Data/readinPy.csv')
from mechanize import Browser, HTTPError
import time
for i in range(0,636):
br = Browser()
kinase = a.siRNA[i]
print i, kinase #Status Check
br.open('http://www.drugkinet.ca/KinaseCompoundQuery.aspx')
br.select_form(nr=0)
br['ctl00$ContentPlaceHolder1$KinaseSpecTextbox'] = kinase
item = br.find_control(name="ctl00$ContentPlaceHolder1$DataOutputDropdown", type="select").get("")
item.selected = True
try:
br.submit()
except HTTPError:
print 'Server Error', i
continue
br.select_form(nr=0)
try:
br.form.find_control("ctl00$ContentPlaceHolder1$RegenTableButton")
except:
print 'not in database error', i
br["login"] = args.username
br["password"] = args.password
if args.debug:
print("Logging in . . .")
br.submit()
writeout("past_login", br)
if args.debug:
print("Getting search page . . .")
br.open("https://hrsa.cunyfirst.cuny.edu/psc/cnyhcprd/EMPLOYEE/HRMS/c/SA_LEARNER_SERVICES.CLASS_SEARCH.GBL")
writeout("on_search_form", br)
br.select_form(nr=0)
# How do I change this next semester?
mods = br.find_control('CLASS_SRCH_WRK2_STRM$45$')
item1 = Item(mods, {"contents": "1162", "value": "1162"})
item1.selected = True
mods = br.find_control('SSR_CLSRCH_WRK_SUBJECT_SRCH$0')
item2 = Item(mods, {"contents": args.subject, "value": args.subject})
item2.selected = True
# Next semester?
br['CLASS_SRCH_WRK2_STRM$45$'] = ['1162']
br['DERIVED_SSTSNAV_SSTS_MAIN_GOTO$155$'] = ['9999']
br['CLASS_SRCH_WRK2_INSTITUTION$42$'] = [args.college + '01']
br['SSR_CLSRCH_WRK_SSR_EXACT_MATCH1$1'] = ['E']
br.form['SSR_CLSRCH_WRK_CATALOG_NBR$1'] = args.klass
br['SSR_CLSRCH_WRK_SUBJECT_SRCH$0'] = [args.subject]
writeout("without_ICAction", br)
class Bugzilla:
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
self.committers = committers
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
def short_bug_url_for_bug_id(self, bug_id):
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
def _parse_attachment_flag(self, element, flag_name, attachment, result_key):
flag = element.find('flag', attrs={'name' : flag_name})
if flag and flag['status'] == '+':
attachment[result_key] = flag['setter']
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = str(element.find('attachid').string)
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['type'] = str(element.find('type').string)
self._parse_attachment_flag(element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(element, 'commit-queue', attachment, 'committer_email')
return attachment
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll('attachment'):
attachment = self._parse_attachment_element(element, bug_id)
attachments.append(attachment)
return attachments
def fetch_title_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
return soup.find('short_desc').string
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment['is_patch'] and not attachment['is_obsolete']:
patches.append(attachment)
return patches
# _view_source_link belongs in some sort of webkit_config.py module.
def _view_source_link(self, local_path):
return "http://trac.webkit.org/browser/trunk/%s" % local_path
def _validate_setter_email(self, patch, result_key, lookup_function, rejection_function, reject_invalid_patches):
setter_email = patch.get(result_key + '_email')
if not setter_email:
return None
committer = lookup_function(setter_email)
if committer:
patch[result_key] = committer.full_name
return patch[result_key]
if reject_invalid_patches:
committer_list = "WebKitTools/Scripts/modules/committers.py"
failure_message = "%s does not have %s permissions according to %s." % (setter_email, result_key, self._view_source_link(committer_list))
rejection_function(patch['id'], failure_message)
else:
log("Warning, attachment %s on bug %s has invalid %s (%s)", (patch['id'], patch['bug_id'], result_key, setter_email))
return None
def _validate_reviewer(self, patch, reject_invalid_patches):
return self._validate_setter_email(patch, 'reviewer', self.committers.reviewer_by_bugzilla_email, self.reject_patch_from_review_queue, reject_invalid_patches)
def _validate_committer(self, patch, reject_invalid_patches):
return self._validate_setter_email(patch, 'committer', self.committers.committer_by_bugzilla_email, self.reject_patch_from_commit_queue, reject_invalid_patches)
def fetch_reviewed_patches_from_bug(self, bug_id, reject_invalid_patches=False):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if self._validate_reviewer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_commit_queue_patches_from_bug(self, bug_id, reject_invalid_patches=False):
commit_queue_patches = []
for attachment in self.fetch_reviewed_patches_from_bug(bug_id, reject_invalid_patches):
if self._validate_committer(attachment, reject_invalid_patches) and not attachment['is_obsolete']:
commit_queue_patches.append(attachment)
return commit_queue_patches
def fetch_bug_ids_from_commit_queue(self):
commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self, reject_invalid_patches=False):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
raise BugzillaError("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(raw_input("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
if match:
return match.group('bug_id')
match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join([" " + line.strip() for line in text_lines if line.strip()])
raise BugzillaError("Bug not created: %s" % error_message)
def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False):
self.authenticate()
log('Creating bug with patch description "%s"' % patch_description)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component or component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
if cc:
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
if bug_description:
log(bug_description)
self.browser['comment'] = bug_description
self.browser['description'] = patch_description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "%s.patch" % timestamp(), 'data')
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
if flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self, attachment_id, additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
# FIXME: We need a way to test this on a live bugzilla instance.
def _set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text):
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
def reject_patch_from_commit_queue(self, attachment_id, additional_comment_text=None):
comment_text = "Rejecting patch %s from commit-queue." % attachment_id
self._set_flag_on_attachment(attachment_id, 'commit-queue', '-', comment_text, additional_comment_text)
def reject_patch_from_review_queue(self, attachment_id, additional_comment_text=None):
comment_text = "Rejecting patch %s from review queue." % attachment_id
self._set_flag_on_attachment(attachment_id, 'review', '-', comment_text, additional_comment_text)
def obsolete_attachment(self, attachment_id, comment_text = None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['comment'] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
log(comment_text) # Bugzilla requires a comment when re-opening a bug, so we know it will never be None.
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['bug_status'] = ['REOPENED']
self.browser['comment'] = comment_text
self.browser.submit()
if not os.path.exists('EMFAC2014_County/' + year):
os.makedirs('EMFAC2014_County/' + year)
for county in counties:
br = Browser()
response = br.open("https://www.arb.ca.gov/emfac/2014/")
print br.title()
forms_list = []
for f in br.forms():
forms_list.append(f)
# print(forms_list[1])
br.select_form(nr=1)
br.form['geo_level'] = ['county']
region = br.find_control('region')
# inject the current county into form because javascript blocks them
item = Item(region, {
"contents": county.keys()[0],
"value": county.keys()[0]
})
# now set the region to the injected county
br.form['region'] = [county.keys()[0]]
br.form['cal_year[]'] = [year]
br.form['season'] = ['annual']
br.form['veh_cat_type'] = ['emfac2011']
# https://www.arb.ca.gov/emfac/2014/combobox.php?_id=veh_cat_type&_name=veh_cat_type&_value=emfac2011
veh_cat_option = br.find_control('veh_cat_option')
# inject veh category
# set up mechanize header
headers = [('User-Agent', 'Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1')]
# define target URL
url = "http://www.bseindia.com/getquote.htm"
br = Browser()
# browser parameters
br.set_handle_equiv(True)
br.set_handle_gzip(True)
br.set_handle_redirect(True)
br.set_handle_referer(True)
br.set_handle_robots(False)
br.addheaders = headers
# make request
main_page = br.open(url)
# select the default form
br.select_form(nr=0)
br.find_control(id="suggestBoxEQ").value = "CAREERP"
# submit form
br.submit()
page_response = bs(br.response().read())
quote = page_response.find_all('div',{'class':'newsrheadingdiv'})
print quote
def fetch_isochrones(isoc_kind='parsec_CAF09_v1.2S',
photsys_version='yang',
photsys_file='tab_mag_odfnew/tab_mag_ubvrijhk.dat',
kind_cspecmag='aringer09',
dust_sourceM='nodustM',
dust_sourceC='nodustC',
extinction_av='0.0',
imf_file='tab_imf/imf_chabrier_lognormal.dat',
sequence_type= 'single_isochrone',
isoc_age=False,
isoc_z =False,
isoc_z0=False,
isoc_z1=False,
isoc_dz=False,
isoc_lage0=False,
isoc_lage1=False,
isoc_dlage=False,
path='',
filename='Isochrone_teste.dat'):
#Sequence_type = 'single_isochrone', 'sequence_constant_metallicity', 'sequence_constant_age'
if sequence_type == 'single_isochrone' or sequence_type == 0: sequence_type = 0
elif sequence_type == 'constant_metallicity' or sequence_type == 1 : sequence_type = 1
elif sequence_type == 'constant_age' or sequence_type == 2: sequence_type = 2
else: raise ValueError("Argument sequence_type must be in ('single_isochrone', 'constant_metallicity', "
"'constant_age')")
warnings.simplefilter('always', UserWarning)
#Handling bad values given for different sequence types
if sequence_type == 0:
if not isoc_age: raise ValueError("For sequence_type == 'single_isochrone', argument isoc_age must be provided")
if not isoc_z: raise ValueError("For sequence_type == 'single_isochrone', argument isoc_z must be provided")
if any((isoc_z0, isoc_z1, isoc_dz, isoc_lage0, isoc_lage1, isoc_dlage)):
warnings.warn("For sequence_type == 'single_isochrone', arguments isoc_z0, isoc_z1, isoc_dz, isoc_lage0, isoc_lage1 and isoc_dlage are not used")
elif sequence_type == 1:
if not isoc_z: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_z must be provided")
if not isoc_lage0: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_lage0 must be provided")
if not isoc_lage1: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_lage1 must be provided")
if not isoc_dlage: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_dlage must be provided")
if any((isoc_age, isoc_z0, isoc_z1, isoc_dz)):
warnings.warn("For sequence_type == 'constant_metallicity', arguments isoc_age, isoc_z0, isoc_z1, and isoc_dz are not used")
elif sequence_type == 2:
if not isoc_age: raise ValueError("For sequence_type == 'constant_age', argument isoc_age must be provided")
if not isoc_z0: raise ValueError("For sequence_type == 'constant_age', argument isoc_z0 must be provided")
if not isoc_z1: raise ValueError("For sequence_type == 'constant_age', argument isoc_z1 must be provided")
if not isoc_dz: raise ValueError("For sequence_type == 'constant_age', argument isoc_dz must be provided")
if any((isoc_z, isoc_lage0, isoc_lage1, isoc_dlage)):
warnings.warn("For sequence_type == 'constant_age', arguments isoc_z, isoc_lage0, isoc_lage1, and isoc_dlage are not used")
#Error raised when too many isochrones are requested
if sequence_type == 1:
N_isoc = len(np.arange(isoc_lage0, isoc_lage1, isoc_dlage))
if N_isoc > 400:
raise ValueError("you requested too many isochrones ({0}), maximum allowed is 400.\nTry to increase isoc_dlage or lower the difference between isoc_lage0 and isoc_lage1".fotmat(N_isoc))
elif sequence_type == 2:
N_isoc = len(np.arange(isoc_z0, isoc_z1, isoc_dz))
if N_isoc > 400:
raise ValueError("you requested too many isochrones ({0}), maximum allowed is 400.\nTry to increase isoc_dz or lower the difference between isoc_z0 and isoc_z1".format(N_isoc))
#print 'Opening browser'
br = Browser()
br.open('http://stev.oapd.inaf.it/cgi-bin/cmd')
br.select_form(nr = 0)
#print 'Filling form'
br.form['isoc_kind'] = [isoc_kind]
br.form['photsys_version'] = [photsys_version]
br.form['photsys_file'] = [photsys_file]
br.form['kind_cspecmag'] = [kind_cspecmag]
br.form['dust_sourceM'] = [dust_sourceM]
br.form['dust_sourceC'] = [dust_sourceC]
br.form['extinction_av'] = (extinction_av)
br.form['imf_file'] = [imf_file]
br.find_control("isoc_val").items[sequence_type].selected = True
if sequence_type == 0:
br.form['isoc_age'] = str(isoc_age) # Isochrone age
br.form['isoc_zeta'] = str(isoc_z) # Isochrone metallicity
elif sequence_type == 1:
br.form['isoc_zeta0'] = str(isoc_z) # Isochrone metallicity
br.form['isoc_lage0'] = str(isoc_lage0) # Isochrone log initial age
br.form['isoc_lage1'] = str(isoc_lage1) # Isochrone log final age
br.form['isoc_dlage'] = str(isoc_dlage) # Isochrone log age step
elif sequence_type == 2:
br.form['isoc_age0'] = str(isoc_age) # Isochrone age
br.form['isoc_z0'] = str(isoc_z0) # Isochrone initial metallicity
br.form['isoc_z1'] = str(isoc_z1) # Isochrone final metallicity
br.form['isoc_dz'] = str(isoc_dz) # Isochrone metallicity step
#print('Submitting form')
br.submit()
#print('Downloading data')
download_link = list(br.links())[0].absolute_url
geturl(download_link, path+'/'+filename)
br.close()
print('File ' + path+'/'+filename + ' created')
exit(0)
classtype = open('classcategory.txt', 'w')
class_type=[]
browser = Browser()
browser.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
# Cookie Jar
cj = cookielib.LWPCookieJar()
browser.set_cookiejar(cj)
browser.set_handle_robots(False)
browser.open("https://banner.uregina.ca/prod/sct/bwckschd.p_disp_dyn_sched") #to get semester specifics
browser.select_form(nr=0)
item = browser.find_control(id="term_input_id").get("%s" % semester)
item.selected = True
response = browser.submit()
content = response.read()
flag = 0
for line in StringIO.StringIO(content):
if flag == 0:
if '<SELECT NAME="sel_subj" SIZE="10" MULTIPLE ID="subj_id">' in line:
flag = 1
else:
if '</SELECT>' not in line:
x = line.split('"')
try:
class Bugzilla:
def __init__(self, dryrun=False):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub("\.", "\\.", bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
# This could eventually be a text file
reviewer_usernames_to_full_names = {
"abarth": "Adam Barth",
"adele": "Adele Peterson",
"aroben": "Adam Roben",
"ap": "Alexey Proskuryakov",
"ariya.hidayat": "Ariya Hidayat",
"barraclough": "Gavin Barraclough",
"beidson": "Brady Eidson",
"darin": "Darin Adler",
"ddkilzer": "David Kilzer",
"dglazkov": "Dimitri Glazkov",
"eric": "Eric Seidel",
"fishd": "Darin Fisher",
"gns": "Gustavo Noronha",
"hausmann": "Simon Hausmann",
"hyatt": "David Hyatt",
"jmalonzo": "Jan Alonzo",
"justin.garcia": "Justin Garcia",
"kevino": "Kevin Ollivier",
"koivisto": "Antti Koivisto",
"levin": "David Levin",
"mitz": "Dan Bernstein",
"mjs": "Maciej Stachowiak",
"mrowe": "Mark Rowe",
"oliver": "Oliver Hunt",
"sam": "Sam Weinig",
"simon.fraser": "Simon Fraser",
"staikos": "George Staikos",
"timothy": "Timothy Hatcher",
"treat": "Adam Treat",
"vestbo": u"Tor Arne Vestb\xf8",
"xan.lopez": "Xan Lopez",
"zecke": "Holger Freyther",
"zimmermann": "Nikolas Zimmermann",
}
def full_name_from_bugzilla_name(self, bugzilla_name):
if not bugzilla_name in self.reviewer_usernames_to_full_names:
raise Exception("ERROR: Unknown reviewer! " + bugzilla_name)
return self.reviewer_usernames_to_full_names[bugzilla_name]
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=" + action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: " + bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll("attachment"):
attachment = {}
attachment["bug_id"] = bug_id
attachment["is_obsolete"] = element.has_key("isobsolete") and element["isobsolete"] == "1"
attachment["is_patch"] = element.has_key("ispatch") and element["ispatch"] == "1"
attachment["id"] = str(element.find("attachid").string)
attachment["url"] = self.attachment_url_for_id(attachment["id"])
attachment["name"] = unicode(element.find("desc").string)
attachment["type"] = str(element.find("type").string)
review_flag = element.find("flag", attrs={"name": "review"})
if review_flag and review_flag["status"] == "+":
reviewer_email = review_flag["setter"]
# We could lookup the full email address instead once we update full_name_from_bugzilla_name
bugzilla_name = reviewer_email.split("@")[0]
attachment["reviewer"] = self.full_name_from_bugzilla_name(bugzilla_name)
attachments.append(attachment)
return attachments
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment["is_patch"] and not attachment["is_obsolete"]:
patches.append(attachment)
return patches
def fetch_reviewed_patches_from_bug(self, bug_id):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if "reviewer" in attachment and not attachment["is_obsolete"]:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_bug_ids_from_commit_queue(self):
# FIXME: We should have an option for restricting the search by email. Example:
# unassigned_only = "&emailassigned_to1=1&emailtype1=substring&email1=unassigned"
commit_queue_url = (
self.bug_server_url
+ "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B"
)
log("Loading commit queue")
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup("td", "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_reviewed_patches_from_bug(bug_id)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser["Bugzilla_login"] = username
self.browser["Bugzilla_password"] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
error("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id)
self.browser.select_form(name="entryform")
self.browser["description"] = description
self.browser["ispatch"] = ("1",)
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["flag_type-1"] = ("?",) if mark_for_review else ("X",)
self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, "edit"))
self.browser.select_form(nr=1)
self.browser.find_control("isobsolete").items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self.browser.find_control(type="select", nr=0).value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name="comment", nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["bug_status"] = ["RESOLVED"]
self.browser["resolution"] = ["FIXED"]
self.browser.submit()
#.*?(\d+).*?(\d{4}-\d{2}-\d{2}\s*?[0-9:]+).*?([A-Z0-9]+).*?([A-Z0-9]+).*?([0-9.]+).*?(\d+).*?([A-Za-z0-9\+\-]+)
regex = re.compile('.*?(\d+).*?(\d{4}-\d{2}-\d{2}\s*?[0-9:]+).*?([A-Z0-9]+).*?([A-Z0-9]+).*?([0-9.]+).*?(\d+).*?([A-Za-z0-9\+\-]+)')
auth = False
while not auth:
user = raw_input("Username: "******"autologin").items[0].selected = True
br.submit()
for link in br.links():
if link.text == 'my account':
auth = True
break
if not auth:
print "Authentication Failed"
print "Authenticated"
sublist = br.open(base_url+"status/"+user+"/signedlist")
sublist = sublist.read()
allsub = regex.findall(sublist)
def getSolutions(path_prefix, path_proxy):
global br, username, password
# create a browser object
br = Browser()
# add proxy support to browser
if len(path_proxy) != 0:
protocol, proxy = options.proxy.split("://")
br.set_proxies({protocol: proxy})
# let browser fool robots.txt
br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; \
rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
br.set_handle_robots(False)
print "Enter yout SPOJ username :"******"Authenticating " + username
br.open("http://www.spoj.com")
br.select_form(name="login")
br["login_user"] = username
br["password"] = password
# sign in for a day to avoid timeouts
br.find_control(name="autologin").items[0].selected = True
br.form.action = "http://www.spoj.com/"
response = br.submit()
verify = response.read()
if (verify.find("Authentication failed!") != -1):
print "Error authenticating - " + username
exit(0)
else:
print "Successfully Authenticated !!"
# Whether or not to download the multiple accepted solution of same problem
global dupdown
global subidinc
print "Do You want to download multiple ACed Solution of same problem? (The latest one will be downloaded otherwise) [Y/N] :",
dupdown = raw_input()
if dupdown != 'Y' and dupdown != 'y':
print "Do You Submission ID as a part of file-name? [Y/N] :",
subidinc = raw_input()
# grab the signed submissions list
print "Grabbing siglist for " + username
siglist = br.open("http://www.spoj.com/status/" + username + "/signedlist")
# dump first nine useless lines in signed list for formatting
for i in xrange(9):
siglist.readline()
# make a list of all AC's and challenges
print "Filtering siglist for AC/Challenge solutions..."
mysublist = list()
while True:
temp = siglist.readline()
if temp == '\------------------------------------------------------------------------------/\n':
# reached end of siglist
break
if not len(temp):
print "Reached EOF, siglist format has probably changed," + \
" contact author."
exit(1)
entry = [x.strip() for x in temp.split('|')]
if entry[4] == 'AC' or entry[4].isdigit():
dupflag = 0
if dupdown != 'Y' and dupdown != 'y':
for xdup in mysublist:
if xdup[3] == entry[3]:
dupflag = 1
break
if dupflag == 0:
mysublist.append(entry)
print "Done !!!"
return mysublist
class Login(object):
"""
IPS client area login class
"""
# Form constants
LOGIN_URL = 'https://www.invisionpower.com/clients/index.php?app=core&module=global§ion=login'
TEST_URL = 'https://www.invisionpower.com/clients/index.php?app=nexus&module=clients§ion=purchases'
USERNAME_FIELD = 'ips_username'
PASSWORD_FIELD = 'ips_password'
REMEMBER_FIELD = 'rememberMe'
# Cookie constants
LOGIN_COOKIE = 'ips_pass_hash'
def __init__(self, ctx):
"""
Initialize a new Login Handler instance
@type ctx: ips_vagrant.cli.Context
"""
# Debug log
self.log = logging.getLogger('ipsv.login')
self.cookiejar = cookiejar()
self.cookies = {cookie.name: cookie.value for cookie in self.cookiejar}
self.browser = Browser()
self.browser.set_cookiejar(self.cookiejar)
def check(self):
"""
Check if we have an active login session set
@rtype: bool
"""
self.log.debug('Testing for a valid login session')
# If our cookie jar is empty, we obviously don't have a valid login session
if not len(self.cookiejar):
return False
# Test our login session and make sure it's still active
return requests.get(self.TEST_URL, cookies=self.cookiejar).status_code == 200
def process(self, username, password, remember=True):
"""
Process a login request
@type username: str
@type password: str
@param remember: Save the login session to disk
@type remember: bool
@raise BadLoginException: Login request failed
@return: Session cookies
@rtype: cookielib.LWPCookieJar
"""
self.log.debug('Processing login request')
self.browser.open(self.LOGIN_URL)
self.log.info('Login page loaded: %s', self.browser.title())
self.browser.select_form(nr=0)
# Set the fields
self.log.debug('Username: %s', username)
self.log.debug('Password: %s', (password[0] + '*' * (len(password) - 2) + password[-1]))
self.log.debug('Remember: %s', remember)
self.browser.form[self.USERNAME_FIELD] = username
self.browser.form[self.PASSWORD_FIELD] = password
self.browser.find_control(self.REMEMBER_FIELD).items[0].selected = remember
# Submit the request
self.browser.submit()
self.log.debug('Response code: %s', self.browser.response().code)
self.log.debug('== Cookies ==')
for cookie in self.cookiejar:
self.log.debug(cookie)
self.cookies[cookie.name] = cookie.value
self.log.debug('== End Cookies ==')
# Make sure we successfully logged in
if self.LOGIN_COOKIE not in self.cookies:
raise BadLoginException('No login cookie returned, this probably means an invalid login was provided')
# Should we save our login session?
if remember:
self.log.info('Saving login session to disk')
self.cookiejar.save()
self.log.info('Login request successful')
return self.cookiejar
class Bugzilla(object):
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.queries = BugzillaQueries(self)
self.committers = committers
# FIXME: We should use some sort of Browser mock object when in dryrun
# mode (to prevent any mistakes).
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this
# script.
self.browser.set_handle_robots(False)
# FIXME: Much of this should go into some sort of config module:
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
unassigned_email = "*****@*****.**"
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
bug_id,
content_type)
def short_bug_url_for_bug_id(self, bug_id):
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id,
action_param)
def _parse_attachment_flag(self,
element,
flag_name,
attachment,
result_key):
flag = element.find('flag', attrs={'name': flag_name})
if flag:
attachment[flag_name] = flag['status']
if flag['status'] == '+':
attachment[result_key] = flag['setter']
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = int(element.find('attachid').string)
# FIXME: No need to parse out the url here.
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['attacher_email'] = str(element.find('attacher').string)
attachment['type'] = str(element.find('type').string)
self._parse_attachment_flag(
element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(
element, 'commit-queue', attachment, 'committer_email')
return attachment
def _parse_bug_page(self, page):
soup = BeautifulSoup(page)
bug = {}
bug["id"] = int(soup.find("bug_id").string)
bug["title"] = unicode(soup.find("short_desc").string)
bug["reporter_email"] = str(soup.find("reporter").string)
bug["assigned_to_email"] = str(soup.find("assigned_to").string)
bug["cc_emails"] = [str(element.string)
for element in soup.findAll('cc')]
bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
return bug
# Makes testing fetch_*_from_bug() possible until we have a better
# BugzillaNetwork abstration.
def _fetch_bug_page(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
return self.browser.open(bug_url)
def fetch_bug_dictionary(self, bug_id):
return self._parse_bug_page(self._fetch_bug_page(bug_id))
# FIXME: A BugzillaCache object should provide all these fetch_ methods.
def fetch_bug(self, bug_id):
return Bug(self.fetch_bug_dictionary(bug_id), self)
def _parse_bug_id_from_attachment_page(self, page):
# The "Up" relation happens to point to the bug.
up_link = BeautifulSoup(page).find('link', rel='Up')
if not up_link:
# This attachment does not exist (or you don't have permissions to
# view it).
return None
match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
return int(match.group('bug_id'))
def bug_id_for_attachment_id(self, attachment_id):
self.authenticate()
attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
log("Fetching: %s" % attachment_url)
page = self.browser.open(attachment_url)
return self._parse_bug_id_from_attachment_page(page)
# FIXME: This should just return Attachment(id), which should be able to
# lazily fetch needed data.
def fetch_attachment(self, attachment_id):
# We could grab all the attachment details off of the attachment edit
# page but we already have working code to do so off of the bugs page,
# so re-use that.
bug_id = self.bug_id_for_attachment_id(attachment_id)
if not bug_id:
return None
attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
for attachment in attachments:
if attachment.id() == int(attachment_id):
return attachment
return None # This should never be hit.
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
attempts = 0
while not self.authenticated:
attempts += 1
(username, password) = Credentials(
self.bug_server_host, git_prefix="bugzilla").read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url +
"index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word
# "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
errorMessage = "Bugzilla login failed: %s" % match.group(1)
# raise an exception only if this was the last attempt
if attempts < 5:
log(errorMessage)
else:
raise Exception(errorMessage)
else:
self.authenticated = True
def _fill_attachment_form(self,
description,
patch_file_object,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False, bug_id=None):
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
if mark_for_landing:
self.browser['flag_type-3'] = ('+',)
elif mark_for_commit_queue:
self.browser['flag_type-3'] = ('?',)
else:
self.browser['flag_type-3'] = ('X',)
if bug_id:
patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
else:
patch_name ="%s.patch" % timestamp()
self.browser.add_file(patch_file_object,
"text/plain",
patch_name,
'data')
def add_patch_to_bug(self,
bug_id,
patch_file_object,
description,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False):
self.authenticate()
log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
self.bug_server_url,
bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
self._fill_attachment_form(description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
mark_for_landing=mark_for_landing,
bug_id=bug_id)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(User.prompt("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
response_html)
if match:
return match.group('bug_id')
match = re.search(
'<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
response_html,
re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(
match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join(
[" " + line.strip()
for line in text_lines if line.strip()])
raise Exception("Bug not created: %s" % error_message)
def create_bug(self,
bug_title,
bug_description,
component=None,
patch_file_object=None,
patch_description=None,
cc=None,
mark_for_review=False,
mark_for_commit_queue=False):
self.authenticate()
log('Creating bug with title "%s"' % bug_title)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component:
component = "New Bugs"
if component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
if cc:
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
self.browser['comment'] = bug_description
if patch_file_object:
self._fill_attachment_form(
patch_description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue)
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
if flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self,
attachment_id,
additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
def set_flag_on_attachment(self,
attachment_id,
flag_name,
flag_value,
comment_text,
additional_comment_text):
# FIXME: We need a way to test this function on a live bugzilla
# instance.
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
# FIXME: All of these bug editing methods have a ridiculous amount of
# copy/paste code.
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow
# hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def add_cc_to_bug(self, bug_id, email_address_list):
self.authenticate()
log("Adding %s to the CC list for bug %s" % (email_address_list,
bug_id))
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["newcc"] = ", ".join(email_address_list)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text, cc=None):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
if cc:
self.browser["newcc"] = ", ".join(cc)
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reassign_bug(self, bug_id, assignee, comment_text=None):
self.authenticate()
log("Assigning bug %s to %s" % (bug_id, assignee))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["assigned_to"] = assignee
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
# Bugzilla requires a comment when re-opening a bug, so we know it will
# never be None.
log(comment_text)
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
bug_status = self.browser.find_control("bug_status", type="select")
# This is a hack around the fact that ClientForm.ListControl seems to
# have no simpler way to ask if a control has an item named "REOPENED"
# without using exceptions for control flow.
possible_bug_statuses = map(lambda item: item.name, bug_status.items)
if "REOPENED" in possible_bug_statuses:
bug_status.value = ["REOPENED"]
else:
log("Did not reopen bug %s. " +
"It appears to already be open with status %s." % (
bug_id, bug_status.value))
self.browser['comment'] = comment_text
self.browser.submit()
class LEAMsite:
def __init__(self, site, user, passwd):
self.site = site
self.error = False
self.b = Browser()
self.b.set_handle_robots(False)
try:
self.b.open(site)
except urllib2.URLError:
self.error = True
return
try:
# try and log in from the portlet
self.b.select_form('loginform')
except:
# try logging in from the main login page
self.b.open('/'.join((site, "login_form")))
self.b.select_form(nr=1)
if not user or not passwd:
raise ValueError('user and password are required')
self.b['__ac_name'] = user
self.b['__ac_password'] = passwd
r = self.b.open(self.b.click())
# plone changes have rendered this inoperable
# capture the response and look in the content
#
#if 'logged_in' in path.split(self.b.geturl()):
# sys.stderr.write("Error: unable to login to LEAM Plone site\n")
# sys.exit(1)
def getURL(self, url, data=None, filename=None):
"""Simple interface for retrieving the contents of a URL
and writing it to a file or returning it as stringIO.
"""
#sys.stderr.write('getURL %s\n' % url)
rsp = self.b.open(url, data)
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return StringIO(rsp.read())
def putFileURL(self,
filename,
url,
fileobj=None,
title=None,
type='text/plain'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url, 'createObject?type_name=File')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def getFile(self, url, filename=None):
""" getFile -- gets a file using standard download from Plone site
url: The URL is pointer to the file on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url, 'at_download/file'))
if filename:
f = open(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return rsp.read()
def saveFile(self, url, dir=".", at_download=False):
"""Reads the response from a URL and saves it to a local
file based on the name provided in the Content-Disposition
header.
The dir field specifies to the directory where the file will be stored.
If the at_download flag is True then 'at_download/file' will
be appended the URL.
"""
if at_download:
rsp = self.b.open(url_join(url, 'at_download/file'))
else:
rsp = self.b.open(url)
fname = get_filename(rsp)
if fname:
f = open('/'.join([dir, fname]), 'wb')
f.write(rsp.read())
f.close()
return fname
def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'):
sys.stderr.write('putImageURL %s to %s\n' % (imgfile, url))
self.b.open('/'.join((url, 'createObject?type_name=Image')))
self.b.select_form("edit_form")
if not title: title = path.basename(imgfile)
self.b['title'] = title
self.b.add_file(open(imgfile), type, path.basename(imgfile))
try:
# doesn't seem necessary but it is required for file upload
form = self.b.find_control('image_delete')
form.value = [
"",
]
# print "This really does need to happen.."
except:
# print "That delete stuff never happens..."
pass
self.b.submit("form.button.save")
def putImageURL(self,
filename,
url,
fileobj=None,
title=None,
type='image/jpg'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url, 'createObject?type_name=Image')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def putDocument(self, doc, url, title):
"""Creates a new document and add the doc (file-like object) to it."""
self.b.open('/'.join((url, 'createObject?type_name=Document')))
self.b.select_form("edit_form")
self.b['title'] = title
doc.seek(0)
self.b['text'] = doc.read()
self.b.submit("form.button.save")
return self.b.geturl()
def getDocument(self, url):
"""Returns a string with the text of the current document."""
self.b.open('/'.join((url, 'edit')))
self.b.select_form("edit_form")
s = self.b['text']
self.b.submit("form.button.cancel")
return s
def editDocument(self, doc, url, title=None):
"""Replaces the contents of a document"""
self.b.open('/'.join((url, 'edit')))
self.b.select_form("edit_form")
# update the title if given
if title: self.b['title'] = title
# slightly dangerous approach where we seek first
# to test for a file-like object. If exception is
# thrown then assume doc is a string.
try:
doc.seek(0)
self.b['text'] = doc.read()
except:
self.b['text'] = doc
self.b.submit("form.button.save")
return self.b.geturl()
def createFolder(self, folder, url):
"""Creates a folder titled <folder> at the location <url> if
it doesn't already exist. Returns the full path of new folder.
"""
pathurl = '/'.join((url, folder.lower().replace(' ', '-')))
try:
self.b.open(pathurl)
except:
self.b.open('/'.join((url, 'createObject?type_name=Folder')))
self.b.select_form("edit_form")
self.b['title'] = folder
self.b.submit("form.button.save")
return self.b.geturl()
def editFolder(self, url, title="", description=""):
"""Edit the basic fields of the Folder. Mostly useful for
setting the title AFTER creating the folder with a reasonable
short name.
"""
try:
self.b.open(url)
except:
self.error = True
return None
self.b.open(url + '/edit')
self.b.select_form("edit_form")
if title:
self.b['title'] = title
if description:
self.b['description'] = description
self.b.submit("form.button.save")
return self.b.geturl()
def deleteFolder(self, url):
"Deletes folder and all of its contents"
sys.stderr.write('DELETING folder %s\n' % url)
return
# Puts SimMaps on to the site
def putSimMap(self,
simmap,
mapfile,
url,
simmap_file=None,
mapfile_file=None,
title=None,
description=None,
trans=.7,
details=None,
zoom=11):
""" putSimMap
Required Input: simmap, mapfile, url
simmap is a file that contains the desired GIS layer
mapfile is the standard .map file that maps GIS layer to image
url - is the full URL to the folder where the SimMap will be stored
Optional Inputs: simmap_file, mapfile_file, title, trans, details
simmap_file - file-like object of the simmap if None simmap
will be openned and read.
mapfile_file - file-like object of the mapfile. If None mapfile
will be openned and read.
title - title sting of the SimMap defaults to basename(simmap)
trans - transparency level
details - description of SimMap as stored in the details field
"""
self.b.open('/'.join((url, 'createObject?type_name=SimMap')))
self.b.select_form("edit_form")
if not simmap_file: simmap_file = open(simmap, 'rb')
if not mapfile_file: mapfile_file = open(mapfile, 'rb')
if not title: title = path.splitext(path.basename(simmap))[0]
self.b['title'] = str(title)
self.b.add_file(simmap_file, 'application/octet-stream',
path.basename(simmap), "simImage_file")
self.b.add_file(mapfile_file, 'application/octet-stream',
path.basename(mapfile), "mapFile_file")
if description: self.b['description'] = str(description)
self.b['transparency'] = str(trans)
self.b['zoom'] = str(zoom)
if details: self.b['details'] = str(details)
self.b.submit("form.button.save")
return self.b.geturl()
def getSimMap(self, url):
""" getSimMap
Gets the metadata associated with SimMap including title,
description, location, transparency, and zoom.
"""
self.b.open(url + '/edit')
self.b.select_form('edit_form')
d = dict(
title=self.b['title'],
description=self.b['description'],
details=self.b['details'],
location=self.b['location'],
transparency=self.b['Transparency'],
zoom=self.b['zoom'],
)
self.b.submit("form.button.cancel")
return d
def getSimMapData(self, url, filename=None):
""" getSimMapData -- gets the data component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
bufsize = 15 * 1024 * 1024
rsp = self.b.open(url_join(url, 'get_layer'))
if filename:
f = file(filename, 'wb')
while 1:
b = rsp.read(bufsize)
f.write(b)
if len(b) < bufsize: break
f.close()
return None
else:
return StringIO(rsp.read())
def getSimMapMapfile(self, url, filename=None):
""" getSimMapData -- gets the mapfile component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url, 'get_mapfile'))
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
data = StringIO()
data.write(rsp.read())
return data
# DELETE FUNCTIONS -----------------------------------------
# These functions delete items from the LEAM Plone sites
# ----------------------------------------------------------
def deleteItem(self, fname, url):
""" deleteItem
Deletes <fname> from the folder <url>
"""
print '/'.join((url, fname, 'delete_confirmation'))
print self.b.open('/'.join((url, fname, 'delete_confirmation')))
self.b.select_form(None, None, 1)
print self.b.submit()
import mechanize
import urllib3
browser = Browser()
browser.open("http://localhost:8080/test_suite")
browser.select_form(nr=1)
print(browser)
tipos = browser.form.possible_items("type")
print(tipos)
browser['title'] = ''
browser.set(True, tipos[0], "type")
browser.find_control("checkbox").items[0].selected = False
print('current estatus')
print(browser['title'])
print(browser['type'])
print(browser['checkbox'])
if browser['title'] == '':
print('val tit: ', 0)
else:
print('val tit: ', 1)
if len(browser['checkbox']) == 0:
print('val check: ', 0)
else:
class Enviroment:
def __init__(self):
#self.fields = fields
self.states = {}
self.status_submit = '0'
self.http_response = ''
self.browser = Browser()
self.browser.open("http://localhost:8080/test_suite")
self.browser.select_form(nr=1)
self.tipos = self.browser.form.possible_items("type")
def initialize(self):
self.browser = Browser()
self.browser.open("http://localhost:8080/test_suite")
self.browser.select_form(nr=1)
self.tipos = self.browser.form.possible_items("type")
self.status_submit = '0'
self.http_response = ''
def apply_action(self, action):
if action == 0:
self.browser['title'] = ''
self.status_submit = '0'
elif action == 1:
self.browser['title'] = 'Prueba'
self.status_submit = '0'
elif action == 2:
self.browser.find_control("checkbox").items[0].selected = False
self.status_submit = '0'
elif action == 3:
self.browser.find_control("checkbox").items[0].selected = True
self.status_submit = '0'
elif action == 4:
self.browser.set(True, self.tipos[0], "type")
self.status_submit = '0'
elif action == 5:
self.browser.set(True, self.tipos[1], "type")
self.status_submit = '0'
elif action == 6:
self.browser.set(True, self.tipos[2], "type")
self.status_submit = '0'
elif action == 7:
self.status_submit = '1'
#just submit if all fields are filled
if self.browser['title'] != '' and self.browser['type'][
0] != '0' and len(self.browser['checkbox']) != 0:
try:
response = self.browser.submit()
#print('after submit')
#content = response.read()
code = response.code
#print(code)
self.http_response = code
except mechanize.HTTPError as e:
#print('Error:', e)
self.http_response = 500
def get_current_state(self):
if self.browser['title'] == '':
val_tit = '0'
else:
val_tit = '1'
if len(self.browser['checkbox']) == 0:
val_che = '0'
else:
val_che = '1'
if self.browser['type'][0] == '0':
val_typ = '0'
elif self.browser['type'][0] == '1':
val_typ = '1'
elif self.browser['type'][0] == '2':
val_typ = '2'
state = val_tit + val_che + val_typ + self.status_submit
#0110
return state
def get_all_possible_actions(self):
actions = [0, 1, 2, 3, 4, 5, 6, 7]
return actions
def get_next_state(self, action):
#(0110, 2)
self.apply_action(action)
if action == 7:
return '0000'
else:
return self.get_current_state()
class Bugzilla:
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this script
self.browser.set_handle_robots(False)
self.committers = committers
# Defaults (until we support better option parsing):
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type)
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=" + action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param)
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = str(element.find('attachid').string)
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['type'] = str(element.find('type').string)
review_flag = element.find('flag', attrs={"name" : "review"})
if review_flag and review_flag['status'] == '+':
reviewer_email = review_flag['setter']
reviewer = self.committers.reviewer_by_bugzilla_email(reviewer_email)
attachment['reviewer'] = reviewer.full_name
commit_queue_flag = element.find('flag', attrs={"name" : "commit-queue"})
if commit_queue_flag and commit_queue_flag['status'] == '+':
committer_email = commit_queue_flag['setter']
committer = self.committers.committer_by_bugzilla_email(committer_email)
attachment['commit-queue'] = committer.full_name
return attachment
def fetch_attachments_from_bug(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: " + bug_url)
page = urllib2.urlopen(bug_url)
soup = BeautifulSoup(page)
attachments = []
for element in soup.findAll('attachment'):
attachment = self._parse_attachment_element(element, bug_id)
attachments.append(attachment)
return attachments
def fetch_patches_from_bug(self, bug_id):
patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if attachment['is_patch'] and not attachment['is_obsolete']:
patches.append(attachment)
return patches
def fetch_reviewed_patches_from_bug(self, bug_id):
reviewed_patches = []
for attachment in self.fetch_attachments_from_bug(bug_id):
if 'reviewer' in attachment and not attachment['is_obsolete']:
reviewed_patches.append(attachment)
return reviewed_patches
def fetch_commit_queue_patches_from_bug(self, bug_id):
commit_queue_patches = []
for attachment in self.fetch_reviewed_patches_from_bug(bug_id):
if 'commit-queue' in attachment and not attachment['is_obsolete']:
commit_queue_patches.append(attachment)
return commit_queue_patches
def fetch_bug_ids_from_commit_queue(self):
commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B"
page = urllib2.urlopen(commit_queue_url)
soup = BeautifulSoup(page)
bug_ids = []
# Grab the cells in the first column (which happens to be the bug ids)
for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child"
bug_link = bug_link_cell.find("a")
bug_ids.append(bug_link.string) # the contents happen to be the bug id
return bug_ids
def fetch_patches_from_commit_queue(self):
patches_to_land = []
for bug_id in self.fetch_bug_ids_from_commit_queue():
patches = self.fetch_commit_queue_patches_from_bug(bug_id)
patches_to_land += patches
return patches_to_land
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
(username, password) = read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
# FIXME: We could add the ability to try again on failure.
raise ScriptError("Bugzilla login failed: %s" % match.group(1))
self.authenticated = True
def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False):
self.authenticate()
log('Adding patch "%s" to bug %s' % (description, bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id)
self.browser.select_form(name="entryform")
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp()))
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(raw_input("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html)
if match:
return match.group('bug_id')
match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join([" " + line.strip() for line in text_lines if line.strip()])
raise ScriptError("Bug not created: %s" % error_message)
def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False):
self.authenticate()
log('Creating bug with patch description "%s"' % patch_description)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component or component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
if bug_description:
log(bug_description)
self.browser['comment'] = bug_description
self.browser['description'] = patch_description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
self.browser.add_file(patch_file_object, "text/plain", "%s.patch" % timestamp(), 'data')
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log(self.bug_server_url + "show_bug.cgi?id=" + bug_id)
return bug_id
def clear_attachment_review_flag(self, attachment_id, additional_comment_text=None):
self.authenticate()
comment_text = "Clearing review flag on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n" + additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.find_control(type='select', nr=0).value = ("X",)
self.browser.submit()
def obsolete_attachment(self, attachment_id, comment_text = None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self.browser.find_control(type='select', nr=0).value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser['comment'] = comment_text
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
class Installer(object):
# License fields
FIELD_LICENSE_KEY = 'lkey'
# Server Detail fields
FIELD_SERVER_SQL_HOST = 'sql_host'
FIELD_SERVER_SQL_USER = '******'
FIELD_SERVER_SQL_PASS = '******'
FIELD_SERVER_SQL_DATABASE = 'sql_database'
# Admin fields
FIELD_ADMIN_USER = '******'
FIELD_ADMIN_PASS = '******'
FIELD_ADMIN_PASS_CONFIRM = 'admin_pass2'
FIELD_ADMIN_EMAIL = 'admin_email'
def __init__(self, ctx, site, force=False):
"""
Initialize a new Installer instance
@type ctx: ips_vagrant.cli.Context
@param site: The IPS Site we are installing
@type site: ips_vagrant.models.sites.Site
@param force: Overwrite existing files / databases
@type force: bool
"""
self.log = logging.getLogger('ipsv.installer')
self.ctx = ctx
self.force = force
self._previous_title = None
self.url = '{scheme}://{host}/admin/install'.format(
scheme='https' if site.ssl else 'http', host=site.domain.name)
self.site = site
self.mysql = create_engine('mysql://*****:*****@localhost')
self._sessions = {}
self.cookiejar = cookiejar()
self.cookies = {cookie.name: cookie.value for cookie in self.cookiejar}
self.browser = Browser()
self.browser.set_cookiejar(self.cookiejar)
def _check_title(self, title):
"""
If we're on the same page, we got an error and need to raise an exception
@type title: str
@raise InstallationError: Title matches the previous page requests title (We're on the same page)
"""
self.log.info('Installation page loaded: %s', title)
if self._previous_title and title == self._previous_title:
raise InstallationError('Unexpected page error')
self._previous_title = title
def start(self):
"""
Start the installation wizard
"""
self.log.debug('Starting the installation process')
self.browser.open(self.url)
continue_link = next(
self.browser.links(text_regex='Start Installation'))
self.browser.follow_link(continue_link)
self.system_check()
def system_check(self):
"""
System requirements check
"""
self._check_title(self.browser.title())
p = Echo('Running system check...')
rsoup = BeautifulSoup(self.browser.response().read(), "html.parser")
# Check for any errors
errors = []
for ul in rsoup.find_all('ul', {'class': 'ipsList_checks'}):
for li in ul.find_all('li', {'class': 'fail'}):
errors.append(li.text)
if errors:
raise InstallationError(errors)
# Continue
continue_link = next(self.browser.links(text_regex='Continue'))
p.done()
self.browser.follow_link(continue_link)
self.license()
def license(self):
"""
Submit our license to IPS' servers
"""
p = Echo('Submitting license key...')
self._check_title(self.browser.title())
self.browser.select_form(nr=0)
# Set the fields
self.browser.form[
self.FIELD_LICENSE_KEY] = '{license}-TESTINSTALL'.format(
license=self.site.license_key)
self.browser.find_control(
'eula_checkbox').items[0].selected = True # TODO: User prompt?
# Submit the request
self.log.debug('Submitting our license')
self.browser.submit()
self.log.debug('Response code: %s', self.browser.response().code)
p.done()
self.applications()
def applications(self):
"""
Select the applications to install (currently hardcoded to install all applications)
"""
# Check for license submission errors
try:
self._check_title(self.browser.title())
except InstallationError:
rsoup = BeautifulSoup(self.browser.response().read(),
"html.parser")
error = rsoup.find('li', id='license_lkey').find(
'span', {
'class': 'ipsType_warning'
}).text
raise InstallationError(error)
# TODO: Make this configurable
p = Echo('Setting applications to install...')
self.browser.select_form(nr=0)
self.browser.submit()
p.done()
self.server_details()
def server_details(self):
"""
Input server details (database information, etc.)
"""
self._check_title(self.browser.title())
p = Echo('Creating MySQL database...')
# Create the database
md5hex = md5(self.site.domain.name + self.site.slug).hexdigest()
db_name = 'ipsv_{md5}'.format(md5=md5hex)
# MySQL usernames are limited to 16 characters max
db_user = '******'.format(md5=md5hex[:11])
rand_pass = ''.join(random.SystemRandom().choice(string.ascii_letters +
string.digits)
for _ in range(random.randint(16, 24)))
db_pass = rand_pass
try:
self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name))
except SQLAlchemyError:
if not self.force:
click.confirm(
'A previous database for this installation already exists.\n'
'Would you like to drop it now? The installation will be aborted if you do not',
abort=True)
self.log.info(
'Dropping existing database: {db}'.format(db=db_name))
self.mysql.execute(
'DROP DATABASE IF EXISTS `{db}`'.format(db=db_name))
self.mysql.execute('DROP USER `{u}`'.format(u=db_user))
self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name))
self.mysql.execute(
"GRANT ALL ON {db}.* TO '{u}'@'localhost' IDENTIFIED BY '{p}'".
format(db=db_name, u=db_user, p=db_pass))
# Save the database connection information
self.site.db_host = 'localhost'
self.site.db_name = db_name
self.site.db_user = db_user
self.site.db_pass = db_pass
self.ctx.db.commit()
self.log.debug('MySQL Database Name: %s', db_name)
self.log.debug('MySQL Database User: %s', db_user)
self.log.debug('MySQL Database Password: %s', db_pass)
# Set form fields and submit
self.browser.select_form(nr=0)
self.browser.form[self.FIELD_SERVER_SQL_HOST] = 'localhost'
self.browser.form[self.FIELD_SERVER_SQL_USER] = db_user
self.browser.form[self.FIELD_SERVER_SQL_PASS] = db_pass
self.browser.form[self.FIELD_SERVER_SQL_DATABASE] = db_name
self.browser.submit()
p.done()
self.admin()
def admin(self):
"""
Provide admin login credentials
"""
self._check_title(self.browser.title())
self.browser.select_form(nr=0)
# Get the admin credentials
prompted = []
user = self.ctx.config.get('User', 'AdminUser')
if not user:
user = click.prompt('Admin display name')
prompted.append('user')
password = self.ctx.config.get('User', 'AdminPass')
if not password:
password = click.prompt(
'Admin password',
hide_input=True,
confirmation_prompt='Confirm admin password')
prompted.append('password')
email = self.ctx.config.get('User', 'AdminEmail')
if not email:
email = click.prompt('Admin email')
prompted.append('email')
self.browser.form[self.FIELD_ADMIN_USER] = user
self.browser.form[self.FIELD_ADMIN_PASS] = password
self.browser.form[self.FIELD_ADMIN_PASS_CONFIRM] = password
self.browser.form[self.FIELD_ADMIN_EMAIL] = email
Echo('Submitting admin information...').done()
if len(prompted) >= 3:
save = click.confirm(
'Would you like to save and use these admin credentials for future installations?'
)
if save:
self.log.info('Saving admin login credentials')
self.ctx.config.set('User', 'AdminUser', user)
self.ctx.config.set('User', 'AdminPass', password)
self.ctx.config.set('User', 'AdminEmail', email)
with open(self.ctx.config_path, 'wb') as cf:
self.ctx.config.write(cf)
self.install()
def _start_install(self):
"""
Start the installation
"""
self.browser.submit()
self._check_title(self.browser.title())
def _get_mr_link(self):
"""
Get the MultipleRedirect URL
@rtype: str
"""
rsoup = BeautifulSoup(self.browser.response().read(), "html.parser")
mr_link = rsoup.find('a', {'class': 'button'}).get('href')
self.log.debug('MultipleRedirect link: %s', mr_link)
return mr_link
def _ajax(self,
url,
method='get',
params=None,
load_json=True,
raise_request=True):
"""
Perform an Ajax request
@type url: str
@type method: str
@type params: dict or None
@type load_json: bool
@return: Tuple with the decoded JSON response and actual response, or just the response if load_json is False
@rtype: requests.Response or tuple of (dict or list, requests.Response)
"""
if 'ajax' in self._sessions:
ajax = self._sessions['ajax']
else:
self.log.debug('Instantiating a new Ajax session')
ajax = requests.Session()
ajax.headers.update({'X-Requested-With': 'XMLHttpRequest'})
ajax.cookies.update(cookiejar())
ajax.verify = False
self._sessions['ajax'] = ajax
response = ajax.request(method, url, params)
self.log.debug('Ajax response: %s', response.text)
if raise_request:
response.raise_for_status()
if load_json:
return byteify(json.loads(response.text)), response
return response
def _request(self, url, method='get', params=None, raise_request=True):
"""
Perform a standard HTTP request
@type url: str
@type method: str
@type params: dict or None
@param raise_request: Raise exceptions for HTTP status errors
@type raise_request: bool
@rtype: requests.Response
"""
if 'http' in self._sessions:
http = self._sessions['http']
else:
self.log.debug('Instantiating a new HTTP session')
http = requests.Session()
http.cookies.update(cookiejar())
http.verify = False
self._sessions['http'] = http
response = http.request(method, url, params)
self.log.debug('HTTP response code: %s', response.status_code)
if raise_request:
response.raise_for_status()
return response
def _parse_response(self, url, json_response):
"""
Parse response data and return the next request URL
@type url: str
@type json_response: list or dict
@rtype: str
"""
parts = list(urlparse(url))
query = parse_qs(parts[4])
query['mr'] = str(json_response[0]).replace('\'', '"')
parts[4] = urlencode(query, True)
return urlunparse(parts)
def _get_stage(self, json_response):
"""
Get the current installation stage
@type json_response: list or dict
@rtype: str
"""
try:
return json_response[1]
except IndexError:
return 'Installation complete!'
def _get_progress(self, json_response):
"""
Get the current installation progress
@type json_response: list or dict
@rtype: str
"""
try:
return round(float(json_response[2]))
except IndexError:
return 0
def _check_if_complete(self, url, json_response):
"""
Check if a request has been completed and return the redirect URL if it has
@type url: str
@type json_response: list or dict
@rtype: str or bool
"""
if 'redirect' in json_response and isinstance(json_response, dict):
self.log.info('Installation complete')
return json_response['redirect']
return False
def _finalize(self, response):
"""
Finalize the installation and display a link to the suite
"""
rsoup = BeautifulSoup(response.text, "html.parser")
click.echo('------')
click.secho(rsoup.find('h1', id='elInstaller_welcome').text.strip(),
fg='yellow',
bold=True)
click.secho(rsoup.find('p', {
'class': 'ipsType_light'
}).text.strip(),
fg='yellow',
dim=True)
link = rsoup.find('a', {'class': 'ipsButton_primary'}).get('href')
click.echo(click.style('Go to the suite: ', bold=True) + link + '\n')
# noinspection PyUnboundLocalVariable
def install(self):
"""
Run the actual installation
"""
self._start_install()
mr_link = self._get_mr_link()
# Set up the progress bar
pbar = ProgressBar(100, 'Running installation...')
pbar.start()
mr_j, mr_r = self._ajax(mr_link)
# Loop until we get a redirect json response
while True:
mr_link = self._parse_response(mr_link, mr_j)
stage = self._get_stage(mr_j)
progress = self._get_progress(mr_j)
mr_j, mr_r = self._ajax(mr_link)
pbar.update(
min([progress, 100]),
stage) # NOTE: Response may return progress values above 100
# If we're done, finalize the installation and break
redirect = self._check_if_complete(mr_link, mr_j)
if redirect:
pbar.finish()
break
p = Echo('Finalizing...')
mr_r = self._request(redirect, raise_request=False)
p.done()
# Install developer tools
if self.site.in_dev:
DevToolsInstaller(self.ctx, self.site).install()
# Get the link to our community homepage
self._finalize(mr_r)
cmssw2rank = {}
rank = 10
for release in release_list:
cmssw2rank[release] = rank
rank += 10
## Loop overall releases in Savannah and check if there are valid
## Hide deprecated versions
for links in br.links(text_regex="CMSSW_*_*"):
if links.text not in release_list:
br.follow_link(links)
print "Hide deprecated version %s" % links.text
br.select_form(nr=1)
control = br.find_control("status")
control.value = ['H']
br.submit()
## Doing a loop in all releases
## Create entry in Savannah for releases. If it already exist, update the rank
for release in release_list:
try:
#First, look if this CMSSW release is already there
br.follow_link(text=release, nr=0)
rank = cmssw2rank[release]
print("Release %s already exists. Changing rank to %s" %
(release, rank))
br.select_form(nr=1)
br['order_id'] = str(rank)
br['status'] = ['A']
class Bugzilla(object):
def __init__(self, dryrun=False, committers=CommitterList()):
self.dryrun = dryrun
self.authenticated = False
self.queries = BugzillaQueries(self)
self.committers = committers
# FIXME: We should use some sort of Browser mock object when in dryrun
# mode (to prevent any mistakes).
self.browser = Browser()
# Ignore bugs.webkit.org/robots.txt until we fix it to allow this
# script.
self.browser.set_handle_robots(False)
# FIXME: Much of this should go into some sort of config module:
bug_server_host = "bugs.webkit.org"
bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host)
bug_server_url = "https://%s/" % bug_server_host
unassigned_email = "*****@*****.**"
def bug_url_for_bug_id(self, bug_id, xml=False):
content_type = "&ctype=xml" if xml else ""
return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url,
bug_id,
content_type)
def short_bug_url_for_bug_id(self, bug_id):
return "http://webkit.org/b/%s" % bug_id
def attachment_url_for_id(self, attachment_id, action="view"):
action_param = ""
if action and action != "view":
action_param = "&action=%s" % action
return "%sattachment.cgi?id=%s%s" % (self.bug_server_url,
attachment_id,
action_param)
def _parse_attachment_flag(self,
element,
flag_name,
attachment,
result_key):
flag = element.find('flag', attrs={'name': flag_name})
if flag:
attachment[flag_name] = flag['status']
if flag['status'] == '+':
attachment[result_key] = flag['setter']
def _parse_attachment_element(self, element, bug_id):
attachment = {}
attachment['bug_id'] = bug_id
attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1")
attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1")
attachment['id'] = int(element.find('attachid').string)
# FIXME: No need to parse out the url here.
attachment['url'] = self.attachment_url_for_id(attachment['id'])
attachment['name'] = unicode(element.find('desc').string)
attachment['attacher_email'] = str(element.find('attacher').string)
attachment['type'] = str(element.find('type').string)
self._parse_attachment_flag(
element, 'review', attachment, 'reviewer_email')
self._parse_attachment_flag(
element, 'commit-queue', attachment, 'committer_email')
return attachment
def _parse_bug_page(self, page):
soup = BeautifulSoup(page)
bug = {}
bug["id"] = int(soup.find("bug_id").string)
bug["title"] = unicode(soup.find("short_desc").string)
bug["reporter_email"] = str(soup.find("reporter").string)
bug["assigned_to_email"] = str(soup.find("assigned_to").string)
bug["cc_emails"] = [str(element.string)
for element in soup.findAll('cc')]
bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')]
return bug
# Makes testing fetch_*_from_bug() possible until we have a better
# BugzillaNetwork abstration.
def _fetch_bug_page(self, bug_id):
bug_url = self.bug_url_for_bug_id(bug_id, xml=True)
log("Fetching: %s" % bug_url)
return self.browser.open(bug_url)
def fetch_bug_dictionary(self, bug_id):
return self._parse_bug_page(self._fetch_bug_page(bug_id))
# FIXME: A BugzillaCache object should provide all these fetch_ methods.
def fetch_bug(self, bug_id):
return Bug(self.fetch_bug_dictionary(bug_id), self)
def _parse_bug_id_from_attachment_page(self, page):
# The "Up" relation happens to point to the bug.
up_link = BeautifulSoup(page).find('link', rel='Up')
if not up_link:
# This attachment does not exist (or you don't have permissions to
# view it).
return None
match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href'])
return int(match.group('bug_id'))
def bug_id_for_attachment_id(self, attachment_id):
self.authenticate()
attachment_url = self.attachment_url_for_id(attachment_id, 'edit')
log("Fetching: %s" % attachment_url)
page = self.browser.open(attachment_url)
return self._parse_bug_id_from_attachment_page(page)
# FIXME: This should just return Attachment(id), which should be able to
# lazily fetch needed data.
def fetch_attachment(self, attachment_id):
# We could grab all the attachment details off of the attachment edit
# page but we already have working code to do so off of the bugs page,
# so re-use that.
bug_id = self.bug_id_for_attachment_id(attachment_id)
if not bug_id:
return None
attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True)
for attachment in attachments:
if attachment.id() == int(attachment_id):
return attachment
return None # This should never be hit.
def authenticate(self):
if self.authenticated:
return
if self.dryrun:
log("Skipping log in for dry run...")
self.authenticated = True
return
attempts = 0
while not self.authenticated:
attempts += 1
(username, password) = Credentials(
self.bug_server_host, git_prefix="bugzilla").read_credentials()
log("Logging in as %s..." % username)
self.browser.open(self.bug_server_url +
"index.cgi?GoAheadAndLogIn=1")
self.browser.select_form(name="login")
self.browser['Bugzilla_login'] = username
self.browser['Bugzilla_password'] = password
response = self.browser.submit()
match = re.search("<title>(.+?)</title>", response.read())
# If the resulting page has a title, and it contains the word
# "invalid" assume it's the login failure page.
if match and re.search("Invalid", match.group(1), re.IGNORECASE):
errorMessage = "Bugzilla login failed: %s" % match.group(1)
# raise an exception only if this was the last attempt
if attempts < 5:
log(errorMessage)
else:
raise Exception(errorMessage)
else:
self.authenticated = True
def _fill_attachment_form(self,
description,
patch_file_object,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False, bug_id=None):
self.browser['description'] = description
self.browser['ispatch'] = ("1",)
self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',)
if mark_for_landing:
self.browser['flag_type-3'] = ('+',)
elif mark_for_commit_queue:
self.browser['flag_type-3'] = ('?',)
else:
self.browser['flag_type-3'] = ('X',)
if bug_id:
patch_name = "bug-%s-%s.patch" % (bug_id, timestamp())
else:
patch_name ="%s.patch" % timestamp()
self.browser.add_file(patch_file_object,
"text/plain",
patch_name,
'data')
def add_patch_to_bug(self,
bug_id,
patch_file_object,
description,
comment_text=None,
mark_for_review=False,
mark_for_commit_queue=False,
mark_for_landing=False):
self.authenticate()
log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description,
self.bug_server_url,
bug_id))
if self.dryrun:
log(comment_text)
return
self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (
self.bug_server_url, bug_id))
self.browser.select_form(name="entryform")
self._fill_attachment_form(description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue,
mark_for_landing=mark_for_landing,
bug_id=bug_id)
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser.submit()
def prompt_for_component(self, components):
log("Please pick a component:")
i = 0
for name in components:
i += 1
log("%2d. %s" % (i, name))
result = int(User.prompt("Enter a number: ")) - 1
return components[result]
def _check_create_bug_response(self, response_html):
match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>",
response_html)
if match:
return match.group('bug_id')
match = re.search(
'<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">',
response_html,
re.DOTALL)
error_message = "FAIL"
if match:
text_lines = BeautifulSoup(
match.group('error_message')).findAll(text=True)
error_message = "\n" + '\n'.join(
[" " + line.strip()
for line in text_lines if line.strip()])
raise Exception("Bug not created: %s" % error_message)
def create_bug(self,
bug_title,
bug_description,
component=None,
patch_file_object=None,
patch_description=None,
cc=None,
mark_for_review=False,
mark_for_commit_queue=False):
self.authenticate()
log('Creating bug with title "%s"' % bug_title)
if self.dryrun:
log(bug_description)
return
self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit")
self.browser.select_form(name="Create")
component_items = self.browser.find_control('component').items
component_names = map(lambda item: item.name, component_items)
if not component:
component = "New Bugs"
if component not in component_names:
component = self.prompt_for_component(component_names)
self.browser['component'] = [component]
if cc:
self.browser['cc'] = cc
self.browser['short_desc'] = bug_title
self.browser['comment'] = bug_description
if patch_file_object:
self._fill_attachment_form(
patch_description,
patch_file_object,
mark_for_review=mark_for_review,
mark_for_commit_queue=mark_for_commit_queue)
response = self.browser.submit()
bug_id = self._check_create_bug_response(response.read())
log("Bug %s created." % bug_id)
log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id))
return bug_id
def _find_select_element_for_flag(self, flag_name):
# FIXME: This will break if we ever re-order attachment flags
if flag_name == "review":
return self.browser.find_control(type='select', nr=0)
if flag_name == "commit-queue":
return self.browser.find_control(type='select', nr=1)
raise Exception("Don't know how to find flag named \"%s\"" % flag_name)
def clear_attachment_flags(self,
attachment_id,
additional_comment_text=None):
self.authenticate()
comment_text = "Clearing flags on attachment: %s" % attachment_id
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
self.browser.submit()
def set_flag_on_attachment(self,
attachment_id,
flag_name,
flag_value,
comment_text,
additional_comment_text):
# FIXME: We need a way to test this function on a live bugzilla
# instance.
self.authenticate()
if additional_comment_text:
comment_text += "\n\n%s" % additional_comment_text
log(comment_text)
if self.dryrun:
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.set_value(comment_text, name='comment', nr=0)
self._find_select_element_for_flag(flag_name).value = (flag_value,)
self.browser.submit()
# FIXME: All of these bug editing methods have a ridiculous amount of
# copy/paste code.
def obsolete_attachment(self, attachment_id, comment_text=None):
self.authenticate()
log("Obsoleting attachment: %s" % attachment_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.attachment_url_for_id(attachment_id, 'edit'))
self.browser.select_form(nr=1)
self.browser.find_control('isobsolete').items[0].selected = True
# Also clear any review flag (to remove it from review/commit queues)
self._find_select_element_for_flag('review').value = ("X",)
self._find_select_element_for_flag('commit-queue').value = ("X",)
if comment_text:
log(comment_text)
# Bugzilla has two textareas named 'comment', one is somehow
# hidden. We want the first.
self.browser.set_value(comment_text, name='comment', nr=0)
self.browser.submit()
def add_cc_to_bug(self, bug_id, email_address_list):
self.authenticate()
log("Adding %s to the CC list for bug %s" % (email_address_list,
bug_id))
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["newcc"] = ", ".join(email_address_list)
self.browser.submit()
def post_comment_to_bug(self, bug_id, comment_text, cc=None):
self.authenticate()
log("Adding comment to bug %s" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
self.browser["comment"] = comment_text
if cc:
self.browser["newcc"] = ", ".join(cc)
self.browser.submit()
def close_bug_as_fixed(self, bug_id, comment_text=None):
self.authenticate()
log("Closing bug %s as fixed" % bug_id)
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser['comment'] = comment_text
self.browser['bug_status'] = ['RESOLVED']
self.browser['resolution'] = ['FIXED']
self.browser.submit()
def reassign_bug(self, bug_id, assignee, comment_text=None):
self.authenticate()
log("Assigning bug %s to %s" % (bug_id, assignee))
if self.dryrun:
log(comment_text)
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
if comment_text:
log(comment_text)
self.browser["comment"] = comment_text
self.browser["assigned_to"] = assignee
self.browser.submit()
def reopen_bug(self, bug_id, comment_text):
self.authenticate()
log("Re-opening bug %s" % bug_id)
# Bugzilla requires a comment when re-opening a bug, so we know it will
# never be None.
log(comment_text)
if self.dryrun:
return
self.browser.open(self.bug_url_for_bug_id(bug_id))
self.browser.select_form(name="changeform")
bug_status = self.browser.find_control("bug_status", type="select")
# This is a hack around the fact that ClientForm.ListControl seems to
# have no simpler way to ask if a control has an item named "REOPENED"
# without using exceptions for control flow.
possible_bug_statuses = map(lambda item: item.name, bug_status.items)
if "REOPENED" in possible_bug_statuses:
bug_status.value = ["REOPENED"]
else:
log("Did not reopen bug %s. " +
"It appears to already be open with status %s." % (
bug_id, bug_status.value))
self.browser['comment'] = comment_text
self.browser.submit()
class LEAMsite:
def __init__(self, site, user, passwd):
self.site = site
self.error = False
self.b = Browser()
self.b.set_handle_robots(False)
try:
self.b.open(site)
except urllib2.URLError:
self.error = True
return
try:
# try and log in from the portlet
self.b.select_form('loginform')
except:
# try logging in from the main login page
self.b.open('/'.join((site,"login_form")))
self.b.select_form(nr=1)
self.b['__ac_name'] = user
self.b['__ac_password'] = passwd
r = self.b.open(self.b.click())
# plone changes have rendered this inoperable
# capture the response and look in the content
# body tag has class with login failure
def checkURL(self, url):
"""Tries to open a URL and return true if successful (object exists)
or false if error occurs.
"""
try:
rsp = self.b.open(url)
except:
return False
return True
def getURL(self, url, data=None, filename=None):
"""Simple interface for retrieving the contents of a URL
and writing it to a file or returning it as stringIO.
"""
#sys.stderr.write('getURL %s\n' % url)
rsp = self.b.open(url, data)
if filename:
f = file("./Inputs/"+filename, 'wb') # always write to Input folder
f.write(rsp.read())
f.close()
return None
else:
return StringIO(rsp.read())
def putFileURL(self, filename, url,
fileobj=None, title=None, type='text/plain'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url,'createObject?type_name=File')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def getFile(self, url, filename=None):
""" getFile -- gets a file using standard download from Plone site
url: The URL is pointer to the file on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url,'at_download/file'))
if filename:
f = open(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
return rsp.read()
def saveFile(self, url, dir=".", at_download=False):
"""Reads the response from a URL and saves it to a local
file based on the name provided in the Content-Disposition
header.
The dir field specifies to the directory where the file will
be stored.
If the at_download flag is True then 'at_download/file' will
be appended the URL.
"""
if at_download:
rsp = self.b.open(url_join(url,'at_download/file'))
else:
rsp = self.b.open(url)
fname = get_filename(rsp)
if fname:
f = open('/'.join([dir, fname]), 'wb')
f.write(rsp.read())
f.close()
return fname
def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'):
sys.stderr.write('putImageURL %s to %s\n' % (imgfile,url))
self.b.open('/'.join((url,'createObject?type_name=Image')))
self.b.select_form("edit_form")
if not title: title = path.basename(imgfile)
self.b['title'] = title
self.b.add_file(open(imgfile), type, path.basename(imgfile))
try:
# doesn't seem necessary but it is required for file upload
form = self.b.find_control('image_delete')
form.value = ["",]
# print "This really does need to happen.."
except:
# print "That delete stuff never happens..."
pass
self.b.submit("form.button.save")
def putImageURL(self, filename, url,
fileobj=None, title=None, type='image/jpg'):
"""Simple interface for uploading a file to a plone site.
<URL> should be an existing folder on the site and
<filename> should be a readable file.
"""
#sys.stderr.write('putFileURL %s to %s\n' % (filename,url))
if not title: title = path.basename(filename)
if not fileobj: fileobj = open(filename, 'rb')
self.b.open('/'.join((url,'createObject?type_name=Image')))
self.b.select_form("edit_form")
self.b['title'] = title
self.b.add_file(fileobj, type, path.basename(filename))
# form = self.b.find_control('file_delete')
# form.value = ["",]
self.b.submit("form.button.save")
# r = self.b.open(self.b.click())
# should check that it worked
def putDocument(self, doc, url, title):
"""Creates a new document and add the doc (file-like object) to it."""
self.b.open('/'.join((url,'createObject?type_name=Document')))
self.b.select_form("edit_form")
self.b['title'] = title
doc.seek(0)
self.b['text'] = doc.read()
self.b.submit("form.button.save")
return self.b.geturl()
def getDocument(self, url):
"""Returns a string with the text of the current document."""
self.b.open('/'.join((url,'edit')))
self.b.select_form("edit_form")
s = self.b['text']
self.b.submit("form.button.cancel")
return s
def editDocument(self, doc, url, title=None):
"""Replaces the contents of a document"""
self.b.open('/'.join((url,'edit')))
self.b.select_form("edit_form")
# update the title if given
if title: self.b['title'] = title
# slightly dangerous approach where we seek first
# to test for a file-like object. If exception is
# thrown then assume doc is a string.
try:
doc.seek(0)
self.b['text'] = doc.read()
except:
self.b['text'] = doc
self.b.submit("form.button.save")
return self.b.geturl()
def createFolder(self, folder, url):
"""Creates a folder titled <folder> at the location <url> if
it doesn't already exist. Returns the full path of new folder.
"""
pathurl = '/'.join((url,folder.lower().replace(' ','-')))
print pathurl
try:
self.b.open(pathurl)
except:
self.b.open('/'.join((url,'createObject?type_name=Folder')))
self.b.select_form("edit_form")
self.b['title'] = folder
self.b.submit("form.button.save")
return self.b.geturl()
def editFolder(self, url, title="", description=""):
"""Edit the basic fields of the Folder. Mostly useful for
setting the title AFTER creating the folder with a reasonable
short name.
"""
try:
self.b.open(url)
except:
self.error = True
return None
self.b.open(url+'/edit')
self.b.select_form("edit_form")
if title:
self.b['title'] = title
if description:
self.b['description'] = description
self.b.submit("form.button.save")
return self.b.geturl()
def deleteFolder(self, url):
"Deletes folder and all of its contents"
sys.stderr.write('DELETING folder %s\n' % url)
return
# Puts SimMaps on to the site
def putSimMap(self, simmap, mapfile, url,
simmap_file=None, mapfile_file=None,
title=None, description=None,
trans=.7, details=None, zoom=8):
""" putSimMap
Required Input: simmap, mapfile, url
simmap is a file that contains the desired GIS layer
mapfile is the standard .map file that maps GIS layer to image
url - is the full URL to the folder where the SimMap will be stored
Optional Inputs: simmap_file, mapfile_file, title, trans, details
simmap_file - file-like object of the simmap if None simmap
will be openned and read.
mapfile_file - file-like object of the mapfile. If None mapfile
will be openned and read.
title - title sting of the SimMap defaults to basename(simmap)
trans - transparency level
details - description of SimMap as stored in the details field
"""
self.b.open('/'.join((url,'createObject?type_name=SimMap')))
self.b.select_form("edit_form")
if not simmap_file: simmap_file = open(simmap, 'rb')
if not mapfile_file: mapfile_file = open(mapfile, 'rb')
if not title: title = path.splitext(path.basename(simmap))[0]
self.b['title'] = str(title)
self.b.add_file(simmap_file, 'application/octet-stream',
path.basename(simmap), "simImage_file")
self.b.add_file(mapfile_file, 'application/octet-stream',
path.basename(mapfile), "mapFile_file")
if description: self.b['description'] = str(description)
self.b['transparency'] = str(trans)
self.b['zoom'] = str(zoom)
if details: self.b['details'] = str(details)
self.b.submit("form.button.save")
return self.b.geturl()
def getSimMap(self, url):
""" return the SimMap metadata
Gets the metadata associated with SimMap including title,
description, location, transparency, and zoom.
= """
self.b.open(url+'/edit')
self.b.select_form('edit_form')
d = dict(
title = self.b['title'],
description = self.b['description'],
details = self.b['details'],
location = self.b['location'],
transparency = self.b['Transparency'],
zoom = self.b['zoom'],
)
self.b.submit("form.button.cancel")
return d
def updateSimMap(self, url, **kwargs):
""" update the SimMap metadata
Keywords must match the field names from the edit form exactly,
extra keywords (or mispelled ones) will be ignored.
"""
self.b.open(url+'/edit')
self.b.select_form('edit_form')
for k in kwargs:
if k in self.b:
self.b[k] = str(kwargs[k])
self.b.submit("form.button.save")
def getSimMapData(self, url, filename=None):
""" getSimMapData -- gets the data component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
bufsize = 15 * 1024 * 1024
rsp = self.b.open(url_join(url,'at_download/simImage'))
if filename:
f = file(filename, 'wb')
while 1:
b = rsp.read(bufsize)
f.write(b)
if len(b) < bufsize: break
f.close()
return None
else:
return StringIO(rsp.read())
def getSimMapMapfile(self, url, filename=None):
""" getSimMapData -- gets the mapfile component of the the SimMap
url: The URL is pointer to the SimMap on the Plone site
filename: optional filename where data will be written
"""
rsp = self.b.open(url_join(url,'at_download/mapFile'))
if filename:
f = file(filename, 'wb')
f.write(rsp.read())
f.close()
return None
else:
data = StringIO()
data.write(rsp.read())
return data
def putProbmapCache(self, url, filename):
"""store precomputed probmaps as part of the Driver Set
This is a temporary method until the REST interface
is ready. It's really depricated before it written!
"""
self.b.open(url+'/edit')
self.b.select_form('edit_form')
with open(filename, 'rb') as f:
self.b.add_file(f, 'application/octet-stream',
path.basename(filename), name='probfile_file')
self.b.submit("form.button.save")
# def putAttrmapCache(self, url, filepathbasename, filename):
# """store precomputed maps as part of the Driver Set
# This is a temporary method until the REST interface
# is ready. It's really depricated before it written!
# """
# self.b.open(url+'/edit')
# self.b.select_form('edit_form')
# with open(filename, 'w') as f:
# self.b.add_file(f, 'application/octet-stream',
# path.basename(filepathbasename), name=filename)
# self.b.submit("form.button.save")
# DELETE FUNCTIONS -----------------------------------------
# These functions delete items from the LEAM Plone sites
# ----------------------------------------------------------
def deleteItem(self,fname,url):
""" deleteItem
Deletes <fname> from the folder <url>
"""
print '/'.join((url,fname,'delete_confirmation'))
print self.b.open('/'.join((url,fname,'delete_confirmation')))
self.b.select_form(None,None,1)
print self.b.submit()
def getSolutions(path_prefix, path_proxy):
global br, username, password
# create a browser object
br = Browser()
# add proxy support to browser
if len(path_proxy) != 0:
protocol, proxy = options.proxy.split("://")
br.set_proxies({protocol: proxy})
# let browser fool robots.txt
br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; \
rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')]
br.set_handle_robots(False)
print "Enter yout SPOJ username :"******"Authenticating " + username
br.open("http://spoj.pl")
br.select_form(name="login")
br["login_user"] = username
br["password"] = password
# sign in for a day to avoid timeouts
br.find_control(name="autologin").items[0].selected = True
br.form.action = "http://www.spoj.pl"
response = br.submit()
verify = response.read()
if (verify.find("Authentication failed!") != -1):
print "Error authenticating - " + username
exit(0)
# grab the signed submissions list
print "Grabbing siglist for " + username
siglist = br.open("http://www.spoj.pl/status/" + username + "/signedlist")
# dump first nine useless lines in signed list for formatting
for i in xrange(9):
siglist.readline()
# make a list of all AC's and challenges
print "Filtering siglist for AC/Challenge solutions..."
mysublist = list()
while True:
temp = siglist.readline()
if temp == '\------------------------------------------------------------------------------/\n':
# reached end of siglist
break
if not len(temp):
print "Reached EOF, siglist format has probably changed," + \
" contact author."
exit(1)
entry = [x.strip() for x in temp.split('|')]
if entry[4] == 'AC' or entry[4].isdigit():
mysublist.append(entry)
print "Done !!!"
return mysublist
class RequestQuery:
def __init__(self,config):
self.br=Browser()
self.config = config
# Initialise connections
self.mySiteDB = SiteDBJSON()
self.phedex = PhEDEx({"endpoint":"https://cmsweb.cern.ch/phedex/datasvc/json/prod/"}, "json")
self.dbsPhys01 = DbsApi(url = dbs_base_url+"phys01/DBSReader/")
self.dbsPhys02 = DbsApi(url = dbs_base_url+"phys02/DBSReader/")
self.dbsPhys03 = DbsApi(url = dbs_base_url+"phys03/DBSReader/")
def __del__(self):
self.br.close()
def login2Savannah(self):
"""
login2Savannah log into savannah with the given parameters in the config (username and password)
User must have admin priviledges for store results requests
"""
login_page='https://savannah.cern.ch/account/login.php?uri=%2F'
savannah_page='https://savannah.cern.ch/task/?group=cms-storeresults'
self.br.open(login_page)
## 'Search' form is form 0
## login form is form 1
self.br.select_form(nr=1)
username = self.config["SavannahUser"]
self.br['form_loginname']=username
self.br['form_pw']=self.config["SavannahPasswd"]
self.br.submit()
response = self.br.open(savannah_page)
# Check to see if login was successful
if not re.search('Logged in as ' + username, response.read()):
print('login unsuccessful, please check your username and password')
return False
else:
return True
def selectQueryForm(self,**kargs):
"""
selectQueryForm create the browser view to get all the store result tickets from savannah
"""
if self.isLoggedIn:
self.br.select_form(name="bug_form")
## Use right query form labelled Test
control = self.br.find_control("report_id",type="select")
for item in control.items:
if item.attrs['label'] == "Test":
control.value = [item.attrs['value']]
##select number of entries displayed per page
control = self.br.find_control("chunksz",type="text")
control.value = "150"
##check additional searching parameter
for arg in kargs:
if arg == "approval_status":
control = self.br.find_control("resolution_id",type="select")
for item in control.items:
if item.attrs['label'] == kargs[arg].strip():
control.value = [item.attrs['value']]
elif arg == "task_status":
control = self.br.find_control("status_id",type="select")
for item in control.items:
if item.attrs['label'] == kargs[arg].strip():
control.value = [item.attrs['value']]
elif arg == "team":
control = self.br.find_control("custom_sb5",type="select")
for item in control.items:
if item.attrs['label'] == kargs[arg].strip():
control.value = [item.attrs['value']]
response = self.br.submit()
response.read()
return
def getScramArchByCMSSW(self):
"""
Get from the list of available CMSSW releases
return a dictionary of ScramArchitecture by CMSSW
"""
# Set temporary conection to the server and get the response from cmstags
url = 'https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML'
br = Browser()
br.set_handle_robots(False)
response=br.open(url)
soup = BeautifulSoup(response.read())
# Dictionary form
# {'CMSSW_X_X_X':[slc5_amd64_gcc472], ... }
archByCmssw={}
# Fill the dictionary
for arch in soup.find_all('architecture'):
for cmssw in arch.find_all('project'):
# CMSSW release
cmsswLabel = cmssw.get('label').encode('ascii', 'ignore')
if cmsswLabel not in archByCmssw:
archByCmssw[cmsswLabel]=[]
# ScramArch related to this CMSSW release
archName = arch.get('name').encode('ascii', 'ignore')
archByCmssw[cmsswLabel].append(archName)
return archByCmssw
def createValueDicts(self):
"""
Init dictionaries by value/label:
- Releases by Value
- Physics group by value
- DBS url by value
- DBS rul by label
- Status of savannah request by value
- Status of savannah ticket by value (Open/Closed/Any)
"""
if self.isLoggedIn:
self.br.select_form(name="bug_form")
control = self.br.find_control("custom_sb2",type="select")
self.ReleaseByValueDict = self.getLabelByValueDict(control)
control = self.br.find_control("custom_sb3",type="select")
self.GroupByValueDict = self.getLabelByValueDict(control)
control = self.br.find_control("custom_sb4",type="select")
self.DBSByValueDict = self.getLabelByValueDict(control)
self.DBSByLabelDict = self.getValueByLabelDict(control)
control = self.br.find_control("resolution_id",type="select")
self.StatusByValueDict = self.getLabelByValueDict(control)
control = self.br.find_control("status_id",type="select")
self.TicketStatusByLabelDict = self.getValueByLabelDict(control)
return
def getDatasetOriginSites(self, dbs_url, data):
"""
Get the origin sites for each block of the dataset.
Return a list block origin sites.
"""
local_dbs = dbs_url.split('/')[5]
if local_dbs == 'phys01':
response = self.dbsPhys01.listBlocks(detail=True,dataset=data)
elif local_dbs == 'phys02':
response = self.dbsPhys02.listBlocks(detail=True,dataset=data)
elif local_dbs == 'phys03':
response = self.dbsPhys03.listBlocks(detail=True,dataset=data)
pnnList = set()
for block in response:
pnnList.add(block['origin_site_name'])
psnList = self.mySiteDB.PNNstoPSNs(pnnList)
return psnList, list(pnnList)
def phEDExNodetocmsName(self, nodeList):
"""
Convert PhEDEx node name list to cms names list
"""
names = []
for node in nodeList:
name = node.replace('_MSS',
'').replace('_Disk',
'').replace('_Buffer',
'').replace('_Export', '')
if name not in names:
names.append(name)
return names
def setGlobalTagFromOrigin(self, dbs_url,input_dataset):
"""
Get the global tag of the dataset from the source dbs url. If it is not set, then set global tag to 'UNKNOWN'
"""
globalTag = ""
local_dbs = dbs_url.split('/')[5]
if local_dbs == 'phys01':
response = self.dbsPhys01.listOutputConfigs(dataset=input_dataset)
elif local_dbs == 'phys02':
response = self.dbsPhys02.listOutputConfigs(dataset=input_dataset)
elif local_dbs == 'phys03':
response = self.dbsPhys03.listOutputConfigs(dataset=input_dataset)
globalTag = response[0]['global_tag']
# GlobalTag cannot be empty
if globalTag == '':
globalTag = 'UNKNOWN'
return globalTag
def isDataAtUrl(self, dbs_url,input_dataset):
"""
Returns True if the dataset is at the dbs url, if not returns False
"""
local_dbs = dbs_url.split('/')[5]
if local_dbs == 'phys01':
response = self.dbsPhys01.listDatasets(dataset=input_dataset)
elif local_dbs == 'phys02':
response = self.dbsPhys02.listDatasets(dataset=input_dataset)
elif local_dbs == 'phys03':
response = self.dbsPhys03.listDatasets(dataset=input_dataset)
# This means that the dataset is not at the url
if not response:
return False
else:
return True
def getLabelByValueDict(self, control):
"""
From control items, create a dictionary by values
"""
d = {}
for item in control.items:
value = item.attrs['value']
label = item.attrs['label']
d[value] = label
return d
def getValueByLabelDict(self, control):
"""
From control items, create a dictionary by labels
"""
d = {}
for item in control.items:
value = item.attrs['value']
label = item.attrs['label']
d[label] = value
return d
def getRequests(self,**kargs):
"""
getRequests Actually goes through all the savannah requests and create json files if the
ticket is not Closed and the status of the item is Done.
It also reports back the summary of the requests in savannah
"""
requests = []
# Open Browser and login into Savannah
self.br=Browser()
self.isLoggedIn = self.login2Savannah()
if self.isLoggedIn:
if not kargs:
self.selectQueryForm(approval_status='1',task_status='0')
else:
self.selectQueryForm(**kargs)
self.createValueDicts()
self.br.select_form(name="bug_form")
response = self.br.submit()
html_ouput = response.read()
scramArchByCMSSW = self.getScramArchByCMSSW()
self.nodeMappings = self.phedex.getNodeMap()
for link in self.br.links(text_regex="#[0-9]+"):
response = self.br.follow_link(link)
try:
## Get Information
self.br.select_form(name="item_form")
## remove leading   and # from task
task = link.text.replace('#','').decode('utf-8').strip()
print("Processing ticket: %s" % task)
## Get input dataset name
control = self.br.find_control("custom_tf1",type="text")
input_dataset = control.value
input_primary_dataset = input_dataset.split('/')[1].replace(' ','')
input_processed_dataset = input_dataset.split('/')[2].replace(' ','')
data_tier = input_dataset.split('/')[3].replace(' ','')
## Get DBS URL by Drop Down
control = self.br.find_control("custom_sb4",type="select")
dbs_url = self.DBSByValueDict[control.value[0]]
## Get DBS URL by text field (for old entries)
if dbs_url=='None':
control = self.br.find_control("custom_tf4",type="text")
dbs_url = control.value.replace(' ','')
else: # Transform input value to a valid DBS url
#dbs_url = "https://cmsweb.cern.ch/dbs/prod/"+dbs_url+"/DBSReader"
dbs_url = dbs_base_url+dbs_url+"/DBSReader"
## Get Release
control = self.br.find_control("custom_sb2",type="select")
release_id = control.value
## Get current request status
control = self.br.find_control("status_id",type="select")
request_status_id = control.value
RequestStatusByValueDict = self.getLabelByValueDict(control)
# close the request if deprecated release was used
try:
release = self.ReleaseByValueDict[release_id[0]]
except:
if len(self.ReleaseByValueDict)>0 and RequestStatusByValueDict[request_status_id[0]] != "Closed":
msg = "Your request is not valid anymore, since the given CMSSW release is deprecated. If your request should be still processed, please reopen the request and update the CMSSW release to a more recent *working* release.\n"
msg+= "\n"
msg+= "Thanks,\n"
msg+= "Your StoreResults team"
self.closeRequest(task,msg)
self.br.back()
print("I tried to Close ticket %s due to CMSSW not valid" % task)
continue
# close the request if release has not ScramArch match
if release not in scramArchByCMSSW:
if len(self.ReleaseByValueDict)>0 and RequestStatusByValueDict[request_status_id[0]] != "Closed":
msg = "Your request is not valid, there is no ScramArch match for the given CMSSW release.\n"
msg+= "If your request should be still processed, please reopen the request and update the CMSSW release according to: https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML \n"
msg+= "\n"
msg+= "Thanks,\n"
msg+= "Your StoreResults team"
self.closeRequest(task,msg)
self.br.back()
print("I tried to Close ticket %s due to ScramArch mismatch" % task)
continue
else:
index=len(scramArchByCMSSW[release])
scram_arch = scramArchByCMSSW[release][index-1]
# close the request if dataset is not at dbs url
try:
data_at_url = self.isDataAtUrl(dbs_url,input_dataset)
except:
print('I got an error trying to look for dataset %s at %s, please look at this ticket: %s' %(input_dataset,dbs_url,task))
continue
if not data_at_url:
msg = "Your request is not valid, I could not find the given dataset at %s\n" % dbs_url
msg+= "If your request should be still processed, please reopen the request and change DBS url properly \n"
msg+= "\n"
msg+= "Thanks,\n"
msg+= "Your StoreResults team"
self.closeRequest(task,msg)
self.br.back()
print("I tried to Close ticket %s, dataset is not at DBS url" % task)
continue
# Avoid not approved Tickets
#if not RequestStatusByValueDict[request_status_id[0]] == "Done":
# continue
## Get Physics Group
control = self.br.find_control("custom_sb3",type="select")
group_id = control.value[0]
group_squad = 'cms-storeresults-'+self.GroupByValueDict[group_id].replace("-","_").lower()
## Get Dataset Version
control = self.br.find_control("custom_tf3",type="text")
dataset_version = control.value.replace(' ','')
if dataset_version == "": dataset_version = '1'
## Get current status
control = self.br.find_control("resolution_id",type="select")
status_id = control.value
## Get assigned to
control = self.br.find_control("assigned_to",type="select")
AssignedToByValueDict = self.getLabelByValueDict(control)
assignedTo_id = control.value
##Assign task to the physics group squad
if AssignedToByValueDict[assignedTo_id[0]]!=group_squad:
assignedTo_id = [self.getValueByLabelDict(control)[group_squad]]
control.value = assignedTo_id
self.br.submit()
# Set default Adquisition Era for StoreResults
acquisitionEra = "StoreResults"
## Construction of the new dataset name (ProcessingString)
## remove leading hypernews or physics group name and StoreResults+Version
if input_processed_dataset.find(self.GroupByValueDict[group_id])==0:
new_dataset = input_processed_dataset.replace(self.GroupByValueDict[group_id],"",1)
else:
stripped_dataset = input_processed_dataset.split("-")[1:]
new_dataset = '_'.join(stripped_dataset)
except Exception as ex:
self.br.back()
print("There is a problem with this ticket %s, please have a look to the error:" % task)
print(str(ex))
print(traceback.format_exc())
continue
self.br.back()
# Get dataset site info:
psnList, pnnList = self.getDatasetOriginSites(dbs_url,input_dataset)
infoDict = {}
# Build store results json
# First add all the defaults values
infoDict["RequestType"] = "StoreResults"
infoDict["UnmergedLFNBase"] = "/store/unmerged"
infoDict["MergedLFNBase"] = "/store/results/" + self.GroupByValueDict[group_id].replace("-","_").lower()
infoDict["MinMergeSize"] = 1500000000
infoDict["MaxMergeSize"] = 5000000000
infoDict["MaxMergeEvents"] = 100000
infoDict["TimePerEvent"] = 40
infoDict["SizePerEvent"] = 512.0
infoDict["Memory"] = 2394
infoDict["CmsPath"] = "/uscmst1/prod/sw/cms"
infoDict["Group"] = "DATAOPS"
infoDict["DbsUrl"] = dbs_url
# Add all the information pulled from Savannah
infoDict["AcquisitionEra"] = acquisitionEra
infoDict["GlobalTag"] = self.setGlobalTagFromOrigin(dbs_url,input_dataset)
infoDict["DataTier"] = data_tier
infoDict["InputDataset"] = input_dataset
infoDict["ProcessingString"] = new_dataset
infoDict["CMSSWVersion"] = release
infoDict["ScramArch"] = scram_arch
infoDict["ProcessingVersion"] = dataset_version
infoDict["SiteWhitelist"] = psnList
# Create report for Migration2Global
report = {}
#Fill json file, if status is done
if self.StatusByValueDict[status_id[0]]=='Done' and RequestStatusByValueDict[request_status_id[0]] != "Closed":
self.writeJSONFile(task, infoDict)
report["json"] = 'y'
else:
report["json"] = 'n'
report["task"] = int(task)
report["InputDataset"] = input_dataset
report["ProcessingString"] = new_dataset
report["ticketStatus"] = self.StatusByValueDict[status_id[0]]
report["assignedTo"] = AssignedToByValueDict[assignedTo_id[0]]
report["localUrl"] = dbs_url
report["sites"] = psnList
report["pnns"] = pnnList
# if the request is closed, change the item status to report to Closed
if report["ticketStatus"] == "Done" and RequestStatusByValueDict[request_status_id[0]] == "Closed":
report["ticketStatus"] = "Closed"
requests.append(report)
# Print out report
self.printReport(requests)
# Close connections
self.br.close()
return requests
def closeRequest(self,task,msg):
"""
This close a specific savannag ticket
Insert a message in the ticket
"""
if self.isLoggedIn:
#self.createValueDicts()
response = self.br.open('https://savannah.cern.ch/task/?'+str(task))
html = response.read()
self.br.select_form(name="item_form")
control = self.br.find_control("status_id",type="select")
control.value = [self.TicketStatusByLabelDict["Closed"]]
#Put reason to the comment field
control = self.br.find_control("comment",type="textarea")
control.value = msg
#DBS Drop Down is a mandatory field, if set to None (for old requests), it is not possible to close the request
self.setDBSDropDown()
self.br.submit()
#remove JSON ticket
self.removeJSONFile(task)
self.br.back()
return
def setDBSDropDown(self):
## Get DBS URL by Drop Down
control = self.br.find_control("custom_sb4",type="select")
dbs_url = self.DBSByValueDict[control.value[0]]
## Get DBS URL by text field (for old entries)
if dbs_url=='None':
tmp = self.br.find_control("custom_tf4",type="text")
dbs_url = tmp.value.replace(' ','')
if dbs_url.find("phys01")!=-1:
control.value = [self.DBSByLabelDict["phys01"]]
elif dbs_url.find("phys02")!=-1:
control.value = [self.DBSByLabelDict["phys02"]]
elif dbs_url.find("phys03")!=-1:
control.value = [self.DBSByLabelDict["phys03"]]
else:
msg = 'DBS URL of the old request is neither phys01, phys02 nor phys03. Please, check!'
print(msg)
raise RuntimeError(msg)
return
def writeJSONFile(self, task, infoDict):
"""
This writes a JSON file at ComponentDir
"""
##check if file already exists
filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json'
if not os.access(filename,os.F_OK):
jsonfile = open(filename,'w')
request = {'createRequest':infoDict} ## CHECK THIS BEFORE FINISHING
jsonfile.write(json.dumps(request,sort_keys=True, indent=4))
jsonfile.close
return
def removeJSONFile(self,task):
"""
This removes the JSON file at ComponentDir if it was created
"""
filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json'
if os.access(filename,os.F_OK):
os.remove(filename)
return
def printReport(self, requests):
"""
Print out a report
"""
print("%20s %10s %5s %35s %10s %50s %50s" %( 'Savannah Ticket','Status','json','Assigned to','local DBS','Sites','pnns'))
print("%20s %10s %5s %35s %10s %50s %50s" %( '-'*20,'-'*10,'-'*5,'-'*35,'-'*10,'-'*50,'-'*50 ))
for report in requests:
json = report["json"]
ticket = report["task"]
status = report["ticketStatus"]
assigned = report["assignedTo"]
localUrl = report["localUrl"].split('/')[5]
site = ', '.join(report["sites"])
pnns = ', '.join(report["pnns"])
print("%20s %10s %5s %35s %10s %50s %50s" %(ticket,status,json,assigned,localUrl,site,pnns))
class Installer(object):
# License fields
FIELD_LICENSE_KEY = 'lkey'
# Server Detail fields
FIELD_SERVER_SQL_HOST = 'sql_host'
FIELD_SERVER_SQL_USER = '******'
FIELD_SERVER_SQL_PASS = '******'
FIELD_SERVER_SQL_DATABASE = 'sql_database'
# Admin fields
FIELD_ADMIN_USER = '******'
FIELD_ADMIN_PASS = '******'
FIELD_ADMIN_PASS_CONFIRM = 'admin_pass2'
FIELD_ADMIN_EMAIL = 'admin_email'
def __init__(self, ctx, site, force=False):
"""
Initialize a new Installer instance
@type ctx: ips_vagrant.cli.Context
@param site: The IPS Site we are installing
@type site: ips_vagrant.models.sites.Site
@param force: Overwrite existing files / databases
@type force: bool
"""
self.log = logging.getLogger('ipsv.installer')
self.ctx = ctx
self.force = force
self._previous_title = None
self.url = '{scheme}://{host}/admin/install'.format(
scheme='https' if site.ssl else 'http', host=site.domain.name
)
self.site = site
self.mysql = create_engine('mysql://*****:*****@localhost')
self._sessions = {}
self.cookiejar = cookiejar()
self.cookies = {cookie.name: cookie.value for cookie in self.cookiejar}
self.browser = Browser()
self.browser.set_cookiejar(self.cookiejar)
def _check_title(self, title):
"""
If we're on the same page, we got an error and need to raise an exception
@type title: str
@raise InstallationError: Title matches the previous page requests title (We're on the same page)
"""
self.log.info('Installation page loaded: %s', title)
if self._previous_title and title == self._previous_title:
raise InstallationError('Unexpected page error')
self._previous_title = title
def start(self):
"""
Start the installation wizard
"""
self.log.debug('Starting the installation process')
self.system_check()
def system_check(self):
"""
System requirements check
"""
self.browser.open(self.url)
self._check_title(self.browser.title())
p = Echo('Running system check...')
rsoup = BeautifulSoup(self.browser.response().read(), "html.parser")
# Check for any errors
errors = []
for ul in rsoup.find_all('ul', {'class': 'ipsList_checks'}):
for li in ul.find_all('li', {'class': 'fail'}):
errors.append(li.text)
if errors:
raise InstallationError(errors)
# Continue
continue_link = next(self.browser.links(text_regex='Continue'))
p.done()
self.browser.follow_link(continue_link)
self.license()
def license(self):
"""
Submit our license to IPS' servers
"""
p = Echo('Submitting license key...')
self._check_title(self.browser.title())
self.browser.select_form(nr=0)
# Set the fields
self.browser.form[self.FIELD_LICENSE_KEY] = '{license}-TESTINSTALL'.format(license=self.site.license_key)
self.browser.find_control('eula_checkbox').items[0].selected = True # TODO: User prompt?
# Submit the request
self.log.debug('Submitting our license')
self.browser.submit()
self.log.debug('Response code: %s', self.browser.response().code)
p.done()
self.applications()
def applications(self):
"""
Select the applications to install (currently hardcoded to install all applications)
"""
# Check for license submission errors
try:
self._check_title(self.browser.title())
except InstallationError:
rsoup = BeautifulSoup(self.browser.response().read(), "html.parser")
error = rsoup.find('li', id='license_lkey').find('span', {'class': 'ipsType_warning'}).text
raise InstallationError(error)
# TODO: Make this configurable
p = Echo('Setting applications to install...')
self.browser.select_form(nr=0)
self.browser.submit()
p.done()
self.server_details()
def server_details(self):
"""
Input server details (database information, etc.)
"""
self._check_title(self.browser.title())
p = Echo('Creating MySQL database...')
# Create the database
md5hex = md5(self.site.domain.name + self.site.slug).hexdigest()
db_name = 'ipsv_{md5}'.format(md5=md5hex)
# MySQL usernames are limited to 16 characters max
db_user = '******'.format(md5=md5hex[:11])
rand_pass = ''.join(random.SystemRandom()
.choice(string.ascii_letters + string.digits) for _ in range(random.randint(16, 24)))
db_pass = rand_pass
try:
self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name))
except SQLAlchemyError:
if not self.force:
click.confirm('A previous database for this installation already exists.\n'
'Would you like to drop it now? The installation will be aborted if you do not',
abort=True)
self.log.info('Dropping existing database: {db}'.format(db=db_name))
self.mysql.execute('DROP DATABASE IF EXISTS `{db}`'.format(db=db_name))
self.mysql.execute('DROP USER IF EXISTS `{u}`'.format(u=db_user))
self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name))
self.mysql.execute("GRANT ALL ON {db}.* TO '{u}'@'localhost' IDENTIFIED BY '{p}'"
.format(db=db_name, u=db_user, p=db_pass))
# Save the database connection information
self.site.db_host = 'localhost'
self.site.db_name = db_name
self.site.db_user = db_user
self.site.db_pass = db_pass
self.ctx.db.commit()
self.log.debug('MySQL Database Name: %s', db_name)
self.log.debug('MySQL Database User: %s', db_user)
self.log.debug('MySQL Database Password: %s', db_pass)
# Set form fields and submit
self.browser.select_form(nr=0)
self.browser.form[self.FIELD_SERVER_SQL_HOST] = 'localhost'
self.browser.form[self.FIELD_SERVER_SQL_USER] = db_user
self.browser.form[self.FIELD_SERVER_SQL_PASS] = db_pass
self.browser.form[self.FIELD_SERVER_SQL_DATABASE] = db_name
self.browser.submit()
p.done()
self.admin()
def admin(self):
"""
Provide admin login credentials
"""
self._check_title(self.browser.title())
self.browser.select_form(nr=0)
# Get the admin credentials
prompted = []
user = self.ctx.config.get('User', 'AdminUser')
if not user:
user = click.prompt('Admin display name')
prompted.append('user')
password = self.ctx.config.get('User', 'AdminPass')
if not password:
password = click.prompt('Admin password', hide_input=True, confirmation_prompt='Confirm admin password')
prompted.append('password')
email = self.ctx.config.get('User', 'AdminEmail')
if not email:
email = click.prompt('Admin email')
prompted.append('email')
self.browser.form[self.FIELD_ADMIN_USER] = user
self.browser.form[self.FIELD_ADMIN_PASS] = password
self.browser.form[self.FIELD_ADMIN_PASS_CONFIRM] = password
self.browser.form[self.FIELD_ADMIN_EMAIL] = email
p = Echo('Submitting admin information...')
self.browser.submit()
p.done()
if len(prompted) >= 3:
save = click.confirm('Would you like to save and use these admin credentials for future installations?')
if save:
self.log.info('Saving admin login credentials')
self.ctx.config.set('User', 'AdminUser', user)
self.ctx.config.set('User', 'AdminPass', password)
self.ctx.config.set('User', 'AdminEmail', email)
with open(self.ctx.config_path, 'wb') as cf:
self.ctx.config.write(cf)
self.install()
def _start_install(self):
"""
Start the installation
"""
self._check_title(self.browser.title())
continue_link = next(self.browser.links(text_regex='Start Installation'))
self.browser.follow_link(continue_link)
def _get_mr_link(self):
"""
Get the MultipleRedirect URL
@rtype: str
"""
rsoup = BeautifulSoup(self.browser.response().read(), "html.parser")
mr_link = rsoup.find('a', {'class': 'button'}).get('href')
self.log.debug('MultipleRedirect link: %s', mr_link)
return mr_link
def _ajax(self, url, method='get', params=None, load_json=True, raise_request=True):
"""
Perform an Ajax request
@type url: str
@type method: str
@type params: dict or None
@type load_json: bool
@return: Tuple with the decoded JSON response and actual response, or just the response if load_json is False
@rtype: requests.Response or tuple of (dict or list, requests.Response)
"""
if 'ajax' in self._sessions:
ajax = self._sessions['ajax']
else:
self.log.debug('Instantiating a new Ajax session')
ajax = requests.Session()
ajax.headers.update({'X-Requested-With': 'XMLHttpRequest'})
ajax.cookies.update(cookiejar())
ajax.verify = False
self._sessions['ajax'] = ajax
response = ajax.request(method, url, params)
self.log.debug('Ajax response: %s', response.text)
if raise_request:
response.raise_for_status()
if load_json:
return byteify(json.loads(response.text)), response
return response
def _request(self, url, method='get', params=None, raise_request=True):
"""
Perform a standard HTTP request
@type url: str
@type method: str
@type params: dict or None
@param raise_request: Raise exceptions for HTTP status errors
@type raise_request: bool
@rtype: requests.Response
"""
if 'http' in self._sessions:
http = self._sessions['http']
else:
self.log.debug('Instantiating a new HTTP session')
http = requests.Session()
http.cookies.update(cookiejar())
http.verify = False
self._sessions['http'] = http
response = http.request(method, url, params)
self.log.debug('HTTP response code: %s', response.status_code)
if raise_request:
response.raise_for_status()
return response
def _parse_response(self, url, json_response):
"""
Parse response data and return the next request URL
@type url: str
@type json_response: list or dict
@rtype: str
"""
parts = list(urlparse(url))
query = parse_qs(parts[4])
query['mr'] = str(json_response[0]).replace('\'', '"')
parts[4] = urlencode(query, True)
return urlunparse(parts)
def _get_stage(self, json_response):
"""
Get the current installation stage
@type json_response: list or dict
@rtype: str
"""
try:
return json_response[1]
except IndexError:
return 'Installation complete!'
def _get_progress(self, json_response):
"""
Get the current installation progress
@type json_response: list or dict
@rtype: str
"""
try:
return round(float(json_response[2]))
except IndexError:
return 0
def _check_if_complete(self, url, json_response):
"""
Check if a request has been completed and return the redirect URL if it has
@type url: str
@type json_response: list or dict
@rtype: str or bool
"""
if 'redirect' in json_response and isinstance(json_response, dict):
self.log.info('Installation complete')
return json_response['redirect']
return False
def _finalize(self, response):
"""
Finalize the installation and display a link to the suite
"""
rsoup = BeautifulSoup(response.text, "html.parser")
click.echo('------')
click.secho(rsoup.find('h1', id='elInstaller_welcome').text.strip(), fg='yellow', bold=True)
click.secho(rsoup.find('p', {'class': 'ipsType_light'}).text.strip(), fg='yellow', dim=True)
link = rsoup.find('a', {'class': 'ipsButton_primary'}).get('href')
click.echo(click.style('Go to the suite: ', bold=True) + link + '\n')
# noinspection PyUnboundLocalVariable
def install(self):
"""
Run the actual installation
"""
self._start_install()
mr_link = self._get_mr_link()
# Set up the progress bar
pbar = ProgressBar(100, 'Running installation...')
pbar.start()
mr_j, mr_r = self._ajax(mr_link)
# Loop until we get a redirect json response
while True:
mr_link = self._parse_response(mr_link, mr_j)
stage = self._get_stage(mr_j)
progress = self._get_progress(mr_j)
mr_j, mr_r = self._ajax(mr_link)
pbar.update(min([progress, 100]), stage) # NOTE: Response may return progress values above 100
# If we're done, finalize the installation and break
redirect = self._check_if_complete(mr_link, mr_j)
if redirect:
pbar.finish()
break
p = Echo('Finalizing...')
mr_r = self._request(redirect, raise_request=False)
p.done()
# Install developer tools
if self.site.in_dev:
DevToolsInstaller(self.ctx, self.site).install()
# Get the link to our community homepage
self._finalize(mr_r)
