def backup_database(url, username, password, database, destination): if not url.endswith("/"): url+="/" br = Browser() br.open(url) br.select_form(name="login_form") br["pma_username"] = username br["pma_password"] = password login_response = br.submit() resp = br.follow_link(url_regex=re.compile("^main\.php.*")) resp = br.follow_link(url_regex=re.compile("\./server_export\.php.*")) br.select_form(name="dump") # Select SQL export br.find_control(name="what").get(id="radio_plugin_sql").selected = True # Select database to export br.find_control(name="db_select[]").get(name=database).selected = True # Add SQL DROP statements to export br.find_control(name="sql_drop").get(id="checkbox_sql_drop").selected = True # Send as file br.find_control(name="asfile").get("sendit").selected = True # Compress file with bzip br.find_control(name="compression").get(id="radio_compression_bzip").selected = True ret = br.submit() open(destination, 'w').write(ret.read())
def do_processing(street1, street2, locationdetail, email, phone): br = Browser() #br.set_handle_robots(False) #br.set_handle_equiv(False) #br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')] br.open("https://secure.toronto.ca/webwizard/html/litter_bin_overflow.htm") br.select_form(nr=0) for i in range(0, len(br.find_control(type="checkbox").items)): br.find_control(type="checkbox").items[i].selected = True #checkbox_select(br) br.submit() #br.select_form(nr=0) #for i in range(0, len(br.find_control(type="checkbox").items)): # br.find_control(type="checkbox").items[i].selected =True checkbox_select(br) br.submit() #br.select_form(nr=0) #for i in range(0, len(br.find_control(type="checkbox").items)): # br.find_control(type="checkbox").items[i].selected =True checkbox_select(br) br["probCrossStreet1"] = street1 br["probCrossStreet2"] = street2 br["probLocationDetails"] = locationdetail br["ctctEmail"] = email br["ctctPhoneNumb"] = str(phone) br.submit() br.select_form(nr=0) br["additional_information"] = "Reported by the waterfront BIA" br.submit() br.select_form(nr=0) response = br.submit() return (response.read())
def download(): b=Browser() b.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')] r1=b.open(URL) b.select_form(nr=0) b.set_all_readonly(False) b["__EVENTTARGET"] = "ctl00$CPH1$dgMain" b["__EVENTARGUMENT"] = "Page$2" b.find_control("ctl00$CPH1$tbSearch").disabled=True r2=b.submit() return [r1,r2]
def test_login(login_url, studentid, username, password): #create a browser object br = Browser() #br.set_handle_redirect(False) br.set_handle_robots(False) br.addheaders = [( 'User-agent', 'Mozilla/5.0 (Windows NT 5.2; ' 'WOW64) AppleWebKit/536.11 (KHTML, like Gecko) ' ' Chrome/20.0.1132.47 Safari/536.11')] #open the login page br.open(login_url) br.select_form(nr=0) #read only? try me. br.find_control("credentialsForm2_hf_0").readonly = False #whoa. via http://stackoverflow.com/a/1549036/561698 br.form.new_control('text', 'username', {'value': ''}) br.form.new_control('text', 'password', {'value': ''}) br.form.fixup() br['username'] = username br['password'] = password outcome = None while True: try: br.submit() #print br.title() if br.title() == 'Fastt Math NG Stretch': outcome = 'PASSED' else: outcome = 'FAILED (no error)' break except HTTPError: #print "Got error code", e.code outcome = 'FAILED' + ' (u:' + username + '/p: ' + password + ')' break #return a dictionary with test results resp_dict = { 'product': 'FASTT Math', 'studentid': studentid, 'tested_on': datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), 'outcome': outcome } return resp_dict
def yapistir(): br = Browser() br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0')] br.set_handle_robots(False) br.open("http://paste.ubuntu.com") br.select_form("pasteform") br['poster'] = coder br.find_control(name="syntax").value = ["python"] dosya_ac = open(dosya) kodlar = dosya_ac.read() br['content'] = kodlar br.submit() for link in br.links(): k_link.append(link.url)
def yapistir(): br = Browser() br.addheaders = [( 'User-agent', 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0' )] br.set_handle_robots(False) br.open("http://paste.ubuntu.com") br.select_form("pasteform") br['poster'] = coder br.find_control(name="syntax").value = ["python"] dosya_ac = open(dosya) kodlar = dosya_ac.read() br['content'] = kodlar br.submit() for link in br.links(): k_link.append(link.url)
'Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1') ] # define target URL url = "http://www.bseindia.com/getquote.htm" br = Browser() # browser parameters br.set_handle_equiv(True) br.set_handle_gzip(True) br.set_handle_redirect(True) br.set_handle_referer(True) br.set_handle_robots(False) br.addheaders = headers # make request main_page = br.open(url) # select the default form br.select_form(nr=0) br.find_control(id="suggestBoxEQ").value = "CAREERP" # submit form br.submit() page_response = bs(br.response().read()) quote = page_response.find_all('div', {'class': 'newsrheadingdiv'}) print quote
def getSolutions (path_prefix, path_proxy): global br, username, password # create a browser object br = Browser() # add proxy support to browser if len(path_proxy) != 0: protocol,proxy = options.proxy.split("://") br.set_proxies({protocol:proxy}) # let browser fool robots.txt br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; \ rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')] br.set_handle_robots(False) print "Enter yout SPOJ username :"******"Authenticating " + username br.open ("http://spoj.pl") br.select_form (name="login") br["login_user"] = username br["password"] = password # sign in for a day to avoid timeouts br.find_control(name="autologin").items[0].selected = True br.form.action = "http://www.spoj.pl" response = br.submit() verify = response.read() if (verify.find("Authentication failed!") != -1): print "Error authenticating - " + username exit(0) # grab the signed submissions list print "Grabbing siglist for " + username siglist = br.open("http://www.spoj.pl/status/" + username + "/signedlist") # dump first nine useless lines in signed list for formatting for i in xrange(9): siglist.readline() # make a list of all AC's and challenges print "Filtering siglist for AC/Challenge solutions..." mysublist = list() while True: temp = siglist.readline() if temp=='\------------------------------------------------------------------------------/\n': # reached end of siglist break if not len(temp) : print "Reached EOF, siglist format has probably changed," + \ " contact author." exit(1) entry = [x.strip() for x in temp.split('|')] if entry[4] == 'AC' or entry[4].isdigit(): mysublist.append (entry) print "Done !!!" return mysublist
class CoreEmulator(Emulator): def __init__(self, username, password): super(CoreEmulator, self).__init__(username, password) self.setup_emulator() def setup_emulator(self): self.browser = Browser() self.browser.set_handle_robots(False) self.browser.addheaders = moodle.USER_AGENT self.cookiejar = CookieJar() self.browser.set_cookiejar(self.cookiejar) def session_expired(self): return self.browser.geturl().endswith(moodle.LOGIN_LOCATION) @throws_moodlefuse_error(exception.LoginException) def login(self): self.open_login_page(self.browser.open) self.browser.select_form( predicate=lambda form: form.attrs.get('id') == attributes.LOGIN ) self.browser.form.set_value(self.username, name='username') self.browser.form.set_value(self.password, name='password') resp = self.browser.submit() if resp.geturl().endswith(moodle.LOGIN_LOCATION): raise Exception @throws_moodlefuse_error(resource_errors.UnableToDownloadResource) def download(self, destination, source): source = str(source) if not source.startswith('http://') and not source.startswith('file://'): source = config['TEST_DATA'] + '/' + source self.browser.retrieve(source, destination) def open_link(self, url): response = self.browser.open(url) return BeautifulSoup(response.read()) def check_form_checkbox(self, checkboxname): self.browser.find_control(checkboxname).items[0].selected = True def uncheck_form_checkbox(self, checkboxname): self.browser.find_control(checkboxname).items[0].selected = False def add_form_content(self, inputname, content): self.browser.form.set_value(content, name=inputname) def close_form(self): self.browser.submit() def set_form_to_first_form(self): self.browser.select_form(nr=0) def set_form_to_form_with_control_value(self, value): for form in self.browser.forms(): for control in form.controls: if control.value == value: self.browser.form = form @throws_moodlefuse_error(exception.UnableToToggleEditing) def turn_course_editing_on(self): self.set_form_to_form_with_control_value(moodle.EDIT_ON_MOODLE_BUTTON_TEXT) response = self.browser.submit() return BeautifulSoup(response.read()) def _setup_assignments_for_parsing(self, submission_filter): self.set_form_to_form_with_control_value('Save and update table') self.browser.form["filter"] = [submission_filter] self.browser.form["perpage"] = ["100"] self.uncheck_form_checkbox('quickgrading') response = self.browser.submit() return BeautifulSoup(response.read()) def filter_assignment_submissions(self): return self._setup_assignments_for_parsing("submitted") def unfilter_assignment_submissions(self): return self._setup_assignments_for_parsing("") @throws_moodlefuse_error(exception.UnableToToggleEditing) def turn_course_editing_off(self): self.set_form_to_form_with_control_value(moodle.EDIT_OFF_MOODLE_BUTTON_TEXT) response = self.browser.submit() return BeautifulSoup(response.read()) @throws_moodlefuse_error(course_errors.InvalidMoodleIndex) def get_courses(self): return self.open_link(config['MOODLE_INDEX_ADDRESS']) @throws_moodlefuse_error(course_errors.UnableToObtainCategoryList) def get_course_categories(self, url): return self.open_link(url) @throws_moodlefuse_error(resource_errors.UnableToObtainResourceList) def get_course_resource_names(self, url): return self.open_link(url) def close(self): self.browser.close()
br.form.find_control("dummy3").readonly=False #så att vi kan manipulera värdena br.form.find_control("dummy2").readonly=False br.form.find_control("dummy1").readonly=False br.form["dummy1"]="3" br.form["dummy2"]="filter" br.form["dummy3"]= "akt" #detta är för att bara få aktiva AB. De övriga två kontrollerna vet jag inte exakt vad de gör... print br.form print "nu är vi här" response7 = br.submit() # 26 april: skumt, detta fungerar om jag inte tidigare preciserat vilket län jag vill söka i. Med länet utvalt får jag däremot noll träffar. Torde ha något att göra med vad br.form har för värden på sina olika attribut, men vad? br.select_form(name="mainform") dagens_org_nr = str(br.find_control("uss")) #här går vi "bakvägen" genom att extrahera strängen som innehåller samtliga org-nr för bolag med kunggörelser i dag. #print br.form dagens_org_nr=dagens_org_nr[30:-15] #en enda lång sträng med de orgnr som har kunggörelser i dag. Denna sträng stoppar vi tillbaka i formuläret lite längre ner. print "dagens org_nr:"+dagens_org_nr print "längd dagens org nr:" + str(len(dagens_org_nr)) print br #print response7.read() response8 = br.follow_link(text="F\xf6retag") #print br br.select_form(name="mainform") print "mainform, rad 129" print br.form #br.form.find_control("limit").readonly = False #sätt till False så att vi kan ändra värdet
br.open(url) br.select_form(nr=0) br["login"] = args.username br["password"] = args.password if args.debug: print("Logging in . . .") br.submit() writeout("past_login", br) if args.debug: print("Getting search page . . .") br.open("https://hrsa.cunyfirst.cuny.edu/psc/cnyhcprd/EMPLOYEE/HRMS/c/SA_LEARNER_SERVICES.CLASS_SEARCH.GBL") writeout("on_search_form", br) br.select_form(nr=0) # How do I change this next semester? mods = br.find_control('CLASS_SRCH_WRK2_STRM$45$') item1 = Item(mods, {"contents": "1162", "value": "1162"}) item1.selected=True mods = br.find_control('SSR_CLSRCH_WRK_SUBJECT_SRCH$0') item2 = Item(mods, {"contents": args.subject, "value": args.subject}) item2.selected=True # Next semester? br['CLASS_SRCH_WRK2_STRM$45$']=['1162'] br['DERIVED_SSTSNAV_SSTS_MAIN_GOTO$155$']=['9999'] br['CLASS_SRCH_WRK2_INSTITUTION$42$']=[args.college + '01'] br['SSR_CLSRCH_WRK_SSR_EXACT_MATCH1$1']=['E'] br.form['SSR_CLSRCH_WRK_CATALOG_NBR$1']=args.klass br['SSR_CLSRCH_WRK_SUBJECT_SRCH$0']=[args.subject] writeout("without_ICAction", br)
class RequestQuery: def __init__(self,config): self.br=Browser() self.config = config self.isLoggedIn = self.login2Savannah() def __del__(self): self.br.close() def closeRequest(self,task,msg): if self.isLoggedIn: self.createValueDicts() response = self.br.open('https://savannah.cern.ch/task/?'+str(task)) html = response.read() self.br.select_form(name="item_form") control = self.br.find_control("status_id",type="select") control.value = [self.TicketStatusByLabelDict["Closed"]] #Put reason to the comment field control = self.br.find_control("comment",type="textarea") control.value = msg #DBS Drop Down is a mandatory field, if set to None (for old requests), it is not possible to close the request self.setDBSDropDown() self.br.submit() #remove JSON ticket self.removeJSONFile(task) return def createValueDicts(self): if self.isLoggedIn: self.br.select_form(name="bug_form") control = self.br.find_control("custom_sb2",type="select") self.ReleaseByValueDict = self.getLabelByValueDict(control) control = self.br.find_control("custom_sb3",type="select") self.GroupByValueDict = self.getLabelByValueDict(control) control = self.br.find_control("custom_sb4",type="select") self.DBSByValueDict = self.getLabelByValueDict(control) self.DBSByLabelDict = self.getValueByLabelDict(control) control = self.br.find_control("resolution_id",type="select") self.StatusByValueDict = self.getLabelByValueDict(control) control = self.br.find_control("status_id",type="select") self.TicketStatusByLabelDict = self.getValueByLabelDict(control) return def setDBSDropDown(self): ## Get DBS URL by Drop Down control = self.br.find_control("custom_sb4",type="select") dbs_url = self.DBSByValueDict[control.value[0]] ## Get DBS URL by text field (for old entries) if dbs_url=='None': tmp = self.br.find_control("custom_tf4",type="text") dbs_url = tmp.value.replace(' ','') if dbs_url.find("analysis_02")!=-1: control.value = [self.DBSByLabelDict["cms_dbs_ph_analysis_02"]] elif dbs_url.find("analysis_01")!=-1: control.value = [self.DBSByLabelDict["cms_dbs_ph_analysis_01"]] elif dbs_url.find("local_09")!=-1: control.value = [self.DBSByLabelDict["cms_dbs_ph_prod_local_09"]] else: msg = 'DBS URL of the old request is neither analysis_01, analysis_02 nor local_09. Please, check!' logging.error(msg) raise RuntimeError, msg return def getLabelByValueDict(self, control): d = {} for item in control.items: value = item.attrs['value'] label = item.attrs['label'] d[value] = label return d def getRequests(self,**kargs): requests = [] if self.isLoggedIn: self.selectQueryForm(**kargs) self.createValueDicts() self.br.select_form(name="bug_form") response = self.br.submit() html_ouput = response.read() for link in self.br.links(text_regex="#[0-9]+"): response = self.br.follow_link(link) ## Get Information self.br.select_form(name="item_form") ## Get input dataset name control = self.br.find_control("custom_tf1",type="text") input_dataset = control.value.split('/') input_primary_dataset = input_dataset[1].replace(' ','') input_processed_dataset = input_dataset[2].replace(' ','') ## Get DBS URL by Drop Down control = self.br.find_control("custom_sb4",type="select") dbs_url = self.DBSByValueDict[control.value[0]] ## Get DBS URL by text field (for old entries) if dbs_url=='None': control = self.br.find_control("custom_tf4",type="text") dbs_url = control.value.replace(' ','') else: # Transform input value to a valid DBS url dbs_url = "http://cmsdbsprod.cern.ch/"+dbs_url+"/servlet/DBSServlet" ## Get Release control = self.br.find_control("custom_sb2",type="select") release_id = control.value ## Get Physics Group control = self.br.find_control("custom_sb3",type="select") group_id = control.value[0] group_squad = 'cms-storeresults-'+self.GroupByValueDict[group_id].replace("-","_").lower() ## Get Dataset Version control = self.br.find_control("custom_tf3",type="text") dataset_version = control.value.replace(' ','') ## Get current status control = self.br.find_control("resolution_id",type="select") status_id = control.value ## Get current request status control = self.br.find_control("status_id",type="select") request_status_id = control.value RequestStatusByValueDict = self.getLabelByValueDict(control) ## Get assigned to control = self.br.find_control("assigned_to",type="select") AssignedToByValueDict = self.getLabelByValueDict(control) assignedTo_id = control.value ##Assign task to the physics group squad if AssignedToByValueDict[assignedTo_id[0]]!=group_squad: control.value = [self.getValueByLabelDict(control)[group_squad]] self.br.submit() ## Construction of the new dataset name ## remove leading hypernews or physics group name and StoreResults+Version if len(dataset_version)>0: dataset_prefix = "StoreResults-"+dataset_version else: dataset_prefix = "StoreResults" if input_processed_dataset.find(self.GroupByValueDict[group_id])==0: new_dataset = input_processed_dataset.replace(self.GroupByValueDict[group_id],dataset_prefix,1) else: stripped_dataset = input_processed_dataset.split("-")[1:] new_dataset = dataset_prefix+'-'+'-'.join(stripped_dataset) self.br.back() ## remove leading   and # from task task = link.text.replace('#','').decode('utf-8').strip() infoDict = {} infoDict["primaryDataset"] = input_primary_dataset infoDict["processedDataset"] = input_processed_dataset infoDict["outputDataset"] = new_dataset infoDict["physicsGroup"] = self.GroupByValueDict[group_id] infoDict["inputDBSURL"] = dbs_url # close the request if deprecated release was used try: infoDict["cmsswRelease"] = self.ReleaseByValueDict[release_id[0]] except: if len(self.ReleaseByValueDict)>0 and RequestStatusByValueDict[request_status_id[0]] != "Closed": msg = "Your request is not valid anymore, since the given CMSSW release is deprecated. If your request should be still processed, please reopen the request and update the CMSSW release to a more recent *working* release.\n" msg+= "\n" msg+= "Thanks,\n" msg+= "Your StoreResults team" self.closeRequest(task,msg) #Fill json file, if status is done if self.StatusByValueDict[status_id[0]]=='Done' and RequestStatusByValueDict[request_status_id[0]] != "Closed": self.writeJSONFile(task, infoDict) infoDict["task"] = int(task) infoDict["ticketStatus"] = self.StatusByValueDict[status_id[0]] infoDict["assignedTo"] = AssignedToByValueDict[assignedTo_id[0]] if infoDict["ticketStatus"] == "Done" and RequestStatusByValueDict[request_status_id[0]] == "Closed": infoDict["ticketStatus"] = "Closed" requests.append(infoDict) return requests def getValueByLabelDict(self, control): d = {} for item in control.items: value = item.attrs['value'] label = item.attrs['label'] d[label] = value return d def login2Savannah(self): login_page='https://savannah.cern.ch/account/login.php?uri=%2F' savannah_page='https://savannah.cern.ch/task/?group=cms-storeresults' self.br.open(login_page) ## 'Search' form is form 0 ## login form is form 1 self.br.select_form(nr=1) username = self.config["SavannahUser"] self.br['form_loginname']=username self.br['form_pw']=self.config["SavannahPasswd"] self.br.submit() response = self.br.open(savannah_page) # Check to see if login was successful if not re.search('Logged in as ' + username, response.read()): logging.error('login unsuccessful, please check your username and password') return False else: return True def selectQueryForm(self,**kargs): if self.isLoggedIn: self.br.select_form(name="bug_form") ## Use right query form labelled Test control = self.br.find_control("report_id",type="select") for item in control.items: if item.attrs['label'] == "Test": control.value = [item.attrs['value']] ##select number of entries displayed per page control = self.br.find_control("chunksz",type="text") control.value = "150" ##check additional searching parameter for arg in kargs: if arg == "approval_status": control = self.br.find_control("resolution_id",type="select") for item in control.items: if item.attrs['label'] == kargs[arg].strip(): control.value = [item.attrs['value']] elif arg == "task_status": control = self.br.find_control("status_id",type="select") for item in control.items: if item.attrs['label'] == kargs[arg].strip(): control.value = [item.attrs['value']] elif arg == "team": control = self.br.find_control("custom_sb5",type="select") for item in control.items: if item.attrs['label'] == kargs[arg].strip(): control.value = [item.attrs['value']] response = self.br.submit() response.read() return def removeJSONFile(self,task): filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json' if os.access(filename,os.F_OK): os.remove(filename) return def writeJSONFile(self, task, infoDict): ##check if file already exists filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json' if not os.access(filename,os.F_OK): jsonfile = open(filename,'w') jsonfile.write(json.dumps(infoDict,sort_keys=True, indent=4)) jsonfile.close return
class Bugzilla: def __init__(self, dryrun=False): self.dryrun = dryrun self.authenticated = False self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script self.browser.set_handle_robots(False) # Defaults (until we support better option parsing): bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host # This could eventually be a text file reviewer_usernames_to_full_names = { "abarth": "Adam Barth", "adele": "Adele Peterson", "aroben": "Adam Roben", "ap": "Alexey Proskuryakov", "ariya.hidayat": "Ariya Hidayat", "barraclough": "Gavin Barraclough", "beidson": "Brady Eidson", "darin": "Darin Adler", "ddkilzer": "David Kilzer", "dglazkov": "Dimitri Glazkov", "eric": "Eric Seidel", "fishd": "Darin Fisher", "gns": "Gustavo Noronha", "hausmann": "Simon Hausmann", "hyatt": "David Hyatt", "jmalonzo": "Jan Alonzo", "justin.garcia": "Justin Garcia", "kevino": "Kevin Ollivier", "koivisto": "Antti Koivisto", "levin": "David Levin", "mitz": "Dan Bernstein", "mjs": "Maciej Stachowiak", "mrowe": "Mark Rowe", "oliver": "Oliver Hunt", "sam": "Sam Weinig", "simon.fraser": "Simon Fraser", "staikos": "George Staikos", "timothy": "Timothy Hatcher", "treat": "Adam Treat", "vestbo": u'Tor Arne Vestb\xf8', "xan.lopez": "Xan Lopez", "zecke": "Holger Freyther", "zimmermann": "Nikolas Zimmermann", } def full_name_from_bugzilla_name(self, bugzilla_name): if not bugzilla_name in self.reviewer_usernames_to_full_names: raise Exception("ERROR: Unknown reviewer! " + bugzilla_name) return self.reviewer_usernames_to_full_names[bugzilla_name] def bug_url_for_bug_id(self, bug_id, xml=False): content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def attachment_url_for_id(self, attachment_id, action="view"): action_param = "" if action and action != "view": action_param = "&action=" + action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def fetch_attachments_from_bug(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: " + bug_url) page = urllib2.urlopen(bug_url) soup = BeautifulSoup(page) attachments = [] for element in soup.findAll('attachment'): attachment = {} attachment['bug_id'] = bug_id attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") attachment['id'] = str(element.find('attachid').string) attachment['url'] = self.attachment_url_for_id(attachment['id']) attachment['name'] = unicode(element.find('desc').string) attachment['type'] = str(element.find('type').string) review_flag = element.find('flag', attrs={"name": "review"}) if review_flag and review_flag['status'] == '+': reviewer_email = review_flag['setter'] # We could lookup the full email address instead once we update full_name_from_bugzilla_name bugzilla_name = reviewer_email.split('@')[0] attachment['reviewer'] = self.full_name_from_bugzilla_name( bugzilla_name) attachments.append(attachment) return attachments def fetch_patches_from_bug(self, bug_id): patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if attachment['is_patch'] and not attachment['is_obsolete']: patches.append(attachment) return patches def fetch_reviewed_patches_from_bug(self, bug_id): reviewed_patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if 'reviewer' in attachment and not attachment['is_obsolete']: reviewed_patches.append(attachment) return reviewed_patches def fetch_bug_ids_from_commit_queue(self): # FIXME: We should have an option for restricting the search by email. Example: # unassigned_only = "&emailassigned_to1=1&emailtype1=substring&email1=unassigned" commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B" log("Loading commit queue") page = urllib2.urlopen(commit_queue_url) soup = BeautifulSoup(page) bug_ids = [] # Grab the cells in the first column (which happens to be the bug ids) for bug_link_cell in soup( 'td', "first-child"): # tds with the class "first-child" bug_link = bug_link_cell.find("a") bug_ids.append( bug_link.string) # the contents happen to be the bug id return bug_ids def fetch_patches_from_commit_queue(self): patches_to_land = [] for bug_id in self.fetch_bug_ids_from_commit_queue(): patches = self.fetch_reviewed_patches_from_bug(bug_id) patches_to_land += patches return patches_to_land def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return (username, password) = read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser['Bugzilla_login'] = username self.browser['Bugzilla_password'] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): # FIXME: We could add the ability to try again on failure. error("Bugzilla login failed: %s" % match.group(1)) self.authenticated = True def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False): self.authenticate() log('Adding patch "%s" to bug %s' % (description, bug_id)) if self.dryrun: log(comment_text) return self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id) self.browser.select_form(name="entryform") self.browser['description'] = description self.browser['ispatch'] = ("1", ) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['flag_type-1'] = ('?', ) if mark_for_review else ('X', ) self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp())) self.browser.submit() def obsolete_attachment(self, attachment_id, comment_text=None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.find_control('isobsolete').items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self.browser.find_control(type='select', nr=0).value = ("X", ) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first. self.browser.set_value(comment_text, name='comment', nr=0) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser['comment'] = comment_text self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['bug_status'] = ['RESOLVED'] self.browser['resolution'] = ['FIXED'] self.browser.submit()
cmssw2rank = {} rank = 10 for release in release_list: cmssw2rank[release] = rank rank += 10 ## Loop overall releases in Savannah and check if there are valid ## Hide deprecated versions for links in br.links(text_regex="CMSSW_*_*"): if links.text not in release_list: br.follow_link(links) print "Hide deprecated version %s" % links.text br.select_form(nr=1) control = br.find_control("status") control.value = ['H'] br.submit() ## Doing a loop in all releases ## Create entry in Savannah for releases. If it already exist, update the rank for release in release_list: try: #First, look if this CMSSW release is already there br.follow_link(text=release, nr=0) rank = cmssw2rank[release] print("Release %s already exists. Changing rank to %s" %(release,rank)) br.select_form(nr=1) br['order_id']= str(rank) br['status']=['A'] br.submit()
br.set_handle_robots(False) br.open('https://m.facebook.com/login.php?next=https://m.facebook.com/settings/security/?active_sessions') br._factory.is_html = True br.select_form(nr=0) br.form["email"] = USER br.form["pass"] = PASSWD br.submit() # Infinite loop checking for unauthorized sessions while True: time.sleep(TIME) br.open("https://m.facebook.com/settings/security/?active_sessions") br._factory.is_html = True br.select_form(nr=1) try: for i in range(0, len(br.find_control(type="checkbox").items)): br.find_control(type="checkbox").items[i].selected = True sessions_detected = True except: print '[+] No active sessions' sessions_detected = False if sessions_detected == True: print '[+] Active sessions detected. Checking Latch status...' status = api.status(accountId) statusData = status.get_data() try: if(statusData['operations'][APP_ID]['status'] == 'off'): print '[!] INTRUDER' br.submit() else:
import pandas as pd a=pd.read_csv('Data/readinPy.csv') from mechanize import Browser, HTTPError import time for i in range(0,636): br = Browser() kinase = a.siRNA[i] print i, kinase #Status Check br.open('http://www.drugkinet.ca/KinaseCompoundQuery.aspx') br.select_form(nr=0) br['ctl00$ContentPlaceHolder1$KinaseSpecTextbox'] = kinase item = br.find_control(name="ctl00$ContentPlaceHolder1$DataOutputDropdown", type="select").get("") item.selected = True try: br.submit() except HTTPError: print 'Server Error', i continue br.select_form(nr=0) try: br.form.find_control("ctl00$ContentPlaceHolder1$RegenTableButton") except: print 'not in database error', i
br["login"] = args.username br["password"] = args.password if args.debug: print("Logging in . . .") br.submit() writeout("past_login", br) if args.debug: print("Getting search page . . .") br.open("https://hrsa.cunyfirst.cuny.edu/psc/cnyhcprd/EMPLOYEE/HRMS/c/SA_LEARNER_SERVICES.CLASS_SEARCH.GBL") writeout("on_search_form", br) br.select_form(nr=0) # How do I change this next semester? mods = br.find_control('CLASS_SRCH_WRK2_STRM$45$') item1 = Item(mods, {"contents": "1162", "value": "1162"}) item1.selected = True mods = br.find_control('SSR_CLSRCH_WRK_SUBJECT_SRCH$0') item2 = Item(mods, {"contents": args.subject, "value": args.subject}) item2.selected = True # Next semester? br['CLASS_SRCH_WRK2_STRM$45$'] = ['1162'] br['DERIVED_SSTSNAV_SSTS_MAIN_GOTO$155$'] = ['9999'] br['CLASS_SRCH_WRK2_INSTITUTION$42$'] = [args.college + '01'] br['SSR_CLSRCH_WRK_SSR_EXACT_MATCH1$1'] = ['E'] br.form['SSR_CLSRCH_WRK_CATALOG_NBR$1'] = args.klass br['SSR_CLSRCH_WRK_SUBJECT_SRCH$0'] = [args.subject] writeout("without_ICAction", br)
class Bugzilla: def __init__(self, dryrun=False, committers=CommitterList()): self.dryrun = dryrun self.authenticated = False self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script self.browser.set_handle_robots(False) self.committers = committers # Defaults (until we support better option parsing): bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host def bug_url_for_bug_id(self, bug_id, xml=False): content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def short_bug_url_for_bug_id(self, bug_id): return "http://webkit.org/b/%s" % bug_id def attachment_url_for_id(self, attachment_id, action="view"): action_param = "" if action and action != "view": action_param = "&action=%s" % action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def _parse_attachment_flag(self, element, flag_name, attachment, result_key): flag = element.find('flag', attrs={'name' : flag_name}) if flag and flag['status'] == '+': attachment[result_key] = flag['setter'] def _parse_attachment_element(self, element, bug_id): attachment = {} attachment['bug_id'] = bug_id attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") attachment['id'] = str(element.find('attachid').string) attachment['url'] = self.attachment_url_for_id(attachment['id']) attachment['name'] = unicode(element.find('desc').string) attachment['type'] = str(element.find('type').string) self._parse_attachment_flag(element, 'review', attachment, 'reviewer_email') self._parse_attachment_flag(element, 'commit-queue', attachment, 'committer_email') return attachment def fetch_attachments_from_bug(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: %s" % bug_url) page = urllib2.urlopen(bug_url) soup = BeautifulSoup(page) attachments = [] for element in soup.findAll('attachment'): attachment = self._parse_attachment_element(element, bug_id) attachments.append(attachment) return attachments def fetch_title_from_bug(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) page = urllib2.urlopen(bug_url) soup = BeautifulSoup(page) return soup.find('short_desc').string def fetch_patches_from_bug(self, bug_id): patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if attachment['is_patch'] and not attachment['is_obsolete']: patches.append(attachment) return patches # _view_source_link belongs in some sort of webkit_config.py module. def _view_source_link(self, local_path): return "http://trac.webkit.org/browser/trunk/%s" % local_path def _validate_setter_email(self, patch, result_key, lookup_function, rejection_function, reject_invalid_patches): setter_email = patch.get(result_key + '_email') if not setter_email: return None committer = lookup_function(setter_email) if committer: patch[result_key] = committer.full_name return patch[result_key] if reject_invalid_patches: committer_list = "WebKitTools/Scripts/modules/committers.py" failure_message = "%s does not have %s permissions according to %s." % (setter_email, result_key, self._view_source_link(committer_list)) rejection_function(patch['id'], failure_message) else: log("Warning, attachment %s on bug %s has invalid %s (%s)", (patch['id'], patch['bug_id'], result_key, setter_email)) return None def _validate_reviewer(self, patch, reject_invalid_patches): return self._validate_setter_email(patch, 'reviewer', self.committers.reviewer_by_bugzilla_email, self.reject_patch_from_review_queue, reject_invalid_patches) def _validate_committer(self, patch, reject_invalid_patches): return self._validate_setter_email(patch, 'committer', self.committers.committer_by_bugzilla_email, self.reject_patch_from_commit_queue, reject_invalid_patches) def fetch_reviewed_patches_from_bug(self, bug_id, reject_invalid_patches=False): reviewed_patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if self._validate_reviewer(attachment, reject_invalid_patches) and not attachment['is_obsolete']: reviewed_patches.append(attachment) return reviewed_patches def fetch_commit_queue_patches_from_bug(self, bug_id, reject_invalid_patches=False): commit_queue_patches = [] for attachment in self.fetch_reviewed_patches_from_bug(bug_id, reject_invalid_patches): if self._validate_committer(attachment, reject_invalid_patches) and not attachment['is_obsolete']: commit_queue_patches.append(attachment) return commit_queue_patches def fetch_bug_ids_from_commit_queue(self): commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B" page = urllib2.urlopen(commit_queue_url) soup = BeautifulSoup(page) bug_ids = [] # Grab the cells in the first column (which happens to be the bug ids) for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child" bug_link = bug_link_cell.find("a") bug_ids.append(bug_link.string) # the contents happen to be the bug id return bug_ids def fetch_patches_from_commit_queue(self, reject_invalid_patches=False): patches_to_land = [] for bug_id in self.fetch_bug_ids_from_commit_queue(): patches = self.fetch_commit_queue_patches_from_bug(bug_id, reject_invalid_patches) patches_to_land += patches return patches_to_land def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return (username, password) = read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser['Bugzilla_login'] = username self.browser['Bugzilla_password'] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): # FIXME: We could add the ability to try again on failure. raise BugzillaError("Bugzilla login failed: %s" % match.group(1)) self.authenticated = True def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False): self.authenticate() log('Adding patch "%s" to bug %s' % (description, bug_id)) if self.dryrun: log(comment_text) return self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % (self.bug_server_url, bug_id)) self.browser.select_form(name="entryform") self.browser['description'] = description self.browser['ispatch'] = ("1",) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp())) self.browser.submit() def prompt_for_component(self, components): log("Please pick a component:") i = 0 for name in components: i += 1 log("%2d. %s" % (i, name)) result = int(raw_input("Enter a number: ")) - 1 return components[result] def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) if match: return match.group('bug_id') match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL) error_message = "FAIL" if match: text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True) error_message = "\n" + '\n'.join([" " + line.strip() for line in text_lines if line.strip()]) raise BugzillaError("Bug not created: %s" % error_message) def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False): self.authenticate() log('Creating bug with patch description "%s"' % patch_description) if self.dryrun: log(bug_description) return self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit") self.browser.select_form(name="Create") component_items = self.browser.find_control('component').items component_names = map(lambda item: item.name, component_items) if not component or component not in component_names: component = self.prompt_for_component(component_names) self.browser['component'] = [component] if cc: self.browser['cc'] = cc self.browser['short_desc'] = bug_title if bug_description: log(bug_description) self.browser['comment'] = bug_description self.browser['description'] = patch_description self.browser['ispatch'] = ("1",) self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) self.browser.add_file(patch_file_object, "text/plain", "%s.patch" % timestamp(), 'data') response = self.browser.submit() bug_id = self._check_create_bug_response(response.read()) log("Bug %s created." % bug_id) log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id)) return bug_id def _find_select_element_for_flag(self, flag_name): # FIXME: This will break if we ever re-order attachment flags if flag_name == "review": return self.browser.find_control(type='select', nr=0) if flag_name == "commit-queue": return self.browser.find_control(type='select', nr=1) raise Exception("Don't know how to find flag named \"%s\"" % flag_name) def clear_attachment_flags(self, attachment_id, additional_comment_text=None): self.authenticate() comment_text = "Clearing flags on attachment: %s" % attachment_id if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) self.browser.submit() # FIXME: We need a way to test this on a live bugzilla instance. def _set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text): self.authenticate() if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag(flag_name).value = (flag_value,) self.browser.submit() def reject_patch_from_commit_queue(self, attachment_id, additional_comment_text=None): comment_text = "Rejecting patch %s from commit-queue." % attachment_id self._set_flag_on_attachment(attachment_id, 'commit-queue', '-', comment_text, additional_comment_text) def reject_patch_from_review_queue(self, attachment_id, additional_comment_text=None): comment_text = "Rejecting patch %s from review queue." % attachment_id self._set_flag_on_attachment(attachment_id, 'review', '-', comment_text, additional_comment_text) def obsolete_attachment(self, attachment_id, comment_text = None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.find_control('isobsolete').items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first. self.browser.set_value(comment_text, name='comment', nr=0) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser['comment'] = comment_text self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['bug_status'] = ['RESOLVED'] self.browser['resolution'] = ['FIXED'] self.browser.submit() def reopen_bug(self, bug_id, comment_text): self.authenticate() log("Re-opening bug %s" % bug_id) log(comment_text) # Bugzilla requires a comment when re-opening a bug, so we know it will never be None. if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser['bug_status'] = ['REOPENED'] self.browser['comment'] = comment_text self.browser.submit()
if not os.path.exists('EMFAC2014_County/' + year): os.makedirs('EMFAC2014_County/' + year) for county in counties: br = Browser() response = br.open("https://www.arb.ca.gov/emfac/2014/") print br.title() forms_list = [] for f in br.forms(): forms_list.append(f) # print(forms_list[1]) br.select_form(nr=1) br.form['geo_level'] = ['county'] region = br.find_control('region') # inject the current county into form because javascript blocks them item = Item(region, { "contents": county.keys()[0], "value": county.keys()[0] }) # now set the region to the injected county br.form['region'] = [county.keys()[0]] br.form['cal_year[]'] = [year] br.form['season'] = ['annual'] br.form['veh_cat_type'] = ['emfac2011'] # https://www.arb.ca.gov/emfac/2014/combobox.php?_id=veh_cat_type&_name=veh_cat_type&_value=emfac2011 veh_cat_option = br.find_control('veh_cat_option') # inject veh category
# set up mechanize header headers = [('User-Agent', 'Mozilla/5.0 (Windows NT 5.1; rv:14.0) Gecko/20100101 Firefox/14.0.1')] # define target URL url = "http://www.bseindia.com/getquote.htm" br = Browser() # browser parameters br.set_handle_equiv(True) br.set_handle_gzip(True) br.set_handle_redirect(True) br.set_handle_referer(True) br.set_handle_robots(False) br.addheaders = headers # make request main_page = br.open(url) # select the default form br.select_form(nr=0) br.find_control(id="suggestBoxEQ").value = "CAREERP" # submit form br.submit() page_response = bs(br.response().read()) quote = page_response.find_all('div',{'class':'newsrheadingdiv'}) print quote
def fetch_isochrones(isoc_kind='parsec_CAF09_v1.2S', photsys_version='yang', photsys_file='tab_mag_odfnew/tab_mag_ubvrijhk.dat', kind_cspecmag='aringer09', dust_sourceM='nodustM', dust_sourceC='nodustC', extinction_av='0.0', imf_file='tab_imf/imf_chabrier_lognormal.dat', sequence_type= 'single_isochrone', isoc_age=False, isoc_z =False, isoc_z0=False, isoc_z1=False, isoc_dz=False, isoc_lage0=False, isoc_lage1=False, isoc_dlage=False, path='', filename='Isochrone_teste.dat'): #Sequence_type = 'single_isochrone', 'sequence_constant_metallicity', 'sequence_constant_age' if sequence_type == 'single_isochrone' or sequence_type == 0: sequence_type = 0 elif sequence_type == 'constant_metallicity' or sequence_type == 1 : sequence_type = 1 elif sequence_type == 'constant_age' or sequence_type == 2: sequence_type = 2 else: raise ValueError("Argument sequence_type must be in ('single_isochrone', 'constant_metallicity', " "'constant_age')") warnings.simplefilter('always', UserWarning) #Handling bad values given for different sequence types if sequence_type == 0: if not isoc_age: raise ValueError("For sequence_type == 'single_isochrone', argument isoc_age must be provided") if not isoc_z: raise ValueError("For sequence_type == 'single_isochrone', argument isoc_z must be provided") if any((isoc_z0, isoc_z1, isoc_dz, isoc_lage0, isoc_lage1, isoc_dlage)): warnings.warn("For sequence_type == 'single_isochrone', arguments isoc_z0, isoc_z1, isoc_dz, isoc_lage0, isoc_lage1 and isoc_dlage are not used") elif sequence_type == 1: if not isoc_z: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_z must be provided") if not isoc_lage0: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_lage0 must be provided") if not isoc_lage1: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_lage1 must be provided") if not isoc_dlage: raise ValueError("For sequence_type == 'constant_metallicity', argument isoc_dlage must be provided") if any((isoc_age, isoc_z0, isoc_z1, isoc_dz)): warnings.warn("For sequence_type == 'constant_metallicity', arguments isoc_age, isoc_z0, isoc_z1, and isoc_dz are not used") elif sequence_type == 2: if not isoc_age: raise ValueError("For sequence_type == 'constant_age', argument isoc_age must be provided") if not isoc_z0: raise ValueError("For sequence_type == 'constant_age', argument isoc_z0 must be provided") if not isoc_z1: raise ValueError("For sequence_type == 'constant_age', argument isoc_z1 must be provided") if not isoc_dz: raise ValueError("For sequence_type == 'constant_age', argument isoc_dz must be provided") if any((isoc_z, isoc_lage0, isoc_lage1, isoc_dlage)): warnings.warn("For sequence_type == 'constant_age', arguments isoc_z, isoc_lage0, isoc_lage1, and isoc_dlage are not used") #Error raised when too many isochrones are requested if sequence_type == 1: N_isoc = len(np.arange(isoc_lage0, isoc_lage1, isoc_dlage)) if N_isoc > 400: raise ValueError("you requested too many isochrones ({0}), maximum allowed is 400.\nTry to increase isoc_dlage or lower the difference between isoc_lage0 and isoc_lage1".fotmat(N_isoc)) elif sequence_type == 2: N_isoc = len(np.arange(isoc_z0, isoc_z1, isoc_dz)) if N_isoc > 400: raise ValueError("you requested too many isochrones ({0}), maximum allowed is 400.\nTry to increase isoc_dz or lower the difference between isoc_z0 and isoc_z1".format(N_isoc)) #print 'Opening browser' br = Browser() br.open('http://stev.oapd.inaf.it/cgi-bin/cmd') br.select_form(nr = 0) #print 'Filling form' br.form['isoc_kind'] = [isoc_kind] br.form['photsys_version'] = [photsys_version] br.form['photsys_file'] = [photsys_file] br.form['kind_cspecmag'] = [kind_cspecmag] br.form['dust_sourceM'] = [dust_sourceM] br.form['dust_sourceC'] = [dust_sourceC] br.form['extinction_av'] = (extinction_av) br.form['imf_file'] = [imf_file] br.find_control("isoc_val").items[sequence_type].selected = True if sequence_type == 0: br.form['isoc_age'] = str(isoc_age) # Isochrone age br.form['isoc_zeta'] = str(isoc_z) # Isochrone metallicity elif sequence_type == 1: br.form['isoc_zeta0'] = str(isoc_z) # Isochrone metallicity br.form['isoc_lage0'] = str(isoc_lage0) # Isochrone log initial age br.form['isoc_lage1'] = str(isoc_lage1) # Isochrone log final age br.form['isoc_dlage'] = str(isoc_dlage) # Isochrone log age step elif sequence_type == 2: br.form['isoc_age0'] = str(isoc_age) # Isochrone age br.form['isoc_z0'] = str(isoc_z0) # Isochrone initial metallicity br.form['isoc_z1'] = str(isoc_z1) # Isochrone final metallicity br.form['isoc_dz'] = str(isoc_dz) # Isochrone metallicity step #print('Submitting form') br.submit() #print('Downloading data') download_link = list(br.links())[0].absolute_url geturl(download_link, path+'/'+filename) br.close() print('File ' + path+'/'+filename + ' created')
exit(0) classtype = open('classcategory.txt', 'w') class_type=[] browser = Browser() browser.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')] # Cookie Jar cj = cookielib.LWPCookieJar() browser.set_cookiejar(cj) browser.set_handle_robots(False) browser.open("https://banner.uregina.ca/prod/sct/bwckschd.p_disp_dyn_sched") #to get semester specifics browser.select_form(nr=0) item = browser.find_control(id="term_input_id").get("%s" % semester) item.selected = True response = browser.submit() content = response.read() flag = 0 for line in StringIO.StringIO(content): if flag == 0: if '<SELECT NAME="sel_subj" SIZE="10" MULTIPLE ID="subj_id">' in line: flag = 1 else: if '</SELECT>' not in line: x = line.split('"') try:
class Bugzilla: def __init__(self, dryrun=False): self.dryrun = dryrun self.authenticated = False self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script self.browser.set_handle_robots(False) # Defaults (until we support better option parsing): bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub("\.", "\\.", bug_server_host) bug_server_url = "https://%s/" % bug_server_host # This could eventually be a text file reviewer_usernames_to_full_names = { "abarth": "Adam Barth", "adele": "Adele Peterson", "aroben": "Adam Roben", "ap": "Alexey Proskuryakov", "ariya.hidayat": "Ariya Hidayat", "barraclough": "Gavin Barraclough", "beidson": "Brady Eidson", "darin": "Darin Adler", "ddkilzer": "David Kilzer", "dglazkov": "Dimitri Glazkov", "eric": "Eric Seidel", "fishd": "Darin Fisher", "gns": "Gustavo Noronha", "hausmann": "Simon Hausmann", "hyatt": "David Hyatt", "jmalonzo": "Jan Alonzo", "justin.garcia": "Justin Garcia", "kevino": "Kevin Ollivier", "koivisto": "Antti Koivisto", "levin": "David Levin", "mitz": "Dan Bernstein", "mjs": "Maciej Stachowiak", "mrowe": "Mark Rowe", "oliver": "Oliver Hunt", "sam": "Sam Weinig", "simon.fraser": "Simon Fraser", "staikos": "George Staikos", "timothy": "Timothy Hatcher", "treat": "Adam Treat", "vestbo": u"Tor Arne Vestb\xf8", "xan.lopez": "Xan Lopez", "zecke": "Holger Freyther", "zimmermann": "Nikolas Zimmermann", } def full_name_from_bugzilla_name(self, bugzilla_name): if not bugzilla_name in self.reviewer_usernames_to_full_names: raise Exception("ERROR: Unknown reviewer! " + bugzilla_name) return self.reviewer_usernames_to_full_names[bugzilla_name] def bug_url_for_bug_id(self, bug_id, xml=False): content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def attachment_url_for_id(self, attachment_id, action="view"): action_param = "" if action and action != "view": action_param = "&action=" + action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def fetch_attachments_from_bug(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: " + bug_url) page = urllib2.urlopen(bug_url) soup = BeautifulSoup(page) attachments = [] for element in soup.findAll("attachment"): attachment = {} attachment["bug_id"] = bug_id attachment["is_obsolete"] = element.has_key("isobsolete") and element["isobsolete"] == "1" attachment["is_patch"] = element.has_key("ispatch") and element["ispatch"] == "1" attachment["id"] = str(element.find("attachid").string) attachment["url"] = self.attachment_url_for_id(attachment["id"]) attachment["name"] = unicode(element.find("desc").string) attachment["type"] = str(element.find("type").string) review_flag = element.find("flag", attrs={"name": "review"}) if review_flag and review_flag["status"] == "+": reviewer_email = review_flag["setter"] # We could lookup the full email address instead once we update full_name_from_bugzilla_name bugzilla_name = reviewer_email.split("@")[0] attachment["reviewer"] = self.full_name_from_bugzilla_name(bugzilla_name) attachments.append(attachment) return attachments def fetch_patches_from_bug(self, bug_id): patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if attachment["is_patch"] and not attachment["is_obsolete"]: patches.append(attachment) return patches def fetch_reviewed_patches_from_bug(self, bug_id): reviewed_patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if "reviewer" in attachment and not attachment["is_obsolete"]: reviewed_patches.append(attachment) return reviewed_patches def fetch_bug_ids_from_commit_queue(self): # FIXME: We should have an option for restricting the search by email. Example: # unassigned_only = "&emailassigned_to1=1&emailtype1=substring&email1=unassigned" commit_queue_url = ( self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=review%2B" ) log("Loading commit queue") page = urllib2.urlopen(commit_queue_url) soup = BeautifulSoup(page) bug_ids = [] # Grab the cells in the first column (which happens to be the bug ids) for bug_link_cell in soup("td", "first-child"): # tds with the class "first-child" bug_link = bug_link_cell.find("a") bug_ids.append(bug_link.string) # the contents happen to be the bug id return bug_ids def fetch_patches_from_commit_queue(self): patches_to_land = [] for bug_id in self.fetch_bug_ids_from_commit_queue(): patches = self.fetch_reviewed_patches_from_bug(bug_id) patches_to_land += patches return patches_to_land def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return (username, password) = read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser["Bugzilla_login"] = username self.browser["Bugzilla_password"] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): # FIXME: We could add the ability to try again on failure. error("Bugzilla login failed: %s" % match.group(1)) self.authenticated = True def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False): self.authenticate() log('Adding patch "%s" to bug %s' % (description, bug_id)) if self.dryrun: log(comment_text) return self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id) self.browser.select_form(name="entryform") self.browser["description"] = description self.browser["ispatch"] = ("1",) if comment_text: log(comment_text) self.browser["comment"] = comment_text self.browser["flag_type-1"] = ("?",) if mark_for_review else ("X",) self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp())) self.browser.submit() def obsolete_attachment(self, attachment_id, comment_text=None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, "edit")) self.browser.select_form(nr=1) self.browser.find_control("isobsolete").items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self.browser.find_control(type="select", nr=0).value = ("X",) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first. self.browser.set_value(comment_text, name="comment", nr=0) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["comment"] = comment_text self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser["comment"] = comment_text self.browser["bug_status"] = ["RESOLVED"] self.browser["resolution"] = ["FIXED"] self.browser.submit()
#.*?(\d+).*?(\d{4}-\d{2}-\d{2}\s*?[0-9:]+).*?([A-Z0-9]+).*?([A-Z0-9]+).*?([0-9.]+).*?(\d+).*?([A-Za-z0-9\+\-]+) regex = re.compile('.*?(\d+).*?(\d{4}-\d{2}-\d{2}\s*?[0-9:]+).*?([A-Z0-9]+).*?([A-Z0-9]+).*?([0-9.]+).*?(\d+).*?([A-Za-z0-9\+\-]+)') auth = False while not auth: user = raw_input("Username: "******"autologin").items[0].selected = True br.submit() for link in br.links(): if link.text == 'my account': auth = True break if not auth: print "Authentication Failed" print "Authenticated" sublist = br.open(base_url+"status/"+user+"/signedlist") sublist = sublist.read() allsub = regex.findall(sublist)
def getSolutions(path_prefix, path_proxy): global br, username, password # create a browser object br = Browser() # add proxy support to browser if len(path_proxy) != 0: protocol, proxy = options.proxy.split("://") br.set_proxies({protocol: proxy}) # let browser fool robots.txt br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; \ rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')] br.set_handle_robots(False) print "Enter yout SPOJ username :"******"Authenticating " + username br.open("http://www.spoj.com") br.select_form(name="login") br["login_user"] = username br["password"] = password # sign in for a day to avoid timeouts br.find_control(name="autologin").items[0].selected = True br.form.action = "http://www.spoj.com/" response = br.submit() verify = response.read() if (verify.find("Authentication failed!") != -1): print "Error authenticating - " + username exit(0) else: print "Successfully Authenticated !!" # Whether or not to download the multiple accepted solution of same problem global dupdown global subidinc print "Do You want to download multiple ACed Solution of same problem? (The latest one will be downloaded otherwise) [Y/N] :", dupdown = raw_input() if dupdown != 'Y' and dupdown != 'y': print "Do You Submission ID as a part of file-name? [Y/N] :", subidinc = raw_input() # grab the signed submissions list print "Grabbing siglist for " + username siglist = br.open("http://www.spoj.com/status/" + username + "/signedlist") # dump first nine useless lines in signed list for formatting for i in xrange(9): siglist.readline() # make a list of all AC's and challenges print "Filtering siglist for AC/Challenge solutions..." mysublist = list() while True: temp = siglist.readline() if temp == '\------------------------------------------------------------------------------/\n': # reached end of siglist break if not len(temp): print "Reached EOF, siglist format has probably changed," + \ " contact author." exit(1) entry = [x.strip() for x in temp.split('|')] if entry[4] == 'AC' or entry[4].isdigit(): dupflag = 0 if dupdown != 'Y' and dupdown != 'y': for xdup in mysublist: if xdup[3] == entry[3]: dupflag = 1 break if dupflag == 0: mysublist.append(entry) print "Done !!!" return mysublist
class Login(object): """ IPS client area login class """ # Form constants LOGIN_URL = 'https://www.invisionpower.com/clients/index.php?app=core&module=global§ion=login' TEST_URL = 'https://www.invisionpower.com/clients/index.php?app=nexus&module=clients§ion=purchases' USERNAME_FIELD = 'ips_username' PASSWORD_FIELD = 'ips_password' REMEMBER_FIELD = 'rememberMe' # Cookie constants LOGIN_COOKIE = 'ips_pass_hash' def __init__(self, ctx): """ Initialize a new Login Handler instance @type ctx: ips_vagrant.cli.Context """ # Debug log self.log = logging.getLogger('ipsv.login') self.cookiejar = cookiejar() self.cookies = {cookie.name: cookie.value for cookie in self.cookiejar} self.browser = Browser() self.browser.set_cookiejar(self.cookiejar) def check(self): """ Check if we have an active login session set @rtype: bool """ self.log.debug('Testing for a valid login session') # If our cookie jar is empty, we obviously don't have a valid login session if not len(self.cookiejar): return False # Test our login session and make sure it's still active return requests.get(self.TEST_URL, cookies=self.cookiejar).status_code == 200 def process(self, username, password, remember=True): """ Process a login request @type username: str @type password: str @param remember: Save the login session to disk @type remember: bool @raise BadLoginException: Login request failed @return: Session cookies @rtype: cookielib.LWPCookieJar """ self.log.debug('Processing login request') self.browser.open(self.LOGIN_URL) self.log.info('Login page loaded: %s', self.browser.title()) self.browser.select_form(nr=0) # Set the fields self.log.debug('Username: %s', username) self.log.debug('Password: %s', (password[0] + '*' * (len(password) - 2) + password[-1])) self.log.debug('Remember: %s', remember) self.browser.form[self.USERNAME_FIELD] = username self.browser.form[self.PASSWORD_FIELD] = password self.browser.find_control(self.REMEMBER_FIELD).items[0].selected = remember # Submit the request self.browser.submit() self.log.debug('Response code: %s', self.browser.response().code) self.log.debug('== Cookies ==') for cookie in self.cookiejar: self.log.debug(cookie) self.cookies[cookie.name] = cookie.value self.log.debug('== End Cookies ==') # Make sure we successfully logged in if self.LOGIN_COOKIE not in self.cookies: raise BadLoginException('No login cookie returned, this probably means an invalid login was provided') # Should we save our login session? if remember: self.log.info('Saving login session to disk') self.cookiejar.save() self.log.info('Login request successful') return self.cookiejar
class Bugzilla(object): def __init__(self, dryrun=False, committers=CommitterList()): self.dryrun = dryrun self.authenticated = False self.queries = BugzillaQueries(self) self.committers = committers # FIXME: We should use some sort of Browser mock object when in dryrun # mode (to prevent any mistakes). self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this # script. self.browser.set_handle_robots(False) # FIXME: Much of this should go into some sort of config module: bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host unassigned_email = "*****@*****.**" def bug_url_for_bug_id(self, bug_id, xml=False): content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def short_bug_url_for_bug_id(self, bug_id): return "http://webkit.org/b/%s" % bug_id def attachment_url_for_id(self, attachment_id, action="view"): action_param = "" if action and action != "view": action_param = "&action=%s" % action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def _parse_attachment_flag(self, element, flag_name, attachment, result_key): flag = element.find('flag', attrs={'name': flag_name}) if flag: attachment[flag_name] = flag['status'] if flag['status'] == '+': attachment[result_key] = flag['setter'] def _parse_attachment_element(self, element, bug_id): attachment = {} attachment['bug_id'] = bug_id attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") attachment['id'] = int(element.find('attachid').string) # FIXME: No need to parse out the url here. attachment['url'] = self.attachment_url_for_id(attachment['id']) attachment['name'] = unicode(element.find('desc').string) attachment['attacher_email'] = str(element.find('attacher').string) attachment['type'] = str(element.find('type').string) self._parse_attachment_flag( element, 'review', attachment, 'reviewer_email') self._parse_attachment_flag( element, 'commit-queue', attachment, 'committer_email') return attachment def _parse_bug_page(self, page): soup = BeautifulSoup(page) bug = {} bug["id"] = int(soup.find("bug_id").string) bug["title"] = unicode(soup.find("short_desc").string) bug["reporter_email"] = str(soup.find("reporter").string) bug["assigned_to_email"] = str(soup.find("assigned_to").string) bug["cc_emails"] = [str(element.string) for element in soup.findAll('cc')] bug["attachments"] = [self._parse_attachment_element(element, bug["id"]) for element in soup.findAll('attachment')] return bug # Makes testing fetch_*_from_bug() possible until we have a better # BugzillaNetwork abstration. def _fetch_bug_page(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: %s" % bug_url) return self.browser.open(bug_url) def fetch_bug_dictionary(self, bug_id): return self._parse_bug_page(self._fetch_bug_page(bug_id)) # FIXME: A BugzillaCache object should provide all these fetch_ methods. def fetch_bug(self, bug_id): return Bug(self.fetch_bug_dictionary(bug_id), self) def _parse_bug_id_from_attachment_page(self, page): # The "Up" relation happens to point to the bug. up_link = BeautifulSoup(page).find('link', rel='Up') if not up_link: # This attachment does not exist (or you don't have permissions to # view it). return None match = re.search("show_bug.cgi\?id=(?P<bug_id>\d+)", up_link['href']) return int(match.group('bug_id')) def bug_id_for_attachment_id(self, attachment_id): self.authenticate() attachment_url = self.attachment_url_for_id(attachment_id, 'edit') log("Fetching: %s" % attachment_url) page = self.browser.open(attachment_url) return self._parse_bug_id_from_attachment_page(page) # FIXME: This should just return Attachment(id), which should be able to # lazily fetch needed data. def fetch_attachment(self, attachment_id): # We could grab all the attachment details off of the attachment edit # page but we already have working code to do so off of the bugs page, # so re-use that. bug_id = self.bug_id_for_attachment_id(attachment_id) if not bug_id: return None attachments = self.fetch_bug(bug_id).attachments(include_obsolete=True) for attachment in attachments: if attachment.id() == int(attachment_id): return attachment return None # This should never be hit. def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return attempts = 0 while not self.authenticated: attempts += 1 (username, password) = Credentials( self.bug_server_host, git_prefix="bugzilla").read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser['Bugzilla_login'] = username self.browser['Bugzilla_password'] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word # "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): errorMessage = "Bugzilla login failed: %s" % match.group(1) # raise an exception only if this was the last attempt if attempts < 5: log(errorMessage) else: raise Exception(errorMessage) else: self.authenticated = True def _fill_attachment_form(self, description, patch_file_object, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False, bug_id=None): self.browser['description'] = description self.browser['ispatch'] = ("1",) self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) if mark_for_landing: self.browser['flag_type-3'] = ('+',) elif mark_for_commit_queue: self.browser['flag_type-3'] = ('?',) else: self.browser['flag_type-3'] = ('X',) if bug_id: patch_name = "bug-%s-%s.patch" % (bug_id, timestamp()) else: patch_name ="%s.patch" % timestamp() self.browser.add_file(patch_file_object, "text/plain", patch_name, 'data') def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False, mark_for_commit_queue=False, mark_for_landing=False): self.authenticate() log('Adding patch "%s" to %sshow_bug.cgi?id=%s' % (description, self.bug_server_url, bug_id)) if self.dryrun: log(comment_text) return self.browser.open("%sattachment.cgi?action=enter&bugid=%s" % ( self.bug_server_url, bug_id)) self.browser.select_form(name="entryform") self._fill_attachment_form(description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue, mark_for_landing=mark_for_landing, bug_id=bug_id) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser.submit() def prompt_for_component(self, components): log("Please pick a component:") i = 0 for name in components: i += 1 log("%2d. %s" % (i, name)) result = int(User.prompt("Enter a number: ")) - 1 return components[result] def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) if match: return match.group('bug_id') match = re.search( '<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL) error_message = "FAIL" if match: text_lines = BeautifulSoup( match.group('error_message')).findAll(text=True) error_message = "\n" + '\n'.join( [" " + line.strip() for line in text_lines if line.strip()]) raise Exception("Bug not created: %s" % error_message) def create_bug(self, bug_title, bug_description, component=None, patch_file_object=None, patch_description=None, cc=None, mark_for_review=False, mark_for_commit_queue=False): self.authenticate() log('Creating bug with title "%s"' % bug_title) if self.dryrun: log(bug_description) return self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit") self.browser.select_form(name="Create") component_items = self.browser.find_control('component').items component_names = map(lambda item: item.name, component_items) if not component: component = "New Bugs" if component not in component_names: component = self.prompt_for_component(component_names) self.browser['component'] = [component] if cc: self.browser['cc'] = cc self.browser['short_desc'] = bug_title self.browser['comment'] = bug_description if patch_file_object: self._fill_attachment_form( patch_description, patch_file_object, mark_for_review=mark_for_review, mark_for_commit_queue=mark_for_commit_queue) response = self.browser.submit() bug_id = self._check_create_bug_response(response.read()) log("Bug %s created." % bug_id) log("%sshow_bug.cgi?id=%s" % (self.bug_server_url, bug_id)) return bug_id def _find_select_element_for_flag(self, flag_name): # FIXME: This will break if we ever re-order attachment flags if flag_name == "review": return self.browser.find_control(type='select', nr=0) if flag_name == "commit-queue": return self.browser.find_control(type='select', nr=1) raise Exception("Don't know how to find flag named \"%s\"" % flag_name) def clear_attachment_flags(self, attachment_id, additional_comment_text=None): self.authenticate() comment_text = "Clearing flags on attachment: %s" % attachment_id if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) self.browser.submit() def set_flag_on_attachment(self, attachment_id, flag_name, flag_value, comment_text, additional_comment_text): # FIXME: We need a way to test this function on a live bugzilla # instance. self.authenticate() if additional_comment_text: comment_text += "\n\n%s" % additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self._find_select_element_for_flag(flag_name).value = (flag_value,) self.browser.submit() # FIXME: All of these bug editing methods have a ridiculous amount of # copy/paste code. def obsolete_attachment(self, attachment_id, comment_text=None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.find_control('isobsolete').items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self._find_select_element_for_flag('review').value = ("X",) self._find_select_element_for_flag('commit-queue').value = ("X",) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow # hidden. We want the first. self.browser.set_value(comment_text, name='comment', nr=0) self.browser.submit() def add_cc_to_bug(self, bug_id, email_address_list): self.authenticate() log("Adding %s to the CC list for bug %s" % (email_address_list, bug_id)) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["newcc"] = ", ".join(email_address_list) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text, cc=None): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser["comment"] = comment_text if cc: self.browser["newcc"] = ", ".join(cc) self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['bug_status'] = ['RESOLVED'] self.browser['resolution'] = ['FIXED'] self.browser.submit() def reassign_bug(self, bug_id, assignee, comment_text=None): self.authenticate() log("Assigning bug %s to %s" % (bug_id, assignee)) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser["comment"] = comment_text self.browser["assigned_to"] = assignee self.browser.submit() def reopen_bug(self, bug_id, comment_text): self.authenticate() log("Re-opening bug %s" % bug_id) # Bugzilla requires a comment when re-opening a bug, so we know it will # never be None. log(comment_text) if self.dryrun: return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") bug_status = self.browser.find_control("bug_status", type="select") # This is a hack around the fact that ClientForm.ListControl seems to # have no simpler way to ask if a control has an item named "REOPENED" # without using exceptions for control flow. possible_bug_statuses = map(lambda item: item.name, bug_status.items) if "REOPENED" in possible_bug_statuses: bug_status.value = ["REOPENED"] else: log("Did not reopen bug %s. " + "It appears to already be open with status %s." % ( bug_id, bug_status.value)) self.browser['comment'] = comment_text self.browser.submit()
class LEAMsite: def __init__(self, site, user, passwd): self.site = site self.error = False self.b = Browser() self.b.set_handle_robots(False) try: self.b.open(site) except urllib2.URLError: self.error = True return try: # try and log in from the portlet self.b.select_form('loginform') except: # try logging in from the main login page self.b.open('/'.join((site, "login_form"))) self.b.select_form(nr=1) if not user or not passwd: raise ValueError('user and password are required') self.b['__ac_name'] = user self.b['__ac_password'] = passwd r = self.b.open(self.b.click()) # plone changes have rendered this inoperable # capture the response and look in the content # #if 'logged_in' in path.split(self.b.geturl()): # sys.stderr.write("Error: unable to login to LEAM Plone site\n") # sys.exit(1) def getURL(self, url, data=None, filename=None): """Simple interface for retrieving the contents of a URL and writing it to a file or returning it as stringIO. """ #sys.stderr.write('getURL %s\n' % url) rsp = self.b.open(url, data) if filename: f = file(filename, 'wb') f.write(rsp.read()) f.close() return None else: return StringIO(rsp.read()) def putFileURL(self, filename, url, fileobj=None, title=None, type='text/plain'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url, 'createObject?type_name=File'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def getFile(self, url, filename=None): """ getFile -- gets a file using standard download from Plone site url: The URL is pointer to the file on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url, 'at_download/file')) if filename: f = open(filename, 'wb') f.write(rsp.read()) f.close() return None else: return rsp.read() def saveFile(self, url, dir=".", at_download=False): """Reads the response from a URL and saves it to a local file based on the name provided in the Content-Disposition header. The dir field specifies to the directory where the file will be stored. If the at_download flag is True then 'at_download/file' will be appended the URL. """ if at_download: rsp = self.b.open(url_join(url, 'at_download/file')) else: rsp = self.b.open(url) fname = get_filename(rsp) if fname: f = open('/'.join([dir, fname]), 'wb') f.write(rsp.read()) f.close() return fname def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'): sys.stderr.write('putImageURL %s to %s\n' % (imgfile, url)) self.b.open('/'.join((url, 'createObject?type_name=Image'))) self.b.select_form("edit_form") if not title: title = path.basename(imgfile) self.b['title'] = title self.b.add_file(open(imgfile), type, path.basename(imgfile)) try: # doesn't seem necessary but it is required for file upload form = self.b.find_control('image_delete') form.value = [ "", ] # print "This really does need to happen.." except: # print "That delete stuff never happens..." pass self.b.submit("form.button.save") def putImageURL(self, filename, url, fileobj=None, title=None, type='image/jpg'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url, 'createObject?type_name=Image'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def putDocument(self, doc, url, title): """Creates a new document and add the doc (file-like object) to it.""" self.b.open('/'.join((url, 'createObject?type_name=Document'))) self.b.select_form("edit_form") self.b['title'] = title doc.seek(0) self.b['text'] = doc.read() self.b.submit("form.button.save") return self.b.geturl() def getDocument(self, url): """Returns a string with the text of the current document.""" self.b.open('/'.join((url, 'edit'))) self.b.select_form("edit_form") s = self.b['text'] self.b.submit("form.button.cancel") return s def editDocument(self, doc, url, title=None): """Replaces the contents of a document""" self.b.open('/'.join((url, 'edit'))) self.b.select_form("edit_form") # update the title if given if title: self.b['title'] = title # slightly dangerous approach where we seek first # to test for a file-like object. If exception is # thrown then assume doc is a string. try: doc.seek(0) self.b['text'] = doc.read() except: self.b['text'] = doc self.b.submit("form.button.save") return self.b.geturl() def createFolder(self, folder, url): """Creates a folder titled <folder> at the location <url> if it doesn't already exist. Returns the full path of new folder. """ pathurl = '/'.join((url, folder.lower().replace(' ', '-'))) try: self.b.open(pathurl) except: self.b.open('/'.join((url, 'createObject?type_name=Folder'))) self.b.select_form("edit_form") self.b['title'] = folder self.b.submit("form.button.save") return self.b.geturl() def editFolder(self, url, title="", description=""): """Edit the basic fields of the Folder. Mostly useful for setting the title AFTER creating the folder with a reasonable short name. """ try: self.b.open(url) except: self.error = True return None self.b.open(url + '/edit') self.b.select_form("edit_form") if title: self.b['title'] = title if description: self.b['description'] = description self.b.submit("form.button.save") return self.b.geturl() def deleteFolder(self, url): "Deletes folder and all of its contents" sys.stderr.write('DELETING folder %s\n' % url) return # Puts SimMaps on to the site def putSimMap(self, simmap, mapfile, url, simmap_file=None, mapfile_file=None, title=None, description=None, trans=.7, details=None, zoom=11): """ putSimMap Required Input: simmap, mapfile, url simmap is a file that contains the desired GIS layer mapfile is the standard .map file that maps GIS layer to image url - is the full URL to the folder where the SimMap will be stored Optional Inputs: simmap_file, mapfile_file, title, trans, details simmap_file - file-like object of the simmap if None simmap will be openned and read. mapfile_file - file-like object of the mapfile. If None mapfile will be openned and read. title - title sting of the SimMap defaults to basename(simmap) trans - transparency level details - description of SimMap as stored in the details field """ self.b.open('/'.join((url, 'createObject?type_name=SimMap'))) self.b.select_form("edit_form") if not simmap_file: simmap_file = open(simmap, 'rb') if not mapfile_file: mapfile_file = open(mapfile, 'rb') if not title: title = path.splitext(path.basename(simmap))[0] self.b['title'] = str(title) self.b.add_file(simmap_file, 'application/octet-stream', path.basename(simmap), "simImage_file") self.b.add_file(mapfile_file, 'application/octet-stream', path.basename(mapfile), "mapFile_file") if description: self.b['description'] = str(description) self.b['transparency'] = str(trans) self.b['zoom'] = str(zoom) if details: self.b['details'] = str(details) self.b.submit("form.button.save") return self.b.geturl() def getSimMap(self, url): """ getSimMap Gets the metadata associated with SimMap including title, description, location, transparency, and zoom. """ self.b.open(url + '/edit') self.b.select_form('edit_form') d = dict( title=self.b['title'], description=self.b['description'], details=self.b['details'], location=self.b['location'], transparency=self.b['Transparency'], zoom=self.b['zoom'], ) self.b.submit("form.button.cancel") return d def getSimMapData(self, url, filename=None): """ getSimMapData -- gets the data component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ bufsize = 15 * 1024 * 1024 rsp = self.b.open(url_join(url, 'get_layer')) if filename: f = file(filename, 'wb') while 1: b = rsp.read(bufsize) f.write(b) if len(b) < bufsize: break f.close() return None else: return StringIO(rsp.read()) def getSimMapMapfile(self, url, filename=None): """ getSimMapData -- gets the mapfile component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url, 'get_mapfile')) if filename: f = file(filename, 'wb') f.write(rsp.read()) f.close() return None else: data = StringIO() data.write(rsp.read()) return data # DELETE FUNCTIONS ----------------------------------------- # These functions delete items from the LEAM Plone sites # ---------------------------------------------------------- def deleteItem(self, fname, url): """ deleteItem Deletes <fname> from the folder <url> """ print '/'.join((url, fname, 'delete_confirmation')) print self.b.open('/'.join((url, fname, 'delete_confirmation'))) self.b.select_form(None, None, 1) print self.b.submit()
import mechanize import urllib3 browser = Browser() browser.open("http://localhost:8080/test_suite") browser.select_form(nr=1) print(browser) tipos = browser.form.possible_items("type") print(tipos) browser['title'] = '' browser.set(True, tipos[0], "type") browser.find_control("checkbox").items[0].selected = False print('current estatus') print(browser['title']) print(browser['type']) print(browser['checkbox']) if browser['title'] == '': print('val tit: ', 0) else: print('val tit: ', 1) if len(browser['checkbox']) == 0: print('val check: ', 0) else:
class Enviroment: def __init__(self): #self.fields = fields self.states = {} self.status_submit = '0' self.http_response = '' self.browser = Browser() self.browser.open("http://localhost:8080/test_suite") self.browser.select_form(nr=1) self.tipos = self.browser.form.possible_items("type") def initialize(self): self.browser = Browser() self.browser.open("http://localhost:8080/test_suite") self.browser.select_form(nr=1) self.tipos = self.browser.form.possible_items("type") self.status_submit = '0' self.http_response = '' def apply_action(self, action): if action == 0: self.browser['title'] = '' self.status_submit = '0' elif action == 1: self.browser['title'] = 'Prueba' self.status_submit = '0' elif action == 2: self.browser.find_control("checkbox").items[0].selected = False self.status_submit = '0' elif action == 3: self.browser.find_control("checkbox").items[0].selected = True self.status_submit = '0' elif action == 4: self.browser.set(True, self.tipos[0], "type") self.status_submit = '0' elif action == 5: self.browser.set(True, self.tipos[1], "type") self.status_submit = '0' elif action == 6: self.browser.set(True, self.tipos[2], "type") self.status_submit = '0' elif action == 7: self.status_submit = '1' #just submit if all fields are filled if self.browser['title'] != '' and self.browser['type'][ 0] != '0' and len(self.browser['checkbox']) != 0: try: response = self.browser.submit() #print('after submit') #content = response.read() code = response.code #print(code) self.http_response = code except mechanize.HTTPError as e: #print('Error:', e) self.http_response = 500 def get_current_state(self): if self.browser['title'] == '': val_tit = '0' else: val_tit = '1' if len(self.browser['checkbox']) == 0: val_che = '0' else: val_che = '1' if self.browser['type'][0] == '0': val_typ = '0' elif self.browser['type'][0] == '1': val_typ = '1' elif self.browser['type'][0] == '2': val_typ = '2' state = val_tit + val_che + val_typ + self.status_submit #0110 return state def get_all_possible_actions(self): actions = [0, 1, 2, 3, 4, 5, 6, 7] return actions def get_next_state(self, action): #(0110, 2) self.apply_action(action) if action == 7: return '0000' else: return self.get_current_state()
class Bugzilla: def __init__(self, dryrun=False, committers=CommitterList()): self.dryrun = dryrun self.authenticated = False self.browser = Browser() # Ignore bugs.webkit.org/robots.txt until we fix it to allow this script self.browser.set_handle_robots(False) self.committers = committers # Defaults (until we support better option parsing): bug_server_host = "bugs.webkit.org" bug_server_regex = "https?://%s/" % re.sub('\.', '\\.', bug_server_host) bug_server_url = "https://%s/" % bug_server_host def bug_url_for_bug_id(self, bug_id, xml=False): content_type = "&ctype=xml" if xml else "" return "%sshow_bug.cgi?id=%s%s" % (self.bug_server_url, bug_id, content_type) def attachment_url_for_id(self, attachment_id, action="view"): action_param = "" if action and action != "view": action_param = "&action=" + action return "%sattachment.cgi?id=%s%s" % (self.bug_server_url, attachment_id, action_param) def _parse_attachment_element(self, element, bug_id): attachment = {} attachment['bug_id'] = bug_id attachment['is_obsolete'] = (element.has_key('isobsolete') and element['isobsolete'] == "1") attachment['is_patch'] = (element.has_key('ispatch') and element['ispatch'] == "1") attachment['id'] = str(element.find('attachid').string) attachment['url'] = self.attachment_url_for_id(attachment['id']) attachment['name'] = unicode(element.find('desc').string) attachment['type'] = str(element.find('type').string) review_flag = element.find('flag', attrs={"name" : "review"}) if review_flag and review_flag['status'] == '+': reviewer_email = review_flag['setter'] reviewer = self.committers.reviewer_by_bugzilla_email(reviewer_email) attachment['reviewer'] = reviewer.full_name commit_queue_flag = element.find('flag', attrs={"name" : "commit-queue"}) if commit_queue_flag and commit_queue_flag['status'] == '+': committer_email = commit_queue_flag['setter'] committer = self.committers.committer_by_bugzilla_email(committer_email) attachment['commit-queue'] = committer.full_name return attachment def fetch_attachments_from_bug(self, bug_id): bug_url = self.bug_url_for_bug_id(bug_id, xml=True) log("Fetching: " + bug_url) page = urllib2.urlopen(bug_url) soup = BeautifulSoup(page) attachments = [] for element in soup.findAll('attachment'): attachment = self._parse_attachment_element(element, bug_id) attachments.append(attachment) return attachments def fetch_patches_from_bug(self, bug_id): patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if attachment['is_patch'] and not attachment['is_obsolete']: patches.append(attachment) return patches def fetch_reviewed_patches_from_bug(self, bug_id): reviewed_patches = [] for attachment in self.fetch_attachments_from_bug(bug_id): if 'reviewer' in attachment and not attachment['is_obsolete']: reviewed_patches.append(attachment) return reviewed_patches def fetch_commit_queue_patches_from_bug(self, bug_id): commit_queue_patches = [] for attachment in self.fetch_reviewed_patches_from_bug(bug_id): if 'commit-queue' in attachment and not attachment['is_obsolete']: commit_queue_patches.append(attachment) return commit_queue_patches def fetch_bug_ids_from_commit_queue(self): commit_queue_url = self.bug_server_url + "buglist.cgi?query_format=advanced&bug_status=UNCONFIRMED&bug_status=NEW&bug_status=ASSIGNED&bug_status=REOPENED&field0-0-0=flagtypes.name&type0-0-0=equals&value0-0-0=commit-queue%2B" page = urllib2.urlopen(commit_queue_url) soup = BeautifulSoup(page) bug_ids = [] # Grab the cells in the first column (which happens to be the bug ids) for bug_link_cell in soup('td', "first-child"): # tds with the class "first-child" bug_link = bug_link_cell.find("a") bug_ids.append(bug_link.string) # the contents happen to be the bug id return bug_ids def fetch_patches_from_commit_queue(self): patches_to_land = [] for bug_id in self.fetch_bug_ids_from_commit_queue(): patches = self.fetch_commit_queue_patches_from_bug(bug_id) patches_to_land += patches return patches_to_land def authenticate(self): if self.authenticated: return if self.dryrun: log("Skipping log in for dry run...") self.authenticated = True return (username, password) = read_credentials() log("Logging in as %s..." % username) self.browser.open(self.bug_server_url + "index.cgi?GoAheadAndLogIn=1") self.browser.select_form(name="login") self.browser['Bugzilla_login'] = username self.browser['Bugzilla_password'] = password response = self.browser.submit() match = re.search("<title>(.+?)</title>", response.read()) # If the resulting page has a title, and it contains the word "invalid" assume it's the login failure page. if match and re.search("Invalid", match.group(1), re.IGNORECASE): # FIXME: We could add the ability to try again on failure. raise ScriptError("Bugzilla login failed: %s" % match.group(1)) self.authenticated = True def add_patch_to_bug(self, bug_id, patch_file_object, description, comment_text=None, mark_for_review=False): self.authenticate() log('Adding patch "%s" to bug %s' % (description, bug_id)) if self.dryrun: log(comment_text) return self.browser.open(self.bug_server_url + "attachment.cgi?action=enter&bugid=" + bug_id) self.browser.select_form(name="entryform") self.browser['description'] = description self.browser['ispatch'] = ("1",) if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) self.browser.add_file(patch_file_object, "text/plain", "bug-%s-%s.patch" % (bug_id, timestamp())) self.browser.submit() def prompt_for_component(self, components): log("Please pick a component:") i = 0 for name in components: i += 1 log("%2d. %s" % (i, name)) result = int(raw_input("Enter a number: ")) - 1 return components[result] def _check_create_bug_response(self, response_html): match = re.search("<title>Bug (?P<bug_id>\d+) Submitted</title>", response_html) if match: return match.group('bug_id') match = re.search('<div id="bugzilla-body">(?P<error_message>.+)<div id="footer">', response_html, re.DOTALL) error_message = "FAIL" if match: text_lines = BeautifulSoup(match.group('error_message')).findAll(text=True) error_message = "\n" + '\n'.join([" " + line.strip() for line in text_lines if line.strip()]) raise ScriptError("Bug not created: %s" % error_message) def create_bug_with_patch(self, bug_title, bug_description, component, patch_file_object, patch_description, cc, mark_for_review=False): self.authenticate() log('Creating bug with patch description "%s"' % patch_description) if self.dryrun: log(bug_description) return self.browser.open(self.bug_server_url + "enter_bug.cgi?product=WebKit") self.browser.select_form(name="Create") component_items = self.browser.find_control('component').items component_names = map(lambda item: item.name, component_items) if not component or component not in component_names: component = self.prompt_for_component(component_names) self.browser['component'] = [component] self.browser['cc'] = cc self.browser['short_desc'] = bug_title if bug_description: log(bug_description) self.browser['comment'] = bug_description self.browser['description'] = patch_description self.browser['ispatch'] = ("1",) self.browser['flag_type-1'] = ('?',) if mark_for_review else ('X',) self.browser.add_file(patch_file_object, "text/plain", "%s.patch" % timestamp(), 'data') response = self.browser.submit() bug_id = self._check_create_bug_response(response.read()) log("Bug %s created." % bug_id) log(self.bug_server_url + "show_bug.cgi?id=" + bug_id) return bug_id def clear_attachment_review_flag(self, attachment_id, additional_comment_text=None): self.authenticate() comment_text = "Clearing review flag on attachment: %s" % attachment_id if additional_comment_text: comment_text += "\n\n" + additional_comment_text log(comment_text) if self.dryrun: return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.set_value(comment_text, name='comment', nr=0) self.browser.find_control(type='select', nr=0).value = ("X",) self.browser.submit() def obsolete_attachment(self, attachment_id, comment_text = None): self.authenticate() log("Obsoleting attachment: %s" % attachment_id) if self.dryrun: log(comment_text) return self.browser.open(self.attachment_url_for_id(attachment_id, 'edit')) self.browser.select_form(nr=1) self.browser.find_control('isobsolete').items[0].selected = True # Also clear any review flag (to remove it from review/commit queues) self.browser.find_control(type='select', nr=0).value = ("X",) if comment_text: log(comment_text) # Bugzilla has two textareas named 'comment', one is somehow hidden. We want the first. self.browser.set_value(comment_text, name='comment', nr=0) self.browser.submit() def post_comment_to_bug(self, bug_id, comment_text): self.authenticate() log("Adding comment to bug %s" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") self.browser['comment'] = comment_text self.browser.submit() def close_bug_as_fixed(self, bug_id, comment_text=None): self.authenticate() log("Closing bug %s as fixed" % bug_id) if self.dryrun: log(comment_text) return self.browser.open(self.bug_url_for_bug_id(bug_id)) self.browser.select_form(name="changeform") if comment_text: log(comment_text) self.browser['comment'] = comment_text self.browser['bug_status'] = ['RESOLVED'] self.browser['resolution'] = ['FIXED'] self.browser.submit()
class Installer(object): # License fields FIELD_LICENSE_KEY = 'lkey' # Server Detail fields FIELD_SERVER_SQL_HOST = 'sql_host' FIELD_SERVER_SQL_USER = '******' FIELD_SERVER_SQL_PASS = '******' FIELD_SERVER_SQL_DATABASE = 'sql_database' # Admin fields FIELD_ADMIN_USER = '******' FIELD_ADMIN_PASS = '******' FIELD_ADMIN_PASS_CONFIRM = 'admin_pass2' FIELD_ADMIN_EMAIL = 'admin_email' def __init__(self, ctx, site, force=False): """ Initialize a new Installer instance @type ctx: ips_vagrant.cli.Context @param site: The IPS Site we are installing @type site: ips_vagrant.models.sites.Site @param force: Overwrite existing files / databases @type force: bool """ self.log = logging.getLogger('ipsv.installer') self.ctx = ctx self.force = force self._previous_title = None self.url = '{scheme}://{host}/admin/install'.format( scheme='https' if site.ssl else 'http', host=site.domain.name) self.site = site self.mysql = create_engine('mysql://*****:*****@localhost') self._sessions = {} self.cookiejar = cookiejar() self.cookies = {cookie.name: cookie.value for cookie in self.cookiejar} self.browser = Browser() self.browser.set_cookiejar(self.cookiejar) def _check_title(self, title): """ If we're on the same page, we got an error and need to raise an exception @type title: str @raise InstallationError: Title matches the previous page requests title (We're on the same page) """ self.log.info('Installation page loaded: %s', title) if self._previous_title and title == self._previous_title: raise InstallationError('Unexpected page error') self._previous_title = title def start(self): """ Start the installation wizard """ self.log.debug('Starting the installation process') self.browser.open(self.url) continue_link = next( self.browser.links(text_regex='Start Installation')) self.browser.follow_link(continue_link) self.system_check() def system_check(self): """ System requirements check """ self._check_title(self.browser.title()) p = Echo('Running system check...') rsoup = BeautifulSoup(self.browser.response().read(), "html.parser") # Check for any errors errors = [] for ul in rsoup.find_all('ul', {'class': 'ipsList_checks'}): for li in ul.find_all('li', {'class': 'fail'}): errors.append(li.text) if errors: raise InstallationError(errors) # Continue continue_link = next(self.browser.links(text_regex='Continue')) p.done() self.browser.follow_link(continue_link) self.license() def license(self): """ Submit our license to IPS' servers """ p = Echo('Submitting license key...') self._check_title(self.browser.title()) self.browser.select_form(nr=0) # Set the fields self.browser.form[ self.FIELD_LICENSE_KEY] = '{license}-TESTINSTALL'.format( license=self.site.license_key) self.browser.find_control( 'eula_checkbox').items[0].selected = True # TODO: User prompt? # Submit the request self.log.debug('Submitting our license') self.browser.submit() self.log.debug('Response code: %s', self.browser.response().code) p.done() self.applications() def applications(self): """ Select the applications to install (currently hardcoded to install all applications) """ # Check for license submission errors try: self._check_title(self.browser.title()) except InstallationError: rsoup = BeautifulSoup(self.browser.response().read(), "html.parser") error = rsoup.find('li', id='license_lkey').find( 'span', { 'class': 'ipsType_warning' }).text raise InstallationError(error) # TODO: Make this configurable p = Echo('Setting applications to install...') self.browser.select_form(nr=0) self.browser.submit() p.done() self.server_details() def server_details(self): """ Input server details (database information, etc.) """ self._check_title(self.browser.title()) p = Echo('Creating MySQL database...') # Create the database md5hex = md5(self.site.domain.name + self.site.slug).hexdigest() db_name = 'ipsv_{md5}'.format(md5=md5hex) # MySQL usernames are limited to 16 characters max db_user = '******'.format(md5=md5hex[:11]) rand_pass = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(random.randint(16, 24))) db_pass = rand_pass try: self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name)) except SQLAlchemyError: if not self.force: click.confirm( 'A previous database for this installation already exists.\n' 'Would you like to drop it now? The installation will be aborted if you do not', abort=True) self.log.info( 'Dropping existing database: {db}'.format(db=db_name)) self.mysql.execute( 'DROP DATABASE IF EXISTS `{db}`'.format(db=db_name)) self.mysql.execute('DROP USER `{u}`'.format(u=db_user)) self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name)) self.mysql.execute( "GRANT ALL ON {db}.* TO '{u}'@'localhost' IDENTIFIED BY '{p}'". format(db=db_name, u=db_user, p=db_pass)) # Save the database connection information self.site.db_host = 'localhost' self.site.db_name = db_name self.site.db_user = db_user self.site.db_pass = db_pass self.ctx.db.commit() self.log.debug('MySQL Database Name: %s', db_name) self.log.debug('MySQL Database User: %s', db_user) self.log.debug('MySQL Database Password: %s', db_pass) # Set form fields and submit self.browser.select_form(nr=0) self.browser.form[self.FIELD_SERVER_SQL_HOST] = 'localhost' self.browser.form[self.FIELD_SERVER_SQL_USER] = db_user self.browser.form[self.FIELD_SERVER_SQL_PASS] = db_pass self.browser.form[self.FIELD_SERVER_SQL_DATABASE] = db_name self.browser.submit() p.done() self.admin() def admin(self): """ Provide admin login credentials """ self._check_title(self.browser.title()) self.browser.select_form(nr=0) # Get the admin credentials prompted = [] user = self.ctx.config.get('User', 'AdminUser') if not user: user = click.prompt('Admin display name') prompted.append('user') password = self.ctx.config.get('User', 'AdminPass') if not password: password = click.prompt( 'Admin password', hide_input=True, confirmation_prompt='Confirm admin password') prompted.append('password') email = self.ctx.config.get('User', 'AdminEmail') if not email: email = click.prompt('Admin email') prompted.append('email') self.browser.form[self.FIELD_ADMIN_USER] = user self.browser.form[self.FIELD_ADMIN_PASS] = password self.browser.form[self.FIELD_ADMIN_PASS_CONFIRM] = password self.browser.form[self.FIELD_ADMIN_EMAIL] = email Echo('Submitting admin information...').done() if len(prompted) >= 3: save = click.confirm( 'Would you like to save and use these admin credentials for future installations?' ) if save: self.log.info('Saving admin login credentials') self.ctx.config.set('User', 'AdminUser', user) self.ctx.config.set('User', 'AdminPass', password) self.ctx.config.set('User', 'AdminEmail', email) with open(self.ctx.config_path, 'wb') as cf: self.ctx.config.write(cf) self.install() def _start_install(self): """ Start the installation """ self.browser.submit() self._check_title(self.browser.title()) def _get_mr_link(self): """ Get the MultipleRedirect URL @rtype: str """ rsoup = BeautifulSoup(self.browser.response().read(), "html.parser") mr_link = rsoup.find('a', {'class': 'button'}).get('href') self.log.debug('MultipleRedirect link: %s', mr_link) return mr_link def _ajax(self, url, method='get', params=None, load_json=True, raise_request=True): """ Perform an Ajax request @type url: str @type method: str @type params: dict or None @type load_json: bool @return: Tuple with the decoded JSON response and actual response, or just the response if load_json is False @rtype: requests.Response or tuple of (dict or list, requests.Response) """ if 'ajax' in self._sessions: ajax = self._sessions['ajax'] else: self.log.debug('Instantiating a new Ajax session') ajax = requests.Session() ajax.headers.update({'X-Requested-With': 'XMLHttpRequest'}) ajax.cookies.update(cookiejar()) ajax.verify = False self._sessions['ajax'] = ajax response = ajax.request(method, url, params) self.log.debug('Ajax response: %s', response.text) if raise_request: response.raise_for_status() if load_json: return byteify(json.loads(response.text)), response return response def _request(self, url, method='get', params=None, raise_request=True): """ Perform a standard HTTP request @type url: str @type method: str @type params: dict or None @param raise_request: Raise exceptions for HTTP status errors @type raise_request: bool @rtype: requests.Response """ if 'http' in self._sessions: http = self._sessions['http'] else: self.log.debug('Instantiating a new HTTP session') http = requests.Session() http.cookies.update(cookiejar()) http.verify = False self._sessions['http'] = http response = http.request(method, url, params) self.log.debug('HTTP response code: %s', response.status_code) if raise_request: response.raise_for_status() return response def _parse_response(self, url, json_response): """ Parse response data and return the next request URL @type url: str @type json_response: list or dict @rtype: str """ parts = list(urlparse(url)) query = parse_qs(parts[4]) query['mr'] = str(json_response[0]).replace('\'', '"') parts[4] = urlencode(query, True) return urlunparse(parts) def _get_stage(self, json_response): """ Get the current installation stage @type json_response: list or dict @rtype: str """ try: return json_response[1] except IndexError: return 'Installation complete!' def _get_progress(self, json_response): """ Get the current installation progress @type json_response: list or dict @rtype: str """ try: return round(float(json_response[2])) except IndexError: return 0 def _check_if_complete(self, url, json_response): """ Check if a request has been completed and return the redirect URL if it has @type url: str @type json_response: list or dict @rtype: str or bool """ if 'redirect' in json_response and isinstance(json_response, dict): self.log.info('Installation complete') return json_response['redirect'] return False def _finalize(self, response): """ Finalize the installation and display a link to the suite """ rsoup = BeautifulSoup(response.text, "html.parser") click.echo('------') click.secho(rsoup.find('h1', id='elInstaller_welcome').text.strip(), fg='yellow', bold=True) click.secho(rsoup.find('p', { 'class': 'ipsType_light' }).text.strip(), fg='yellow', dim=True) link = rsoup.find('a', {'class': 'ipsButton_primary'}).get('href') click.echo(click.style('Go to the suite: ', bold=True) + link + '\n') # noinspection PyUnboundLocalVariable def install(self): """ Run the actual installation """ self._start_install() mr_link = self._get_mr_link() # Set up the progress bar pbar = ProgressBar(100, 'Running installation...') pbar.start() mr_j, mr_r = self._ajax(mr_link) # Loop until we get a redirect json response while True: mr_link = self._parse_response(mr_link, mr_j) stage = self._get_stage(mr_j) progress = self._get_progress(mr_j) mr_j, mr_r = self._ajax(mr_link) pbar.update( min([progress, 100]), stage) # NOTE: Response may return progress values above 100 # If we're done, finalize the installation and break redirect = self._check_if_complete(mr_link, mr_j) if redirect: pbar.finish() break p = Echo('Finalizing...') mr_r = self._request(redirect, raise_request=False) p.done() # Install developer tools if self.site.in_dev: DevToolsInstaller(self.ctx, self.site).install() # Get the link to our community homepage self._finalize(mr_r)
cmssw2rank = {} rank = 10 for release in release_list: cmssw2rank[release] = rank rank += 10 ## Loop overall releases in Savannah and check if there are valid ## Hide deprecated versions for links in br.links(text_regex="CMSSW_*_*"): if links.text not in release_list: br.follow_link(links) print "Hide deprecated version %s" % links.text br.select_form(nr=1) control = br.find_control("status") control.value = ['H'] br.submit() ## Doing a loop in all releases ## Create entry in Savannah for releases. If it already exist, update the rank for release in release_list: try: #First, look if this CMSSW release is already there br.follow_link(text=release, nr=0) rank = cmssw2rank[release] print("Release %s already exists. Changing rank to %s" % (release, rank)) br.select_form(nr=1) br['order_id'] = str(rank) br['status'] = ['A']
class LEAMsite: def __init__(self, site, user, passwd): self.site = site self.error = False self.b = Browser() self.b.set_handle_robots(False) try: self.b.open(site) except urllib2.URLError: self.error = True return try: # try and log in from the portlet self.b.select_form('loginform') except: # try logging in from the main login page self.b.open('/'.join((site,"login_form"))) self.b.select_form(nr=1) self.b['__ac_name'] = user self.b['__ac_password'] = passwd r = self.b.open(self.b.click()) # plone changes have rendered this inoperable # capture the response and look in the content # body tag has class with login failure def checkURL(self, url): """Tries to open a URL and return true if successful (object exists) or false if error occurs. """ try: rsp = self.b.open(url) except: return False return True def getURL(self, url, data=None, filename=None): """Simple interface for retrieving the contents of a URL and writing it to a file or returning it as stringIO. """ #sys.stderr.write('getURL %s\n' % url) rsp = self.b.open(url, data) if filename: f = file("./Inputs/"+filename, 'wb') # always write to Input folder f.write(rsp.read()) f.close() return None else: return StringIO(rsp.read()) def putFileURL(self, filename, url, fileobj=None, title=None, type='text/plain'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url,'createObject?type_name=File'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def getFile(self, url, filename=None): """ getFile -- gets a file using standard download from Plone site url: The URL is pointer to the file on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url,'at_download/file')) if filename: f = open(filename, 'wb') f.write(rsp.read()) f.close() return None else: return rsp.read() def saveFile(self, url, dir=".", at_download=False): """Reads the response from a URL and saves it to a local file based on the name provided in the Content-Disposition header. The dir field specifies to the directory where the file will be stored. If the at_download flag is True then 'at_download/file' will be appended the URL. """ if at_download: rsp = self.b.open(url_join(url,'at_download/file')) else: rsp = self.b.open(url) fname = get_filename(rsp) if fname: f = open('/'.join([dir, fname]), 'wb') f.write(rsp.read()) f.close() return fname def putImageURL_old(self, imgfile, url, title=None, type='image/jpg'): sys.stderr.write('putImageURL %s to %s\n' % (imgfile,url)) self.b.open('/'.join((url,'createObject?type_name=Image'))) self.b.select_form("edit_form") if not title: title = path.basename(imgfile) self.b['title'] = title self.b.add_file(open(imgfile), type, path.basename(imgfile)) try: # doesn't seem necessary but it is required for file upload form = self.b.find_control('image_delete') form.value = ["",] # print "This really does need to happen.." except: # print "That delete stuff never happens..." pass self.b.submit("form.button.save") def putImageURL(self, filename, url, fileobj=None, title=None, type='image/jpg'): """Simple interface for uploading a file to a plone site. <URL> should be an existing folder on the site and <filename> should be a readable file. """ #sys.stderr.write('putFileURL %s to %s\n' % (filename,url)) if not title: title = path.basename(filename) if not fileobj: fileobj = open(filename, 'rb') self.b.open('/'.join((url,'createObject?type_name=Image'))) self.b.select_form("edit_form") self.b['title'] = title self.b.add_file(fileobj, type, path.basename(filename)) # form = self.b.find_control('file_delete') # form.value = ["",] self.b.submit("form.button.save") # r = self.b.open(self.b.click()) # should check that it worked def putDocument(self, doc, url, title): """Creates a new document and add the doc (file-like object) to it.""" self.b.open('/'.join((url,'createObject?type_name=Document'))) self.b.select_form("edit_form") self.b['title'] = title doc.seek(0) self.b['text'] = doc.read() self.b.submit("form.button.save") return self.b.geturl() def getDocument(self, url): """Returns a string with the text of the current document.""" self.b.open('/'.join((url,'edit'))) self.b.select_form("edit_form") s = self.b['text'] self.b.submit("form.button.cancel") return s def editDocument(self, doc, url, title=None): """Replaces the contents of a document""" self.b.open('/'.join((url,'edit'))) self.b.select_form("edit_form") # update the title if given if title: self.b['title'] = title # slightly dangerous approach where we seek first # to test for a file-like object. If exception is # thrown then assume doc is a string. try: doc.seek(0) self.b['text'] = doc.read() except: self.b['text'] = doc self.b.submit("form.button.save") return self.b.geturl() def createFolder(self, folder, url): """Creates a folder titled <folder> at the location <url> if it doesn't already exist. Returns the full path of new folder. """ pathurl = '/'.join((url,folder.lower().replace(' ','-'))) print pathurl try: self.b.open(pathurl) except: self.b.open('/'.join((url,'createObject?type_name=Folder'))) self.b.select_form("edit_form") self.b['title'] = folder self.b.submit("form.button.save") return self.b.geturl() def editFolder(self, url, title="", description=""): """Edit the basic fields of the Folder. Mostly useful for setting the title AFTER creating the folder with a reasonable short name. """ try: self.b.open(url) except: self.error = True return None self.b.open(url+'/edit') self.b.select_form("edit_form") if title: self.b['title'] = title if description: self.b['description'] = description self.b.submit("form.button.save") return self.b.geturl() def deleteFolder(self, url): "Deletes folder and all of its contents" sys.stderr.write('DELETING folder %s\n' % url) return # Puts SimMaps on to the site def putSimMap(self, simmap, mapfile, url, simmap_file=None, mapfile_file=None, title=None, description=None, trans=.7, details=None, zoom=8): """ putSimMap Required Input: simmap, mapfile, url simmap is a file that contains the desired GIS layer mapfile is the standard .map file that maps GIS layer to image url - is the full URL to the folder where the SimMap will be stored Optional Inputs: simmap_file, mapfile_file, title, trans, details simmap_file - file-like object of the simmap if None simmap will be openned and read. mapfile_file - file-like object of the mapfile. If None mapfile will be openned and read. title - title sting of the SimMap defaults to basename(simmap) trans - transparency level details - description of SimMap as stored in the details field """ self.b.open('/'.join((url,'createObject?type_name=SimMap'))) self.b.select_form("edit_form") if not simmap_file: simmap_file = open(simmap, 'rb') if not mapfile_file: mapfile_file = open(mapfile, 'rb') if not title: title = path.splitext(path.basename(simmap))[0] self.b['title'] = str(title) self.b.add_file(simmap_file, 'application/octet-stream', path.basename(simmap), "simImage_file") self.b.add_file(mapfile_file, 'application/octet-stream', path.basename(mapfile), "mapFile_file") if description: self.b['description'] = str(description) self.b['transparency'] = str(trans) self.b['zoom'] = str(zoom) if details: self.b['details'] = str(details) self.b.submit("form.button.save") return self.b.geturl() def getSimMap(self, url): """ return the SimMap metadata Gets the metadata associated with SimMap including title, description, location, transparency, and zoom. = """ self.b.open(url+'/edit') self.b.select_form('edit_form') d = dict( title = self.b['title'], description = self.b['description'], details = self.b['details'], location = self.b['location'], transparency = self.b['Transparency'], zoom = self.b['zoom'], ) self.b.submit("form.button.cancel") return d def updateSimMap(self, url, **kwargs): """ update the SimMap metadata Keywords must match the field names from the edit form exactly, extra keywords (or mispelled ones) will be ignored. """ self.b.open(url+'/edit') self.b.select_form('edit_form') for k in kwargs: if k in self.b: self.b[k] = str(kwargs[k]) self.b.submit("form.button.save") def getSimMapData(self, url, filename=None): """ getSimMapData -- gets the data component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ bufsize = 15 * 1024 * 1024 rsp = self.b.open(url_join(url,'at_download/simImage')) if filename: f = file(filename, 'wb') while 1: b = rsp.read(bufsize) f.write(b) if len(b) < bufsize: break f.close() return None else: return StringIO(rsp.read()) def getSimMapMapfile(self, url, filename=None): """ getSimMapData -- gets the mapfile component of the the SimMap url: The URL is pointer to the SimMap on the Plone site filename: optional filename where data will be written """ rsp = self.b.open(url_join(url,'at_download/mapFile')) if filename: f = file(filename, 'wb') f.write(rsp.read()) f.close() return None else: data = StringIO() data.write(rsp.read()) return data def putProbmapCache(self, url, filename): """store precomputed probmaps as part of the Driver Set This is a temporary method until the REST interface is ready. It's really depricated before it written! """ self.b.open(url+'/edit') self.b.select_form('edit_form') with open(filename, 'rb') as f: self.b.add_file(f, 'application/octet-stream', path.basename(filename), name='probfile_file') self.b.submit("form.button.save") # def putAttrmapCache(self, url, filepathbasename, filename): # """store precomputed maps as part of the Driver Set # This is a temporary method until the REST interface # is ready. It's really depricated before it written! # """ # self.b.open(url+'/edit') # self.b.select_form('edit_form') # with open(filename, 'w') as f: # self.b.add_file(f, 'application/octet-stream', # path.basename(filepathbasename), name=filename) # self.b.submit("form.button.save") # DELETE FUNCTIONS ----------------------------------------- # These functions delete items from the LEAM Plone sites # ---------------------------------------------------------- def deleteItem(self,fname,url): """ deleteItem Deletes <fname> from the folder <url> """ print '/'.join((url,fname,'delete_confirmation')) print self.b.open('/'.join((url,fname,'delete_confirmation'))) self.b.select_form(None,None,1) print self.b.submit()
def getSolutions(path_prefix, path_proxy): global br, username, password # create a browser object br = Browser() # add proxy support to browser if len(path_proxy) != 0: protocol, proxy = options.proxy.split("://") br.set_proxies({protocol: proxy}) # let browser fool robots.txt br.addheaders = [('User-agent', 'Mozilla/5.0 (X11; U; Linux i686; en-US; \ rv:1.9.0.1) Gecko/2008071615 Fedora/3.0.1-1.fc9 Firefox/3.0.1')] br.set_handle_robots(False) print "Enter yout SPOJ username :"******"Authenticating " + username br.open("http://spoj.pl") br.select_form(name="login") br["login_user"] = username br["password"] = password # sign in for a day to avoid timeouts br.find_control(name="autologin").items[0].selected = True br.form.action = "http://www.spoj.pl" response = br.submit() verify = response.read() if (verify.find("Authentication failed!") != -1): print "Error authenticating - " + username exit(0) # grab the signed submissions list print "Grabbing siglist for " + username siglist = br.open("http://www.spoj.pl/status/" + username + "/signedlist") # dump first nine useless lines in signed list for formatting for i in xrange(9): siglist.readline() # make a list of all AC's and challenges print "Filtering siglist for AC/Challenge solutions..." mysublist = list() while True: temp = siglist.readline() if temp == '\------------------------------------------------------------------------------/\n': # reached end of siglist break if not len(temp): print "Reached EOF, siglist format has probably changed," + \ " contact author." exit(1) entry = [x.strip() for x in temp.split('|')] if entry[4] == 'AC' or entry[4].isdigit(): mysublist.append(entry) print "Done !!!" return mysublist
class RequestQuery: def __init__(self,config): self.br=Browser() self.config = config # Initialise connections self.mySiteDB = SiteDBJSON() self.phedex = PhEDEx({"endpoint":"https://cmsweb.cern.ch/phedex/datasvc/json/prod/"}, "json") self.dbsPhys01 = DbsApi(url = dbs_base_url+"phys01/DBSReader/") self.dbsPhys02 = DbsApi(url = dbs_base_url+"phys02/DBSReader/") self.dbsPhys03 = DbsApi(url = dbs_base_url+"phys03/DBSReader/") def __del__(self): self.br.close() def login2Savannah(self): """ login2Savannah log into savannah with the given parameters in the config (username and password) User must have admin priviledges for store results requests """ login_page='https://savannah.cern.ch/account/login.php?uri=%2F' savannah_page='https://savannah.cern.ch/task/?group=cms-storeresults' self.br.open(login_page) ## 'Search' form is form 0 ## login form is form 1 self.br.select_form(nr=1) username = self.config["SavannahUser"] self.br['form_loginname']=username self.br['form_pw']=self.config["SavannahPasswd"] self.br.submit() response = self.br.open(savannah_page) # Check to see if login was successful if not re.search('Logged in as ' + username, response.read()): print('login unsuccessful, please check your username and password') return False else: return True def selectQueryForm(self,**kargs): """ selectQueryForm create the browser view to get all the store result tickets from savannah """ if self.isLoggedIn: self.br.select_form(name="bug_form") ## Use right query form labelled Test control = self.br.find_control("report_id",type="select") for item in control.items: if item.attrs['label'] == "Test": control.value = [item.attrs['value']] ##select number of entries displayed per page control = self.br.find_control("chunksz",type="text") control.value = "150" ##check additional searching parameter for arg in kargs: if arg == "approval_status": control = self.br.find_control("resolution_id",type="select") for item in control.items: if item.attrs['label'] == kargs[arg].strip(): control.value = [item.attrs['value']] elif arg == "task_status": control = self.br.find_control("status_id",type="select") for item in control.items: if item.attrs['label'] == kargs[arg].strip(): control.value = [item.attrs['value']] elif arg == "team": control = self.br.find_control("custom_sb5",type="select") for item in control.items: if item.attrs['label'] == kargs[arg].strip(): control.value = [item.attrs['value']] response = self.br.submit() response.read() return def getScramArchByCMSSW(self): """ Get from the list of available CMSSW releases return a dictionary of ScramArchitecture by CMSSW """ # Set temporary conection to the server and get the response from cmstags url = 'https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML' br = Browser() br.set_handle_robots(False) response=br.open(url) soup = BeautifulSoup(response.read()) # Dictionary form # {'CMSSW_X_X_X':[slc5_amd64_gcc472], ... } archByCmssw={} # Fill the dictionary for arch in soup.find_all('architecture'): for cmssw in arch.find_all('project'): # CMSSW release cmsswLabel = cmssw.get('label').encode('ascii', 'ignore') if cmsswLabel not in archByCmssw: archByCmssw[cmsswLabel]=[] # ScramArch related to this CMSSW release archName = arch.get('name').encode('ascii', 'ignore') archByCmssw[cmsswLabel].append(archName) return archByCmssw def createValueDicts(self): """ Init dictionaries by value/label: - Releases by Value - Physics group by value - DBS url by value - DBS rul by label - Status of savannah request by value - Status of savannah ticket by value (Open/Closed/Any) """ if self.isLoggedIn: self.br.select_form(name="bug_form") control = self.br.find_control("custom_sb2",type="select") self.ReleaseByValueDict = self.getLabelByValueDict(control) control = self.br.find_control("custom_sb3",type="select") self.GroupByValueDict = self.getLabelByValueDict(control) control = self.br.find_control("custom_sb4",type="select") self.DBSByValueDict = self.getLabelByValueDict(control) self.DBSByLabelDict = self.getValueByLabelDict(control) control = self.br.find_control("resolution_id",type="select") self.StatusByValueDict = self.getLabelByValueDict(control) control = self.br.find_control("status_id",type="select") self.TicketStatusByLabelDict = self.getValueByLabelDict(control) return def getDatasetOriginSites(self, dbs_url, data): """ Get the origin sites for each block of the dataset. Return a list block origin sites. """ local_dbs = dbs_url.split('/')[5] if local_dbs == 'phys01': response = self.dbsPhys01.listBlocks(detail=True,dataset=data) elif local_dbs == 'phys02': response = self.dbsPhys02.listBlocks(detail=True,dataset=data) elif local_dbs == 'phys03': response = self.dbsPhys03.listBlocks(detail=True,dataset=data) pnnList = set() for block in response: pnnList.add(block['origin_site_name']) psnList = self.mySiteDB.PNNstoPSNs(pnnList) return psnList, list(pnnList) def phEDExNodetocmsName(self, nodeList): """ Convert PhEDEx node name list to cms names list """ names = [] for node in nodeList: name = node.replace('_MSS', '').replace('_Disk', '').replace('_Buffer', '').replace('_Export', '') if name not in names: names.append(name) return names def setGlobalTagFromOrigin(self, dbs_url,input_dataset): """ Get the global tag of the dataset from the source dbs url. If it is not set, then set global tag to 'UNKNOWN' """ globalTag = "" local_dbs = dbs_url.split('/')[5] if local_dbs == 'phys01': response = self.dbsPhys01.listOutputConfigs(dataset=input_dataset) elif local_dbs == 'phys02': response = self.dbsPhys02.listOutputConfigs(dataset=input_dataset) elif local_dbs == 'phys03': response = self.dbsPhys03.listOutputConfigs(dataset=input_dataset) globalTag = response[0]['global_tag'] # GlobalTag cannot be empty if globalTag == '': globalTag = 'UNKNOWN' return globalTag def isDataAtUrl(self, dbs_url,input_dataset): """ Returns True if the dataset is at the dbs url, if not returns False """ local_dbs = dbs_url.split('/')[5] if local_dbs == 'phys01': response = self.dbsPhys01.listDatasets(dataset=input_dataset) elif local_dbs == 'phys02': response = self.dbsPhys02.listDatasets(dataset=input_dataset) elif local_dbs == 'phys03': response = self.dbsPhys03.listDatasets(dataset=input_dataset) # This means that the dataset is not at the url if not response: return False else: return True def getLabelByValueDict(self, control): """ From control items, create a dictionary by values """ d = {} for item in control.items: value = item.attrs['value'] label = item.attrs['label'] d[value] = label return d def getValueByLabelDict(self, control): """ From control items, create a dictionary by labels """ d = {} for item in control.items: value = item.attrs['value'] label = item.attrs['label'] d[label] = value return d def getRequests(self,**kargs): """ getRequests Actually goes through all the savannah requests and create json files if the ticket is not Closed and the status of the item is Done. It also reports back the summary of the requests in savannah """ requests = [] # Open Browser and login into Savannah self.br=Browser() self.isLoggedIn = self.login2Savannah() if self.isLoggedIn: if not kargs: self.selectQueryForm(approval_status='1',task_status='0') else: self.selectQueryForm(**kargs) self.createValueDicts() self.br.select_form(name="bug_form") response = self.br.submit() html_ouput = response.read() scramArchByCMSSW = self.getScramArchByCMSSW() self.nodeMappings = self.phedex.getNodeMap() for link in self.br.links(text_regex="#[0-9]+"): response = self.br.follow_link(link) try: ## Get Information self.br.select_form(name="item_form") ## remove leading   and # from task task = link.text.replace('#','').decode('utf-8').strip() print("Processing ticket: %s" % task) ## Get input dataset name control = self.br.find_control("custom_tf1",type="text") input_dataset = control.value input_primary_dataset = input_dataset.split('/')[1].replace(' ','') input_processed_dataset = input_dataset.split('/')[2].replace(' ','') data_tier = input_dataset.split('/')[3].replace(' ','') ## Get DBS URL by Drop Down control = self.br.find_control("custom_sb4",type="select") dbs_url = self.DBSByValueDict[control.value[0]] ## Get DBS URL by text field (for old entries) if dbs_url=='None': control = self.br.find_control("custom_tf4",type="text") dbs_url = control.value.replace(' ','') else: # Transform input value to a valid DBS url #dbs_url = "https://cmsweb.cern.ch/dbs/prod/"+dbs_url+"/DBSReader" dbs_url = dbs_base_url+dbs_url+"/DBSReader" ## Get Release control = self.br.find_control("custom_sb2",type="select") release_id = control.value ## Get current request status control = self.br.find_control("status_id",type="select") request_status_id = control.value RequestStatusByValueDict = self.getLabelByValueDict(control) # close the request if deprecated release was used try: release = self.ReleaseByValueDict[release_id[0]] except: if len(self.ReleaseByValueDict)>0 and RequestStatusByValueDict[request_status_id[0]] != "Closed": msg = "Your request is not valid anymore, since the given CMSSW release is deprecated. If your request should be still processed, please reopen the request and update the CMSSW release to a more recent *working* release.\n" msg+= "\n" msg+= "Thanks,\n" msg+= "Your StoreResults team" self.closeRequest(task,msg) self.br.back() print("I tried to Close ticket %s due to CMSSW not valid" % task) continue # close the request if release has not ScramArch match if release not in scramArchByCMSSW: if len(self.ReleaseByValueDict)>0 and RequestStatusByValueDict[request_status_id[0]] != "Closed": msg = "Your request is not valid, there is no ScramArch match for the given CMSSW release.\n" msg+= "If your request should be still processed, please reopen the request and update the CMSSW release according to: https://cmssdt.cern.ch/SDT/cgi-bin/ReleasesXML \n" msg+= "\n" msg+= "Thanks,\n" msg+= "Your StoreResults team" self.closeRequest(task,msg) self.br.back() print("I tried to Close ticket %s due to ScramArch mismatch" % task) continue else: index=len(scramArchByCMSSW[release]) scram_arch = scramArchByCMSSW[release][index-1] # close the request if dataset is not at dbs url try: data_at_url = self.isDataAtUrl(dbs_url,input_dataset) except: print('I got an error trying to look for dataset %s at %s, please look at this ticket: %s' %(input_dataset,dbs_url,task)) continue if not data_at_url: msg = "Your request is not valid, I could not find the given dataset at %s\n" % dbs_url msg+= "If your request should be still processed, please reopen the request and change DBS url properly \n" msg+= "\n" msg+= "Thanks,\n" msg+= "Your StoreResults team" self.closeRequest(task,msg) self.br.back() print("I tried to Close ticket %s, dataset is not at DBS url" % task) continue # Avoid not approved Tickets #if not RequestStatusByValueDict[request_status_id[0]] == "Done": # continue ## Get Physics Group control = self.br.find_control("custom_sb3",type="select") group_id = control.value[0] group_squad = 'cms-storeresults-'+self.GroupByValueDict[group_id].replace("-","_").lower() ## Get Dataset Version control = self.br.find_control("custom_tf3",type="text") dataset_version = control.value.replace(' ','') if dataset_version == "": dataset_version = '1' ## Get current status control = self.br.find_control("resolution_id",type="select") status_id = control.value ## Get assigned to control = self.br.find_control("assigned_to",type="select") AssignedToByValueDict = self.getLabelByValueDict(control) assignedTo_id = control.value ##Assign task to the physics group squad if AssignedToByValueDict[assignedTo_id[0]]!=group_squad: assignedTo_id = [self.getValueByLabelDict(control)[group_squad]] control.value = assignedTo_id self.br.submit() # Set default Adquisition Era for StoreResults acquisitionEra = "StoreResults" ## Construction of the new dataset name (ProcessingString) ## remove leading hypernews or physics group name and StoreResults+Version if input_processed_dataset.find(self.GroupByValueDict[group_id])==0: new_dataset = input_processed_dataset.replace(self.GroupByValueDict[group_id],"",1) else: stripped_dataset = input_processed_dataset.split("-")[1:] new_dataset = '_'.join(stripped_dataset) except Exception as ex: self.br.back() print("There is a problem with this ticket %s, please have a look to the error:" % task) print(str(ex)) print(traceback.format_exc()) continue self.br.back() # Get dataset site info: psnList, pnnList = self.getDatasetOriginSites(dbs_url,input_dataset) infoDict = {} # Build store results json # First add all the defaults values infoDict["RequestType"] = "StoreResults" infoDict["UnmergedLFNBase"] = "/store/unmerged" infoDict["MergedLFNBase"] = "/store/results/" + self.GroupByValueDict[group_id].replace("-","_").lower() infoDict["MinMergeSize"] = 1500000000 infoDict["MaxMergeSize"] = 5000000000 infoDict["MaxMergeEvents"] = 100000 infoDict["TimePerEvent"] = 40 infoDict["SizePerEvent"] = 512.0 infoDict["Memory"] = 2394 infoDict["CmsPath"] = "/uscmst1/prod/sw/cms" infoDict["Group"] = "DATAOPS" infoDict["DbsUrl"] = dbs_url # Add all the information pulled from Savannah infoDict["AcquisitionEra"] = acquisitionEra infoDict["GlobalTag"] = self.setGlobalTagFromOrigin(dbs_url,input_dataset) infoDict["DataTier"] = data_tier infoDict["InputDataset"] = input_dataset infoDict["ProcessingString"] = new_dataset infoDict["CMSSWVersion"] = release infoDict["ScramArch"] = scram_arch infoDict["ProcessingVersion"] = dataset_version infoDict["SiteWhitelist"] = psnList # Create report for Migration2Global report = {} #Fill json file, if status is done if self.StatusByValueDict[status_id[0]]=='Done' and RequestStatusByValueDict[request_status_id[0]] != "Closed": self.writeJSONFile(task, infoDict) report["json"] = 'y' else: report["json"] = 'n' report["task"] = int(task) report["InputDataset"] = input_dataset report["ProcessingString"] = new_dataset report["ticketStatus"] = self.StatusByValueDict[status_id[0]] report["assignedTo"] = AssignedToByValueDict[assignedTo_id[0]] report["localUrl"] = dbs_url report["sites"] = psnList report["pnns"] = pnnList # if the request is closed, change the item status to report to Closed if report["ticketStatus"] == "Done" and RequestStatusByValueDict[request_status_id[0]] == "Closed": report["ticketStatus"] = "Closed" requests.append(report) # Print out report self.printReport(requests) # Close connections self.br.close() return requests def closeRequest(self,task,msg): """ This close a specific savannag ticket Insert a message in the ticket """ if self.isLoggedIn: #self.createValueDicts() response = self.br.open('https://savannah.cern.ch/task/?'+str(task)) html = response.read() self.br.select_form(name="item_form") control = self.br.find_control("status_id",type="select") control.value = [self.TicketStatusByLabelDict["Closed"]] #Put reason to the comment field control = self.br.find_control("comment",type="textarea") control.value = msg #DBS Drop Down is a mandatory field, if set to None (for old requests), it is not possible to close the request self.setDBSDropDown() self.br.submit() #remove JSON ticket self.removeJSONFile(task) self.br.back() return def setDBSDropDown(self): ## Get DBS URL by Drop Down control = self.br.find_control("custom_sb4",type="select") dbs_url = self.DBSByValueDict[control.value[0]] ## Get DBS URL by text field (for old entries) if dbs_url=='None': tmp = self.br.find_control("custom_tf4",type="text") dbs_url = tmp.value.replace(' ','') if dbs_url.find("phys01")!=-1: control.value = [self.DBSByLabelDict["phys01"]] elif dbs_url.find("phys02")!=-1: control.value = [self.DBSByLabelDict["phys02"]] elif dbs_url.find("phys03")!=-1: control.value = [self.DBSByLabelDict["phys03"]] else: msg = 'DBS URL of the old request is neither phys01, phys02 nor phys03. Please, check!' print(msg) raise RuntimeError(msg) return def writeJSONFile(self, task, infoDict): """ This writes a JSON file at ComponentDir """ ##check if file already exists filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json' if not os.access(filename,os.F_OK): jsonfile = open(filename,'w') request = {'createRequest':infoDict} ## CHECK THIS BEFORE FINISHING jsonfile.write(json.dumps(request,sort_keys=True, indent=4)) jsonfile.close return def removeJSONFile(self,task): """ This removes the JSON file at ComponentDir if it was created """ filename = self.config["ComponentDir"]+'/Ticket_'+str(task)+'.json' if os.access(filename,os.F_OK): os.remove(filename) return def printReport(self, requests): """ Print out a report """ print("%20s %10s %5s %35s %10s %50s %50s" %( 'Savannah Ticket','Status','json','Assigned to','local DBS','Sites','pnns')) print("%20s %10s %5s %35s %10s %50s %50s" %( '-'*20,'-'*10,'-'*5,'-'*35,'-'*10,'-'*50,'-'*50 )) for report in requests: json = report["json"] ticket = report["task"] status = report["ticketStatus"] assigned = report["assignedTo"] localUrl = report["localUrl"].split('/')[5] site = ', '.join(report["sites"]) pnns = ', '.join(report["pnns"]) print("%20s %10s %5s %35s %10s %50s %50s" %(ticket,status,json,assigned,localUrl,site,pnns))
class Installer(object): # License fields FIELD_LICENSE_KEY = 'lkey' # Server Detail fields FIELD_SERVER_SQL_HOST = 'sql_host' FIELD_SERVER_SQL_USER = '******' FIELD_SERVER_SQL_PASS = '******' FIELD_SERVER_SQL_DATABASE = 'sql_database' # Admin fields FIELD_ADMIN_USER = '******' FIELD_ADMIN_PASS = '******' FIELD_ADMIN_PASS_CONFIRM = 'admin_pass2' FIELD_ADMIN_EMAIL = 'admin_email' def __init__(self, ctx, site, force=False): """ Initialize a new Installer instance @type ctx: ips_vagrant.cli.Context @param site: The IPS Site we are installing @type site: ips_vagrant.models.sites.Site @param force: Overwrite existing files / databases @type force: bool """ self.log = logging.getLogger('ipsv.installer') self.ctx = ctx self.force = force self._previous_title = None self.url = '{scheme}://{host}/admin/install'.format( scheme='https' if site.ssl else 'http', host=site.domain.name ) self.site = site self.mysql = create_engine('mysql://*****:*****@localhost') self._sessions = {} self.cookiejar = cookiejar() self.cookies = {cookie.name: cookie.value for cookie in self.cookiejar} self.browser = Browser() self.browser.set_cookiejar(self.cookiejar) def _check_title(self, title): """ If we're on the same page, we got an error and need to raise an exception @type title: str @raise InstallationError: Title matches the previous page requests title (We're on the same page) """ self.log.info('Installation page loaded: %s', title) if self._previous_title and title == self._previous_title: raise InstallationError('Unexpected page error') self._previous_title = title def start(self): """ Start the installation wizard """ self.log.debug('Starting the installation process') self.system_check() def system_check(self): """ System requirements check """ self.browser.open(self.url) self._check_title(self.browser.title()) p = Echo('Running system check...') rsoup = BeautifulSoup(self.browser.response().read(), "html.parser") # Check for any errors errors = [] for ul in rsoup.find_all('ul', {'class': 'ipsList_checks'}): for li in ul.find_all('li', {'class': 'fail'}): errors.append(li.text) if errors: raise InstallationError(errors) # Continue continue_link = next(self.browser.links(text_regex='Continue')) p.done() self.browser.follow_link(continue_link) self.license() def license(self): """ Submit our license to IPS' servers """ p = Echo('Submitting license key...') self._check_title(self.browser.title()) self.browser.select_form(nr=0) # Set the fields self.browser.form[self.FIELD_LICENSE_KEY] = '{license}-TESTINSTALL'.format(license=self.site.license_key) self.browser.find_control('eula_checkbox').items[0].selected = True # TODO: User prompt? # Submit the request self.log.debug('Submitting our license') self.browser.submit() self.log.debug('Response code: %s', self.browser.response().code) p.done() self.applications() def applications(self): """ Select the applications to install (currently hardcoded to install all applications) """ # Check for license submission errors try: self._check_title(self.browser.title()) except InstallationError: rsoup = BeautifulSoup(self.browser.response().read(), "html.parser") error = rsoup.find('li', id='license_lkey').find('span', {'class': 'ipsType_warning'}).text raise InstallationError(error) # TODO: Make this configurable p = Echo('Setting applications to install...') self.browser.select_form(nr=0) self.browser.submit() p.done() self.server_details() def server_details(self): """ Input server details (database information, etc.) """ self._check_title(self.browser.title()) p = Echo('Creating MySQL database...') # Create the database md5hex = md5(self.site.domain.name + self.site.slug).hexdigest() db_name = 'ipsv_{md5}'.format(md5=md5hex) # MySQL usernames are limited to 16 characters max db_user = '******'.format(md5=md5hex[:11]) rand_pass = ''.join(random.SystemRandom() .choice(string.ascii_letters + string.digits) for _ in range(random.randint(16, 24))) db_pass = rand_pass try: self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name)) except SQLAlchemyError: if not self.force: click.confirm('A previous database for this installation already exists.\n' 'Would you like to drop it now? The installation will be aborted if you do not', abort=True) self.log.info('Dropping existing database: {db}'.format(db=db_name)) self.mysql.execute('DROP DATABASE IF EXISTS `{db}`'.format(db=db_name)) self.mysql.execute('DROP USER IF EXISTS `{u}`'.format(u=db_user)) self.mysql.execute('CREATE DATABASE `{db}`'.format(db=db_name)) self.mysql.execute("GRANT ALL ON {db}.* TO '{u}'@'localhost' IDENTIFIED BY '{p}'" .format(db=db_name, u=db_user, p=db_pass)) # Save the database connection information self.site.db_host = 'localhost' self.site.db_name = db_name self.site.db_user = db_user self.site.db_pass = db_pass self.ctx.db.commit() self.log.debug('MySQL Database Name: %s', db_name) self.log.debug('MySQL Database User: %s', db_user) self.log.debug('MySQL Database Password: %s', db_pass) # Set form fields and submit self.browser.select_form(nr=0) self.browser.form[self.FIELD_SERVER_SQL_HOST] = 'localhost' self.browser.form[self.FIELD_SERVER_SQL_USER] = db_user self.browser.form[self.FIELD_SERVER_SQL_PASS] = db_pass self.browser.form[self.FIELD_SERVER_SQL_DATABASE] = db_name self.browser.submit() p.done() self.admin() def admin(self): """ Provide admin login credentials """ self._check_title(self.browser.title()) self.browser.select_form(nr=0) # Get the admin credentials prompted = [] user = self.ctx.config.get('User', 'AdminUser') if not user: user = click.prompt('Admin display name') prompted.append('user') password = self.ctx.config.get('User', 'AdminPass') if not password: password = click.prompt('Admin password', hide_input=True, confirmation_prompt='Confirm admin password') prompted.append('password') email = self.ctx.config.get('User', 'AdminEmail') if not email: email = click.prompt('Admin email') prompted.append('email') self.browser.form[self.FIELD_ADMIN_USER] = user self.browser.form[self.FIELD_ADMIN_PASS] = password self.browser.form[self.FIELD_ADMIN_PASS_CONFIRM] = password self.browser.form[self.FIELD_ADMIN_EMAIL] = email p = Echo('Submitting admin information...') self.browser.submit() p.done() if len(prompted) >= 3: save = click.confirm('Would you like to save and use these admin credentials for future installations?') if save: self.log.info('Saving admin login credentials') self.ctx.config.set('User', 'AdminUser', user) self.ctx.config.set('User', 'AdminPass', password) self.ctx.config.set('User', 'AdminEmail', email) with open(self.ctx.config_path, 'wb') as cf: self.ctx.config.write(cf) self.install() def _start_install(self): """ Start the installation """ self._check_title(self.browser.title()) continue_link = next(self.browser.links(text_regex='Start Installation')) self.browser.follow_link(continue_link) def _get_mr_link(self): """ Get the MultipleRedirect URL @rtype: str """ rsoup = BeautifulSoup(self.browser.response().read(), "html.parser") mr_link = rsoup.find('a', {'class': 'button'}).get('href') self.log.debug('MultipleRedirect link: %s', mr_link) return mr_link def _ajax(self, url, method='get', params=None, load_json=True, raise_request=True): """ Perform an Ajax request @type url: str @type method: str @type params: dict or None @type load_json: bool @return: Tuple with the decoded JSON response and actual response, or just the response if load_json is False @rtype: requests.Response or tuple of (dict or list, requests.Response) """ if 'ajax' in self._sessions: ajax = self._sessions['ajax'] else: self.log.debug('Instantiating a new Ajax session') ajax = requests.Session() ajax.headers.update({'X-Requested-With': 'XMLHttpRequest'}) ajax.cookies.update(cookiejar()) ajax.verify = False self._sessions['ajax'] = ajax response = ajax.request(method, url, params) self.log.debug('Ajax response: %s', response.text) if raise_request: response.raise_for_status() if load_json: return byteify(json.loads(response.text)), response return response def _request(self, url, method='get', params=None, raise_request=True): """ Perform a standard HTTP request @type url: str @type method: str @type params: dict or None @param raise_request: Raise exceptions for HTTP status errors @type raise_request: bool @rtype: requests.Response """ if 'http' in self._sessions: http = self._sessions['http'] else: self.log.debug('Instantiating a new HTTP session') http = requests.Session() http.cookies.update(cookiejar()) http.verify = False self._sessions['http'] = http response = http.request(method, url, params) self.log.debug('HTTP response code: %s', response.status_code) if raise_request: response.raise_for_status() return response def _parse_response(self, url, json_response): """ Parse response data and return the next request URL @type url: str @type json_response: list or dict @rtype: str """ parts = list(urlparse(url)) query = parse_qs(parts[4]) query['mr'] = str(json_response[0]).replace('\'', '"') parts[4] = urlencode(query, True) return urlunparse(parts) def _get_stage(self, json_response): """ Get the current installation stage @type json_response: list or dict @rtype: str """ try: return json_response[1] except IndexError: return 'Installation complete!' def _get_progress(self, json_response): """ Get the current installation progress @type json_response: list or dict @rtype: str """ try: return round(float(json_response[2])) except IndexError: return 0 def _check_if_complete(self, url, json_response): """ Check if a request has been completed and return the redirect URL if it has @type url: str @type json_response: list or dict @rtype: str or bool """ if 'redirect' in json_response and isinstance(json_response, dict): self.log.info('Installation complete') return json_response['redirect'] return False def _finalize(self, response): """ Finalize the installation and display a link to the suite """ rsoup = BeautifulSoup(response.text, "html.parser") click.echo('------') click.secho(rsoup.find('h1', id='elInstaller_welcome').text.strip(), fg='yellow', bold=True) click.secho(rsoup.find('p', {'class': 'ipsType_light'}).text.strip(), fg='yellow', dim=True) link = rsoup.find('a', {'class': 'ipsButton_primary'}).get('href') click.echo(click.style('Go to the suite: ', bold=True) + link + '\n') # noinspection PyUnboundLocalVariable def install(self): """ Run the actual installation """ self._start_install() mr_link = self._get_mr_link() # Set up the progress bar pbar = ProgressBar(100, 'Running installation...') pbar.start() mr_j, mr_r = self._ajax(mr_link) # Loop until we get a redirect json response while True: mr_link = self._parse_response(mr_link, mr_j) stage = self._get_stage(mr_j) progress = self._get_progress(mr_j) mr_j, mr_r = self._ajax(mr_link) pbar.update(min([progress, 100]), stage) # NOTE: Response may return progress values above 100 # If we're done, finalize the installation and break redirect = self._check_if_complete(mr_link, mr_j) if redirect: pbar.finish() break p = Echo('Finalizing...') mr_r = self._request(redirect, raise_request=False) p.done() # Install developer tools if self.site.in_dev: DevToolsInstaller(self.ctx, self.site).install() # Get the link to our community homepage self._finalize(mr_r)