def get_genbank_accession_from_ucsc_name(db):
"""Queries NCBI EUtils for the GenBank accession of a UCSC asseembly name
"""
global time_ncbi
t0 = time_ms()
logger.info('Fetching GenBank accession from NCBI EUtils for: ' + db)
eutils = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/'
esearch = eutils + 'esearch.fcgi?retmode=json'
esummary = eutils + 'esummary.fcgi?retmode=json'
asm_search = esearch + '&db=assembly&term=' + db
# Example: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=assembly&retmode=json&term=panTro4
data = json.loads(request(asm_search))
id_list = data['esearchresult']['idlist']
if len(id_list) > 0:
assembly_uid = id_list[0]
else:
unfound_dbs.append(db)
return ''
asm_summary = esummary + '&db=assembly&id=' + assembly_uid
# Example: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?retmode=json&db=assembly&id=255628
data = json.loads(request(asm_summary))
result = data['result'][assembly_uid]
acc = result['assemblyaccession'] # Accession.version
# Return GenBank accession if it's default, else find and return it
if "GCA_" not in acc:
acc = result['synonym']['genbank']
time_ncbi += time_ms() - t0
return acc
def test_development_env(self):
"""
Test traceback thrown in development env
"""
setup_server()
# test 404
resp = json.loads(request("/tasks/blah").read())
self.assertEquals("404 Not Found", resp.get("code"))
# test 405 wrong method
resp = json.loads(request("/", None, "DELETE").read())
msg = u"WOKAPI0002E: Delete is not allowed for wokroot"
self.assertEquals("405 Method Not Allowed", resp.get("code"))
self.assertEquals(msg, resp.get("reason"))
# test 400 parse error
resp = json.loads(request("/tasks", "{", "POST").read())
msg = u"WOKAPI0006E: Unable to parse JSON request"
self.assertEquals("400 Bad Request", resp.get("code"))
self.assertEquals(msg, resp.get("reason"))
self.assertIn("call_stack", resp)
# test 405 method not allowed
req = json.dumps({})
resp = json.loads(request("/tasks", req, "POST").read())
m = u"WOKAPI0005E: Create is not allowed for tasks"
self.assertEquals("405 Method Not Allowed", resp.get("code"))
self.assertEquals(m, resp.get("reason"))
self.assertIn("call_stack", resp)
def get_product_url_for_category(category_url):
links = []
soup = utils.request(category_url) # r = requests.get(category_url)
boutonNext = soup.find("li", {"class": "next"})
index = 1
articles = soup.find_all('article', 'product_pod')
for article in articles:
a = article.find('a')
link = a['href']
url_article = link.replace('../../../',
'http://books.toscrape.com/catalogue/')
links.append(url_article)
while boutonNext is not None:
index += 1
category_url = category_url.replace("index.html",
"page-" + str(index) + '.html')
soup = utils.request(category_url) # r = requests.get(category_url)
articles = soup.findAll('article', 'product_pod')
for article in articles:
a = article.find('a')
link = a['href']
url_article = link.replace('../../../',
'http://books.toscrape.com/catalogue/')
links.append(url_article)
boutonNext = soup.find("li", {"class": "next"})
for link in links:
soup = utils.request(link) #r = requests.get(link)
book = scraping.scrap_book(link)
file = dictwriter_csv.write_csv(book)
return file
def test_development_env(self):
"""
Test traceback thrown in development env
"""
setup_server()
# test 404
resp = json.loads(request(host, port, '/vms/blah').read())
self.assertEquals('404 Not Found', resp.get('code'))
# test 405 wrong method
resp = json.loads(request(host, port, '/', None, 'DELETE').read())
msg = u'KCHAPI0002E: Delete is not allowed for kimchiroot'
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
# test 400 parse error
resp = json.loads(request(host, port, '/vms', '{', 'POST').read())
msg = u'KCHAPI0006E: Unable to parse JSON request'
self.assertEquals('400 Bad Request', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
self.assertIn('call_stack', resp)
# test 400 missing required parameter
req = json.dumps({})
resp = json.loads(request(host, port, '/vms', req, 'POST').read())
msg = u"KCHVM0016E: Specify a template to create a virtual machine from"
self.assertEquals('400 Bad Request', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
self.assertIn('call_stack', resp)
def test_development_env(self):
"""
Test traceback thrown in development env
"""
setup_server()
# test 404
resp = json.loads(request(host, ssl_port, '/vms/blah').read())
self.assertEquals('404 Not Found', resp.get('code'))
# test 405 wrong method
resp = json.loads(request(host, ssl_port, '/', None, 'DELETE').read())
msg = u'KCHAPI0002E: Delete is not allowed for kimchiroot'
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
# test 400 parse error
resp = json.loads(request(host, ssl_port, '/vms', '{', 'POST').read())
msg = u'KCHAPI0006E: Unable to parse JSON request'
self.assertEquals('400 Bad Request', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
self.assertIn('call_stack', resp)
# test 400 missing required parameter
req = json.dumps({})
resp = json.loads(request(host, ssl_port, '/vms', req, 'POST').read())
msg = u"KCHVM0016E: Specify a template to create a virtual machine from"
self.assertEquals('400 Bad Request', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
self.assertIn('call_stack', resp)
def test_development_env(self):
"""
Test traceback thrown in development env
"""
setup_server()
# test 404
resp = json.loads(request('/tasks/blah').read())
self.assertEquals('404 Not Found', resp.get('code'))
# test 405 wrong method
resp = json.loads(request('/', None, 'DELETE').read())
msg = u'WOKAPI0002E: Delete is not allowed for wokroot'
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
# test 400 parse error
resp = json.loads(request('/tasks', '{', 'POST').read())
msg = u'WOKAPI0006E: Unable to parse JSON request'
self.assertEquals('400 Bad Request', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
self.assertIn('call_stack', resp)
# test 405 method not allowed
req = json.dumps({})
resp = json.loads(request('/tasks', req, 'POST').read())
m = u"WOKAPI0005E: Create is not allowed for tasks"
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(m, resp.get('reason'))
self.assertIn('call_stack', resp)
def test_development_env(self):
"""
Test traceback thrown in development env
"""
setup_server()
# test 404
resp = json.loads(request('/tasks/blah').read())
self.assertEquals('404 Not Found', resp.get('code'))
# test 405 wrong method
resp = json.loads(request('/', None, 'DELETE').read())
msg = u'WOKAPI0002E: Delete is not allowed for wokroot'
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
# test 400 parse error
resp = json.loads(request('/tasks', '{', 'POST').read())
msg = u'WOKAPI0006E: Unable to parse JSON request'
self.assertEquals('400 Bad Request', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
self.assertIn('call_stack', resp)
# test 405 method not allowed
req = json.dumps({})
resp = json.loads(request('/tasks', req, 'POST').read())
m = u"WOKAPI0005E: Create is not allowed for tasks"
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(m, resp.get('reason'))
self.assertIn('call_stack', resp)
def test_posts_comm_get(param_posts):
# get запрос постов, группировка postId, 2 способа запроса одного контента.
url1 = gbl.posts_url + str(param_posts) + "/comments"
q = request(url1, gbl.schema_posts_comments_get(param_posts))
url2 = gbl.comments_url
payload = {"postId": param_posts}
q1 = request(url2, gbl.schema_posts_comments_get(param_posts), payload)
q.get_simple()
q1.get_payload()
def test_albums_phot_get(param_albums):
# get запрос альбомов, группировка albumId, 2 способа запроса 1 контента.
url1 = gbl.albums_url + str(param_albums) + "/photos"
url2 = gbl.photos_url
payload = {"albumId": param_albums}
q = request(url1, gbl.schema_albums_phot_get(param_albums))
q1 = request(url2, gbl.schema_albums_phot_get(param_albums), payload)
q.get_simple()
q1.get_payload()
def cleanup(self):
print("Complete server cleanup")
file_list = json.loads(
request(self.__json_data, 'list', self.__service_url,
self.__client_name, {}))['List']
for file in file_list:
request(self.__json_data, 'remove', self.__service_url,
self.__client_name, {'Hash': file['Hash']})
logging.info("Removed " + file['Hash'] + " from " +
self.__service_url)
def test_screenshot_refresh(self):
# Create a VM
req = json.dumps({'name': 'test', 'cdrom': '/nonexistent.iso'})
request(host, port, '/templates', req, 'POST')
req = json.dumps({'name': 'test-vm', 'template': '/templates/test'})
request(host, port, '/vms', req, 'POST')
# Test screenshot refresh for running vm
request(host, port, '/vms/test-vm/start', '{}', 'POST')
resp = request(host, port, '/vms/test-vm/screenshot')
self.assertEquals(200, resp.status)
self.assertEquals('image/png', resp.getheader('content-type'))
resp1 = request(host, port, '/vms/test-vm')
rspBody = resp1.read()
testvm_Data = json.loads(rspBody)
screenshotURL = testvm_Data['screenshot']
time.sleep(5)
resp2 = request(host, port, screenshotURL)
self.assertEquals(200, resp2.status)
self.assertEquals(resp2.getheader('content-type'),
resp.getheader('content-type'))
self.assertEquals(resp2.getheader('content-length'),
resp.getheader('content-length'))
self.assertEquals(resp2.getheader('last-modified'),
resp.getheader('last-modified'))
def test_screenshot_refresh(self):
# Create a VM
req = json.dumps({'name': 'test', 'cdrom': '/nonexistent.iso'})
request(host, ssl_port, '/templates', req, 'POST')
req = json.dumps({'name': 'test-vm', 'template': '/templates/test'})
request(host, ssl_port, '/vms', req, 'POST')
# Test screenshot refresh for running vm
request(host, ssl_port, '/vms/test-vm/start', '{}', 'POST')
resp = request(host, ssl_port, '/vms/test-vm/screenshot')
self.assertEquals(200, resp.status)
self.assertEquals('image/png', resp.getheader('content-type'))
resp1 = request(host, ssl_port, '/vms/test-vm')
rspBody = resp1.read()
testvm_Data = json.loads(rspBody)
screenshotURL = testvm_Data['screenshot']
time.sleep(5)
resp2 = request(host, ssl_port, screenshotURL)
self.assertEquals(200, resp2.status)
self.assertEquals(resp2.getheader('content-type'),
resp.getheader('content-type'))
self.assertEquals(resp2.getheader('content-length'),
resp.getheader('content-length'))
self.assertEquals(resp2.getheader('last-modified'),
resp.getheader('last-modified'))
def set_card_cover(card_id, attachment_id):
target = f"{TRELLO_API}/cards/{card_id}"
params = {
"idAttachmentCover": attachment_id,
"key": TRELLO_CLIENT_ID,
"token": TRELLO_ACCESS_TOKEN
}
request(put, target, params)
def close_and_delete_board(board_id):
target = f"{TRELLO_API}/boards/{board_id}"
params = {
"closed": "true",
"key": TRELLO_CLIENT_ID,
"token": TRELLO_ACCESS_TOKEN
}
request(put, target, params)
request(delete, target, params)
def test_screenshot_refresh(self):
# Create a VM
req = json.dumps({'name': 'test', 'cdrom': fake_iso})
request(host, ssl_port, '/plugins/kimchi/templates', req, 'POST')
req = json.dumps({
'name': 'test-vm',
'template': '/plugins/kimchi/templates/test'
})
resp = request(host, ssl_port, '/plugins/kimchi/vms', req, 'POST')
task = json.loads(resp.read())
wait_task(model.task_lookup, task['id'])
# Test screenshot refresh for running vm
request(host, ssl_port, '/plugins/kimchi/vms/test-vm/start', '{}',
'POST')
resp = request(host, ssl_port,
'/plugins/kimchi/vms/test-vm/screenshot')
self.assertEquals(200, resp.status)
self.assertEquals('image/png', resp.getheader('content-type'))
resp1 = request(host, ssl_port, '/plugins/kimchi/vms/test-vm')
rspBody = resp1.read()
testvm_Data = json.loads(rspBody)
screenshotURL = '/' + testvm_Data['screenshot']
time.sleep(5)
resp2 = request(host, ssl_port, screenshotURL)
self.assertEquals(200, resp2.status)
self.assertEquals(resp2.getheader('content-type'),
resp.getheader('content-type'))
self.assertEquals(resp2.getheader('content-length'),
resp.getheader('content-length'))
self.assertEquals(resp2.getheader('last-modified'),
resp.getheader('last-modified'))
def get_products(self):
page = 1
total_page = 1
total_page_scraped = False
while total_page != 0:
resp = request(MAIN_URL.format(page))
if resp:
soup = BeautifulSoup(resp.text, 'lxml')
_prods = soup.select('div.all_products_table div.row')
if not total_page_scraped:
total_page = int(
soup.find(
'span',
class_='pages').get_text().strip().split()[-1])
total_page_scraped = True
for div in _prods[1:]:
div_name = div.find('div', class_='product_name')
name = div_name.get_text().strip()
profile = 'https://reviews.financesonline.com' + div_name.find(
'a').get('href')
category = div.find(
'div', class_='category_name').get_text().strip()
score = div.find(
'div', class_='smart_score_name').get_text().strip()
price = div.find('div',
class_='price_name').get_text().strip()
user_sat = div.find(
'div', class_='user_sat_name').get_text().strip()
support = div.find(
'div', class_='support_name').get_text().strip()
resp = request(profile)
if resp:
desc, features, website, reviews = profile_data(resp)
else:
resp = request_restart(profile)
desc, features, website, reviews = profile_data(resp)
data = [[
name, website, desc, features, price, score, user_sat,
reviews, support, category, profile
]]
write_data(data, name)
page = page + 1
total_page = total_page - 1
def test_vm_list_sorted(self):
req = json.dumps({'name': 'test', 'cdrom': fake_iso})
request(host, ssl_port, '/templates', req, 'POST')
def add_vm(name):
# Create a VM
req = json.dumps({'name': name, 'template': '/templates/test'})
request(host, ssl_port, '/vms', req, 'POST')
vms = [u'abc', u'bca', u'cab', u'xba']
for vm in vms:
add_vm(vm)
vms.append(u'test')
self.assertEqual(model.vms_get_list(), sorted(vms))
def test_vm_list_sorted(self):
req = json.dumps({'name': 'test', 'cdrom': fake_iso})
request(host, ssl_port, '/templates', req, 'POST')
def add_vm(name):
# Create a VM
req = json.dumps({'name': name, 'template': '/templates/test'})
request(host, ssl_port, '/vms', req, 'POST')
vms = [u'abc', u'bca', u'cab', u'xba']
for vm in vms:
add_vm(vm)
vms.append(u'test')
self.assertEqual(model.vms_get_list(), sorted(vms))
def get_data(self, url):
logger.info(f'Getting source - {url}')
resp = request(url)
if resp:
soup = BeautifulSoup(resp.text, 'lxml')
name = self.get_name(soup)
if not name:
return False
website = self.get_website(soup)
hq = self.get_address(soup)
phone = self.get_phone(soup)
desc = self.get_description(soup)
rating = self.get_rating(soup)
reviews = self.get_reviews(soup)
customers = self.get_customers(soup)
category = self.get_category(soup)
profile = url
data = [[
name,
website,
desc,
hq,
phone,
customers,
rating,
reviews,
category,
profile
]]
write_data(data, name)
def test_users_put(param_users):
# Изменяем данные крайних юзеров и одного случайного.
url = gbl.users_url + str(param_users)
payload = {
"id": param_users,
"name": "Glenna Reichert",
"username": "******",
"email": "*****@*****.**",
"address": {
"street": "Dayna Park",
"suite": "Suite 449",
"city": "Bartholomebury",
"zipcode": "76495-3109",
"geo": {
"lat": "24.6463",
"lng": "-168.8889"
}
},
"phone": "(775)976-6794 x41206",
"website": "conrad.com",
"company": {
"name": "Yost and Sons",
"catchPhrase": "Switchable contextually-based project",
"bs": "aggregate real-time technologies"
}
}
q = request(url, gbl.schema_users_post(param_users), payload)
q.put_simple()
def test_users_post():
# post запрос на добавление нового юзера.
k = req(gbl.users_url)
le = k.json_lenght() + 1
payload = {
"name": "Glenna Reichert",
"username": "******",
"email": "*****@*****.**",
"address": {
"street": "Dayna Park",
"suite": "Suite 449",
"city": "Bartholomebury",
"zipcode": "76495-3109",
"geo": {
"lat": "24.6463",
"lng": "-168.8889"
}
},
"phone": "(775)976-6794 x41206",
"website": "conrad.com",
"company": {
"name": "Yost and Sons",
"catchPhrase": "Switchable contextually-based project",
"bs": "aggregate real-time technologies"
}
}
q = request(gbl.users_url, gbl.schema_users_post(le), payload)
q.post_simple()
def get_data(self, url, country):
resp = request(url)
if resp:
soup = BeautifulSoup(resp.text, 'lxml')
startups = soup.select('.startup')
for card in startups:
name = card.get('data-name').replace(';', ',').strip()
desc = card.find('p')
if desc:
desc = desc.get_text().replace(';', ',').strip()
try:
site = card.find(
'i',
class_='fa-globe').find_previous().get('href').strip()
if not site.startswith('http'):
site = 'http://' + site
site = get_fld(site)
except:
site = None
try:
fb = card.find(
'i', class_='fa-facebook-square').find_previous().get(
'href').strip()
except:
fb = None
try:
tw = card.find('i',
class_='fa-twitter').find_previous().get(
'href').strip()
except:
tw = None
data = [[name, site, desc, fb, tw, country]]
write_data(data, name)
def update(who, what, where, host, port=80, url='/uss/sql/update', headers={}):
"""
:param host: HTTP Request host
:param port: HTTP Request port
:param url: HTTP Request url
:param who: SQL option on which table
:param what: {"hostname":"node2", "password":"******"},
:param where: {"ip":"192.168.1.1"}
:returns: { id:0 }, id of the node updated.
"""
d = {}
d['who'] = who
d['what'] = what
d['where'] = where
try:
params = json.dumps(d)
headers["Content-Type"] = "application/json"
rslt = request(host=host, method='POST', url=url,
headers=headers, params=params, port=port)
return json.loads(rslt)
except:
raise
def add_vm(name):
# Create a VM
req = json.dumps({'name': name,
'template': '/plugins/kimchi/templates/test'})
task = json.loads(request(host, ssl_port, '/plugins/kimchi/vms',
req, 'POST').read())
wait_task(model.task_lookup, task['id'])
def test_vm_list_sorted(self):
req = json.dumps({"name": "test", "cdrom": fake_iso})
request(host, ssl_port, "/plugins/kimchi/templates", req, "POST")
def add_vm(name):
# Create a VM
req = json.dumps({"name": name, "template": "/plugins/kimchi/templates/test"})
task = json.loads(request(host, ssl_port, "/plugins/kimchi/vms", req, "POST").read())
wait_task(model.task_lookup, task["id"])
vms = [u"abc", u"bca", u"cab", u"xba"]
for vm in vms:
add_vm(vm)
vms.append(u"test")
self.assertEqual(model.vms_get_list(), sorted(vms))
def get_engineers(cookies, merchant):
"""获取京东的在职师傅"""
api_url = 'http://jdfw.jd.com/engineer/queryEngineerBy.json'
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',
'Host': 'jdfw.jd.com',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36Name',
'X-Requested-With': 'XMLHttpRequest',
'Referer': 'http://jdfw.jd.com/engineer/engineerPage',
'Cookie': cookies
}
form_data = {
'belongsBranchId': '4',
'belongLargewarehouseId': 'lw_4_992%%4_2',
'websiteNo': merchant.website_id,
'websiteName': '成华区智诚电器经营部',
'currentStatus': '1',
'mobile': '',
'engineerName': '',
'personId': '',
'jdAccount': '',
'page': '1',
'rows': '50',
}
# form_data = json.dumps(form_data)
r = request(api_url, method='post', data=form_data, headers=headers)
# print('get_engineers:%s'%r)
return r
def _get_bitbucket_license(license_url):
logger.info("Downloading license file from Bitbucket")
license_content = utils.request("GET",
license_url,
json_output=False,
stream=True)
return license_content
def addrepairmachine(self,data):
url = self.aux_base_url + 'addrepairmachine/'
requestData = {
'orderno':data['orderno'],
'repairinfo':data['repairinfo'],
'piclist':data['piclist'],
'lng':data['lng'],
'guarantee':data['guarantee'],
'userid':data['userid'],
'mac':data['mac'],
'machinetype':'空调',
'token':data['token'],
'buydate':data['buydate'],
'traftype':data['traftype'],
'servdesc':data['servdesc'],
'kilometer2':data['kilometer2'],
'appid':0,
'sn':data['sn'],
'brand':'奥克斯',
'lat':data['lat'],
'servmethod':data['servmethod']
}
obj = utils.request(url = url,method = 'post',data = requestData ,headers = self.headers)
if obj['ret'] !='0':
return utils.errorReport(obj['msg']+' ord_id:'+ self.ord_id + " " +sys._getframe().f_code.co_name,system_name = self.system_name)
return obj
def _send_url_request(self, method, url, data=None):
req = None
if data:
req = json.dumps(data)
resp = request(host, ssl_port, url, req, method)
rsp_body = resp.read()
return json.loads(rsp_body), resp.status
def getmaintainitem(self,row_id,userid,token):
url = self.aux_base_url + 'getmaintainitem/'
requestData = {'row_id':row_id,'userid':userid,'token':token}
obj = utils.request(url = url,method = 'post',data = requestData ,headers = self.headers)
if obj['ret']!='0':
return utils.errorReport(obj['msg']+' '+sys._getframe().f_code.co_name,system_name = self.system_name)
return obj['element']
def test_vm_list_sorted(self):
req = json.dumps({'name': 'test', 'cdrom': '/nonexistent.iso'})
request(host, port, '/templates', req, 'POST')
def add_vm(name):
# Create a VM
req = json.dumps({'name': name, 'template': '/templates/test'})
request(host, port, '/vms', req, 'POST')
add_vm('bca')
add_vm('xba')
add_vm('abc')
add_vm('cab')
self.assertEqual(model.vms_get_list(), ['abc', 'bca', 'cab', 'xba'])
def main(stdscr):
cursor = Cursor(stdscr)
url = utils.ip + "token/get"
token = utils.request(url)['token']
grid = Grid(stdscr, x_limit, y_limit, cursor)
sel_bool = False
sel_soldiers = []
while True:
key = stdscr.getch()
# quit game
if key == ord('q'):
exit(stdscr)
# create soldier
elif key == ord('c'):
create_soldier(token)
# move cursor
elif key in directions:
cursor.move_cursor(key)
# select tiles
elif key == ord('s') and not sel_bool:
cursor.select()
key = 0
sel_bool = True
sel_soldiers = []
# finish selecting
elif key is ord('s') and sel_bool:
cursor.deselect()
x_r = sorted((cursor.select_coords[0], cursor.x))
y_r = sorted((cursor.select_coords[1], cursor.y))
debug_file.write(str(x_r) + "\n" + str(y_r) + "\n")
for x in range(x_r[0], x_r[1]+1):
for y in range(y_r[0], y_r[1]+1):
if (x,y) in grid.grid:
debug_file.write("inserting")
sel_soldiers.append(grid.grid[(x,y)])
sel_bool = False
if key is ord('q'):
exit(stdscr)
# move soldiers (soldiers must be selected first)
elif key == ord('m'):
key = 69
debug_file.write(str(sel_soldiers))
if sel_soldiers:
debug_file.write("moving soldiers\n")
move_soldiers(cursor.position(), sel_soldiers)
sel_bool = False
elif key == ord('d'):
raise Exception(grid.request())
grid.debug(str(key))
grid.update(key, sel_bool)
grid.display()
def get_categories(self):
logger.info('Getting categories.')
resp = request(MAIN_URL)
if resp:
soup = BeautifulSoup(resp.text, 'lxml')
return [a.get('href').strip() for a in soup.select('.saas-category-list-body a')]
return []
def getJSON(url,
params=None,
headers=None,
cookies=None,
auth=None,
redirection=True,
timeout=60):
"""A shortcut to :func:`utils.request` that performs a ``GET`` operation and returns json content.
:param url: the url of the request.
:type url: ``str``.
:param params: mapping of url parameters.
:type params: :class:`dict`.
:param headers: the headers of the request.
:type headers: :class:`dict`.
:param cookies: the cookies of the request.
:type cookies: :class:`dict`.
:param auth: the authentication information to be used.
:type auth: :class:`dict`.
:param redirection: a flag indicating whether redirection is allowed or not.
:type redirection: ``boolean``.
:param timeout: a timeout for the request.
:type timeout: ``int``.
:return: the content obtained from executing the request.
:rtype: ``json``.
"""
return request('GET', url, 'json', None, params, headers, cookies,
auth, redirection, timeout)
def test_vm_list_sorted(self):
req = json.dumps({'name': 'test', 'cdrom': '/nonexistent.iso'})
request(host, ssl_port, '/templates', req, 'POST')
def add_vm(name):
# Create a VM
req = json.dumps({'name': name, 'template': '/templates/test'})
request(host, ssl_port, '/vms', req, 'POST')
add_vm('bca')
add_vm('xba')
add_vm('abc')
add_vm('cab')
self.assertEqual(model.vms_get_list(), ['abc', 'bca', 'cab', 'xba'])
def get_data(self, url):
resp = request(url)
if resp:
soup = BeautifulSoup(resp.text, 'lxml')
name = self.get_name(soup)
if not name:
return False
desc = self.get_description(soup)
rating = self.get_rating(soup)
reviews = self.get_reviews(soup)
listed = self.get_listed_on(soup)
latest = self.get_latest_release(soup)
cat = self.get_categories(soup)
data = [[
name,
desc,
rating,
reviews,
listed,
latest,
cat,
url
]]
write_data(data, name)
self.apps_done.append(url)
def get_hotness(cookie, f):
def get_name(tag):
return tag.find_all("h3")[0].text.encode("utf-8").strip()
def get_likehood(tag):
raw_likehood = tag.find_all("h4")[0].text
return int(float(raw_likehood.split(":")[1][:-1]) * 10000)
url = "http://energy.tv.weibo.cn/e/10173/index?display=0&retcode=6102"
response = utils.request(url, "html", cookies=cookie)
if response is None:
return
start_time = response.start_time.strftime("%Y%m%d,%H:%M:%S.%f")
finish_time = response.finish_time.strftime("%Y%m%d,%H:%M:%S.%f")
soup = BeautifulSoup(response.get_html(), "lxml")
for tag in soup.find_all("div", class_="card25"):
measures = tag.find_all("span")
name = get_name(tag)
likehood = get_likehood(tag)
mentioned = measures[0].text
interaction = measures[1].text
cheer_cards = measures[2].text
print(utils.to_csv_line(start_time, finish_time, name, likehood,
mentioned, interaction, cheer_cards),
file=f)
def get_cookie(username, password):
payload = {
"username": username,
"password": password,
"savestate": "1",
"ec": "0",
"entry": "mweibo",
"mainpageflag": "1"
}
headers = {
"Accept-Encoding":
"gzip, deflate, br",
"Connection":
"keep-alive",
"Content-Length":
"162",
"Content-Type":
"application/x-www-form-urlencoded",
"Host":
"passport.weibo.cn",
"Origin":
"https://passport.weibo.cn",
"Referer":
"https://passport.weibo.cn/signin/login",
"User-Agent":
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.167 Safari/537.36"
}
url = "https://passport.weibo.cn/sso/login"
response = utils.request(url, "cookie", data=payload, headers=headers)
if response is not None:
return response.get_cookie_data()
else:
return None
def receive_order(order_no, cookies):
api_url = 'http://jdfw.jd.com/receipt/batch/batchAccept'
headers = {
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',
'Content-Type': 'application/x-www-form-urlencoded',
'Host': 'jdfw.jd.com',
'Referer':
'http://jdfw.jd.com/receipt/search?serviceType=0&esSwitch=1',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.78 Safari/537.36Name',
'X-Requested-With': 'XMLHttpRequest',
'Cookie': cookies
}
form_data = {
'noList': [order_no],
'subCompanyId': '4',
'outletsId': '028861803',
'baseWebsiteNo': '028861803',
'outletName': '成华区智诚电器经营部',
'receivePersonName': '傅敏',
'engineerFirstName': '',
'engineerSecondName': '',
'outletsAssignerName': '',
'reservationPeriodName': '',
'reservationPersonName': '',
'feedbackInfoName': '',
'receivePerson': '傅敏'
}
form_data = json.dumps(form_data)
r = request(api_url, method='post', data=form_data, headers=headers)
print('receive_order:%s' % r)
return r
def get_data(self, url):
resp = request(url)
if resp:
if 'rate limit exceeded' in resp.text.lower():
if self.err_count == 5:
return False
self.err_count = self.err_count + 1
sleep()
self.get_data(url)
else:
soup = BeautifulSoup(resp.text, 'lxml')
name = self.get_name(soup)
if not name:
logger.warning('Unable to get resp.text!')
return False
website = self.get_website(soup)
desc = self.get_description(soup)
reviews = self.get_reviews(soup)
rating = self.get_rating(soup)
categories = self.get_categories(soup)
data = [[
name, website, desc, rating, reviews, categories, url
]]
write_data(data, name)
self.err_count = 0
else:
if self.err_count == 5:
return False
self.err_count = self.err_count + 1
sleep()
self.get_data(url)
def getContent(self, agent, keyword, page, timescope, region):
''' '''
url = 'http://search.t.qq.com/index.php?k=%s&is_dup=1&pos=174&p=%d' % (keyword, page)
log.debug('Starting request url: %s' % url)
result = yield request(agent, url)
returnValue(result)
def github_repo_search(self, repo_name):
"""
Search for github repos
:param repo_name: search string
:return:
"""
try:
search_url = "https://api.github.com/search/repositories"
query_params = {"q": str(repo_name)}
search_results = utils.request("GET",
search_url,
params=query_params,
custom_headers=self.custom_header)
if "total_count" in search_results and search_results[
"total_count"] > 0:
# select search result with highest score. Default sorted based on score.
item = search_results["items"][0]
package = {"name": item["name"], "url": item["url"]}
return package
else:
logger.info("No search results found")
return None
except Exception as exp:
logger.error("Failed to search Github repository: %s" % exp)
return None
def test_get_storagepools(self):
storagepools = json.loads(
self.request('/plugins/kimchi/storagepools').read()
)
self.assertIn('default', [pool['name'] for pool in storagepools])
with RollbackContext() as rollback:
# Now add a couple of storage pools
for i in xrange(3):
name = u'kīмсhī-storagepool-%i' % i
req = json.dumps({'name': name, 'type': 'dir',
'path': '/var/lib/libvirt/images/%i' % i})
resp = self.request('/plugins/kimchi/storagepools', req,
'POST')
rollback.prependDefer(model.storagepool_delete, name)
self.assertEquals(201, resp.status)
# Pool name must be unique
req = json.dumps({'name': name, 'type': 'dir',
'path': '/var/lib/libvirt/images/%i' % i})
resp = self.request('/plugins/kimchi/storagepools', req,
'POST')
self.assertEquals(400, resp.status)
# Verify pool information
resp = self.request('/plugins/kimchi/storagepools/%s' %
name.encode("utf-8"))
p = json.loads(resp.read())
keys = [u'name', u'state', u'capacity', u'allocated',
u'available', u'path', u'source', u'type',
u'nr_volumes', u'autostart', u'persistent']
self.assertEquals(sorted(keys), sorted(p.keys()))
self.assertEquals(name, p['name'])
self.assertEquals('inactive', p['state'])
self.assertEquals(True, p['persistent'])
self.assertEquals(True, p['autostart'])
self.assertEquals(0, p['nr_volumes'])
pools = json.loads(
self.request('/plugins/kimchi/storagepools').read()
)
self.assertEquals(len(storagepools) + 3, len(pools))
# Create a pool with an existing path
tmp_path = tempfile.mkdtemp(dir='/var/lib/kimchi')
rollback.prependDefer(os.rmdir, tmp_path)
req = json.dumps({'name': 'existing_path', 'type': 'dir',
'path': tmp_path})
resp = self.request('/plugins/kimchi/storagepools', req, 'POST')
rollback.prependDefer(model.storagepool_delete, 'existing_path')
self.assertEquals(201, resp.status)
# Reserved pool return 400
req = json.dumps({'name': 'kimchi_isos', 'type': 'dir',
'path': '/var/lib/libvirt/images/%i' % i})
resp = request(host, ssl_port, '/plugins/kimchi/storagepools', req,
'POST')
self.assertEquals(400, resp.status)
def test_vm_list_sorted(self):
req = json.dumps({'name': 'test', 'cdrom': fake_iso})
request(host, ssl_port, '/plugins/kimchi/templates', req, 'POST')
def add_vm(name):
# Create a VM
req = json.dumps({'name': name,
'template': '/plugins/kimchi/templates/test'})
task = json.loads(request(host, ssl_port, '/plugins/kimchi/vms',
req, 'POST').read())
wait_task(model.task_lookup, task['id'])
vms = [u'abc', u'bca', u'cab', u'xba']
for vm in vms:
add_vm(vm)
vms.append(u'test')
self.assertEqual(model.vms_get_list(), sorted(vms))
def test_create_debugreport(self):
req = json.dumps({'name': 'report1'})
with RollbackContext() as rollback:
resp = request(host, ssl_port, '/debugreports', req, 'POST')
self.assertEquals(202, resp.status)
task = json.loads(resp.read())
# make sure the debugreport doesn't exist until the
# the task is finished
wait_task(self._task_lookup, task['id'])
rollback.prependDefer(self._report_delete, 'report2')
resp = request(host, ssl_port, '/debugreports/report1')
debugreport = json.loads(resp.read())
self.assertEquals("report1", debugreport['name'])
self.assertEquals(200, resp.status)
req = json.dumps({'name': 'report2'})
resp = request(host, ssl_port, '/debugreports/report1',
req, 'PUT')
self.assertEquals(303, resp.status)
def test_development_env(self):
"""
Test traceback thrown in development env
"""
server_root = '/test'
setup_server(server_root=server_root)
# check if server_root in config is the same used to start server
resp = request(server_root + '/config').read()
conf = json.loads(resp)
self.assertEquals(len(conf), 5)
def test_production_env(self):
"""
Test reasons sanitized in production env
"""
server_root = '/test'
setup_server('production', server_root)
# check if server_root in config is the same used to start server
resp = request(server_root + '/config').read()
conf = json.loads(resp)
self.assertEquals(len(conf), 5)
def provisioning_uninstallProduct(self, product, release, fqn, errorType, content):
"""
uninstall a product-release in a VM
:param release:
:param fqn:
:param NOT_ERROR:
:param content:
"""
name = fqn+"_"+product+"_"+release
world.response = utils.request("DELETE", self.__create__url("uninstallProduct", name), self.__create__headers(content), None, errorType)
pass
def test_development_env(self):
"""
Test traceback thrown in development env
"""
setup_server()
# test 404
resp = json.loads(request(host, ssl_port, '/tasks/blah').read())
self.assertEquals('404 Not Found', resp.get('code'))
# test 405 wrong method
resp = json.loads(request(host, ssl_port, '/', None, 'DELETE').read())
msg = u'WOKAPI0002E: Delete is not allowed for wokroot'
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
# test 400 parse error
resp = json.loads(request(host, ssl_port, '/tasks', '{',
'POST').read())
msg = u'WOKAPI0006E: Unable to parse JSON request'
self.assertEquals('400 Bad Request', resp.get('code'))
self.assertEquals(msg, resp.get('reason'))
self.assertIn('call_stack', resp)
# test 400 missing required parameter
# TODO: need add this test when some REST API from wok accepts POST
# req = json.dumps({})
# resp = json.loads(request(host, ssl_port, '/tasks', req,
# 'POST').read())
# m = u"KCHVM0016E: Specify a template to create a virtual machine from"
# self.assertEquals('400 Bad Request', resp.get('code'))
# self.assertEquals(m, resp.get('reason'))
# self.assertIn('call_stack', resp)
# test 405 method not allowed
req = json.dumps({})
resp = json.loads(request(host, ssl_port, '/tasks', req,
'POST').read())
m = u"WOKAPI0005E: Create is not allowed for tasks"
self.assertEquals('405 Method Not Allowed', resp.get('code'))
self.assertEquals(m, resp.get('reason'))
self.assertIn('call_stack', resp)
def _fetch(self, headers={}, d={}, method='POST', url=None):
params = json.dumps(d)
headers["Content-Type"] = "application/json"
host = self.host
port = self.port
method = method
try:
resp = request(host=host, method=method, url=url,
headers=headers, params=params, port=port)
print d, headers, host, method, url, port, resp
return json.loads(resp)
except:
raise Exception('Bad response from %s' %(host))
def POST(self):
self.stage = web.data()
store.ctx.last_call = time.time()
try:
resp = utils.request(utils.request_url, 'GET', {'count': 60, 'mode': 'lite'})
data = json.loads(resp.read())
for body in self.filter(data):
add_task(web.ctx, '/whale/send', json.dumps(body))
finally:
if self.stage != utils.stage():
add_task(web.ctx, '/whale/hello', self.stage)
else:
time.sleep(4)
add_task(web.ctx, '/whale/greet', self.stage)
def main():
loader = FreednsConfigLoader()
parser = OptionParser(option_list=loader.options, version=loader.version)
(options, args) = parser.parse_args()
try:
loader.set_timeout(options.timeout)
loader.set_auth(options.username, options.password)
res = utils.request(loader.get_request_url(), loader.timeout)
utils.update(res, loader.timeout)
except Exception, e:
import sys
import traceback
sys.stderr.write(traceback.format_exc())
parser.print_help()
def test_debugreport_download(self):
req = json.dumps({'name': 'report1'})
with RollbackContext() as rollback:
resp = request(host, ssl_port, '/plugins/gingerbase/debugreports',
req, 'POST')
self.assertEquals(202, resp.status)
task = json.loads(resp.read())
# make sure the debugreport doesn't exist until the
# the task is finished
wait_task(self._task_lookup, task['id'], 20)
rollback.prependDefer(self._report_delete, 'report1')
resp = request(host, ssl_port,
'/plugins/gingerbase/debugreports/report1')
debugreport = json.loads(resp.read())
self.assertEquals("report1", debugreport['name'])
self.assertEquals(200, resp.status)
resp = request(host, ssl_port,
'/plugins/gingerbase/debugreports/report1/content')
self.assertEquals(200, resp.status)
resp = request(host, ssl_port,
'/plugins/gingerbase/debugreports/report1')
debugre = json.loads(resp.read())
resp = request(host, ssl_port, '/' + debugre['uri'])
self.assertEquals(200, resp.status)
def test_server_start(self):
"""
Test that we can start a server and receive HTTP:200.
"""
host = '127.0.0.1'
port = utils.get_free_port()
model = kimchi.mockmodel.MockModel('/tmp/obj-store-test')
s = utils.run_server(host, port, None, test_mode=True, model=model)
try:
resp = utils.request(host, port, '/')
self.assertEquals(200, resp.status)
except:
raise
finally:
os.unlink('/tmp/obj-store-test')
s.stop()
def test_server_start(self):
"""
Test that we can start a server and receive a response. Right now we
have no content so we expect HTTP:404
"""
host = '127.0.0.1'
port = utils.get_free_port()
s = utils.run_server(host, port, test_mode=True)
try:
resp = utils.request(host, port, '/')
data = json.loads(resp.read())
self.assertEquals(200, resp.status)
self.assertEquals('localhost', data['hostname'])
except:
raise
finally:
s.stop()
def getJSON(url, params=None, headers=None, cookies=None, auth=None, redirection=True, timeout=60):
"""A shortcut to :func:`utils.request` that performs a ``GET`` operation and returns json content.
:param url: the url of the request.
:type url: ``str``.
:param params: mapping of url parameters.
:type params: :class:`dict`.
:param headers: the headers of the request.
:type headers: :class:`dict`.
:param cookies: the cookies of the request.
:type cookies: :class:`dict`.
:param auth: the authentication information to be used.
:type auth: :class:`dict`.
:param redirection: a flag indicating whether redirection is allowed or not.
:type redirection: ``boolean``.
:param timeout: a timeout for the request.
:type timeout: ``int``.
:return: the content obtained from executing the request.
:rtype: ``json``.
"""
return request('GET', url, 'json', None, params, headers, cookies, auth, redirection, timeout)
def test_screenshot_refresh(self):
# Create a VM
req = json.dumps({"name": "test", "cdrom": fake_iso})
request(host, ssl_port, "/plugins/kimchi/templates", req, "POST")
req = json.dumps({"name": "test-vm", "template": "/plugins/kimchi/templates/test"})
resp = request(host, ssl_port, "/plugins/kimchi/vms", req, "POST")
task = json.loads(resp.read())
wait_task(model.task_lookup, task["id"])
# Test screenshot refresh for running vm
request(host, ssl_port, "/plugins/kimchi/vms/test-vm/start", "{}", "POST")
resp = request(host, ssl_port, "/plugins/kimchi/vms/test-vm/screenshot")
self.assertEquals(200, resp.status)
self.assertEquals("image/png", resp.getheader("content-type"))
resp1 = request(host, ssl_port, "/plugins/kimchi/vms/test-vm")
rspBody = resp1.read()
testvm_Data = json.loads(rspBody)
screenshotURL = "/" + testvm_Data["screenshot"]
time.sleep(5)
resp2 = request(host, ssl_port, screenshotURL)
self.assertEquals(200, resp2.status)
self.assertEquals(resp2.getheader("content-type"), resp.getheader("content-type"))
self.assertEquals(resp2.getheader("content-length"), resp.getheader("content-length"))
self.assertEquals(resp2.getheader("last-modified"), resp.getheader("last-modified"))
def delete(who, where, host, port=80, url='/uss/sql/delete', headers={}):
"""
:param host: HTTP Request host
:param port: HTTP Request port
:param url: HTTP Request url
:param who: SQL option on which table
:param where: {"id":0}
:returns: { id:0 }
"""
d = {}
d['who'] = who
d['where'] = where
try:
params = json.dumps(d)
headers["Content-Type"] = "application/json"
rslt = request(host=host, method='POST', url=url,
headers=headers, params=params, port=port)
return json.loads(rslt)
except:
raise
def insert(who, what, host, port=80, url='/uss/sql/insert', headers={}):
"""
:param host: HTTP Request host
:param port: HTTP Request port
:param url: HTTP Request url
:param who: SQL option on which table
:param what: { "ip":"192.168.1.1", "hostname":"node1", "user":"******", "passwd":"root", "rack":"1" }
:param headers: HTTP Request headers
:returns: { id:0 }, id of the node inserted.
"""
d = {}
d['who'] = who
d['what'] = what
try:
params = json.dumps(d)
headers["Content-Type"] = "application/json"
rslt = request(host=host, method='POST', url=url,
headers=headers, params=params, port=port)
return json.loads(rslt)
except:
raise
def provisioning_installProduct (self, ip, hostname, osType, product, release, errorType, content):
"""
install a product in a VM
:param product:
:param release:
:param ip:
:param errorType:
:param content:
"""
vmElement = [{'label': 'ip', 'value': ip},
{'label': 'hostname', 'value': hostname},
{'label': 'domain', 'value': None},
{'label': 'fqn', 'value': 'fqn'},
{'label': 'osType', 'value': osType}]
productElement = [{'label': 'name', 'value': product},
{'label': 'version', 'value': release},
{'label': 'type', 'value': None}]
self.__init_body(content)
self.INSTALL_BODY = utils.body_elements(self.INSTALL_BODY, vmElement, "vm", "installProduct", content)
self.INSTALL_BODY = utils.body_elements(self.INSTALL_BODY, productElement, "product", "installProduct", content)
world.response = utils.request("POST", self.__create__url("installProduct"), self.__create__headers(content), self.INSTALL_BODY, errorType)
