def _get_cluster_periodically(self, period):
from retry import retry
retry(self._get_cluster,
logger=self.logger,
context="Gather cluster details",
delay=30)
timer = Timer(period, self._get_cluster_periodically, args=[period])
timer.start()
def turn_off_machine(self):
"""
Turn off gcloud instance, wait until it's off.
"""
compute().instances().stop(project='near-core',
zone=self.zone,
instance=self.instance_name)
retry.retry(lambda: self.machine_status() == 'STOPPED', 60)
def turn_on_machine(self):
"""
Turn on gcloud instance, wait until it's on.
"""
compute().instances().start(project='near-core',
zone=self.zone,
instance=self.instance_name)
retry.retry(lambda: self.machine_status() == 'RUNNING', 60)
self.exec("tmux new -s node -d")
def _rest_retry(self, method, resource, query_id=None, subcategory=None,
second_query_id=None, data=None, **options):
"""Invokes the specified HTTP method, with retries if they are specified in the configuration
:return: an instance of OpenCGAResponseList"""
query_ids_str = self._get_query_id_str(query_id)
def exec_retry():
return execute(host=self._cfg.host,
version=self._cfg.version,
sid=self.session_id,
category=self._category,
subcategory=subcategory,
method=method,
query_id=query_ids_str,
second_query_id=second_query_id,
resource=resource,
data=data,
options=options)
def notify_retry(exc_type, exc_val, exc_tb):
if self.on_retry:
self.on_retry(self, exc_type, exc_val, exc_tb, dict(
method=method, resource=resource, query_id=query_id,
category=self._category, subcategory=subcategory,
second_query_id=second_query_id, data=data,
options=options
))
response = retry(
exec_retry, self._cfg.max_attempts, self._cfg.min_retry_secs, self._cfg.max_retry_secs,
login_handler=self._client_login_handler if self.login_handler else None,
on_retry=notify_retry)
return OpenCGAResponseList(response, query_ids_str)
def get_assisted_service_url_by_args(args, wait=True):
if hasattr(args, 'inventory_url') and args.inventory_url:
return args.inventory_url
kwargs = {
'service': args.service_name,
'namespace': args.namespace
}
if args.oc_mode:
get_url = get_remote_assisted_service_url
kwargs['oc'] = oc_utils.get_oc_api_client(
token=args.oc_token,
server=args.oc_server
)
kwargs['scheme'] = args.oc_scheme
else:
get_url = get_local_assisted_service_url
kwargs['profile'] = args.profile
return retry(
tries=5 if wait else 1,
delay=3,
backoff=2,
exceptions=(
requests.ConnectionError,
requests.ConnectTimeout,
requests.RequestException
)
)(get_url)(**kwargs)
def retry_when(errors):
return retry(errors, 3,
lambda err, cnt: (
print("sleep 3s"),
sleep(3)
)
)
def main():
if sys.version_info < (3, 9):
print("Must use Python 3.9+")
sys.exit(1)
cfg = read_config()
now = this_minute()
# Read the channel list from Slack
chans = channel_manager.ChannelManager(token=cfg['oauth_token'])
retry(5, "listing conversations", lambda: chans.list())
# Read all posts from Google Sheets
all_posts = retry(
5, "reading Google Sheet", lambda: get_posts_csv(
cfg['posts']['csv'], cfg['posts']['poll_timeout']))
sent_posts = cfg['sent']
pending_posts = []
for post in all_posts:
if post in sent_posts:
continue
when = parse_date(post['When'])
if when is None:
pending_posts.append({
'Message':
f"Could not process the timestamp for {post['Message']}",
'Channel': cfg["botadmin"]
})
continue
if when != now:
continue
pending_posts.append(post)
sent_posts.extend(pending_posts)
# Write the config back with the current time, to avoid spamming.
update_config(cfg, now)
for post in pending_posts:
try:
chan = post["Channel"]
message = post["Message"]
retry(3, f"Posting message to {chan}: {message}",
lambda: chans.post(chan, message))
except:
print(f"Failed to post to {chan}: {message}")
def test_backoff(self):
""" Retries with exponential backoff. """
r = retry.retry(tries=10, delay=1, backoff=2)(fail_n(9))
fake_time = FakeTime()
with fake_time:
r()
self.assertGreaterEqual(fake_time.mock_sleep.total, 2**9 - 1)
def test_n_retry(self):
""" Retries a fixed number of times. """
r = retry.retry(tries=10, logger=None)(fail_n(9))
fake_time = FakeTime()
with fake_time:
r()
self.assertEqual(fake_time.mock_sleep.calls, 9)
def _get_available_host_retry(self):
def notify_retry(exc_type, exc_val, exc_tb):
if self._on_retry:
self._on_retry(self, exc_type, exc_val, exc_tb,
dict(config=self._configuration_input))
return retry(self._get_available_host,
self.max_attempts,
self.min_retry_secs,
self.max_retry_secs,
on_retry=notify_retry)
def test_interval():
"""Interval expected is the interval to complete an action"""
def _success_interval(in_dict):
in_dict['num'] += 1
return in_dict['num']
baz_with_interval = retry.retry(
success=lambda x: x > 5, interval=1)(_success_interval)
start = datetime.now()
baz_with_interval({'num': 0})
elapsed = datetime.now() - start
assert elapsed.seconds >= 5
def test_successful_timeout():
"""Success with a timeout still works"""
def _success_timeout(in_dict):
in_dict['num'] += 1
return in_dict['num']
try:
_test_func = retry.retry(
success=lambda x: x == 5, timeout=10, interval=1)(
_success_timeout)
_test_func({'num': 0})
except retry.MaximumTimeoutExceeded:
pytest.fail('Expected the timeout not to be exceeded')
def get_assisted_service_url_by_args(args, wait=True):
if hasattr(args, "inventory_url") and args.inventory_url:
return args.inventory_url
kwargs = {"service": args.service_name, "namespace": args.namespace}
if args.oc_mode:
get_url = get_remote_assisted_service_url
kwargs["oc"] = oc_utils.get_oc_api_client(token=args.oc_token, server=args.oc_server)
kwargs["scheme"] = args.oc_scheme
else:
get_url = get_local_assisted_service_url
kwargs["deploy_target"] = args.deploy_target
return retry(
tries=5 if wait else 1,
delay=3,
backoff=2,
exceptions=(requests.ConnectionError, requests.ConnectTimeout, requests.RequestException),
)(get_url)(**kwargs)
def get_relationship(query_text, intent_info):
str_parse_tree = parse_tree.get_parse_tree(query_text)
matched_words_col = recColoumns_temp(query_text)
if not matched_words_col:
rows = retry.no_col_match(query_text)
if rows:
rows = [row.tolist() for row in rows]
print(rows)
return [rows]
print(str_parse_tree)
print(matched_words_col)
db_inp_dic = {}
col_type, col_pos = util.get_col_pos()
print(col_pos)
adj_dic = util.get_adj(query_text)
rows = []
for key, value in matched_words_col.items():
pos_tag = col_pos[value]
node, val = parse_tree.get_relation(str_parse_tree, key, pos_tag)
print("\n\n\n\n")
print(node)
print("\n\n\n\n")
print(val)
if(val != False and val is not None):
db_inp_dic[value.lower()] = val
print(db_inp_dic)
rows.append(db_handler.db_select(db_inp_dic, intent_info, col_type, adj_dic))
print(rows)
else:
matched_rows = retry.retry(value, query_text)
rows.append( [row.tolist() for row in matched_rows])
print(rows)
return rows
def gameOver(self):
x = (config.SCREENX - config.GAMEOVERX) / 2
y = (config.SCREENY - config.GAMEOVERY) / 2
self.gameScreen.blit(pygame.image.load("./images/gameover.png"),
[x, y])
b = retry.retry(self.gameScreen)
pygame.display.update()
running = True
while running:
for event in pygame.event.get():
if event.type == pygame.QUIT:
sys.exit()
if event.type == pygame.MOUSEBUTTONDOWN:
if event.button == 1:
if b.rect.collidepoint(event.pos):
self.music['CATDOG'].stop()
self.music['robot'].play(-1)
self.initVars(self.level)
return
def sign_in_out(u: dgutXgxt,
flag: int,
workAssignmentId: int = None,
key: str = None):
"""签到/签退
Args:
u (dgutXgxt): 学工系统类
flag (int): 签到=>1，签退=>2
workAssignmentId (int, optional): 考勤职位的id. Defaults to None.
key (str, optional): server酱的key. Defaults to None.
"""
u.attendance_retry = retry(tries=20, delay=10, backoff=2,
max_delay=30)(u.attendance)
res = u.attendance_retry(flag=flag, workAssignmentId=workAssignmentId)
now = get_beijing_datetime().strftime('%Y-%m-%d %H:%M:%S')
print(f"[考勤结果] {now} - {res}")
if key:
message = "签到成功" if flag == 1 else "签退成功"
post_server(key, now + message)
def get_platform(driver):
"""Return info about the platform used by the given driver."""
name = driver.name.title()
caps = driver.capabilities
return '{} {} ({})'.format(name, caps['version'], caps['platform'].title())
def format_subprocess_error(error):
"""Return an error message string including the subprocess output."""
return '{}: {}'.format(error, error.output)
retry_process_error = retry(
subprocess.CalledProcessError, tries=2,
format_error=format_subprocess_error)
class TestCase(unittest.TestCase):
"""Helper base class that supports running browser tests."""
@classmethod
def setUpClass(cls):
# The Selenium web driver is instantiated here for each test case.
# This way the suite should be more reliable, especially when running
# Firefox tests, in which cases the GUI WebSocket connection is often
# problematic (i.e. connection errors) when switching from the sandbox
# mode back to the staging backend.
local_prefix = 'local-'
if browser_name.startswith(local_prefix):
class Chuzhou(Crawler):
def __init__(self):
self.url = 'http://www.czhome.com.cn/complexPro.asp?page=1&districtID=0&projectAdr=&projectName=&buildingType=0&houseArea=0&averagePrice=0&selState=-1'
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36',
}
def start_crawler(self):
self.start()
@retry(retry(3))
def get_all_page(self):
response = requests.get(url=self.url, headers=self.headers)
html = response.content.decode('gbk')
tree = etree.HTML(html)
page = tree.xpath('//*[@id="Table7"]/tr/td[2]/text()')[1]
page = re.search('\d+', page).group()
print(page)
return page
def is_none(self, xpath_adv):
if xpath_adv:
xpath_adv = xpath_adv[0]
else:
xpath_adv = None
return xpath_adv
@retry(retry(3))
def start(self):
page = self.get_all_page()
for i in range(1, int(page) + 1):
url = 'http://www.czhome.com.cn/complexPro.asp?page=' + str(
i
) + '&districtID=0&projectAdr=&projectName=&buildingType=0&houseArea=0&averagePrice=0&selState=-1'
response = requests.get(url, headers=self.headers)
html = response.content.decode('gbk')
tree = etree.HTML(html)
comm_url_list = tree.xpath('//*[@id="Table8"]/tr/td[2]/a/@href')
for i in range(len(comm_url_list)):
try:
comm = Comm(7)
comm_url = 'http://www.czhome.com.cn/' + comm_url_list[i]
self.get_comm_info(comm_url, comm)
except Exception as e:
print("co_index={},小区:{}无法提取".format(co_index, comm_url))
print(e)
@retry(retry(3))
def get_comm_info(self, comm_url, comm):
response = requests.get(comm_url, headers=self.headers)
html = response.content.decode('gbk').replace(' ', '').replace(
'\n', '').replace('\r', '').replace('\t', '')
# 小区id
co_id = re.search('projectID=(\d+?)&', comm_url).group(1)
# 小区名称
co_name = re.search('项目名称：(.*?)<', html).group(1)
# 所在区域
co_area = re.search('所在区名：(.*?)<', html).group(1)
# 项目地址
co_address = re.search('项目地址：(.*?)<', html).group(1)
# 开发商
co_develops = re.search('企业名称：<(.*?)>(.*?)<', html).group(2)
# 小区总套数
co_all_house = re.search('总套数<(.*?)><(.*?)<(.*?)>(.*?)<',
html).group(4)
# 占地面的
co_all_size = re.search('总面积<(.*?)><(.*?)<(.*?)>(.*?)<',
html).group(4).replace('㎡', '')
# 建筑面积
co_residential_size = re.search('住宅面积<(.*?)<(.*?)<(.*?)<(.*?)>(.*?)<',
html).group(5).replace('㎡', '')
comm.co_id = co_id
comm.co_name = co_name
comm.co_area = co_area
comm.co_address = co_address
comm.co_develops = co_develops
comm.co_all_house = co_all_house
comm.co_all_size = co_all_size
comm.co_residential_size = co_residential_size
comm.insert_db()
self.get_build_info(co_id, co_name)
global count
count += 1
print(count)
@retry(retry(3))
def get_build_info(self, co_id, co_name):
url = 'http://www.czhome.com.cn/Presell.asp?projectID=' + co_id + '&projectname=' + co_name
response = requests.get(url, headers=self.headers)
html = response.content.decode('gbk')
tree = etree.HTML(html)
xpath_list = tree.xpath('//tr[@class="indextabletxt"]')
for i in xpath_list[1:]:
build_url = i.xpath('td[2]/a/@href')[0]
url = 'http://www.czhome.com.cn/' + build_url
result = requests.get(url, headers=self.headers)
if result.status_code is not 200:
print("co_index={},预售url:{}连接失败".format(co_index, url))
continue
html = result.content.decode('gbk')
tree = etree.HTML(html)
# 总套数
bu_xpath = tree.xpath(
'/html/body/table/tr/td/table/tr/td/table/tr')[1:]
for i in bu_xpath:
try:
building = Building(7)
global building_id
building_id += 1
building.bu_id = building_id
bu_all_house = i.xpath('td[7]/text()')[0]
bu_url = i.xpath('td[1]/a/@href')[0]
url = 'http://www.czhome.com.cn/' + bu_url
response = requests.get(url, headers=self.headers)
if response.status_code is not 200:
print("co_index={},楼栋url:{}连接失败".format(co_index, url))
continue
html = response.content.decode('gbk')
tree = etree.HTML(html)
# 楼层
bu_floor = tree.xpath(
'//*[@id="Table4"]/tr[2]/td/table[3]/tr/td[1]/u/text()'
)[-1]
house_url_list = tree.xpath(
'//*[@id="Table4"]/tr[2]/td/table[3]/tr/td/a/@href')
bu_address = re.search(
'<center><font color=.*? (.*?)<', html,
re.S | re.M).group(1)
building.bu_all_house = bu_all_house
building.bu_address = bu_address
building.bu_floor = bu_floor
building.bu_id = building_id
building.co_id = co_id
building.insert_db()
for i in house_url_list:
try:
house = House(7)
house_url = 'http://www.czhome.com.cn/' + i
self.get_house_info(house_url, house, co_id,
building_id, building)
except Exception as e:
print(e)
except Exception as e:
print(e)
@retry(retry(3))
def get_house_info(self, house_url, house, co_id, building_id, building):
try:
response = requests.get(house_url, headers=self.headers)
html = response.content.decode('gbk').replace('\t', '').replace(
'\n', '').replace('\r', '').replace(' ', '')
if response.status_code is not 200:
return
# 房号id
ho_name = re.search('室号<(.*?)<(.*?)>(.*?)<', html).group(3)
# 房屋类型
ho_type = re.search('房屋类型<(.*?)<(.*?)>(.*?)<', html).group(3)
# 户型
ho_room_type = re.search('房型<(.*?)<(.*?)>(.*?)<', html).group(3)
# 楼层
ho_floor = re.search('名义层/实际层<(.*?)<(.*?)>(.*?)<', html).group(3)
ho_floor = re.search('/(.*?)$', ho_floor).group(1)
# 建筑面积
ho_build_size = re.search('预测建筑面积<(.*?)<(.*?)>(.*?)<',
html).group(3)
# 预测套内面积
ho_true_size = re.search('预测套内面积<(.*?)<(.*?)>(.*?)<',
html).group(3)
# 分摊面积
ho_share_size = re.search('预测分摊面积<(.*?)<(.*?)>(.*?)<',
html).group(3)
house.co_id = co_id
house.bu_id = building_id
house.ho_name = ho_name
house.ho_type = ho_type
house.ho_room_type = ho_room_type
house.ho_floor = ho_floor
house.ho_build_size = ho_build_size
house.ho_true_size = ho_true_size
house.ho_share_size = ho_share_size
house.insert_db()
except Exception as e:
print(e)
def fight_norman():
global firsttime
global your_hp
global norm_health
if firsttime == 0:
firsttime += 1
your_hp = 15
norm_health = 20
inventory.get_inventory()
if inventory.held_item == "Fists":
your_atk = random.randint(1, 2)
elif inventory.held_item == "Gun":
your_atk = random.randint(10, 15)
elif inventory.held_item == "Crowbar":
your_atk = random.randint(2, 4)
elif inventory.held_item == "Starting pistol":
your_atk = random.randrange(1, 9, 2)
elif inventory.held_item == "Rusty Knife":
your_atk = 0
elif inventory.held_item == "Bow and arrow":
your_atk = random.randrange(1, 8)
elif inventory.held_item == "Manual Revolver":
your_atk = random.randrange(1, 10, 3)
elif inventory.held_item == "Umbrella":
your_atk = random.randint(1, 3)
elif inventory.held_item == "Hunting knife":
your_atk = random.randint(3, 6)
else:
your_atk = 0
f = False
while f == False:
if your_hp <= 0:
read_text.read_letters(
"""\033[1;31;40m\n Norman triumphs over you, resulting in the end of your life.
Unfortunately Norman Bates was the last face you ever saw
And his reign of terror will proceed to rage on.\n""")
your_hp = 15
norm_health = 20
firsttime -= 1
f = True
retry.retry()
read_text.read_letters(
f"\033[1;37;40m\n What would you like to do?:\n")
print(
f"\033[1;37;40m (A)ttack Norman with {inventory.held_item}\n(D)istract Norman\n(R)eturn to reception\n"
)
answer = msvcrt.getch().decode().lower()
if answer == "a":
norm_health -= your_atk
if your_atk == 0:
read_text.read_letters(
f"\033[1;31;40m Norman chuckles. You honestly thought your {inventory.held_item} would do something to me? FOOL\n Norman took no damage from that attack and launches you back into the reception\n"
)
f = True
motel_navigation.enter_reception()
elif norm_health > 16:
read_text.read_letters(
f"\033[1;31;40m You hit Norman for {your_atk} damage\n 'I appreciate the effort but you've barely wounded me!'\n"
)
read_text.read_letters(
f"\033[1;31;40m Norman has {norm_health} hp left\n")
norm_atk = random.randint(1, 4)
your_hp -= norm_atk
read_text.read_letters(
f"\033[1;37;40m Norman hits you for {norm_atk} hp,\n Watch out, you only have {your_hp} hp left\n"
)
elif norm_health <= 16 and norm_health > 12:
read_text.read_letters(
f"\033[1;31;40m\n You hit Norman for {your_atk} damage\n 'You really think this is hurting me?' Norman exclaims\n"
)
read_text.read_letters(
f"\033[1;31;40m Norman has {norm_health} hp left\n")
norm_atk = random.randint(2, 5)
your_hp -= norm_atk
read_text.read_letters(
f"\033[1;37;40m Norman hits you for {norm_atk} hp,\n Watch out, you only have {your_hp} hp left\n"
)
elif norm_health <= 12 and norm_health > 6:
read_text.read_letters(
f"\033[1;31;40m You hit Norman for {your_atk} damage\n 'You're not doing a great job!' Normal stumbles as he proclaims\n"
)
read_text.read_letters(
f"\033[1;31;40m Norman has {norm_health} hp left\n")
norm_atk = random.randint(1, 5)
your_hp -= norm_atk
read_text.read_letters(
f"\033[1;37;40m Norman hits you for {norm_atk} hp,\n Watch out, you only have {your_hp} hp left\n"
)
elif norm_health <= 6 and norm_health >= 1:
read_text.read_letters(
f"\033[1;31;40m You hit Norman for {your_atk} damage\n 'I told you I'm not letting you leave here alive!' Normal shouts with murderous intent\n"
)
read_text.read_letters(
f"\033[1;31;40m Norman has {norm_health} hp left\n")
norm_atk = random.randint(2, 6)
your_hp -= norm_atk
read_text.read_letters(
f"\033[1;37;40m Norman hits you for {norm_atk} hp,\n Watch out, you only have {your_hp} hp left\n"
)
elif norm_health < 1:
read_text.read_letters(
"\033[1;31;40\n You hit Norman for the remainder of his health."
)
read_text.read_letters(
"\033[1;31;40m\n 'Impossible, you... Can't... L e a v e......'"
)
firsttime -= 1
f = True
retry.end()
elif answer == "d":
read_text.read_letters(
" Norman doesn't fall for your pitiful attempt to point and distract him,\n He proceeds to launch you back into the reception.\n"
)
f = True
motel_navigation.enter_reception()
elif answer == "r":
read_text.read_letters(
" You decide to return to the reception. Probably for the better..."
)
f = True
motel_navigation.enter_reception()
"""
A stdlib-like logger that implements the method used by :module:`retry`.
"""
@staticmethod
def warning(fmt, error, delay):
Message.new(
message_type="flocker_bb:ec2:retry",
error=str(error), delay=delay,
).write()
class RequestLimitExceeded(Exception):
pass
retry_on_request_limit = retry(
RequestLimitExceeded, delay=10, backoff=2, max_delay=240, jitter=(0, 10),
logger=RetryLogger(),
)
# A metadata key name which can be found in image metadata. This metadata
# items identifies the general purpose of the image. It may not be unique if
# multiple versions of the image have been created. It corresponds to the
# names used in the manifest.yml file used to build the images.
BASE_NAME_TAG = "base_name"
class Input(Names):
REQUEST_START = NamedConstant()
INSTANCE_STARTED = NamedConstant()
START_FAILED = NamedConstant()
REQUEST_STOP = NamedConstant()
def test_exception_criteria():
"""Exceptions specified are raised on MaximumRetriesExceeded"""
foo_with_exception = retry.retry(exceptions=(ArithmeticError,))(foo)
with pytest.raises(ArithmeticError) as exc_info:
foo_with_exception(-1)
assert exception_message in str(exc_info.value)
class Chengdu(Crawler):
def __init__(self):
self.url = 'http://www.funi.com/loupan/region_0_0_0_0_1'
self.headers = {
'User-Agent':
'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36',
}
def request_proxy(self, url):
data = {"app_name": 'gv_chengdu'}
while True:
ip = requests.post(url='http://192.168.0.235:8999/get_one_proxy',
data=data).text
print(ip)
proxy = {'http': ip}
try:
res = requests.get(url=url,
headers=self.headers,
proxies=proxy,
timeout=10)
print(res.status_code)
if res.status_code == 200:
print(res.content)
return res
except Exception as e:
form_data = {
"app_name": 'gv_chengdu',
"status_code": 1,
"ip": ip
}
requests.post(
url='http://192.168.0.235:8999/send_proxy_status',
data=form_data)
print('ip有问题')
def start_crawler(self):
self.start()
def get_all_page(self):
response = self.request_proxy(self.url)
html = response.text
tree = etree.HTML(html)
page = tree.xpath('//div[@class="pages"]/a/text()')[-2]
print(page)
return page
@retry(retry(3))
def start(self):
page = self.get_all_page()
for i in range(1, int(page) + 1):
url = 'http://www.funi.com/loupan/region_0_0_0_0_' + str(i)
response = self.request_proxy(url)
html = response.text
tree = etree.HTML(html)
comm_url_list = tree.xpath('//dt[@class="clearfix"]/h2/a/@href')
for i in comm_url_list:
comm = Comm(co_index)
i = i.split(';')
if i:
i = i[0]
detail_url = 'http://www.funi.com/' + i + '/detail.htm'
comm_index_url = 'http://www.funi.com/' + i
try:
comm = self.get_comm_info(comm_index_url, comm)
self.get_comm_detail(detail_url, comm)
except Exception as e:
print(
'小区错误：co_index={},url={}'.format(co_index, detail_url),
e)
def get_comm_info(self, comm_index_url, comm):
response = self.request_proxy(comm_index_url)
html = response.text
co_open_time = re.search('开盘时间：.*?class.*?>(.*?)<', html, re.S | re.M)
if co_open_time:
comm.co_open_time = co_open_time.group(1)
co_build_type = re.search('建筑类型：.*?<i.*?>(.*?)<', html, re.S | re.M)
if co_build_type:
comm.co_build_type = co_build_type.group(1)
co_all_house = re.search('规划户数：.*?<i.*?>(.*?)<', html, re.S | re.M)
if co_all_house:
comm.co_all_house = co_all_house.group(1)
return comm
@retry(retry(3))
def get_comm_detail(self, url, comm):
try:
response = requests.get(url, headers=self.headers)
html = response.text
tree = etree.HTML(html)
co_name = tree.xpath(
'//div[@class="info"]/h1/text()')[0].strip() # 小区名字
co_build_size = tree.xpath(
'//div[@class="intro"]/dl[1]/dd/ul/li[1]/em/i/text()') # 建筑面积
if co_build_size:
co_build_size = co_build_size[0].replace('㎡', '').strip()
else:
co_build_size = None
co_size = tree.xpath(
'//div[@class="intro"]/dl[1]/dd/ul/li[2]/em/i/text()') # 占地面的
if co_size:
co_size = co_size[0].replace('㎡', '').strip()
else:
co_size = None
co_volumetric = tree.xpath(
'//div[@class="intro"]/dl[1]/dd/ul/li[3]/em/i/text()') # 容积率
if co_volumetric:
co_volumetric = co_volumetric[0]
else:
co_volumetric = None
co_green = tree.xpath(
'//div[@class="intro"]/dl[1]/dd/ul/li[4]/em/i/text()') # 绿化率
if co_green:
co_green = co_green[0]
else:
co_green = None
co_address = tree.xpath(
'//div[@class="intro"]/dl[1]/dd/ul/li[6]/em/i/text()') # 小区地址
if co_address:
co_address = co_address[0]
else:
co_address = None
co_develops = tree.xpath(
'//div[@class="intro"]/dl[1]/dd/ul/li[8]/em/text()') # 开发商
if co_develops:
co_develops = co_develops[0]
else:
co_develops = None
co_pre_sale = tree.xpath(
'//div[@class="intro"]/dl[1]/dd/ul/li[9]/em/text()') # 预售证书
if co_pre_sale:
co_pre_sale = co_pre_sale[0]
else:
co_pre_sale = None
co_type = tree.xpath(
'//div[@class="intro"]/dl[2]/dd/ul/li[1]/em/text()') # 小区类型
if co_type:
co_type = co_type[0]
else:
co_type = None
co_handed_time = tree.xpath(
'//div[@class="intro"]/dl[2]/dd/ul/li[12]/em/text()') # 小区交房时间
if co_handed_time:
co_handed_time = co_handed_time[0]
else:
co_handed_time = None
comm.co_name = co_name
comm.co_address = co_address
comm.co_type = co_type
comm.co_green = co_green
comm.co_size = co_size
comm.co_build_size = co_build_size
comm.co_develops = co_develops
comm.co_pre_sale = co_pre_sale
comm.co_handed_time = co_handed_time
comm.co_volumetric = co_volumetric
comm.insert_db()
global count
count += 1
print(count)
except Exception as e:
print('小区错误，co_index={},url={}'.format(co_index, url), e)
def test_disarm_signal_on_success():
"""Success with a timeout disarms signal"""
_test_func = retry.retry(success=lambda x: True, timeout=1, interval=0.5)(foo)
_test_func(1)
sleep(1.2)
def test_unsuccessful_timeout():
"""Unsuccessful functions with a timeout work"""
foo_with_timeout = retry.retry(
success=lambda x: x > 0, timeout=5, interval=1)(foo)
with pytest.raises(retry.MaximumTimeoutExceeded):
foo_with_timeout(-1)
# -*- coding: utf-8 -*-
# Build-in Modules
import json
# 3rd-party Modules
import xmltodict
from retry import retry
import requests
retry_for_requests = retry((requests.ConnectionError, requests.Timeout),
tries=3,
delay=1,
backoff=2,
jitter=(1, 2))
def ensure_str(s):
if isinstance(s, unicode):
return s.encode('utf8')
else:
return str(s).decode('utf8').encode('utf8')
def parse_response(response):
resp_content_type = response.headers.get('content-type') or ''
resp_content_type = resp_content_type.lower().split(';')[0].strip()
if resp_content_type == 'application/json':
return response.json()
def test_execution():
"""Expected execution of a successful runstill works"""
foo_with_both = retry.retry(
exceptions=(ArithmeticError,), success=lambda x: x > 0)(foo)
assert foo_with_both(1) == 1
class Fuzhou(Crawler):
def __init__(self):
self.url = 'http://www.fzfgj.cn/website/search/q.html?type=spf'
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36',
}
def start_crawler(self):
self.start()
@retry(retry(3))
def start(self):
b = AllListUrl(first_page_url=self.url,
request_method='get',
analyzer_type='regex',
encode='gbk',
page_count_rule='共(.*?)页', )
page = b.get_page_count()
for i in range(1, int(page) + 1):
all_page_url = self.url + '&page=' + str(i)
response = requests.get(url=all_page_url, headers=self.headers)
html = response.text
tree = etree.HTML(html)
comm_url_list = tree.xpath('//dt[@class="name"]/a/@href')
area_list = tree.xpath('//dl[@class="houseList_n"]/dd[3]/text()')
for i in range(len(comm_url_list)):
url = 'http://www.fzfgj.cn/' + comm_url_list[i]
try:
comm = Comm(11)
comm.area = area_list[i].replace('所属区域:', '')
self.get_comm_info(url, comm)
except BaseException as e:
print('小区错误，co_index={},url={}'.format(co_index, url), e)
@retry(retry(3))
def get_comm_info(self, url, comm):
try:
response = requests.get(url=url, headers=self.headers)
html = response.text
tree = etree.HTML(html)
# 小区名称
co_name = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[1]/td[2]/text()')[0].strip()
# 小区地址
co_address = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[2]/td[2]/text()')[0].strip()
# 开工时间
co_build_start_time = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[3]/td[2]/text()')[
0].strip()
# 竣工时间
co_build_end_time = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[3]/td[4]/text()')[0].strip()
# 建筑结构
co_build_structural = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[4]/td[2]/text()')[
0].strip()
# 容积率
co_volumetric = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[6]/td[4]/text()')[0].strip()
# 绿化率
co_green = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[6]/td[2]/text()')[0].strip()
# 占地面的
co_size = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox3"]/div/table/tr[5]/td[2]/text()')[0].strip()
co_id = re.search('home/(.*?).html', url).group(1)
comm.co_name = co_name
comm.co_address = co_address
comm.co_build_start_time = co_build_start_time
comm.co_build_end_time = co_build_end_time
comm.co_build_structural = co_build_structural
comm.co_volumetric = co_volumetric
comm.co_green = co_green
comm.co_size = co_size
comm.co_id = co_id
comm.insert_db()
build_info_list = tree.xpath('//*[@id="ctl00_CPH_M_sm_spfBox1"]/div/table/tr[@class="hobuild"]')
for i in build_info_list:
try:
build = Building(11)
# 楼栋名称
bu_name = i.xpath('string(td[1])')[0]
bu_all_house = i.xpath('td[2]/text()')[0]
# 楼栋id
bu_id = i.xpath('td[1]/strong/a/@href')[0]
bu_id = re.search('building_id=(.*?)$', bu_id).group(1)
# 建筑面积
bu_build_size = i.xpath('string(td[3])').replace('�O', '')
build.co_id = co_id
build.bu_id = bu_id
build.bu_all_house = bu_all_house
build.bu_name = bu_name
build.bu_build_size = bu_build_size
build.insert_db()
self.get_house_info(bu_id, co_id)
except Exception as e:
print('楼栋错误，co_index={},url={}'.format(co_index, url), e)
except BaseException as e:
print('楼栋错误，co_index={},url={}'.format(co_index, url), e)
@retry(retry(3))
def get_house_info(self, bu_id, co_id):
url = 'http://www.fzfgj.cn/website/presale/home/HouseTableControl/GetData.aspx?Building_ID=' + bu_id
try:
response = requests.get(url=url, headers=self.headers)
xml = response.text
tree = etree.XML(xml)
logo = tree.xpath('//LOGICBUILDING_ID/text()')[0]
url_2 = 'http://www.fzfgj.cn/website/presale/home/HouseTableControl/GetData.aspx?LogicBuilding_ID=' + logo
result = requests.get(url_2, headers=self.headers)
xml_2 = result.text
tree_2 = etree.XML(xml_2)
house_info_list = tree_2.xpath('T_HOUSE')
for i in house_info_list:
try:
house = House(11)
ho_name = i.xpath('ROOM_NUMBER/text()')[0]
ho_build_size = i.xpath('BUILD_AREA/text()')[0]
ho_true_size = i.xpath('BUILD_AREA_INSIDE/text()')[0]
ho_share_size = i.xpath('BUILD_AREA_SHARE/text()')[0]
ho_floor = i.xpath('FLOOR_REALRIGHT/text()')[0]
ho_type = i.xpath('USE_FACT/text()')[0]
house.co_id = co_id
house.bu_id = bu_id
house.ho_build_size = ho_build_size
house.ho_true_size = ho_true_size
house.ho_share_size = ho_share_size
house.ho_floor = ho_floor
house.ho_name = ho_name
house.ho_type = ho_type
house.insert_db()
except Exception as e:
print('房号错误，co_index={},url={}'.format(co_index, url_2), e)
except BaseException as e:
print('房号错误，co_index={},url={}'.format(co_index, url), e)
def get_platform(driver):
"""Return info about the platform used by the given driver."""
name = driver.name.title()
caps = driver.capabilities
return '{} {} ({})'.format(name, caps['version'], caps['platform'].title())
def format_subprocess_error(error):
"""Return an error message string including the subprocess output."""
return '{}: {}'.format(error, error.output)
retry_process_error = retry(subprocess.CalledProcessError,
tries=2,
format_error=format_subprocess_error)
class TestCase(unittest.TestCase):
"""Helper base class that supports running browser tests."""
@classmethod
def setUpClass(cls):
# The Selenium web driver is instantiated here for each test case.
# This way the suite should be more reliable, especially when running
# Firefox tests, in which cases the GUI WebSocket connection is often
# problematic (i.e. connection errors) when switching from the sandbox
# mode back to the staging backend.
local_prefix = 'local-'
if browser_name.startswith(local_prefix):
# If the browser name has the "local-" prefix, i.e. "local-chrome',
def test_success_criteria():
"""Success criteria successfully raises MaximumRetriesExceeded"""
foo_with_success = retry.retry(success=lambda x: x > 0)(foo)
with pytest.raises(retry.MaximumRetriesExceeded):
foo_with_success(0)
import re
import arrow
from pkg_resources import RequirementParseError
import requests
import requirements
from retry import retry
import structlog
from . import PyLibrary
log = structlog.get_logger()
pypi_retry = retry(requests.RequestException, tries=5, delay=2, backoff=2)
def analyze(repository, path):
"""Parse requirements files for python packages."""
req_file_paths = path.glob("*requirements*.txt")
for req_file_path in req_file_paths:
if req_file_path.name == "requirements.txt":
for_production = True
elif req_file_path.name in {
"test-requirements.txt",
"dev-requirements.txt",
"build-requirements.txt",
"docs-requirements.txt",
}:
for_production = False
else:
for_production = None
def test_invalid_parameters():
"""The exceptions and success parameter can not both be None"""
with pytest.raises(TypeError):
retry.retry(exceptions=None, success=None)(foo)
