def search(self, query, pages=cfg.search_pages):
'''
Queries the search engine, goes through the pages and collects the results.
:param query: str The search query
:param pages: int Optional, the number of pages to search
:returns Results object
'''
utl.console('\rSearching {}'.format(self._name))
self._query = utl.decode_bytes(query)
ref, (url, data) = None, self._first_page()
for page in range(1, pages + 1):
try:
resp = self._get_page(url, data, ref)
if resp.http != 200:
msg = ('HTTP ' + str(resp.http)) if resp.http else resp.html
utl.console(u'\rError: {:<20}'.format(msg))
break
tags = BeautifulSoup(resp.html, "html.parser")
items = self._filter_results(tags)
self._collect_results(items)
msg = '\rpage: {:<8} links: {}'.format(page, len(self.results))
utl.console(msg, end='')
ref, (url, data) = url, self._next_page(tags)
if not url:
break
sleep(random_uniform(*self._delay))
except KeyboardInterrupt:
break
utl.console('\r\t\t\t\t\t\t\r', end='')
return self.results
def act(self, observation):
"""
Our agent attempts to manipulate the world.
Acting according to epsilon greedy policy.
args
observation (np.array)
returns
action (np.array)
"""
self.counter += 1
epsilon = self.epsilon_getter.epsilon
logging.debug('epsilon is {}'.format(epsilon))
if epsilon > random_uniform():
action = self.env.action_space.sample()
logging.debug('acting randomly - action is {}'.format(action))
else:
action = self.predict_online(observation)
logging.debug('acting optimally action is {}'.format(action))
epsilon_sum = tf.Summary(
value=[tf.Summary.Value(tag='epsilon', simple_value=epsilon)])
self.acting_writer.add_summary(epsilon_sum, self.counter)
self.acting_writer.flush()
# return np.array(action).reshape(1, *self.action_space_shape)
return action
def random_network(height, level):
self = Network(height, level, [])
for i in range(height):
self.weight.append([])
for j in range(level[i + 1]):
self.weight[-1].append(random_uniform(-1, 1))
return self
def search(self, query, pages=cfg.SEARCH_ENGINE_RESULTS_PAGES):
'''Queries the search engine, goes through the pages and collects the results.
:param query: str The search query
:param pages: int Optional, the maximum number of results pages to search
:returns SearchResults object
'''
out.console('Searching {}'.format(self.__class__.__name__))
self._query = utils.decode_bytes(query)
request = self._first_page()
for page in range(1, pages + 1):
try:
response = self._get_page(request['url'], request['data'])
if not self._is_ok(response):
break
tags = BeautifulSoup(response.html, "html.parser")
items = self._filter_results(tags)
self._collect_results(items)
msg = 'page: {:<8} links: {}'.format(page, len(self.results))
out.console(msg, end='')
request = self._next_page(tags)
if not request['url']:
break
if page < pages:
sleep(random_uniform(*self._delay))
except KeyboardInterrupt:
break
out.console('', end='')
return self.results
def spawn_player_projectiles(self):
with self._lmb_event_lock:
current_lmb_event = self._lmb_event
if current_lmb_event is not None:
moving_unit_vector: Vector2D = (
self._get_player_hand_cursor_unit_vector(
Vector2D(current_lmb_event.x, current_lmb_event.y)))
spawn_multiplier: float = 20
spawn_location: Vector2D = Vector2D(
self._game_map.movable_objects[0].location.x +
Player.HAND_LOCATION.x +
moving_unit_vector.x * spawn_multiplier,
self._game_map.movable_objects[0].location.y +
Player.HAND_LOCATION.y +
moving_unit_vector.y * spawn_multiplier)
if self._selected_weapon is self._Weapons.Handgun:
if current_lmb_event.type.name == 'ButtonRelease':
self._handgun_can_fire = True
elif (current_lmb_event.type.name == 'ButtonPress'
and self._handgun_can_fire):
moving_vector: Vector2D = Vector2D(
moving_unit_vector.x *
ProjectileObject.PROJECTILE_SPEED,
moving_unit_vector.y *
ProjectileObject.PROJECTILE_SPEED)
self._game_map.movable_objects.append(
HandgunProjectile(moving_vector, spawn_location))
self._handgun_can_fire = False
elif self._selected_weapon is self._Weapons.MachineGun:
if (current_lmb_event.type.name
in ['ButtonPress', 'Motion']):
moving_vector: Vector2D = Vector2D(0, 0)
# In radians
rotation_angle: float = random_uniform(
-MachineGunProjectile.ANGLE_SCATTER_RADIUS,
MachineGunProjectile.ANGLE_SCATTER_RADIUS)
moving_vector.x = (
cos(rotation_angle) * moving_unit_vector.x *
ProjectileObject.PROJECTILE_SPEED -
sin(rotation_angle) * moving_unit_vector.y *
ProjectileObject.PROJECTILE_SPEED)
moving_vector.y = (
sin(rotation_angle) * moving_unit_vector.x *
ProjectileObject.PROJECTILE_SPEED +
cos(rotation_angle) * moving_unit_vector.y *
ProjectileObject.PROJECTILE_SPEED)
self._game_map.movable_objects.append(
MachineGunProjectile(moving_vector,
spawn_location))
def ununiform_mutation(self):
for i in self.chromosome:
for j in self.mutation_dict.items():
value = random_random()
if value < j[1][0]:
i[j[0]] = random_uniform(
j[1][1] + i[j[0]],
j[1][2] + i[j[0]])
def start_badoo_liker(count):
login('cookies_badoo_(keep_this_file_save_!_).txt', url=badoo_matches_url)
for i in tqdm(iterable=range(int(count)), desc='swipes done',
unit='swipe'): # tqdm for progress bar
try:
swipe_badoo_user()
time_sleep(random_uniform(1.0, 2.0))
except Exception as e:
print(e)
def swipe_badoo_user():
# Send key to whole page instead of particular element (Skip ads pop up if exists). Why "perform" ?
webdriver.ActionChains(driver).send_keys(Keys.ESCAPE).perform()
if random_uniform(
0, 4) > 1: # Random value for clicking like to every 4rd user
webdriver.ActionChains(driver).send_keys(
1).perform() # 1 is like, 2 is dislike
else: # click dislike
webdriver.ActionChains(driver).send_keys(
2).perform() # 1 is like, 2 is dislike
def search(self, query, pages=cfg.max_pages, unique=False):
'''
Queries the search engine, goes through all pages and collects the results.
:param query: str The search query
:param pages: int Optional, the number of search results pages
:param unique: bool Optional, collects unique domains only
:returns Results
'''
self._query, self._filter = self._filter_query(query)
_pages = [self._first_page()]
utl.console('\rSearching {}'.format(self._name))
while len(_pages) <= pages and _pages[-1]['url'] is not None:
ref = _pages[-2]['url'] if len(_pages) > 1 else None
try:
request = self._get_page(_pages[-1]['url'], _pages[-1]['data'],
ref)
if request['http'] == 200:
tags = BeautifulSoup(request['html'], "html.parser")
items = self._collect_results(tags)
for item in items:
domain, url = item['host'], item['link']
if not utl.is_url(url):
continue
if (unique and domain
in self._domains) or domain in self.blacklist:
continue
if item in self.results.results():
continue
self.results._results += [item]
self._domains += [domain]
msg = '\rpage: {:<8} links: {}'.format(
len(_pages), len(self.results))
utl.console(msg, end='')
_pages += [self._next_page(tags)]
else:
msg = ('HTTP ' + str(request['http'])
) if request['http'] else request['html']
utl.console(u'\rError: {:<20}'.format(msg))
break
sleep(random_uniform(*self._delay))
except KeyboardInterrupt:
break
utl.console('\r\t\t\t\t\r', end='')
return self.results
def __init__(self):
self.directionX = 0
self.directionY = 0
pygame.sprite.Sprite.__init__(self)
#Type of asteroid
self.type_asteroid = random_randrange(7)
if self.type_asteroid == const.BIG_ASTEROID1:
self.image = const.SPRITE_B_AST1
elif self.type_asteroid == const.BIG_ASTEROID2:
self.image = const.SPRITE_B_AST2
elif self.type_asteroid == const.BIG_ASTEROID3:
self.image = const.SPRITE_B_AST3
elif self.type_asteroid == const.BIG_ASTEROID4:
self.image = const.SPRITE_B_AST4
elif self.type_asteroid == const.BIG_ASTEROID5:
self.image = const.SPRITE_B_AST5
elif self.type_asteroid == const.AN_ASTEROID1:
self.image = const.SPRITE_A_AST1
elif self.type_asteroid == const.AN_ASTEROID2:
self.image = const.SPRITE_A_AST2
screen = pygame.display.get_surface()
self.area = screen.get_rect()
self.location = [
random_uniform(0, const.WINDOW_WIDTH - self.image.get_width()), 0
]
if self.location[0] + self.image.get_width() > const.WINDOW_WIDTH:
self.directionX = random_uniform(-1, 1)
else:
self.directionX = random_uniform(-1, 2)
if self.location[1] + self.image.get_height() > const.WINDOW_HEIGHT:
self.directionY = random_uniform(-1, 1)
elif self.location[
1] + 1 < const.WINDOW_HEIGHT and self.directionX == 0:
self.directionY = random_uniform(0, 2)
else:
self.directionY = random_uniform(0, 2)
def __init__(self):
self.directionX = 0
self.directionY = 0
pygame.sprite.Sprite.__init__(self)
#Type of asteroid
self.type_asteroid = random_randrange(7)
if self.type_asteroid == const.BIG_ASTEROID1:
self.image = const.SPRITE_B_AST1
elif self.type_asteroid == const.BIG_ASTEROID2:
self.image = const.SPRITE_B_AST2
elif self.type_asteroid == const.BIG_ASTEROID3:
self.image = const.SPRITE_B_AST3
elif self.type_asteroid == const.BIG_ASTEROID4:
self.image = const.SPRITE_B_AST4
elif self.type_asteroid == const.BIG_ASTEROID5:
self.image = const.SPRITE_B_AST5
elif self.type_asteroid == const.AN_ASTEROID1:
self.image = const.SPRITE_A_AST1
elif self.type_asteroid == const.AN_ASTEROID2:
self.image = const.SPRITE_A_AST2
screen = pygame.display.get_surface()
self.area = screen.get_rect()
self.location = [random_uniform(0, const.WINDOW_WIDTH - self.image.get_width()) ,0]
if self.location[0] + self.image.get_width() > const.WINDOW_WIDTH:
self.directionX = random_uniform(-1, 1)
else:
self.directionX = random_uniform(-1, 2)
if self.location[1] + self.image.get_height() > const.WINDOW_HEIGHT:
self.directionY = random_uniform(-1, 1)
elif self.location[1] + 1 < const.WINDOW_HEIGHT and self.directionX == 0:
self.directionY = random_uniform(0, 2)
else:
self.directionY = random_uniform(0, 2)
def get_hash_func(self):
"""initialize each L2Hash with a different random projection vector
and offset
"""
return L2Hash(self.get_projection(), random_uniform(0, self.w), self.w)
async def _fck_run(self):
"""
main
:return:
"""
while True:
try:
if get_shanghai_time().hour == 0:
await async_sleep(60 * 60 * 3.5)
continue
self.db_res = await self.get_db_res()
all_tasks_params_list_obj = await self.get_all_tasks_params_list_obj(
)
tasks_params_list_obj = TasksParamsListObj(
tasks_params_list=all_tasks_params_list_obj,
step=self.concurrency,
slice_start_index=0,
)
while True:
try:
slice_params_list = tasks_params_list_obj.__next__()
except AssertionError:
break
coupon_url_list = await self.get_coupon_url_list_by_goods_id_list(
slice_params_list=slice_params_list)
# pprint(coupon_url_list)
# 测试
# coupon_url = 'https://uland.taobao.com/coupon/edetail?e=5M3kt6O%2FfZqa2P%2BN2ppgB2X2iX5OaVULVb9%2F1Hxlj5NQYhkEFAI5hGSlkL8%2BFO6JZSEGEhAo6u3FrE8HH4fiD8KUixUTTLeu0WMS0ZKY%2BzmLVIDjuHwzlw%3D%3D&af=1&pid=mm_55371245_39912139_149806421'
# coupon_url_list = [coupon_url for i in range(6)]
# # goods_id得对应上面的领券地址
# goods_id_and_coupon_url_queue.put({
# 'goods_id': '562016826663',
# 'coupon_url': coupon_url,
# })
if coupon_url_list == []:
# 此处也回收下
collect()
self.lg.info('coupon_url_list为空list, 跳过!')
random_sleep_time = random_uniform(3., 6.)
self.lg.info('休眠{}s ...'.format(random_sleep_time))
await async_sleep(random_sleep_time)
continue
# 划分coupon_url_list, 避免多开使内存崩溃
tasks_params_list_obj2 = TasksParamsListObj(
tasks_params_list=coupon_url_list,
step=self.concurrency2,
slice_start_index=0,
)
while True:
try:
slice_params_list2 = tasks_params_list_obj2.__next__(
)
except AssertionError:
break
tasks = []
for coupon_url in slice_params_list2:
self.lg.info(
'create task[where coupon_url: {}] ...'.format(
coupon_url))
tasks.append(
self.loop.create_task(
self.intercept_target_api(
coupon_url=coupon_url)))
try:
one_res = await wait_for(
fut=async_wait_tasks_finished(tasks=tasks),
timeout=60 * 2,
)
except AsyncTimeoutError:
self.lg.error('遇到错误:', exc_info=True)
continue
# 成功总数
success_count = 0
for item in one_res:
if item:
success_count += 1
self.lg.info('成功个数: {}, 成功概率: {:.3f}'.format(
success_count, success_count / self.concurrency2))
collect()
collect()
self.lg.info('一次大循环结束!!')
except Exception:
self.lg.error('遇到错误:', exc_info=True)
await async_sleep(30)
finally:
self.lg.info('休眠6s...')
await async_sleep(6.)
collect()
@app.route('/api/users')
def users():
return jsonify({'users': get_all_items_by_key('users')})
@app.route('/api/users/<username>')
def user(username):
u = get_items_by_filter('users', 'username', username)
if len(u) > 0:
return jsonify({'user': u[0]})
else:
abort(404)
@app.route('/api/purchases/by_user/<username>')
def purchases_by_user(username):
p = get_items_by_filter('purchases', 'username', username)
return get_purchases(p)
@app.route('/api/purchases/by_product/<int:product_id>')
def purchases_by_product(product_id):
p = get_items_by_filter('purchases', 'product_id', product_id)
return get_purchases(p)
if __name__ == "__main__":
data = load_data()
app.before_request(lambda: sleep(random_uniform(0.5, 1)))
app.run(host='0.0.0.0', port=8085)
def get_tm_coupon_url_from_lq5u(goods_id='',
goods_name_or_m_url: str='',
proxy_type=PROXY_TYPE_HTTP,
num_retries=5,
logger=None,) -> str:
"""
从领券无忧根据goods_name搜索tm优惠券
:param goods_id: 推荐使用商品id来查券
:param goods_name_or_m_url: 商品名 or 商品地址
:return:
"""
# todo 测试发现无需搜索, 只需把goods_id 改为领券无忧的对应的url即可查询是否有券
# 基于领券无忧来根据商品名获取其优惠券
headers = get_random_headers(
user_agent_type=1,
connection_status_keep_alive=False,
)
headers.update({
'Proxy-Connection': 'keep-alive',
'Origin': 'http://www.lq5u.com',
'Content-Type': 'application/x-www-form-urlencoded',
'Referer': 'http://www.lq5u.com/',
})
# # 只搜索天猫的
# data = {
# 'p': '1',
# 'cid': '0',
# 'sort': '0',
# 'b2c': '1', # '0'为搜索tb, tm | '1'为只搜索tm
# 'coupon': '1',
# 'k': goods_name_or_m_url,
# }
# body = Requests.get_url_body(
# method='post',
# url='http://www.lq5u.com/',
# headers=headers,
# # cookies=cookies,
# data=data,
# verify=False,
# ip_pool_type=IP_POOL_TYPE,
# num_retries=num_retries,
# proxy_type=proxy_type,)
# assert body != ''
# # print(body)
#
# lq5u_url_list_sel = {
# 'method': 'css',
# 'selector': 'li a ::attr("onmousedown")',
# }
# ori_lq5u_url_list = parse_field(
# parser=lq5u_url_list_sel,
# target_obj=body,
# is_first=False,)
# lq5u_url_list = []
# for item in ori_lq5u_url_list:
# try:
# url = re.compile('this.href=\'(.*?)\'').findall(item)[0]
# assert url != ''
# except Exception:
# continue
#
# lq5u_url_list.append('http://www.lq5u.com' + url)
#
# assert lq5u_url_list != []
# pprint(lq5u_url_list)
# 领券无忧对应页面如下
# url = 'http://www.lq5u.com/item/index/iid/{}.html'.format(goods_id)
# body = Requests.get_url_body(
# method='get',
# url=url,
# headers=headers,
# verify=False,
# ip_pool_type=IP_POOL_TYPE,
# num_retries=num_retries,
# proxy_type=proxy_type, )
# assert body != ''
# print(body)
#
# coupon_info_sel = {
# 'method': 'css',
# 'selector': 'span.b.red ::text',
# }
# coupon_info = parse_field(
# parser=coupon_info_sel,
# target_obj=body,
# )
# if '很遗憾，该商品没有优惠券' in coupon_info:
# return []
# else:
# _print(msg='goods_id: {}, 存在优惠券'.format(goods_id), logger=logger)
# return []
# 根据领券无忧接口
from random import uniform as random_uniform
headers = get_random_headers(
user_agent_type=1,
connection_status_keep_alive=False,
)
headers.update({
'Origin': 'http://www.lq5u.com',
'Content-Type': 'application/x-www-form-urlencoded',
'Referer': 'http://www.lq5u.com/item/index/iid/{}.html'.format(goods_id),
'X-Requested-With': 'XMLHttpRequest',
})
params = (
('rnd', str(random_uniform(0, 1))), # eg: '0.4925945510743117'
)
data = {
'iid': goods_id
}
body = Requests.get_url_body(
method='post',
url='http://www.lq5u.com/item/ajax_get_auction_code.html',
headers=headers,
params=params,
data=data,
verify=False,
ip_pool_type=IP_POOL_TYPE,
num_retries=num_retries,
proxy_type=proxy_type,)
assert body != ''
# print(body)
data = json_2_dict(
json_str=body,
default_res={},
logger=logger,).get('data', {})
# pprint(data)
coupon_url = data.get('coupon_click_url', '')
if coupon_url != '':
_print(msg='该goods_id: {} 含 有优惠券'.format(goods_id), logger=logger)
else:
_print(msg='该goods_id: {} 不含 有优惠券'.format(goods_id), logger=logger)
return coupon_url
def __init__(self):
self.directionX = 0
self.directionY = 0
pygame.sprite.Sprite.__init__(self)
#Type of asteroid
self.type_asteroid = random_randrange(7)
if self.type_asteroid == const.BIG_ASTEROID1:
self.image = const.SPRITE_B_AST1
elif self.type_asteroid == const.BIG_ASTEROID2:
self.image = const.SPRITE_B_AST2
elif self.type_asteroid == const.BIG_ASTEROID3:
self.image = const.SPRITE_B_AST3
elif self.type_asteroid == const.BIG_ASTEROID4:
self.image = const.SPRITE_B_AST4
elif self.type_asteroid == const.BIG_ASTEROID5:
self.image = const.SPRITE_B_AST5
elif self.type_asteroid == const.AN_ASTEROID1:
self.image = const.SPRITE_A_AST1
elif self.type_asteroid == const.AN_ASTEROID2:
self.image = const.SPRITE_A_AST2
screen = pygame.display.get_surface()
self.area = screen.get_rect()
self.starting_side = random_randrange(4)
self.rect = self.image.get_rect()
if self.starting_side == 0:
self.location = [random_uniform(0, const.WINDOW_WIDTH - self.image.get_width() / 2 - 1 ), - self.image.get_height()/2 + 1]
self.directionX = random_uniform(-1, 1)
self.directionY = random_uniform(0.1, 1)
elif self.starting_side == 1:
self.location = [random_uniform(0, const.WINDOW_WIDTH - self.image.get_width() / 2 - 1), const.WINDOW_HEIGHT - self.image.get_height()/2 - 1]
self.directionX = random_uniform(-1, 1)
self.directionY = random_uniform(-0.1, -1)
elif self.starting_side == 2:
self.location = [-self.image.get_width() / 2 + 1, random_uniform(0, const.WINDOW_HEIGHT - self.image.get_height()/2 - 1)]
self.directionX = random_uniform(0.1, 1)
self.directionY = random_uniform(-1, 1)
elif self.starting_side == 3:
self.location = [const.WINDOW_WIDTH - self.image.get_width() / 2 - 1, random_uniform(0, const.WINDOW_HEIGHT - self.image.get_height()/2 - 1)]
self.directionX = random_uniform(-0.1, -1)
self.directionY = random_uniform(-1, 1)
if -0.1 < self.directionX < 0.1:
self.directionX = 0.1 * self.directionX / abs(self.directionX)
if -0.1 < self.directionY < 0.1:
self.directionY = 0.1 * self.directionY / abs(self.directionY)
def generate_random_points(n, min_xy, max_xy):
return [(random_uniform(min_xy, max_xy), random_uniform(min_xy, max_xy))
for i in range(n)]
def __init__(self):
self.directionX = 0
self.directionY = 0
pygame.sprite.Sprite.__init__(self)
#Type of asteroid
self.type_asteroid = random_randrange(7)
if self.type_asteroid == const.BIG_ASTEROID1:
self.image = const.SPRITE_B_AST1
elif self.type_asteroid == const.BIG_ASTEROID2:
self.image = const.SPRITE_B_AST2
elif self.type_asteroid == const.BIG_ASTEROID3:
self.image = const.SPRITE_B_AST3
elif self.type_asteroid == const.BIG_ASTEROID4:
self.image = const.SPRITE_B_AST4
elif self.type_asteroid == const.BIG_ASTEROID5:
self.image = const.SPRITE_B_AST5
elif self.type_asteroid == const.AN_ASTEROID1:
self.image = const.SPRITE_A_AST1
elif self.type_asteroid == const.AN_ASTEROID2:
self.image = const.SPRITE_A_AST2
screen = pygame.display.get_surface()
self.area = screen.get_rect()
self.starting_side = random_randrange(4)
if self.starting_side == 0:
self.location = [random_uniform(0, const.WINDOW_WIDTH - self.image.get_width() / 2 - 1 ), - self.image.get_height()/2 + 1]
self.directionX = random_uniform(-1, 1)
self.directionY = random_uniform(0.1, 1)
elif self.starting_side == 1:
self.location = [random_uniform(0, const.WINDOW_WIDTH - self.image.get_width() / 2 - 1), const.WINDOW_HEIGHT - self.image.get_height()/2 - 1]
self.directionX = random_uniform(-1, 1)
self.directionY = random_uniform(-0.1, -1)
elif self.starting_side == 2:
self.location = [-self.image.get_width() / 2 + 1, random_uniform(0, const.WINDOW_HEIGHT - self.image.get_height()/2 - 1)]
self.directionX = random_uniform(0.1, 1)
self.directionY = random_uniform(-1, 1)
elif self.starting_side == 3:
self.location = [const.WINDOW_WIDTH - self.image.get_width() / 2 - 1, random_uniform(0, const.WINDOW_HEIGHT - self.image.get_height()/2 - 1)]
self.directionX = random_uniform(-0.1, -1)
self.directionY = random_uniform(-1, 1)
if -0.1 < self.directionX < 0.1:
self.directionX = 0.1 * self.directionX / abs(self.directionX)
if -0.1 < self.directionY < 0.1:
self.directionY = 0.1 * self.directionY / abs(self.directionY)
