def creat_zombie(c):
print(f'''Processing: {c}''')
c = val.clean_str(c).replace(' ', '_')
driver = Chrome(driver_path)
# first load a page
driver.get(config.base_url)
# then load the cookie
cookies = pickle.load(open(cookie_path, "rb"))
for cookie in cookies:
driver.add_cookie(cookie)
tars = []
with open(r'{}/pages_{}.json'.format(config.path_data, c), 'r') as f:
urls = json.loads(f.read())
for u in urls:
driver.get(u)
time.sleep(config.sleep_long)
soup = BeautifulSoup(driver.page_source, 'html5lib')
# imgs = soup.findAll( 'img', class_='ImageThumbnail-image' )
imgs = soup.findAll('img', class_='info-card-media-user')
if (len(imgs) == 0):
l.commit(type='error', msg=f'Failed! Country: {c}, url: {u}')
break
# store the image urls
for e in imgs:
tars.append(e['src'])
# commit the result to file
rw.write_to_json(
r'{}/{}'.format(config.path_data, 'imgs_{}.json'.format(c)), tars)
# safely quit the driver
driver.quit()
class Bot:
def __init__(self, current_user):
from selenium import webdriver
from seleniumrequests import Chrome
self.current_user = current_user
self.headers = {
"Authority":
"www.supremenewyork.com",
"Method":
"GET",
"Path":
"/",
"Scheme":
"https",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"Accept-Encoding":
"gzip,deflate,br",
"Accept-Language":
"en-US,en;q=0.9,ru;q=0.8",
"Cache-Control":
"max-age=0",
"Upgrade-Insecure-Requests":
"1",
"User-Agent":
"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)"
}
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument("--headless")
self.browser = Chrome(os.path.dirname(os.path.realpath(__file__)) +
'/chromedriver',
chrome_options=chrome_options)
# получаем информацию о пользователе из файла
@staticmethod
def get_user_billing_info(filename):
with open('json/' + filename, mode='r') as fout:
user = json.load(fout)
return user
# получаем список вещей на покупку из файла
@staticmethod
def get_buy_list():
with open('json/items_to_buy.json', mode='r') as fout:
return json.load(fout)
# получаем список всех вещей, находящихся на данный момент в продаже
def fetch_stock(self):
# print(self.utc_to_est(), '-----> Fetching stock.')
return requests.request(
method='GET',
url='http://www.supremenewyork.com/mobile_stock.json',
headers=self.headers).json()
def get_item_info(self, id):
print(self.utc_to_est(), '-----> Getting item info.')
return requests.request(
method='GET',
url='http://www.supremenewyork.com/shop/{}.json'.format(id),
headers=self.headers).json()
def pure_cart_generator(self, data):
print(self.utc_to_est(), '-----> Generating cart.')
sizes = ''
total = 0
sizes_colors = ''
for piece in data:
total += int(piece[3])
sizes += '"{}":1,'.format(piece[2])
sizes_colors += '{},{}-'.format(piece[2], piece[1])
count = len(data)
if count == 1:
cookie = '"cookie":"1 item--"'
else:
cookie = '"cookie":"{}+items--{}"'.format(count, sizes_colors[:-1])
from urllib.request import quote
return quote('{' + sizes + cookie + ',"total":"€{}"'.format(total) +
'}',
safe='')
def checkout(self, response, data):
print(self.utc_to_est(), '-----> Checkout.')
sess = cart = ''
for item in re_s('[;,]', response.headers['Set-Cookie']):
if item[:9] == ' _supreme':
sess = item
break
for item in re_s('[;,]', response.headers['Set-Cookie']):
if item[:4] == 'cart':
cart = item
break
pure_cart = self.pure_cart_generator(data)
url = 'https://www.supremenewyork.com/checkout'
headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
'Accept-Encoding':
'gzip, deflate, sdch, br',
'Accept-Language':
'en-US,en;q=0.8',
'Connection':
'keep-alive',
'Cookie':
'pure_cart={0};cart={1};_supreme_sess={2};'.format(
pure_cart, cart, sess),
'Host':
'www.supremenewyork.com',
'If-None-Match':
'W/"7cd6c2d3c1278e6ec2f8f895e92dc2dd"',
'Referer':
'https:/www.supremenewyork.com/checkout',
'Upgrade-Insecure-Requests':
'1',
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87'
}
auth_r = requests.post(url=url, headers=headers, timeout=2)
x_csrf_token = findall('<meta name="csrf-token" content="(.*==)" />',
auth_r.text)[0]
print(self.utc_to_est(), '-----> Done.')
user = self.get_user_billing_info(self.current_user)
form_data = {
'authenticity_token': x_csrf_token,
'order[billing_name]': user['name'],
'order[email]': user['email'],
'order[tel]': user['tel'],
'order[billing_address]': user['address'],
'order[billing_address_2]': user['address2'],
'order[billing_address_3]': user['address3'],
'order[billing_city]': user['city'],
'order[billing_zip]': user['postcode'],
'order[billing_country]': user['country'],
'same_as_billing_address': '1',
'store_credit_id': '0',
'credit_card[type]': user['card_type'],
'credit_card[cnb]': user['card_number'],
'credit_card[month]': user['card_month'],
'credit_card[year]': user['card_year'],
'credit_card[vval]': user['card_cvv'],
'order[terms]': '1',
'hpcvv': ''
}
print(self.utc_to_est(), '-----> Opening browser.')
self.browser.get(url='https://www.supremenewyork.com/shop/cart')
self.browser.delete_all_cookies()
for key, value in requests.utils.dict_from_cookiejar(
response.cookies).items():
self.browser.add_cookie({'name': key, 'value': value})
print(self.utc_to_est(), '-----> Refreshing browser.')
self.browser.refresh()
print(self.utc_to_est(), '-----> Checkout request.')
self.browser.get(url='https://www.supremenewyork.com/checkout')
response_ = self.browser.request(
method='POST',
url='https://www.supremenewyork.com/checkout',
data=form_data)
print(self.utc_to_est(), '-----> Done.')
print(response_.content)
def add_to_cart(self, data):
print(self.utc_to_est(), '-----> Adding to cart.')
response = None
cookies = {}
for element in data:
product_id = element[0]
color_id = element[1]
size_id = element[2]
form_data = {
'utf8': '%E2%9C%93',
'style': str(color_id),
'size': str(size_id),
'commit': 'add+to+basket'
}
headers = {
'Authotiry':
'www.supremenewyork.com',
'Method':
'POST',
'Path':
'/shop/{}/add'.format(product_id),
'Scheme':
'https',
'accept':
'*/*;q=0.5, text/javascript, application/javascript, application/ecmascript, '
'application/x-ecmascript',
'accept-encoding':
'gzip, deflate, br',
'accept-language':
'en-US,en;q=0.9',
'content-length':
'58',
'content-type':
'application/x-www-form-urlencoded; charset=UTF-8',
'origin':
'https://www.supremenewyork.com',
'referer':
'https://www.supremenewyork.com/shop/',
'user-agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/69.0.3497.100 Safari/537.36',
'x-requested-with':
'XMLHttpRequest',
}
response = requests.request(
'POST',
'https://www.supremenewyork.com/shop/{}/add.json'.format(
product_id),
data=form_data,
headers=headers,
cookies=cookies)
cookies = response.cookies
self.checkout(response, data)
def choose_size_and_color(self, element, item_info):
print(self.utc_to_est(), '-----> Choosing color and size.')
# Приоритетные цвета, если цвет не задан или задан любой цвет
priority_colors = ['Black', 'White', 'Green']
# Представленные в магазине цвета по данной вещи
represented_colors = item_info['styles']
# Флаги цвета и размера
color_is_not_set = 'Any' in element['color'] or element['color'] == ''
size_is_not_set = 'Any' in element['size'] or element['size'] == ''
# Начало выбора
# Цвет не задан
if color_is_not_set:
# Итерация по представленным цветам
for color in represented_colors:
# Проверка на присутствие представленного в магазине цвета в приоритетных
if color['name'] in priority_colors:
# Если размер не выставлен
if size_is_not_set:
for size in color['sizes']:
# Проверка только на доступность данного размера
if size['stock_level'] is not 0:
print(self.utc_to_est(), '-----> Found (323).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Размер выставлен
else:
for size in color['sizes']:
# Проверка на доступность данного размера и на присутствие его в списке пользователя
if size['name'] in element['size'] and size[
'stock_level'] is not 0:
print(self.utc_to_est(), '-----> Found (330).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Если не найдено в приоритетных - идем по всем
else:
for color in represented_colors:
if color['name'] not in priority_colors:
# Размер не выставлен
if size_is_not_set:
for size in color['sizes']:
if size['stock_level'] is not 0:
print(self.utc_to_est(),
'-----> Found (340).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Размер выставлен
else:
for size in color['sizes']:
# Проверка на доступность данного размера и на присутствие его в списке пользователя
if size['name'] in element['size'] and size[
'stock_level'] is not 0:
print(self.utc_to_est(),
'-----> Found (347).')
return {
'color_id': color['id'],
'size_id': size['id']
}
else:
for color in represented_colors:
for size in color['sizes']:
if size['name'] in element['size'] and size[
'stock_level'] is not 0:
print(self.utc_to_est(), '-----> Found (353).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Цвет задан
else:
# Итерация по представленным цветам
for color in represented_colors:
# Проверка на присутствие представленного в магазине цвета в пользовательских цветах
if color['name'] in element['color']:
# Если размер не выставлен
if size_is_not_set:
for size in color['sizes']:
# Проверка только на доступность данного размера
if size['stock_level'] is not 0:
print(self.utc_to_est(), '-----> Found (367).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Размер выставлен
else:
for size in color['sizes']:
# Проверка на доступность данного размера и на присутствие его в списке пользователя
if size['name'] in element['size'] and size[
'stock_level'] is not 0:
print(self.utc_to_est(), '-----> Found (374).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Ни одного пользовательского цвета не найдено - идем по приоритетным
else:
# Итерация по приоритетным цветам
for color in represented_colors:
# Проверка на присутствие представленного в магазине цвета в приоритетных
if color['name'] in priority_colors:
# Если размер не выставлен
if size_is_not_set:
for size in color['sizes']:
# Проверка только на доступность данного размера
if size['stock_level'] is not 0:
print(self.utc_to_est(),
'-----> Found (387).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Размер выставлен
else:
for size in color['sizes']:
# Проверка на доступность данного размера и на присутствие его в списке пользователя
if size['name'] in element['size'] and size[
'stock_level'] is not 0:
print(self.utc_to_est(),
'-----> Found (394).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Если не найдено в приоритетных - идем по всем
else:
for color in represented_colors:
if color['name'] not in priority_colors:
if size_is_not_set:
for size in color['sizes']:
if size['stock_level'] is not 0:
print(self.utc_to_est(),
'-----> Found (403).')
return {
'color_id': color['id'],
'size_id': size['id']
}
else:
for color in represented_colors:
for size in color['sizes']:
if size['name'] in element['size'] and size[
'stock_level'] is not 0:
print(self.utc_to_est(),
'-----> Found (409).')
return {
'color_id': color['id'],
'size_id': size['id']
}
# Конец выбора
# Если предмет не найден
print(self.utc_to_est(), '-----> Not found.')
return {'color_id': None, 'size_id': None}
def find_items(self):
print(self.utc_to_est(), '-----> Started processing.')
my_items = self.get_buy_list()
requests_data = []
for element in my_items: # items_to_buy
item_found = False # flag
# split desired product name into pieces
elem_split = element['name'].split(' ')
while not item_found:
stock = self.fetch_stock() # supreme stock
# replace 'Tops/Sweaters' with 'Tops/Sweaters'
# if element['type'] == 'tops-sweaters':
# element['type'] = 'Tops/Sweaters'
#
# elif element['type'] == 't-shirts':
# element['type'] = 'Shirts'
#
# else:
# element['type'] = element['type'].title()
# iterating through represented items in market
for item in stock['products_and_categories'][element['type']]:
# searching for most appropriate item name
'''
comparing intersection length with desired product name split length minus one
e.g.: user wants to buy 'Supreme/The Killer Trust Tee'
but in real its name sounds like 'The Killer Trust Tee'
the way is to split desired item name: ['Supreme/The', 'Killer', 'Trust', 'Tee'] and
to split item name represented in shop: ['The', 'Killer', 'Trust', 'Tee']
By finding the longest intersection of 2 sets, we can be sure we found desired item.
The longest intersection usually has length of desired product name split length minus one
'''
if len(list(
set(elem_split)
& set(item['name'].split(' ')))) >= (
len(elem_split) - 1):
# item found
print(self.utc_to_est(),
'-----> {} found.'.format(element['name']))
item_found = True
# collecting info
price = round(item['price_euro'] / 100)
item_info = self.get_item_info(item['id'])
color_and_size = self.choose_size_and_color(
element, item_info)
# if size and color info successfully found, add data to request data
if color_and_size['color_id'] is None or color_and_size[
'size_id'] is None:
print(self.utc_to_est(),
'-----> Desired item has been sold out.')
else:
requests_data.append([
item['id'], color_and_size['color_id'],
color_and_size['size_id'], price
])
# when all desired items data collected, add them to cart
if len(requests_data) == len(my_items):
self.add_to_cart(requests_data)
@staticmethod
def utc_to_est():
from datetime import datetime
return datetime.now()
class UmsConnection():
def __init__(self, user_phone, user_password, cookies=None):
self._user_phone = user_phone
self._user_password = user_password
self._is_authorized = False
self._captcha = None # REMOVE
options = Options()
options.add_argument("--headless")
WINDOW_SIZE = "1620,880" # ?
options.add_argument("--window-size=%s" % WINDOW_SIZE)
self.webdriver = Chrome(chrome_options=options)
if cookies:
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
for cookie in cookies:
self.webdriver.add_cookie(cookie)
self.webdriver.get('https://messages.megafon.ru/onebox/mix.do')
self._is_authorized = self.try_is_authorized()
if not self.is_authorized:
self.webdriver.get(r"https://messages.megafon.ru/onebox/mix.do")
phone_input = self.webdriver.find_element_by_id('user12')
phone_input.send_keys(user_phone)
password_input_secinfo = self.webdriver.find_element_by_id('secinfo')
password_input_secinfo.click()
password_input = self.webdriver.find_element_by_id("secinfo2")
password_input.send_keys(user_password)
def get_captcha(self) -> bytes:
captcha = self.webdriver.find_element_by_id('imageC')
location = captcha.location
size = captcha.size
screenshot = self.webdriver.get_screenshot_as_png()
im = Image.open(BytesIO(screenshot))
left = location['x']
top = location['y']
right = location['x'] + size['width']
bottom = location['y'] + size['height']
im = im.crop((left, top, right, bottom))
self._captcha = im # REMOVE
stream = BytesIO()
im.save(stream, format="PNG")
return stream.getvalue()
def send_captcha_key(self, key) -> bool:
captcha_input = self.webdriver.find_element_by_id('imageObj')
captcha_input.send_keys(key)
submit_button = self.webdriver.find_element_by_id('index_login')
submit_button.click()
try:
WebDriverWait(self.webdriver, 2).until(
EC.url_contains(("messages.megafon.ru/onebox/mix.do")))
self._is_authorized = True
self._captcha.save(f'captchas/{key}.png') # REMOVE
return True
except exceptions.TimeoutException:
return False
def try_is_authorized(self) -> bool:
# TODO
url = r"https://messages.megafon.ru/onebox/getChatList.do"
get_chats_url_params = {
'startNum': '1',
'endNum': 10,
'reFreshFlag': '1',
'operation': '1',
'chatMsgType': '10100000000110000000000000000000',
't': random.random()
}
response = self.webdriver.request('GET', url, params=get_chats_url_params)
return response.status_code == 200
@property
def is_authorized(self):
return self._is_authorized
def get_chat_list(self, number=10):
url = r"https://messages.megafon.ru/onebox/getChatList.do"
get_chats_url_params = {
'startNum': '1',
'endNum': number,
'reFreshFlag': '1',
'operation': '1',
'chatMsgType': '10100000000110000000000000000000',
't': random.random()
}
response = self.webdriver.request('GET', url, params=get_chats_url_params)
return response.text
def get_one_box(self, address, number=10, from_num=1):
url = r"https://messages.megafon.ru/onebox/oneboxList.do"
params = [
('umReq.ctlg', '1,2'),
('umReq.numFlg', '1'),
('umReq.mType', '6149'),
('umReq.srt', '0'),
('umReq.lType', '0'),
('umReq.dFlg', '0'),
('umReq.srtDr', '0'),
('umReq.rdFlg', '0'),
('umReq.bNum', from_num),
('umReq.eNum', number),
('umReq.snd', address),
('umReq.rcv', self._user_phone),
('umReq.bTime', ''),
('umReq.eTime', ''),
('umReq.impt', '-1'),
('umReq.t', ''),
('umReq.threadFlag', '1'),
('rownid', random.random()),
]
response = self.webdriver.request('GET', url, params=params)
return response.text
def close(self):
self.webdriver.close()
def get_cookies_json(self):
cookies = self.webdriver.get_cookies()
return json.dumps(cookies)
