def creat_zombie(c): print(f'''Processing: {c}''') c = val.clean_str(c).replace(' ', '_') driver = Chrome(driver_path) # first load a page driver.get(config.base_url) # then load the cookie cookies = pickle.load(open(cookie_path, "rb")) for cookie in cookies: driver.add_cookie(cookie) tars = [] with open(r'{}/pages_{}.json'.format(config.path_data, c), 'r') as f: urls = json.loads(f.read()) for u in urls: driver.get(u) time.sleep(config.sleep_long) soup = BeautifulSoup(driver.page_source, 'html5lib') # imgs = soup.findAll( 'img', class_='ImageThumbnail-image' ) imgs = soup.findAll('img', class_='info-card-media-user') if (len(imgs) == 0): l.commit(type='error', msg=f'Failed! Country: {c}, url: {u}') break # store the image urls for e in imgs: tars.append(e['src']) # commit the result to file rw.write_to_json( r'{}/{}'.format(config.path_data, 'imgs_{}.json'.format(c)), tars) # safely quit the driver driver.quit()
class Bot: def __init__(self, current_user): from selenium import webdriver from seleniumrequests import Chrome self.current_user = current_user self.headers = { "Authority": "www.supremenewyork.com", "Method": "GET", "Path": "/", "Scheme": "https", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", "Accept-Encoding": "gzip,deflate,br", "Accept-Language": "en-US,en;q=0.9,ru;q=0.8", "Cache-Control": "max-age=0", "Upgrade-Insecure-Requests": "1", "User-Agent": "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)" } chrome_options = webdriver.ChromeOptions() chrome_options.add_argument("--headless") self.browser = Chrome(os.path.dirname(os.path.realpath(__file__)) + '/chromedriver', chrome_options=chrome_options) # получаем информацию о пользователе из файла @staticmethod def get_user_billing_info(filename): with open('json/' + filename, mode='r') as fout: user = json.load(fout) return user # получаем список вещей на покупку из файла @staticmethod def get_buy_list(): with open('json/items_to_buy.json', mode='r') as fout: return json.load(fout) # получаем список всех вещей, находящихся на данный момент в продаже def fetch_stock(self): # print(self.utc_to_est(), '-----> Fetching stock.') return requests.request( method='GET', url='http://www.supremenewyork.com/mobile_stock.json', headers=self.headers).json() def get_item_info(self, id): print(self.utc_to_est(), '-----> Getting item info.') return requests.request( method='GET', url='http://www.supremenewyork.com/shop/{}.json'.format(id), headers=self.headers).json() def pure_cart_generator(self, data): print(self.utc_to_est(), '-----> Generating cart.') sizes = '' total = 0 sizes_colors = '' for piece in data: total += int(piece[3]) sizes += '"{}":1,'.format(piece[2]) sizes_colors += '{},{}-'.format(piece[2], piece[1]) count = len(data) if count == 1: cookie = '"cookie":"1 item--"' else: cookie = '"cookie":"{}+items--{}"'.format(count, sizes_colors[:-1]) from urllib.request import quote return quote('{' + sizes + cookie + ',"total":"€{}"'.format(total) + '}', safe='') def checkout(self, response, data): print(self.utc_to_est(), '-----> Checkout.') sess = cart = '' for item in re_s('[;,]', response.headers['Set-Cookie']): if item[:9] == ' _supreme': sess = item break for item in re_s('[;,]', response.headers['Set-Cookie']): if item[:4] == 'cart': cart = item break pure_cart = self.pure_cart_generator(data) url = 'https://www.supremenewyork.com/checkout' headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Encoding': 'gzip, deflate, sdch, br', 'Accept-Language': 'en-US,en;q=0.8', 'Connection': 'keep-alive', 'Cookie': 'pure_cart={0};cart={1};_supreme_sess={2};'.format( pure_cart, cart, sess), 'Host': 'www.supremenewyork.com', 'If-None-Match': 'W/"7cd6c2d3c1278e6ec2f8f895e92dc2dd"', 'Referer': 'https:/www.supremenewyork.com/checkout', 'Upgrade-Insecure-Requests': '1', 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87' } auth_r = requests.post(url=url, headers=headers, timeout=2) x_csrf_token = findall('<meta name="csrf-token" content="(.*==)" />', auth_r.text)[0] print(self.utc_to_est(), '-----> Done.') user = self.get_user_billing_info(self.current_user) form_data = { 'authenticity_token': x_csrf_token, 'order[billing_name]': user['name'], 'order[email]': user['email'], 'order[tel]': user['tel'], 'order[billing_address]': user['address'], 'order[billing_address_2]': user['address2'], 'order[billing_address_3]': user['address3'], 'order[billing_city]': user['city'], 'order[billing_zip]': user['postcode'], 'order[billing_country]': user['country'], 'same_as_billing_address': '1', 'store_credit_id': '0', 'credit_card[type]': user['card_type'], 'credit_card[cnb]': user['card_number'], 'credit_card[month]': user['card_month'], 'credit_card[year]': user['card_year'], 'credit_card[vval]': user['card_cvv'], 'order[terms]': '1', 'hpcvv': '' } print(self.utc_to_est(), '-----> Opening browser.') self.browser.get(url='https://www.supremenewyork.com/shop/cart') self.browser.delete_all_cookies() for key, value in requests.utils.dict_from_cookiejar( response.cookies).items(): self.browser.add_cookie({'name': key, 'value': value}) print(self.utc_to_est(), '-----> Refreshing browser.') self.browser.refresh() print(self.utc_to_est(), '-----> Checkout request.') self.browser.get(url='https://www.supremenewyork.com/checkout') response_ = self.browser.request( method='POST', url='https://www.supremenewyork.com/checkout', data=form_data) print(self.utc_to_est(), '-----> Done.') print(response_.content) def add_to_cart(self, data): print(self.utc_to_est(), '-----> Adding to cart.') response = None cookies = {} for element in data: product_id = element[0] color_id = element[1] size_id = element[2] form_data = { 'utf8': '%E2%9C%93', 'style': str(color_id), 'size': str(size_id), 'commit': 'add+to+basket' } headers = { 'Authotiry': 'www.supremenewyork.com', 'Method': 'POST', 'Path': '/shop/{}/add'.format(product_id), 'Scheme': 'https', 'accept': '*/*;q=0.5, text/javascript, application/javascript, application/ecmascript, ' 'application/x-ecmascript', 'accept-encoding': 'gzip, deflate, br', 'accept-language': 'en-US,en;q=0.9', 'content-length': '58', 'content-type': 'application/x-www-form-urlencoded; charset=UTF-8', 'origin': 'https://www.supremenewyork.com', 'referer': 'https://www.supremenewyork.com/shop/', 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/69.0.3497.100 Safari/537.36', 'x-requested-with': 'XMLHttpRequest', } response = requests.request( 'POST', 'https://www.supremenewyork.com/shop/{}/add.json'.format( product_id), data=form_data, headers=headers, cookies=cookies) cookies = response.cookies self.checkout(response, data) def choose_size_and_color(self, element, item_info): print(self.utc_to_est(), '-----> Choosing color and size.') # Приоритетные цвета, если цвет не задан или задан любой цвет priority_colors = ['Black', 'White', 'Green'] # Представленные в магазине цвета по данной вещи represented_colors = item_info['styles'] # Флаги цвета и размера color_is_not_set = 'Any' in element['color'] or element['color'] == '' size_is_not_set = 'Any' in element['size'] or element['size'] == '' # Начало выбора # Цвет не задан if color_is_not_set: # Итерация по представленным цветам for color in represented_colors: # Проверка на присутствие представленного в магазине цвета в приоритетных if color['name'] in priority_colors: # Если размер не выставлен if size_is_not_set: for size in color['sizes']: # Проверка только на доступность данного размера if size['stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (323).') return { 'color_id': color['id'], 'size_id': size['id'] } # Размер выставлен else: for size in color['sizes']: # Проверка на доступность данного размера и на присутствие его в списке пользователя if size['name'] in element['size'] and size[ 'stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (330).') return { 'color_id': color['id'], 'size_id': size['id'] } # Если не найдено в приоритетных - идем по всем else: for color in represented_colors: if color['name'] not in priority_colors: # Размер не выставлен if size_is_not_set: for size in color['sizes']: if size['stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (340).') return { 'color_id': color['id'], 'size_id': size['id'] } # Размер выставлен else: for size in color['sizes']: # Проверка на доступность данного размера и на присутствие его в списке пользователя if size['name'] in element['size'] and size[ 'stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (347).') return { 'color_id': color['id'], 'size_id': size['id'] } else: for color in represented_colors: for size in color['sizes']: if size['name'] in element['size'] and size[ 'stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (353).') return { 'color_id': color['id'], 'size_id': size['id'] } # Цвет задан else: # Итерация по представленным цветам for color in represented_colors: # Проверка на присутствие представленного в магазине цвета в пользовательских цветах if color['name'] in element['color']: # Если размер не выставлен if size_is_not_set: for size in color['sizes']: # Проверка только на доступность данного размера if size['stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (367).') return { 'color_id': color['id'], 'size_id': size['id'] } # Размер выставлен else: for size in color['sizes']: # Проверка на доступность данного размера и на присутствие его в списке пользователя if size['name'] in element['size'] and size[ 'stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (374).') return { 'color_id': color['id'], 'size_id': size['id'] } # Ни одного пользовательского цвета не найдено - идем по приоритетным else: # Итерация по приоритетным цветам for color in represented_colors: # Проверка на присутствие представленного в магазине цвета в приоритетных if color['name'] in priority_colors: # Если размер не выставлен if size_is_not_set: for size in color['sizes']: # Проверка только на доступность данного размера if size['stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (387).') return { 'color_id': color['id'], 'size_id': size['id'] } # Размер выставлен else: for size in color['sizes']: # Проверка на доступность данного размера и на присутствие его в списке пользователя if size['name'] in element['size'] and size[ 'stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (394).') return { 'color_id': color['id'], 'size_id': size['id'] } # Если не найдено в приоритетных - идем по всем else: for color in represented_colors: if color['name'] not in priority_colors: if size_is_not_set: for size in color['sizes']: if size['stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (403).') return { 'color_id': color['id'], 'size_id': size['id'] } else: for color in represented_colors: for size in color['sizes']: if size['name'] in element['size'] and size[ 'stock_level'] is not 0: print(self.utc_to_est(), '-----> Found (409).') return { 'color_id': color['id'], 'size_id': size['id'] } # Конец выбора # Если предмет не найден print(self.utc_to_est(), '-----> Not found.') return {'color_id': None, 'size_id': None} def find_items(self): print(self.utc_to_est(), '-----> Started processing.') my_items = self.get_buy_list() requests_data = [] for element in my_items: # items_to_buy item_found = False # flag # split desired product name into pieces elem_split = element['name'].split(' ') while not item_found: stock = self.fetch_stock() # supreme stock # replace 'Tops/Sweaters' with 'Tops/Sweaters' # if element['type'] == 'tops-sweaters': # element['type'] = 'Tops/Sweaters' # # elif element['type'] == 't-shirts': # element['type'] = 'Shirts' # # else: # element['type'] = element['type'].title() # iterating through represented items in market for item in stock['products_and_categories'][element['type']]: # searching for most appropriate item name ''' comparing intersection length with desired product name split length minus one e.g.: user wants to buy 'Supreme/The Killer Trust Tee' but in real its name sounds like 'The Killer Trust Tee' the way is to split desired item name: ['Supreme/The', 'Killer', 'Trust', 'Tee'] and to split item name represented in shop: ['The', 'Killer', 'Trust', 'Tee'] By finding the longest intersection of 2 sets, we can be sure we found desired item. The longest intersection usually has length of desired product name split length minus one ''' if len(list( set(elem_split) & set(item['name'].split(' ')))) >= ( len(elem_split) - 1): # item found print(self.utc_to_est(), '-----> {} found.'.format(element['name'])) item_found = True # collecting info price = round(item['price_euro'] / 100) item_info = self.get_item_info(item['id']) color_and_size = self.choose_size_and_color( element, item_info) # if size and color info successfully found, add data to request data if color_and_size['color_id'] is None or color_and_size[ 'size_id'] is None: print(self.utc_to_est(), '-----> Desired item has been sold out.') else: requests_data.append([ item['id'], color_and_size['color_id'], color_and_size['size_id'], price ]) # when all desired items data collected, add them to cart if len(requests_data) == len(my_items): self.add_to_cart(requests_data) @staticmethod def utc_to_est(): from datetime import datetime return datetime.now()
class UmsConnection(): def __init__(self, user_phone, user_password, cookies=None): self._user_phone = user_phone self._user_password = user_password self._is_authorized = False self._captcha = None # REMOVE options = Options() options.add_argument("--headless") WINDOW_SIZE = "1620,880" # ? options.add_argument("--window-size=%s" % WINDOW_SIZE) self.webdriver = Chrome(chrome_options=options) if cookies: self.webdriver.get('https://messages.megafon.ru/onebox/mix.do') for cookie in cookies: self.webdriver.add_cookie(cookie) self.webdriver.get('https://messages.megafon.ru/onebox/mix.do') self._is_authorized = self.try_is_authorized() if not self.is_authorized: self.webdriver.get(r"https://messages.megafon.ru/onebox/mix.do") phone_input = self.webdriver.find_element_by_id('user12') phone_input.send_keys(user_phone) password_input_secinfo = self.webdriver.find_element_by_id('secinfo') password_input_secinfo.click() password_input = self.webdriver.find_element_by_id("secinfo2") password_input.send_keys(user_password) def get_captcha(self) -> bytes: captcha = self.webdriver.find_element_by_id('imageC') location = captcha.location size = captcha.size screenshot = self.webdriver.get_screenshot_as_png() im = Image.open(BytesIO(screenshot)) left = location['x'] top = location['y'] right = location['x'] + size['width'] bottom = location['y'] + size['height'] im = im.crop((left, top, right, bottom)) self._captcha = im # REMOVE stream = BytesIO() im.save(stream, format="PNG") return stream.getvalue() def send_captcha_key(self, key) -> bool: captcha_input = self.webdriver.find_element_by_id('imageObj') captcha_input.send_keys(key) submit_button = self.webdriver.find_element_by_id('index_login') submit_button.click() try: WebDriverWait(self.webdriver, 2).until( EC.url_contains(("messages.megafon.ru/onebox/mix.do"))) self._is_authorized = True self._captcha.save(f'captchas/{key}.png') # REMOVE return True except exceptions.TimeoutException: return False def try_is_authorized(self) -> bool: # TODO url = r"https://messages.megafon.ru/onebox/getChatList.do" get_chats_url_params = { 'startNum': '1', 'endNum': 10, 'reFreshFlag': '1', 'operation': '1', 'chatMsgType': '10100000000110000000000000000000', 't': random.random() } response = self.webdriver.request('GET', url, params=get_chats_url_params) return response.status_code == 200 @property def is_authorized(self): return self._is_authorized def get_chat_list(self, number=10): url = r"https://messages.megafon.ru/onebox/getChatList.do" get_chats_url_params = { 'startNum': '1', 'endNum': number, 'reFreshFlag': '1', 'operation': '1', 'chatMsgType': '10100000000110000000000000000000', 't': random.random() } response = self.webdriver.request('GET', url, params=get_chats_url_params) return response.text def get_one_box(self, address, number=10, from_num=1): url = r"https://messages.megafon.ru/onebox/oneboxList.do" params = [ ('umReq.ctlg', '1,2'), ('umReq.numFlg', '1'), ('umReq.mType', '6149'), ('umReq.srt', '0'), ('umReq.lType', '0'), ('umReq.dFlg', '0'), ('umReq.srtDr', '0'), ('umReq.rdFlg', '0'), ('umReq.bNum', from_num), ('umReq.eNum', number), ('umReq.snd', address), ('umReq.rcv', self._user_phone), ('umReq.bTime', ''), ('umReq.eTime', ''), ('umReq.impt', '-1'), ('umReq.t', ''), ('umReq.threadFlag', '1'), ('rownid', random.random()), ] response = self.webdriver.request('GET', url, params=params) return response.text def close(self): self.webdriver.close() def get_cookies_json(self): cookies = self.webdriver.get_cookies() return json.dumps(cookies)