class WalletService: def __init__(self): print('Wallet service initialized') collection_name = 'wallet' self.db = MongoHelper(collection_name) def create_wallet(self, name): ''' data: name ''' data = { 'name': name, 'amount': 10000 # initial amount } wallet_id = self.db.insert(data) return wallet_id def deposit_in_wallet(self, wallet_id, amount): wallet_data = self.db.increment_by_id(wallet_id, {'amount': amount}) return wallet_data def withdraw_from_wallet(self, wallet_id, amount): amount = int(amount) amount = -amount return self.deposit_in_wallet(wallet_id, amount)
class OtpService: def __init__(self): print('OTP service initialized') collection_name = 'otp' self.db = MongoHelper(collection_name) def generate_code(self): n = 4 code = ''.join(["%s" % randint(0, 9) for num in range(0, n)]) return code def create_otp(self, phone): otp = self.generate_code() self.db.insert({'otp': otp, 'used': False, 'phone': phone}) return otp def validate_otp(self, code): otp_data = self.db.find_one({'otp': code, 'used': False}) print('Otp data', otp_data, 'code', code) if otp_data: self.db.update_by_id(otp_data['_id'], {'used': True}) return True, otp_data['phone'] else: return False, None
class TransactionsService: def __init__(self): print('Transactions service initialized') collection_name = 'transactions' self.db = MongoHelper(collection_name) def create_transaction(self, data): ''' data: agent_phone, wallet_id, amount, type ''' return self.db.insert(data) def get_transactions(self): return self.db.find_all()
class AgentsService: def __init__(self): print('Agents service initialized') collection_name = 'agents' self.db = MongoHelper(collection_name) def create_agent(self, data): ''' data: name, pin, phone ''' hashed = bcrypt.hashpw(data['pin'].encode(), bcrypt.gensalt()) data['pin'] = hashed.decode() self.db.insert(data) def validate_agent(self, phone, pin): agent = self.db.find_one({'phone': phone}) return bcrypt.checkpw(pin.encode(), agent['pin'].encode())
def check_active_listings(search_term): flag_old_30_days_old_listings(search_term) mongo_helper = MongoHelper() mongo_helper.start_client_and_connect() # we are taking only active listings, which are not marked as old and or multiitem active_listings = mongo_helper.get_active_singleitem_and_not_flaged_as_old_listings( search_term) print("There are {0} active listings in db".format( active_listings.count())) driver_helper = DriverHelper() driver_helper.start_driver() #TODO: replace threshold with retry after testing number_of_failed_tries_to_scrape_listing = 0 error_threshold = 20 for listing in active_listings.batch_size(10): try: retry(update_active_listing, listing) except Exception as e: trace = traceback.format_exc() print("Failed to scrape the listing {0}".format(listing["link"])) driver_helper.take_screenshot() print(e) print(trace) number_of_failed_tries_to_scrape_listing += 1 if (number_of_failed_tries_to_scrape_listing > error_threshold): raise Exception( "More than {0} scraping failures for the search term {1}". format(error_threshold, search_term)) continue driver_helper.quit_driver() mongo_helper.close_connection()
def flag_old_30_days_old_listings(search_term): mongo_helper = MongoHelper() mongo_helper.start_client_and_connect() active_listings = mongo_helper.get_active_and_not_flaged_as_old_listings( search_term) for listing in active_listings.batch_size(10): if ("last_update" in listing): creation_date = datetime.datetime.strptime( re.sub(r"\d{2}\:\d{2}\s", "", listing["creation_date"]), '%d. %b. %Y') last_update = datetime.datetime.strptime(listing["last_update"], '%d. %b. %Y') age = (last_update - creation_date).days else: continue if (age > 30): mongo_helper.update_listing(listing["_id"], "old", True) mongo_helper.close_connection()
def scrape_new_listings(search_term): # open initial search page for search_term with predefined search criterias. See search criterias in SearchPage driver_helper = DriverHelper() driver_helper.start_driver() mongo_helper = MongoHelper() mongo_helper.start_client_and_connect() print("Searching new listings for '{0}' keyword".format(search_term)) search_page = SearchPage.open_it_via_url(search_term) there_are_new_listings = True there_are_more_pages = True while (there_are_new_listings & there_are_more_pages): there_are_new_listings = scrape_search_results_from_the_currenst_search_page( search_term) there_are_more_pages = search_page.go_to_the_next_page_if_available() print( "There are new pages available: {0}\nThere are new listings: {1}". format(there_are_more_pages, there_are_new_listings)) driver_helper.quit_driver() mongo_helper.close_connection()
def __init__(self): print('Wallet service initialized') collection_name = 'wallet' self.db = MongoHelper(collection_name)
def scrape_search_results_from_the_currenst_search_page(search_term): search_page = SearchPage() driver_helper = DriverHelper() mongo_helper = MongoHelper() advert_links_and_dates = search_page.get_listing_links_and_dates() there_are_new_listings = True if len(advert_links_and_dates) > 0 else False # Could be that scraping of the listing page will fail because of some edge cases and network issues # I'd like to have info about the fils in the logs, but still continue scraping if the amount of failures is small # TODO: replace threshold with retry after testing number_of_failed_tries_to_scrape_listing = 0 error_per_serch_page_threshold = 2 # total number of listings on the search page is 50 # Sometimes after navigation to the next page we see the last listing form the previos page on it # Adding threshold to avoid false conclusion that there are only already saved listings on the page number_of_existing_listings_on_the_page = 0 existing_listings_on_the_page_threshold = 3 for link_and_date in advert_links_and_dates: print("Listing date: {}".format(link_and_date["date"])) #next line was used for initial run #if (("Mai" in link_and_date["date"]) or ("Apr" in link_and_date["date"])): #return False link = link_and_date["link"] date = "{0} {1}".format(link_and_date["date"], datetime.datetime.now().year) try: print("Scraping listing {0}".format(link)) listing_page = ListingPage.open_url_in_the_new_tab(link) listing = listing_page.get_listing_data() except Exception as e: trace = traceback.format_exc() print("Failed to scrape the listing {0}".format(link)) driver_helper.take_screenshot() print(e) print(trace) number_of_failed_tries_to_scrape_listing += 1 if (number_of_failed_tries_to_scrape_listing > error_per_serch_page_threshold): raise Exception( "More than {0} scraping failures for the search result page" .format(error_per_serch_page_threshold)) driver_helper.close_current_tab_and_go_to_the_first_one() continue listing.creation_date = date listing.search_term = search_term driver_helper.close_current_tab_and_go_to_the_first_one() if ((len(mongo_helper.get_listing_by_id(listing._id)) > 0) & (number_of_existing_listings_on_the_page > existing_listings_on_the_page_threshold)): print("This listing is already in the db") print( "Number of the listings that were already saved in the db on the current search page is {0}" .format(number_of_existing_listings_on_the_page)) there_are_new_listings = False break elif (mongo_helper.get_listing_by_id(listing._id)): print("This listing is already in the db: {0}".format(link)) if (mongo_helper.get_listing_by_id( listing._id)[0]["search_term"] == search_term): number_of_existing_listings_on_the_page += 1 else: print( "This listing was saved in database for the different search term" ) continue mongo_helper.insert_listing(listing) return there_are_new_listings
def update_active_listing(listing): print("Checking listing {0}".format(listing["link"])) driver_helper = DriverHelper() mongo_helper = MongoHelper() try: listing_page = ListingPage.open_url(listing["link"]) except: ListingNotFoundPage() mongo_helper.update_listing(listing["_id"], "page_not_found", True) return today = datetime.datetime.now().strftime('%d. %b. %Y') if listing_page.is_active(): print("The listing {0} page is ACTIVE".format(listing["link"])) print("Updating listing {0}".format(listing["_id"])) mongo_helper.update_listing(listing["_id"], "last_update", today) else: print("The listing {0} page is NOT ACTIVE".format(listing["link"])) closure_date = None try: closure_date = listing_page.get_closure_date() except: # it's fine if the product is just out of stock print( "there is no closure data for this closed listing. Checking if it's just out of stock" ) reason = listing_page.get_closure_reason() print(reason.encode('utf-8')) if (reason != 'Dieser Artikel ist nicht vorrätig.'): raise Exception( "Closure reason is ont 'out of stock' and closur date not provided" ) listing_page.open_original_listing_if_the_link_is_available() closure_reason = listing_page.get_closure_reason() final_price = listing_page.get_price() fianl_shipping_cost = listing_page.get_shipping_cost() print("Updating listing {0}".format(listing["_id"])) mongo_helper.update_listing(listing["_id"], "last_update", today) mongo_helper.update_listing(listing["_id"], "final_price", final_price) mongo_helper.update_listing(listing["_id"], "fianl_shipping_cost", fianl_shipping_cost) mongo_helper.update_listing(listing["_id"], "closure_reason", closure_reason) mongo_helper.update_listing(listing["_id"], "closure_date", closure_date) mongo_helper.update_listing(listing["_id"], "active", False)
def __init__(self): print('OTP service initialized') collection_name = 'otp' self.db = MongoHelper(collection_name)
def __init__(self): print('Transactions service initialized') collection_name = 'transactions' self.db = MongoHelper(collection_name)
def __init__(self): print('Agents service initialized') collection_name = 'agents' self.db = MongoHelper(collection_name)