def sort_col_docs():
collections_names = ['ended', 'upcoming', 'ongoing']
for col_name in collections_names:
mongo = pymongodb.MongoDB('icobazaar')
sorted_docs = mongo.find_with_sort(col_name, 'ico_star_rating')
mongo.drop_collection(col_name)
mongo.insert_many(sorted_docs, col_name)
def check_on_clear_db():
"""
Method which check database on clear by test find document in collection.
:return:
"""
mongo = pymongodb.MongoDB('btt')
document = mongo.find({}, 'en_altcoins', 1)
return True if document else False
def write_data(*args, **kwargs):
"""
Method which insert data in specific collection.
:param kwargs: dict/s of data.
:return:
"""
mongo = pymongodb.MongoDB('btt')
mongo.insert_one(kwargs, 'en_altcoins')
return args[0]
def __init__(self):
self.url = 'https://bitjournal.media/wp-admin/admin-ajax.php'
self.mongo = pymongodb.MongoDB('bitjournal')
self.last_new = None
self.next = False
self.today = datetime.today().strftime('%d-%m-%Y').zfill(2)
self.yesterday = (datetime.today() -
timedelta(1)).strftime('%d-%m-%Y').zfill(2)
self.before_yesterday = (datetime.today() -
timedelta(2)).strftime('%d-%m-%Y').zfill(2)
def __init__(self):
self.ICO_LIST_URL = 'https://icobazaar.com/v2/ico-list'
self.ICO_LIST_PAGINATION_URL = 'https://icobazaar.com/v2/ico-list?cats%5B%5D={0}&page={1}'
self.UPCOMING_URL = 'https://icobazaar.com//v2/ico-list?status%5B0%5D=upcoming&page={}'
self.ONGOING_URL = 'https://icobazaar.com//v2/ico-list?status%5B0%5D=ongoing&page={}'
self.ENDED_URL = 'https://icobazaar.com//v2/ico-list?status%5B0%5D=ended&page={}'
self.SPECIFIC_URLS_LIST = [
self.UPCOMING_URL, self.ONGOING_URL, self.ENDED_URL
]
self.mongo = pymongodb.MongoDB('icobazaar')
def write_data(self, **kwargs):
"""
Method which insert data in specific collection.
:param kwargs: dict of data.
:return:
"""
self.lock.acquire()
try:
mongo = pymongodb.MongoDB('cmc')
mongo.insert_one(kwargs, 'currencies')
finally:
self.lock.release()
def __init__(self):
self.mongo = pymongodb.MongoDB('btt')
self.documents = self.mongo.find({}, 'en_altcoins')
self.form_url = ''
self.client = requests.sessions.Session()
def find_and_write_data(bs_obj, cat_title, count=None):
"""
Method which find specific data in page and write it into db.
:param bs_obj:
:param cat_title:
:param count:
:return:
"""
# Get ico full description links.
ico_full_desc_links = bs_obj.findAll('a', {'class': 'ico-link'})
ico_full_desc_links = [
ico_link['href'] for ico_link in ico_full_desc_links
]
# Check on clear list.
if not ico_full_desc_links:
print('page #{} not exist'.format(count))
return False
# Get ico logos.
imgs_src = bs_obj.findAll('div', {'class': 'ico-image'})
imgs_src = [img_url.find('img')['src'] for img_url in imgs_src]
# Get ico names.
ico_names = bs_obj.findAll('h5')
ico_names = [name.text for name in ico_names]
# Get updated date.
updated_dates = bs_obj.findAll('div',
{'class': 'campaign_update_widget'})
updated_dates = [
updated_date.find('span').text for updated_date in updated_dates
]
# Get ico short description texts.
ico_texts = bs_obj.findAll('div', {'class': 'ico-text'})
ico_texts = [ico_text.text for ico_text in ico_texts]
# Get ico statuses.
ico_statuses = bs_obj.findAll('div', {'class': 'ico-condition'})
ico_statuses = [
ico_status.find('div').text for ico_status in ico_statuses
]
# Get ico dates.
ico_dates = bs_obj.findAll('div', {'class': 'ico-date'})
ico_dates = [ico_date.text for ico_date in ico_dates]
# Get ico text ratings.
ico_text_ratings = bs_obj.findAll('div', {'class': 'ico-eva_class'})
ico_text_ratings = [
ico_text_rating.text for ico_text_rating in ico_text_ratings
]
# Get ico stars ratings.
ico_stars_ratings = bs_obj.findAll(
'i', {'class': re.compile('[star] \w{0,4}')})
ico_stars_ratings = [
ico_stars_rating['class'] for ico_stars_rating in ico_stars_ratings
]
ico_stars_ratings = utils.run_rate_transform(ico_stars_ratings)
# Clear category title. Change symbols (whitespaces and &) in collections names to _.
title2 = utils.clear_title(cat_title)
# Insert data into db.
mongo = pymongodb.MongoDB('icobazaar')
for i in range(0, len(ico_full_desc_links)):
mongo.insert_one(
{
'ico_full_desc_link': ico_full_desc_links[i],
'img_src': imgs_src[i],
'ico_name': ico_names[i],
'updated_date': updated_dates[i],
'ico_text': ico_texts[i],
'ico_status': ico_statuses[i],
'ico_date': ico_dates[i],
'ico_text_rating': ico_text_ratings[i],
'ico_star_rating': ico_stars_ratings[i]
}, title2)
mongo.finish()
def drop_db():
mongo = pymongodb.MongoDB('cmc')
mongo.drop_database()
def __init__(self):
self.url = 'https://bits.media/news/nav_feed/page-{}/'
self.mongo = pymongodb.MongoDB('bitsmedia')
self.last_new = None
self.next = False