def add_list_to_db(item_list):
db = get_db()
try:
db.products.insert(item_list)
util.loginfo('ad(s) inserted successfully')
except Exception as e:
util.logerr(str(e))
def add_ad_to_db(item):
db = get_db()
try:
db.products.insert_one(item)
util.loginfo('ad inserted successfully')
except Exception as e:
util.logerr(str(e))
def deactivate(item):
db = get_db()
try:
db.products.update({'source': item.get('source'),"ad_id": item.get('ad_id')}, {'$set': {'active': False}})
util.loginfo('ad is deactivated successful')
except Exception as e:
util.logerr(str(e))
def delete(item):
db = get_db()
try:
db.products.delete_many({'source': item.get('source'),"ad_id": item.get('ad_id')})
util.loginfo('ad deleted successful')
except Exception as e:
util.logerr(str(e))
def get_db(env=settings['ENVIRONMENT']):
db = None
try:
client = MongoClient(settings[env]['Mongo']['DbClient'])
db = client.get_database(settings[env]['Mongo']['DbName'])
except Exception as e:
util.logerr('Error connecting to database or not reachable')
return db
def get_models():
db = get_db()
try:
models = list(db.models.find({'active': True}))
util.loginfo('models retrived successfully')
except Exception as e:
util.logerr(str(e))
return models
def get_brand(brand_id):
db = get_db()
try:
if db.brands.find({'_id':ObjectId(brand_id),'active': True}).count() != 0:
return db.brands.find({'_id': ObjectId(brand_id), 'active': True})[0]
except Exception as e:
util.logerr(str(e))
return None
def get_brands():
db = get_db()
try:
brands = list(db.brands.find({'active': True}))
util.loginfo('brands retrived successfully')
except Exception as e:
util.logerr(str(e))
return brands
def get_ad_by_query(q):
db = get_db()
try:
ads = list(db.products.find(q))
util.loginfo('ads retrived successfully')
except Exception as e:
util.logerr(str(e))
return ads
def get_ad():
db = get_db()
try:
ad = db.products.find_one()
util.loginfo('ad retrived successfully')
except Exception as e:
util.logerr(str(e))
return ad
def prepare_ad_to_query(ad, fields_to_project):
try:
ad_ = []
temp = []
for key in ad[0]:
if key in fields_to_project:
temp.append(hash(ad[0][key]))
ad_.append(temp)
ad_ = np.array(ad_)
# # lebeling rest of columns
# for i in range (ad_.shape[1]):
# ad_[:, i] = le.transform(ad_[:,i])
return ad_
except (Exception, e):
util.logerr(str(e))
return
def update(items):
db = get_db()
for item in items:
try:
db.products.update(
{
'source': item.get('source'),
"ad_id": item.get('ad_id')
},
{
"$set": {
'language_override': item.get('language_override'),
'ad_page_link': item.get('ad_page_link'),
'last_update': item.get('last_update'),
'title': item.get('title'),
'description': item.get('description'),
'brand': item.get('brand'),
'model': item.get('model'),
'year': item.get('year'),
'body_type': item.get('body_type'),
'mileage': item.get('mileage'),
'engine': item.get('engine'),
'power': item.get('power'),
'specs': item.get('specs'),
'transmission': item.get('transmission'),
'fuel': item.get('fuel'),
'condition': item.get('condition'),
'color': item.get('color'),
'price': item.get('price'),
'image_link': item.get('image_link'),
'location': item.get('location'),
'active': item.get('active'),
'keyFeatures': item.get('keyFeatures'),
'features': item.get('features'),
'tags':item.get('tags'),
'variants': item.get('variants'),
'ad_cat': item.get('ad_cat')
}
}
)
util.loginfo("ad(s) updated successfully")
except Exception as e:
util.logerr(str(e))
def get_ads_dataset(filters, fields_to_project, target_field):
db = dao.get_db()
try:
# {
# "brand": ObjectId("5916d82fff4a4215c6cb2ac7"),
# "model": ""
# "year": "٢٠١٥",
# "body_type": "",
# "engine": 0,
# "power": 0,
# "fuel": "",
# "transmission": "اوتوماتيك",
# "location": "جدة",
# "specs": ""
# }
# {"body_type": {"$ne": ""}, "$and": [{"engine": {"$ne": "0"}}, {"engine": {"$ne": 0}}],
# "power": {"$ne": 0}, "fuel": {"$ne": ""}, "specs": {"$ne": ""}},
# {"brand": 1, "model": 1, "year": 1, "body_type": 1, "engine": 1, "power": 1, "fuel": 1,
# "transmission": 1, "location": 1, "specs": 1, "_id": 0}):
ads = []
for ad in db.products.find(
filters, {
"brand": 1,
"model": 1,
"year": 1,
"body_type": 1,
"engine": 1,
"power": 1,
"fuel": 1,
"transmission": 1,
"location": 1,
"specs": 1,
"_id": 0
}):
temp = []
target_field_index = 0
counter = -1
for key in ad:
if key in fields_to_project:
if key != target_field:
temp.append(hash(ad[key]))
else:
temp.append(ad[key])
counter += 1
if key == target_field:
target_field_index = counter
ads.append(temp)
ads = np.array(ads)
# swapping columns so that y i.e target is col 0
ads[:,
0], ads[:,
target_field_index] = ads[:,
target_field_index], ads[:,
0].copy(
)
# # cleaning engine capcity less than 1000 cc
# ads[ads[:, 0] > 900, :]
# # lebeling rest of columns
# le = preprocessing.LabelEncoder()
# for i in range (1,ads.shape[1]):
# le.fit(ads[:,i])
# ads[:, i] = le.fit_transform(ads[:,i])
# Calculate mode of each column except target key
# max(set(list), key=list.count)
return ads
except (Exception, e):
util.logerr(str(e))
return