def update_user_email_is_validated_status(coll: pymongo.collection.Collection,
user_id: ObjectId,
yes=False,
no=False,
token=""):
assert isinstance(user_id, ObjectId)
if token:
res = coll.update_one({"_id": user_id}, {
"$set": {
"email_validated": "pending",
"email_validation_token": token,
}
})
elif yes:
coll.update_one({"_id": user_id}, {
"$set": {
"email_validated": "yes",
},
"$unset": {
"email_validation_token": "",
},
})
elif no:
coll.update_one({"_id": user_id}, {
"$set": {
"email_validated": "no",
},
"$unset": {
"email_validation_token": "",
},
})
def update_single_offer(self,
collection: pymongo.collection.Collection,
driver: webdriver.Chrome,
offer: dict,
old_db_corresponding_record=None):
"""Parses the info for a single offer
Args:
collection (pymongo.collection.Collection): MongoDB collection to run the update against
driver (webdriver.Chrome): Selenium webdriver to get offer details
offer (dict): A single offer from MongoDB
old_db_corresponding_record (Dict, optional): Corresponding record in MongoDB, if exists. Defaults to None.
"""
doc = {
"id": offer['id'],
"date": offer['published'],
"title": offer['title'],
"position": offer['title'].split('@')[0],
"author": offer['author'],
"link": offer['link'],
}
offer_soup = BeautifulSoup(offer.summary, 'html.parser')
text = offer_soup.text.split('\n')
address = text[5].split('Location: ', 1)[1]
city = address.split(',')[-1].strip()
address = ','.join(address.split(',')[:-1])
salaries_raw = text[4].split('Salary: ')[1]
salaries = [
salaries_raw
] if len(re.findall("\(.{1,17},.{1,17}\)",
salaries_raw)) > 0 else salaries_raw.split(',')
salary_ranges = []
for salary in salaries:
salary_ranges.append(
self.get_salary_details(salary.replace(" ", ""), salaries_raw))
if city in self.cities_translations:
city = self.cities_translations[city]
resp = driver.get(offer['id'])
page_text = driver.find_element_by_id('root')
doc["address"] = address
doc["city"] = city
doc["salary"] = salary_ranges
doc["raw_salary"] = salaries_raw
doc["full_description"] = page_text.text
offer["full_description"] = page_text.text
try:
description_details = self.parse_single_description(offer)
doc.update(description_details)
except IndexError as e:
print(f'Error parsing description for offer {offer["id"]} : {e}')
if city not in self.cities:
self.cities.append(city)
if old_db_corresponding_record:
collection.update_one({"id": offer['id']}, {"$set": doc})
else:
collection.insert_one(doc)
def update_api(
apis_collection: pymongo.collection.Collection,
api_provider: str,
number_of_calls: int,
):
apis_collection.update_one({"provider": api_provider},
{"$set": {
"number_of_calls": number_of_calls
}})
def delete_user_address(coll: pymongo.collection.Collection, user_id: ObjectId,
address_type: str):
assert isinstance(user_id, ObjectId)
coll.update_one({"_id": user_id},
{"$pull": {
"addresses": {
"type": address_type
}
}})
def delete_user_document(coll: pymongo.collection.Collection,
user_id: ObjectId, document_id: ObjectId):
assert isinstance(user_id, ObjectId)
assert isinstance(document_id, ObjectId)
coll.update_one({"_id": user_id},
{"$pull": {
"documents": {
"_id": document_id
}
}})
def add_user_contract_scan(coll: pymongo.collection.Collection,
user_id: ObjectId, contract_id: ObjectId, file_id):
assert isinstance(user_id, ObjectId)
assert isinstance(contract_id, ObjectId)
coll.update_one({
"_id": user_id,
"contracts._id": contract_id
}, {"$set": {
"contracts.$.scan_file": file_id,
}})
def invalidate_user_contract(coll: pymongo.collection.Collection,
user_id: ObjectId, contract_id: ObjectId,
invalidation_date: datetime):
assert isinstance(user_id, ObjectId)
assert isinstance(contract_id, ObjectId)
coll.update_one({
"_id": user_id,
"contracts._id": contract_id
}, {"$set": {
"contracts.$.invalidation_date": invalidation_date,
}})
def invalidate_user_document(coll: pymongo.collection.Collection,
user_id: ObjectId, document_id: ObjectId,
invalidation_date: datetime):
assert isinstance(user_id, ObjectId)
assert isinstance(document_id, ObjectId)
coll.update_one({
"_id": user_id,
"documents._id": document_id
}, {"$set": {
"documents.$.invalidation_date": invalidation_date
}})
def fix_all_descriptions(self, collection: pymongo.collection.Collection):
"""One time function to fix all descriptions
Args:
collection (pymongo.collection.Collection): MongoDB collection to be updated
"""
offers = collection.find()
for offer in tqdm.tqdm(offers, desc="Fixing the descriptions"):
try:
description_details = self.parse_single_description(offer)
collection.update_one({"id": offer['id']},
{"$set": description_details})
except IndexError:
print("ERROR")
def update_dataset(old_dataset: Dict, h5_dataset: H5Dataset, folder: str,
collection: pymongo.collection.Collection):
try:
rm_rf(folder)
h5_dataset.convert(folder)
collection.update_one({'_id': old_dataset['_id']},
{'$set': h5_dataset.dataset})
logging.info(f"Successfully updated dataset {h5_dataset.token}")
except pymongo.errors.DuplicateKeyError as e:
rm_rf(folder)
logging.error(f"Duplicated token in dataset {h5_dataset.token}")
except Exception as e:
rm_rf(folder)
logging.error(
f"Failed to update dataset {h5_dataset.token} due to: {e}")
def add_embedded_mdoc_to_mdoc_array(coll: pymongo.collection.Collection,
user_id: ObjectId,
array_field: str,
document: dict,
document_id: ObjectId = None,
filter_values=(None, "")):
if filter_values:
document = {
key: value
for key, value in document.items() if value not in filter_values
}
if document_id:
document["_id"] = document_id
coll.update_one({"_id": user_id}, {"$addToSet": {array_field: document}})
def _update_user_client_metric(user_collection: pymongo.collection.Collection,
user: dict[str, Any], dry_run: bool) -> None:
user_id = user['_id']
client_metrics = proto.create_from_mongo(user.get('clientMetrics'),
user_pb2.ClientSideMetrics)
assert client_metrics
updated_metrics = user_pb2.ClientSideMetrics()
if client_metrics.amplitude_id:
amplitude_id = client_metrics.amplitude_id
else:
try:
amplitude_id = _get_amplitude_id(user_id)
except KeyError:
logging.info('Could not find user "%s" on Amplitude.', user_id)
amplitude_id = _AMPLITUDE_ID_NOT_FOUND
updated_metrics.amplitude_id = amplitude_id
if amplitude_id != _AMPLITUDE_ID_NOT_FOUND:
if not client_metrics.first_session_duration_seconds:
events = _get_amplitude_events(amplitude_id)
first_session_duration = compute_first_session_duration(events)
updated_metrics.first_session_duration_seconds = \
round(first_session_duration.total_seconds())
else:
events = []
if not client_metrics.is_first_session_mobile:
if not events:
events = _get_amplitude_events(amplitude_id, limit=5)
updated_metrics.is_first_session_mobile = _compute_is_mobile(
events[:5])
dict_update = json_format.MessageToDict(updated_metrics)
if not dict_update:
logging.info('Nothing to update for user "%s"', user_id)
return
if dry_run:
logging.info('Update user "%s":\n%s', user_id, dict_update)
else:
user_collection.update_one(user, {
'$set': {f'clientMetrics.{k}': v
for k, v in dict_update.items()}
})
def write_to_fomo(id: int, id_cursor: pymongo.collection.Collection, source: str):
# check if it exists first
record = exists_on_fomo(id, id_cursor)
if record:
if source == 'file':
deleted_ids.append(id)
id_cursor.delete_one({'id': id})
if source == 'api':
return id_cursor.update_one({'id': id}, {'$set': {'updated': datetime.datetime.utcnow()}})
else:
id_cursor.insert_one({"id": id, "updated": datetime.datetime.utcnow()})
def update_user(coll: pymongo.collection.Collection,
user_id: ObjectId,
data: dict,
embedded_1to1_docs=("name", )):
assert isinstance(user_id, ObjectId)
to_unset = {key: "" for key, value in data.items() if value == ""}
for key in to_unset:
del data[key]
operation_dict = {}
if data:
operation_dict["$set"] = data
if to_unset:
operation_dict["$unset"] = to_unset
updated = coll.find_one_and_update({"_id": user_id},
operation_dict,
return_document=ReturnDocument.AFTER)
for key in embedded_1to1_docs:
if key in updated and not updated[key]:
coll.update_one({"_id": user_id}, {"$unset": {"name": ""}})
def auto_time_update(col: pymongo.collection.Collection,
filter_dict: dict, update_dict: dict):
dt0 = datetime.datetime.utcnow()
update_dict['updated_at'] = dt0
update_dict = {'$set': update_dict, '$setOnInsert': {'created_at': dt0}}
return col.update_one(filter_dict, update_dict, upsert=True)
def user_unset_password_change_token(coll: pymongo.collection.Collection,
user_id):
coll.update_one({"_id": user_id},
{"$unset": {
"password_change_token": "",
}})
def user_set_password_change_token(coll: pymongo.collection.Collection,
user_id, token):
coll.update_one({"_id": user_id},
{"$set": {
"password_change_token": token,
}})
def reset_api(apis_collection: pymongo.collection.Collection,
api_provider: str):
apis_collection.update_one({"provider": api_provider},
{"$set": {
"number_of_calls": 0
}})
def update_meme(memes_collection: pymongo.collection.Collection, meme_id,
attribute: str, value):
memes_collection.update_one({"_id": meme_id}, {"$set": {attribute: value}})
def push(self, db_collection: pymongo.collection.Collection):
db_collection.update_one({"symbol": self._symbol}, self._document)
return self
def convert_currencies(self, rates: RatesConverter,
collection: pymongo.collection.Collection):
"""Maps the wages to PLN currencies for all offers in DB - we run it on all offers to adjust for currency fluctuation
Args:
rates (RatesConverter): RatesConverter instance to convert
collection (pymongo.collection.Collection): MongoDB collection to be updated
"""
def map_single_salary_field(salary: Dict, salary_field: str,
currency_rate: float):
"""Helper method to convert a single field - adds a new field with _pln ending
Args:
salary (Dict): nested document with salary info
salary_field (str): name of the salary field
currency_rate (float): Currency rate from target currency to PLN
Returns:
Dict: Updated salary chunk
"""
if salary[salary_field] != 'undisclosed':
salary[salary_field] = int(salary[salary_field])
salary[f'{salary_field}_pln'] = int(
int(salary[salary_field]) * currency_rate)
else:
salary[f'{salary_field}_pln'] = 'undisclosed'
return salary
def map_single_salary(salary: Dict, currency_rate: float):
"""Maps all fields in a single salary field
Args:
salary (Dict): nested document with salary info
currency_rate (float): Currency rate from target currency to PLN
Returns:
Dict: Updated salary chunk
"""
salary = map_single_salary_field(salary, 'upper_range',
currency_rate)
salary = map_single_salary_field(salary, 'lower_range',
currency_rate)
if salary['upper_range'] != 'undisclosed':
salary['average'] = (salary['upper_range'] +
salary['lower_range']) / 2
else:
salary['average'] = 'undisclosed'
salary = map_single_salary_field(salary, 'average', currency_rate)
return salary
offers = collection.find()
for offer in tqdm.tqdm(offers, desc="Mapping the currencies to PLN"):
try:
new_salaries = []
for salary in offer["salary"]:
new_salaries.append(
map_single_salary(salary, rates[salary['currency']]))
collection.update_one({"id": offer['id']},
{"$set": {
"salary": new_salaries
}})
except IndexError:
print("ERROR converting currencies")
