def update_user_email_is_validated_status(coll: pymongo.collection.Collection, user_id: ObjectId, yes=False, no=False, token=""): assert isinstance(user_id, ObjectId) if token: res = coll.update_one({"_id": user_id}, { "$set": { "email_validated": "pending", "email_validation_token": token, } }) elif yes: coll.update_one({"_id": user_id}, { "$set": { "email_validated": "yes", }, "$unset": { "email_validation_token": "", }, }) elif no: coll.update_one({"_id": user_id}, { "$set": { "email_validated": "no", }, "$unset": { "email_validation_token": "", }, })
def update_single_offer(self, collection: pymongo.collection.Collection, driver: webdriver.Chrome, offer: dict, old_db_corresponding_record=None): """Parses the info for a single offer Args: collection (pymongo.collection.Collection): MongoDB collection to run the update against driver (webdriver.Chrome): Selenium webdriver to get offer details offer (dict): A single offer from MongoDB old_db_corresponding_record (Dict, optional): Corresponding record in MongoDB, if exists. Defaults to None. """ doc = { "id": offer['id'], "date": offer['published'], "title": offer['title'], "position": offer['title'].split('@')[0], "author": offer['author'], "link": offer['link'], } offer_soup = BeautifulSoup(offer.summary, 'html.parser') text = offer_soup.text.split('\n') address = text[5].split('Location: ', 1)[1] city = address.split(',')[-1].strip() address = ','.join(address.split(',')[:-1]) salaries_raw = text[4].split('Salary: ')[1] salaries = [ salaries_raw ] if len(re.findall("\(.{1,17},.{1,17}\)", salaries_raw)) > 0 else salaries_raw.split(',') salary_ranges = [] for salary in salaries: salary_ranges.append( self.get_salary_details(salary.replace(" ", ""), salaries_raw)) if city in self.cities_translations: city = self.cities_translations[city] resp = driver.get(offer['id']) page_text = driver.find_element_by_id('root') doc["address"] = address doc["city"] = city doc["salary"] = salary_ranges doc["raw_salary"] = salaries_raw doc["full_description"] = page_text.text offer["full_description"] = page_text.text try: description_details = self.parse_single_description(offer) doc.update(description_details) except IndexError as e: print(f'Error parsing description for offer {offer["id"]} : {e}') if city not in self.cities: self.cities.append(city) if old_db_corresponding_record: collection.update_one({"id": offer['id']}, {"$set": doc}) else: collection.insert_one(doc)
def update_api( apis_collection: pymongo.collection.Collection, api_provider: str, number_of_calls: int, ): apis_collection.update_one({"provider": api_provider}, {"$set": { "number_of_calls": number_of_calls }})
def delete_user_address(coll: pymongo.collection.Collection, user_id: ObjectId, address_type: str): assert isinstance(user_id, ObjectId) coll.update_one({"_id": user_id}, {"$pull": { "addresses": { "type": address_type } }})
def delete_user_document(coll: pymongo.collection.Collection, user_id: ObjectId, document_id: ObjectId): assert isinstance(user_id, ObjectId) assert isinstance(document_id, ObjectId) coll.update_one({"_id": user_id}, {"$pull": { "documents": { "_id": document_id } }})
def add_user_contract_scan(coll: pymongo.collection.Collection, user_id: ObjectId, contract_id: ObjectId, file_id): assert isinstance(user_id, ObjectId) assert isinstance(contract_id, ObjectId) coll.update_one({ "_id": user_id, "contracts._id": contract_id }, {"$set": { "contracts.$.scan_file": file_id, }})
def invalidate_user_contract(coll: pymongo.collection.Collection, user_id: ObjectId, contract_id: ObjectId, invalidation_date: datetime): assert isinstance(user_id, ObjectId) assert isinstance(contract_id, ObjectId) coll.update_one({ "_id": user_id, "contracts._id": contract_id }, {"$set": { "contracts.$.invalidation_date": invalidation_date, }})
def invalidate_user_document(coll: pymongo.collection.Collection, user_id: ObjectId, document_id: ObjectId, invalidation_date: datetime): assert isinstance(user_id, ObjectId) assert isinstance(document_id, ObjectId) coll.update_one({ "_id": user_id, "documents._id": document_id }, {"$set": { "documents.$.invalidation_date": invalidation_date }})
def fix_all_descriptions(self, collection: pymongo.collection.Collection): """One time function to fix all descriptions Args: collection (pymongo.collection.Collection): MongoDB collection to be updated """ offers = collection.find() for offer in tqdm.tqdm(offers, desc="Fixing the descriptions"): try: description_details = self.parse_single_description(offer) collection.update_one({"id": offer['id']}, {"$set": description_details}) except IndexError: print("ERROR")
def update_dataset(old_dataset: Dict, h5_dataset: H5Dataset, folder: str, collection: pymongo.collection.Collection): try: rm_rf(folder) h5_dataset.convert(folder) collection.update_one({'_id': old_dataset['_id']}, {'$set': h5_dataset.dataset}) logging.info(f"Successfully updated dataset {h5_dataset.token}") except pymongo.errors.DuplicateKeyError as e: rm_rf(folder) logging.error(f"Duplicated token in dataset {h5_dataset.token}") except Exception as e: rm_rf(folder) logging.error( f"Failed to update dataset {h5_dataset.token} due to: {e}")
def add_embedded_mdoc_to_mdoc_array(coll: pymongo.collection.Collection, user_id: ObjectId, array_field: str, document: dict, document_id: ObjectId = None, filter_values=(None, "")): if filter_values: document = { key: value for key, value in document.items() if value not in filter_values } if document_id: document["_id"] = document_id coll.update_one({"_id": user_id}, {"$addToSet": {array_field: document}})
def _update_user_client_metric(user_collection: pymongo.collection.Collection, user: dict[str, Any], dry_run: bool) -> None: user_id = user['_id'] client_metrics = proto.create_from_mongo(user.get('clientMetrics'), user_pb2.ClientSideMetrics) assert client_metrics updated_metrics = user_pb2.ClientSideMetrics() if client_metrics.amplitude_id: amplitude_id = client_metrics.amplitude_id else: try: amplitude_id = _get_amplitude_id(user_id) except KeyError: logging.info('Could not find user "%s" on Amplitude.', user_id) amplitude_id = _AMPLITUDE_ID_NOT_FOUND updated_metrics.amplitude_id = amplitude_id if amplitude_id != _AMPLITUDE_ID_NOT_FOUND: if not client_metrics.first_session_duration_seconds: events = _get_amplitude_events(amplitude_id) first_session_duration = compute_first_session_duration(events) updated_metrics.first_session_duration_seconds = \ round(first_session_duration.total_seconds()) else: events = [] if not client_metrics.is_first_session_mobile: if not events: events = _get_amplitude_events(amplitude_id, limit=5) updated_metrics.is_first_session_mobile = _compute_is_mobile( events[:5]) dict_update = json_format.MessageToDict(updated_metrics) if not dict_update: logging.info('Nothing to update for user "%s"', user_id) return if dry_run: logging.info('Update user "%s":\n%s', user_id, dict_update) else: user_collection.update_one(user, { '$set': {f'clientMetrics.{k}': v for k, v in dict_update.items()} })
def write_to_fomo(id: int, id_cursor: pymongo.collection.Collection, source: str): # check if it exists first record = exists_on_fomo(id, id_cursor) if record: if source == 'file': deleted_ids.append(id) id_cursor.delete_one({'id': id}) if source == 'api': return id_cursor.update_one({'id': id}, {'$set': {'updated': datetime.datetime.utcnow()}}) else: id_cursor.insert_one({"id": id, "updated": datetime.datetime.utcnow()})
def update_user(coll: pymongo.collection.Collection, user_id: ObjectId, data: dict, embedded_1to1_docs=("name", )): assert isinstance(user_id, ObjectId) to_unset = {key: "" for key, value in data.items() if value == ""} for key in to_unset: del data[key] operation_dict = {} if data: operation_dict["$set"] = data if to_unset: operation_dict["$unset"] = to_unset updated = coll.find_one_and_update({"_id": user_id}, operation_dict, return_document=ReturnDocument.AFTER) for key in embedded_1to1_docs: if key in updated and not updated[key]: coll.update_one({"_id": user_id}, {"$unset": {"name": ""}})
def auto_time_update(col: pymongo.collection.Collection, filter_dict: dict, update_dict: dict): dt0 = datetime.datetime.utcnow() update_dict['updated_at'] = dt0 update_dict = {'$set': update_dict, '$setOnInsert': {'created_at': dt0}} return col.update_one(filter_dict, update_dict, upsert=True)
def user_unset_password_change_token(coll: pymongo.collection.Collection, user_id): coll.update_one({"_id": user_id}, {"$unset": { "password_change_token": "", }})
def user_set_password_change_token(coll: pymongo.collection.Collection, user_id, token): coll.update_one({"_id": user_id}, {"$set": { "password_change_token": token, }})
def reset_api(apis_collection: pymongo.collection.Collection, api_provider: str): apis_collection.update_one({"provider": api_provider}, {"$set": { "number_of_calls": 0 }})
def update_meme(memes_collection: pymongo.collection.Collection, meme_id, attribute: str, value): memes_collection.update_one({"_id": meme_id}, {"$set": {attribute: value}})
def push(self, db_collection: pymongo.collection.Collection): db_collection.update_one({"symbol": self._symbol}, self._document) return self
def convert_currencies(self, rates: RatesConverter, collection: pymongo.collection.Collection): """Maps the wages to PLN currencies for all offers in DB - we run it on all offers to adjust for currency fluctuation Args: rates (RatesConverter): RatesConverter instance to convert collection (pymongo.collection.Collection): MongoDB collection to be updated """ def map_single_salary_field(salary: Dict, salary_field: str, currency_rate: float): """Helper method to convert a single field - adds a new field with _pln ending Args: salary (Dict): nested document with salary info salary_field (str): name of the salary field currency_rate (float): Currency rate from target currency to PLN Returns: Dict: Updated salary chunk """ if salary[salary_field] != 'undisclosed': salary[salary_field] = int(salary[salary_field]) salary[f'{salary_field}_pln'] = int( int(salary[salary_field]) * currency_rate) else: salary[f'{salary_field}_pln'] = 'undisclosed' return salary def map_single_salary(salary: Dict, currency_rate: float): """Maps all fields in a single salary field Args: salary (Dict): nested document with salary info currency_rate (float): Currency rate from target currency to PLN Returns: Dict: Updated salary chunk """ salary = map_single_salary_field(salary, 'upper_range', currency_rate) salary = map_single_salary_field(salary, 'lower_range', currency_rate) if salary['upper_range'] != 'undisclosed': salary['average'] = (salary['upper_range'] + salary['lower_range']) / 2 else: salary['average'] = 'undisclosed' salary = map_single_salary_field(salary, 'average', currency_rate) return salary offers = collection.find() for offer in tqdm.tqdm(offers, desc="Mapping the currencies to PLN"): try: new_salaries = [] for salary in offer["salary"]: new_salaries.append( map_single_salary(salary, rates[salary['currency']])) collection.update_one({"id": offer['id']}, {"$set": { "salary": new_salaries }}) except IndexError: print("ERROR converting currencies")