def write_contract_to_mongo(reqId, contract: Contract,
symbols: pymongo.collection.Collection):
"""write a contract to symbols database with valid from attribute"""
contract = vars(contract)
contract["reqId"] = reqId
contract["validFrom"] = datetime.datetime.now()
symbols.insert_one(contract)
def update_single_offer(self,
collection: pymongo.collection.Collection,
driver: webdriver.Chrome,
offer: dict,
old_db_corresponding_record=None):
"""Parses the info for a single offer
Args:
collection (pymongo.collection.Collection): MongoDB collection to run the update against
driver (webdriver.Chrome): Selenium webdriver to get offer details
offer (dict): A single offer from MongoDB
old_db_corresponding_record (Dict, optional): Corresponding record in MongoDB, if exists. Defaults to None.
"""
doc = {
"id": offer['id'],
"date": offer['published'],
"title": offer['title'],
"position": offer['title'].split('@')[0],
"author": offer['author'],
"link": offer['link'],
}
offer_soup = BeautifulSoup(offer.summary, 'html.parser')
text = offer_soup.text.split('\n')
address = text[5].split('Location: ', 1)[1]
city = address.split(',')[-1].strip()
address = ','.join(address.split(',')[:-1])
salaries_raw = text[4].split('Salary: ')[1]
salaries = [
salaries_raw
] if len(re.findall("\(.{1,17},.{1,17}\)",
salaries_raw)) > 0 else salaries_raw.split(',')
salary_ranges = []
for salary in salaries:
salary_ranges.append(
self.get_salary_details(salary.replace(" ", ""), salaries_raw))
if city in self.cities_translations:
city = self.cities_translations[city]
resp = driver.get(offer['id'])
page_text = driver.find_element_by_id('root')
doc["address"] = address
doc["city"] = city
doc["salary"] = salary_ranges
doc["raw_salary"] = salaries_raw
doc["full_description"] = page_text.text
offer["full_description"] = page_text.text
try:
description_details = self.parse_single_description(offer)
doc.update(description_details)
except IndexError as e:
print(f'Error parsing description for offer {offer["id"]} : {e}')
if city not in self.cities:
self.cities.append(city)
if old_db_corresponding_record:
collection.update_one({"id": offer['id']}, {"$set": doc})
else:
collection.insert_one(doc)
def put_dataset(h5_dataset: H5Dataset, folder: str,
collection: pymongo.collection.Collection):
try:
h5_dataset.convert(folder)
collection.insert_one(h5_dataset.dataset)
logging.info(f"Successfully inserted dataset {h5_dataset.token}")
except pymongo.errors.DuplicateKeyError as e:
rm_rf(folder)
logging.error(f"Duplicated token in dataset {h5_dataset.token}")
def write_to_fomo(id: int, id_cursor: pymongo.collection.Collection, source: str):
# check if it exists first
record = exists_on_fomo(id, id_cursor)
if record:
if source == 'file':
deleted_ids.append(id)
id_cursor.delete_one({'id': id})
if source == 'api':
return id_cursor.update_one({'id': id}, {'$set': {'updated': datetime.datetime.utcnow()}})
else:
id_cursor.insert_one({"id": id, "updated": datetime.datetime.utcnow()})
def insert_meme(
memes_collection: pymongo.collection.Collection,
meme_name: str,
meme_url: str,
meme_description: str = "*new meme*",
):
meme = {
"name": meme_name,
"description": meme_description,
"times_used": 0,
"url": meme_url,
}
memes_collection.insert_one(meme)
def insert_doc(doc: dict, col: pymongo.collection.Collection):
if_insert = True
if doc["danmaku"]:
for each in col.find({"danmaku": doc["danmaku"]}):
if_insert = False
break
elif doc["uid"]:
for each in col.find({"uid": doc["uid"]}):
if_insert = False
break
if if_insert:
col.insert_one(doc)
print("insert: " + str(doc))
def add_mdoc(coll: pymongo.collection.Collection, mdoc: dict):
mdoc_filter = {"_id": mdoc["_id"]}
current_mdoc_in_db = coll.find_one(mdoc_filter)
if current_mdoc_in_db is not None:
print(f"Stávanící mdokument '{mdoc['_id']}': {current_mdoc_in_db}")
rewrite = input(f"'{mdoc['_id']}' mdokument už v kolekci existuje, mám ho přepsat (a/N)? ")
if rewrite.lower() not in ["a", "ano", "y", "yes"]:
print("Zachovávám původní mdokument.")
return
coll.delete_one(mdoc_filter)
coll.insert_one(mdoc)
print(f"Přidán mdokument {mdoc['_id']}")
def create_item(collection: pymongo.collection.Collection,
model: Union[Type[BaseModel], Type[dict]],
new_item_obj: BaseModel,
*,
id_key: str = "id_",
projection: dict = None,
raise_exc: bool = True) -> BaseModelClass:
"""
Create an item in the collection
:param collection: Collection to query
:param model: Class which the JSON in the collection represents
:param new_item_obj: Object to place in collection
:param id_key: If the UUID is stored outside of id_, specify here
:param projection: Filter to exclude from mongo query result
:param raise_exc: Whether to raise exception if item cannot be created.
:return: Created object in collection
"""
try:
if not getattr(new_item_obj, id_key, None):
setattr(new_item_obj, id_key, uuid4())
r = collection.insert_one(dict(new_item_obj))
if r.acknowledged:
return get_item(collection, model, getattr(new_item_obj, id_key), id_key=id_key, projection=projection,
raise_exc=False)
except pymongo.errors.DuplicateKeyError:
if raise_exc:
raise problems.UniquenessException("create", model.__name__, id_and_name(model, new_item_obj))
def insert_tables_mongo(coll_tables: pymongo.collection.Collection,
detected_table: dict) -> list:
"""
Insert tables into mongodb
"""
result = coll_tables.insert_one(detected_table)
return [f'Inserted table: {result}']
def import_new_records(base_id: str,
table: str,
mongo_table: pymongo.collection.Collection,
view: Optional[str] = None) -> None:
"""Import new records from Airtable to MongoDB."""
if not _AIRTABLE_API_KEY:
raise ValueError(
'No API key found. Create an airtable API key at '
'https://airtable.com/account and set it in the AIRTABLE_API_KEY '
'env var.')
client = airtable.Airtable(base_id, _AIRTABLE_API_KEY)
records = client.iterate(table, view=view)
converter = airtable_to_protos.ProtoAirtableConverter(
proto_type=review_pb2.DocumentToReview,
id_field=None,
required_fields=('anonymized_url', ))
num_inserted = 0
num_updated = 0
for record in records:
mongo_id = record.get('fields', {}).get('mongo_id')
proto_data = converter.convert_record(record)
airtable_id = proto_data.pop('_id')
if record['fields'].get('anonymized_url'):
proto_data['anonymizedUrl'] = record['fields']['anonymized_url'][
0]['url']
if mongo_id:
# Already added, let's update it.
document_json = mongo_table.find_one_and_update(
{'_id': objectid.ObjectId(mongo_id)},
{'$set': proto_data},
)
any_pending_or_done_review = document_json.get('numPendingReviews', 0) or \
document_json.get('numDoneReviews', 0)
timeout_review_count = sum(
1 for review in document_json.get('reviews', [])
if review.get('status') == 'REVIEW_TIME_OUT')
client.update(
table, airtable_id, {
'Bayes help needed': not any_pending_or_done_review,
'review_timeouts': timeout_review_count,
})
num_updated += 1
continue
result = mongo_table.insert_one(proto_data)
mongo_id = str(result.inserted_id)
client.update(table, airtable_id, {
'mongo_id': mongo_id,
'Bayes help needed': True
})
num_inserted += 1
print(f'{num_updated:d} documents updated.')
print(f'{num_inserted:d} documents added.')
def addPunishItem(cmp: pymongo.collection.Collection, apDataList: list):
resultList = []
for apData in apDataList:
dcdCode = apData['dcdCode']
try:
requests.adapters.DEFAULT_RETRIES = 5
# 获取具体信息
r3 = requests.get(
'http://210.76.74.232/appr-law-datacenter-service/law/datacenter/publicity/2/'
+ dcdCode,
timeout=(5, 5))
except Exception as e:
resultList.append(apData)
print("Request failed :" + dcdCode)
continue
deatil = json.loads(r3.text)['data']
item = {
'dcdCode': deatil['dcdCode'],
# 1:法人及其他组织|2:自然人|3:个体工商户
'admCounterCategory': deatil['admCounterCategory'],
'admCounterName': deatil['admCounterName'],
'caseName': deatil['caseName'],
'decisionNum': deatil['decisionNum'],
'illegalBasis': deatil['illegalBasis'],
'illegalFact': deatil['illegalFact'],
'punishBasis': deatil['punishBasis'],
'punishDate': deatil['punishDate'],
'punishDate': deatil['punishDate'],
'punishType': deatil['punishTypeModels'][0]['punishType'], # 2:罚款
'punishAmount':
deatil['punishTypeModels'][0]['punishAmount'], # 单位:万
'remark': deatil['remark']
}
print('Loaded successfully: ' + dcdCode)
cmp.insert_one(item)
time.sleep(0.1)
return resultList
def auto_time_insert(col: pymongo.collection.Collection,
insert_dict: dict):
dt0 = datetime.datetime.utcnow()
insert_dict['created_at'] = dt0
return col.insert_one(insert_dict)
def insert_id_if_not_exist(col: pymongo.collection.Collection, key_name: str, value):
result = col.find_one({"_id": key_name})
# logger.warning(f'result: {result}, key_name: {key_name}')
if result is None:
# logger.warning(f'insert key_name: {key_name}')
col.insert_one({"_id": key_name, "sequence_value": value})
def insert_todo(todos_collection: pymongo.collection.Collection,
timestamp: str, todo_content: str):
# Convert date string to Date object.
timestamp = parser.parse(timestamp)
todo = {"timestamp": timestamp, "content": todo_content}
todos_collection.insert_one(todo)
def insert_id_if_not_exist(col: pymongo.collection.Collection, key_name: str,
value):
result = col.find_one({"_id": key_name})
if result is None:
col.insert_one({"_id": key_name, "sequence_value": value})