dien['lng'] = geocode_result[0]['geometry']['location']['lng']
print(address, dien['lat'], dien['lng'])
time.sleep(1.5)
except Exception as e:
print(e)
print(address + "無經緯度資料")
client = pymongo.MongoClient('localhost',
27017,
username='******',
password='******')
collection = client.rawData.info3Store
ids = [data["name"] for data in dienList]
operations = [
UpdateOne({"_id": idn}, {'$set': data}, upsert=True)
for idn, data in zip(ids, dienList)
]
try:
collection.bulk_write(operations, ordered=False)
except Exception as er:
print(er)
mailTo("CrawlerThreeStoreSuccess", ["*****@*****.**"],
"轉換地址完畢 共{}筆".format(Alln))
except Exception as er:
errorstr = "出錯" + str(er) + "出錯位置:" + str(sys.exc_info()[2].tb_lineno)
mailTo("CrawlerThreeStoreError", ["*****@*****.**"],
"轉換地址失敗" + errorstr)
def _load_institution_with_aggregate_variable(
self, formatted_sheet_data: FormattedSheetData
):
"""
Load the special institution that is also a composite variable in to the db.
Parameters
----------
formatted_sheet_data: FormattedSheetData
The data to be loaded into the database. Please see the FormattedSheetData class to view its attributes.
"""
# Create the institution primary keys
institution = {
InstitutionField.spreadsheet_id: formatted_sheet_data.spreadsheet_id,
InstitutionField.sheet_id: formatted_sheet_data.sheet_id,
InstitutionField.name: formatted_sheet_data.meta_data.get(
MetaDataField.variable_heading
),
InstitutionField.country: formatted_sheet_data.meta_data.get(
InstitutionField.country
),
InstitutionField.category: formatted_sheet_data.meta_data.get(
InstitutionField.category
),
}
# Find the specific institution
institution_doc = self._find_one(db_collection.institutions, institution)
log.info(
f"Loaded 1 {db_collection.institutions} "
f"from sheet: {formatted_sheet_data.sheet_title}"
)
# Create a dict with variable heading(category of rights) as keys and the list of rights as the values
variable_heading_dict = {}
variable_heading_list = []
for datum in formatted_sheet_data.formatted_data:
# Category of a right is the first element in sigla_answers field of datum
variable_heading = datum.get(CompositeVariableField.sigla_answers)[0].get(
SiglaAnswerField.answer
)
sigla_answers = datum.get(CompositeVariableField.sigla_answers)[1:]
if variable_heading in variable_heading_dict:
variable_heading_dict.get(variable_heading).append(sigla_answers)
else:
variable_heading_list.append(variable_heading)
variable_heading_dict[variable_heading] = [sigla_answers]
# Create the list of variables
variables = [
{
VariableField.institution: institution_doc.get(InstitutionField._id),
VariableField.name: variable_heading,
VariableField.heading: variable_heading,
VariableField.sigla_answer: variable_heading_dict.get(variable_heading),
VariableField.type: VariableType.aggregate,
VariableField.variable_index: i,
}
for i, variable_heading in enumerate(variable_heading_list)
]
# Create the list of update requests into the db, one for each variable
update_requests = [
UpdateOne(
{
VariableField.institution: variable.get(VariableField.institution),
VariableField.name: variable.get(VariableField.name),
VariableField.variable_index: variable.get(
VariableField.variable_index
),
},
{"$set": variable},
upsert=True,
)
for i, variable in enumerate(variables)
]
# Bulk write the variables into the db
update_requests_results = self._db.get_collection(
db_collection.variables
).bulk_write(update_requests)
log.info(
f"Loaded {update_requests_results.upserted_count} {db_collection.variables} "
f"from sheet: {formatted_sheet_data.sheet_title}"
)
movies_to_migrate = []
for doc in cursor:
doc_id = doc.get('_id')
lastupdated = doc.get('lastupdated', None)
movies_to_migrate.append({
"doc_id": ObjectId(doc_id),
"lastupdated": parser.parse(lastupdated)
})
print(f"{len(movies_to_migrate)} documents to migrate")
try:
# TODO: Complete the UpdateOne statement below
# build the UpdateOne so it updates the "lastupdated" field to contain
# the new ISODate() type
bulk_updates = [
UpdateOne({"_id": movie.get("doc_id")},
{"$set": {
"lastupdated": movie.get("lastupdated")
}}) for movie in movies_to_migrate
]
# here's where the bulk operation is sent to MongoDB
bulk_results = mflix.movies.bulk_write(bulk_updates)
print(f"{bulk_results.modified_count} documents updated")
except InvalidOperation:
print("no updates necessary")
except Exception as e:
print(str(e))
counter = tqdm(desc="Writing title to Mongo ", unit="piece")
bkh = BulkHelper(100000, lambda e: collection.bulk_write(e))
with open(r"rsc/CID-Title", "r", encoding="latin-1") as f:
try:
while 1:
counter.update()
new_line = next(f).strip("\n")
id_string, title = new_line.split("\t")
uid = make_uid(src_name, id_string)
bkh.append(
UpdateOne({"uid": uid},
{"$set": {
"title": title,
"_sync": False
}},
upsert=True))
except StopIteration:
pass
bkh.close()
counter = tqdm(desc="Writing SMILES and formulas to Mongo ", unit="piece")
bkh = BulkHelper(100000, lambda e: collection.bulk_write(e))
with open(r"rsc/CID-SMILES", "r", encoding="latin-1") as f:
try:
while 1:
counter.update()
from pymongo import UpdateOne
from pymongo.errors import BulkWriteError
uri = os.getenv("MONGODB_URI")
# uri = "mongodb://localhost:27017/test" # Use this to test on local database.
client = MongoClient(uri)
database_name = pymongo.uri_parser.parse_uri(uri)['database']
db = client[database_name]
with open(os.path.join(os.path.dirname(__file__),
'consolidated_entries.json')) as f:
updates = json.load(f)
bulk_updates = []
for update in updates:
bulk_updates.append(
UpdateOne(
{
"_id": ObjectId(update["_id"]["$oid"]),
"_revisionIndex": update["_revisionIndex"]
}, {
"$set": {
"parsed_notes": update["parsed_notes"],
"consolidated_notes": update["consolidated_notes"]
}
}))
try:
db.entries.bulk_write(bulk_updates, ordered=False)
except BulkWriteError as bwe:
print(bwe.details)
def add_point(uid: str, point: Point):
return UpdateOne({"_id": uid},
{"$set": make_nested_dict(prepare_pt(point), 'points')},
upsert=True)
def fill_daily_k_at_suspension_days_at_date_one_collection(
basics, all_dates, collection):
"""
更新单个数据集的单个日期的数据
:param basics:
:param all_dates:
:param collection:
:return:
"""
code_last_trading_daily_dict = dict()
total = len(all_dates)
for i, date in enumerate(all_dates):
_tic = time.process_time()
update_requests = []
last_daily_code_set = set(code_last_trading_daily_dict.keys())
for basic in basics:
code = basic['code']
# 如果循环日期小于
if date < basic['timeToMarket']:
# print('日期：%s, %s 还没上市，上市日期: %s' % (date, code, basic['timeToMarket']), flush=True)
pass
else:
# 找到当日数据
daily = DB_CONN[collection].find_one({
'code': code,
'date': date,
'index': False
})
if daily is not None:
code_last_trading_daily_dict[code] = daily
last_daily_code_set.add(code)
else:
if code in last_daily_code_set:
last_trading_daily = code_last_trading_daily_dict[code]
suspension_daily_doc = {
'code': code,
'date': date,
'close': last_trading_daily['close'],
'open': last_trading_daily['close'],
'high': last_trading_daily['close'],
'low': last_trading_daily['close'],
'volume': 0,
'is_trading': False
}
update_requests.append(
UpdateOne(
{
'code': code,
'date': date,
'index': False
}, {'$set': suspension_daily_doc},
upsert=True))
_toc = time.process_time()
expect_time = (_toc - _tic) * (total - i - 1)
if len(update_requests) > 0:
update_result = DB_CONN[collection].bulk_write(update_requests,
ordered=False)
print(
'填充停牌数据进度: (%s/%s)， 日期：%s，数据集：%s，插入：%4d条，更新：%4d条, 预计还需要%.2fs' %
(i + 1, total, date, collection, update_result.upserted_count,
update_result.modified_count, expect_time),
flush=True)
def update_many(self, collection_name, docs: list):
with self._client:
self._db[collection_name].bulk_write([
UpdateOne({'id': doc["id"]}, {'$set': doc}) for doc in docs
if "id" in doc
])
def compute(self, begin_date, end_date):
"""
计算指定日期内的信号
:param begin_date: 开始日期
:param end_date: 结束日期
"""
all_codes = get_all_codes()
dm = DataModule()
N = 20
k = 2
for code in all_codes:
try:
df_daily = dm.get_k_data(code,
autype='hfq',
begin_date=begin_date,
end_date=end_date)
# 计算MB，盘后计算，这里用当日的Close
df_daily['MID'] = df_daily['close'].rolling(N).mean()
# 计算STD20
df_daily['std'] = df_daily['close'].rolling(N).std()
# 计算UP
df_daily['UP'] = df_daily['MID'] + k * df_daily['std']
# 计算down
df_daily['DOWN'] = df_daily['MID'] - k * df_daily['std']
# 将日期作为索引
df_daily.set_index(['date'], inplace=True)
# 上轨和中轨右移一位
shifted_up = df_daily['UP'].shift(1)
shifted_middle = df_daily['MID'].shift(1)
# 收盘价突破或者跌破中轨的幅度占上轨和中轨宽度的比例
ref_line = (df_daily['close'] -
shifted_middle) / (shifted_up - shifted_middle)
ref_prev = ref_line.shift(1)
# 找到时间窗口内的最小值
min_val = ref_line.rolling(10).min()
# 找到时间窗口内最低点前的最大值
max_leading_value = ref_line.rolling(10).apply(
lambda vec: vec[:np.argmin(vec) + 1].max().astype(float),
raw=True)
# 中轨支撑的作用的范围
delta = 0.15
# 判断是否存在中轨支撑反弹的信号，要求：
# 时间窗口的最低点之前的最大值大于delta，最小值的绝对值小于delta，就有一个穿越阈值分界线的动作；
# 当前日期在也在阈值之上，表示又从最低点穿越到阈值分界线之上；
# 而判断前一日在阈值分界线之下，表示穿越是在当前交易日完成
m_rebound_mask = (abs(min_val) <= delta) & (ref_line > delta) & (ref_prev <= delta) & \
(max_leading_value > delta)
# 将信号保存到数据库
update_requests = []
df_daily['m_rebound_mask'] = m_rebound_mask
df_daily = df_daily[df_daily['m_rebound_mask']]
for date in df_daily.index:
doc = {'code': code, 'date': date, 'signal': 'mid_rebound'}
update_requests.append(
UpdateOne(doc, {'$set': doc}, upsert=True))
if len(update_requests) > 0:
update_result = self.collection.bulk_write(update_requests,
ordered=False)
print('%s, upserted: %4d, modified: %4d' %
(code, update_result.upserted_count,
update_result.modified_count),
flush=True)
except:
traceback.print_exc()
async def post_process(self):
"""
url: POST /api/plan_management/plan/status_update
监控app发往计划管理app的数据结构:
[
{
"plan_no": "A181120203-00",
"status": 2, //2表示进行中，3表示暂停，4表示已完工
"progress_detail": //仅更新状态为暂停和已完工时传合格数和不合格数，对于进行中没有这个字段
{
"qualified_count": 1000,
"unqualified_count": 2
}
}
]
:return:
{
"code":"success",
"info": "",
"data": null
}
"""
req_data_list = json.loads(self.request.body)
try:
plan_collection = get_plan_db_collection()
# 转存成字典方便操作
status_data_dict = dict()
for req_data in req_data_list:
plan_no = req_data['plan_no']
status_data_dict[plan_no] = req_data
task_no_list = [
status_data['plan_no'] for status_data in req_data_list
]
query = {'task_no': {'$in': task_no_list}}
update_plan_list = []
async for document in plan_collection.find(query):
document_id = document['_id']
task_no = document['task_no']
status_data = status_data_dict[task_no]
updated_status = status_data['status']
update_clause = {'plan_status': updated_status}
if updated_status == PlanStatusType.in_progress.value:
real_start_date = document['real_start_date']
if not real_start_date:
# 如果是计划刚开始加工了，则设置真实开工时间为当前日期
update_clause['real_start_date'] = str(
datetime.now().date())
if 'progress_detail' in status_data:
# 如果是暂停或者可完工，则会传过来合格数和不合格数
qualified_count = status_data['progress_detail'][
'qualified_count']
unqualified_count = status_data['progress_detail'][
'unqualified_count']
update_clause['qualified_count'] = qualified_count
update_clause['unqualified_count'] = unqualified_count
update_data = UpdateOne({'_id': document_id},
{'$set': update_clause})
update_plan_list.append(update_data)
if update_plan_list:
# 批量更新计划
plan_collection.bulk_write(update_plan_list)
self.send_response_data(MesCode.success, None, '')
get_logger().info('更新计划状态成功: {}'.format(req_data_list))
except Exception as e:
log_exception(e, '更新计划状态数据失败')
self.send_response_data(MesCode.fail, None,
'更新计划状态数据失败: {}'.format(req_data_list))
def fill_daily_k_at_suspension_days_at_date_one_collection(self,
basics, all_dates, collection):
"""
更新单个数据集的单个日期的数据
:param basics: 上一交易日基础数据
:param all_dates:
:param collection:
:return:
"""
code_last_trading_daily_dict = dict()
for date in all_dates: # 按升序来
update_requests = []
last_daily_code_set = set(code_last_trading_daily_dict.keys())
for basic in basics: #轮询股票
code = basic['code']
# 如果循环日期小于
if date < basic['listDate']:
print('日期：%s, %s 还没上市，上市日期: %s' % (date, code, basic['list_date']), flush=True)
else:
# 找到当日数据
daily = DB_CONN[collection].find_one({'code': code, 'date': date})
if daily is not None:
code_last_trading_daily_dict[code] = daily
last_daily_code_set.add(code)
trading_daily_doc = {
'is_trading': True
}
update_requests.append(
UpdateOne(
{'code': code, 'date': date},
{'$set': trading_daily_doc},
upsert=True))
stock_node = "Stock" + date
query = "match(p:%s) where p.stock_code='%s' set p.is_trading = %r" % (stock_node, trading_daily_doc['is_trading'])
try:
GRAPH_DB_CONN.run(query)
except Exception as e:
print(e)
else:
suspension_daily_doc = {}
print(last_daily_code_set)
if code in last_daily_code_set: #从此日起开始停牌
print("begin to tingpai")
last_trading_daily = code_last_trading_daily_dict[code]
suspension_daily_doc = {
'code': code,
'date': date,
'close': last_trading_daily['close'],
'open': last_trading_daily['close'],
'high': last_trading_daily['close'],
'low': last_trading_daily['close'],
'vol': 0,
'change': 0,
'pct_chg': 0,
'amount': 0,
'is_trading': False
}
update_requests.append(
UpdateOne(
{'code': code, 'date': date},
{'$set': suspension_daily_doc},
upsert=True))
print(suspension_daily_doc)
if len(suspension_daily_doc) > 0 and collection == 'daily':
stock_node = "Stock" + date
query = "match(p:%s) where p.stock_code='%s' set p.close = %f, p.open = %f, p.high = %f, p.low = %f," \
"p.is_trading = %r" % (
stock_node, code, suspension_daily_doc['close'], suspension_daily_doc['open'], suspension_daily_doc['high'],
suspension_daily_doc['low'], suspension_daily_doc['is_trading'])
try:
GRAPH_DB_CONN.run(query)
except Exception as e:
print(e)
if len(update_requests) > 0:
update_result = DB_CONN[collection].bulk_write(update_requests, ordered=False)
print('填充停牌数据，日期：%s，数据集：%s，插入：%4d条，更新：%4d条' %
(date, collection, update_result.upserted_count, update_result.modified_count), flush=True)
def cache_start(self, _, data):
yield "shards", UpdateOne({"_id": "meta"},
{"$set": {
"_id": "meta",
**data
}})
def cache_guild_role_create(self, _, data, *, upsert=True):
role = data["role"]
role["_id"] = role["id"]
role["guild_id"] = data["guild_id"]
yield "roles", UpdateOne({"_id": role["id"]}, {"$set": role},
upsert=upsert)
def cache_channel_create(self, _, data, *, upsert=True):
data["_id"] = data["id"]
yield "channels", UpdateOne({"_id": data["id"]}, {"$set": data},
upsert=upsert)
def BulkSetData(_id, gvkey):
return UpdateOne({"_id":ObjectId(_id)},{"$set":{"gvkey":gvkey}})
def migrate(self):
bulk = [
UpdateOne(
{"_id": bson.ObjectId("5b6d6819d706360001a0b716")},
{
"$set": {
"name": "Group",
"uuid": UUID("8874518c-effd-41fe-81bf-d67f1519ccf2"),
"description": "Grouping element",
"single_service": False,
"single_client": False,
"allow_children": True,
"bi_id": Int64(3859384814270643576),
}
},
upsert=True,
),
UpdateOne(
{"_id": bson.ObjectId("5b6d6c9fd706360001f5c053")},
{
"$set": {
"name": "Network | Controller",
"uuid": UUID("bcf7ad57-81a4-4da0-8e6d-e429c9e21532"),
"description": "Controller - CPE relation",
"service_model": "sa.ManagedObject",
"client_model": "sa.ManagedObject",
"single_service": False,
"single_client": True,
"allow_children": False,
"bi_id": Int64(2204453448135692504),
}
},
upsert=True,
),
UpdateOne(
{"_id": bson.ObjectId("5b6dbbefd70636000170b980")},
{
"$set": {
"name": "Object Group",
"uuid": UUID("f4c6d51d-d597-4183-918e-23efd748fd12"),
"description": "Arbitrary group of Managed Objects",
"service_model": "sa.ManagedObject",
"single_service": False,
"single_client": False,
"allow_children": False,
"bi_id": Int64(4062440225872880146),
}
},
upsert=True,
),
UpdateOne(
{"_id": bson.ObjectId("5b6d6be1d706360001f5c04e")},
{
"$set": {
"name": "Network | IPoE Termination",
"uuid": UUID("ef42d9fe-d217-4754-b628-a1f71f6159da"),
"description":
"IPoE Temination (access equipment -> BRAS)",
"service_model": "sa.ManagedObject",
"client_model": "sa.ManagedObject",
"single_service": False,
"single_client": False,
"allow_children": False,
"bi_id": Int64(4546441601898809637),
}
},
upsert=True,
),
UpdateOne(
{"_id": bson.ObjectId("5b6d6beed706360001f5c04f")},
{
"$set": {
"name": "Network | PPPoE Termination",
"uuid": UUID("a8ddcd67-d8c4-471d-9a9b-9f4749e09011"),
"description":
"PPPoE Temination (access equipment -> BRAS)",
"service_model": "sa.ManagedObject",
"client_model": "sa.ManagedObject",
"single_service": False,
"single_client": False,
"allow_children": False,
"bi_id": Int64(3384545658468911814),
}
},
upsert=True,
),
UpdateOne(
{"_id": bson.ObjectId("5b6d6c56d706360001f5c052")},
{
"$set": {
"name": "Network | PPTP Termination",
"uuid": UUID("8ce08fc8-a5b1-448d-9c2c-ac1419ad9816"),
"description":
"PPTP Temination (access equipment -> BRAS)",
"service_model": "sa.ManagedObject",
"client_model": "sa.ManagedObject",
"single_service": False,
"single_client": False,
"allow_children": False,
"bi_id": Int64(2085768785416150430),
}
},
upsert=True,
),
UpdateOne(
{"_id": bson.ObjectId("5b6e785ed70636000170b9a6")},
{
"$set": {
"name": "Voice | SIP Termination",
"uuid": UUID("3e15a3ea-f4c1-49a1-a183-d61dd79531c2"),
"description":
"SIP Temination (media gateway -> softswitch)",
"service_model": "sa.ManagedObject",
"client_model": "sa.ManagedObject",
"single_service": False,
"single_client": False,
"allow_children": False,
"bi_id": Int64(4632306658633376591),
}
},
upsert=True,
),
]
self.mongo_db.technologies.bulk_write(bulk)
def create0(link):
'''Populates mongoDB database with our index'''
print("############### Starting storage #####################")
## root = 'C:/Users/richa/Desktop/webpages/WEBPAGES_RAW'
root = "C:/Users/spsan/OneDrive/Documents/UCI 2018-19/CS 121/Project 3/webpages/WEBPAGES_RAW"
counter = 0
bulk_update_list = []
# https://www.tutorialspoint.com/python3/os_walk.htm
for root, dirs, files in sorted(os.walk(root)):
for f in files:
if not f.endswith('.json') and not f.endswith(
'.tsv') and not f.endswith('.DS_Store'):
path = os.path.join(root, f)
## doc_id = path[33:] # Richard's path
doc_id = path[85:] # Sandy's path
tokenDict = preprocess.tokenizer(path)
#========== insert here ========
for token, freq in tokenDict.items():
append_list = {"token": token}
append_list2 = {
"$set": {
"posting" + str(counter): {
"docID": doc_id,
"freq": freq,
"tf-idf": "None"
}
}
}
bulk_update_list.append(append_list)
bulk_update_list.append(append_list2)
counter += 1
print(counter)
if (counter % 10 == 0):
link.bulk_write([
UpdateOne(bulk_update_list[0], bulk_update_list[1],
True),
UpdateOne(bulk_update_list[2], bulk_update_list[3],
True),
UpdateOne(bulk_update_list[4], bulk_update_list[5],
True),
UpdateOne(bulk_update_list[6], bulk_update_list[7],
True),
UpdateOne(bulk_update_list[8], bulk_update_list[9],
True),
UpdateOne(bulk_update_list[10],
bulk_update_list[11], True),
UpdateOne(bulk_update_list[12],
bulk_update_list[13], True),
UpdateOne(bulk_update_list[14],
bulk_update_list[15], True),
UpdateOne(bulk_update_list[16],
bulk_update_list[17], True),
UpdateOne(bulk_update_list[18],
bulk_update_list[19], True)
])
bulk_update_list = []
#=========== end here==========
print("############### Finished storage #####################")
async def incoming(message,
chain_name=None,
tx_hash=None,
height=None,
seen_ids=None,
check_message=False,
retrying=False,
bulk_operation=False):
""" New incoming message from underlying chain.
For regular messages it will be marked as confirmed
if existing in database, created if not.
"""
hash = message['item_hash']
sender = message['sender']
chain = chain_name
ids_key = (hash, sender, chain)
if chain_name and tx_hash and height and seen_ids is not None:
if ids_key in seen_ids.keys():
if height > seen_ids[ids_key]:
return True
filters = {
'item_hash': hash,
'chain': message['chain'],
'sender': message['sender'],
'type': message['type']
}
existing = await Message.collection.find_one(filters,
projection={
'confirmed': 1,
'confirmations': 1,
'time': 1,
'signature': 1
})
if check_message:
if existing is None or (existing['signature'] != message['signature']):
# check/sanitize the message if needed
message = await check_message_fn(message,
from_chain=(chain_name
is not None))
if message is None:
return True # message handled.
if retrying:
LOGGER.debug("(Re)trying %s." % hash)
else:
LOGGER.info("Incoming %s." % hash)
# we set the incoming chain as default for signature
message['chain'] = message.get('chain', chain_name)
# if existing is None:
# # TODO: verify if search key is ok. do we need an unique key for messages?
# existing = await Message.collection.find_one(
# filters, projection={'confirmed': 1, 'confirmations': 1, 'time': 1})
if chain_name and tx_hash and height:
# We are getting a confirmation here
new_values = await mark_confirmed_data(chain_name, tx_hash, height)
updates = {
'$set': {
'confirmed': True,
},
'$min': {
'time': message['time']
},
'$addToSet': {
'confirmations': new_values['confirmations'][0]
}
}
else:
updates = {
'$max': {
'confirmed': False,
},
'$min': {
'time': message['time']
}
}
# new_values = {'confirmed': False} # this should be our default.
should_commit = False
if existing:
if seen_ids is not None:
if ids_key in seen_ids.keys():
if height > seen_ids[ids_key]:
return True
else:
seen_ids[ids_key] = height
else:
seen_ids[ids_key] = height
# THIS CODE SHOULD BE HERE...
# But, if a race condition appeared, we might have the message twice.
# if (existing['confirmed'] and
# chain_name in [c['chain'] for c in existing['confirmations']]):
# return
LOGGER.debug("Updating %s." % hash)
if chain_name and tx_hash and height:
# we need to update messages adding the confirmation
#await Message.collection.update_many(filters, updates)
should_commit = True
else:
# if not (chain_name and tx_hash and height):
# new_values = {'confirmed': False} # this should be our default.
try:
content, size = await get_message_content(message)
except Exception:
LOGGER.exception("Can't get content of object %r" % hash)
content = None
if content is None:
LOGGER.info("Can't get content of object %r, retrying later." %
hash)
if not retrying:
await PendingMessage.collection.insert_one({
'message':
message,
'source':
dict(
chain_name=chain_name,
tx_hash=tx_hash,
height=height,
check_message=check_message # should we store this?
)
})
return
if content == -1:
LOGGER.warning("Can't get content of object %r, won't retry." %
hash)
return -1
if content.get('address', None) is None:
content['address'] = message['sender']
if content.get('time', None) is None:
content['time'] = message['time']
# warning: those handlers can modify message and content in place
# and return a status. None has to be retried, -1 is discarded, True is
# handled and kept.
# TODO: change this, it's messy.
try:
handling_result = await handle_incoming_message(message, content)
except Exception:
LOGGER.exception("Error using the message type handler")
handling_result = None
if handling_result is None:
LOGGER.debug("Message type handler has failed, retrying later.")
if not retrying:
await PendingMessage.collection.insert_one({
'message':
message,
'source':
dict(
chain_name=chain_name,
tx_hash=tx_hash,
height=height,
check_message=check_message # should we store this?
)
})
return
if handling_result != True:
LOGGER.warning("Message type handler has failed permanently for "
"%r, won't retry." % hash)
return -1
if not await check_sender_authorization(message, content):
LOGGER.warn("Invalid sender for %s" % hash)
return True # message handled.
if seen_ids is not None:
if ids_key in seen_ids.keys():
if height > seen_ids[ids_key]:
return True
else:
seen_ids[ids_key] = height
else:
seen_ids[ids_key] = height
LOGGER.debug("New message to store for %s." % hash)
# message.update(new_values)
updates['$set'] = {
'content': content,
'size': size,
'item_content': message.get('item_content'),
'item_type': message.get('item_type'),
'channel': message.get('channel'),
'signature': message.get('signature')
}
should_commit = True
#await Message.collection.insert_one(message)
# since it's on-chain, we need to keep that content.
# if message['item_type'] == 'ipfs' and app['config'].ipfs.enabled.value:
# LOGGER.debug("Pining hash %s" % hash)
# await pin_hash(hash)
if should_commit:
action = UpdateOne(filters, updates, upsert=True)
if not bulk_operation:
await Message.collection.bulk_write([action])
else:
return action
return True # message handled.
def add_fields(uid: str, fields: Mapping):
return UpdateOne({"_id": uid},
{"$set": make_nested_dict(fields, 'fields')},
upsert=True)
lastupdated = doc.get('lastupdated', None)
movies_to_migrate.append(
{
"doc_id": ObjectId(doc_id),
"lastupdated": parser.parse(lastupdated)
}
)
print(f"{len(movies_to_migrate)} documents to migrate")
try:
# TODO: Complete the UpdateOne statement below
# build the UpdateOne so it updates the "lastupdated" field to contain
# the new ISODate() type
bulk_updates = [UpdateOne(
{"_id": movie.get("doc_id")},
#{"$some_update_operator": {"some_field_to_update"}}
{"$set": {"lastupdated": movie.get("lastupdated")}}
) for movie in movies_to_migrate]
# here's where the bulk operation is sent to MongoDB
bulk_results = mflix.movies.bulk_write(bulk_updates)
print(f"{bulk_results.modified_count} documents updated")
except InvalidOperation:
print("no updates necessary")
except Exception as e:
print(str(e))
def fill_au_factor_pre_close(begin_date, end_date):
"""
为daily数据集填充：
1. 复权因子au_factor，复权的因子计算方式：au_factor = hfq_close/close
2. pre_close = close(-1) * au_factor(-1)/au_factor
:param begin_date: 开始日期
:param end_date: 结束日期
"""
all_codes = ts.get_stock_basics().index.tolist() # get_all_codes()
total = len(all_codes)
for i, code in enumerate(all_codes):
hfq_daily_cursor = DB_CONN['daily_hfq'].find(
{
'code': code,
'date': {
'$lte': end_date,
'$gte': begin_date
},
'index': False
},
sort=[('date', ASCENDING)],
projection={
'date': True,
'close': True
})
date_hfq_close_dict = dict([(x['date'], x['close'])
for x in hfq_daily_cursor])
daily_cursor = DB_CONN['daily'].find(
{
'code': code,
'date': {
'$lte': end_date,
'$gte': begin_date
},
'index': False
},
sort=[('date', ASCENDING)],
projection={
'date': True,
'close': True
})
last_close = -1
last_au_factor = -1
_tic = time.process_time()
update_requests = []
for daily in daily_cursor:
date = daily['date']
try:
close = daily['close']
doc = dict()
au_factor = np.round(date_hfq_close_dict[date] / close, 2)
doc['au_factor'] = au_factor
if last_close != -1 and last_au_factor != -1:
pre_close = last_close * last_au_factor / au_factor
doc['pre_close'] = np.round(pre_close, 2)
last_au_factor = au_factor
last_close = close
update_requests.append(
UpdateOne({
'code': code,
'date': date,
'index': False
}, {'$set': doc}))
except:
print('计算复权因子时发生错误，股票代码：%s，日期：%s' % (code, date), flush=True)
# 恢复成初始值，防止用错
last_close = -1
last_au_factor = -1
_toc = time.process_time()
expect_time = (_toc - _tic) * (total - i - 1)
if len(update_requests) > 0:
update_result = DB_CONN['daily'].bulk_write(update_requests,
ordered=False)
print('填充复权因子和前收，进度：(%s / %s), 股票：%s，更新：%4d条, 预计还需%.2fs' %
(i + 1, total, code, update_result.modified_count,
expect_time),
flush=True)
def post(self, authority, dataclass):
"""
[資料集批次寫入模式]
可上傳一組完整格式的資料，且重複資料複寫
命令格式： /v1/traffic_data/authority/{authority}/class/{dataclass}/standard/MOTC_traffic_v2/method/batch -X POST -d {data}
---
tags:
- MOTC Traffic v2 Upload API (提供上傳交通部「即時路況資料標準(V2.0)」格式資料)
parameters:
- in: path
name: authority
type: string
required: true
description: 業管機關簡碼(https://traffic-api-documentation.gitbook.io/traffic/xiang-dai-zhao-biao)
enum: ['NFB', 'THB', 'TNN']
- in: path
name: dataclass
type: string
required: true
description: 資料型態(依即時路況資料標準V2.0資料類型訂定，如VD、VDLive、LiveTraffic...)
enum: ['VD', 'VDLive', 'CCTV', 'CMS', 'CMSLive', 'AVI', 'AVIPair', 'AVIPairLive',
'ETag', 'ETagPair', 'ETagPairLive', 'GVPLiveTraffic', 'CVPLiveTraffic',
'Section', 'SectionLink', 'LiveTraffic', 'CongestionLevel', 'SectionShape',
'News']
- in: body
name: data
required: true
description: 輸入一組資料(JSON格式)
responses:
200:
description: OK
"""
from api import mongo_client
message = ''
# 讀取API傳入參數
args = self.parser.parse_args()
# 參數轉小寫處理
dataclass_lower = dataclass.lower()
authority_lower = authority.lower()
# MongoDB連結設定參數處理
database = 'traffic_data_' + authority_lower
collection = dataclass_lower
db = mongo_client[database]
collection = db[collection]
# pyspark寫入語法
# 輸入JSON文件
data = request.get_json()
update_time = data['UpdateTime']
one_records = data[dataclass_record[dataclass]]
update_records = []
for one_record in one_records:
if not ('DataCollectTime' in one_record) and not ('UpdateTime'
in one_record):
# 靜態資料附加UpdateTime資訊
one_record.update(
{'UpdateTime': aniso8601.parse_datetime(update_time)})
update_records.append(
UpdateOne(
{
dataclass_id[dataclass]:
one_record[dataclass_id[dataclass]],
'UpdateTime':
one_record['UpdateTime']
}, {"$set": one_record},
upsert=True))
else:
one_record.update({
'DataCollectTime':
aniso8601.parse_datetime(one_record['DataCollectTime'])
})
update_records.append(
UpdateOne(
{
dataclass_id[dataclass]:
one_record[dataclass_id[dataclass]],
'DataCollectTime':
one_record['DataCollectTime']
}, {"$set": one_record},
upsert=True))
try:
collection.bulk_write(update_records)
except BulkWriteError as e:
message = e.details
return {'message': message}, 404
message = 'upload succeeded'
return {'message': message}, 200
async def post(self):
r_dict = {'code': 0}
try:
subject_dimension_id_list = self.get_body_arguments(
'subject_dimension_id_list[]', [])
if subject_dimension_id_list:
operate = self.get_argument('operate', None)
if operate is not None:
status = None
id_list = [
ObjectId(subject_id)
for subject_id in subject_dimension_id_list
]
if int(operate) == 1:
status = STATUS_SUBJECT_DIMENSION_ACTIVE
update_requests = []
for dimension_id in subject_dimension_id_list:
update_requests.append(
UpdateOne({'_id': ObjectId(dimension_id)}, {
'$set': {
'status': status,
'updated_dt': datetime.datetime.now(),
'updated_id': self.current_user.oid
}
}))
await SubjectDimension.update_many(update_requests)
elif int(operate) == 0:
status = STATUS_SUBJECT_DIMENSION_INACTIVE
update_requests = []
for dimension_id in subject_dimension_id_list:
update_requests.append(
UpdateOne({'_id': ObjectId(dimension_id)}, {
'$set': {
'status': status,
'updated_dt': datetime.datetime.now(),
'updated_id': self.current_user.oid
}
}))
await SubjectDimension.update_many(update_requests)
elif int(operate) == -1:
cid_list = await SubjectDimension.distinct(
'cid', {
'_id': {
'$in': id_list
},
'parent_cid': {
'$in': [None, '']
}
})
await SubjectDimension.delete_many(
{'_id': {
'$in': id_list
}})
await SubjectDimension.delete_many(
{'parent_cid': {
'$in': cid_list
}})
# 状态变更子维度也变更
if int(operate) in [1, 0] and status:
cid_list = await SubjectDimension.distinct(
'cid', {
'_id': {
'$in': id_list
},
'parent_cid': {
'$in': [None, '']
}
})
await update_subject_sub_dimension_status(
self.current_user.oid, status, cid_list)
r_dict['code'] = 1
except RuntimeError:
logger.error(traceback.format_exc())
return r_dict
def post(self, authority, dataclass, date):
"""
[單筆靜態資料模式]
僅可寫入單筆靜態資料，且重複資料複寫
命令格式： /v1/traffic_data/authority/{authority}/class/{dataclass}/update/{date}/standard/MOTC_traffic_v2/method/one_record -X POST -d {data}
---
tags:
- MOTC Traffic v2 Upload API (提供上傳交通部「即時路況資料標準(V2.0)」格式資料)
parameters:
- in: path
name: authority
type: string
required: true
description: 業管機關簡碼(https://traffic-api-documentation.gitbook.io/traffic/xiang-dai-zhao-biao)
enum: ['NFB', 'THB', 'TNN']
- in: path
name: dataclass
type: string
required: true
description: 資料型態(依即時路況資料標準V2.0資料類型訂定，如VD、VDLive、LiveTraffic...)
enum: ['VD', 'CCTV', 'CMS', 'AVI', 'AVIPair',
'ETag', 'ETagPair',
'Section', 'SectionLink', 'CongestionLevel', 'SectionShape']
- in: path
name: date
type: string
required: true
description: 資料代表之時間(靜態資料參照欄位：UpdateTime)[格式：ISO8601]
default: '2020-12-20T15:00:00+08:00'
- in: body
name: data
required: true
description: 輸入一筆資料(JSON格式)
responses:
200:
description: OK
"""
from api import mongo_client
message = ''
# 讀取API傳入參數
args = self.parser.parse_args()
# 參數轉小寫處理
dataclass_lower = dataclass.lower()
authority_lower = authority.lower()
# MongoDB連結設定參數處理
database = 'traffic_data_' + authority_lower
collection = dataclass_lower
db = mongo_client[database]
collection = db[collection]
# pyspark寫入語法
# 輸入JSON文件
one_record = request.get_json()
update_time = date
update_records = []
one_record.update(
{'UpdateTime': aniso8601.parse_datetime(update_time)})
update_records.append(
UpdateOne(
{
dataclass_id[dataclass]:
one_record[dataclass_id[dataclass]],
'UpdateTime': one_record['UpdateTime']
}, {"$set": one_record},
upsert=True))
try:
collection.bulk_write(update_records)
except BulkWriteError as e:
message = e.details
return {'message': message}, 404
message = 'upload succeeded'
return {'message': message}, 200
def get_botscore_by_userid(user):
"""
Collects the botscore from Botometer
Keyword arguments:
user -- Twitter users' identificator
"""
user_id = user.id
try:
botometer_instance = random.choice(keys)
consumer_key = botometer_instance.consumer_key
result = botometer_instance.check_account(user_id)
return UpdateOne({'_id': make_objid(user_id)},
{'$set': {
'scores': result,
'screen_name': user.screen_name,
'id': user.id,
'error': 'None',
'checked': datetime.now().strftime("%d/%m/%Y")
}},
upsert=True
)
except Exception as e:
# Locked account (private)
auth_match = re.search('Not authorized', str(e))
timeline_match = re.search('has no tweets in timeline', str(e))
notExist_match = re.search('Sorry, that page does not exist', str(e))
overCapacity_match = re.search('Over capacity', str(e))
if auth_match:
return UpdateOne({'_id': make_objid(user_id)},
{'$unset': {'scores': ""},
'$set': {'screen_name': user.screen_name,
'id': user.id,
'error': 'not authorized',
'checked': datetime.now().strftime("%d/%m/%Y")},
'$push': {'error_key_used': consumer_key}},
upsert=True
)
elif overCapacity_match:
return UpdateOne({'_id': make_objid(user_id)},
{'$unset': {'scores': ""},
'$set': {'screen_name': user.screen_name,
'id': user.id,
'error': 'over capacity',
'checked': datetime.now().strftime("%d/%m/%Y")},
'$push': {'error_key_used': consumer_key}},
upsert=True
)
elif timeline_match:
# print("User", user_id, " has no tweets in timeline")
return UpdateOne({'_id': make_objid(user_id)},
{'$unset': {'scores': ""},
'$set': {'screen_name': user.screen_name,
'id': user.id,
'error': 'has no tweets in timeline',
'checked': datetime.now().strftime("%d/%m/%Y")}},
upsert=True
)
elif notExist_match:
# print("User", user_id, " does not exists anymore")
return UpdateOne({'_id': make_objid(user_id)},
{'$unset': {'scores': ""},
'$set': {'error': 'does not exists anymore',
'checked': datetime.now().strftime("%d/%m/%Y")}},
upsert=True
)
else:
print("Exception. User:"******"API:", consumer_key, "Message:", e)
return UpdateOne({'_id': make_objid(user_id)},
{'$unset': {'scores': ""},
'$set': {
'screen_name': user.screen_name,
'id': user.id,
'error': str(e),
'checked': datetime.now().strftime("%d/%m/%Y")}
},
upsert=True
)
def compute_fractal(begin_date, end_date):
# 获取所有股票代码
codes = get_all_codes()
# 计算每个股票的信号
for code in codes:
try:
# 获取后复权的价格，使用后复权的价格计算分型信号
daily_cursor = DB_CONN['daily_hfq'].find(
{
'code': code,
'date': {
'$gte': begin_date,
'$lte': end_date
},
'index': False
},
sort=[('date', ASCENDING)],
projection={
'date': True,
'high': True,
'low': True,
'_id': False
})
df_daily = DataFrame([daily for daily in daily_cursor])
# 设置日期作为索引
df_daily.set_index(['date'], 1, inplace=True)
# 通过shift，将前两天和后两天对齐到中间一天
df_daily_shift_1 = df_daily.shift(1)
df_daily_shift_2 = df_daily.shift(2)
df_daily_shift_3 = df_daily.shift(3)
df_daily_shift_4 = df_daily.shift(4)
# 顶分型，中间日的最高价既大于前两天的最高价，也大于后两天的最高价
df_daily['up'] = (df_daily_shift_3['high'] > df_daily_shift_1['high']) & \
(df_daily_shift_3['high'] > df_daily_shift_2['high']) & \
(df_daily_shift_3['high'] > df_daily_shift_4['high']) & \
(df_daily_shift_3['high'] > df_daily['high'])
# 底分型，中间日的最低价既小于前两天的最低价，也小于后两天的最低价
df_daily['down'] = (df_daily_shift_3['low'] < df_daily_shift_1['low']) & \
(df_daily_shift_3['low'] < df_daily_shift_2['low']) & \
(df_daily_shift_3['low'] < df_daily_shift_4['low']) & \
(df_daily_shift_3['low'] < df_daily['low'])
# 只保留了出现顶分型和低分型信号的日期
df_daily = df_daily[(df_daily['up'] | df_daily['down'])]
# 抛掉不用的数据
df_daily.drop(['high', 'low'], 1, inplace=True)
print(df_daily)
# 将信号保存到数据库 ,
update_requests = []
# 保存的数据结果时，code、date和信号的方向
for date in df_daily.index:
doc = {
'code': code,
'date': date,
# up: 顶分型， down：底分型
'direction': 'up' if df_daily.loc[date]['up'] else 'down'
}
# 保存时以code、date和direction做条件，那么就需要在这三个字段上建立索引
# db.fractal_signal.createIndex({'code': 1, 'date': 1, 'direction': 1})
update_requests.append(
UpdateOne(doc, {'$set': doc}, upsert=True))
if len(update_requests) > 0:
update_result = DB_CONN['fractal_signal'].bulk_write(
update_requests, ordered=False)
print('%s, upserted: %4d, modified: %4d' %
(code, update_result.upserted_count,
update_result.modified_count),
flush=True)
except:
print('错误发生： %s' % code, flush=True)
traceback.print_exc()
def _load_institutions(
self,
formatted_sheet_data: FormattedSheetData,
):
"""
Load institutions and their variables in to the database.
Parameters
----------
formatted_sheet_data: FormattedSheetData
The data to be loaded into the database. Please see the FormattedSheetData class to view its attributes.
"""
institution_primary_keys = [InstitutionField.name, InstitutionField.category]
if InstitutionField.country in formatted_sheet_data.meta_data:
institution_primary_keys.append(InstitutionField.country)
# Create the list of update requests into the db, one for each institution
institution_requests = [
UpdateOne(
{pk: institution.get(pk) for pk in institution_primary_keys},
{
"$set": {
key: institution.get(key)
for key in institution.keys()
if key != "childs"
}
},
upsert=True,
)
for institution in formatted_sheet_data.formatted_data
]
# Bulk write the institutions in the db
institution_requests_results = self._db.get_collection(
db_collection.institutions
).bulk_write(institution_requests)
log.info(
f"Loaded {institution_requests_results.upserted_count} {db_collection.institutions} "
f"from sheet: {formatted_sheet_data.sheet_title}"
)
# Get doc id for each institution
institution_doc_id_dict = {}
for i, institution in enumerate(formatted_sheet_data.formatted_data):
upserted_id = institution_requests_results.upserted_ids.get(i)
if upserted_id is None:
# The institution wasn't upserted
# Find the doc
institution_doc = self._db.get_collection(
db_collection.institutions
).find_one({pk: institution.get(pk) for pk in institution_primary_keys})
institution_doc_id_dict[i] = institution_doc.get(InstitutionField._id)
else:
institution_doc_id_dict[i] = upserted_id
# Create the list of update requests into the db, one for each variable
variable_requests = [
UpdateOne(
{
VariableField.institution: institution_doc_id_dict.get(i),
VariableField.heading: child.get(VariableField.heading),
VariableField.name: child.get(VariableField.name),
VariableField.variable_index: child.get(
VariableField.variable_index
),
},
{
"$set": {
VariableField.institution: institution_doc_id_dict.get(i),
**child,
}
},
upsert=True,
)
for i, institution in enumerate(formatted_sheet_data.formatted_data)
for child in institution.get("childs")
]
# Bulk write the variables in the db
variable_requests_results = self._db.get_collection(
db_collection.variables
).bulk_write(variable_requests)
log.info(
f"Loaded {variable_requests_results.upserted_count} {db_collection.variables} "
f"from sheet: {formatted_sheet_data.sheet_title}"
)
async def process_cursor(self, skip=0):
"""Process mongodb cursor
Transform data and move to new database
Args:
docs(:obj:`pymongo.Cursor`): documents to be processed
"""
bulk_write = []
query = {}
if self.max_entries == float('inf'):
limit = 0
else:
limit = self.max_entries
docs = self.from_collection.find(filter=query, projection={'_id': 0},
no_cursor_timeout=True, batch_size=500,
skip=skip, limit=limit)
i = 0
async for doc in docs:
i += 1
if i == self.max_entries:
break
if i != 0 and i % 50 == 0:
print("Processing file {}".format(i + skip))
try:
await self.to_collection.bulk_write(bulk_write)
bulk_write = []
except BulkWriteError as bwe:
pprint(bwe.details)
bulk_write = []
uniprot_id = doc.get('uniprot_id')
doc["add_id"] = [{"name_space": "gene_name_alt", "value": doc.get("gene_name_alt")},
{"name_space": "gene_name_orf", "value": doc.get("gene_name_orf")},
{"name_space": "gene_name_oln", "value": doc.get("gene_name_oln")}]
doc.pop('gene_name_alt', None)
doc.pop('gene_name_orf', None)
doc.pop('gene_name_oln', None)
doc['schema_version'] = "2"
tax_doc = await motor_client_manager.client.get_database(
"datanator-test")["taxon_tree"].find_one(filter={"tax_id": doc["ncbi_taxonomy_id"]},
projection={'canon_anc_ids': 1, 'canon_anc_names': 1})
if tax_doc is not None:
doc['canon_anc_names'] = tax_doc["canon_anc_names"]
doc['canon_anc_ids'] = tax_doc["canon_anc_ids"]
modifications = doc.get('modifications')
if modifications is not None:
bw = []
for mod in modifications:
mod['uniprot_id'] = uniprot_id
mod['schema_version'] = "2"
reference = mod['reference']
mod['reference'] = {"namespace": "doi", "value": reference["doi"]}
bw.append(json.loads(json.dumps(mod, ignore_nan=True)))
motor_client_manager.client.get_database(self.to_database)['protein_modifications'].insert_many(bw)
doc.pop('modifications', None)
bulk_write.append(UpdateOne({'uniprot_id': uniprot_id}, {'$set': json.loads(json.dumps(doc, ignore_nan=True))}, upsert=True))
if len(bulk_write) != 0:
try:
self.to_collection.bulk_write(bulk_write)
except BulkWriteError as bwe:
pprint(bwe.details)
finally:
print("Done.")
def run():
calendar_list = []
calendar_file_path = os.path.join(sys.path[0], 'calendar.dat')
try:
mongo = Mongo()
# 交易日历
exchange_calendar = mongo.gettable('z3_exchange_calendar')
tmp_calendar_list = exchange_calendar.find({'open_close': 2})
for calendar in tmp_calendar_list:
calendar_list.append(calendar['trade_date'])
except ServerSelectionTimeoutError:
if not os.path.exists(calendar_file_path):
return
with open(calendar_file_path) as rfile:
for calendar in rfile.readlines():
calendar_list.append(calendar)
calendar_dict = {}
with open(calendar_file_path, 'w') as wfile:
for calendar in calendar_list:
wfile.write(str(calendar) + '\n')
calendar_dict[int(calendar)] = 1
calendar_date = int(time.strftime('%Y%m%d', time.localtime(time.time())))
if calendar_date in calendar_dict:
return
for key, value in TABLES.items():
if not value:
continue
logger.info('===init init===table = %s', key)
collection = mongo.gettable(key)
if collection.name == 'Z3_TOPIC_CHANGE':
result = collection.find_one()
value['_id'] = int(datetime.datetime.now().strftime("%Y%m%d"))
value['topic_num'] = result.get('topic_num')
Mongo.remove(collection)
Mongo.insert(collection, value)
logger.info('===update update===table = %s', key)
elif collection.name == 'Z3_INDU_CHANGE':
result = collection.find_one()
value['_id'] = int(datetime.datetime.now().strftime("%Y%m%d"))
value['indu_num'] = result.get('indu_num')
Mongo.remove(collection)
Mongo.insert(collection, value)
else:
if value.get('operator'):
Mongo.remove(collection)
logger.info('===remove remove===table = %s', key)
else:
Mongo.update(collection, value)
logger.info('===update update===table = %s', key)
# add by 20170731
equity_profile_collection = mongo.gettable('z3_equity_profile')
equity_profile_list = equity_profile_collection.find({'sec_type': 1, 'chi_spel': {'$ne': None}}, {'_id': 1, 'chi_spel': 1})
requests = []
for equity_profile in equity_profile_list:
innercode = equity_profile.get('_id')
chi_spel = equity_profile.get('chi_spel')
requests.append(UpdateOne({'_id': innercode, 'type': 1}, {'$set': {'chi_spel': chi_spel}}))
cap_info_subs_collection = mongo.gettable('z3_cap_info_subs')
cap_info_subs_collection.bulk_write(requests, ordered=False)
def fixate(self, new_version):
"""
Фиксация изменений
@return:
"""
if new_version == self.meta.version:
raise CannotFixate('New version equals current version')
if new_version in [v.version for v in self.meta.versions]:
raise CannotFixate(
'New version already present in previous versions')
from pymongo import InsertOne, UpdateOne, DeleteMany
old_version = self.meta.version or new_version
# noinspection PyListCreation
requests = []
# Если справочник был неверсионным и у нас записи без меты, надо забить её умолчаниями.
self._fix_meta(new_version)
# Новые удалённые - просто удаляем
requests.append(
DeleteMany(
{'$and': [{
'_meta.delete': True
}, {
'_meta.draft': True
}]}))
# Старые удалённые - помечаем конечной версией
for db_record in self.collection.find(
{'$and': [{
'_meta.delete': True
}, {
'_meta.draft': False
}]}):
rb_record = self.record_factory(db_record)
rb_record.meta.delete = False
rb_record.meta.draft = False
rb_record.meta.edit = None
rb_record.meta.end_version = old_version
requests.append(
UpdateOne(
{'_id': rb_record._id},
{'$set': rb_record.as_db_record()},
))
# Новые записи - помечаем начальной версией
for db_record in self.collection.find(
{'$and': [
{
'_meta.delete': False
},
{
'_meta.draft': True
},
]}):
rb_record = self.record_factory(db_record)
rb_record.meta.draft = False
rb_record.meta.delete = False
rb_record.meta.end_version = None
rb_record.meta.beg_version = new_version
requests.append(
UpdateOne(
{'_id': rb_record._id},
{'$set': rb_record.as_db_record()},
))
# Изменённые - создаём новую, а старую помечаем конечной версией
for db_record in self.collection.find({'_meta.edit': {'$ne': None}}):
rb_record = self.record_factory(db_record)
new = self.record_factory(rb_record.meta.edit)
new.meta.beg_version = new_version
new.meta.end_version = None
new.meta.delete = False
new.meta.draft = False
requests.append(InsertOne(new.as_db_record()))
rb_record.meta.end_version = old_version
rb_record.meta.edit = None
rb_record.meta.delete = False
rb_record.meta.draft = False
requests.append(
UpdateOne(
{'_id': rb_record._id},
{'$set': rb_record.as_db_record()},
))
self.collection.bulk_write(requests)
self.meta.version = new_version
from nvesta.library.rb.rbmeta import RefBookVersionMeta
version_meta = RefBookVersionMeta()
version_meta.version = new_version
version_meta.fix_datetime = datetime.datetime.utcnow()
self.meta.versions.append(version_meta)
self.meta.reshape()
