def load_legit(legit_list):
''' Load the 'legit' instants to the remote database and delete from temp.
This process does not delete the
:param collection: the collection you want to pull instants from
:type collection: pymongo.collection.Collection
'''
from pymongo.errors import DuplicateKeyError
import config
from config import remote_client
from config import client
from config import database
from db_ops import dbncol
### this should load to the remote_client.owmap.legit_inst for production ###
remote_col = dbncol(remote_client, 'legit_inst', database=database)
col = dbncol(client, 'instant_temp', database=database)
# col = dbncol(client, 'legit_inst', database=database)
try:
check = remote_col.insert_one(legit_list)
if not check:
print('was not able to insert to remote db')
except DuplicateKeyError:
col.delete_one(legit_list)
### saved for later, when doing it on bulk ###
# col.insert_many(legit_list)
# Now go to the temp_instants collection and delete the instants just
# loaded to legit_inst.
# col.delete_many(legit_list)
return
def to_dbncol(self, collection='test'):
''' Load the data to the database.
:param collection: the collection name
:type collection: string
'''
from config import database
from db_ops import dbncol
col = dbncol(client, collection, database=database)
col.update_one({'_id': self._id}, {'$set': self.as_dict}, upsert=True)
def sweep(instants):
''' Move any instant that has a ref_time less than the current next
ref_time and with self.count less than 40. This is getting rid of the
instnats that are not and will never be legit.
:param instants: a itterable of Instant objects
:type instants: dict, list, pymongo cursor
'''
import time
import pymongo
from pymongo.cursor import Cursor
from Extract.make_instants import find_data
from config import client, database
from db_ops import dbncol
col = dbncol(client, 'instant_temp', database=database)
n = 0
# Check the instant type- it could be a dict if it came from the database,
# or it could be a list if it's a bunch of instant objects, or a pymongo
# cursor over the database.
if type(instants) == dict:
for key, doc in instants:
if key['instant'] < time.time()-453000: # 453000sec: about 5 days
col.delete_one(doc)
n += 1
elif type(instants) == list:
for doc in instants:
if doc['instant'] < time.time()-453000:
col.delete_one(doc)
n += 1
elif type(instants) == pymongo.cursor.Cursor:
for doc in instants:
if doc['instant'] < time.time()-453000:
col.delete_one(doc)
n += 1
else:
print(f'You want me to sweep instants that are {type(instants)}\'s.')
return
# col = dbncol(client, 'legit_inst', database=database)
try:
check = remote_col.insert_one(legit_list)
if not check:
print('was not able to insert to remote db')
except DuplicateKeyError:
col.delete_one(legit_list)
### saved for later, when doing it on bulk ###
# col.insert_many(legit_list)
# Now go to the temp_instants collection and delete the instants just
# loaded to legit_inst.
# col.delete_many(legit_list)
return
import time
start_time = time.time() # This is to get the total runtime if this script is
# run as __main__
if __name__ == '__main__':
''' Connect to the database, then move all the legit instants to the remote
database and clear out any instants that are past and not legit.
'''
import config
import db_ops
collection = 'instant_temp'
col = db_ops.dbncol(config.client, collection, database=config.database)
cast_count_all(col.find({}))
sweep(col.find({}))
print(f'Total op time for instant.py was {time.time()-start_time} seconds')