def fetchLongUrl(short_url): alias_name = "_CREATE_" + str(random.randint(100000000, 999999999)) try: db_name = "url_db" connect(db=db_name, alias="default", host=os.environ["DB_URL"]) connect(db=db_name, alias=alias_name, host=os.environ["DB_URL"]) if short_url == "": raise Exception("Short url cannot be empty") try: logging.warning( "[services.url.shortener.fetchLongUrl] - Running fresh query. Not from cache" ) object_fetched = Urls.objects.get(short_url=short_url) #Update the last_clicked_on timestamp in order to use it further for stale records Urls.objects.get(short_url=short_url).update( last_clicked_on=time.time()) return { "status": "success", "message": object_fetched["long_url"] } except: raise Exception("Could not find matching URL") disconnect(alias=alias_name) disconnect(alias="default") except Exception as e: logging.warning("[services.url.shortener.fetchLongUrl] - %s", e) return {"status": "failure", "message": str(e)}
def flush_market_task(start_date: str = None, end_date: str = None): """ 刷新交易数据 :param start_date: :param end_date: :return: """ start_date = start_date if start_date else yesterday end_date = end_date if end_date else yesterday job = MarketData() connect('jiucai', host='master:17585') code_list = [obj.code for obj in ShareDocument.objects().all()] # executed_code_list = MarketDocument.objects(date__gte=end_date).distinct("code") # code_list = list(set(code_list).difference(executed_code_list)) bar = tqdm(code_list) for code in bar: data = job.get_by_code_and_date(code, start_date, end_date) if data: bulk = [ UpdateOne({ "code": row["code"], "date": row["date"] }, {"$set": row}, upsert=True) for row in data ] # noinspection PyProtectedMember MarketDocument._get_collection().bulk_write(bulk, ordered=False) bar.set_description(f"{code} flush {len(data)} records.") disconnect()
def setUpClass(cls): disconnect() connect('mongoenginetest', host='mongomock://localhost') machines = [ CoffeeMachine(name='m1', type="ES", model="ss", water_line=True), CoffeeMachine(name='m2', type="SMALL", model="ss", water_line=False), CoffeeMachine(name='m3', type="LARGE", model="ss", water_line=True), CoffeeMachine(name='m4', type="LARGE", model="ss", water_line=False), ] pods = [ CoffeePod(name='p1', type="ES", flavor="VANILLA", size=2), CoffeePod(name='p2', type="SMALL", flavor="MOCHA", size=5), CoffeePod(name='p3', type="LARGE", flavor="PSL", size=7), CoffeePod(name='p4', type="LARGE", flavor="VANILLA", size=5), ] CoffeeMachine.objects.insert(machines) CoffeePod.objects.insert(pods)
def setUpClass(cls): ''' 在该函数中定义需要用到的模型,必须要先disconnect然后连接mock.之后的 定义要多少自己定 ''' disconnect() connect('mongoenginetest', host='mongomock://localhost') Stat.objects.create(name='size_of_collection', data={ 'user': 0, 'venue': 0, 'court': 0, 'feedback': 0, 'notification': 0, 'share_notification': 0, 'reservation': 0, 'course': 0 }) Notification.objects.create(name='tongzhi', content='i am coming', time=datetime.now(), notice_id=1) User.create(password='******', user_id=1, name='test', api_id='2018000000', privilege=1)
def GetTweetsByHashtag(self, hashtag): self.hashtag = hashtag self.response = [] self.dict_ = {'userName': [], 'userId': [], 'followers': [], 'lang': [], 'location': [], 'date': [], 'text': [], 'tag': []} self.query = self.search(q='{}'.format(self.hashtag), result_type='recent', count=100) connect('tweetcatcher', host='mongo') for tweet in self.query['statuses']: self.userName = tweet['user']['screen_name'] self.userId = tweet['user']['id'] self.followers = tweet['user']['followers_count'] self.date = tweet['created_at'] self.lang = tweet['metadata']['iso_language_code'] self.location = tweet['user']['location'] self.body = tweet['text'].replace('\n', ' ').replace('\r', '').replace('\'', '').replace('\"', '') self.tag = self.hashtag self.tweetInfo = json.loads(r'''{{"userName": "******", "userId": "{}", "followers": "{}", "dateTime": "{}", "tag": "{}", "body": "{}", "lang": "{}", "location": "{}"}}'''.format(self.userName, self.userId, self.followers, self.date, self.tag, self.body, self.lang, self.location).replace("\\", ""), strict=False) self.tweetRecord = Tweet(**self.tweetInfo) self.tweetRecord.save() self.dict_['userName'].append(tweet['user']['screen_name']) self.dict_['userId'].append(tweet['user']['id']) self.dict_['followers'].append(tweet['user']['followers_count']) self.dict_['lang'].append(tweet['metadata']['iso_language_code']) self.dict_['location'].append(tweet['user']['location']) self.dict_['date'].append(tweet['created_at']) self.dict_['text'].append(tweet['text']) self.dict_['tag'].append(self.hashtag) self.dataFrame = pd.DataFrame(self.dict_) self.dataFrame.sort_values(by='followers', inplace=True, ascending=False) #print(self.dataFrame.head(5)) #print(self.dataFrame.head(5).to_json(orient='records')) self.response.append(json.loads(self.dataFrame.head(5).to_json(orient='records'))) disconnect() return self.response
def index(request): template = 'login/login.html' form = LoginForm() if request.method == 'POST': form = LoginForm(request.POST) if form.is_valid(): _username = form.cleaned_data.get('username') disconnect() connect('eagle') user = Users.objects(username=_username).first() disconnect() if user.is_active: request.session['username'] = user.username request.session['permissions'] = user.permissions request.session['admin'] = user.is_admin return redirect('home') else: messages.warning( request, 'Por favor, entre em contato com o gestor de sua área.') context = {'login': True, 'form': form} return render(request, template, context)
def teardown_method(self, method): """ teardown any state that was previously setup with a setup_method call. """ db = _get_db() db.drop_collection('video_info') disconnect(alias='test_video')
def phoneLogin(phone): alias_name = "_PHONELOGIN_" + str(random.randint(100000000, 999999999)) try: connect(db=DATABASE, alias="default", host=DBURL) connect(db=DATABASE, alias=alias_name, host=DBURL) if phone == "": raise Exception("Phone number can't be empty") otpString = str(random.randint(100000, 999999)) user = User.objects.get(phone_no=phone).update(otp_timestamp=str( int(time.time())), otp=otpString) if not Service.sendSMS(phone, otpString): raise Exception("SMS not sent") disconnect(alias=alias_name) disconnect(alias="default") except Exception as e: logging.error( "[APIService.phoneLogin] - Login with Phone failed for %s : %s", phone, e) return False logging.info( "[APIService.phoneLogin] - Login with Phone executed for %s : %s", phone, 'Success') return True
def otpVerification(phone, otp): alias_name = "_PHONELOGIN_" + str(random.randint(100000000, 999999999)) try: connect(db=DATABASE, alias="default", host=DBURL) connect(db=DATABASE, alias=alias_name, host=DBURL) if phone == "" or otp == "": raise Exception("Phone number or OTP can't be empty") user = User.objects.get(phone_no=phone, otp=otp) user = user.to_mongo().to_dict() if int(user['otptimestamp']) + 600 < int(time.time()): raise Exception("OTP Expired") disconnect(alias=alias_name) disconnect(alias="default") except Exception as e: logging.error( "[APIService.otpVerification] - OTP Verification Phone failed for %s : %s", phone, e) return False logging.info( "[APIService.otpVerification] - OTP Verification for %s : %s", phone, 'Success') return True
def test_get_http_action_invalid_db_url(self): disconnect() try: ActionUtility.get_http_action_config("mongodb://localhost:8000/test", "bot", "http_action") assert False except HttpActionFailure: assert True
def emailLogin(email, password): """ No password hashing is done since no registration method is to be done Thus without hashing testing becomes easier (like insertion) """ alias_name = "_EMAILLOGIN_" + str(random.randint(100000000, 999999999)) try: connect(db=DATABASE, alias="default", host=DBURL) connect(db=DATABASE, alias=alias_name, host=DBURL) print("Connected") if email == "" or password == "": raise Exception("Email or password can't be empty") user = User.objects.get(email_id=email, password=password) disconnect(alias=alias_name) disconnect(alias="default") except Exception as e: logging.error( "[APIService.emailLogin] - Login with Email failed for %s : %s", email, e) return False logging.info( "[APIService.emailLogin] - Login with Email executed for %s : %s", email, 'Success') return True
def del_tx_reference_bulk(bundle): with Mongo_Wrapper() as (alias, db_client): with switch_db(TxReference_db, alias) as TxReference_db_switch: for b in bundle: address, tx_reference, log_handler = b tx_hash = bytes(tx_reference[0]) tx_index = tx_reference[1] if log_handler is not None: log_handler("[-] Attempting to delete tx reference: {}:{}".format(tx_hash.hex(), tx_index)) try: tx_reference_docs = TxReference_db_switch.objects(tx_hash=tx_hash, tx_index=tx_index) except (CursorNotFound): disconnect(alias) raise DatabaseQueryFailed("Database query failed for tx reference {}:{} with address: {}".format(tx_hash.hex(), tx_index, address.hex())) if len(tx_reference_docs) == 1: tx_reference_doc = tx_reference_docs[0] if log_handler is not None: log_handler("[-] Tx reference {}:{} is already added to database".format(tx_hash.hex(), tx_index)) tx_reference_doc.delete() if log_handler is not None: log_handler("[-] Tx reference {}:{} deleted".format(tx_hash.hex(), tx_index)) else: if log_handler is not None: log_handler("[-] Tx reference {}:{} is not in database".format(tx_hash.hex(), tx_index))
def get_salary(request, id): disconnect() connect('eagle') depto = Functions.objects(id=id).first() disconnect() return JsonResponse(depto.salary, safe=False)
def setUpClass(cls): ''' 在该函数中定义需要用到的模型,必须要先disconnect然后连接mock.之后的 定义要多少自己定 ''' disconnect() connect('mongoenginetest', host='mongomock://localhost') Stat.objects.create(name='size_of_collection', data={ 'user': 0, 'venue': 0, 'court': 0, 'feedback': 0, 'notification': 0, 'share_notification': 0, 'reservation': 0, 'course': 0 }) Venue.objects.create(name='name', description='description', img=None, venue_id=1) Court.objects.create(name='name', court_id=1, venue_id=1) User.create(password='******', user_id=1, name='test', api_id='2018000000', privilege=1) Court.objects.create(name='name', court_id=2, venue_id=1)
def create_app(config=None): app = Flask(__name__) with app.app_context(): if config is not None: app.config.from_object(config) else: app.config.from_object(Config) disconnect(alias='default') connect( db=app.config['DB'], alias='default', host=app.config['MONGODB_HOST'] ) session.init_app(app) login_manager.init_app(app) login_manager.login_view = 'authorization.login' from app.main import bp as main_bp app.register_blueprint(main_bp) from app.errors import bp as errors_bp app.register_blueprint(errors_bp) from app.posts import posts_bp app.register_blueprint(posts_bp, url_prefix='/blog') from app.authorization import auth_bp app.register_blueprint(auth_bp) return app
def app(): app = create_app({ 'TESTING': True, 'JWT_ACCESS_TOKEN_EXPIRES': timedelta(hours=1), 'JWT_COOKIE_SECURE': False, 'JWT_SECRET_KEY': "test_key", 'JWT_TOKEN_LOCATION': ["cookies"], 'MONGODB_CONNECT': False }) with app.app_context(): disconnect() db = connect('test', host='mongomock://localhost') test_user1 = User(username='******', email='*****@*****.**', password='******') test_user1.save() test_user2 = User(username='******', email='*****@*****.**', password='******') test_user2.save() jwt = JWTManager() jwt.init_app(app) @jwt.user_lookup_loader def user_lookup_callback(_jwt_header, jwt_data): identity = jwt_data["sub"] return User.objects.get(email=identity) yield app # fixture cleanup db.drop_database('test') disconnect()
def connect_db(credentials_file=settings.DEFAULT_CREDENTIALS_FILE, alias="default"): """Connects to MongoDB using credentials. Args: credentials_file (str): path to the credentials file, or the Python dictionary containing the contents thereof. alias (str): alias for the connection. Returns: A MongoEngine connection instance. """ # Disconnect if needed try: mongo.disconnect(alias=alias) except: pass credentials = get_credentials(credentials_file) # Return connection return mongo.connect(credentials["database"], host=credentials["host"], port=credentials["port"], username=credentials["username"], password=credentials["password"])
def clean(self): id = self.cleaned_data.get('id') name = self.cleaned_data.get('name') code = self.cleaned_data.get('code') company = self.cleaned_data.get('company') type = self.cleaned_data.get('costcenter_type') disconnect() connect('eagle') if not id: name = CostCenters.objects(name=name).filter( company=company).filter(type=type).first() code = CostCenters.objects(code=code).filter( company=company).filter(type=type).first() else: name = CostCenters.objects.filter(name=name).filter( company=company).filter(type=type).filter(id__ne=id).first() code = CostCenters.objects.filter(code=code).filter( company=company).filter(type=type).filter(id__ne=id).first() disconnect() if name: self.add_error( 'name', 'Desculpe, este centro de custo já está cadastrado') if code: self.add_error('code', 'Desculpe, este código já está cadastrado')
def create_gametime_record(config_data, game_id, user_id, start_timestamp, end_timestamp): logging.basicConfig(format='%(levelname)s:%(message)s', level=getattr(logging, config_data.logLevel.upper(), None)) logging.debug('Logger loaded in create_gametime_record(...).') logging.debug('Creating MongoDB connection.') mongoengine.connect('gameplay_tracker', host=config_data.databases.mongodb.host, port=config_data.databases.mongodb.port, username=config_data.databases.mongodb.username, password=config_data.databases.mongodb.password, authentication_source=config_data.databases.mongodb. authentication_source) game_record = Game.objects(game_id=game_id).first() if game_record is None: logging.error(f'{game_id} is not a valid GameID') return -1 # TODO Pull user given UserID user_record = User.objects(user_id=user_id).first() if user_record is None: logging.error(f'{user_id} is not a valid UserID') game_time_record = GameTime(game_id=game_record.id, user_id=user_record.id, start_date=start_timestamp, end_date=end_timestamp) game_time_record.save() mongoengine.disconnect()
def show_search_result(self): """Show search result on searching""" print("show button1") search_by_txt = self.admin_search_verb_page.search_verb_by_comboBox.currentText( ) search_txt = self.admin_search_verb_page.search_verb_searchbar_lineEdit.text( ) print("show button2") print("Search by text:", search_by_txt) print("Search text:", search_txt) connect(db='VocabsEasyy', host='localhost', port=27017) print('Connected successfully for searching') try: if search_by_txt == 'German Verb': self.current_verb = Verb.objects(verb_name=search_txt).get() self.fetch_result_from_db() else: self.current_verb = Verb.objects( eng_meaning__contains=search_txt).get() self.fetch_result_from_db() except MultipleObjectsReturned as e: self.admin_search_verb_result_window.hide() self.popup_be_specific() except DoesNotExist as e: self.admin_search_verb_result_window.hide() self.popup_no_results() finally: disconnect()
def clean(): db = connectdb() db.drop_collection('buffer') db.drop_collection('stream') db.drop_collection('window') db.drop_collection('processed') disconnect('minibatch')
def get(self, pan): try: num = randrange(10) if num in (8, 9): raise BackendError client = None connect(db='SurePass', host=db_uri) for clients in PanClient.objects: if clients.pan == pan: client = clients break #Exception for not able to find the client in the database if client is None: disconnect() raise ClientNotFound js = { 'pan': client.pan, 'name': client.name, 'dob': client.dob.strftime('%Y-%m-%d'), 'father_name': client.father_name, 'client_id': str(client.id) } disconnect() return js except Exception as e: return {'msg': str(e)}
def setUp(self): disconnect() self.test_app = app.get_flask_app(test_config) self.test_app.testing = True self.test_app.debug = True self.test_client = self.test_app.test_client() connect('testing_db', is_mock=True)
def get(self, id): try: client = None connect(db='SurePass', host=db_uri) for clients in PanClient.objects: if str(clients.id) == id: client = clients break #Exception for not able to find the object id in the database if client is None: disconnect() raise ObjectIDNotFound js = { 'pan': client.pan, 'name': client.name, 'dob': client.dob.strftime('%Y-%m-%d'), 'father_name': client.father_name, 'client_id': str(client.id) } disconnect() return js except Exception as e: return {'msg': str(e)}
def get_http_action_config(db_url: str, bot: str, action_name: str): """ Fetch HTTP action configuration parameters from the MongoDB database :param db_url: MongoDB connection string :param bot: BotID :param action_name: Action name :return: HttpActionConfig object containing configuration for the action """ if ActionUtility.is_empty(db_url) or ActionUtility.is_empty( bot) or ActionUtility.is_empty( action_name): raise HttpActionFailure("Database url, bot name and action name are required") try: connect(host=db_url) http_config_dict = HttpActionConfig.objects().get(bot=bot, action_name=action_name).to_mongo().to_dict() if dict is None: raise DoesNotExist except DoesNotExist: raise HttpActionFailure("No HTTP action found for bot " + bot + " and action " + action_name) except Exception as ex: raise HttpActionFailure(ex) finally: disconnect() return http_config_dict
def mongo(request): """Use this fixtures for setup and teardown database every test method. How to use this fixture for a method: ``` def test_xxx(self, mongo): ``` How to use this fixture for every method on a class: ``` import pytest from utils.fixtures import mongo @pytest.mark.usefixtures('mongo') class SiteTests: ``` """ # Set up disconnect() db = connect("testdb", host="mongomock://localhost") # Run test method yield db # Tear down db.drop_database("testdb") disconnect()
def flush_adj_task(last_date: str = None): """ 刷新复权数据 :param last_date: 最后一个报告期对应日期 :return: """ job = AdjData() connect('jiucai', host='master:17585') code_list = [obj.code for obj in ShareDocument.objects().all()] if last_date: executed_code_list = AdjDocument.objects( trade_date__gte=last_date).distinct("code") code_list = list(set(code_list).difference(executed_code_list)) bar = tqdm(code_list) for code in bar: data = job.get_by_code(code) # noinspection PyProtectedMember bulk = [ UpdateOne({ "code": row.code, "trade_date": row.trade_date }, {"$set": row._asdict()}, upsert=True) for row in data ] if bulk: # noinspection PyProtectedMember AdjDocument._get_collection().bulk_write(bulk, ordered=False) bar.set_description(f"{code} flush {len(bulk)} records.") disconnect()
def clean(self): _username = self.cleaned_data.get('username') _email = self.cleaned_data.get('email') _id = self.cleaned_data.get('id') disconnect() connect('eagle') if not _id: user_username = Users.objects(username=_username).first() user_email = Users.objects(email=_email).first() else: user_username = Users.objects.filter(username=_username).filter( id__ne=_id).first() user_email = Users.objects.filter(email=_email).filter( id__ne=_id).first() disconnect() if user_username: self.add_error( 'username', 'Desculpe, este nome de usuário já está cadastrado') if user_email: self.add_error('email', 'Desculpe, este e-mail já está cadastrado')
def process_data(self, file): try: csv_data = file.decode("utf-8") reader = DictReader(StringIO(csv_data)) records = list(reader) records_processed = 0 number_of_records = len(records) for row in records: record = create_record(row) if record: try: record.save() except Exception as e: logging.error(e) records_processed += 1 self.update_state( state="PROGRESS", meta={"done": records_processed, "total": number_of_records}, ) disconnect() except Exception as ex: self.update_state( state=states.FAILURE, meta={ "exc_type": type(ex).__name__, "exc_message": traceback.format_exc().split("\n"), }, ) raise ex
def fetchShortUrl(long_url): alias_name = "_CREATE_" + str(random.randint(100000000, 999999999)) try: db_name = "url_db" connect(db=db_name, alias="default", host=os.environ["DB_URL"]) connect(db=db_name, alias=alias_name, host=os.environ["DB_URL"]) if long_url == "": raise Exception("Long url cannot be empty") maxNumOfChar = 6 #It can be changed dynamically based on the length of input string #Regex to validate the url format (Can be done in client side) # r0 = urlparse(long_url) # print(r0) # if not r0.scheme or not r0.netloc: # raise Exception("Invalid URL passed") #Check if any short_url corresponding to the longurl already exists try: object_fetched = Urls.objects.get(long_url=long_url) return { "status": "success", "message": object_fetched["short_url"] } except: pass #Else try to create one flag = True retry = 3 while (retry > 0 and flag): short_url = fake.lexify(text='?' * maxNumOfChar, letters=store) try: url_doc = Urls() url_doc.short_url = short_url url_doc.long_url = long_url url_doc.created_at = time.time() url_doc.save() flag = False except: logging.warning("Retrying with retry #%s", str(retry)) retry = retry - 1 if not flag: #successful insertion return {"status": "success", "message": short_url} return { "status": "failure", "message": "Something went wrong. Please try again" } disconnect(alias=alias_name) disconnect(alias="default") except Exception as e: logging.warning("[services.url.shortener.fetchShortUrl] - %s", e) return {"status": "failure", "message": str(e)}