def fetchLongUrl(short_url):
alias_name = "_CREATE_" + str(random.randint(100000000, 999999999))
try:
db_name = "url_db"
connect(db=db_name, alias="default", host=os.environ["DB_URL"])
connect(db=db_name, alias=alias_name, host=os.environ["DB_URL"])
if short_url == "":
raise Exception("Short url cannot be empty")
try:
logging.warning(
"[services.url.shortener.fetchLongUrl] - Running fresh query. Not from cache"
)
object_fetched = Urls.objects.get(short_url=short_url)
#Update the last_clicked_on timestamp in order to use it further for stale records
Urls.objects.get(short_url=short_url).update(
last_clicked_on=time.time())
return {
"status": "success",
"message": object_fetched["long_url"]
}
except:
raise Exception("Could not find matching URL")
disconnect(alias=alias_name)
disconnect(alias="default")
except Exception as e:
logging.warning("[services.url.shortener.fetchLongUrl] - %s", e)
return {"status": "failure", "message": str(e)}
def flush_market_task(start_date: str = None, end_date: str = None):
"""
刷新交易数据
:param start_date:
:param end_date:
:return:
"""
start_date = start_date if start_date else yesterday
end_date = end_date if end_date else yesterday
job = MarketData()
connect('jiucai', host='master:17585')
code_list = [obj.code for obj in ShareDocument.objects().all()]
# executed_code_list = MarketDocument.objects(date__gte=end_date).distinct("code")
# code_list = list(set(code_list).difference(executed_code_list))
bar = tqdm(code_list)
for code in bar:
data = job.get_by_code_and_date(code, start_date, end_date)
if data:
bulk = [
UpdateOne({
"code": row["code"],
"date": row["date"]
}, {"$set": row},
upsert=True) for row in data
]
# noinspection PyProtectedMember
MarketDocument._get_collection().bulk_write(bulk, ordered=False)
bar.set_description(f"{code} flush {len(data)} records.")
disconnect()
def setUpClass(cls):
disconnect()
connect('mongoenginetest', host='mongomock://localhost')
machines = [
CoffeeMachine(name='m1', type="ES", model="ss", water_line=True),
CoffeeMachine(name='m2',
type="SMALL",
model="ss",
water_line=False),
CoffeeMachine(name='m3', type="LARGE", model="ss",
water_line=True),
CoffeeMachine(name='m4',
type="LARGE",
model="ss",
water_line=False),
]
pods = [
CoffeePod(name='p1', type="ES", flavor="VANILLA", size=2),
CoffeePod(name='p2', type="SMALL", flavor="MOCHA", size=5),
CoffeePod(name='p3', type="LARGE", flavor="PSL", size=7),
CoffeePod(name='p4', type="LARGE", flavor="VANILLA", size=5),
]
CoffeeMachine.objects.insert(machines)
CoffeePod.objects.insert(pods)
def setUpClass(cls):
'''
在该函数中定义需要用到的模型，必须要先disconnect然后连接mock.之后的
定义要多少自己定
'''
disconnect()
connect('mongoenginetest', host='mongomock://localhost')
Stat.objects.create(name='size_of_collection',
data={
'user': 0,
'venue': 0,
'court': 0,
'feedback': 0,
'notification': 0,
'share_notification': 0,
'reservation': 0,
'course': 0
})
Notification.objects.create(name='tongzhi',
content='i am coming',
time=datetime.now(),
notice_id=1)
User.create(password='******',
user_id=1,
name='test',
api_id='2018000000',
privilege=1)
def GetTweetsByHashtag(self, hashtag):
self.hashtag = hashtag
self.response = []
self.dict_ = {'userName': [], 'userId': [], 'followers': [], 'lang': [], 'location': [], 'date': [], 'text': [], 'tag': []}
self.query = self.search(q='{}'.format(self.hashtag), result_type='recent', count=100)
connect('tweetcatcher', host='mongo')
for tweet in self.query['statuses']:
self.userName = tweet['user']['screen_name']
self.userId = tweet['user']['id']
self.followers = tweet['user']['followers_count']
self.date = tweet['created_at']
self.lang = tweet['metadata']['iso_language_code']
self.location = tweet['user']['location']
self.body = tweet['text'].replace('\n', ' ').replace('\r', '').replace('\'', '').replace('\"', '')
self.tag = self.hashtag
self.tweetInfo = json.loads(r'''{{"userName": "******", "userId": "{}", "followers": "{}", "dateTime": "{}", "tag": "{}", "body": "{}", "lang": "{}", "location": "{}"}}'''.format(self.userName, self.userId, self.followers, self.date, self.tag, self.body, self.lang, self.location).replace("\\", ""), strict=False)
self.tweetRecord = Tweet(**self.tweetInfo)
self.tweetRecord.save()
self.dict_['userName'].append(tweet['user']['screen_name'])
self.dict_['userId'].append(tweet['user']['id'])
self.dict_['followers'].append(tweet['user']['followers_count'])
self.dict_['lang'].append(tweet['metadata']['iso_language_code'])
self.dict_['location'].append(tweet['user']['location'])
self.dict_['date'].append(tweet['created_at'])
self.dict_['text'].append(tweet['text'])
self.dict_['tag'].append(self.hashtag)
self.dataFrame = pd.DataFrame(self.dict_)
self.dataFrame.sort_values(by='followers', inplace=True, ascending=False)
#print(self.dataFrame.head(5))
#print(self.dataFrame.head(5).to_json(orient='records'))
self.response.append(json.loads(self.dataFrame.head(5).to_json(orient='records')))
disconnect()
return self.response
def index(request):
template = 'login/login.html'
form = LoginForm()
if request.method == 'POST':
form = LoginForm(request.POST)
if form.is_valid():
_username = form.cleaned_data.get('username')
disconnect()
connect('eagle')
user = Users.objects(username=_username).first()
disconnect()
if user.is_active:
request.session['username'] = user.username
request.session['permissions'] = user.permissions
request.session['admin'] = user.is_admin
return redirect('home')
else:
messages.warning(
request,
'Por favor, entre em contato com o gestor de sua área.')
context = {'login': True, 'form': form}
return render(request, template, context)
def teardown_method(self, method):
""" teardown any state that was previously setup with a setup_method
call.
"""
db = _get_db()
db.drop_collection('video_info')
disconnect(alias='test_video')
def phoneLogin(phone):
alias_name = "_PHONELOGIN_" + str(random.randint(100000000, 999999999))
try:
connect(db=DATABASE, alias="default", host=DBURL)
connect(db=DATABASE, alias=alias_name, host=DBURL)
if phone == "":
raise Exception("Phone number can't be empty")
otpString = str(random.randint(100000, 999999))
user = User.objects.get(phone_no=phone).update(otp_timestamp=str(
int(time.time())),
otp=otpString)
if not Service.sendSMS(phone, otpString):
raise Exception("SMS not sent")
disconnect(alias=alias_name)
disconnect(alias="default")
except Exception as e:
logging.error(
"[APIService.phoneLogin] - Login with Phone failed for %s : %s",
phone, e)
return False
logging.info(
"[APIService.phoneLogin] - Login with Phone executed for %s : %s",
phone, 'Success')
return True
def otpVerification(phone, otp):
alias_name = "_PHONELOGIN_" + str(random.randint(100000000, 999999999))
try:
connect(db=DATABASE, alias="default", host=DBURL)
connect(db=DATABASE, alias=alias_name, host=DBURL)
if phone == "" or otp == "":
raise Exception("Phone number or OTP can't be empty")
user = User.objects.get(phone_no=phone, otp=otp)
user = user.to_mongo().to_dict()
if int(user['otptimestamp']) + 600 < int(time.time()):
raise Exception("OTP Expired")
disconnect(alias=alias_name)
disconnect(alias="default")
except Exception as e:
logging.error(
"[APIService.otpVerification] - OTP Verification Phone failed for %s : %s",
phone, e)
return False
logging.info(
"[APIService.otpVerification] - OTP Verification for %s : %s",
phone, 'Success')
return True
def test_get_http_action_invalid_db_url(self):
disconnect()
try:
ActionUtility.get_http_action_config("mongodb://localhost:8000/test", "bot", "http_action")
assert False
except HttpActionFailure:
assert True
def emailLogin(email, password):
"""
No password hashing is done since no registration method is to be done
Thus without hashing testing becomes easier (like insertion)
"""
alias_name = "_EMAILLOGIN_" + str(random.randint(100000000, 999999999))
try:
connect(db=DATABASE, alias="default", host=DBURL)
connect(db=DATABASE, alias=alias_name, host=DBURL)
print("Connected")
if email == "" or password == "":
raise Exception("Email or password can't be empty")
user = User.objects.get(email_id=email, password=password)
disconnect(alias=alias_name)
disconnect(alias="default")
except Exception as e:
logging.error(
"[APIService.emailLogin] - Login with Email failed for %s : %s",
email, e)
return False
logging.info(
"[APIService.emailLogin] - Login with Email executed for %s : %s",
email, 'Success')
return True
def del_tx_reference_bulk(bundle):
with Mongo_Wrapper() as (alias, db_client):
with switch_db(TxReference_db, alias) as TxReference_db_switch:
for b in bundle:
address, tx_reference, log_handler = b
tx_hash = bytes(tx_reference[0])
tx_index = tx_reference[1]
if log_handler is not None:
log_handler("[-] Attempting to delete tx reference: {}:{}".format(tx_hash.hex(), tx_index))
try:
tx_reference_docs = TxReference_db_switch.objects(tx_hash=tx_hash, tx_index=tx_index)
except (CursorNotFound):
disconnect(alias)
raise DatabaseQueryFailed("Database query failed for tx reference {}:{} with address: {}".format(tx_hash.hex(), tx_index, address.hex()))
if len(tx_reference_docs) == 1:
tx_reference_doc = tx_reference_docs[0]
if log_handler is not None:
log_handler("[-] Tx reference {}:{} is already added to database".format(tx_hash.hex(), tx_index))
tx_reference_doc.delete()
if log_handler is not None:
log_handler("[-] Tx reference {}:{} deleted".format(tx_hash.hex(), tx_index))
else:
if log_handler is not None:
log_handler("[-] Tx reference {}:{} is not in database".format(tx_hash.hex(), tx_index))
def get_salary(request, id):
disconnect()
connect('eagle')
depto = Functions.objects(id=id).first()
disconnect()
return JsonResponse(depto.salary, safe=False)
def setUpClass(cls):
'''
在该函数中定义需要用到的模型，必须要先disconnect然后连接mock.之后的
定义要多少自己定
'''
disconnect()
connect('mongoenginetest', host='mongomock://localhost')
Stat.objects.create(name='size_of_collection',
data={
'user': 0,
'venue': 0,
'court': 0,
'feedback': 0,
'notification': 0,
'share_notification': 0,
'reservation': 0,
'course': 0
})
Venue.objects.create(name='name',
description='description',
img=None,
venue_id=1)
Court.objects.create(name='name', court_id=1, venue_id=1)
User.create(password='******',
user_id=1,
name='test',
api_id='2018000000',
privilege=1)
Court.objects.create(name='name', court_id=2, venue_id=1)
def create_app(config=None):
app = Flask(__name__)
with app.app_context():
if config is not None:
app.config.from_object(config)
else: app.config.from_object(Config)
disconnect(alias='default')
connect(
db=app.config['DB'],
alias='default',
host=app.config['MONGODB_HOST']
)
session.init_app(app)
login_manager.init_app(app)
login_manager.login_view = 'authorization.login'
from app.main import bp as main_bp
app.register_blueprint(main_bp)
from app.errors import bp as errors_bp
app.register_blueprint(errors_bp)
from app.posts import posts_bp
app.register_blueprint(posts_bp, url_prefix='/blog')
from app.authorization import auth_bp
app.register_blueprint(auth_bp)
return app
def app():
app = create_app({
'TESTING': True,
'JWT_ACCESS_TOKEN_EXPIRES': timedelta(hours=1),
'JWT_COOKIE_SECURE': False,
'JWT_SECRET_KEY': "test_key",
'JWT_TOKEN_LOCATION': ["cookies"],
'MONGODB_CONNECT': False
})
with app.app_context():
disconnect()
db = connect('test', host='mongomock://localhost')
test_user1 = User(username='******',
email='*****@*****.**',
password='******')
test_user1.save()
test_user2 = User(username='******',
email='*****@*****.**',
password='******')
test_user2.save()
jwt = JWTManager()
jwt.init_app(app)
@jwt.user_lookup_loader
def user_lookup_callback(_jwt_header, jwt_data):
identity = jwt_data["sub"]
return User.objects.get(email=identity)
yield app
# fixture cleanup
db.drop_database('test')
disconnect()
def connect_db(credentials_file=settings.DEFAULT_CREDENTIALS_FILE,
alias="default"):
"""Connects to MongoDB using credentials.
Args:
credentials_file (str): path to the credentials file,
or the Python dictionary containing the contents thereof.
alias (str): alias for the connection.
Returns:
A MongoEngine connection instance.
"""
# Disconnect if needed
try:
mongo.disconnect(alias=alias)
except:
pass
credentials = get_credentials(credentials_file)
# Return connection
return mongo.connect(credentials["database"],
host=credentials["host"],
port=credentials["port"],
username=credentials["username"],
password=credentials["password"])
def clean(self):
id = self.cleaned_data.get('id')
name = self.cleaned_data.get('name')
code = self.cleaned_data.get('code')
company = self.cleaned_data.get('company')
type = self.cleaned_data.get('costcenter_type')
disconnect()
connect('eagle')
if not id:
name = CostCenters.objects(name=name).filter(
company=company).filter(type=type).first()
code = CostCenters.objects(code=code).filter(
company=company).filter(type=type).first()
else:
name = CostCenters.objects.filter(name=name).filter(
company=company).filter(type=type).filter(id__ne=id).first()
code = CostCenters.objects.filter(code=code).filter(
company=company).filter(type=type).filter(id__ne=id).first()
disconnect()
if name:
self.add_error(
'name', 'Desculpe, este centro de custo já está cadastrado')
if code:
self.add_error('code', 'Desculpe, este código já está cadastrado')
def create_gametime_record(config_data, game_id, user_id, start_timestamp,
end_timestamp):
logging.basicConfig(format='%(levelname)s:%(message)s',
level=getattr(logging, config_data.logLevel.upper(),
None))
logging.debug('Logger loaded in create_gametime_record(...).')
logging.debug('Creating MongoDB connection.')
mongoengine.connect('gameplay_tracker',
host=config_data.databases.mongodb.host,
port=config_data.databases.mongodb.port,
username=config_data.databases.mongodb.username,
password=config_data.databases.mongodb.password,
authentication_source=config_data.databases.mongodb.
authentication_source)
game_record = Game.objects(game_id=game_id).first()
if game_record is None:
logging.error(f'{game_id} is not a valid GameID')
return -1
# TODO Pull user given UserID
user_record = User.objects(user_id=user_id).first()
if user_record is None:
logging.error(f'{user_id} is not a valid UserID')
game_time_record = GameTime(game_id=game_record.id,
user_id=user_record.id,
start_date=start_timestamp,
end_date=end_timestamp)
game_time_record.save()
mongoengine.disconnect()
def show_search_result(self):
"""Show search result on searching"""
print("show button1")
search_by_txt = self.admin_search_verb_page.search_verb_by_comboBox.currentText(
)
search_txt = self.admin_search_verb_page.search_verb_searchbar_lineEdit.text(
)
print("show button2")
print("Search by text:", search_by_txt)
print("Search text:", search_txt)
connect(db='VocabsEasyy', host='localhost', port=27017)
print('Connected successfully for searching')
try:
if search_by_txt == 'German Verb':
self.current_verb = Verb.objects(verb_name=search_txt).get()
self.fetch_result_from_db()
else:
self.current_verb = Verb.objects(
eng_meaning__contains=search_txt).get()
self.fetch_result_from_db()
except MultipleObjectsReturned as e:
self.admin_search_verb_result_window.hide()
self.popup_be_specific()
except DoesNotExist as e:
self.admin_search_verb_result_window.hide()
self.popup_no_results()
finally:
disconnect()
def clean():
db = connectdb()
db.drop_collection('buffer')
db.drop_collection('stream')
db.drop_collection('window')
db.drop_collection('processed')
disconnect('minibatch')
def get(self, pan):
try:
num = randrange(10)
if num in (8, 9):
raise BackendError
client = None
connect(db='SurePass', host=db_uri)
for clients in PanClient.objects:
if clients.pan == pan:
client = clients
break
#Exception for not able to find the client in the database
if client is None:
disconnect()
raise ClientNotFound
js = {
'pan': client.pan,
'name': client.name,
'dob': client.dob.strftime('%Y-%m-%d'),
'father_name': client.father_name,
'client_id': str(client.id)
}
disconnect()
return js
except Exception as e:
return {'msg': str(e)}
def setUp(self):
disconnect()
self.test_app = app.get_flask_app(test_config)
self.test_app.testing = True
self.test_app.debug = True
self.test_client = self.test_app.test_client()
connect('testing_db', is_mock=True)
def get(self, id):
try:
client = None
connect(db='SurePass', host=db_uri)
for clients in PanClient.objects:
if str(clients.id) == id:
client = clients
break
#Exception for not able to find the object id in the database
if client is None:
disconnect()
raise ObjectIDNotFound
js = {
'pan': client.pan,
'name': client.name,
'dob': client.dob.strftime('%Y-%m-%d'),
'father_name': client.father_name,
'client_id': str(client.id)
}
disconnect()
return js
except Exception as e:
return {'msg': str(e)}
def get_http_action_config(db_url: str, bot: str, action_name: str):
"""
Fetch HTTP action configuration parameters from the MongoDB database
:param db_url: MongoDB connection string
:param bot: BotID
:param action_name: Action name
:return: HttpActionConfig object containing configuration for the action
"""
if ActionUtility.is_empty(db_url) or ActionUtility.is_empty(
bot) or ActionUtility.is_empty(
action_name):
raise HttpActionFailure("Database url, bot name and action name are required")
try:
connect(host=db_url)
http_config_dict = HttpActionConfig.objects().get(bot=bot,
action_name=action_name).to_mongo().to_dict()
if dict is None:
raise DoesNotExist
except DoesNotExist:
raise HttpActionFailure("No HTTP action found for bot " + bot + " and action " + action_name)
except Exception as ex:
raise HttpActionFailure(ex)
finally:
disconnect()
return http_config_dict
def mongo(request):
"""Use this fixtures for setup and teardown database every test method.
How to use this fixture for a method:
```
def test_xxx(self, mongo):
```
How to use this fixture for every method on a class:
```
import pytest
from utils.fixtures import mongo
@pytest.mark.usefixtures('mongo')
class SiteTests:
```
"""
# Set up
disconnect()
db = connect("testdb", host="mongomock://localhost")
# Run test method
yield db
# Tear down
db.drop_database("testdb")
disconnect()
def flush_adj_task(last_date: str = None):
"""
刷新复权数据
:param last_date: 最后一个报告期对应日期
:return:
"""
job = AdjData()
connect('jiucai', host='master:17585')
code_list = [obj.code for obj in ShareDocument.objects().all()]
if last_date:
executed_code_list = AdjDocument.objects(
trade_date__gte=last_date).distinct("code")
code_list = list(set(code_list).difference(executed_code_list))
bar = tqdm(code_list)
for code in bar:
data = job.get_by_code(code)
# noinspection PyProtectedMember
bulk = [
UpdateOne({
"code": row.code,
"trade_date": row.trade_date
}, {"$set": row._asdict()},
upsert=True) for row in data
]
if bulk:
# noinspection PyProtectedMember
AdjDocument._get_collection().bulk_write(bulk, ordered=False)
bar.set_description(f"{code} flush {len(bulk)} records.")
disconnect()
def clean(self):
_username = self.cleaned_data.get('username')
_email = self.cleaned_data.get('email')
_id = self.cleaned_data.get('id')
disconnect()
connect('eagle')
if not _id:
user_username = Users.objects(username=_username).first()
user_email = Users.objects(email=_email).first()
else:
user_username = Users.objects.filter(username=_username).filter(
id__ne=_id).first()
user_email = Users.objects.filter(email=_email).filter(
id__ne=_id).first()
disconnect()
if user_username:
self.add_error(
'username',
'Desculpe, este nome de usuário já está cadastrado')
if user_email:
self.add_error('email', 'Desculpe, este e-mail já está cadastrado')
def process_data(self, file):
try:
csv_data = file.decode("utf-8")
reader = DictReader(StringIO(csv_data))
records = list(reader)
records_processed = 0
number_of_records = len(records)
for row in records:
record = create_record(row)
if record:
try:
record.save()
except Exception as e:
logging.error(e)
records_processed += 1
self.update_state(
state="PROGRESS",
meta={"done": records_processed, "total": number_of_records},
)
disconnect()
except Exception as ex:
self.update_state(
state=states.FAILURE,
meta={
"exc_type": type(ex).__name__,
"exc_message": traceback.format_exc().split("\n"),
},
)
raise ex
def fetchShortUrl(long_url):
alias_name = "_CREATE_" + str(random.randint(100000000, 999999999))
try:
db_name = "url_db"
connect(db=db_name, alias="default", host=os.environ["DB_URL"])
connect(db=db_name, alias=alias_name, host=os.environ["DB_URL"])
if long_url == "":
raise Exception("Long url cannot be empty")
maxNumOfChar = 6 #It can be changed dynamically based on the length of input string
#Regex to validate the url format (Can be done in client side)
# r0 = urlparse(long_url)
# print(r0)
# if not r0.scheme or not r0.netloc:
# raise Exception("Invalid URL passed")
#Check if any short_url corresponding to the longurl already exists
try:
object_fetched = Urls.objects.get(long_url=long_url)
return {
"status": "success",
"message": object_fetched["short_url"]
}
except:
pass
#Else try to create one
flag = True
retry = 3
while (retry > 0 and flag):
short_url = fake.lexify(text='?' * maxNumOfChar, letters=store)
try:
url_doc = Urls()
url_doc.short_url = short_url
url_doc.long_url = long_url
url_doc.created_at = time.time()
url_doc.save()
flag = False
except:
logging.warning("Retrying with retry #%s", str(retry))
retry = retry - 1
if not flag:
#successful insertion
return {"status": "success", "message": short_url}
return {
"status": "failure",
"message": "Something went wrong. Please try again"
}
disconnect(alias=alias_name)
disconnect(alias="default")
except Exception as e:
logging.warning("[services.url.shortener.fetchShortUrl] - %s", e)
return {"status": "failure", "message": str(e)}
