def _login_to_tracker(self, engine=None):
with DBSession() as db:
cred = db.query(LostFilmTVCredentials).first()
if not cred:
return False
username = cred.username
password = cred.password
if not username or not password:
return False
self.tracker.setup(cred.c_uid, cred.c_pass, cred.c_usess)
if self.tracker.verify():
if engine:
engine.log.info('Cookies are valid')
return True
if engine:
engine.log.info('Login to <b>lostfilm.tv</b>')
try:
self.tracker.login(username, password)
with DBSession() as db:
cred = db.query(LostFilmTVCredentials).first()
cred.c_uid = self.tracker.c_uid
cred.c_pass = self.tracker.c_pass
cred.c_usess = self.tracker.c_usess
except Exception as e:
if engine:
engine.log.failed(
'Login to <b>lostfilm.tv</b> failed: {0}'.format(
e.message))
return self.tracker.verify()
def create_one(author_id, content, title) -> ArticleModel:
article = ArticleModel(author_id=author_id,
content=content,
title=title)
DBSession().add(article)
DBSession().flush()
return article
def GetInstances(self, request: GetInstancesRequest,
context: ServicerContext,
claims: TokenClaims) -> GetInstancesResponse:
session = DBSession()
user = get_user_from_claims(session, claims)
instances = user.instances
session.close()
# find the instances
client = create_client()
resp = GetInstancesResponse()
for db_server in instances:
server = client.connection.get_server_by_id(db_server.id)
flavor = client.connection.get_flavor_by_id(server.flavor["id"])
i = Instance()
i.id = server.id
i.name = server.name
i.imageName = db_server.image_name
i.flavor.name = flavor.name
i.flavor.cpu = flavor.vcpus
i.flavor.memory = flavor.ram
i.flavor.rootDisk = flavor.disk
i.status = server.status
i.vmState = server.vm_state
i.powerState = str(server.power_state)
if server.task_state is not None:
i.taskState = server.task_state
i.ip = server.public_v4
i.createTime = server.created_at
resp.instances.append(i)
return resp
def get_one(**args) -> CommentModel:
query = DBSession().query(CommentModel)
if "id" in args:
query = query.filter(CommentModel.id == args["id"])
return query.first()
def delete_all_votes():
"""
WARNING: Sending a DELETE request to /delete-votes deletes ALL votes.
"""
# this query deletes all votes
DBSession.query(Vote).delete()
DBSession.commit()
return json.dumps({"result": "OK", "message" : "All votes have been deleted."})
def get_one(**args) -> ArticleModel:
query = DBSession().query(ArticleModel)
if "id" in args:
query = query.filter(ArticleModel.id == args["id"])
if "title" in args:
pass
return query.first()
def get_one(**args) -> AuthorModel:
query = DBSession().query(AuthorModel)
if "id" in args:
query = query.filter(AuthorModel.id == args["id"])
if "name" in args:
pass
return query.first()
def get_one(**args) -> OrdinaryWriterModel:
query = DBSession().query(OrdinaryWriterModel)
if "id" in args:
query = query.filter(OrdinaryWriterModel.id == args["id"])
if "author_id" in args:
query = query.filter(
OrdinaryWriterModel.author_id == args["author_id"])
return query.first()
def show_menu(restaurant_id: int):
session = DBSession()
restaurant = session.query(Restaurant).filter_by(id=restaurant_id).one_or_none()
items = session.query(MenuItem).filter_by(restaurant_id=restaurant_id).all()
return render_template(
"4_menu.html",
restaurant=restaurant,
items=items,
)
def batch_load_fn(self, ids):
query = DBSession().query(CommentModel).filter(
CommentModel.article_id.in_(ids))
article_comments = {}
for comment in query.all():
if comment.article_id in article_comments:
article_comments[comment.article_id].append(comment)
else:
article_comments[comment.article_id] = [comment]
return Promise.resolve(
[article_comments.get(article_id, None) for article_id in ids])
def __init__(self):
self.user_id = ''
self.db = DBSession()
self.session = requests.session()
self.corn_types = CORN_TYPE
self.corns = {}
# stop/active
self.status = 'stop'
self.warn_interval = WARN_INTERVAL
self.warn_time = None
self.mail = Mail()
self.session.headers.update(HEADER)
def delete_plot(id):
if id == None:
return 'Invalid ID'
plot = DBSession.query(Restraurant)\
.filter(Restraurant.id == id)\
.first()
if plot == None:
return 'Invalid ID'
if plot.created_by != session['user_id']:
return 'Invalid ID'
DBSession.delete(plot)
DBSession.commit()
return 'ok'
def get_list(_fields: list = None, _paginated: bool = False, **kwargs):
page = 1 if "page" not in kwargs else int(kwargs["page"])
limit = 20 if "limit" not in kwargs else int(kwargs["limit"])
query = DBSession().query(ArticleModel)
if "created_at_ge" in kwargs:
query = query.filter(
ArticleModel.created_at >= kwargs["created_at_ge"])
if "created_at_le" in kwargs:
query = query.filter(
ArticleModel.created_at <= kwargs["created_at_le"])
if _fields is not None:
query = query.with_entities(
*[getattr(ArticleModel, _field) for _field in _fields])
if limit != 0:
datas = (query.order_by(desc(
ArticleModel.created_at)).limit(limit).offset(
(page - 1) * limit))
else:
datas = query.order_by(desc(ArticleModel.created_at))
if _paginated:
total = query.count()
return datas, schemas.PageInfoSchema.paginate(total, page, limit)
else:
return datas
def delete_menu(restaurant_id: int, menu_id: int):
session = DBSession()
restaurant = session.query(Restaurant).filter_by(
id=restaurant_id).one_or_none()
item = (session.query(MenuItem).filter_by(
id=menu_id, restaurant_id=restaurant_id).one_or_none())
if restaurant is None:
return (
jsonify({"details": f"Restaurant {restaurant_id} does not exist"}),
404,
{},
)
if item is None:
return (
jsonify(
{
"details":
f"Menu item {menu_id} does not exist in restaurant {restaurant.name}",
}, ),
404,
{},
)
session.delete(item)
session.commit()
return (
None,
204,
{},
)
def cancel_result(message_id, user_id):
"""
This method is responsible to cancel sending particular sms if it is not yet delivered.
"""
transaction.commit()
message_id = int(message_id.decode(encoding='ascii'))
smses = DBSession.query(Sms).filter_by(sms_type='outgoing', id=message_id, user_id=user_id).first()
if(smses is None):
return False
elif(smses.status == 'delivered'):
return command_status.ESME_RCANCELFAIL
else:
DBSession.delete(smses)
transaction.commit()
return True
def get_quotes():
response.content_type = "application/json"
limit = request.query.limit or 100
offset = request.query.offset or 0
try:
limit = int(limit)
offset = int(offset)
assert 0 <= int(limit) <= 1000 or 0 < int(offset)
except Exception:
return dumps({"Error": "Too much you wish."})
session = DBSession()
quotes = session.query(Quote).order_by(func.random()).limit(limit).offset(offset).all()
return dumps(list(map(lambda x: x.jsonify(), quotes)))
def check_active_trades():
with app.app_context():
session = DBSession()
threads = []
users = session.query(User).filter_by(is_active=True).all()
for user in users:
if len(threads) >= NUM_WORKER_THREADS:
print("waiting for threads to join")
for thread in threads:
thread.join()
threads = []
thread = Thread(target=check_smart_trades_for_user, args=(user, ))
thread.daemon = True
thread.start()
threads.append(thread)
session.close()
def oauth_redirect():
print(session)
if request.args.get('state') != app.config['SESSION_STATE']:
return 'Invalid state'
data = urllib.request.urlopen(
'https://www.googleapis.com/oauth2/v4/token',
urllib.parse.urlencode({
'code': request.args.get('code'),
'client_id': app.config['CLIENT_ID'],
'client_secret': app.config['CLIENT_SECRET'],
'redirect_uri': app.config['REDICET_URL'],
'grant_type': 'authorization_code'
}).encode('utf8')).read()
data = json.loads(data.decode('utf8'))
id_token = data['id_token'].split('.')[1]
id_token = id_token + '=' * (4 - len(id_token) % 4
) # パディングが足りなかったりするっぽいので補う
id_token = base64.b64decode(id_token, '-_')
id_token = json.loads(id_token.decode('utf8'))
session['user_id'] = id_token['sub']
users = DBSession.query(User)\
.filter(User.id == id_token['sub'])\
.all()
print(session)
print(users)
if len(users) != 0:
session['name'] = users[0].name
return redirect(url_for('application'))
return redirect(url_for('register'))
def post_register():
if 'user_id' not in session or \
'name' not in request.form:
return 'Invalied Data'
name = request.form['name']
id = session['user_id']
user = User()
user.id = id
user.name = name
DBSession.add(user)
DBSession.commit()
session['name'] = name
return redirect(url_for('application'))
def connect_to_server(ip, port, system_id, password, system_type, recipient, message, sms_id, sender_number):
"""
This method is responsible to create client instance to communicate with destination network server as client.
"""
try:
client = SMPPClient(ip, port)
connection = False
notification = 0
while connection is False: # while connection is not established with server, try to connect.
if client.connect():
connection = True
# this thread is checking the socket for getting responses from other server and save in dictionary.
background_thread4 = threading.Thread(target=client.session.storing_recieved_pdus, args=()) # to recieve pdu response from other smpp server to whom it has sent the request.
background_thread4.start()
if client.login('TX', system_id, password, system_type): # if client successfully logins
client.session.send_sms(recipient, message, sender_number)
# till client receives no response for sending sms
while notification == 0:
notification = client.session.notifications_4_client()
client.session.processing_recieved_pdus()
smses = DBSession.query(Sms).filter_by(id=int(sms_id)).first()
if smses:
smses.status = 'delivered'
client.session.unbind()
while client.session.state != SessionState.UNBOUND:
client.session.processing_recieved_pdus()
client.sc.close()
transaction.commit()
background_thread4.join()
except:
pass
def manageSession(request):
session = DBSession()
def closeSession(request):
session.close()
request.add_finished_callback(closeSession)
return session
def set_plot():
if 'x' not in request.form or \
'y' not in request.form or \
'comment' not in request.form or \
'name' not in request.form :
return 'Invalid Form'
plot = Restraurant()
plot.x = request.form['x']
plot.y = request.form['y']
plot.comment = request.form['comment']
plot.name = request.form['name']
plot.created_by = session['user_id']
DBSession.add(plot)
DBSession.commit()
return 'ok'
def remove_watch(self, url):
with DBSession() as db:
topic = db.query(RutorOrgTopic).filter(
RutorOrgTopic.url == url).first()
if topic is None:
return False
db.delete(topic)
return True
def __init__(self, session=None, autoinit=True):
if not autoinit:
self._session = None
elif session is None or not isinstance(session, Union[Session, scoped_session].__args__):
self._session: scoped_session = DBSession()
else:
self._session = session
self._session.expire_on_commit = False
def get_next_timestamp_by_path(path):
with DBSession() as session:
storage_file = session.query(Storage).filter(
Storage.path == path).one()
sql_map = session.query(Map).get(storage_file.map_id)
sql_file = found_next_map_in_storage(
session, sql_map, storage_file.timestamp + timedelta(seconds=1))
return sql_file.timestamp if sql_file else None
def set_settings(self, settings):
with DBSession() as db:
cred = db.query(LostFilmTVCredentials).first()
if not cred:
cred = LostFilmTVCredentials()
db.add(cred)
cred.username = settings['username']
cred.password = settings['password']
def list_votes():
"""
Get the list of all votes.
"""
votes = DBSession.query(Vote).order_by(Vote.id)
json_data = {'votes': [vote.as_dict() for vote in votes]}
return json.dumps(json_data)
def remove_watch(self, url):
with DBSession() as db:
topic = db.query(LostFilmTVSeries).filter(
LostFilmTVSeries.url == url).first()
if topic is None:
return False
db.delete(topic)
return True
def render_GET(self, request):
# self.count +=1
sessionweb = DBSession()
spiderRule = sessionweb.query(SpiderRule).filter(
SpiderRule.enable and SpiderRule.cron == None).one()
# spiderRule.name = '%s-%s'%(spiderRule.name,self.count)
# print('我来了%s次!!!'%spiderRule.name)
self.scheduler.add_job(crawl,
'date',
args=[spiderRule],
name=spiderRule.name,
id='%s' % spiderRule.id,
replace_existing=True)
sessionweb.close()
request.setHeader("Content-Type", "text/html; charset=utf-8")
return ("<html>Hello, world!</html>").encode('utf-8')
def get_fees(symbol, side, start, end):
with DBSession() as session:
query = session.query(func.sum(Order.fill_fees)). \
filter(Order.symbol == symbol). \
filter(Order.side == side)
if start and end:
query = query.filter(Order.created_at >= start,
Order.created_at < end)
return float(query.scalar())
def list_people():
"""
Get the list of all people ordered by name.
"""
people = DBSession.query(Person).order_by(Person.name)
# this is more compact, using a list comprehension.
json_data = {'people': [person.as_dict() for person in people]}
return json.dumps(json_data)
def update_one(article_id, **args) -> ArticleModel:
article = ArticleManager.get_one(article_id=article_id)
if "content" in args:
article.content = args["content"]
if "title" in args:
article.title = args["title"]
DBSession().flush()
return article
def fetch_incoming_sms(user_id):
"""
This method query database to retrieve pending incoming smses for logged in client..
"""
smses = DBSession.query(Sms).filter_by(sms_type='incoming', user_id=user_id, status='recieved').first()
if smses:
smses.status = 'delivered'
return(smses)
def save_data(self):
if self.resp:
for item in self.resp:
flag = DBSession.save( item )
if not flag:
print 'write failed with data=', json.dumps( item )
else:
print 'write successed '
else:
print 'error with no self.resp or self.resp is not list type'
def db_storage(recipients, message, user_id, sender):
"""
This method is responsible to store Sms and related fields provided by client in database
"""
sms_ids = '' # contains all message ids that are sended by client to recipient of same or different network.
#smses_ids = '' # contains only message ids that are sended by client to recipient of some other network.
sender = sender.decode(encoding='ascii')
recipients = recipients.decode(encoding='ascii').splitlines()
selected_package = selected_packages(user_id)
for recipient in recipients:
processed_fields = process_outgoing_sms(sender, user_id, recipient)
# storing vaues to database
S = Sms()
S.sms_type = 'outgoing'
S.sms_from = processed_fields['sender_number']
S.sms_to = recipient
S.schedule_delivery_time = datetime.date.today()
S.validity_period = datetime.date.today()+datetime.timedelta(days=1)
S.msg = message
S.timestamp = datetime.date.today()
S.status = 'scheduled'
S.msg_type = 'text'
S.user_id = user_id
S.package_name = selected_package['package_name']
S.rates = selected_package['rates']
S.target_network = processed_fields['target_network'] # process sms file would use it to send to respective network of which server is.
S.client_type = 'smpp'
DBSession.add(S)
transaction.commit()
sms = DBSession.query(Sms)[-1] # to send id to the client for ancilliary operations and querying.
sms_ids = sms_ids + str(sms.id) + '\n'
if processed_fields['target_network'] != processed_fields['source_network']: # if destination and source network is different
connect_info(recipient, message, processed_fields['target_network'], sms.id,
processed_fields['sender_number']) # connect to the destination's smpp server.
#smses_ids = smses_ids + str(s) + '\n'
#updating_status(smses_ids)
return(sms_ids)
def updating_status(sms_ids):
"""
This method is responsible to update status of messages from scheduled to delivered when messages are for server of
other network.
"""
if sms_ids:
sms_ids = sms_ids.splitlines()
print(sms_ids)
for sms_id in sms_ids:
smses = DBSession.query(Sms).filter_by(id=int(sms_id)).first()
smses.status = 'delivered'
transaction.commit()
def selected_packages(user_id):
"""
This method is used by server to ensure if logged in user has selected any package or not.
"""
total_selected_package = DBSession.query(Selected_package).filter_by(user_id=user_id).count()
if(total_selected_package > 0):
selected_package = DBSession.query(Selected_package).filter_by(user_id=user_id)[-1] # retrieve last selected package
else:
selected_package = None
if(selected_package is None):
package_name = None
rates = 1.5
else:
end_date = int(selected_package.end_date.strftime('%d'))
end_month = int(selected_package.end_date.strftime('%m'))
end_year = int(selected_package.end_date.strftime('%y'))
date = datetime.datetime.now()
today_date = int(date.strftime('%d'))
today_month = int(date.strftime('%m'))
today_year = int(date.strftime('%y'))
if(end_year > today_year and int(selected_package.smses) > 0):
package_name = selected_package.package_name
rates = 0.0
selected_package.smses = selected_package.smses-1
elif(end_year == today_year and end_month > today_month and int(selected_package.smses) > 0):
package_name = selected_package.package_name
rates = 0.0
selected_package.smses = selected_package.smses-1
elif(end_month == today_month):
if(end_date >= today_date and int(selected_package.smses) > 0):
package_name = selected_package.package_name
rates = 0.0
selected_package.smses = selected_package.smses-1
else:
package_name = None
rates = 1.5
return(dict(package_name=package_name, rates=rates))
def process_incoming_sms():
"""
This method is responsible for processing incoming smses.
"""
sms_ids = ''
smses = DBSession.query(Sms).filter_by(sms_type='incoming', status='recieved', target_network=None, user_id=None).all()
if smses:
for S in smses:
sms_to = S.sms_to
sms_from = S.sms_from
mnp = DBSession.query(Mnp).filter_by(cell_number=sms_to).first() # querying for mobile number conversion
if mnp:
target_network = mnp.target_network
S.sms_type = 'outgoing'
S.status = 'scheduled'
S.validity_period = datetime.date.today()+datetime.timedelta(days=1)
else:
source_prefix = sms_from[0:6] # extract prefix of sender number
dest_prefix = sms_to[0:6] # extract prefix of recipient
s_network = DBSession.query(Prefix_Match).filter_by(prefix=source_prefix).first()
source_network = s_network.network # refers to sender network, getting the network of source from prefixes
d_network = DBSession.query(Prefix_Match).filter_by(prefix=dest_prefix).first()
dest_network = d_network.network # refers to recipient network
user = DBSession.query(User_Number).filter_by(cell_number=sms_to).first() # user refers to normal user
if user:
S.user_id = user.user_id
target_network = None # bECAUSE IT IS FOR CLIENT
else:
# IF network is same or different.
target_network = dest_network # member variable
S.sms_type = 'outgoing'
S.status = 'scheduled'
S.validity_period = datetime.date.today()+datetime.timedelta(days=1)
S.target_network = target_network # from sms table
if target_network != source_network:
connect_info(sms_to, S.msg, dest_network, S.id, sms_from)
def query_result(message_id, user_id):
"""
This method is responsible to query database for provided message id to view the status of Sms
"""
message_id = int(message_id.decode(encoding='ascii'))
smses = DBSession.query(Sms).filter_by(sms_type='outgoing', id=message_id, user_id=user_id).first() #check for all outgoing sms
if(smses is None):
return(command_status.ESME_RINVMSGID)
elif(smses.status == 'delivered'):
return(dict(state=message_state.DELIVERED, final_date=smses.validity_period))
elif(smses.status == 'scheduled' and smses.validity_period >= datetime.datetime.now()):
return(dict(state=message_state.SCHEDULED, final_date=smses.validity_period))
elif(smses.status == 'scheduled' and smses.validity_period < datetime.datetime.now()):
return(dict(state=message_state.EXPIRED, final_date=smses.validity_period))
def validate(system_id, password, system_type):
"""
This method is used by server to validate credentials provided by client against database
"""
system_id = system_id.decode(encoding='ascii')
passhash = hashlib.sha1(bytes(password.decode(encoding='ascii'), encoding="utf8")).hexdigest()
system_type = system_type.decode(encoding='ascii')
record = DBSession.query(User).filter_by(user_id=system_id, password=passhash, system_type=system_type).first()
if(record):
print("Credentials Validated successfully!!!")
return 'True'
else:
print("Validation failed")
return 'false'
def process_outgoing_sms(sender, user_id, recipient):
"""
This method is responsible for processing outgoing smses.
"""
if sender is not '':
sender_number = sender
else:
user = DBSession.query(User_Number).filter_by(user_id=user_id).first() # user refers to normal user
sender_number = user.cell_number # cell number of sender
source_prefix = sender_number[0:6] # extract prefix of sender number
s_network = DBSession.query(Prefix_Match).filter_by(prefix=source_prefix).first()
source_network = s_network.network # refers to sender network, getting the network of source from prefixes
dest_prefix = recipient[0:6] # extract prefix of recipient
d_network = DBSession.query(Prefix_Match).filter_by(prefix=dest_prefix).first()
dest_network = d_network.network # refers to recipient network
mnp = DBSession.query(Mnp).filter_by(cell_number=recipient).first() # querying for mobile number conversion
if mnp:
target_network = mnp.target_network
elif(source_network == dest_network):
target_network = dest_network
else:
target_network = dest_network
return(dict(sender_number=sender_number, source_network=source_network, target_network=target_network))
def tween (request): try: response = handler(request) DBSession.commit() except: exc_info = sys.exc_info() #workaround for DBSession.rollback() breaking traceback DBSession.rollback() raise exc_info[0], exc_info[1], exc_info[2] finally: DBSession.remove() return response
def replace_result(message_id, message, user_id):
"""
This method is responsible to replace particular message which is not yet delivered.
"""
message_id = int(message_id.decode(encoding='ascii'))
message = message.decode(encoding='ascii')
smses = DBSession.query(Sms).filter_by(sms_type='outgoing', id=message_id, user_id=user_id).first()
if(smses is None):
return False
elif(smses.status == 'delivered'):
return(command_status.ESME_RREPLACEFAIL)
elif(smses.status == 'scheduled'):
smses.schedule_delivery_time = datetime.datetime.now()
smses. validity_period = datetime.datetime.now()+datetime.timedelta(days=1)
smses.msg = message
transaction.commit()
return True
def connect_info(recipient, message, dest_network, sms_id, sender_number):
"""
This method is responsible to get connection paramters from database to communicate with server of destination network
in case when sender sends a message to some other network.
"""
# here we have to make client object
server = DBSession.query(Network).filter_by(network=dest_network).first()
if server:
print("Establishing Connection with " + server.network.upper() + " Server...")
system_id = server.username
password = server.password
system_type = server.system_type
ip = server.ip
port = server.port
# this thread is responsible for connecting with other smpp server.
background_thread3 = threading.Thread(target=connect_to_server,
args=(ip, port, system_id, password, system_type, recipient, message,
sms_id, sender_number))
background_thread3.start()
background_thread3.join()
'''This background process is responsible for reading the outgoing smses from database sent by client
and post them to the url on bottle server
'''
import urllib2
import urllib
import db
import transaction
from db import DBSession
from models import Sms, User
import datetime
import time
try:
db.bind_session()
while True:
smses = DBSession.query(Sms).filter_by(status='scheduled', sms_type='outgoing', target_network='zong').all()
if(smses):
for S in smses:
sms_to = S.sms_to
sms_from = S.sms_from
message = S.msg
post_data = {'sms_to': sms_to, 'sms_from': sms_from, 'message': message}
post_str = urllib.urlencode(post_data)
# urlencode converts the dict to a string suitable for POST
# Note when some data is specified in urlopen then HTTP request type
# POST is used instead of GET request type
result = urllib2.urlopen("http://192.168.5.34:50111/" + 'sendsms', post_str).read()
S.status = 'delivered'
S.timestamp = datetime.date.today()
print(result)
transaction.commit()
S = Sms() S.sms_type = 'outgoing' S.sms_from = '+9233365195924' S.sms_to = '+923366767999' S.schedule_delivery_time = d # give date 2nd december S.validity_period = d+datetime.timedelta(days=1) S.msg = "dont disturb.." S.timestamp = d S.status = 'delivered' S.msg_type = 'text' S.user_id = 'ASMA' S.package_name = 'dhamaka' S.rates = 0.0 S.target_network = 'ufone' # process sms file would use it to send to respective network of which server is. S.client_type = 'smpp' DBSession.add(S) #user = DBSession.query(User_Number).filter_by(user_id='ASMA').first() # user refers to normal user #cell_number = user.cell_number #source_prefix = cell_number[0:6] #dest_prefix = '+92300' #source_network = DBSession.query(Prefix_Match).filter_by(prefix=source_prefix).first() # t_user refers to network #dest_network = DBSession.query(Prefix_Match).filter_by(prefix=dest_prefix).first() # t_user refers to network #print(source_network.network) #print(dest_network.network) transaction.commit() #data = b'\x00\x00\x00/\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01SMPP3TEST\x00secret08\x00' + \ #b'SUBMIT1\x00P\x01\x01\x00' #P = BindTransmitter.decode(data) #print(DBSession.query(User).count()) #system=P.system_id.value.decode(encoding='ascii') #for U in DBSession.query(User).all():
def process_steps (test_id, tests_cache, registry, step_ques, finish_que, workers_last_activity, timers):
#print "tick", test_id
#TODO what if this gets interrupted by kill during io?
#TODO dont forget actual finish time
test = tests_cache[test_id]
#new_rx = get_rx()
#rx = new_rx - test['rx_snapshot']
#test['rx_snapshot'] = new_rx
#new_tx = get_tx()
#tx = new_tx - test['tx_snapshot']
#test['tx_snapshot'] = new_tx
connect_errors = []
while True:
steps = step_ques[test_id].next()
if steps:
buf_statuses = defaultdict(int)
buf_resp_time = defaultdict(list)
buf_conn_time = defaultdict(list)
buf_errors = defaultdict(int)
buf_concur_users_num_max = 0
buf_concur_users_num_min = 0
buf_concur_conns_num_min = 0
buf_concur_conns_num_max = 0
buf_start_session = 0
buf_request_sent = 0
is_finish_only_step = all(len(data) == 1 and data[0]['type'] == stypes.FINISH_TEST for _node_id, data in steps.items())
for node_id, data in steps.items():
node_id = int(node_id)
workers_last_activity[test_id][node_id] = time.time()
for rec in data:
data_type = rec['type']
if data_type == stypes.RESPONSE_STATUS:
grp, status = rec['value']
if int(status) not in (200, 201, 202): #TODO move to test logic
test['resp_bad_statuses_total'] += 1
else:
test['resp_successful_total'] += 1
status = str(status) + " " + grp
tests_cache[test_id]['resp_statuses'].add(status)
buf_statuses[status] += 1
elif data_type == stypes.RESPONSE_TIME:
grp_name, resp_time = rec['value']
buf_resp_time[grp_name].append(resp_time)
tests_cache[test_id]['groups'].add(grp_name)
elif data_type == stypes.CONNECT_TIME:
test['conns_total'] += 1
grp_name, timelen = rec['value']
buf_conn_time[grp_name].append(timelen)
tests_cache[test_id]['groups'].add(grp_name)
elif data_type == stypes.CONCUR_USERS_NUM_MAX:
buf_concur_users_num_max += rec['value']
elif data_type == stypes.CONCUR_USERS_NUM_MIN:
buf_concur_users_num_min += rec['value']
elif data_type == stypes.CONCUR_CONNS_NUM_MIN:
buf_concur_conns_num_min += rec['value']
elif data_type == stypes.CONCUR_CONNS_NUM_MAX:
buf_concur_conns_num_max += rec['value']
elif data_type == stypes.START_SESSION:
buf_start_session += rec['value']
elif data_type == stypes.REQUEST_SENT:
test['reqs_total'] += rec['value']
buf_request_sent += rec['value']
elif data_type == stypes.CONNECT_ERROR:
test['conns_errors_total'] += 1
try:
tests_cache[test_id]['errors'].add("connect " + rec['value']['msg'])
except:
print "rec:", rec
raise
buf_errors[rec['value']['msg']] += 1
if "not enough ports" not in rec['value']['msg']:
connect_errors.append(u'%s\t%s\t%s' % (rec['value']['time'], rec['value']['ip'], rec['value']['msg']))
elif data_type == stypes.RESPONSE_ERROR:
if "timeout" in rec['value']:
test['resp_timeouts_total'] += 1
else:
test['resp_errors_total'] += 1
ern = "response " + rec['value']
tests_cache[test_id]['errors'].add(ern)
buf_errors[ern] += 1
elif data_type == stypes.FINISH_TEST:
finish_que[test_id][node_id] = True
else:
raise NotImplementedError(rec['type'])
if not is_finish_only_step:
res = test['result']
rt = {}
rm = {}
for grp, times in buf_resp_time.iteritems():
resp_time_med = util.get_median(times)
rt[grp] = resp_time_med
rm[grp] = util.get_median(abs(t - resp_time_med) for t in times)
res['resp_time'].append(rt)
res['resp_time_meav'].append(rm) #TODO rename to med_abs_dev
rt = {}
rm = {}
for grp, times in buf_conn_time.iteritems():
resp_time_med = util.get_median(times)
rt[grp] = resp_time_med
rm[grp] = util.get_median(abs(t - resp_time_med) for t in times)
res['conn_time'].append(rt)
res['conn_time_meav'].append(rm) #TODO rename to med_abs_dev
res['start_session'].append(buf_start_session)
res['resp_status'].append(buf_statuses)
res['req_sent'].append(buf_request_sent)
res['errors'].append(buf_errors)
res['concur_users_num_max'].append(buf_concur_users_num_max)
res['concur_users_num_min'].append(buf_concur_users_num_min)
res['concur_conns_num_min'].append(buf_concur_conns_num_min)
res['concur_conns_num_max'].append(buf_concur_conns_num_max)
#res['network_received'].append(float(rx) / 1024.0)
#res['network_sent'].append(float(tx) / 1024.0)
else:
break
if connect_errors and test['write_connect_error_log']:
with open("conn_err_%s.log" % test_id, 'ab') as f:
f.write("\n".join(connect_errors).encode('utf-8') + "\n")
is_crashed = False
now = time.time()
for _node_id, ts_last in workers_last_activity[test_id].items():
if now - ts_last >= WORKERS_TIMEOUT:
is_crashed = True
break
is_finished = len(finish_que[test_id]) == test['worker_num']
if is_crashed:
is_finished = True
if is_finished:
tests_cache[test_id]['finished'] = (now - WORKERS_TIMEOUT) if is_crashed else now
#DBSession.query(Test).filter_by(id = test_id).update({Test.data: dbdump(tests_cache[test_id])})
t = Test.query.filter_by(id = test_id).first()
if not t:
raise Exception("no test with id = %s (maybe it was deleted before finish timeout happened?)" % test_id)
t.data = dbdump(tests_cache[test_id])
DBSession.add(t)
DBSession.commit()
if registry.has_listeners:
registry.notify(OnFinishTest(t, registry.settings))
del tests_cache[test_id]
del finish_que[test_id]
del step_ques[test_id]
del workers_last_activity[test_id]
gevent.kill(timers[test_id])
del timers[test_id]
log.info("finished test #%s%s" % (test_id, " (timeout)" if is_crashed else ""))
