def _login_to_tracker(self, engine=None): with DBSession() as db: cred = db.query(LostFilmTVCredentials).first() if not cred: return False username = cred.username password = cred.password if not username or not password: return False self.tracker.setup(cred.c_uid, cred.c_pass, cred.c_usess) if self.tracker.verify(): if engine: engine.log.info('Cookies are valid') return True if engine: engine.log.info('Login to <b>lostfilm.tv</b>') try: self.tracker.login(username, password) with DBSession() as db: cred = db.query(LostFilmTVCredentials).first() cred.c_uid = self.tracker.c_uid cred.c_pass = self.tracker.c_pass cred.c_usess = self.tracker.c_usess except Exception as e: if engine: engine.log.failed( 'Login to <b>lostfilm.tv</b> failed: {0}'.format( e.message)) return self.tracker.verify()
def create_one(author_id, content, title) -> ArticleModel: article = ArticleModel(author_id=author_id, content=content, title=title) DBSession().add(article) DBSession().flush() return article
def GetInstances(self, request: GetInstancesRequest, context: ServicerContext, claims: TokenClaims) -> GetInstancesResponse: session = DBSession() user = get_user_from_claims(session, claims) instances = user.instances session.close() # find the instances client = create_client() resp = GetInstancesResponse() for db_server in instances: server = client.connection.get_server_by_id(db_server.id) flavor = client.connection.get_flavor_by_id(server.flavor["id"]) i = Instance() i.id = server.id i.name = server.name i.imageName = db_server.image_name i.flavor.name = flavor.name i.flavor.cpu = flavor.vcpus i.flavor.memory = flavor.ram i.flavor.rootDisk = flavor.disk i.status = server.status i.vmState = server.vm_state i.powerState = str(server.power_state) if server.task_state is not None: i.taskState = server.task_state i.ip = server.public_v4 i.createTime = server.created_at resp.instances.append(i) return resp
def get_one(**args) -> CommentModel: query = DBSession().query(CommentModel) if "id" in args: query = query.filter(CommentModel.id == args["id"]) return query.first()
def delete_all_votes(): """ WARNING: Sending a DELETE request to /delete-votes deletes ALL votes. """ # this query deletes all votes DBSession.query(Vote).delete() DBSession.commit() return json.dumps({"result": "OK", "message" : "All votes have been deleted."})
def get_one(**args) -> ArticleModel: query = DBSession().query(ArticleModel) if "id" in args: query = query.filter(ArticleModel.id == args["id"]) if "title" in args: pass return query.first()
def get_one(**args) -> AuthorModel: query = DBSession().query(AuthorModel) if "id" in args: query = query.filter(AuthorModel.id == args["id"]) if "name" in args: pass return query.first()
def get_one(**args) -> OrdinaryWriterModel: query = DBSession().query(OrdinaryWriterModel) if "id" in args: query = query.filter(OrdinaryWriterModel.id == args["id"]) if "author_id" in args: query = query.filter( OrdinaryWriterModel.author_id == args["author_id"]) return query.first()
def show_menu(restaurant_id: int): session = DBSession() restaurant = session.query(Restaurant).filter_by(id=restaurant_id).one_or_none() items = session.query(MenuItem).filter_by(restaurant_id=restaurant_id).all() return render_template( "4_menu.html", restaurant=restaurant, items=items, )
def batch_load_fn(self, ids): query = DBSession().query(CommentModel).filter( CommentModel.article_id.in_(ids)) article_comments = {} for comment in query.all(): if comment.article_id in article_comments: article_comments[comment.article_id].append(comment) else: article_comments[comment.article_id] = [comment] return Promise.resolve( [article_comments.get(article_id, None) for article_id in ids])
def __init__(self): self.user_id = '' self.db = DBSession() self.session = requests.session() self.corn_types = CORN_TYPE self.corns = {} # stop/active self.status = 'stop' self.warn_interval = WARN_INTERVAL self.warn_time = None self.mail = Mail() self.session.headers.update(HEADER)
def delete_plot(id): if id == None: return 'Invalid ID' plot = DBSession.query(Restraurant)\ .filter(Restraurant.id == id)\ .first() if plot == None: return 'Invalid ID' if plot.created_by != session['user_id']: return 'Invalid ID' DBSession.delete(plot) DBSession.commit() return 'ok'
def get_list(_fields: list = None, _paginated: bool = False, **kwargs): page = 1 if "page" not in kwargs else int(kwargs["page"]) limit = 20 if "limit" not in kwargs else int(kwargs["limit"]) query = DBSession().query(ArticleModel) if "created_at_ge" in kwargs: query = query.filter( ArticleModel.created_at >= kwargs["created_at_ge"]) if "created_at_le" in kwargs: query = query.filter( ArticleModel.created_at <= kwargs["created_at_le"]) if _fields is not None: query = query.with_entities( *[getattr(ArticleModel, _field) for _field in _fields]) if limit != 0: datas = (query.order_by(desc( ArticleModel.created_at)).limit(limit).offset( (page - 1) * limit)) else: datas = query.order_by(desc(ArticleModel.created_at)) if _paginated: total = query.count() return datas, schemas.PageInfoSchema.paginate(total, page, limit) else: return datas
def delete_menu(restaurant_id: int, menu_id: int): session = DBSession() restaurant = session.query(Restaurant).filter_by( id=restaurant_id).one_or_none() item = (session.query(MenuItem).filter_by( id=menu_id, restaurant_id=restaurant_id).one_or_none()) if restaurant is None: return ( jsonify({"details": f"Restaurant {restaurant_id} does not exist"}), 404, {}, ) if item is None: return ( jsonify( { "details": f"Menu item {menu_id} does not exist in restaurant {restaurant.name}", }, ), 404, {}, ) session.delete(item) session.commit() return ( None, 204, {}, )
def cancel_result(message_id, user_id): """ This method is responsible to cancel sending particular sms if it is not yet delivered. """ transaction.commit() message_id = int(message_id.decode(encoding='ascii')) smses = DBSession.query(Sms).filter_by(sms_type='outgoing', id=message_id, user_id=user_id).first() if(smses is None): return False elif(smses.status == 'delivered'): return command_status.ESME_RCANCELFAIL else: DBSession.delete(smses) transaction.commit() return True
def get_quotes(): response.content_type = "application/json" limit = request.query.limit or 100 offset = request.query.offset or 0 try: limit = int(limit) offset = int(offset) assert 0 <= int(limit) <= 1000 or 0 < int(offset) except Exception: return dumps({"Error": "Too much you wish."}) session = DBSession() quotes = session.query(Quote).order_by(func.random()).limit(limit).offset(offset).all() return dumps(list(map(lambda x: x.jsonify(), quotes)))
def check_active_trades(): with app.app_context(): session = DBSession() threads = [] users = session.query(User).filter_by(is_active=True).all() for user in users: if len(threads) >= NUM_WORKER_THREADS: print("waiting for threads to join") for thread in threads: thread.join() threads = [] thread = Thread(target=check_smart_trades_for_user, args=(user, )) thread.daemon = True thread.start() threads.append(thread) session.close()
def oauth_redirect(): print(session) if request.args.get('state') != app.config['SESSION_STATE']: return 'Invalid state' data = urllib.request.urlopen( 'https://www.googleapis.com/oauth2/v4/token', urllib.parse.urlencode({ 'code': request.args.get('code'), 'client_id': app.config['CLIENT_ID'], 'client_secret': app.config['CLIENT_SECRET'], 'redirect_uri': app.config['REDICET_URL'], 'grant_type': 'authorization_code' }).encode('utf8')).read() data = json.loads(data.decode('utf8')) id_token = data['id_token'].split('.')[1] id_token = id_token + '=' * (4 - len(id_token) % 4 ) # パディングが足りなかったりするっぽいので補う id_token = base64.b64decode(id_token, '-_') id_token = json.loads(id_token.decode('utf8')) session['user_id'] = id_token['sub'] users = DBSession.query(User)\ .filter(User.id == id_token['sub'])\ .all() print(session) print(users) if len(users) != 0: session['name'] = users[0].name return redirect(url_for('application')) return redirect(url_for('register'))
def post_register(): if 'user_id' not in session or \ 'name' not in request.form: return 'Invalied Data' name = request.form['name'] id = session['user_id'] user = User() user.id = id user.name = name DBSession.add(user) DBSession.commit() session['name'] = name return redirect(url_for('application'))
def connect_to_server(ip, port, system_id, password, system_type, recipient, message, sms_id, sender_number): """ This method is responsible to create client instance to communicate with destination network server as client. """ try: client = SMPPClient(ip, port) connection = False notification = 0 while connection is False: # while connection is not established with server, try to connect. if client.connect(): connection = True # this thread is checking the socket for getting responses from other server and save in dictionary. background_thread4 = threading.Thread(target=client.session.storing_recieved_pdus, args=()) # to recieve pdu response from other smpp server to whom it has sent the request. background_thread4.start() if client.login('TX', system_id, password, system_type): # if client successfully logins client.session.send_sms(recipient, message, sender_number) # till client receives no response for sending sms while notification == 0: notification = client.session.notifications_4_client() client.session.processing_recieved_pdus() smses = DBSession.query(Sms).filter_by(id=int(sms_id)).first() if smses: smses.status = 'delivered' client.session.unbind() while client.session.state != SessionState.UNBOUND: client.session.processing_recieved_pdus() client.sc.close() transaction.commit() background_thread4.join() except: pass
def manageSession(request): session = DBSession() def closeSession(request): session.close() request.add_finished_callback(closeSession) return session
def set_plot(): if 'x' not in request.form or \ 'y' not in request.form or \ 'comment' not in request.form or \ 'name' not in request.form : return 'Invalid Form' plot = Restraurant() plot.x = request.form['x'] plot.y = request.form['y'] plot.comment = request.form['comment'] plot.name = request.form['name'] plot.created_by = session['user_id'] DBSession.add(plot) DBSession.commit() return 'ok'
def remove_watch(self, url): with DBSession() as db: topic = db.query(RutorOrgTopic).filter( RutorOrgTopic.url == url).first() if topic is None: return False db.delete(topic) return True
def __init__(self, session=None, autoinit=True): if not autoinit: self._session = None elif session is None or not isinstance(session, Union[Session, scoped_session].__args__): self._session: scoped_session = DBSession() else: self._session = session self._session.expire_on_commit = False
def get_next_timestamp_by_path(path): with DBSession() as session: storage_file = session.query(Storage).filter( Storage.path == path).one() sql_map = session.query(Map).get(storage_file.map_id) sql_file = found_next_map_in_storage( session, sql_map, storage_file.timestamp + timedelta(seconds=1)) return sql_file.timestamp if sql_file else None
def set_settings(self, settings): with DBSession() as db: cred = db.query(LostFilmTVCredentials).first() if not cred: cred = LostFilmTVCredentials() db.add(cred) cred.username = settings['username'] cred.password = settings['password']
def list_votes(): """ Get the list of all votes. """ votes = DBSession.query(Vote).order_by(Vote.id) json_data = {'votes': [vote.as_dict() for vote in votes]} return json.dumps(json_data)
def remove_watch(self, url): with DBSession() as db: topic = db.query(LostFilmTVSeries).filter( LostFilmTVSeries.url == url).first() if topic is None: return False db.delete(topic) return True
def render_GET(self, request): # self.count +=1 sessionweb = DBSession() spiderRule = sessionweb.query(SpiderRule).filter( SpiderRule.enable and SpiderRule.cron == None).one() # spiderRule.name = '%s-%s'%(spiderRule.name,self.count) # print('我来了%s次!!!'%spiderRule.name) self.scheduler.add_job(crawl, 'date', args=[spiderRule], name=spiderRule.name, id='%s' % spiderRule.id, replace_existing=True) sessionweb.close() request.setHeader("Content-Type", "text/html; charset=utf-8") return ("<html>Hello, world!</html>").encode('utf-8')
def get_fees(symbol, side, start, end): with DBSession() as session: query = session.query(func.sum(Order.fill_fees)). \ filter(Order.symbol == symbol). \ filter(Order.side == side) if start and end: query = query.filter(Order.created_at >= start, Order.created_at < end) return float(query.scalar())
def list_people(): """ Get the list of all people ordered by name. """ people = DBSession.query(Person).order_by(Person.name) # this is more compact, using a list comprehension. json_data = {'people': [person.as_dict() for person in people]} return json.dumps(json_data)
def update_one(article_id, **args) -> ArticleModel: article = ArticleManager.get_one(article_id=article_id) if "content" in args: article.content = args["content"] if "title" in args: article.title = args["title"] DBSession().flush() return article
def fetch_incoming_sms(user_id): """ This method query database to retrieve pending incoming smses for logged in client.. """ smses = DBSession.query(Sms).filter_by(sms_type='incoming', user_id=user_id, status='recieved').first() if smses: smses.status = 'delivered' return(smses)
def save_data(self): if self.resp: for item in self.resp: flag = DBSession.save( item ) if not flag: print 'write failed with data=', json.dumps( item ) else: print 'write successed ' else: print 'error with no self.resp or self.resp is not list type'
def db_storage(recipients, message, user_id, sender): """ This method is responsible to store Sms and related fields provided by client in database """ sms_ids = '' # contains all message ids that are sended by client to recipient of same or different network. #smses_ids = '' # contains only message ids that are sended by client to recipient of some other network. sender = sender.decode(encoding='ascii') recipients = recipients.decode(encoding='ascii').splitlines() selected_package = selected_packages(user_id) for recipient in recipients: processed_fields = process_outgoing_sms(sender, user_id, recipient) # storing vaues to database S = Sms() S.sms_type = 'outgoing' S.sms_from = processed_fields['sender_number'] S.sms_to = recipient S.schedule_delivery_time = datetime.date.today() S.validity_period = datetime.date.today()+datetime.timedelta(days=1) S.msg = message S.timestamp = datetime.date.today() S.status = 'scheduled' S.msg_type = 'text' S.user_id = user_id S.package_name = selected_package['package_name'] S.rates = selected_package['rates'] S.target_network = processed_fields['target_network'] # process sms file would use it to send to respective network of which server is. S.client_type = 'smpp' DBSession.add(S) transaction.commit() sms = DBSession.query(Sms)[-1] # to send id to the client for ancilliary operations and querying. sms_ids = sms_ids + str(sms.id) + '\n' if processed_fields['target_network'] != processed_fields['source_network']: # if destination and source network is different connect_info(recipient, message, processed_fields['target_network'], sms.id, processed_fields['sender_number']) # connect to the destination's smpp server. #smses_ids = smses_ids + str(s) + '\n' #updating_status(smses_ids) return(sms_ids)
def updating_status(sms_ids): """ This method is responsible to update status of messages from scheduled to delivered when messages are for server of other network. """ if sms_ids: sms_ids = sms_ids.splitlines() print(sms_ids) for sms_id in sms_ids: smses = DBSession.query(Sms).filter_by(id=int(sms_id)).first() smses.status = 'delivered' transaction.commit()
def selected_packages(user_id): """ This method is used by server to ensure if logged in user has selected any package or not. """ total_selected_package = DBSession.query(Selected_package).filter_by(user_id=user_id).count() if(total_selected_package > 0): selected_package = DBSession.query(Selected_package).filter_by(user_id=user_id)[-1] # retrieve last selected package else: selected_package = None if(selected_package is None): package_name = None rates = 1.5 else: end_date = int(selected_package.end_date.strftime('%d')) end_month = int(selected_package.end_date.strftime('%m')) end_year = int(selected_package.end_date.strftime('%y')) date = datetime.datetime.now() today_date = int(date.strftime('%d')) today_month = int(date.strftime('%m')) today_year = int(date.strftime('%y')) if(end_year > today_year and int(selected_package.smses) > 0): package_name = selected_package.package_name rates = 0.0 selected_package.smses = selected_package.smses-1 elif(end_year == today_year and end_month > today_month and int(selected_package.smses) > 0): package_name = selected_package.package_name rates = 0.0 selected_package.smses = selected_package.smses-1 elif(end_month == today_month): if(end_date >= today_date and int(selected_package.smses) > 0): package_name = selected_package.package_name rates = 0.0 selected_package.smses = selected_package.smses-1 else: package_name = None rates = 1.5 return(dict(package_name=package_name, rates=rates))
def process_incoming_sms(): """ This method is responsible for processing incoming smses. """ sms_ids = '' smses = DBSession.query(Sms).filter_by(sms_type='incoming', status='recieved', target_network=None, user_id=None).all() if smses: for S in smses: sms_to = S.sms_to sms_from = S.sms_from mnp = DBSession.query(Mnp).filter_by(cell_number=sms_to).first() # querying for mobile number conversion if mnp: target_network = mnp.target_network S.sms_type = 'outgoing' S.status = 'scheduled' S.validity_period = datetime.date.today()+datetime.timedelta(days=1) else: source_prefix = sms_from[0:6] # extract prefix of sender number dest_prefix = sms_to[0:6] # extract prefix of recipient s_network = DBSession.query(Prefix_Match).filter_by(prefix=source_prefix).first() source_network = s_network.network # refers to sender network, getting the network of source from prefixes d_network = DBSession.query(Prefix_Match).filter_by(prefix=dest_prefix).first() dest_network = d_network.network # refers to recipient network user = DBSession.query(User_Number).filter_by(cell_number=sms_to).first() # user refers to normal user if user: S.user_id = user.user_id target_network = None # bECAUSE IT IS FOR CLIENT else: # IF network is same or different. target_network = dest_network # member variable S.sms_type = 'outgoing' S.status = 'scheduled' S.validity_period = datetime.date.today()+datetime.timedelta(days=1) S.target_network = target_network # from sms table if target_network != source_network: connect_info(sms_to, S.msg, dest_network, S.id, sms_from)
def query_result(message_id, user_id): """ This method is responsible to query database for provided message id to view the status of Sms """ message_id = int(message_id.decode(encoding='ascii')) smses = DBSession.query(Sms).filter_by(sms_type='outgoing', id=message_id, user_id=user_id).first() #check for all outgoing sms if(smses is None): return(command_status.ESME_RINVMSGID) elif(smses.status == 'delivered'): return(dict(state=message_state.DELIVERED, final_date=smses.validity_period)) elif(smses.status == 'scheduled' and smses.validity_period >= datetime.datetime.now()): return(dict(state=message_state.SCHEDULED, final_date=smses.validity_period)) elif(smses.status == 'scheduled' and smses.validity_period < datetime.datetime.now()): return(dict(state=message_state.EXPIRED, final_date=smses.validity_period))
def validate(system_id, password, system_type): """ This method is used by server to validate credentials provided by client against database """ system_id = system_id.decode(encoding='ascii') passhash = hashlib.sha1(bytes(password.decode(encoding='ascii'), encoding="utf8")).hexdigest() system_type = system_type.decode(encoding='ascii') record = DBSession.query(User).filter_by(user_id=system_id, password=passhash, system_type=system_type).first() if(record): print("Credentials Validated successfully!!!") return 'True' else: print("Validation failed") return 'false'
def process_outgoing_sms(sender, user_id, recipient): """ This method is responsible for processing outgoing smses. """ if sender is not '': sender_number = sender else: user = DBSession.query(User_Number).filter_by(user_id=user_id).first() # user refers to normal user sender_number = user.cell_number # cell number of sender source_prefix = sender_number[0:6] # extract prefix of sender number s_network = DBSession.query(Prefix_Match).filter_by(prefix=source_prefix).first() source_network = s_network.network # refers to sender network, getting the network of source from prefixes dest_prefix = recipient[0:6] # extract prefix of recipient d_network = DBSession.query(Prefix_Match).filter_by(prefix=dest_prefix).first() dest_network = d_network.network # refers to recipient network mnp = DBSession.query(Mnp).filter_by(cell_number=recipient).first() # querying for mobile number conversion if mnp: target_network = mnp.target_network elif(source_network == dest_network): target_network = dest_network else: target_network = dest_network return(dict(sender_number=sender_number, source_network=source_network, target_network=target_network))
def tween (request): try: response = handler(request) DBSession.commit() except: exc_info = sys.exc_info() #workaround for DBSession.rollback() breaking traceback DBSession.rollback() raise exc_info[0], exc_info[1], exc_info[2] finally: DBSession.remove() return response
def replace_result(message_id, message, user_id): """ This method is responsible to replace particular message which is not yet delivered. """ message_id = int(message_id.decode(encoding='ascii')) message = message.decode(encoding='ascii') smses = DBSession.query(Sms).filter_by(sms_type='outgoing', id=message_id, user_id=user_id).first() if(smses is None): return False elif(smses.status == 'delivered'): return(command_status.ESME_RREPLACEFAIL) elif(smses.status == 'scheduled'): smses.schedule_delivery_time = datetime.datetime.now() smses. validity_period = datetime.datetime.now()+datetime.timedelta(days=1) smses.msg = message transaction.commit() return True
def connect_info(recipient, message, dest_network, sms_id, sender_number): """ This method is responsible to get connection paramters from database to communicate with server of destination network in case when sender sends a message to some other network. """ # here we have to make client object server = DBSession.query(Network).filter_by(network=dest_network).first() if server: print("Establishing Connection with " + server.network.upper() + " Server...") system_id = server.username password = server.password system_type = server.system_type ip = server.ip port = server.port # this thread is responsible for connecting with other smpp server. background_thread3 = threading.Thread(target=connect_to_server, args=(ip, port, system_id, password, system_type, recipient, message, sms_id, sender_number)) background_thread3.start() background_thread3.join()
'''This background process is responsible for reading the outgoing smses from database sent by client and post them to the url on bottle server ''' import urllib2 import urllib import db import transaction from db import DBSession from models import Sms, User import datetime import time try: db.bind_session() while True: smses = DBSession.query(Sms).filter_by(status='scheduled', sms_type='outgoing', target_network='zong').all() if(smses): for S in smses: sms_to = S.sms_to sms_from = S.sms_from message = S.msg post_data = {'sms_to': sms_to, 'sms_from': sms_from, 'message': message} post_str = urllib.urlencode(post_data) # urlencode converts the dict to a string suitable for POST # Note when some data is specified in urlopen then HTTP request type # POST is used instead of GET request type result = urllib2.urlopen("http://192.168.5.34:50111/" + 'sendsms', post_str).read() S.status = 'delivered' S.timestamp = datetime.date.today() print(result) transaction.commit()
S = Sms() S.sms_type = 'outgoing' S.sms_from = '+9233365195924' S.sms_to = '+923366767999' S.schedule_delivery_time = d # give date 2nd december S.validity_period = d+datetime.timedelta(days=1) S.msg = "dont disturb.." S.timestamp = d S.status = 'delivered' S.msg_type = 'text' S.user_id = 'ASMA' S.package_name = 'dhamaka' S.rates = 0.0 S.target_network = 'ufone' # process sms file would use it to send to respective network of which server is. S.client_type = 'smpp' DBSession.add(S) #user = DBSession.query(User_Number).filter_by(user_id='ASMA').first() # user refers to normal user #cell_number = user.cell_number #source_prefix = cell_number[0:6] #dest_prefix = '+92300' #source_network = DBSession.query(Prefix_Match).filter_by(prefix=source_prefix).first() # t_user refers to network #dest_network = DBSession.query(Prefix_Match).filter_by(prefix=dest_prefix).first() # t_user refers to network #print(source_network.network) #print(dest_network.network) transaction.commit() #data = b'\x00\x00\x00/\x00\x00\x00\x02\x00\x00\x00\x00\x00\x00\x00\x01SMPP3TEST\x00secret08\x00' + \ #b'SUBMIT1\x00P\x01\x01\x00' #P = BindTransmitter.decode(data) #print(DBSession.query(User).count()) #system=P.system_id.value.decode(encoding='ascii') #for U in DBSession.query(User).all():
def process_steps (test_id, tests_cache, registry, step_ques, finish_que, workers_last_activity, timers): #print "tick", test_id #TODO what if this gets interrupted by kill during io? #TODO dont forget actual finish time test = tests_cache[test_id] #new_rx = get_rx() #rx = new_rx - test['rx_snapshot'] #test['rx_snapshot'] = new_rx #new_tx = get_tx() #tx = new_tx - test['tx_snapshot'] #test['tx_snapshot'] = new_tx connect_errors = [] while True: steps = step_ques[test_id].next() if steps: buf_statuses = defaultdict(int) buf_resp_time = defaultdict(list) buf_conn_time = defaultdict(list) buf_errors = defaultdict(int) buf_concur_users_num_max = 0 buf_concur_users_num_min = 0 buf_concur_conns_num_min = 0 buf_concur_conns_num_max = 0 buf_start_session = 0 buf_request_sent = 0 is_finish_only_step = all(len(data) == 1 and data[0]['type'] == stypes.FINISH_TEST for _node_id, data in steps.items()) for node_id, data in steps.items(): node_id = int(node_id) workers_last_activity[test_id][node_id] = time.time() for rec in data: data_type = rec['type'] if data_type == stypes.RESPONSE_STATUS: grp, status = rec['value'] if int(status) not in (200, 201, 202): #TODO move to test logic test['resp_bad_statuses_total'] += 1 else: test['resp_successful_total'] += 1 status = str(status) + " " + grp tests_cache[test_id]['resp_statuses'].add(status) buf_statuses[status] += 1 elif data_type == stypes.RESPONSE_TIME: grp_name, resp_time = rec['value'] buf_resp_time[grp_name].append(resp_time) tests_cache[test_id]['groups'].add(grp_name) elif data_type == stypes.CONNECT_TIME: test['conns_total'] += 1 grp_name, timelen = rec['value'] buf_conn_time[grp_name].append(timelen) tests_cache[test_id]['groups'].add(grp_name) elif data_type == stypes.CONCUR_USERS_NUM_MAX: buf_concur_users_num_max += rec['value'] elif data_type == stypes.CONCUR_USERS_NUM_MIN: buf_concur_users_num_min += rec['value'] elif data_type == stypes.CONCUR_CONNS_NUM_MIN: buf_concur_conns_num_min += rec['value'] elif data_type == stypes.CONCUR_CONNS_NUM_MAX: buf_concur_conns_num_max += rec['value'] elif data_type == stypes.START_SESSION: buf_start_session += rec['value'] elif data_type == stypes.REQUEST_SENT: test['reqs_total'] += rec['value'] buf_request_sent += rec['value'] elif data_type == stypes.CONNECT_ERROR: test['conns_errors_total'] += 1 try: tests_cache[test_id]['errors'].add("connect " + rec['value']['msg']) except: print "rec:", rec raise buf_errors[rec['value']['msg']] += 1 if "not enough ports" not in rec['value']['msg']: connect_errors.append(u'%s\t%s\t%s' % (rec['value']['time'], rec['value']['ip'], rec['value']['msg'])) elif data_type == stypes.RESPONSE_ERROR: if "timeout" in rec['value']: test['resp_timeouts_total'] += 1 else: test['resp_errors_total'] += 1 ern = "response " + rec['value'] tests_cache[test_id]['errors'].add(ern) buf_errors[ern] += 1 elif data_type == stypes.FINISH_TEST: finish_que[test_id][node_id] = True else: raise NotImplementedError(rec['type']) if not is_finish_only_step: res = test['result'] rt = {} rm = {} for grp, times in buf_resp_time.iteritems(): resp_time_med = util.get_median(times) rt[grp] = resp_time_med rm[grp] = util.get_median(abs(t - resp_time_med) for t in times) res['resp_time'].append(rt) res['resp_time_meav'].append(rm) #TODO rename to med_abs_dev rt = {} rm = {} for grp, times in buf_conn_time.iteritems(): resp_time_med = util.get_median(times) rt[grp] = resp_time_med rm[grp] = util.get_median(abs(t - resp_time_med) for t in times) res['conn_time'].append(rt) res['conn_time_meav'].append(rm) #TODO rename to med_abs_dev res['start_session'].append(buf_start_session) res['resp_status'].append(buf_statuses) res['req_sent'].append(buf_request_sent) res['errors'].append(buf_errors) res['concur_users_num_max'].append(buf_concur_users_num_max) res['concur_users_num_min'].append(buf_concur_users_num_min) res['concur_conns_num_min'].append(buf_concur_conns_num_min) res['concur_conns_num_max'].append(buf_concur_conns_num_max) #res['network_received'].append(float(rx) / 1024.0) #res['network_sent'].append(float(tx) / 1024.0) else: break if connect_errors and test['write_connect_error_log']: with open("conn_err_%s.log" % test_id, 'ab') as f: f.write("\n".join(connect_errors).encode('utf-8') + "\n") is_crashed = False now = time.time() for _node_id, ts_last in workers_last_activity[test_id].items(): if now - ts_last >= WORKERS_TIMEOUT: is_crashed = True break is_finished = len(finish_que[test_id]) == test['worker_num'] if is_crashed: is_finished = True if is_finished: tests_cache[test_id]['finished'] = (now - WORKERS_TIMEOUT) if is_crashed else now #DBSession.query(Test).filter_by(id = test_id).update({Test.data: dbdump(tests_cache[test_id])}) t = Test.query.filter_by(id = test_id).first() if not t: raise Exception("no test with id = %s (maybe it was deleted before finish timeout happened?)" % test_id) t.data = dbdump(tests_cache[test_id]) DBSession.add(t) DBSession.commit() if registry.has_listeners: registry.notify(OnFinishTest(t, registry.settings)) del tests_cache[test_id] del finish_que[test_id] del step_ques[test_id] del workers_last_activity[test_id] gevent.kill(timers[test_id]) del timers[test_id] log.info("finished test #%s%s" % (test_id, " (timeout)" if is_crashed else ""))