def profile(queue, func_args):
# give testing.db a brand new pool and don't
# touch the existing pool, since closing a socket
# in the subprocess can affect the parent
testing.db.pool = testing.db.pool.recreate()
gc_collect()
samples = []
max_ = 0
max_grew_for = 0
success = False
until_maxtimes = 0
while True:
if until_maxtimes >= maxtimes // 5:
break
for x in range(5):
func(*func_args)
gc_collect()
samples.append(
get_num_objects() if get_num_objects is not None else
len(get_objects_skipping_sqlite_issue()))
if assert_no_sessions:
assert len(_sessions) == 0
# queue.put(('samples', samples))
latest_max = max(samples[-5:])
if latest_max > max_:
queue.put((
"status",
"Max grew from %s to %s, max has "
"grown for %s samples" %
(max_, latest_max, max_grew_for),
))
max_ = latest_max
max_grew_for += 1
until_maxtimes += 1
continue
else:
queue.put((
"status",
"Max remained at %s, %s more attempts left" %
(max_, max_grew_for),
))
max_grew_for -= 1
if max_grew_for == 0:
success = True
break
if not success:
queue.put((
"result",
False,
"Ran for a total of %d times, memory kept "
"growing: %r" % (maxtimes, samples),
))
else:
queue.put(("result", True, "success"))
def get_census_profile(geo_code, geo_level):
session = get_session()
try:
geo_summary_levels = get_summary_geo_info(geo_code, geo_level, session)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session), level)
# tweaks to make the data nicer
# show X largest groups on their own and group the rest as 'Other'
group_remainder(data['households']['roofing_material_distribution'], 5)
group_remainder(data['households']['wall_material_distribution'], 5)
return data
finally:
session.close()
def get_census_profile(geo_code, geo_level):
session = get_session()
try:
geo_summary_levels = get_summary_geo_info(geo_code, geo_level, session)
data = {}
for section in PROFILE_SECTIONS:
function_name = 'get_%s_profile' % section
if function_name in globals():
func = globals()[function_name]
data[section] = func(geo_code, geo_level, session)
# get profiles for province and/or country
for level, code in geo_summary_levels:
# merge summary profile into current geo profile
merge_dicts(data[section], func(code, level, session), level)
# tweaks to make the data nicer
# show 3 largest groups on their own and group the rest as 'Other'
group_remainder(data['service_delivery']['water_source_distribution'])
group_remainder(data['service_delivery']['refuse_disposal_distribution'])
group_remainder(data['service_delivery']['toilet_facilities_distribution'], 5)
group_remainder(data['demographics']['language_distribution'], 7)
return data
finally:
session.close()
def wrapper(self, *args, **kwArgs):
try:
return func(self, *args, **kwArgs)
except (MPDError, OSError) as ex:
logger.warning('Detect MPD connection error; reconnecting...')
logger.debug(ex)
self._mpd = initialize_mpd(self.conf)
return func(self, *args, **kwArgs)
def _initMappers(self, version = None):
"""
Init database mappers.
"""
if version == None:
version = self.getUptodateVersion()
func = getattr(self, "_initMappers" + self.config.auditdbdriver + "V" + str(version))
func()
def _populateTables(self, version = None):
"""
Populate tables before the first use.
"""
if version == None:
version = self.getUptodateVersion()
func = getattr(self, "_populateTables" + self.config.auditdbdriver + "V" + str(version))
func()
def wrapper(*args, **kwargs):
old_stdout = sys.stdout
sys.stdout = new_stdout = io.BytesIO()
func(*args, **kwargs)
func_out = new_stdout.getvalue()
sys.stdout = old_stdout
new_stdout.close()
return func_out
def wrapper(self, *args, **kwArgs):
try:
return func(self, *args, **kwArgs)
except (MPDError, OSError) as ex:
logger.warning('Detect MPD connection error; reconnecting...')
logger.debug(ex)
self._mpd = initialize_mpd(self.conf)
return func(self, *args, **kwArgs)
def wrapper(*args, **kwargs):
if request.method in ['GET','HEAD','OPTIONS'] or \
(hasattr(current_user, 'token') and current_user.token.type != 'session'):
return func(*args, **kwargs)
if 'X-Requested-With' in request.headers:
return func(*args, **kwargs)
abort(401)
def shutdown_server():
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
def decorator(*args, **kwargs):
session = kwargs.get('session')
if not session:
with open_session(Session, commit=commit) as session:
kwargs['session'] = session
r = func(*args, **kwargs)
else:
r = func(*args, **kwargs)
return r
def _check_existence(db, entity, rolename):
role = orm.Role.find(db, rolename)
if entity is None:
raise ValueError(
f"{entity!r} of kind {type(entity).__name__!r} does not exist")
elif role is None:
raise ValueError("Role %r does not exist" % rolename)
else:
func(db, entity, role)
def decorated_function(*args, **kwargs):
callback = request.args.get('callback', False)
if callback:
data = str(func(*args, **kwargs).data)
content = str(callback) + '(' + data + ')'
mimetype = 'application/javascript'
return current_app.response_class(content, mimetype=mimetype)
else:
return func(*args, **kwargs)
def wrapper(*args, **kws):
try:
return func(*args, **kws)
except Exception as e:
logging.exception(e)
try:
# reconnect_mysql()
return func(*args, **kws)
except:
logging.exception(e)
return None
def wrapper(*args, **kwargs):
if not kwargs.get('session'):
kwargs['session'] = Session(autoflush=True, expire_on_commit=False)
try:
result = func(*args, **kwargs)
kwargs['session'].commit()
return result
finally:
kwargs['session'].close()
else:
return func(*args, **kwargs)
def wrapper(*args, **kwargs):
if not kwargs.get('session'):
kwargs['session'] = Session(autoflush=True, expire_on_commit=False)
try:
result = func(*args, **kwargs)
kwargs['session'].commit()
return result
finally:
kwargs['session'].close()
else:
return func(*args, **kwargs)
def call(db, *args, **kwargs):
# if the Database has an active session, don't create a new one
if db.session is not None:
result = func(db, *args, **kwargs)
if commit:
db.session.commit()
else:
# otherwise, use the session generator
with db_session(db, commit=commit):
result = func(db, *args, **kwargs)
return result
def call(db, *args, **kwargs):
# if the Database has an active session, don't create a new one
if db.session is not None:
result = func(db, *args, **kwargs)
if commit:
db.session.commit()
else:
# otherwise, use the session generator
with DBSession(db, commit=commit):
result = func(db, *args, **kwargs)
return result
def do_task(self, args):
if args['func'] == 'seq':
func = self._calculate_seq_stats
elif args['func'] == 'clone':
func = self._calculate_clone_stats
self.info(('Processing {} for sample {}, include_outliers {}, '
'only_full_reads {}').format(
'sequences' if args['func'] == 'seq' else 'clones',
args['sample_id'], args['include_outliers'],
args['only_full_reads']))
func(args['sample_id'], args['min_cdr3'], args['max_cdr3'],
args['include_outliers'], args['only_full_reads'])
self._session.commit()
def do_task(self, args):
if args['func'] == 'seq':
func = self._calculate_seq_stats
elif args['func'] == 'clone':
func = self._calculate_clone_stats
self.info(('Processing {} for sample {}, include_outliers {}, '
'only_full_reads {}').format(
'sequences' if args['func'] == 'seq' else 'clones',
args['sample_id'], args['include_outliers'],
args['only_full_reads']))
func(args['sample_id'], args['min_cdr3'], args['max_cdr3'],
args['include_outliers'], args['only_full_reads'])
self._session.commit()
def decorated_view(*args, **kwargs):
if request.method in EXEMPT_METHODS:
return func(*args, **kwargs)
elif current_app.login_manager._login_disabled:
return func(*args, **kwargs)
elif not current_user.is_authenticated:
return current_app.login_manager.unauthorized()
else:
if current_user.is_authenticated:
if session['type'] != 'front':
return current_app.login_manager.unauthorized()
# return current_app.login_manager.unauthorized()
return func(*args, **kwargs)
def db_wrapper(func, *args, **kwargs):
self = args[0]
if self.session:
return func(*args, **kwargs)
else:
self.db = get_db()
self.session = self.db.sess
try:
return func(*args, **kwargs)
finally:
self.db.end()
self.db = None
self.session = None
def save(self, _commit=True):
try:
db.session.add(self)
if _commit:
db.session.commit()
except IntegrityError:
db.session.rollback()
raise
_hooks = ('_clean_cache', '_before_save_hook')
for each in _hooks:
if hasattr(self, each) and callable(getattr(self, each)):
func = getattr(self, each)
func()
def openatin(self): #开始相关操作
while True:
print('\033[35;1m讲师界面\033[0m'.center(60, '='))
for index, i in enumerate(config.TEACH_OPEN):
print(index, ':', i[0])
id = input('请选择>>:')
if id.isdigit():
if int(id) >= len(config.TEACH_OPEN): continue
s = config.TEACH_OPEN[int(id)][1]
else:
continue
if hasattr(self, s): #是反射否存在
func = getattr(self, s) #调用
func() #执行
def wrapper(*args, **kwargs):
allowed_users = flask.current_app.config["CACHITO_WORKER_USERNAMES"]
# current_user.is_authenticated is only ever False when auth is disabled
if current_user.is_authenticated and current_user.username not in allowed_users:
raise Forbidden(
"This API endpoint is restricted to Cachito workers")
return func(*args, **kwargs)
def run_hash():
# Timestamp is the key
# Because it is better than pure integers
now = datetime.datetime.now()
# Generate salt
salt = os.urandom(1024)
salt = base64.b64encode(salt)
# The final key
combination = str(now) + str(salt)
# Use corresponding algorithm for specified required length
# Provides better hash and avoid collisions
if length == 32:
func = hashlib.md5
elif length == 40:
func = hashlib.sha1
elif length == 56:
func = hashlib.sha224
elif length == 64:
func = hashlib.sha256
elif length == 96:
func = hashlib.sha384
else:
func = hashlib.sha512
hashid = func(combination).hexdigest()
return unicode(hashid[:length])
def wrapper(*args, **kwargs):
if not 'uid' in session:
abort(403)
user = request.user = User.query.filter_by(name=session['uid']).first()
if getattr(Perms, kwargs.get(request.method, defaccess_or_fn)) not in user.role:
abort(403)
return func(*args, **kwargs)
def get_user_list_info(self, listof_screen_name=None, listof_user_id=None, **kwargs):
process_name = "/users/show/:id"
try:
user_list = list()
if listof_screen_name is None:
for item in listof_user_id:
user_id = self.get_user_info(user_id=item)[0]
user_list.append(user_id)
elif listof_user_id is None:
raise Exception('Unknown Input')
else:
for item in listof_screen_name:
user_id = self.get_user_info(screen_name=item)[0]
user_list.append(user_id)
except TwitterError as e:
""" Error처리는 다른 함수와 동일
"""
sess = Session()
t = TweetErrorHandler(e)
t.add_handler(ErrorNumbers.RATE_LIMIT_ERROR, self.rate_limit_handler)
result = t.invoke(process_name=process_name)
sess.commit()
sess.close()
return result
return func(self, user_list, **kwargs)
def _execute_async_action(self, func, *args):
self.log.info('Scheduling syncback action', func=func, args=args)
g = gevent.Greenlet(retry_with_logging, lambda: func(*args),
logger=self.log)
g.link_value(lambda _: self.log.info('Syncback action completed',
func=func, args=args))
self.worker_pool.start(g)
def wrapper(*args, **kwargs):
if 'apikey' not in session:
return redirect('/')
user = User.query.filter_by(apikey = session['apikey']).first()
if user == None:
return redirect('/')
return func(*args, **kwargs)
def _(user_obj, *args, **kwargs):
if not user_obj.is_active:
return False
elif user_obj.is_superuser:
return True
else:
return func(user_obj, *args, **kwargs)
def init_advanced_task(self, funcname, para):
if datareader_API_dict[funcname]['multithread']:
job = current_app.task_queue.enqueue(
'app.tasks.ngs_data_processing.advanced_task',self.id,funcname,para,job_timeout=3600*10)
t = Task(id=job.get_id(), name=f"Advanced call {self}.")
self.advanced_result[funcname]={'input':para,'output':{'task':t.id,'file':[],'text':[],'img':[]}}
self.save_data()
db.session.add(t)
db.session.commit()
return t
else:
try:
self.advanced_result[funcname]={'input':para,'output':{'task':None,'file':[],'text':[],'img':[]}}
para = eval(f"dict({para})")
dr = self.get_advanced_datareader()
func = getattr(dr, funcname)
return_annotation = signature(func).return_annotation
if return_annotation == _empty:
return None
else:
_return = return_annotation.split(',')
result = func(**para)
if len(_return) == 1:
result = [result]
output = dict(zip(_return, result))
self.advanced_result[funcname]['output'].update(output)
self.save_data()
db.session.commit()
except Exception as e:
self.advanced_result[funcname]['output']['text'] = [f"Error: {e}"]
self.save_data()
db.session.commit()
return None
def with_group_checking(self, message):
if message.chat.id < 0:
self.bot.sendMessage(
chat_id=message.chat.id, text="這個功能沒辦法在聊天室使用，請跟我私聊。", reply_to_message_id=message.message_id
)
return
return func(self, message)
def wrapper(*args, **kwargs):
result = json.dumps(func(*args, **kwargs)
or {"error": "No data found for your request"},
default=serialize_to_json)
headers = [("Content-Type", "application/json"),
("Content-Length", str(len(result)))]
return Response(result, status=200, headers=headers)
def run_and_record_update(self, db, *args, **kwargs):
all_completed = False
for reader_name in self.reader_names:
self.run_datetime = datetime.utcnow()
done = func(self, db, reader_name, *args, **kwargs)
all_completed &= done
logger.info("%s is%s done" % (reader_name,
'' if done else ' not'))
if done:
is_read_all = (func.__name__ == 'read_all')
reader_version = self.get_version(reader_name)
if reader_version is None:
# This effectively indicates no jobs ran.
logger.info("It appears no %s jobs ran. No update "
"will be logged." % reader_name)
continue
logger.info("Recording this reading in reading_updates: "
"%s version %s running at %s reading content "
"between %s and %s."
% (reader_name, reader_version,
self.run_datetime, self.begin_datetime,
self.end_datetime))
db.insert('reading_updates', complete_read=is_read_all,
reader=reader_name,
reader_version=reader_version,
run_datetime=self.run_datetime,
earliest_datetime=self.begin_datetime,
latest_datetime=self.end_datetime)
return all_completed
def wrapper(**kwargs):
#Only track user id if they are authenticated
if current_user.is_authenticated:
user_id = current_user.id
else:
user_id = None
#Get session id if it exists; Reminder, session id lasts 8 years or until cookies are deleted
session_id = session.get('FVID')
# Get To URL
base_url = request.base_url
# Get From Url
referrer_url = request.referrer if request.referrer else 'None'
# get user agent
user_agent = request.headers.get('User-Agent')
# If no session data, then create new session data
if session.get('FVID') is None:
session['FVID'] = secrets.token_hex(32)
session_id = session.get('FVID')
# If user is clicking an outbound link, track this instead out /outboundLinks
to_url = request.args.get('url', default=None, type=None) if base_url == request.url_root + 'outboundLinks' else base_url
pv = userPageView(session_id = session_id, from_page = referrer_url, to_page = to_url, user_id = user_id, user_agent = user_agent)
if track_pageviews:
db.session.add(pv)
db.session.commit()
else:
print(pv)
x = func(**kwargs)
return x
def inner(env, *a, **kw):
key = env.request.full_path
value = cache.get(key)
if value is None:
value = func(env, *a, **kw)
cache.set(key, value, timeout=timeout)
return value
def openatin(self):#开始相关操作
while True:
print('\033[32;1m学员界面\033[0m'.center(60,'='))
print(('\033[36;1m[%s]\033[0m'%self.stu_name).center(60,'='))
for index,i in enumerate(config.STUED_OPEN):
print(index,':',i[0])
id=input('请选择>>:')
if id.isdigit():
if int(id)>=len(config.STUED_OPEN):continue
s=config.STUED_OPEN[int(id)][1]
else:
continue
if hasattr(self,s):#是反射否存在
func=getattr(self,s)#调用
func()#执行
def wrapper(*args, **kwargs):
if not "user_id" in flask.session:
flask.flash("Sorry, you are not allowed to do that. "
"Please log in first.")
flask.session["real_url"] = flask.request.url
return flask.redirect(flask.url_for("login"))
return func(*args, **kwargs)
def display_time(func):
start = time.time()
result = func()
end = time.time()
elapsed_time = round(end - start, 2)
print("Time to Execute: {} seconds".format(elapsed_time))
return result
def disk_path(self, disk_type='publish', refresh=False):
'''
从orm 转换到DiskPath 的对象
:param disk_type: string，通常可以选择 'publish', 'work', 'cache'。（需要和storage_config 的内容保持一致）
:return: DiskPath 对象（实际在硬盘上的存放路径）
'''
if getattr(self, '_cache_{}_disk_path'.format(disk_type), None) and refresh is False:
return getattr(self, '_cache_{}_disk_path'.format(disk_type), None)
from dayu_path import DayuPath
import util
storage = util.get_storage_config(self.storage_config_name)
db_config = util.get_db_config(self.db_config_name)
parents = self.hierarchy
result = storage.config[disk_type][sys.platform]
for index, x in enumerate(parents[1:]):
depth_config = db_config.config[str(index + 1)]
param_list = []
for param in depth_config['to_disk_param'][x.meaning][disk_type]:
match = DiskPathMixin._eval_regex.match(param)
if match:
func = getattr(util, match.groups()[0])
if func:
param_list.append(func())
else:
param_list.append(getattr(x, param))
result += depth_config['to_disk'][x.meaning][disk_type].format(*param_list)
setattr(self, '_cache_{}_disk_path'.format(disk_type), DayuPath(result))
getattr(self, '_cache_{}_disk_path'.format(disk_type), None)._cache_orm = self
return getattr(self, '_cache_{}_disk_path'.format(disk_type), None)
def wrapper(*args, **kwargs):
start = time.time()
res = func(*args, **kwargs)
end = time.time()
print('[{}] {}s elapsed'.format(label, end - start))
return res
def _(user_obj, *args, **kwargs):
if not user_obj.is_active:
return False
elif user_obj.is_superuser:
return True
else:
return func(user_obj, *args, **kwargs)
def wrapper():
result = current_user.is_contributor
if result:
x = func()
return x
else:
flash(f'You do not have permission to do that.', 'info')
return redirect(url_for('home'))
def wrapper(*args, **kwargs):
result = json.dumps(func(*args, **kwargs) or {"error": "No data found for your request"},
default=serialize_to_json)
headers = [
("Content-Type", "application/json"),
("Content-Length", str(len(result)))
]
return Response(result, status=200, headers=headers)
def inner(*args, **kwargs):
saved_session = getattr(threadlocals, 'session_name', None)
res = func(*args, **kwargs)
threadlocals.session_name = saved_session
return res
def wrapper(obj, *args, **kwargs):
try:
with _establish_db_session() as obj.sess:
ret = func(obj, *args, **kwargs)
obj.sess = None
return ret
except SQLAlchemyError as exc:
raise AddressBookError(exc) from exc
def wrapper(*args, **kwargs):
try:
results = func(*args, **kwargs)
except OperationalError as oe:
raise DbConnectionError("Unable to execute {}. Reason: {}".format(
func.__name__, str(oe)))
else:
return results
def wrapper(obj, *args, **kwargs):
try:
with _establish_db_session() as obj.sess:
ret = func(obj, *args, **kwargs)
obj.sess = None
return ret
except SQLAlchemyError as exc:
raise AddressBookError(exc) from exc
def wrapper(*args, **kwargs):
html = func(*args, **kwargs)
if request.args.get('download'):
page = kwargs.get('page')
dataset_id = kwargs.get('dataset_id')
report_name = get_report_name(page, dataset_id)
return get_excel_document(html, report_name)
return html
def wrapper(*args, **kwargs):
user = UserModel.query.get(login.current_user.id)
user_id = kwargs.get('user_id')
# Check if user with user_id exists
get_object_or_404(UserModel, user_id)
if user.id == user_id or user.is_staff:
return func(*args, **kwargs)
else:
raise PermissionDeniedError()
def db_user(self, screen_name=None, user_id=None, **kwargs):
process_name = "/users/show/:id"
try:
result = self.get_user_info(screen_name=screen_name, user_id=user_id, **kwargs)
if len(result) > 1:
return func(self, result[0], user=result[1], **kwargs)
else:
return func(self, result[0], **kwargs)
except TwitterError as e:
""" Error처리는 다른 함수와 동일
"""
sess = Session()
t = TweetErrorHandler(e)
t.add_handler(ErrorNumbers.RATE_LIMIT_ERROR, self.rate_limit_handler)
result = t.invoke(process_name=process_name)
sess.commit()
sess.close()
return result
def _(*args, **kwargs):
obj = args[0]
if hasattr(obj, name):
return getattr(obj, name)
_prop = func(*args, **kwargs)
if not _prop:
_prop = default_prop
setattr(obj, name, _prop)
return _prop
def wrapped(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except Exception as e:
log.err("Unhandled exception in %s" % func.__name__)
log.err(e)
if hasattr(self, 'alerts_proxy') and self.alerts_proxy is not None:
self.alerts_proxy.send_alert(str(e), "Unhandled exception in %s" % func.__name__)
raise e
def wrapper(*args, **kwargs):
name = kwargs.pop('room_name')
room = Room.query.filter_by(name=name).first()
if not room:
abort(404)
elif request.user not in room.users:
abort(403)
request.room = room
return func(*args, room=room, **kwargs)
def _wrap(*args, **kwargs):
lock.acquire()
result = None
try:
result = func(*args, **kwargs)
except Exception as e:
print("Error in lock decorator: ", e)
lock.release()
return result
def wrapper(*args, **kwargs):
status_code = 200
result = func(*args, **kwargs)
if isinstance(result, Response):
return result
elif isinstance(result, tuple):
result, status_code = result
return Response(json.dumps(result, default=scoped_default_handler()),
mimetype='application/json', status=status_code)
def wrapper(*args, **kwargs):
user = getattr(g, 'user', None)
user_id = kwargs.get('user_id')
# Check if user with user_id exists
get_object_or_404(UserModel, user_id)
if user and (user.id == user_id or user.is_staff):
return func(*args, **kwargs)
else:
raise PermissionDeniedError()