def setUp(self):
# Cache current database connections
self.databases = copy(connections._databases)
self.connection = getattr(connections._connections, DEFAULT_DB_ALIAS, None)
del connections.__dict__['databases'] # remove cached_property value
connections._databases = None
connections._connections = local()
def __init__(self, databases=None):
"""
databases is an optional dictionary of database definitions (structured
like settings.DATABASES).
"""
self._databases = databases
self._connections = local()
def setUp(self):
# Cache current database connections
self.databases = copy(connections._databases)
self.connection = getattr(connections._connections, DEFAULT_DB_ALIAS,
None)
del connections.__dict__['databases'] # remove cached_property value
connections._databases = None
connections._connections = local()
def check_task(): #校验文件任务函数
with settings(warn_only=True):
#本地local命令需要配置capture=True才能捕获返回值
lmd5 = local("md5sum /data/logs/access.tar.gz",
capture=True).split(' ')[0]
rmd5 = run("md5sum /data/logs/access.tar.gz").split(' ')[0]
if lmd5 == rmd5: #对比本地及远程的md5信息
print "OK"
else:
print "ERROR"
def clear_thread_locals():
"""
clear the thread locals. Should be run at the top of every request.
"""
global _thread_locals
_thread_locals = local()
from __future__ import absolute_import
# Standard Library
import logging
from _threading_local import local
# External Libraries
from decorator import contextmanager
from django.contrib import auth
from django.contrib.auth import get_user_model
from django.contrib.auth.hashers import UNUSABLE_PASSWORD_PREFIX
from django.contrib.auth.models import AnonymousUser
from funcy import memoize
logger = logging.getLogger(__name__)
_thread_locals = local()
def clear_thread_locals():
"""
clear the thread locals. Should be run at the top of every request.
"""
global _thread_locals
_thread_locals = local()
def get_request():
"""
:returns Request: request for current thread locals.
"""
return getattr(_thread_locals, 'request', None)
# mimic django's language activation machinery. it checks for .mo files
# and we don't need anything nearly as complex.
from _threading_local import local
from django.conf import settings
_active = local()
def activate(language):
"""
Installs the given language as the language for the current thread.
"""
_active.value = language
def deactivate():
"""
Uninstalls the currently active language so that further _ calls
will resolve against the default language, again.
"""
if hasattr(_active, "value"):
del _active.value
def get_language():
"""Returns the currently selected language."""
lang = getattr(_active, "value", None)
if lang is None:
return settings.LANGUAGE_CODE
from _threading_local import local
from threading import current_thread, Thread
local_school = local()
def process_tread(name):
local_school.stu = name
process_stu()
def process_stu():
print("thread name: %s, its corresponding student's name is: %s" %
(current_thread().name, local_school.stu))
t1 = Thread(target=process_tread, args=("Adam", ), name="Thread 1")
t2 = Thread(target=process_tread, args=("Peter", ), name="Thread 2")
t3 = Thread(target=process_tread, args=("Susan", ), name="Thread 3")
t4 = Thread(target=process_tread, args=("Tom", ), name="Thread 4")
t1.start()
t2.start()
t3.start()
t4.start()
class CustomSQLAlchemy(flaskext.sqlalchemy.SQLAlchemy):
def __init__(self, *args, **kws):
super(CustomSQLAlchemy, self).__init__(*args, **kws)
self.session = _create_scoped_session(self)
db = CustomSQLAlchemy(app)
import threading, thread, os
def _get_ident():
return (thread.get_ident(), os.getpid())
threading._get_ident = _get_ident
from _threading_local import local
db.session.registry.registry = local()
db.session.extension = MySessionExtension()
app.db = db
def drop_all(self):
from myojin.utils import drop_all_tables
metadata = self
return drop_all_tables(metadata, metadata.bind)
type(db.metadata).drop_all = drop_all
# mimic django's language activation machinery. it checks for .mo files
# and we don't need anything nearly as complex.
from _threading_local import local
from django.conf import settings
_active = local()
def activate(language):
"""
Installs the given language as the language for the current thread.
"""
_active.value = language
def deactivate():
"""
Uninstalls the currently active language so that further _ calls
will resolve against the default language, again.
"""
if hasattr(_active, "value"):
del _active.value
def get_language():
"""Returns the currently selected language."""
l = getattr(_active, "value", None)
if l is None:
def tar_task(): #本地打包函数，只执行一次
with lcd("/data/logs"):
local("tar -czf access.tar.gz access.log")
if not sohu_data_coll.find_one({'_id': doc_id}):
sohu_data_coll.insert_one({
'_id': doc_id,
'url': url,
'page': Binary(zlib.compress(pickle.dumps(page_html)))
})
self.spider.parse(page_html)
self.spider.status = SpiderStatus.IDLE
def is_any_alive(spider_threads):
return any(SpiderStatus.WORKING == spider_thread.spider.status for spider_thread in spider_threads)
thread_local = local()
hasher_proto = sha1()
task_name = 'm_souhu_task'
def main():
redis_client = Redis(host='127.0.0.1', port=6379)
if not redis_client.exists(task_name):
redis_client.rpush(task_name, 'http://m.sohu.com/')
spider_threads = [SpiderThread(f'spider_thread_{i}', Spider()) for i in range(10)]
for spider_thread in spider_threads:
spider_thread.start()
while redis_client.exists(task_name) or is_any_alive(spider_threads):
sleep(5)
import threading
import _threading_local
thread_ctx = _threading_local.local()
def get_thread_name():
return threading.currentThread().getName()
def with_thread_context(_key, _value):
def fn_with_context(fn):
def wrapped_fn_with_context(*args, **kwargs):
thread_name = get_thread_name() + ' '
try:
print thread_name,
print thread_ctx.__dict__
if _key:
if thread_ctx.__dict__.has_key(_key):
print thread_name + 'WARNING : Context already has key ',
print _key,
print 'Current ' + _value,
print 'Requested ' + thread_ctx.__dict__.get(_key)
else:
print thread_name + 'Adding ' + _key + ' to Context. Value ' + _value
thread_ctx.__dict__[_key] = _value
else:
print thread_name + 'Nothing to add to Context'
fn(*args, **kwargs)
finally:
if _key and thread_ctx.__dict__.has_key(_key):
del thread_ctx.__dict__[_key]
print thread_name + 'Cleaning %s from Context' % _key
else:
print thread_name + 'Nothing to clean from Context'
def local_task():
local("uname -a")
from _threading_local import local
_thread_locals = local()
def set_cid(cid):
setattr(_thread_locals, 'cid', cid)
def get_cid():
return getattr(_thread_locals, 'cid', '')
def __init__(self):
self.storage = local()
