def _get_greenlet_lock(self):
if not hasattr(self._thread_local, 'greenlet_lock'):
greenlet_lock = self._thread_local.greenlet_lock = BoundedSemaphore(
1)
else:
greenlet_lock = self._thread_local.greenlet_lock
return greenlet_lock
def __init__(self,
account,
heartbeat=1,
refresh_frequency=30,
poll_frequency=30,
retry_fail_classes=[],
refresh_flags_max=2000):
self.refresh_frequency = refresh_frequency
self.poll_frequency = poll_frequency
self.syncmanager_lock = BoundedSemaphore(1)
self.refresh_flags_max = refresh_flags_max
self.saved_remote_folders = None
provider_supports_condstore = account.provider_info.get(
'condstore', False)
account_supports_condstore = getattr(account, 'supports_condstore',
False)
if provider_supports_condstore or account_supports_condstore:
self.sync_engine_class = CondstoreFolderSyncEngine
else:
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
BaseMailSyncMonitor.__init__(self, account, heartbeat,
retry_fail_classes)
def __init__(self, module, server_list):
# logging
logger = logging.getLogger(module)
logger.setLevel(logging.INFO)
try:
handler = logging.handlers.RotatingFileHandler(
'/var/log/contrail/' + module + '-zk.log',
maxBytes=10 * 1024 * 1024,
backupCount=5)
except IOError:
print "Cannot open log file in /var/log/contrail/"
else:
log_format = logging.Formatter(
'%(asctime)s [%(name)s]: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
handler.setFormatter(log_format)
logger.addHandler(handler)
self._zk_client = \
kazoo.client.KazooClient(
server_list,
handler=kazoo.handlers.gevent.SequentialGeventHandler(),
logger=logger)
self._logger = logger
self._election = None
self._zk_sem = BoundedSemaphore(1)
self.connect()
def sync(self):
sync_folder_names_ids = self.prepare_sync()
thread_download_lock = BoundedSemaphore(1)
for folder_name, folder_id in sync_folder_names_ids:
log.info('initializing folder sync')
thread = GmailFolderSyncEngine(
thread_download_lock, self.account_id, folder_name, folder_id,
self.email_address, self.provider_name, self.poll_frequency,
self.syncmanager_lock, self.refresh_flags_max,
self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
if thread.should_block:
while not self._thread_polling(thread) and \
not self._thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def __init__(
self,
syncback_id,
process_number,
total_processes,
poll_interval=1,
retry_interval=120,
num_workers=NUM_PARALLEL_ACCOUNTS,
batch_size=20,
fetch_batch_size=100,
):
self.process_number = process_number
self.total_processes = total_processes
self.poll_interval = poll_interval
self.retry_interval = retry_interval
# Amount of log entries to fetch before merging/de-duplication to
# determine which records need to be processed.
self.fetch_batch_size = fetch_batch_size
# Amount of log entries to process in a batch.
self.batch_size = batch_size
self.keep_running = True
self.workers = gevent.pool.Group()
# Dictionary account_id -> semaphore to serialize action syncback for
# any particular account.
# TODO(emfree): We really only need to serialize actions that operate
# on any given object. But IMAP actions are already effectively
# serialized by using an IMAP connection pool of size 1, so it doesn't
# matter too much.
self.account_semaphores = defaultdict(lambda: BoundedSemaphore(1))
# This SyncbackService performs syncback for only and all the accounts
# on shards it is reponsible for; shards are divided up between
# running SyncbackServices.
self.log = logger.new(component="syncback")
syncback_assignments = {
int(k): v
for k, v in config.get("SYNCBACK_ASSIGNMENTS", {}).items()
}
if syncback_id in syncback_assignments:
self.keys = [
key for key in engine_manager.engines
if key in syncback_assignments[syncback_id] and key %
total_processes == process_number
]
else:
self.log.warn("No shards assigned to syncback server",
syncback_id=syncback_id)
self.keys = []
self.log = logger.new(component="syncback")
self.num_workers = num_workers
self.num_idle_workers = 0
self.worker_did_finish = gevent.event.Event()
self.worker_did_finish.clear()
self.task_queue = Queue()
self.running_action_ids = set()
gevent.Greenlet.__init__(self)
def __init__(self, redis_uve_server, logger):
self._local_redis_uve = redis_uve_server
self._redis_uve_list = []
self._logger = logger
self._sem = BoundedSemaphore(1)
self._redis = None
if self._local_redis_uve:
self._redis = redis.StrictRedis(self._local_redis_uve[0],
self._local_redis_uve[1], db=0)
def connectionMade(self):
"""连接建立处理
"""
address = self.transport.getAddress()
logger.info('Client %d login in.[%s,%d]' % (self.transport.sessionno,
address[0], address[1]))
self.factory.connmanager.addConnection(self)
self.factory.doConnectionMade(self)
self.sem = BoundedSemaphore(1)
def __init__(self, socket, client_class):
self.socket = None
self.mutex = BoundedSemaphore(1)
self.socket = TSocket.TSocket(unix_socket=socket)
self.transport = TTransport.TBufferedTransport(self.socket)
self.protocol = TBinaryProtocol.TBinaryProtocolAccelerated(
self.transport)
self.client = client_class(self.protocol)
self.connected = False
def __init__(self, account, heartbeat=1, refresh_frequency=30):
self.refresh_frequency = refresh_frequency
self.syncmanager_lock = BoundedSemaphore(1)
self.saved_remote_folders = None
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
BaseMailSyncMonitor.__init__(self, account, heartbeat)
def __init__(self, poll_interval=1, chunk_size=100):
semaphore_factory = lambda: BoundedSemaphore(CONCURRENCY_LIMIT)
self.semaphore_map = defaultdict(semaphore_factory)
self.keep_running = True
self.running = False
self.log = logger.new(component='syncback')
self.poll_interval = poll_interval
self.chunk_size = chunk_size
self._scheduled_actions = set()
gevent.Greenlet.__init__(self)
def __init__(self, account_id, num_connections, readonly):
logger.info('Creating Crispin connection pool for account {} with {} '
'connections'.format(account_id, num_connections))
self.account_id = account_id
self.readonly = readonly
self._new_conn_lock = BoundedSemaphore(1)
self._set_account_info()
# 1200s == 20min
geventconnpool.ConnectionPool.__init__(
self, num_connections, keepalive=1200,
exc_classes=CONN_DISCARD_EXC_CLASSES)
def __init__(self, redis_uve_server, logger, redis_password=None):
self._local_redis_uve = redis_uve_server
self._redis_uve_list = []
self._logger = logger
self._sem = BoundedSemaphore(1)
self._redis = None
self._redis_password = redis_password
if self._local_redis_uve:
self._redis = redis.StrictRedis(self._local_redis_uve[0],
self._local_redis_uve[1],
password=self._redis_password,
db=1)
def __init__(self,
discServer,
zk_srv_ip='127.0.0.1',
zk_srv_port='2181',
reset_config=False):
self._reset_config = reset_config
self._service_id_to_type = {}
self._ds = discServer
self._zk_sem = BoundedSemaphore(1)
self._election = None
self._restarting = False
zk_endpts = []
for ip in zk_srv_ip.split(','):
zk_endpts.append('%s:%s' % (ip, zk_srv_port))
# logging
logger = logging.getLogger('discovery-service')
logger.setLevel(logging.WARNING)
handler = logging.handlers.RotatingFileHandler(
'/var/log/contrail/discovery_zk.log',
maxBytes=1024 * 1024,
backupCount=10)
log_format = logging.Formatter('%(asctime)s [%(name)s]: %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p')
handler.setFormatter(log_format)
logger.addHandler(handler)
self._zk = kazoo.client.KazooClient(
hosts=','.join(zk_endpts),
handler=kazoo.handlers.gevent.SequentialGeventHandler(),
logger=logger)
self._logger = logger
# connect
self.connect()
if reset_config:
self.delete_node("/services", recursive=True)
self.delete_node("/clients", recursive=True)
self.delete_node("/election", recursive=True)
# create default paths
self.create_node("/services")
self.create_node("/clients")
self.create_node("/election")
self._debug = {
'subscription_expires': 0,
'oos_delete': 0,
'db_excepts': 0,
}
def __init__(self, localServer):
Greenlet.__init__(self)
self.callinfos = {}
self.localServer = localServer
self.recv_queue = Queue()
self.lock = BoundedSemaphore(1)
self.handle_stoping = False
self.recv_stop_evt = Event()
self.timeout_stop_evt = Event()
self.timeout_handle_greenlet = gevent.spawn(self.on_timeout_handle)
self.timeout_handle_greenlet.start()
class Cache(dict):
semaphore = BoundedSemaphore()
def __setattr__(self, key, value):
with self.semaphore:
self[key] = value
def __delattr__(self, key):
with self.semaphore:
try:
del self[key]
except KeyError, k:
raise AttributeError, k
def __init__(self, poll_interval=1, retry_interval=30):
self.log = logger.new(component='syncback')
self.keep_running = True
self.poll_interval = poll_interval
self.retry_interval = retry_interval
self.workers = gevent.pool.Group()
# Dictionary account_id -> semaphore to serialize action syncback for
# any particular account.
# TODO(emfree): We really only need to serialize actions that operate
# on any given object. But IMAP actions are already effectively
# serialized by using an IMAP connection pool of size 1, so it doesn't
# matter too much.
self.account_semaphores = defaultdict(lambda: BoundedSemaphore(1))
gevent.Greenlet.__init__(self)
def __init__(self, f, block=True):
if isinstance(f, file):
self.filename = f.name
self.handle = f if not f.closed else open(f, 'w')
else:
self.filename = f
mkdirp(os.path.dirname(f))
self.handle = open(f, 'w')
if block:
self.lock_op = fcntl.LOCK_EX
else:
self.lock_op = fcntl.LOCK_EX | fcntl.LOCK_NB
self.block = block
self.gevent_lock = BoundedSemaphore(1)
def __init__(self, args):
self.thread = None
self.socket = None
self.cli_args = args
self.force_debug = args['debug']
# timers, counters and triggers
self.pokemon_caught = 0
self._error_counter = 0
self._error_threshold = 10
self.start_time = time()
self.exp_start = None
self._heartbeat_number = 1 # setting this back to one because we make parse a full heartbeat during login!
self._heartbeat_frequency = 3 # 1 = always
self._full_heartbeat_frequency = 15 # 10 = as before (every 10th heartbeat)
self._farm_mode_triggered = False
# objects, order is important!
self.config = None
self._load_config()
self.log = create_logger(__name__,
self.config.log_colors["poketrainer".upper()])
self._open_socket()
self.player = Player({})
self.player_stats = PlayerStats({})
self.inventory = Inventory(self, [])
self.fort_walker = FortWalker(self)
self.map_objects = MapObjects(self)
self.poke_catcher = PokeCatcher(self)
self.incubate = Incubate(self)
self.evolve = Evolve(self)
self.release = Release(self)
self.sniper = Sniper(self)
self._origPosF = (0, 0, 0)
self.api = None
self._load_api()
# config values that might be changed during runtime
self.step_size = self.config.step_size
self.should_catch_pokemon = self.config.should_catch_pokemon
# threading / locking
self.sem = BoundedSemaphore(1) # gevent
self.persist_lock = False
self.locker = None
def __init__(self, size, exc_classes=DEFAULT_EXC_CLASSES, keepalive=None):
self.size = size
self.conn = deque()
self.lock = BoundedSemaphore(size)
self.keepalive = keepalive
# Exceptions list must be in tuple form to be caught properly
self.exc_classes = tuple(exc_classes)
for i in xrange(size):
self.lock.acquire()
for i in xrange(size):
greenlet = TestRunGreenlet(self._addOne)
greenlet.start_later(self.SPAWN_FREQUENCY * i)
if self.keepalive:
greenlet = TestRunGreenlet(self._keepalive_periodic)
greenlet.start_later()
def __init__(self, amqp_info):
Greenlet.__init__(self)
ExampleConsumer.__init__(self, amqp_info)
self.callinfos = {}
self.send_queue = Queue()
self.lock = BoundedSemaphore(1)
self.send_greenlet = None
self.handle_stoping = False
self.send_stop_evt = Event()
self.timeout_stop_evt = Event()
self.timeout_handle_greenlet = gevent.spawn(self.on_timeout_handle)
self.timeout_handle_greenlet.start()
def __init__(self, size, exc_classes=DEFAULT_EXC_CLASSES, keepalive=None):
"""
:param exc_classes: tuple, exceptions which connection should be dropped when it raises
"""
self.size = size
self.connections = deque()
self.lock = BoundedSemaphore(size)
self.keepalive = keepalive
# Exceptions list must be in tuple form to be caught properly
self.exc_classes = tuple(exc_classes)
for i in xrange(size):
self.lock.acquire()
for i in xrange(size):
gevent.spawn_later(self.SPAWN_FREQUENCY * i, self._add_one)
if self.keepalive:
gevent.spawn(self._keepalive_periodic)
def __init__(self, size, exc_classes=DEFAULT_EXC_CLASSES, keepalive=None):
self.size = size
self.conn = deque()
self.lock = BoundedSemaphore(size)
self.keepalive = keepalive
# Exceptions list must be in tuple form to be caught properly
self.exc_classes = tuple(exc_classes)
# http://stackoverflow.com/a/31136897/357578
try:
xrange
except NameError:
xrange = range
for i in xrange(size):
self.lock.acquire()
for i in xrange(size):
gevent.spawn_later(self.SPAWN_FREQUENCY * i, self._addOne)
if self.keepalive:
gevent.spawn(self._keepalive_periodic)
def process_parallely(self, year, last_page):
"""
Process various pages of a particular year parallely
@param year (Number): year
@param last_page (Number): Last page number
"""
threads = []
params = dict(PARAMS)
params["year"] = year
for page in xrange(1, last_page):
params["page"] = page
url = "https://www.codechef.com/submissions?" + urlencode(params)
threads.append(gevent.spawn(self.process_page, year, page, url))
self.semaphore = BoundedSemaphore(len(threads))
gevent.joinall(threads)
def __init__(self,
discServer,
zk_srv_ip='127.0.0.1',
zk_srv_port='2181',
reset_config=False):
self._reset_config = reset_config
self._service_id_to_type = {}
self._ds = discServer
self._zk_sem = BoundedSemaphore(1)
self._zk = kazoo.client.KazooClient(
hosts='%s:%s' % (zk_srv_ip, zk_srv_port),
timeout=120,
handler=kazoo.handlers.gevent.SequentialGeventHandler())
# connect
while True:
try:
self._zk.start()
break
except gevent.event.Timeout as e:
self.syslog(
'Failed to connect with Zookeeper - will retry in a second'
)
gevent.sleep(1)
self.syslog('Connected to ZooKeeper!')
if reset_config:
self._zk.delete("/services", recursive=True)
self._zk.delete("/clients", recursive=True)
self._zk.delete("/publishers", recursive=True)
self._zk.delete("/election", recursive=True)
# create default paths
self.create_node("/services")
self.create_node("/clients")
self.create_node("/publishers")
self.create_node("/election")
self._debug = {
'subscription_expires': 0,
'oos_delete': 0,
'db_excepts': 0,
}
def __init__(self, input, output, func, nthreads=800, maxcnt=None):
"""@todo: to be defined
:param input: @todo
:param output: @todo
:param func: @todo
:param qname: @todo
"""
self._func = func
self._input = input
self._output = output
self._lock = BoundedSemaphore(1)
self._pool = Pool(nthreads)
self._nthreads = nthreads
self._true = 0
self._false = 0
self._nogeo = 0
self._notruth = 0
self.maxcnt = maxcnt
def load(self, spider):
redis_args = dict(host=self.settings.REDIS_URL,
port=self.settings.REDIS_PORT,
db=self.settings.REDIS_DB)
if hasattr(self.settings, 'NAMESPACE'):
redis_args['namespace'] = self.settings.NAMESPACE
else:
redis_args['namespace'] = spider.name
self.url_set = redisds.Set('urlset', **redis_args)
self.url_queue = redisds.Queue('urlqueue',
serializer=Pickle(),
**redis_args)
self.runner = redisds.Lock("runner:%s" % uuid4().hex, **redis_args)
self.runners = redisds.Dict("runner:*", **redis_args)
self.stats = redisds.Dict("stats:*", **redis_args)
self.lock = BoundedSemaphore(1)
self.running_count = 0
self.allowed_urls_regex = self.get_regex(spider.allowed_domains)
self.spider = spider
self.start()
def __init__(self,
account,
heartbeat=1,
refresh_frequency=30,
syncback_frequency=5):
# DEPRECATED.
# TODO[k]: Remove after sync-syncback integration deploy is complete.
self.refresh_frequency = refresh_frequency
self.syncmanager_lock = BoundedSemaphore(1)
self.saved_remote_folders = None
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
self.delete_handler = None
self.syncback_handler = None
self.folder_sync_signals = {}
self.syncback_timestamp = None
self.syncback_frequency = syncback_frequency
BaseMailSyncMonitor.__init__(self, account, heartbeat)
def __init__(self, name, force_new_connection=False):
"""@todo: to be defined
:param name: @todo
:param force_new_connection: @todo
"""
if not INITTED:
log.warn("QUEUE INIT Not called, calling")
init()
self._name = RPC_PREFIX + name
self._connection = connect(force_new_connection)
self._channel = self._connection.channel()
self._queue = self._connection.SimpleQueue(
"", queue_opts={"exclusive": True})
self._marshal = queue.JsonMarshal()
self._producer = messaging.Producer(self._channel,
routing_key=self._name)
self._lock = BoundedSemaphore(1)
self.reqs = {}
def mass_deauth(network, iface):
sniff_ap(iface, network)
channels = set()
for bssid in network:
if 'channel' in network[bssid] and network[bssid]['channel'] < 13:
channels.add(network[bssid]['channel'])
q = Queue()
lock = BoundedSemaphore(1)
t3 = gevent.spawn(switch_channel, lock, iface, list(channels))
t1 = gevent.spawn(sniff_data, q, lock, iface)
t2 = gevent.spawn(add_network, q, network)
t4 = gevent.spawn(deauth_attack, lock, network, iface)
ts = [t1, t2, t3, t4]
gevent.signal(signal.SIGINT, gevent.killall, ts)
try:
gevent.joinall(ts, timeout=600)
except KeyboardInterrupt:
pass
finally:
print('---kill all---')
gevent.killall(ts)
def __init__(self,
syncback_id,
cpu_id,
total_cpus,
poll_interval=1,
retry_interval=30):
self.cpu_id = cpu_id
self.total_cpus = total_cpus
self.poll_interval = poll_interval
self.retry_interval = retry_interval
self.keep_running = True
self.workers = gevent.pool.Group()
# Dictionary account_id -> semaphore to serialize action syncback for
# any particular account.
# TODO(emfree): We really only need to serialize actions that operate
# on any given object. But IMAP actions are already effectively
# serialized by using an IMAP connection pool of size 1, so it doesn't
# matter too much.
self.account_semaphores = defaultdict(lambda: BoundedSemaphore(1))
# This SyncbackService performs syncback for only and all the accounts
# on shards it is reponsible for; shards are divided up between
# running SyncbackServices.
self.log = logger.new(component='syncback')
syncback_assignments = config.get("SYNCBACK_ASSIGNMENTS", {})
if syncback_id in syncback_assignments:
self.keys = [
key for key in engine_manager.engines
if key in syncback_assignments[syncback_id] and key %
total_cpus == cpu_id
]
else:
self.log.warn("No shards assigned to syncback server",
syncback_id=syncback_id)
self.keys = []
self.log = logger.new(component='syncback')
gevent.Greenlet.__init__(self)
