class RandomProxy(object):
def __init__(self):
self.current_proxy = None
self.lock = DeferredLock()
def process_request(self, request, spider):
pass
# user_agent = random.choice(USER_AGENT_LIST)
# request.headers['User-Agent'] = user_agent
# if 'proxy' not in request.meta or self.current_proxy.is_expiring:
# #请求代理
# self.update_proxy()
# request.meta['proxy'] = self.current_proxy.proxy
def process_response(self, request, response, spider):
# 如果对方重定向（302）去验证码的网页，换掉代理IP
# 'captcha' in response.url 指的是有时候验证码的网页返回的状态码是200，所以用这个作为辨识的标志
if response.status != 200 or 'captcha' in response.url:
# 如果来到这里，说明这个请求已经被boss直聘识别为爬虫了
# 所以这个请求就相当于什么都没有获取到
# 所以要重新返回request，让这个请求重新加入到调度中
# 下次再发送
# if not self.current_proxy.blacked:
# self.current_proxy.blacked = True
# self.update_proxy()
# print('%s代理失效' % self.current_proxy.proxy)
# request.meta['proxy'] = self.current_proxy.proxy
return request
# 如果是正常的话，记得最后要返回response
# 如果不返回，这个response就不会被传到爬虫那里去
# 也就得不到解析
return response
def update_proxy(self):
#lock是属于多线程中的一个概念，因为这里scrapy是采用异步的，可以直接看成多线程
#所以有可能出现这样的情况，爬虫在爬取一个网页的时候，忽然被对方封了，这时候就会来到这里
#获取新的IP，但是同时会有多条线程来这里请求，那么就会出现浪费代理IP的请求，所以这这里加上了锁
#锁的作用是在同一时间段，所有线程只能有一条线程可以访问锁内的代码，这个时候一条线程获得新的代理IP
#而这个代理IP是可以用在所有线程的，这样子别的线程就可以继续运行了，减少了代理IP（钱）的浪费
self.lock.acquire()
# 判断换线程的条件
# 1.目前没有使用代理IP
# 2.到线程过期的时间了
# 3.目前IP已经被对方封了
# 满足以上其中一种情况就可以换代理IP了
if not self.current_proxy or self.current_proxy.is_expiring or self.current_proxy.blacked:
url = r'https://h.wandouip.com/get/ip-list?pack=%s&num=1&xy=1&type=2&lb=\r\n&mr=1&' % random.randint(
100, 1000)
response = requests.get(url=url, headers=DEFAULT_REQUEST_HEADERS)
text = json.loads(response.text)
print(text)
data = text['data'][0]
proxy_model = ProxyModel(data)
print('重新获取了一个代理：%s' % proxy_model.proxy)
self.current_proxy = proxy_model
# return proxy_model
self.lock.release()
def initServer(self):
"""Initialize the server after connecting to LabRAD."""
self.knownDevices = {} # maps (server, channel) to (name, idn)
self.deviceServers = {
} # maps device name to list of interested servers.
# each interested server is {'target':<>,'context':<>,'messageID':<>}
self.identFunctions = {} # maps server to (setting, ctx) for ident
self.identLock = DeferredLock()
# named messages are sent with source ID first, which we ignore
def connect_func(c, s_payload):
(s, payload) = s_payload
return self.gpib_device_connect(*payload)
def disconnect_func(c, s_payload):
(s, payload) = s_payload
return self.gpib_device_disconnect(*payload)
mgr = self.client.manager
self._cxn.addListener(connect_func, source=mgr.ID, ID=10)
self._cxn.addListener(disconnect_func, source=mgr.ID, ID=11)
yield mgr.subscribe_to_named_message('GPIB Device Connect', 10, True)
yield mgr.subscribe_to_named_message('GPIB Device Disconnect', 11,
True)
# do an initial scan of the available GPIB devices
yield self.refreshDeviceLists()
def __init__(self, priority):
self._lock = DeferredLock()
self.priority = str(priority)
self.is_shutdown = False
self.state_pub = state_publisher.StatePublisher(())
self.flush()
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
class IPProxyMiddleware():
PROXY_URL = "http://webapi.http.zhimacangku.com/getip?num=3&type=2&pro=&city=0&yys=0&port=11&time=2&ts=1&ys=0&cs=0&lb=1&sb=0&pb=45&mr=1®ions="
def __init__(self):
super(IPProxyMiddleware, self).__init__()
self.current_proxy = None
self.lock = DeferredLock()
def process_request(self, request, spider):
if 'proxy' not in request.meta or self.current_proxy.is_expireing:
#请求代理
self.update_proxy()
request.meta['proxy'] = self.current_proxy.proxy
def process_response(self, request, response, spider):
if response.status != 200 or "captcha" in response.url:
if not self.current_proxy.blacked:
self.current_proxy.blacked = True
print('%s这个代理被加入黑名单了' % self.current_proxy.ip)
self.update_proxy()
return request
return response
def update_proxy(self):
self.lock.acquire()
if not self.current_proxy or self.current_proxy.is_expireing or self.current_proxy.blacked:
response = requests.get(self.PROXY_URL)
text = response.text
print("重新获取了一个代理：", text)
result = json.loads(text)
if len(result['data']) > 0:
data = result['data'][random.randint(0, 2)]
proxy_model = ProxyModel(data)
self.current_proxy = proxy_model
self.lock.release()
class IPProxyDownloadMiddlware(object):
PROXY_URL = '购买的代理链接'
def __init__(self):
super(IPProxyDownloadMiddlware, self).__init__()
self.current_proxy = None
self.lock= DeferredLock()
def prooess_request(self, request, spider):
if 'proxy' not in request.meta or self.current_proxy.is_expiring:
self.update_proxy()
request.meta['proxy'] = self.current_proxy.proxy
# 请求代理
def prooess_response(self,request, response, spider):
if response.status !=200 or 'captcha' in response.url:
self.update_proxy()
return request
return response
def update_proxy(self):
self.lock.acquire()
if not self.current_proxy or self.current_proxy.is_expiring:
response = requests.get(self.PROXY_URL)
text = response.text
result = json.loads(text)
if len(result['data'])>0:
data = result['data'][0]
proxy_model = ProxyModel(data)
self.current_proxy = proxy_model
return proxy_model
self.lock.release()
class Client(object):
def __init__(self, specFilename, exchange='signals'):
self.exchange = exchange
spec = txamqp.spec.load(specFilename)
delegate = TwistedDelegate()
self.clientConnected = ClientCreator(reactor, AMQClient,
delegate=delegate, vhost="/",
spec=spec).connectTCP("localhost", 5672)
self.conn = None
self.chan = None
self.finishLock = DeferredLock()
@inlineCallbacks
def finishChannelOpen(self):
yield self.finishLock.acquire()
if self.conn is None:
print "opening connection for", self
self.conn = yield self.clientConnected
authentication = {"LOGIN": "******", "PASSWORD": "******"}
yield self.conn.start(authentication)
if self.chan is None:
self.chan = yield self.conn.channel(1)
yield self.chan.channel_open()
yield self.newChan()
print "made channel for", self
self.finishLock.release()
def newChan(self):
# called once when the new channel is opened
return succeed(None)
def __init__(self, nodedir=None, executable=None):
self.executable = executable
self.multi_folder_support = True
if nodedir:
self.nodedir = os.path.expanduser(nodedir)
else:
self.nodedir = os.path.join(os.path.expanduser('~'), '.tahoe')
self.rootcap_path = os.path.join(self.nodedir, 'private', 'rootcap')
self.servers_yaml_path = os.path.join(self.nodedir, 'private',
'servers.yaml')
self.config = Config(os.path.join(self.nodedir, 'tahoe.cfg'))
self.pidfile = os.path.join(self.nodedir, 'twistd.pid')
self.nodeurl = None
self.shares_happy = None
self.name = os.path.basename(self.nodedir)
self.api_token = None
self.magic_folders_dir = os.path.join(self.nodedir, 'magic-folders')
self.lock = DeferredLock()
self.rootcap = None
self.magic_folders = defaultdict(dict)
self.remote_magic_folders = defaultdict(dict)
self.use_tor = False
self.monitor = Monitor(self)
self._monitor_started = False
self.state = Tahoe.STOPPED
def __init__(self, nodedir=None, executable=None, reactor=None):
if reactor is None:
from twisted.internet import reactor
self.executable = executable
self.multi_folder_support = True
if nodedir:
self.nodedir = os.path.expanduser(nodedir)
else:
self.nodedir = os.path.join(os.path.expanduser('~'), '.tahoe')
self.rootcap_path = os.path.join(self.nodedir, 'private', 'rootcap')
self.servers_yaml_path = os.path.join(self.nodedir, 'private',
'servers.yaml')
self.config = Config(os.path.join(self.nodedir, 'tahoe.cfg'))
self.pidfile = os.path.join(self.nodedir, 'twistd.pid')
self.nodeurl = None
self.shares_happy = None
self.name = os.path.basename(self.nodedir)
self.api_token = None
self.magic_folders_dir = os.path.join(self.nodedir, 'magic-folders')
self.lock = DeferredLock()
self.rootcap = None
self.magic_folders = defaultdict(dict)
self.remote_magic_folders = defaultdict(dict)
self.use_tor = False
self.monitor = Monitor(self)
streamedlogs_maxlen = None
debug_settings = global_settings.get('debug')
if debug_settings:
log_maxlen = debug_settings.get('log_maxlen')
if log_maxlen is not None:
streamedlogs_maxlen = int(log_maxlen)
self.streamedlogs = StreamedLogs(reactor, streamedlogs_maxlen)
self.state = Tahoe.STOPPED
self.newscap = ""
self.newscap_checker = NewscapChecker(self)
class UAMiddleware(object):
def __init__(self):
self.lock = DeferredLock()
self.update_time = datetime.now()
self.UA_List = USER_AGENT
def process_request(self, request, spider):
self.lock.acquire()
if self.is_expiring:
ua = random.choice(self.UA_List)
request.headers['User-Agent'] = ua
print(request.headers['User-Agent'])
self.lock.release()
def process_response(self, request, response, spider):
return response
def process_exception(self, request, exception, spider):
pass
@property
def is_expiring(self):
now = datetime.now()
if (now - self.update_time) > timedelta(seconds=30):
self.update_time = datetime.now()
print("跟换USER_AGENT")
return True
else:
return False
class IPProxyDownloadMiddleware(object):
PROXY_URL = 'http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=&city=0&yys=0&port=11&pack=61856&ts=1&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions='
def __init__(self):
super().__init__()
self.current_proxy = None
self.lock = DeferredLock()
def process_request(self, request, spider):
if 'proxy' not in request.meta or self.current_proxy.is_expiring:
self.update_proxy()
request.meta['proxy'] = self.current_proxy.proxy
def process_response(self, request, response, spider):
if response.status != 200 or 'captcha' in response.url:
if not self.current_proxy.blacked:
self.current_proxy.blacked = True
self.update_proxy()
print('%s这个代理被加入黑名单了' % self.current_proxy)
return request
return response
def update_proxy(self):
self.lock.acquire()
if not self.current_proxy or self.current_proxy.is_expiring or self.current_proxy.blacked:
response = requests.get(self.PROXY_URL)
text = response.text
print('重新获取了一个代理:', text)
result = json.loads(text)
if len(result['data']) > 0:
data = result['data'][0]
proxy_model = ProxyModel(data)
self.current_proxy = proxy_model
self.current_proxy.blacked = False
self.lock.release()
def __init__(self, node, register, simNum, num=0):
# Node where this qubit is located
self.node = node
# Register where this qubit is simulated
self.register = register
# Number in the register, if known
self.num = num
# Number of the simulated qubit, unique at each virtual node
self.simNum = simNum
# Lock marshalling access to this qubit
self._lock = DeferredLock()
# Mark this qubit as active (still connected to a register)
self.active = True
# Time until retry
self._delay = 1
# Optional parameters for when the simulation is noise
self.noisy = settings.simulaqron_settings.noisy_qubits
self.T1 = settings.simulaqron_settings.t1
self.last_accessed = time.time()
self._logger = get_netqasm_logger(
f"{self.__class__.__name__}(node={node.name}, sim_num={simNum})")
class IPProxyDownloadMiddleware(object):
PROXY_URL = ''
def __init__(self):
super(IPProxyDownloadMiddleware, self).__init__()
self.current_proxy = None
self.lock = DeferredLock()
def process_request(self, request, spider):
if 'proxy' not in request.meta or self.current_proxy.is_expiring:
self.update_proxy()
request.meta['proxy'] = self.current_proxy.proxy
def process_response(self, request, response, spider):
if response.status != 200 or "captcha" in response.url:
if not self.current_proxy.blacked:
self.current_proxy.blacked = True
self.update_proxy()
# 如果来到这里说明这个请求被识别为爬虫，所以这个请求就没有获取到，要重新请求
return request
# 来到这里是正常的，要返回response给爬虫解析
return response
def update_proxy(self):
self.lock.acquire()
if not self.current_proxy or self.current_proxy.is_expiring or self.current_proxy.blacked:
response = requests.get(self.PROXY_URL)
text = response.text
print(text)
result = json.loads(text)
if len(result['data']) > 0:
data = result['data'][0]
proxy_model = ProxyModel(data)
self.current_proxy = proxy_model
self.lock.release()
def __init__(self, config=None):
ApplicationSession.__init__(self, config)
global live
live = self
self.logger = logging.getLogger('Live')
self.logger.info("Config: %s", config)
self.account_id = config.extra['authid']
self.secret = config.extra['secret']
if '-' not in self.account_id:
self.account_id = "local-%s" % self.account_id
self.authid = '%s:%s' % (self.account_id, self.secret[-7:])
self.joined = False
self.lock = DeferredLock()
self.checks = {}
self.workers = {}
self.CallOptions = CallOptions()
class IPProxyDownloadMiddleware(object):
PROXY_URL = 'http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=&city=0&yys=0&port=11&time=1&ts=1&ys=0&cs=0&lb=1&sb=0&pb=45&mr=1®ions='
def __init__(self):
super(IPProxyDownloadMiddleware, self).__init__()
self.lock = DeferredLock()
self.current_proxy = None
self.username = None
def process_request(self, request, spider):
if not self.current_proxy:
self.update_proxy()
return request
if self.current_proxy.is_expiring:
self.update_proxy()
request.meta['proxy'] = self.current_proxy.proxy
print(request.meta['proxy'])
def process_response(self, request, response, spider):
if response.status != 200:
self.update_proxy()
return request
return response
def update_proxy(self):
self.lock.acquire()
if not self.current_proxy or self.current_proxy.is_expiring:
response = requests.get(self.PROXY_URL)
result = json.loads(response.text)
data = result['data']
proxy = ProxyModel(data[0])
print('请求到了：', response.text)
self.current_proxy = proxy
self.lock.release()
class ProxyDownloaderMiddleware(object):
def __init__(self):
super(ProxyDownloaderMiddleware, self).__init__()
self.PROXY_URL = "http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=&city=0&yys=0&port=11&time=1&ts=1&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions="
self.current_proxy = None
self.lock = DeferredLock()
def process_request(self, request, spider):
if 'proxy' not in request.meta or self.current_proxy.is_expiring:
self.upgrade_proxy()
request.meta['proxy'] = self.current_proxy.proxy
def process_process(self, request, response, spider):
if response.status != 200:
self.upgrade_proxy()
return request
return response
def upgrade_proxy(self):
self.lock.acquire()
if not self.current_proxy or self.current_proxy.is_expiring:
resp = requests.get(self.PROXY_URL)
result = resp.text
data = json.loads(result)['data'][0]
if data:
proxy_model = ProxyModel(data)
self.current_proxy = proxy_model
self.lock.release()
class IPProxy(object):
PROXY_URL = "http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=0&city=0&yys=0&port=1&pack=21267&ts=1&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions="
def __init__(self):
self.lock = DeferredLock()
self.current_proxy = None
def process_request(self, request, spider):
if 'proxy' not in request.meta or self.current_proxy.is_expiring:
self.update_proxy()
request.meta['proxy'] = self.current_proxy.proxy
def process_response(self, request, response, spider):
if response.status != 200:
if not self.current_proxy.is_block:
self.current_proxy.is_block = True
self.update_proxy()
return request
return response
def update_proxy(self):
self.lock.acquire()
if self.current_proxy is None or self.current_proxy.is_expiring or self.current_proxy.is_block:
response_json = requests.get(self.PROXY_URL).json()
try:
print(response_json)
self.current_proxy = ProxyModel(response_json['data'][0])
except:
print('出错了！')
print(response_json)
self.lock.release()
def initServer(self):
self.collectionTime = {
0: 1.0,
1: 1.0
} #default collection times in the format channel:time(sec)
self.inCommunication = DeferredLock()
self.connectOKBoard()
def __init__(self, settings):
super(My_RetryMiddleware, self).__init__(settings)
self.lock = DeferredLock()
self.seconds = settings.getint('expire_datetime')
self.proxy_url = settings.get('PROXY_URL')
self.expire_datetime = datetime.datetime.now() + datetime.timedelta(
seconds=self.seconds)
self.proxy = None
def __init__(self, kernel_key, conda_env=conda_env, env_path=env_path, pkg_path=pkg_path, config=ComponentConfig(realm=u"jupyter")):
ApplicationSession.__init__(self, config=config)
self._kernel_key = kernel_key
self._conda_env = conda_env
self._env_path = env_path
self._pkg_path = pkg_path
self._conda_env = conda_env
self._lock = DeferredLock()
def initServer(self):
self.api_dac = api_dac()
self.inCommunication = DeferredLock()
connected = self.api_dac.connectOKBoard()
if not connected:
raise Exception ("Could not connect to DAC")
self.d = yield self.initializeDAC()
self.listeners = set()
def __init__(self, bandwidth=5):
"""
:param bandwidth: максимальная длина очереди, по превышению которой запросы будут отсекаться
"""
self.tasks = []
self.bandwidth = bandwidth
self._lock = DeferredLock()
def __init__(self, spider_name):
"""
初始化付费代理池, 短效代理
ip, port, 来源网站, 获取时间, 是否被封, 是否过期, 过期时间
"""
self.redis = redis.StrictRedis(host=PROXY_REDIS_HOST, port=PROXY_REDIS_PORT, db=PROXY_REDIS_DB)
self.lock = DeferredLock()
self._init_proxy_pool(spider_name)
def __init__(self, cxn, context, dataset):
super(Dataset, self).__init__()
self.accessingData = DeferredLock()
self.cxn = cxn
self.context = context # context of the first dataset in the window
self.dataset = dataset
self.data = None
self.setupDataListener(self.context)
def __init__(self, name, conf):
_log.info("CF_INIT %s", name)
self.name, self.conf = name, conf
self.channel_dict = defaultdict(list)
self.iocs = dict()
self.client = None
self.currentTime = getCurrentTime
self.lock = DeferredLock()
def initServer(self):
self.channelDict = hardwareConfiguration.channelDict
self.collectionTime = hardwareConfiguration.collectionTime
self.collectionMode = hardwareConfiguration.collectionMode
self.sequenceType = hardwareConfiguration.sequenceType
self.isProgrammed = hardwareConfiguration.isProgrammed
self.inCommunication = DeferredLock()
self.connectOKBoard()
self.listeners = set()
def __init__(self):
super(IPProxyDownloadMiddleware, self).__init__()
self.current_proxy = None
self.lock = DeferredLock()
self.blacked = False
self.get_proxy = False
self.proxies = []
# self.proxies_time = 9999999999
self.proxies_time = time.time()
class IPProxyDownloaderMiddleware(object):
'''
IP代理 ，
'''
# 获取代理ip信息地址 例如芝麻代理、快代理等
IP_URL = r'http://127.0.0.1:8000/?types=0&count=1&country=国内'
def __init__(self):
# super(IPProxyDownloaderMiddleware, self).__init__(self)
super(IPProxyDownloaderMiddleware, self).__init__()
self.current_proxy = None
self.lock = DeferredLock()
def process_request(self, request, spider):
if 'proxy' not in request.meta or self.current_proxy.is_expire:
self.updateProxy()
request.meta['proxy'] = self.current_proxy.address
def process_response(self, request, response, spider):
if response.status != 200:
# 如果来到这里，这个请求相当于被识别为爬虫了
# 所以这个请求被废掉了
# 如果不返回request,那么这个请求就是没有获取到数据
# 返回了request，那么这个这个请求会被重新添加到调速器
if not self.current_proxy.blacked:
self.current_proxy.blacked = True
print("被拉黑了")
self.updateProxy()
return request
# 正常的情况下，返回response
return response
def updateProxy(self):
'''
获取新的代理ip
:return:
'''
# 因为是异步请求，为了不同时向芝麻代理发送过多的请求这里在获取代理IP
# 的时候，需要加锁
self.lock.acquire()
if not self.current_proxy or self.current_proxy.is_expire or self.current_proxy.blacked:
response = requests.get(self.IP_URL)
text = response.text
# # 返回值 {"code":0,"success":true,"msg":"0","data":[{"ip":"49.70.152.188","port":4207,"expire_time":"2019-05-28 18:53:15"}]}
# text=text.split(',')
jsonString = json.loads(text)
data = jsonString['data']
if len(data) > 0:
proxyModel = IPProxyModel(data=data[0])
self.current_proxy = proxyModel
self.lock.release()
def __init__(self, iface):
self.iface = iface
self._lock = DeferredLock()
CompositeStatePublisher(lambda x: x, [
netlink_monitor.get_state_publisher(iface, IFSTATE.PLUGGED),
netlink_monitor.get_state_publisher(iface, IFSTATE.UP),
]).subscribe(self._cb)
self._is_shutdown = False
self.state = None
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
def __init__(self, service, clock=reactor):
"""
@param service: An object implementing the same whenConnected() API as
the twisted.application.internet.ClientService class.
@param clock: An object implementing IReactorTime.
"""
self._service = service
self._clock = clock
self._channel = None
self._channel_lock = DeferredLock()
def __init__(self):
# 2. 初始化配置及相关变量
# self.proxies = settings.getlist('PROXIES')
self.lock = DeferredLock()
self.lockproxy = DeferredLock()
self.proxies = {}
self.api = 'http://lianjiaip.v4.dailiyun.com/query.txt?key=NP7CE43238&word=&count=50&rand=false&detail=true'
self.stats = defaultdict(int)
self.max_failed = 10
self.time = time.time()
def call_sync(self, *args, **kw):
"""Call synchronous code in a separate thread outside the twisted event loop."""
if not hasattr(self, '_sync_lock'):
self._sync_lock = DeferredLock()
yield self._sync_lock.acquire()
try:
result = yield deferToThread(*args, **kw)
returnValue(result)
finally:
self._sync_lock.release()
class NotificationConnector(object):
"""Provide ready-to-use AMQP channels."""
def __init__(self, service, clock=reactor):
"""
@param service: An object implementing the same whenConnected() API as
the twisted.application.internet.ClientService class.
@param clock: An object implementing IReactorTime.
"""
self._service = service
self._clock = clock
self._channel = None
self._channel_lock = DeferredLock()
@inlineCallbacks
def __call__(self):
"""
@return: A deferred firing with a ready-to-use txamqp.protocol.Channel.
"""
# Serialize calls, in order to setup new channels only once.
yield self._channel_lock.acquire()
try:
if self._channel and self._channel.client.closed:
# If we have a client but it's closed, let's wait for it to be
# fully disconnected and spin a reactor iteration to give
# change to the AMQClient.connectionLost callback chain to
# settle (in particular our ClientService will be notified and
# will start connecting again).
yield self._channel.client.disconnected.wait()
yield deferLater(self._clock, 0, lambda: None)
client = yield self._service.whenConnected()
channel = yield client.channel(1)
# Check if we got a new channel, and initialize it if so.
if channel is not self._channel:
self._channel = channel
yield self._channel.channel_open()
# This tells the broker to deliver us at most one message at
# a time to support using multiple processes (e.g. in a
# load-balanced/HA deployment). If NotificationSource.get()
# gets called against the same UUID first by process A and then
# when it completes by process B, we're guaranteed that process
# B will see the very next message in the queue, because
# process A hasn't fetched any more messages than the one it
# received. See #729140.
yield self._channel.basic_qos(prefetch_count=1)
finally:
self._channel_lock.release()
returnValue(self._channel)
class emailer( LabradServer ):
name = 'Email Server'
@inlineCallbacks
def initServer( self ):
self.username, self.fromaddr, self.password = yield self.getInfoReg()
self.password = base64.b64decode(self.password)
self.toaddrs = {}
self.smtp = 'smtp.gmail.com:587'
self.sending = DeferredLock()
@inlineCallbacks
def getInfoReg(self):
reg = self.client.registry
yield reg.cd(['Servers','Email Server'])
username = yield reg.get('username')
fromaddr = yield reg.get('address')
password = yield reg.get('password')
returnValue([username,fromaddr,password])
@setting(0, "Set Recipients", recepients = '*s', returns = '')
def setRecepients(self, c, recepients):
"""Set the recipients of the email as a list of strings of email addresses"""
self.toaddrs[c.ID] = recepients
@setting(1, "Send", subject = 's', message = 's', returns = '')
def selectDP(self, c, subject, message):
"""Select Double Pass in the current context"""
if not self.toaddrs[c.ID]: raise Exception("Recipients not set")
yield self.sending.acquire()
session = smtplib.SMTP(self.smtp)
session.starttls()
session.login(self.username,self.password)
toaddrs = self.toaddrs[c.ID]
msg = MIMEMultipart()
msg['From'] = self.fromaddr
msg['To'] = COMMASPACE.join(toaddrs)
msg['Subject'] = subject
msg.attach(MIMEText(message, 'plain'))
session.sendmail(self.fromaddr, toaddrs, msg.as_string())
session.quit()
self.sending.release()
def initContext(self, c):
"""Initialize a new context object."""
pass
def expireContext(self, c):
del(self.toaddrs[c.ID])
class Pulser_729(LabradServer):
name = 'Pulser_729'
def initServer(self):
self.api = api()
self.inCommunication = DeferredLock()
self.initializeBoard()
def initializeBoard(self):
connected = self.api.connectOKBoard()
if not connected:
raise Exception("Pulser Not Found")
@setting(0, 'Reset DDS', returns = '')
def resetDDS(self , c):
"""
Reset the ram position to 0
"""
yield self.inCommunication.acquire()
yield deferToThread(self.api.resetAllDDS)
self.inCommunication.release()
@setting(1, "Program DDS", program = '*(is)', returns = '')
def programDDS(self, c, program):
"""
Programs the DDS, the input is a tuple of channel numbers and buf objects for the channels
"""
yield self.inCommunication.acquire()
yield deferToThread(self._programDDSSequence, program)
self.inCommunication.release()
@setting(2, "Reinitialize DDS", returns = '')
def reinitializeDDS(self, c):
"""
Reprograms the DDS chip to its initial state
"""
yield self.inCommunication.acquire()
yield deferToThread(self.api.initializeDDS)
self.inCommunication.release()
def _programDDSSequence(self, program):
'''takes the parsed dds sequence and programs the board with it'''
for chan, buf in program:
self.api.setDDSchannel(chan)
self.api.programDDS(buf)
self.api.resetAllDDS()
def wait(self, seconds, result=None):
"""Returns a deferred that will be fired later"""
d = Deferred()
reactor.callLater(seconds, d.callback, result)
return d
def initServer(self):
self.api = api()
self.channelDict = hardwareConfiguration.channelDict
self.collectionTime = hardwareConfiguration.collectionTime
self.collectionMode = hardwareConfiguration.collectionMode
self.sequenceType = hardwareConfiguration.sequenceType
self.isProgrammed = hardwareConfiguration.isProgrammed
self.timeResolution = float(hardwareConfiguration.timeResolution)
self.ddsDict = hardwareConfiguration.ddsDict
self.timeResolvedResolution = hardwareConfiguration.timeResolvedResolution
self.remoteChannels = hardwareConfiguration.remoteChannels
self.collectionTimeRange = hardwareConfiguration.collectionTimeRange
self.sequenceTimeRange = hardwareConfiguration.sequenceTimeRange
self.haveSecondPMT = hardwareConfiguration.secondPMT
self.haveDAC = hardwareConfiguration.DAC
self.inCommunication = DeferredLock()
self.clear_next_pmt_counts = 0
self.hwconfigpath = os.path.dirname(inspect.getfile(hardwareConfiguration))
print self.hwconfigpath
#LineTrigger.initialize(self)
#self.initializeBoard()
#yield self.initializeRemote()
#self.initializeSettings()
#yield self.initializeDDS()
self.ddsLock = True
self.listeners = set()
def __init__(self, config=None):
ApplicationSession.__init__(self, config)
global live
live = self
self.logger = logging.getLogger('Live')
self.logger.info("Config: %s", config)
self.account_id = config.extra['authid']
self.secret = config.extra['secret']
if '-' not in self.account_id:
self.account_id = "local-%s" % self.account_id
self.authid = '%s:%s' % (self.account_id, self.secret[-7:])
self.joined = False
self.lock = DeferredLock()
self.checks = {}
self.workers = {}
self.CallOptions = CallOptions()
def __init__(self, omegleProto):
"""
Initializes an L{OmegleBot}.
@param omegleProto: an instance of a protocol that implements:
* typingCallback: when the stranger is typing
* stoppedTypingCallback: stranger no longer typing
* disconnectCallback: stranger OR bot has disconnected
* messageCallback: when the stranger has sent us a message
* recaptchaFailedCallback: when our submitted captcha fails
* recaptchaRequiredCallback: when omegle requires a captcha
* connectCallback: when we have found a stranger
* waitingCallback: when we are waiting for a stranger
"""
for callback_name in ('typingCallback',
'stoppedTypingCallback',
'disconnectCallback',
'messageCallback',
'recaptchaFailedCallback',
'recaptchaRequiredCallback',
'connectCallback',
'waitingCallback',
):
setattr(self, callback_name, getattr(omegleProto, callback_name, None))
self.status = DISCONNECTED
self.server = None
self.id = None
self.lock = DeferredLock()
self.activeRequests = set()
self.challenge = None
self.image = None
def __init__(self, priority):
self._lock = DeferredLock()
self.priority = str(priority)
self.is_shutdown = False
self.state_pub = state_publisher.StatePublisher(())
self.flush()
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
def acquire(self, timeout=None):
"""
This method operates the same as :meth:`DeferredLock.acquire` does
except it requires a timeout argument.
:param int timeout:
The number of seconds to wait before timing out.
:raises LockTimeoutError:
Raised if the timeout was reached before we could acquire
the lock.
"""
assert timeout is None \
or isinstance(timeout, (int, float)) and timeout > 0
lock = DeferredLock.acquire(self)
if timeout is None:
return lock
# Schedule a call to trigger finished.errback() which will raise
# an exception. If lock finishes first however cancel the timeout
# and unlock the lock by calling finished.
finished = Deferred()
lock.addCallback(
self._cancel_timeout,
reactor.callLater(timeout, self._timeout, finished))
lock.addCallback(self._call_callback, finished)
return finished
def __init__(self, publisher, ignoreBuilders=None, send_logs=False,
max_connect_time=600, max_idle_time=300, heartbeat_time=900):
self.publisher = publisher
self.send_logs = send_logs
self.ignoreBuilders = []
if ignoreBuilders:
for i in ignoreBuilders:
if isinstance(i, basestring):
self.ignoreBuilders.append(re.compile(i))
else:
assert hasattr(i, 'match') and callable(i.match)
self.ignoreBuilders.append(i)
self.watched = []
self.max_connect_time = max_connect_time
self.max_idle_time = max_idle_time
self.heartbeat_time = heartbeat_time
self._disconnect_timer = None
self._idle_timer = None
# Lock to make sure only one thread is active at a time
self._thread_lock = DeferredLock()
# Set up heartbeat
self._heartbeat_loop = LoopingCall(self.heartbeat)
StatusPush.__init__(self, PulseStatus.pushEvents, filter=False)
def initServer(self):
self.api_dac = api_dac()
self.inCommunication = DeferredLock()
connected = self.api_dac.connectOKBoard()
if not connected:
raise Exception ("Could not connect to DAC")
self.d = yield self.initializeDAC()
self.listeners = set()
def __init__(self, name, conf):
_log.info("CF_INIT %s", name)
self.name, self.conf = name, conf
self.channel_dict = defaultdict(list)
self.iocs = dict()
self.client = None
self.currentTime = getCurrentTime
self.lock = DeferredLock()
def initServer(self):
self.channelDict = hardwareConfiguration.channelDict
self.collectionTime = hardwareConfiguration.collectionTime
self.collectionMode = hardwareConfiguration.collectionMode
self.sequenceType = hardwareConfiguration.sequenceType
self.isProgrammed = hardwareConfiguration.isProgrammed
self.inCommunication = DeferredLock()
self.connectOKBoard()
self.listeners = set()
def __init__(self, consumer):
self.consumer = consumer
assert IConsumer.implementedBy(consumer.__class__)
self._producers = []
self._sendingLock = DeferredLock()
self._localHeaderLength = 0
self._centralDirectoryLength = 0
def __init__(self):
# so parts of this instance are accessible elsewhere
assert "agent" not in config
config["agent"] = self
self.services = {}
self.register_shutdown_events = False
self.last_free_ram_post = time.time()
self.repeating_call_counter = {}
self.shutdown_timeout = None
self.post_shutdown_lock = DeferredLock()
self.stop_lock = DeferredLock()
self.reannounce_lock = utility.TimedDeferredLock()
self.stopped = False
# Register a callback so self.shutdown_timeout is set when
# "shutting_down" is set or modified.
config.register_callback(
"shutting_down", self.callback_shutting_down_changed)
class MKSPDR2000Wrapper(DeviceWrapper):
@inlineCallbacks
def connect(self, server, port):
'''Connect to the MKS PDR2000'''
print('Connecting to "%s" on port "%s"...' %(server.name, port))
self.server = server
self.ctx = server.context()
self.port = port
p = self.packet()
p.open(port)
# The following parameters were obtained from the MKS PDR2000 manual.
p.baudrate(9600L)
p.stopbits(1L)
p.bytesize(8L)
p.parity('N')
p.timeout(0.1 * u.s)
# Clear out the Rx buffer. This is necessary for some devices.
yield p.send()
def packet(self):
'''Create a new packet in our private context'''
return self.server.packet(context=self.ctx)
def shutdown(self):
'''Disconnect from teh serial port when we shut down.'''
return self.packet().close().send()
def rw_line(self, code):
# Don't allow two concurrent read/write calls.
self._lock = DeferredLock()
return self._lock.run(partial(self._rw_line, code))
@inlineCallbacks
def _rw_line(self, code):
'''Write data to the device.'''
yield self.server.write_line(code, context=self.ctx)
time.sleep(0.2)
ans = yield self.server.read(context=self.ctx)
returnValue(ans)
@inlineCallbacks
def getUnits(self):
yield self.write_line('u')
time.sleep(0.5)
ans = yield self.read_line()
returnValue(ans)
def toFloat(self, val):
try:
if val == 'Off' or val == 'Low':
return np.nan
return float(val)
except:
return None
def __init__(self, iface):
self.iface = iface
self._lock = DeferredLock()
CompositeStatePublisher(lambda x: x, [
netlink_monitor.get_state_publisher(iface, IFSTATE.PLUGGED),
netlink_monitor.get_state_publisher(iface, IFSTATE.UP),
]).subscribe(self._cb)
self._is_shutdown = False
self.state = None
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
def call_sync(self, *args, **kw):
"""Call synchronous code in a separate thread outside the twisted event loop."""
if not hasattr(self, '_sync_lock'):
self._sync_lock = DeferredLock()
yield self._sync_lock.acquire()
try:
result = yield deferToThread(*args, **kw)
returnValue(result)
finally:
self._sync_lock.release()
def __init__(self, ident, signals):
self.pause_lock = DeferredLock()
self.pause_requests = []
self.continue_requests = []
self.already_called_continue = False
self.status = 'Ready'
self.percentage_complete = 0.0
self.should_stop = False
self.ident = ident
self.signals = signals
def add_scan_to_queue(self, scan, priority = 'Normal'):
#increment counter
scan_id = self.scan_ID_counter
self.scan_ID_counter += 1
#add to queue
if priority == 'Normal':
order = self.queue.put_last(1, (scan_id, scan, 1))
elif priority == 'First in Queue':
order = self.queue.put_first(1, (scan_id, scan, 1))
elif priority == 'Pause All Others':
order = self.queue.put_last(0, (scan_id, scan, 0))
else:
raise Exception ("Unrecognized priority type")
self.signals.on_queued_new_script((scan_id, scan.name, order))
d = DeferredLock()
d.acquire()
self.running_locks[scan_id] = d
self.launch_scripts()
return scan_id
def __init__(
self, servers=None, session_timeout=None, connect_timeout=4000):
"""
"""
super(SessionClient, self).__init__(servers, session_timeout)
self._connect_timeout = connect_timeout
self._watches = WatchManager()
self._ephemerals = {}
self._session_notifications = []
self._reconnect_lock = DeferredLock()
self.set_connection_error_callback(self._cb_connection_error)
def __init__(self, cxn, context, dataset, directory):
super(Dataset, self).__init__()
self.accessingData = DeferredLock()
self.cxn = cxn
self.context = context # context of the first dataset in the window
self.dataset = dataset
self.directory = directory
self.data = None
self.hasPlotParameter = False
self.cnt = 0
self.setupDataListener(self.context)
def __init__(self, specFilename, exchange='signals'):
self.exchange = exchange
spec = txamqp.spec.load(specFilename)
delegate = TwistedDelegate()
self.clientConnected = ClientCreator(reactor, AMQClient,
delegate=delegate, vhost="/",
spec=spec).connectTCP("localhost", 5672)
self.conn = None
self.chan = None
self.finishLock = DeferredLock()
def initServer(self):
self.inCommunication = DeferredLock()
self.connectOKBoard()
#create dictionary for triggers and switches in the form 'trigger':channel;'switch:(channel , logicnotnegated, state'
#the state written below represents the initial state of the server
self.dict = {
'Triggers':{'PaulBox':0},
'Switches':{'866':[0x01,True, True], 'BluePI':[0x02,True, False], '397LocalHeating':[0x04,True,False]}
}
self.initializeChannels()
self.listeners = set()
def __init__(self, data_vault, context, dataset_location,reactor):
super(Dataset, self).__init__()
self.data = None
self.accessingData = DeferredLock()
self.reactor = reactor
self.dataset_location = dataset_location
self.data_vault = data_vault
self.updateCounter = 0
self.context = context
self.connectDataVault()
self.setupListeners()
def __init__(self, client, unit, service, unit_path, executor):
self._client = client
self._unit = unit
self._service = service
self._executor = executor
self._unit_path = unit_path
self._relations = {}
self._running = False
self._watching_relation_memberships = False
self._watching_relation_resolved = False
self._run_lock = DeferredLock()
self._log = logging.getLogger("unit.lifecycle")
def initServer( self ):
self.api = api()
self.inCommunication = DeferredLock()
self.registry = self.client.registry
self.dacDict = dict(hc.elec_dict.items() + hc.sma_dict.items())
print("dacDict:")
print(len(self.dacDict.items()))
self.queue = Queue()
self.multipoles = hc.default_multipoles
self.currentVoltages = {}
self.initializeBoard()
self.listeners = set()
yield self.setCalibrations()
self.setPreviousControlFile()
class _DhcpSetterCommon:
# Provides support common code for shutting down on shutdown, and
# handles locking.
def __init__(self):
self._lock = DeferredLock()
self.is_shutdown = False
reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
@inlineCallbacks
def _cb(self, old_state, new_state):
if self.is_shutdown:
return
yield self._lock.acquire()
try:
yield self._locked_cb(old_state, new_state)
finally:
self._lock.release()
@inlineCallbacks
def _shutdown(self):
#print "Shutdown", self
yield self._cb(None, None)
self.is_shutdown = True
