class RandomProxy(object): def __init__(self): self.current_proxy = None self.lock = DeferredLock() def process_request(self, request, spider): pass # user_agent = random.choice(USER_AGENT_LIST) # request.headers['User-Agent'] = user_agent # if 'proxy' not in request.meta or self.current_proxy.is_expiring: # #请求代理 # self.update_proxy() # request.meta['proxy'] = self.current_proxy.proxy def process_response(self, request, response, spider): # 如果对方重定向(302)去验证码的网页,换掉代理IP # 'captcha' in response.url 指的是有时候验证码的网页返回的状态码是200,所以用这个作为辨识的标志 if response.status != 200 or 'captcha' in response.url: # 如果来到这里,说明这个请求已经被boss直聘识别为爬虫了 # 所以这个请求就相当于什么都没有获取到 # 所以要重新返回request,让这个请求重新加入到调度中 # 下次再发送 # if not self.current_proxy.blacked: # self.current_proxy.blacked = True # self.update_proxy() # print('%s代理失效' % self.current_proxy.proxy) # request.meta['proxy'] = self.current_proxy.proxy return request # 如果是正常的话,记得最后要返回response # 如果不返回,这个response就不会被传到爬虫那里去 # 也就得不到解析 return response def update_proxy(self): #lock是属于多线程中的一个概念,因为这里scrapy是采用异步的,可以直接看成多线程 #所以有可能出现这样的情况,爬虫在爬取一个网页的时候,忽然被对方封了,这时候就会来到这里 #获取新的IP,但是同时会有多条线程来这里请求,那么就会出现浪费代理IP的请求,所以这这里加上了锁 #锁的作用是在同一时间段,所有线程只能有一条线程可以访问锁内的代码,这个时候一条线程获得新的代理IP #而这个代理IP是可以用在所有线程的,这样子别的线程就可以继续运行了,减少了代理IP(钱)的浪费 self.lock.acquire() # 判断换线程的条件 # 1.目前没有使用代理IP # 2.到线程过期的时间了 # 3.目前IP已经被对方封了 # 满足以上其中一种情况就可以换代理IP了 if not self.current_proxy or self.current_proxy.is_expiring or self.current_proxy.blacked: url = r'https://h.wandouip.com/get/ip-list?pack=%s&num=1&xy=1&type=2&lb=\r\n&mr=1&' % random.randint( 100, 1000) response = requests.get(url=url, headers=DEFAULT_REQUEST_HEADERS) text = json.loads(response.text) print(text) data = text['data'][0] proxy_model = ProxyModel(data) print('重新获取了一个代理:%s' % proxy_model.proxy) self.current_proxy = proxy_model # return proxy_model self.lock.release()
def initServer(self): """Initialize the server after connecting to LabRAD.""" self.knownDevices = {} # maps (server, channel) to (name, idn) self.deviceServers = { } # maps device name to list of interested servers. # each interested server is {'target':<>,'context':<>,'messageID':<>} self.identFunctions = {} # maps server to (setting, ctx) for ident self.identLock = DeferredLock() # named messages are sent with source ID first, which we ignore def connect_func(c, s_payload): (s, payload) = s_payload return self.gpib_device_connect(*payload) def disconnect_func(c, s_payload): (s, payload) = s_payload return self.gpib_device_disconnect(*payload) mgr = self.client.manager self._cxn.addListener(connect_func, source=mgr.ID, ID=10) self._cxn.addListener(disconnect_func, source=mgr.ID, ID=11) yield mgr.subscribe_to_named_message('GPIB Device Connect', 10, True) yield mgr.subscribe_to_named_message('GPIB Device Disconnect', 11, True) # do an initial scan of the available GPIB devices yield self.refreshDeviceLists()
def __init__(self, priority): self._lock = DeferredLock() self.priority = str(priority) self.is_shutdown = False self.state_pub = state_publisher.StatePublisher(()) self.flush() reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
class IPProxyMiddleware(): PROXY_URL = "http://webapi.http.zhimacangku.com/getip?num=3&type=2&pro=&city=0&yys=0&port=11&time=2&ts=1&ys=0&cs=0&lb=1&sb=0&pb=45&mr=1®ions=" def __init__(self): super(IPProxyMiddleware, self).__init__() self.current_proxy = None self.lock = DeferredLock() def process_request(self, request, spider): if 'proxy' not in request.meta or self.current_proxy.is_expireing: #请求代理 self.update_proxy() request.meta['proxy'] = self.current_proxy.proxy def process_response(self, request, response, spider): if response.status != 200 or "captcha" in response.url: if not self.current_proxy.blacked: self.current_proxy.blacked = True print('%s这个代理被加入黑名单了' % self.current_proxy.ip) self.update_proxy() return request return response def update_proxy(self): self.lock.acquire() if not self.current_proxy or self.current_proxy.is_expireing or self.current_proxy.blacked: response = requests.get(self.PROXY_URL) text = response.text print("重新获取了一个代理:", text) result = json.loads(text) if len(result['data']) > 0: data = result['data'][random.randint(0, 2)] proxy_model = ProxyModel(data) self.current_proxy = proxy_model self.lock.release()
class IPProxyDownloadMiddlware(object): PROXY_URL = '购买的代理链接' def __init__(self): super(IPProxyDownloadMiddlware, self).__init__() self.current_proxy = None self.lock= DeferredLock() def prooess_request(self, request, spider): if 'proxy' not in request.meta or self.current_proxy.is_expiring: self.update_proxy() request.meta['proxy'] = self.current_proxy.proxy # 请求代理 def prooess_response(self,request, response, spider): if response.status !=200 or 'captcha' in response.url: self.update_proxy() return request return response def update_proxy(self): self.lock.acquire() if not self.current_proxy or self.current_proxy.is_expiring: response = requests.get(self.PROXY_URL) text = response.text result = json.loads(text) if len(result['data'])>0: data = result['data'][0] proxy_model = ProxyModel(data) self.current_proxy = proxy_model return proxy_model self.lock.release()
class Client(object): def __init__(self, specFilename, exchange='signals'): self.exchange = exchange spec = txamqp.spec.load(specFilename) delegate = TwistedDelegate() self.clientConnected = ClientCreator(reactor, AMQClient, delegate=delegate, vhost="/", spec=spec).connectTCP("localhost", 5672) self.conn = None self.chan = None self.finishLock = DeferredLock() @inlineCallbacks def finishChannelOpen(self): yield self.finishLock.acquire() if self.conn is None: print "opening connection for", self self.conn = yield self.clientConnected authentication = {"LOGIN": "******", "PASSWORD": "******"} yield self.conn.start(authentication) if self.chan is None: self.chan = yield self.conn.channel(1) yield self.chan.channel_open() yield self.newChan() print "made channel for", self self.finishLock.release() def newChan(self): # called once when the new channel is opened return succeed(None)
def __init__(self, nodedir=None, executable=None): self.executable = executable self.multi_folder_support = True if nodedir: self.nodedir = os.path.expanduser(nodedir) else: self.nodedir = os.path.join(os.path.expanduser('~'), '.tahoe') self.rootcap_path = os.path.join(self.nodedir, 'private', 'rootcap') self.servers_yaml_path = os.path.join(self.nodedir, 'private', 'servers.yaml') self.config = Config(os.path.join(self.nodedir, 'tahoe.cfg')) self.pidfile = os.path.join(self.nodedir, 'twistd.pid') self.nodeurl = None self.shares_happy = None self.name = os.path.basename(self.nodedir) self.api_token = None self.magic_folders_dir = os.path.join(self.nodedir, 'magic-folders') self.lock = DeferredLock() self.rootcap = None self.magic_folders = defaultdict(dict) self.remote_magic_folders = defaultdict(dict) self.use_tor = False self.monitor = Monitor(self) self._monitor_started = False self.state = Tahoe.STOPPED
def __init__(self, nodedir=None, executable=None, reactor=None): if reactor is None: from twisted.internet import reactor self.executable = executable self.multi_folder_support = True if nodedir: self.nodedir = os.path.expanduser(nodedir) else: self.nodedir = os.path.join(os.path.expanduser('~'), '.tahoe') self.rootcap_path = os.path.join(self.nodedir, 'private', 'rootcap') self.servers_yaml_path = os.path.join(self.nodedir, 'private', 'servers.yaml') self.config = Config(os.path.join(self.nodedir, 'tahoe.cfg')) self.pidfile = os.path.join(self.nodedir, 'twistd.pid') self.nodeurl = None self.shares_happy = None self.name = os.path.basename(self.nodedir) self.api_token = None self.magic_folders_dir = os.path.join(self.nodedir, 'magic-folders') self.lock = DeferredLock() self.rootcap = None self.magic_folders = defaultdict(dict) self.remote_magic_folders = defaultdict(dict) self.use_tor = False self.monitor = Monitor(self) streamedlogs_maxlen = None debug_settings = global_settings.get('debug') if debug_settings: log_maxlen = debug_settings.get('log_maxlen') if log_maxlen is not None: streamedlogs_maxlen = int(log_maxlen) self.streamedlogs = StreamedLogs(reactor, streamedlogs_maxlen) self.state = Tahoe.STOPPED self.newscap = "" self.newscap_checker = NewscapChecker(self)
class UAMiddleware(object): def __init__(self): self.lock = DeferredLock() self.update_time = datetime.now() self.UA_List = USER_AGENT def process_request(self, request, spider): self.lock.acquire() if self.is_expiring: ua = random.choice(self.UA_List) request.headers['User-Agent'] = ua print(request.headers['User-Agent']) self.lock.release() def process_response(self, request, response, spider): return response def process_exception(self, request, exception, spider): pass @property def is_expiring(self): now = datetime.now() if (now - self.update_time) > timedelta(seconds=30): self.update_time = datetime.now() print("跟换USER_AGENT") return True else: return False
class IPProxyDownloadMiddleware(object): PROXY_URL = 'http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=&city=0&yys=0&port=11&pack=61856&ts=1&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions=' def __init__(self): super().__init__() self.current_proxy = None self.lock = DeferredLock() def process_request(self, request, spider): if 'proxy' not in request.meta or self.current_proxy.is_expiring: self.update_proxy() request.meta['proxy'] = self.current_proxy.proxy def process_response(self, request, response, spider): if response.status != 200 or 'captcha' in response.url: if not self.current_proxy.blacked: self.current_proxy.blacked = True self.update_proxy() print('%s这个代理被加入黑名单了' % self.current_proxy) return request return response def update_proxy(self): self.lock.acquire() if not self.current_proxy or self.current_proxy.is_expiring or self.current_proxy.blacked: response = requests.get(self.PROXY_URL) text = response.text print('重新获取了一个代理:', text) result = json.loads(text) if len(result['data']) > 0: data = result['data'][0] proxy_model = ProxyModel(data) self.current_proxy = proxy_model self.current_proxy.blacked = False self.lock.release()
def __init__(self, node, register, simNum, num=0): # Node where this qubit is located self.node = node # Register where this qubit is simulated self.register = register # Number in the register, if known self.num = num # Number of the simulated qubit, unique at each virtual node self.simNum = simNum # Lock marshalling access to this qubit self._lock = DeferredLock() # Mark this qubit as active (still connected to a register) self.active = True # Time until retry self._delay = 1 # Optional parameters for when the simulation is noise self.noisy = settings.simulaqron_settings.noisy_qubits self.T1 = settings.simulaqron_settings.t1 self.last_accessed = time.time() self._logger = get_netqasm_logger( f"{self.__class__.__name__}(node={node.name}, sim_num={simNum})")
class IPProxyDownloadMiddleware(object): PROXY_URL = '' def __init__(self): super(IPProxyDownloadMiddleware, self).__init__() self.current_proxy = None self.lock = DeferredLock() def process_request(self, request, spider): if 'proxy' not in request.meta or self.current_proxy.is_expiring: self.update_proxy() request.meta['proxy'] = self.current_proxy.proxy def process_response(self, request, response, spider): if response.status != 200 or "captcha" in response.url: if not self.current_proxy.blacked: self.current_proxy.blacked = True self.update_proxy() # 如果来到这里说明这个请求被识别为爬虫,所以这个请求就没有获取到,要重新请求 return request # 来到这里是正常的,要返回response给爬虫解析 return response def update_proxy(self): self.lock.acquire() if not self.current_proxy or self.current_proxy.is_expiring or self.current_proxy.blacked: response = requests.get(self.PROXY_URL) text = response.text print(text) result = json.loads(text) if len(result['data']) > 0: data = result['data'][0] proxy_model = ProxyModel(data) self.current_proxy = proxy_model self.lock.release()
def __init__(self, config=None): ApplicationSession.__init__(self, config) global live live = self self.logger = logging.getLogger('Live') self.logger.info("Config: %s", config) self.account_id = config.extra['authid'] self.secret = config.extra['secret'] if '-' not in self.account_id: self.account_id = "local-%s" % self.account_id self.authid = '%s:%s' % (self.account_id, self.secret[-7:]) self.joined = False self.lock = DeferredLock() self.checks = {} self.workers = {} self.CallOptions = CallOptions()
class IPProxyDownloadMiddleware(object): PROXY_URL = 'http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=&city=0&yys=0&port=11&time=1&ts=1&ys=0&cs=0&lb=1&sb=0&pb=45&mr=1®ions=' def __init__(self): super(IPProxyDownloadMiddleware, self).__init__() self.lock = DeferredLock() self.current_proxy = None self.username = None def process_request(self, request, spider): if not self.current_proxy: self.update_proxy() return request if self.current_proxy.is_expiring: self.update_proxy() request.meta['proxy'] = self.current_proxy.proxy print(request.meta['proxy']) def process_response(self, request, response, spider): if response.status != 200: self.update_proxy() return request return response def update_proxy(self): self.lock.acquire() if not self.current_proxy or self.current_proxy.is_expiring: response = requests.get(self.PROXY_URL) result = json.loads(response.text) data = result['data'] proxy = ProxyModel(data[0]) print('请求到了:', response.text) self.current_proxy = proxy self.lock.release()
class ProxyDownloaderMiddleware(object): def __init__(self): super(ProxyDownloaderMiddleware, self).__init__() self.PROXY_URL = "http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=&city=0&yys=0&port=11&time=1&ts=1&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions=" self.current_proxy = None self.lock = DeferredLock() def process_request(self, request, spider): if 'proxy' not in request.meta or self.current_proxy.is_expiring: self.upgrade_proxy() request.meta['proxy'] = self.current_proxy.proxy def process_process(self, request, response, spider): if response.status != 200: self.upgrade_proxy() return request return response def upgrade_proxy(self): self.lock.acquire() if not self.current_proxy or self.current_proxy.is_expiring: resp = requests.get(self.PROXY_URL) result = resp.text data = json.loads(result)['data'][0] if data: proxy_model = ProxyModel(data) self.current_proxy = proxy_model self.lock.release()
class IPProxy(object): PROXY_URL = "http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=0&city=0&yys=0&port=1&pack=21267&ts=1&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1®ions=" def __init__(self): self.lock = DeferredLock() self.current_proxy = None def process_request(self, request, spider): if 'proxy' not in request.meta or self.current_proxy.is_expiring: self.update_proxy() request.meta['proxy'] = self.current_proxy.proxy def process_response(self, request, response, spider): if response.status != 200: if not self.current_proxy.is_block: self.current_proxy.is_block = True self.update_proxy() return request return response def update_proxy(self): self.lock.acquire() if self.current_proxy is None or self.current_proxy.is_expiring or self.current_proxy.is_block: response_json = requests.get(self.PROXY_URL).json() try: print(response_json) self.current_proxy = ProxyModel(response_json['data'][0]) except: print('出错了!') print(response_json) self.lock.release()
def initServer(self): self.collectionTime = { 0: 1.0, 1: 1.0 } #default collection times in the format channel:time(sec) self.inCommunication = DeferredLock() self.connectOKBoard()
def __init__(self, settings): super(My_RetryMiddleware, self).__init__(settings) self.lock = DeferredLock() self.seconds = settings.getint('expire_datetime') self.proxy_url = settings.get('PROXY_URL') self.expire_datetime = datetime.datetime.now() + datetime.timedelta( seconds=self.seconds) self.proxy = None
def __init__(self, kernel_key, conda_env=conda_env, env_path=env_path, pkg_path=pkg_path, config=ComponentConfig(realm=u"jupyter")): ApplicationSession.__init__(self, config=config) self._kernel_key = kernel_key self._conda_env = conda_env self._env_path = env_path self._pkg_path = pkg_path self._conda_env = conda_env self._lock = DeferredLock()
def initServer(self): self.api_dac = api_dac() self.inCommunication = DeferredLock() connected = self.api_dac.connectOKBoard() if not connected: raise Exception ("Could not connect to DAC") self.d = yield self.initializeDAC() self.listeners = set()
def __init__(self, bandwidth=5): """ :param bandwidth: максимальная длина очереди, по превышению которой запросы будут отсекаться """ self.tasks = [] self.bandwidth = bandwidth self._lock = DeferredLock()
def __init__(self, spider_name): """ 初始化付费代理池, 短效代理 ip, port, 来源网站, 获取时间, 是否被封, 是否过期, 过期时间 """ self.redis = redis.StrictRedis(host=PROXY_REDIS_HOST, port=PROXY_REDIS_PORT, db=PROXY_REDIS_DB) self.lock = DeferredLock() self._init_proxy_pool(spider_name)
def __init__(self, cxn, context, dataset): super(Dataset, self).__init__() self.accessingData = DeferredLock() self.cxn = cxn self.context = context # context of the first dataset in the window self.dataset = dataset self.data = None self.setupDataListener(self.context)
def __init__(self, name, conf): _log.info("CF_INIT %s", name) self.name, self.conf = name, conf self.channel_dict = defaultdict(list) self.iocs = dict() self.client = None self.currentTime = getCurrentTime self.lock = DeferredLock()
def initServer(self): self.channelDict = hardwareConfiguration.channelDict self.collectionTime = hardwareConfiguration.collectionTime self.collectionMode = hardwareConfiguration.collectionMode self.sequenceType = hardwareConfiguration.sequenceType self.isProgrammed = hardwareConfiguration.isProgrammed self.inCommunication = DeferredLock() self.connectOKBoard() self.listeners = set()
def __init__(self): super(IPProxyDownloadMiddleware, self).__init__() self.current_proxy = None self.lock = DeferredLock() self.blacked = False self.get_proxy = False self.proxies = [] # self.proxies_time = 9999999999 self.proxies_time = time.time()
class IPProxyDownloaderMiddleware(object): ''' IP代理 , ''' # 获取代理ip信息地址 例如芝麻代理、快代理等 IP_URL = r'http://127.0.0.1:8000/?types=0&count=1&country=国内' def __init__(self): # super(IPProxyDownloaderMiddleware, self).__init__(self) super(IPProxyDownloaderMiddleware, self).__init__() self.current_proxy = None self.lock = DeferredLock() def process_request(self, request, spider): if 'proxy' not in request.meta or self.current_proxy.is_expire: self.updateProxy() request.meta['proxy'] = self.current_proxy.address def process_response(self, request, response, spider): if response.status != 200: # 如果来到这里,这个请求相当于被识别为爬虫了 # 所以这个请求被废掉了 # 如果不返回request,那么这个请求就是没有获取到数据 # 返回了request,那么这个这个请求会被重新添加到调速器 if not self.current_proxy.blacked: self.current_proxy.blacked = True print("被拉黑了") self.updateProxy() return request # 正常的情况下,返回response return response def updateProxy(self): ''' 获取新的代理ip :return: ''' # 因为是异步请求,为了不同时向芝麻代理发送过多的请求这里在获取代理IP # 的时候,需要加锁 self.lock.acquire() if not self.current_proxy or self.current_proxy.is_expire or self.current_proxy.blacked: response = requests.get(self.IP_URL) text = response.text # # 返回值 {"code":0,"success":true,"msg":"0","data":[{"ip":"49.70.152.188","port":4207,"expire_time":"2019-05-28 18:53:15"}]} # text=text.split(',') jsonString = json.loads(text) data = jsonString['data'] if len(data) > 0: proxyModel = IPProxyModel(data=data[0]) self.current_proxy = proxyModel self.lock.release()
def __init__(self, iface): self.iface = iface self._lock = DeferredLock() CompositeStatePublisher(lambda x: x, [ netlink_monitor.get_state_publisher(iface, IFSTATE.PLUGGED), netlink_monitor.get_state_publisher(iface, IFSTATE.UP), ]).subscribe(self._cb) self._is_shutdown = False self.state = None reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown)
def __init__(self, service, clock=reactor): """ @param service: An object implementing the same whenConnected() API as the twisted.application.internet.ClientService class. @param clock: An object implementing IReactorTime. """ self._service = service self._clock = clock self._channel = None self._channel_lock = DeferredLock()
def __init__(self): # 2. 初始化配置及相关变量 # self.proxies = settings.getlist('PROXIES') self.lock = DeferredLock() self.lockproxy = DeferredLock() self.proxies = {} self.api = 'http://lianjiaip.v4.dailiyun.com/query.txt?key=NP7CE43238&word=&count=50&rand=false&detail=true' self.stats = defaultdict(int) self.max_failed = 10 self.time = time.time()
def call_sync(self, *args, **kw): """Call synchronous code in a separate thread outside the twisted event loop.""" if not hasattr(self, '_sync_lock'): self._sync_lock = DeferredLock() yield self._sync_lock.acquire() try: result = yield deferToThread(*args, **kw) returnValue(result) finally: self._sync_lock.release()
class NotificationConnector(object): """Provide ready-to-use AMQP channels.""" def __init__(self, service, clock=reactor): """ @param service: An object implementing the same whenConnected() API as the twisted.application.internet.ClientService class. @param clock: An object implementing IReactorTime. """ self._service = service self._clock = clock self._channel = None self._channel_lock = DeferredLock() @inlineCallbacks def __call__(self): """ @return: A deferred firing with a ready-to-use txamqp.protocol.Channel. """ # Serialize calls, in order to setup new channels only once. yield self._channel_lock.acquire() try: if self._channel and self._channel.client.closed: # If we have a client but it's closed, let's wait for it to be # fully disconnected and spin a reactor iteration to give # change to the AMQClient.connectionLost callback chain to # settle (in particular our ClientService will be notified and # will start connecting again). yield self._channel.client.disconnected.wait() yield deferLater(self._clock, 0, lambda: None) client = yield self._service.whenConnected() channel = yield client.channel(1) # Check if we got a new channel, and initialize it if so. if channel is not self._channel: self._channel = channel yield self._channel.channel_open() # This tells the broker to deliver us at most one message at # a time to support using multiple processes (e.g. in a # load-balanced/HA deployment). If NotificationSource.get() # gets called against the same UUID first by process A and then # when it completes by process B, we're guaranteed that process # B will see the very next message in the queue, because # process A hasn't fetched any more messages than the one it # received. See #729140. yield self._channel.basic_qos(prefetch_count=1) finally: self._channel_lock.release() returnValue(self._channel)
class emailer( LabradServer ): name = 'Email Server' @inlineCallbacks def initServer( self ): self.username, self.fromaddr, self.password = yield self.getInfoReg() self.password = base64.b64decode(self.password) self.toaddrs = {} self.smtp = 'smtp.gmail.com:587' self.sending = DeferredLock() @inlineCallbacks def getInfoReg(self): reg = self.client.registry yield reg.cd(['Servers','Email Server']) username = yield reg.get('username') fromaddr = yield reg.get('address') password = yield reg.get('password') returnValue([username,fromaddr,password]) @setting(0, "Set Recipients", recepients = '*s', returns = '') def setRecepients(self, c, recepients): """Set the recipients of the email as a list of strings of email addresses""" self.toaddrs[c.ID] = recepients @setting(1, "Send", subject = 's', message = 's', returns = '') def selectDP(self, c, subject, message): """Select Double Pass in the current context""" if not self.toaddrs[c.ID]: raise Exception("Recipients not set") yield self.sending.acquire() session = smtplib.SMTP(self.smtp) session.starttls() session.login(self.username,self.password) toaddrs = self.toaddrs[c.ID] msg = MIMEMultipart() msg['From'] = self.fromaddr msg['To'] = COMMASPACE.join(toaddrs) msg['Subject'] = subject msg.attach(MIMEText(message, 'plain')) session.sendmail(self.fromaddr, toaddrs, msg.as_string()) session.quit() self.sending.release() def initContext(self, c): """Initialize a new context object.""" pass def expireContext(self, c): del(self.toaddrs[c.ID])
class Pulser_729(LabradServer): name = 'Pulser_729' def initServer(self): self.api = api() self.inCommunication = DeferredLock() self.initializeBoard() def initializeBoard(self): connected = self.api.connectOKBoard() if not connected: raise Exception("Pulser Not Found") @setting(0, 'Reset DDS', returns = '') def resetDDS(self , c): """ Reset the ram position to 0 """ yield self.inCommunication.acquire() yield deferToThread(self.api.resetAllDDS) self.inCommunication.release() @setting(1, "Program DDS", program = '*(is)', returns = '') def programDDS(self, c, program): """ Programs the DDS, the input is a tuple of channel numbers and buf objects for the channels """ yield self.inCommunication.acquire() yield deferToThread(self._programDDSSequence, program) self.inCommunication.release() @setting(2, "Reinitialize DDS", returns = '') def reinitializeDDS(self, c): """ Reprograms the DDS chip to its initial state """ yield self.inCommunication.acquire() yield deferToThread(self.api.initializeDDS) self.inCommunication.release() def _programDDSSequence(self, program): '''takes the parsed dds sequence and programs the board with it''' for chan, buf in program: self.api.setDDSchannel(chan) self.api.programDDS(buf) self.api.resetAllDDS() def wait(self, seconds, result=None): """Returns a deferred that will be fired later""" d = Deferred() reactor.callLater(seconds, d.callback, result) return d
def initServer(self): self.api = api() self.channelDict = hardwareConfiguration.channelDict self.collectionTime = hardwareConfiguration.collectionTime self.collectionMode = hardwareConfiguration.collectionMode self.sequenceType = hardwareConfiguration.sequenceType self.isProgrammed = hardwareConfiguration.isProgrammed self.timeResolution = float(hardwareConfiguration.timeResolution) self.ddsDict = hardwareConfiguration.ddsDict self.timeResolvedResolution = hardwareConfiguration.timeResolvedResolution self.remoteChannels = hardwareConfiguration.remoteChannels self.collectionTimeRange = hardwareConfiguration.collectionTimeRange self.sequenceTimeRange = hardwareConfiguration.sequenceTimeRange self.haveSecondPMT = hardwareConfiguration.secondPMT self.haveDAC = hardwareConfiguration.DAC self.inCommunication = DeferredLock() self.clear_next_pmt_counts = 0 self.hwconfigpath = os.path.dirname(inspect.getfile(hardwareConfiguration)) print self.hwconfigpath #LineTrigger.initialize(self) #self.initializeBoard() #yield self.initializeRemote() #self.initializeSettings() #yield self.initializeDDS() self.ddsLock = True self.listeners = set()
def __init__(self, omegleProto): """ Initializes an L{OmegleBot}. @param omegleProto: an instance of a protocol that implements: * typingCallback: when the stranger is typing * stoppedTypingCallback: stranger no longer typing * disconnectCallback: stranger OR bot has disconnected * messageCallback: when the stranger has sent us a message * recaptchaFailedCallback: when our submitted captcha fails * recaptchaRequiredCallback: when omegle requires a captcha * connectCallback: when we have found a stranger * waitingCallback: when we are waiting for a stranger """ for callback_name in ('typingCallback', 'stoppedTypingCallback', 'disconnectCallback', 'messageCallback', 'recaptchaFailedCallback', 'recaptchaRequiredCallback', 'connectCallback', 'waitingCallback', ): setattr(self, callback_name, getattr(omegleProto, callback_name, None)) self.status = DISCONNECTED self.server = None self.id = None self.lock = DeferredLock() self.activeRequests = set() self.challenge = None self.image = None
def acquire(self, timeout=None): """ This method operates the same as :meth:`DeferredLock.acquire` does except it requires a timeout argument. :param int timeout: The number of seconds to wait before timing out. :raises LockTimeoutError: Raised if the timeout was reached before we could acquire the lock. """ assert timeout is None \ or isinstance(timeout, (int, float)) and timeout > 0 lock = DeferredLock.acquire(self) if timeout is None: return lock # Schedule a call to trigger finished.errback() which will raise # an exception. If lock finishes first however cancel the timeout # and unlock the lock by calling finished. finished = Deferred() lock.addCallback( self._cancel_timeout, reactor.callLater(timeout, self._timeout, finished)) lock.addCallback(self._call_callback, finished) return finished
def __init__(self, publisher, ignoreBuilders=None, send_logs=False, max_connect_time=600, max_idle_time=300, heartbeat_time=900): self.publisher = publisher self.send_logs = send_logs self.ignoreBuilders = [] if ignoreBuilders: for i in ignoreBuilders: if isinstance(i, basestring): self.ignoreBuilders.append(re.compile(i)) else: assert hasattr(i, 'match') and callable(i.match) self.ignoreBuilders.append(i) self.watched = [] self.max_connect_time = max_connect_time self.max_idle_time = max_idle_time self.heartbeat_time = heartbeat_time self._disconnect_timer = None self._idle_timer = None # Lock to make sure only one thread is active at a time self._thread_lock = DeferredLock() # Set up heartbeat self._heartbeat_loop = LoopingCall(self.heartbeat) StatusPush.__init__(self, PulseStatus.pushEvents, filter=False)
def __init__(self, consumer): self.consumer = consumer assert IConsumer.implementedBy(consumer.__class__) self._producers = [] self._sendingLock = DeferredLock() self._localHeaderLength = 0 self._centralDirectoryLength = 0
def __init__(self): # so parts of this instance are accessible elsewhere assert "agent" not in config config["agent"] = self self.services = {} self.register_shutdown_events = False self.last_free_ram_post = time.time() self.repeating_call_counter = {} self.shutdown_timeout = None self.post_shutdown_lock = DeferredLock() self.stop_lock = DeferredLock() self.reannounce_lock = utility.TimedDeferredLock() self.stopped = False # Register a callback so self.shutdown_timeout is set when # "shutting_down" is set or modified. config.register_callback( "shutting_down", self.callback_shutting_down_changed)
class MKSPDR2000Wrapper(DeviceWrapper): @inlineCallbacks def connect(self, server, port): '''Connect to the MKS PDR2000''' print('Connecting to "%s" on port "%s"...' %(server.name, port)) self.server = server self.ctx = server.context() self.port = port p = self.packet() p.open(port) # The following parameters were obtained from the MKS PDR2000 manual. p.baudrate(9600L) p.stopbits(1L) p.bytesize(8L) p.parity('N') p.timeout(0.1 * u.s) # Clear out the Rx buffer. This is necessary for some devices. yield p.send() def packet(self): '''Create a new packet in our private context''' return self.server.packet(context=self.ctx) def shutdown(self): '''Disconnect from teh serial port when we shut down.''' return self.packet().close().send() def rw_line(self, code): # Don't allow two concurrent read/write calls. self._lock = DeferredLock() return self._lock.run(partial(self._rw_line, code)) @inlineCallbacks def _rw_line(self, code): '''Write data to the device.''' yield self.server.write_line(code, context=self.ctx) time.sleep(0.2) ans = yield self.server.read(context=self.ctx) returnValue(ans) @inlineCallbacks def getUnits(self): yield self.write_line('u') time.sleep(0.5) ans = yield self.read_line() returnValue(ans) def toFloat(self, val): try: if val == 'Off' or val == 'Low': return np.nan return float(val) except: return None
def __init__(self, ident, signals): self.pause_lock = DeferredLock() self.pause_requests = [] self.continue_requests = [] self.already_called_continue = False self.status = 'Ready' self.percentage_complete = 0.0 self.should_stop = False self.ident = ident self.signals = signals
def add_scan_to_queue(self, scan, priority = 'Normal'): #increment counter scan_id = self.scan_ID_counter self.scan_ID_counter += 1 #add to queue if priority == 'Normal': order = self.queue.put_last(1, (scan_id, scan, 1)) elif priority == 'First in Queue': order = self.queue.put_first(1, (scan_id, scan, 1)) elif priority == 'Pause All Others': order = self.queue.put_last(0, (scan_id, scan, 0)) else: raise Exception ("Unrecognized priority type") self.signals.on_queued_new_script((scan_id, scan.name, order)) d = DeferredLock() d.acquire() self.running_locks[scan_id] = d self.launch_scripts() return scan_id
def __init__( self, servers=None, session_timeout=None, connect_timeout=4000): """ """ super(SessionClient, self).__init__(servers, session_timeout) self._connect_timeout = connect_timeout self._watches = WatchManager() self._ephemerals = {} self._session_notifications = [] self._reconnect_lock = DeferredLock() self.set_connection_error_callback(self._cb_connection_error)
def __init__(self, cxn, context, dataset, directory): super(Dataset, self).__init__() self.accessingData = DeferredLock() self.cxn = cxn self.context = context # context of the first dataset in the window self.dataset = dataset self.directory = directory self.data = None self.hasPlotParameter = False self.cnt = 0 self.setupDataListener(self.context)
def __init__(self, specFilename, exchange='signals'): self.exchange = exchange spec = txamqp.spec.load(specFilename) delegate = TwistedDelegate() self.clientConnected = ClientCreator(reactor, AMQClient, delegate=delegate, vhost="/", spec=spec).connectTCP("localhost", 5672) self.conn = None self.chan = None self.finishLock = DeferredLock()
def initServer(self): self.inCommunication = DeferredLock() self.connectOKBoard() #create dictionary for triggers and switches in the form 'trigger':channel;'switch:(channel , logicnotnegated, state' #the state written below represents the initial state of the server self.dict = { 'Triggers':{'PaulBox':0}, 'Switches':{'866':[0x01,True, True], 'BluePI':[0x02,True, False], '397LocalHeating':[0x04,True,False]} } self.initializeChannels() self.listeners = set()
def __init__(self, data_vault, context, dataset_location,reactor): super(Dataset, self).__init__() self.data = None self.accessingData = DeferredLock() self.reactor = reactor self.dataset_location = dataset_location self.data_vault = data_vault self.updateCounter = 0 self.context = context self.connectDataVault() self.setupListeners()
def __init__(self, client, unit, service, unit_path, executor): self._client = client self._unit = unit self._service = service self._executor = executor self._unit_path = unit_path self._relations = {} self._running = False self._watching_relation_memberships = False self._watching_relation_resolved = False self._run_lock = DeferredLock() self._log = logging.getLogger("unit.lifecycle")
def initServer( self ): self.api = api() self.inCommunication = DeferredLock() self.registry = self.client.registry self.dacDict = dict(hc.elec_dict.items() + hc.sma_dict.items()) print("dacDict:") print(len(self.dacDict.items())) self.queue = Queue() self.multipoles = hc.default_multipoles self.currentVoltages = {} self.initializeBoard() self.listeners = set() yield self.setCalibrations() self.setPreviousControlFile()
class _DhcpSetterCommon: # Provides support common code for shutting down on shutdown, and # handles locking. def __init__(self): self._lock = DeferredLock() self.is_shutdown = False reactor.addSystemEventTrigger('before', 'shutdown', self._shutdown) @inlineCallbacks def _cb(self, old_state, new_state): if self.is_shutdown: return yield self._lock.acquire() try: yield self._locked_cb(old_state, new_state) finally: self._lock.release() @inlineCallbacks def _shutdown(self): #print "Shutdown", self yield self._cb(None, None) self.is_shutdown = True