def connectionLost(self): dt = self.getDateTime() log.msg("[OUTPUT] Lost Connection with the attacker: %s" % self.endIP) if not self.passwordTried: if self.cfg.get('txtlog', 'enabled') == 'true': txtlog.authLog(dt, self.cfg.get('folders', 'log_path') + "/" + datetime.datetime.now().strftime("%Y%m%d"), self.endIP, '', '', False) if self.loginSuccess: if self.cfg.get('txtlog', 'enabled') == 'true': if os.path.exists(self.txtlog_file): txtlog.log(dt, self.txtlog_file, '[SSH ] Lost Connection with ' + self.endIP) if self.cfg.get('database_mysql', 'enabled') == 'true': self.dbLog.handleConnectionLost(dt, self.sessionID) if self.cfg.get('hpfeeds', 'enabled') == 'true': self.hpLog.handleConnectionLost(dt) if self.cfg.get('email', 'attack') == 'true': threads.deferToThread(self.email, self.sensorName + ' - Attack logged', self.txtlog_file, self.ttyFiles) if self.cfg.has_option('app_hooks', 'connection_lost'): if self.cfg.get('app_hooks', 'connection_lost') != '': cmdString = self.cfg.get('app_hooks', 'connection_lost') + " CONNECTION_LOST " + dt + " " + self.endIP threads.deferToThread(self.runCommand, cmdString) self.connections.delConn(self.sensorName, self.endIP, self.endPort)
def start_instance(self, build): if self.instance is not None: raise ValueError('instance active') if self.spot_instance: return threads.deferToThread(self._request_spot_instance) else: return threads.deferToThread(self._start_instance)
def download_from_legacy_server(): if config.legacy_server is None or config.legacy_server == "": returnValue(0) logging.info("Downloading hosts from legacy server...") rows = yield database.run_query('SELECT `value` FROM info WHERE `key`="last_legacy_sync"') last_legacy_sync_time = int(rows[0][0]) try: server = yield deferToThread(xmlrpclib.ServerProxy, config.legacy_server) response = yield deferToThread(server.get_new_hosts, last_legacy_sync_time, config.legacy_threshold, [], config.legacy_resiliency) try: last_legacy_sync_time = int(response["timestamp"]) except: logging.ERROR("Illegal timestamp {} from legacy server".format(response["timestamp"])) #Registry.DBPOOL.runOperation('UPDATE info SET `value`=%s WHERE `key`="last_legacy_sync"', (str(last_legacy_sync_time),)) database.run_operation('UPDATE info SET `value`=? WHERE `key`="last_legacy_sync"', str(last_legacy_sync_time)) now = time.time() logging.debug("Got {} hosts from legacy server".format(len(response["hosts"]))) for host in response["hosts"]: legacy = yield Legacy.find(where=["ip_address=?",host], limit=1) if legacy is None: logging.debug("New host from legacy server: {}".format(host)) legacy = Legacy(ip_address=host, retrieved_time=now) else: logging.debug("Known host from legacy server: {}".format(host)) legacy.retrieved_time = now yield legacy.save() except Exception, e: logging.error("Error retrieving info from legacy server: {}".format(e))
def loginSuccessful(self, username, password): self.passwordTried = True self.loginSuccess = True dt = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") self.makeSessionFolder() if self.cfg.get('txtlog', 'enabled') == 'true': txtlog.authLog(self.cfg.get('folders', 'log_path') + "/" + datetime.datetime.now().strftime("%Y%m%d"), self.endIP, username, password, True) txtlog.log(self.txtlog_file, self.connectionString) txtlog.log(self.txtlog_file, '[SSH ] Login Successful: ' + username + ':' + password) if self.cfg.get('email', 'login') == 'true': threads.deferToThread(self.email, self.sensorName + ' - Login Successful', self.txtlog_file) if self.cfg.get('database_mysql', 'enabled') == 'true': self.dbLog.handleLoginSucceeded(username, password) self.dbLog.createSession(self.sessionID, self.endIP, self.endPort, self.honeyIP, self.honeyPort, self.sensorName) self.dbLog.handleClientVersion(self.sessionID, self.version) if self.cfg.get('hpfeeds', 'enabled') == 'true': self.hpLog.handleLoginSucceeded(username, password) self.hpLog.createSession(self.sessionID, self.endIP, self.endPort, self.honeyIP, self.honeyPort) self.hpLog.handleClientVersion(self.version) if self.cfg.has_option('app_hooks', 'login_successful'): if self.cfg.get('app_hooks', 'login_successful') != '': cmdString = self.cfg.get('app_hooks', 'login_successful') + " LOGIN_SUCCESSFUL " + dt + " " + self.endIP + " " + username + " " + password threads.deferToThread(self.runCommand, cmdString)
def parse_message(self, sqs_message): e = [] # decode JSON try: parsed_json = simplejson.loads(sqs_message.get_body()) except Exception as exception: logger.error('Error parsing JSON %s' % exception) return False for k in self.message_format: if not k in parsed_json: logger.error('MessageTag %s not defined in sqs_message.id:%s, sqs_message_body:%s' % (k, sqs_message.id, parsed_json)) return False e = parsed_json['event'] for k in self.event_format: if not k in e: logger.error('EventTag %s not defined in sqs_message.id:%s, sqsMessageBody:%s' % (k, sqs_message.id, parsed_json)) return False # Add incident to Database incident = add_incident(e, parsed_json['timestamp']) # Check incident thresholds and fire events if incident and incident.status == 'Active': threads.deferToThread(ProcessIncident, incident, e['message']) logger.info('MsgOk: EventID:%s, Element:%s, Message%s on Timestamp:%s' % (e['eventid'], e['element'], e['message'], parsed_json['timestamp'])) return True
def _setup_all(self, ids): """ Initializing of all the circuits. @param ids: circuits ids @type ids: int @return: list of setup results @rtype: DeferredList """ dl = [] for id in ids: if self.kill_all: self.logger.debug("Dispatcher killed") return DeferredList(dl) if any([True for c in self._circuits if c.id == id]): circuit = self.get(id) if circuit.initialized: deferToThread(circuit.run) else: circuit.install_dispatcher(self) d = circuit.setup() dl.append(d) else: d = maybeDeferred(self._circuit_pre_setup, id) d.addCallback(self._circuit_setup) @d.addErrback def _err(failure): self.logger.error("Circuit pre-setup failed: %s" % failure) dl.append(d) return DeferredList(dl)
def sendBuildbotNetUsageData(master): if master.config.buildbotNetUsageData is None: return data = computeUsageData(master) if data is None: return threads.deferToThread(_sendBuildbotNetUsageData, data)
def _register(self): """ Performs the registration based on the values provided in the form """ self.ui.btnRegister.setEnabled(False) username = self.ui.lblUser.text() password = self.ui.lblPassword.text() password2 = self.ui.lblPassword2.text() ok, msg = basic_password_checks(username, password, password2) if ok: register = SRPRegister(provider_config=self._provider_config) register.registration_finished.connect( self._registration_finished) threads.deferToThread( partial(register.register_user, username.encode("utf8"), password.encode("utf8"))) self._username = username self._password = password self._set_register_status(self.tr("Starting registration...")) else: self._set_register_status(msg, error=True) self._focus_password() self.ui.btnRegister.setEnabled(True)
def start(self): """ Start the buildstep. """ instance_id = self.getProperty('instance_id') assert instance_id volume_tags = dict(self.object_tags) volume_tags['timestamp'] = self._timestamp() conn = yield threads.deferToThread(self._connect) volume_id, device, error = yield threads.deferToThread( create_ami.create_volume, conn, instance_id, self.volume_gib, self.location, volume_tags) self.setProperty('volume_id', volume_id) self.setProperty('device', device) self.setProperty('volume_tags', volume_tags) if volume_id and not error: self.finished(results.SUCCESS) else: self.failed(failure.Failure(error))
def send_successful(self, results): """ Generic callback for when send methods are successful. This method cleans up and shuts down the worker """ if (results): threads.deferToThread(self.shutdown)
def persist_file(self, path, buf, info, meta=None, headers=None): """Upload file to S3 storage""" key_name = '%s%s' % (self.prefix, path) buf.seek(0) if self.is_botocore: extra = self._headers_to_botocore_kwargs(self.HEADERS) if headers: extra.update(self._headers_to_botocore_kwargs(headers)) return threads.deferToThread( self.s3_client.put_object, Bucket=self.bucket, Key=key_name, Body=buf, Metadata={k: str(v) for k, v in six.iteritems(meta or {})}, ACL=self.POLICY, **extra) else: b = self._get_boto_bucket() k = b.new_key(key_name) if meta: for metakey, metavalue in six.iteritems(meta): k.set_metadata(metakey, str(metavalue)) h = self.HEADERS.copy() if headers: h.update(headers) return threads.deferToThread( k.set_contents_from_string, buf.getvalue(), headers=h, policy=self.POLICY)
def delete(self, combo): """HTTP DELETE Invalidate a UAID (and all channels associated with it). """ combo = combo.strip("/") if "/" in combo: (uaid, chid) = combo.split("/", 2) else: uaid = combo chid = None # what is request if not self._validate_auth(uaid): return self._write_response( 401, 109, message="Invalid Authentication") message = self.ap_settings.message if chid: # mark channel as dead self.ap_settings.metrics.increment("updates.client.unregister", tags=self.base_tags()) d = deferToThread(self._deleteChannel, message, uaid, chid) d.addCallback(self._success) d.addErrback(self._chid_not_found_err) d.addErrback(self._response_err) return d # nuke uaid d = deferToThread(self._deleteUaid, message, uaid, self.ap_settings.router) d.addCallback(self._success) d.addErrback(self._uaid_not_found_err) d.addErrback(self._response_err) return d
def _update_user_permissions(self, user_id): """ Updates the permissions for the indicated user. This method downloads (or loads), validates, and saves the specified user's permission settings. @note If a non-remote permissions endpoint is specified, the permissions will be loaded from the offline local file. @note If the indicated user already has permission settings recorded, they will be overridden with the new settings. @throws Throws PermissionsInvalidSchema if the dictionary defined in permission_settings does not conform to the permission settings schema, as checked by _validate_permissions(). @throws PermissionsUserNotFound if the indicated user couldn't be located in the loaded permissions resource. @param user_id The ID of the user that the command permissions are for. @return Returns a deferred that will be fired with the results of the permission settings load/download. That is, an error or the permissions object for the indicated user. """ # Attempt to load the user's permissions if self.use_remote_permissions: defer_download = threads.deferToThread(self._download_remote_permissions, user_id) else: defer_download = threads.deferToThread(self._load_local_permissions, user_id) # Validate & save defer_download.addCallback(self._validate_permissions, user_id) defer_download.addCallback(self._save_permissions, user_id) return defer_download
def auth(client): @fsync_readonly def get_file_status(): file_id = self._state.file try: file = self.usr0.get_node(file_id) except errors.DoesNotExist: return 'Dead' return file.status d = client.dummy_authenticate("open sesame") d.addCallback(lambda r: client.get_root()) d.addCallbacks( lambda r: client.make_file(request.ROOT, r, "hola"), client.test_fail) d.addCallback(lambda req: self._save_state("file", req.new_id)) d.addCallback(lambda _: threads.deferToThread(get_file_status)) d.addCallback(lambda status: self.assert_(status == 'Live')) d.addCallbacks( lambda mkfile_req: client.unlink(request.ROOT, self._state.file), client.test_fail) d.addCallback(lambda _: threads.deferToThread(get_file_status)) d.addCallback(lambda status: self.assert_(status == 'Dead')) d.addCallbacks(client.test_done, client.test_fail)
def start(self, args={}, subtask_key=None, callback=None, callback_args={}, errback=None): """ starts the task. This will spawn the work in a workunit thread. """ # only start if not already running if self._status == STATUS_RUNNING: return #if this was subtask find it and execute just that subtask if subtask_key: logger.debug('Task - starting subtask %s' % (subtask_key)) split = subtask_key.split('.') subtask = self.get_subtask(split) logger.debug('Task - got subtask') self.work_deferred = threads.deferToThread(subtask._start, args, callback, callback_args) #else this is a normal task just execute it else: logger.debug('Task - starting task: %s' % self) self.work_deferred = threads.deferToThread(self._start, args, callback, callback_args) if errback: self.work_deferred.addErrback(errback) return 1
def _updateInBackground(self): if self._updating: return self._updating = True deferToThread(super(CustomLsiModel, self).add_documents, self._currentChunk[:]) self._currentChunk = [] self._updating = False
def update_schedule(self): """ Downloads the most recent version of the schedule from the active source. @note This method loads the schedule from the active source (either a local file or network address) and updates the local copy using callbacks. If use_local_schedule is true, the schedule will be loaded from a local file (specified in the configuration files). If it is false, it will be loaded from the user interface API. @return Returns a deferred that will be called with the result of the file access (the schedule object or a Failure). """ # Setup local variables defer_download = None # Attempt to download the schedule if self.use_network_schedule: defer_download = threads.deferToThread(self._download_remote_schedule) else: defer_download = threads.deferToThread(self._download_local_schedule) # Add a callback to store the schedule defer_download.addCallback(self._validate_schedule) defer_download.addCallback(self._save_schedule) return defer_download
def start(self, c): if not self.isProgrammed: raise Exception ("No Programmed Sequence") yield self.inCommunication.acquire() yield deferToThread(self.api.resetSeqCounter) yield deferToThread(self.api.startSingle) self.sequenceType = 'One' self.inCommunication.release()
def process_speech(self, plist): phrase = '' for phrase_plist in plist['properties']['recognition']['properties']['phrases']: for token in phrase_plist['properties']['interpretations'][0]['properties']['tokens']: if token['properties']['removeSpaceBefore']: phrase = phrase[:-1] phrase += token['properties']['text'] if not token['properties']['removeSpaceAfter']: phrase += ' ' if phrase: self.logger.info('[Speech Recognised] "%s"' % phrase) if self.consumer: self.logger.info("Sending phrase to consumer") self.consumer(phrase) self.consumer = None else: for trigger, function in self.triggers: match = trigger.search(phrase) if match: fname = '%s.%s' % (function.im_class.__name__, function.__func__.__name__) self.logger.info('Phrase matched "%s" for trigger %s' % (trigger.pattern, fname)) groups = match.groups() args = [phrase] if groups: args.append(groups) threads.deferToThread(function, *args)
def handleConnectionLost(self, dt): log.msg('[HPFEEDS] - publishing metadata to hpfeeds') meta = self.sessionMeta meta['endTime'] = dt log.msg("[HPFEEDS] - sessionMeta: " + str(meta)) threads.deferToThread(self.client.publish, HONSSHSESHCHAN, **meta)
def dataReceived(self, data): """ Takes "data" which we assume is json encoded If data has a subject_id attribute, we pass that to the dispatcher as the subject_id so it will get carried through into any return communications and be identifiable to the client falls back to just passing the message along... """ try: address = self.guid data = json.loads(data) threads.deferToThread(send_signal, self.dispatcher, data) if 'hx_subscribe' in data: return self.dispatcher.subscribe(self.transport, data) if 'address' in data: address = data['address'] else: address = self.guid self.dispatcher.send(address, data) except Exception, exc: raise self.dispatcher.send( self.guid, {'message': data, 'error': str(exc)} )
def __init__(self, server, **kwargs): AbstractBackendStore.__init__(self, server, **kwargs) self.name = 'audio CD' self.device_name = kwargs.get('device_name', "/dev/cdom") threads.deferToThread(self.extractAudioCdInfo)
def _handle_event(self, event_type, args=(), respond_to=False): """handles event dispatch""" for event in self.events[event_type]: if self.config['threaded']: threads.deferToThread(self._event, event, args, respond_to) else: defer.execute(self._event, event, args, respond_to)
def __CREA(self, data): """ Massive reading files from file storage, using threads in Twisted """ algorithm = self.gui.FileManager.options_frame.getCryptoAlgorithm() key = self.gui.FileManager.options_frame.getCryptoKey() tmp_dir = self.gui.FileManager.options_frame.tmpFolder user_folder = self.gui.FileManager.files_folder.getUsersDir() def defferedReadFile(user_id, name, path, file_id, servers): new_folders = os.path.normpath(user_folder + '/' + path) try: os.makedirs(new_folders) except OSError: pass for ip, port in servers: src_file = os.path.normpath(user_folder + '/' + path + '/' + name) server_ip = str(ip) server_port = str(port) json_file = os.path.normpath(tmp_dir + '/fsc_' + self.login + '_' + name + '_' + str(randint(0, 100000)) + '.json') dumpConfigToJSON(json_file, "READU_FILE", user_id, file_id, src_file, key, algorithm) self.__SendInfoToFileServer(json_file, server_ip, server_port) evt = UpdateFileListCtrlEvent() wx.PostEvent(self.gui, evt) for name, path, file_id, servers in data['files_read']: threads.deferToThread(defferedReadFile, data['user_id'], name, path, file_id, servers) del data['files_read'] del data['user_id']
def __CLNK(self, data): """ Downloading file by link from another file storage """ tmp_dir = self.gui.FileManager.options_frame.tmpFolder def defferedDownloadByLink(user_id, save_in, filename, file_id, key, server): src_file = os.path.normpath(save_in + '/' + filename) server_ip = str(server[0][0]) server_port = str(server[0][1]) json_file = os.path.normpath(tmp_dir + '/fsc_link_' + self.login + '_' + filename + '_' + str(randint(0, 100000)) + '.json') dumpConfigToJSON(json_file, "READU_FILE", user_id, file_id, src_file, key) self.__SendInfoToFileServer(json_file, server_ip, server_port) wx.MessageBox("File downloading {0} has completed!".format(filename), "Message") file_info = data.get('file_info', None) if file_info: file_id, filename, key, server = file_info if file_id and filename and key and server: dlg = wx.DirDialog(self.gui, "Choose directory for saving file", style=wx.DD_DEFAULT_STYLE) if dlg.ShowModal() == wx.ID_OK: save_in = dlg.GetPath() threads.deferToThread(defferedDownloadByLink, data['user_id'], save_in, filename, file_id, key, server) else: wx.MessageBox("Incorrect link or file has deleted!", "Error") del data['user_id'] del data['file_info'] else: wx.MessageBox("Incorrect link or file has deleted!", "Error")
def afterDownload(self, result, filename, deleteFile=False): if os.path.getsize(filename) > 0: print>>log, "[EPGImport] afterDownload", filename if self.source.parser == 'epg.dat': if twisted.python.runtime.platform.supportsThreads(): print>>log, "[EPGImport] Using twisted thread for DAT file" threads.deferToThread(self.readEpgDatFile, filename, deleteFile).addCallback(lambda ignore: self.nextImport()) else: self.readEpgDatFile(filename, deleteFile) return if filename.endswith('.gz'): self.fd = gzip.open(filename, 'rb') else: self.fd = open(filename, 'rb') if twisted.python.runtime.platform.supportsThreads(): print>>log, "[EPGImport] Using twisted thread!" threads.deferToThread(self.doThreadRead).addCallback(lambda ignore: self.nextImport()) else: self.iterator = self.createIterator() reactor.addReader(self) if deleteFile: try: print>>log, "[EPGImport] unlink", filename os.unlink(filename) except Exception, e: print>>log, "[EPGImport] warning: Could not remove '%s' intermediate" % filename, e
def dataReceived(self, data): self.dataLogger.debug('received: %s' % upperhexstr(data)) if self.state == 'IDLE': self.logger.warning("Discarding data in IDLE state %d bytes" % len(data)) elif self.state == 'RESPONSE_WAIT': try: self.input = MaraFrame.parse(data) except FieldError: self.logger.error("Bad package") return # FIXME: Hacerlos con todos los campos o con ninguno # if self.input.command != self.output.command: # logger.warn("Command not does not match with sent command %d" % self.input.command) # Calcular próxima sequencia # FIXME: Checkear que la secuencia sea == a self.output.sequence self.logger.debug("Message OK") self.incrementSequenceNumber() self.pending = 0 if self.factory.defer_db_save: deferToThread(self.saveInDatabase) else: self.saveInDatabase() #MaraFrame.pretty_print(self.input, show_header=False, show_bcc=False) self.state = 'IDLE'
def listen(self): try: deferToThread(self.server.serve_forever) self.running = True except TFTPServerConfigurationError, e: self.logger.error('TFTP server configuration error: %s!' % e.message)
def connectionMade(self, ip, port, honeyIP, honeyPort, sensorName): dt = datetime.datetime.now().strftime("%Y%m%d_%H%M%S") self.sensorName = sensorName self.honeyIP = honeyIP self.honeyPort = honeyPort self.logLocation = self.cfg.get('folders', 'session_path') + "/" + self.sensorName + "/"+ ip + "/" self.downloadFolder = self.logLocation + 'downloads/' self.txtlog_file = self.logLocation + dt + ".log" self.endIP = ip self.endPort = port self.sessionID = uuid.uuid4().hex self.passwordTried = False self.loginSuccess = False self.ttyFiles = [] if self.cfg.get('txtlog', 'enabled') == 'true': self.connectionString = '[POT ] ' + self.sensorName + ' - ' + self.honeyIP + ':' + str(self.honeyPort) self.addConnectionString('[SSH ] Incoming Connection from ' + ip + ':' + str(port)) country = self.cname(ip) if country != None: self.connectionString = self.connectionString + ' - ' + self.cname(ip) if self.cfg.get('database_mysql', 'enabled') == 'true': self.dbLog = mysql.DBLogger() self.dbLog.setClient(self.dbLogClient, self.cfg) if self.cfg.get('hpfeeds', 'enabled') == 'true': self.hpLog = hpfeeds.HPLogger() self.hpLog.setClient(self.hpLogClient, self.cfg, self.sensorName) if self.cfg.has_option('app_hooks', 'connection_made'): if self.cfg.get('app_hooks', 'connection_made') != '': cmdString = self.cfg.get('app_hooks', 'connection_made') + " CONNECTION_MADE " + dt + " " + self.endIP + " " + str(port) + " " + self.honeyIP threads.deferToThread(self.runCommand, cmdString)
def _gotDMResult(self, feed): plains = [] feed.entry.reverse() hasNew = False for a in feed.entry: entry = doubanapi.Entry(a) entry_id = int(entry.id) if entry_id > self.last_dm_id: self.last_dm_id = entry_id hasNew = True if entry.isRead is True: continue plain = "Got a doumail from %s: %s\n%s" % ( entry.authorName.decode("utf-8"), entry.title.decode("utf-8"), entry.alternateLink, ) plains.append(plain) if len(plains) > 0: log.msg("User: %s got %s new doumail" % (self.uid, len(plains))) conn = protocol.current_conn for jid in self.bare_jids(): conn.send_plain(jid, "\n".join(plains)) if hasNew: threads.deferToThread(self._deferred_write, self.short_jid, "last_dm_id", self.last_dm_id)
def calldefer(): return threads.deferToThread(test)
def get_account_by_email(self, email): return threads.deferToThread(super(DeferredGoogleDataSource, self).get_account_by_email, email)
def get_events_by_email(self, email, **kwargs): return threads.deferToThread(super(DeferredGoogleDataSource, self).get_events_by_email, email, **kwargs)
def cleanUp(self): print 'clean!' yield deferToThread(time.sleep, 5) gc.collect()
def symbolscheck(self): threads.deferToThread(self.JobTask) self.timer.startLongTimer(POLLTIME)
def test_cli_lifecycle(click_runner, random_policy_label, federated_ursulas, blockchain_ursulas, custom_filepath, custom_filepath_2, federated): """ This is an end to end integration test that runs each cli call in it's own process using only CLI character control entry points, and a mock side channel that runs in the control process """ # Boring Setup Stuff alice_config_root = custom_filepath bob_config_root = custom_filepath_2 envvars = {'NUCYPHER_KEYRING_PASSWORD': INSECURE_DEVELOPMENT_PASSWORD} # A side channel exists - Perhaps a dApp side_channel = MockSideChannel() shutil.rmtree(custom_filepath, ignore_errors=True) shutil.rmtree(custom_filepath_2, ignore_errors=True) """ Scene 1: Alice Installs nucypher to a custom filepath and examines her configuration """ # Alice performs an installation for the first time alice_init_args = ('alice', 'init', '--network', TEMPORARY_DOMAIN, '--config-root', alice_config_root) if federated: alice_init_args += ('--federated-only', ) else: alice_init_args += ('--provider-uri', TEST_PROVIDER_URI) alice_init_response = click_runner.invoke(nucypher_cli, alice_init_args, catch_exceptions=False, env=envvars) assert alice_init_response.exit_code == 0 # Alice uses her configuration file to run the character "view" command alice_configuration_file_location = os.path.join( alice_config_root, AliceConfiguration.generate_filename()) alice_view_args = ('--json-ipc', 'alice', 'public-keys', '--config-file', alice_configuration_file_location) alice_view_result = click_runner.invoke(nucypher_cli, alice_view_args, catch_exceptions=False, env=envvars) assert alice_view_result.exit_code == 0 alice_view_response = json.loads(alice_view_result.output) # Alice expresses her desire to participate in data sharing with nucypher # by saving her public key somewhere Bob and Enrico can find it. side_channel.save_alice_pubkey( alice_view_response['result']['alice_verifying_key']) """ Scene 2: Bob installs nucypher, examines his configuration and expresses his interest to participate in data retrieval by posting his public keys somewhere public (side-channel). """ bob_init_args = ('bob', 'init', '--network', TEMPORARY_DOMAIN, '--config-root', bob_config_root) if federated: bob_init_args += ('--federated-only', ) else: bob_init_args += ('--provider-uri', TEST_PROVIDER_URI) bob_init_response = click_runner.invoke(nucypher_cli, bob_init_args, catch_exceptions=False, env=envvars) assert bob_init_response.exit_code == 0 # Alice uses her configuration file to run the character "view" command bob_configuration_file_location = os.path.join( bob_config_root, BobConfiguration.generate_filename()) bob_view_args = ('--json-ipc', 'bob', 'public-keys', '--config-file', bob_configuration_file_location) bob_view_result = click_runner.invoke(nucypher_cli, bob_view_args, catch_exceptions=False, env=envvars) assert bob_view_result.exit_code == 0 bob_view_response = json.loads(bob_view_result.output) # Bob interacts with the sidechannel bob_public_keys = MockSideChannel.BobPublicKeys( bob_view_response['result']['bob_encrypting_key'], bob_view_response['result']['bob_verifying_key']) side_channel.save_bob_public_keys(bob_public_keys) """ Scene 3: Alice derives a policy keypair, and saves it's public key to a sidechannel. """ random_label = random_policy_label.decode() # Unicode string derive_args = ('--mock-networking', '--json-ipc', 'alice', 'derive-policy-pubkey', '--config-file', alice_configuration_file_location, '--label', random_label) derive_response = click_runner.invoke(nucypher_cli, derive_args, catch_exceptions=False, env=envvars) assert derive_response.exit_code == 0 derive_response = json.loads(derive_response.output) assert derive_response['result']['label'] == random_label # Alice and the sidechannel: at Tinagre policy = MockSideChannel.PolicyAndLabel( encrypting_key=derive_response['result']['policy_encrypting_key'], label=derive_response['result']['label']) side_channel.save_policy(policy=policy) """ Scene 4: Enrico encrypts some data for some policy public key and saves it to a side channel. """ def enrico_encrypts(): # Fetch! policy = side_channel.fetch_policy() enrico_args = ('--json-ipc', 'enrico', 'encrypt', '--policy-encrypting-key', policy.encrypting_key, '--message', PLAINTEXT) encrypt_result = click_runner.invoke(nucypher_cli, enrico_args, catch_exceptions=False, env=envvars) assert encrypt_result.exit_code == 0 encrypt_result = json.loads(encrypt_result.output) encrypted_message = encrypt_result['result'][ 'message_kit'] # type: str side_channel.save_message_kit(message_kit=encrypted_message) return encrypt_result def _alice_decrypts(encrypt_result): """ alice forgot what exactly she encrypted for bob. she decrypts it just to make sure. """ policy = side_channel.fetch_policy() alice_signing_key = side_channel.fetch_alice_pubkey() message_kit = encrypt_result['result']['message_kit'] decrypt_args = ( '--mock-networking', '--json-ipc', 'alice', 'decrypt', '--config-file', alice_configuration_file_location, '--message-kit', message_kit, '--label', policy.label, ) if federated: decrypt_args += ('--federated-only', ) decrypt_response_fail = click_runner.invoke(nucypher_cli, decrypt_args[0:7], catch_exceptions=False, env=envvars) assert decrypt_response_fail.exit_code == 2 decrypt_response = click_runner.invoke(nucypher_cli, decrypt_args, catch_exceptions=False, env=envvars) decrypt_result = json.loads(decrypt_response.output) for cleartext in decrypt_result['result']['cleartexts']: assert b64decode(cleartext.encode()).decode() == PLAINTEXT # replenish the side channel side_channel.save_policy(policy=policy) side_channel.save_alice_pubkey(alice_signing_key) return encrypt_result """ Scene 5: Alice grants access to Bob: We catch up with Alice later on, but before she has learned about existing Ursulas... """ if federated: teacher = list(federated_ursulas)[0] else: teacher = list(blockchain_ursulas)[1] teacher_uri = teacher.seed_node_metadata(as_teacher_uri=True) # Some Ursula is running somewhere def _run_teacher(_encrypt_result): start_pytest_ursula_services(ursula=teacher) return teacher_uri def _grant(teacher_uri): # Alice fetched Bob's public keys from the side channel bob_keys = side_channel.fetch_bob_public_keys() bob_encrypting_key = bob_keys.bob_encrypting_key bob_verifying_key = bob_keys.bob_verifying_key grant_args = ('--mock-networking', '--json-ipc', 'alice', 'grant', '--network', TEMPORARY_DOMAIN, '--teacher-uri', teacher_uri, '--config-file', alice_configuration_file_location, '--m', 2, '--n', 3, '--value', Web3.toWei(1, 'ether'), '--expiration', (maya.now() + datetime.timedelta(days=3)).iso8601(), '--label', random_label, '--bob-encrypting-key', bob_encrypting_key, '--bob-verifying-key', bob_verifying_key) if federated: grant_args += ('--federated-only', ) else: grant_args += ('--provider-uri', TEST_PROVIDER_URI) grant_result = click_runner.invoke(nucypher_cli, grant_args, catch_exceptions=False, env=envvars) assert grant_result.exit_code == 0 grant_result = json.loads(grant_result.output) # TODO: Expand test to consider manual treasure map handing # # Alice puts the Treasure Map somewhere Bob can get it. # side_channel.save_treasure_map(treasure_map=grant_result['result']['treasure_map']) return grant_result def _bob_retrieves(_grant_result): """ Scene 6: Bob retrieves encrypted data from the side channel and uses nucypher to re-encrypt it """ # Bob interacts with a sidechannel ciphertext_message_kit = side_channel.fetch_message_kit() policy = side_channel.fetch_policy() policy_encrypting_key, label = policy alice_signing_key = side_channel.fetch_alice_pubkey() retrieve_args = ('--mock-networking', '--json-ipc', 'bob', 'retrieve', '--teacher-uri', teacher_uri, '--config-file', bob_configuration_file_location, '--message-kit', ciphertext_message_kit, '--label', label, '--policy-encrypting-key', policy_encrypting_key, '--alice-verifying-key', alice_signing_key) if federated: retrieve_args += ('--federated-only', ) retrieve_response = click_runner.invoke(nucypher_cli, retrieve_args, catch_exceptions=False, env=envvars) assert retrieve_response.exit_code == 0 retrieve_response = json.loads(retrieve_response.output) for cleartext in retrieve_response['result']['cleartexts']: assert b64decode(cleartext.encode()).decode() == PLAINTEXT return # Run the Callbacks d = threads.deferToThread(enrico_encrypts) # scene 4 d.addCallback(_alice_decrypts) # scene 5 (uncertainty) d.addCallback(_run_teacher) # scene 6 (preamble) d.addCallback(_grant) # scene 7 d.addCallback(_bob_retrieves) # scene 8 yield d
def wrapper(*args, **kwargs): from twisted.internet import threads return threads.deferToThread(f, *args, **kwargs)
class CommandView(DefaultView): context(ICommand) def write_results(self, request, pid, cmd): log.msg('Called %s got result: pid(%s) term writes=%s' % (cmd, pid, len(cmd.write_buffer)), system='command-view') request.write( json.dumps({ 'status': 'ok', 'pid': pid, 'stdout': cmd.write_buffer })) request.finish() def render_PUT(self, request): """ Converts arguments into command-line counterparts and executes the omsh command. Parameters passed as 'arg' are converted into positional arguments, others are converted into named parameters: PUT /bin/ls?arg=/some/path&arg=/another/path&-l&--recursive thus translates to: /bin/ls /some/path /another/path -l --recursive Allows blocking (synchronous) and non-blocking operation using the 'asynchronous' parameter (any value will trigger it). Synchronous operation requires two threads to function. """ def named_args_filter_and_flatten(nargs): for name, vallist in nargs: if name not in ('arg', 'asynchronous'): for val in vallist: yield name yield val def convert_args(args): tokenized_args = args.get('arg', []) return tokenized_args + list( named_args_filter_and_flatten(args.items())) protocol = DetachedProtocol() protocol.interaction = get_interaction( self.context) or request.interaction args = convert_args(request.args) args = filter(None, args) cmd = self.context.cmd(protocol) # Setting write_buffer to a list makes command save the output to the buffer too cmd.write_buffer = [] d0 = defer.Deferred() try: pid = threads.blockingCallFromThread( reactor, cmd.register, d0, args, '%s %s' % (request.path, args)) except ArgumentParsingError, e: raise BadRequest(str(e)) q = Queue.Queue() def cancel_execute(_, d): d.cancel() def execute(q, cmd, args): d = defer.maybeDeferred(cmd, *args) request.notifyFinish().addBoth(cancel_execute, d) d.addBoth(q.put) d.chainDeferred(d0) # cmd() may be synchronous (that are executed inline, without deferreds) or asynchronous. # We want synchronous methods to be executed in a separate thread dt = threads.deferToThread(execute, q, cmd, args) request.notifyFinish().addBoth(cancel_execute, dt) asynchronous = request.args.get('asynchronous', []) if not asynchronous: dt.addBoth(lambda r: threads.deferToThread(q.get, True, 300)) dt.addCallback(lambda r: reactor.callFromThread( self.write_results, request, pid, cmd)) def handleArgumentParsingError(e, pid, cmd): e.trap(ArgumentParsingError) log.err(e, system='http-cmd') request.setResponseCode(400) request.write(str(e.value)) request.finish() def handleQueueEmpty(e, pid, cmd): e.trap(Queue.Empty) msg = 'Timeout waiting for command %s (%s) to complete' % ( request.path, args) log.msg(msg, system='http-cmd') request.setResponseCode(504) request.write(msg) request.finish() def handleUnknownError(e, pid, cmd): e.trap(Exception) log.err(e, system='http-cmd') request.setResponseCode(500) request.write(''.join(traceback.format_exc(e.value))) request.finish() dt.addErrback(handleArgumentParsingError, pid, cmd) dt.addErrback(handleQueueEmpty, pid, cmd) dt.addErrback(handleUnknownError, pid, cmd) return NOT_DONE_YET
def process(self, msg, sender=None): cmd = [self.print_command, msg] return threads.deferToThread(subprocess32.call, cmd)
def _deferToThreadPool(self, f, *a, **kw): return deferToThread(f, *a, **kw)
def submit_entries(self, batch): """ Large parts of this method are adapted from kippo-pyshield by jkakavas Many thanks to their efforts. https://github.com/jkakavas/kippo-pyshield """ # The nonce is predefined as explained in the original script : # trying to avoid sending the authentication key in the "clear" but # not wanting to deal with a full digest like exchange. Using a # fixed nonce to mix up the limited userid. _nonceb64 = 'ElWO1arph+Jifqme6eXD8Uj+QTAmijAWxX1msbJzXDM=' log_output = '' for attempt in self.batch: log_output += '{0}\t{1}\t{2}\t{3}\t{4}\t{5}\n'.format( attempt['date'], attempt['time'], attempt['timezone'], attempt['source_ip'], attempt['user'], attempt['password']) nonce = base64.b64decode(_nonceb64) digest = base64.b64encode( hmac.new('{0}{1}'.format(nonce, self.userid), base64.b64decode(self.auth_key), hashlib.sha256).digest()) auth_header = 'credentials={0} nonce={1} userid={2}'.format( digest, _nonceb64, self.userid) headers = { 'X-ISC-Authorization': auth_header, 'Content-Type': 'text/plain', 'Content-Length': len(log_output) } log.msg(headers) req = threads.deferToThread( requests.request, method='PUT', url='https://secure.dshield.org/api/file/sshlog', headers=headers, timeout=10, data=log_output) def check_response(resp): failed = False response = resp.content if resp.status_code == requests.codes.ok: sha1_regex = re.compile( r'<sha1checksum>([^<]+)<\/sha1checksum>') sha1_match = sha1_regex.search(response) if sha1_match is None: log.err( 'dshield ERROR: Could not find sha1checksum in response' ) failed = True sha1_local = hashlib.sha1() sha1_local.update(log_output) if sha1_match.group(1) != sha1_local.hexdigest(): log.err('dshield ERROR: SHA1 Mismatch {0} {1} .'.format( sha1_match.group(1), sha1_local.hexdigest())) failed = True md5_regex = re.compile(r'<md5checksum>([^<]+)<\/md5checksum>') md5_match = md5_regex.search(response) if md5_match is None: log.err( 'dshield ERROR: Could not find md5checksum in response' ) failed = True md5_local = hashlib.md5() md5_local.update(log_output) if md5_match.group(1) != md5_local.hexdigest(): log.err('dshield ERROR: MD5 Mismatch {0} {1} .'.format( md5_match.group(1), md5_local.hexdigest())) failed = True log.msg( 'dshield SUCCESS: Sent {0} bytes worth of data to secure.dshield.org' .format(len(log_output))) else: log.err('dshield ERROR: error {0}.'.format(resp.status_code)) log.err('Response was {0}'.format(response)) failed = True if failed: # Something went wrong, we need to add them to batch. reactor.callFromThread(self.transmission_error, batch) req.addCallback(check_response)
def test_thread(self): return threads.deferToThread(lambda : None)
def iconcheck(self): try: threads.deferToThread(self.JobTask) except: pass self.timer.startLongTimer(30)
def stop_recording(self): if self.recorder and self.recorder.is_alive(): self.recorder.terminate() d = threads.deferToThread(self._flv2webm) d.addBoth(lambda s: None)
def watch(self, count): self.debug("[watchdog] watching (%d)" % count) if (self.loop_call.running): self.checks.append(deferToThread(self._check, count))
def lineReceived(self, line): d = deferToThread(self.handle_request, line) d.addCallbacks(self.factory.cbSucceed, self.factory.cbFailed)
def route_notification(self, notification, uaid_data): """Route a notification to an internal node, and store it if the node can't deliver immediately or is no longer a valid node """ # Determine if they're connected at the moment node_id = uaid_data.get("node_id") uaid = uaid_data["uaid"] router = self.db.router # Node_id is present, attempt delivery. # - Send Notification to node # - Success: Done, return 200 # - Error (Node busy): Jump to Save notification below # - Error (Client gone, node gone/dead): Clear node entry for user # - Both: Done, return 503 if node_id: result = None try: result = yield self._send_notification(uaid, node_id, notification) except (ConnectError, ConnectionClosed, ResponseFailed, CancelledError, PotentialDataLoss) as exc: self.metrics.increment("updates.client.host_gone") yield deferToThread(router.clear_node, uaid_data).addErrback(self._eat_db_err) if isinstance(exc, ConnectionRefusedError): # Occurs if an IP record is now used by some other node # in AWS or if the connection timesout. self.log.debug("Could not route message: {exc}", exc=exc) if result and result.code == 200: returnValue(self.delivered_response(notification)) # Save notification, node is not present or busy # - Save notification # - Success (older version): Done, return 202 # - Error (db error): Done, return 503 try: yield self._save_notification(uaid_data, notification) except ClientError as e: log_exception = (e.response["Error"]["Code"] != "ProvisionedThroughputExceededException") raise RouterException("Error saving to database", status_code=503, response_body="Retry Request", log_exception=log_exception, errno=201) # - Lookup client again to get latest node state after save. # - Success (node found): Notify node of new notification # - Success: Done, return 200 # - Error (no client): Done, return 202 # - Error (no node): Clear node entry # - Both: Done, return 202 # - Success (no node): Done, return 202 # - Error (db error): Done, return 202 # - Error (no client) : Done, return 404 try: uaid_data = yield deferToThread(router.get_uaid, uaid) except ClientError: returnValue(self.stored_response(notification)) except ItemNotFound: self.metrics.increment("updates.client.deleted") raise RouterException("User was deleted", status_code=410, response_body="Invalid UAID", log_exception=False, errno=105) # Verify there's a node_id in here, if not we're done node_id = uaid_data.get("node_id") if not node_id: returnValue(self.stored_response(notification)) try: result = yield self._send_notification_check(uaid, node_id) except (ConnectError, ConnectionClosed, ResponseFailed) as exc: self.metrics.increment("updates.client.host_gone") if isinstance(exc, ConnectionRefusedError): self.log.debug("Could not route message: {exc}", exc=exc) yield deferToThread(router.clear_node, uaid_data).addErrback(self._eat_db_err) returnValue(self.stored_response(notification)) if result.code == 200: returnValue(self.delivered_response(notification)) else: ret_val = self.stored_response(notification) returnValue(ret_val)
def blockOnThread(*args, **kwargs): return threads.deferToThread(monitored_benchmark, threads.blockingCallFromThread, reactor, *args, **kwargs)
def defer_to_thread(block_method, result_method, *args): d = threads.deferToThread(block_method, *args) d.addCallback(result_method)
def toThread(sio): d = threads.deferToThread(parseAvitoImage, sio, ob) d.addCallback(saveWithPhone) return d
def collect(self, config): return threads.deferToThread(lambda: self.inner(config))
def afterDownload(self, result, filename, deleteFile=False): print("[EPGImport] afterDownload", filename, file=log) try: if not os.path.getsize(filename): raise Exception("File is empty") except Exception as e: self.downloadFail(e) return if self.source.parser == 'epg.dat': if twisted.python.runtime.platform.supportsThreads(): print("[EPGImport] Using twisted thread for DAT file", file=log) threads.deferToThread( self.readEpgDatFile, filename, deleteFile).addCallback(lambda ignore: self.nextImport()) else: self.readEpgDatFile(filename, deleteFile) return if filename.endswith('.gz'): self.fd = gzip.open(filename, 'rb') try: # read a bit to make sure it's a gzip file self.fd.read(10) self.fd.seek(0, 0) except Exception as e: print("[EPGImport] File downloaded is not a valid gzip file", filename, file=log) self.downloadFail(e) return elif filename.endswith('.xz') or filename.endswith('.lzma'): try: import lzma except ImportError: from backports import lzma self.fd = lzma.open(filename, 'rb') try: # read a bit to make sure it's an xz file self.fd.read(10) self.fd.seek(0, 0) except Exception as e: print("[EPGImport] File downloaded is not a valid xz file", filename, file=log) self.downloadFail(e) return else: self.fd = open(filename, 'rb') if deleteFile and self.source.parser != 'epg.dat': try: print("[EPGImport] unlink", filename, file=log) os.unlink(filename) except Exception as e: print( "[EPGImport] warning: Could not remove '%s' intermediate" % filename, e, file=log) self.channelFiles = self.source.channels.downloadables() if not self.channelFiles: self.afterChannelDownload(None, None) else: filename = random.choice(self.channelFiles) self.channelFiles.remove(filename) self.do_download(filename, self.afterChannelDownload, self.channelDownloadFail)
def control(self): if self.active_task: msg = '%r currently busy with a task from %r' % (self, self.active_task) chalk.red(msg) logger.debug(msg) """ First we check our task registry and check if there are any tasks that should be running right this second. If there are, we queue them. Next we execute queued tasks """ now = utc_now() tomorrow = now + datetime.timedelta(days=1) time_tasks = get_task_keys(self.task_groups) _tasks = [ t for t in TIEMPO_REGISTRY.values() if t.group in self.task_groups and t.periodic ] for _task in _tasks: stop_key = '%s:schedule:%s:stop' % (resolve_group_namespace( _task.group), _task.key) if hasattr(_task, 'force_interval'): expire_key = now + datetime.timedelta( seconds=_task.force_interval) else: expire_key = time_tasks.get(_task.get_schedule()) # the expire key is a datetime which signifies the NEXT time this # task would run if it was to be run right now. # # if there is no expire key it means that there are no tasks that # should run right now. if expire_key: # if we are here, it means we have a task that could run # right now if no other worker has already started it. try: # create a key whose existence prevents this task # from executing. This will ensure that it only runs # once across the whole network. if REDIS.setnx(stop_key, 0): logger.debug('running task %r because: %r', _task, _task.get_schedule()) logger.debug('expire key: %r', expire_key) logger.debug('now: %r', now) logger.debug('stop key: %r', stop_key) logger.debug('seconds: %r', (expire_key - now).total_seconds()) # the creation of the key was successful. # this means that previous keys whose existence # postponed execution for this task have expired. # We will now set the expiration on this key to expire # at the time of the next appropriate running of this # task REDIS.expire( stop_key, int(float((expire_key - now).total_seconds())) - 1) # queue it up _task.soon() logger.debug('next will be: %r', expire_key) except ResponseError as e: print e except BaseException as e: print e try: for g in self.task_groups: if not self.active_task: msg = '%r checking for work in group %r' % (self, g) chalk.blue(msg) logger.debug(msg) task = REDIS.lpop(resolve_group_namespace(g)) if task: logger.debug('RUNNING TASK on thread %r: %s' % (self, task)) self.active_task = resolve_group_namespace(g) threads.deferToThread(run_task, task, self) except BaseException as e: chalk.red('%s got an error' % self) print traceback.format_exc()
def store(self, file): return threads.deferToThread(self._store_in_thread, file)
def __init__(self, config, dbname): self.logger = logging.getLogger(__name__) self.logger.info('########################') self.logger.info(' Loading! ') self.logger.info('########################') self.init_finished = False self.init_failure = False self.config = config self.dbname = dbname self.loading_stations = True self.loading_events = True self.root_template = load_template(self.config.template) self.plot_template = load_template(self.config.plot_template) # # Initialize Classes # self.logger.debug( 'QueryParser(): Init DB: Load class twisted.web.resource.Resource.__init__(self)' ) twisted.web.resource.Resource.__init__(self) # # Open db using Dbcentral CLASS # self.logger.debug( "QueryParser(): Init DB: Create Dbcentral object with database(%s)." % self.dbname) self.dbcentral = Dbcentral(self.dbname, self.config.nickname, self.config.debug, ['wfdisc', 'sitechan']) if self.config.debug: self.dbcentral.info() if not self.dbcentral.list(): self.logger.critical('Init DB: No databases to use! (%s)' % self.dbname) sys.exit(twisted.internet.reactor.stop()) self.tvals = { "filters": '<option value="None">None</option>', "dbname": self.dbname, "display_arrivals": '', "display_points": '', "proxy_url": self.config.proxy_url, "dbname": self.dbname, "application_title": self.config.application_title, } for filter in self.config.filters: self.tvals['filters'] += '<option value=' + filter.replace( ' ', '_') + '>' self.tvals['filters'] += filter self.tvals['filters'] += '</option>' if self.config.event == 'true' and self.config.display_arrivals: self.tvals['display_arrivals'] = 'checked="checked"' if self.config.display_points: self.tvals['display_points'] = 'checked="checked"' if not self.dbcentral.list(): self.logger.critical( 'No valid databases to work with! -v or -V for more info') return False d = deferToThread(self._init_in_thread) d.addCallback(self._init_finished) d.addErrback(self._init_failed)
def iconcheck(self): threads.deferToThread(self.JobTask) self.timer.startLongTimer(30)
def testDeferredResult(self): d = threads.deferToThread(lambda x, y=5: x + y, 3, y=4) d.addCallback(self.assertEquals, 7) return d
def wrapped(*a, **kw): return threads.deferToThread(func, *a, **kw)
def start_instance(self, build): if self.instance is not None: raise ValueError('instance active') if self.spot_instance: return threads.deferToThread(self._request_spot_instance) return threads.deferToThread(self._start_instance)
def process_item(self, item, spider): return deferToThread(self._process_item, item, spider)