def connectionLost(self):
dt = self.getDateTime()
log.msg("[OUTPUT] Lost Connection with the attacker: %s" % self.endIP)
if not self.passwordTried:
if self.cfg.get('txtlog', 'enabled') == 'true':
txtlog.authLog(dt, self.cfg.get('folders', 'log_path') + "/" + datetime.datetime.now().strftime("%Y%m%d"), self.endIP, '', '', False)
if self.loginSuccess:
if self.cfg.get('txtlog', 'enabled') == 'true':
if os.path.exists(self.txtlog_file):
txtlog.log(dt, self.txtlog_file, '[SSH ] Lost Connection with ' + self.endIP)
if self.cfg.get('database_mysql', 'enabled') == 'true':
self.dbLog.handleConnectionLost(dt, self.sessionID)
if self.cfg.get('hpfeeds', 'enabled') == 'true':
self.hpLog.handleConnectionLost(dt)
if self.cfg.get('email', 'attack') == 'true':
threads.deferToThread(self.email, self.sensorName + ' - Attack logged', self.txtlog_file, self.ttyFiles)
if self.cfg.has_option('app_hooks', 'connection_lost'):
if self.cfg.get('app_hooks', 'connection_lost') != '':
cmdString = self.cfg.get('app_hooks', 'connection_lost') + " CONNECTION_LOST " + dt + " " + self.endIP
threads.deferToThread(self.runCommand, cmdString)
self.connections.delConn(self.sensorName, self.endIP, self.endPort)
def start_instance(self, build):
if self.instance is not None:
raise ValueError('instance active')
if self.spot_instance:
return threads.deferToThread(self._request_spot_instance)
else:
return threads.deferToThread(self._start_instance)
def download_from_legacy_server():
if config.legacy_server is None or config.legacy_server == "":
returnValue(0)
logging.info("Downloading hosts from legacy server...")
rows = yield database.run_query('SELECT `value` FROM info WHERE `key`="last_legacy_sync"')
last_legacy_sync_time = int(rows[0][0])
try:
server = yield deferToThread(xmlrpclib.ServerProxy, config.legacy_server)
response = yield deferToThread(server.get_new_hosts,
last_legacy_sync_time, config.legacy_threshold, [],
config.legacy_resiliency)
try:
last_legacy_sync_time = int(response["timestamp"])
except:
logging.ERROR("Illegal timestamp {} from legacy server".format(response["timestamp"]))
#Registry.DBPOOL.runOperation('UPDATE info SET `value`=%s WHERE `key`="last_legacy_sync"', (str(last_legacy_sync_time),))
database.run_operation('UPDATE info SET `value`=? WHERE `key`="last_legacy_sync"', str(last_legacy_sync_time))
now = time.time()
logging.debug("Got {} hosts from legacy server".format(len(response["hosts"])))
for host in response["hosts"]:
legacy = yield Legacy.find(where=["ip_address=?",host], limit=1)
if legacy is None:
logging.debug("New host from legacy server: {}".format(host))
legacy = Legacy(ip_address=host, retrieved_time=now)
else:
logging.debug("Known host from legacy server: {}".format(host))
legacy.retrieved_time = now
yield legacy.save()
except Exception, e:
logging.error("Error retrieving info from legacy server: {}".format(e))
def loginSuccessful(self, username, password):
self.passwordTried = True
self.loginSuccess = True
dt = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
self.makeSessionFolder()
if self.cfg.get('txtlog', 'enabled') == 'true':
txtlog.authLog(self.cfg.get('folders', 'log_path') + "/" + datetime.datetime.now().strftime("%Y%m%d"), self.endIP, username, password, True)
txtlog.log(self.txtlog_file, self.connectionString)
txtlog.log(self.txtlog_file, '[SSH ] Login Successful: ' + username + ':' + password)
if self.cfg.get('email', 'login') == 'true':
threads.deferToThread(self.email, self.sensorName + ' - Login Successful', self.txtlog_file)
if self.cfg.get('database_mysql', 'enabled') == 'true':
self.dbLog.handleLoginSucceeded(username, password)
self.dbLog.createSession(self.sessionID, self.endIP, self.endPort, self.honeyIP, self.honeyPort, self.sensorName)
self.dbLog.handleClientVersion(self.sessionID, self.version)
if self.cfg.get('hpfeeds', 'enabled') == 'true':
self.hpLog.handleLoginSucceeded(username, password)
self.hpLog.createSession(self.sessionID, self.endIP, self.endPort, self.honeyIP, self.honeyPort)
self.hpLog.handleClientVersion(self.version)
if self.cfg.has_option('app_hooks', 'login_successful'):
if self.cfg.get('app_hooks', 'login_successful') != '':
cmdString = self.cfg.get('app_hooks', 'login_successful') + " LOGIN_SUCCESSFUL " + dt + " " + self.endIP + " " + username + " " + password
threads.deferToThread(self.runCommand, cmdString)
def parse_message(self, sqs_message):
e = []
# decode JSON
try:
parsed_json = simplejson.loads(sqs_message.get_body())
except Exception as exception:
logger.error('Error parsing JSON %s' % exception)
return False
for k in self.message_format:
if not k in parsed_json:
logger.error('MessageTag %s not defined in sqs_message.id:%s, sqs_message_body:%s' %
(k, sqs_message.id, parsed_json))
return False
e = parsed_json['event']
for k in self.event_format:
if not k in e:
logger.error('EventTag %s not defined in sqs_message.id:%s, sqsMessageBody:%s' %
(k, sqs_message.id, parsed_json))
return False
# Add incident to Database
incident = add_incident(e, parsed_json['timestamp'])
# Check incident thresholds and fire events
if incident and incident.status == 'Active':
threads.deferToThread(ProcessIncident, incident, e['message'])
logger.info('MsgOk: EventID:%s, Element:%s, Message%s on Timestamp:%s' % (e['eventid'],
e['element'],
e['message'],
parsed_json['timestamp']))
return True
def _setup_all(self, ids):
"""
Initializing of all the circuits.
@param ids: circuits ids
@type ids: int
@return: list of setup results
@rtype: DeferredList
"""
dl = []
for id in ids:
if self.kill_all:
self.logger.debug("Dispatcher killed")
return DeferredList(dl)
if any([True for c in self._circuits if c.id == id]):
circuit = self.get(id)
if circuit.initialized:
deferToThread(circuit.run)
else:
circuit.install_dispatcher(self)
d = circuit.setup()
dl.append(d)
else:
d = maybeDeferred(self._circuit_pre_setup, id)
d.addCallback(self._circuit_setup)
@d.addErrback
def _err(failure):
self.logger.error("Circuit pre-setup failed: %s" % failure)
dl.append(d)
return DeferredList(dl)
def sendBuildbotNetUsageData(master):
if master.config.buildbotNetUsageData is None:
return
data = computeUsageData(master)
if data is None:
return
threads.deferToThread(_sendBuildbotNetUsageData, data)
def _register(self):
"""
Performs the registration based on the values provided in the form
"""
self.ui.btnRegister.setEnabled(False)
username = self.ui.lblUser.text()
password = self.ui.lblPassword.text()
password2 = self.ui.lblPassword2.text()
ok, msg = basic_password_checks(username, password, password2)
if ok:
register = SRPRegister(provider_config=self._provider_config)
register.registration_finished.connect(
self._registration_finished)
threads.deferToThread(
partial(register.register_user,
username.encode("utf8"),
password.encode("utf8")))
self._username = username
self._password = password
self._set_register_status(self.tr("Starting registration..."))
else:
self._set_register_status(msg, error=True)
self._focus_password()
self.ui.btnRegister.setEnabled(True)
def start(self):
"""
Start the buildstep.
"""
instance_id = self.getProperty('instance_id')
assert instance_id
volume_tags = dict(self.object_tags)
volume_tags['timestamp'] = self._timestamp()
conn = yield threads.deferToThread(self._connect)
volume_id, device, error = yield threads.deferToThread(
create_ami.create_volume,
conn,
instance_id,
self.volume_gib,
self.location,
volume_tags)
self.setProperty('volume_id', volume_id)
self.setProperty('device', device)
self.setProperty('volume_tags', volume_tags)
if volume_id and not error:
self.finished(results.SUCCESS)
else:
self.failed(failure.Failure(error))
def send_successful(self, results):
"""
Generic callback for when send methods are successful. This method
cleans up and shuts down the worker
"""
if (results):
threads.deferToThread(self.shutdown)
def persist_file(self, path, buf, info, meta=None, headers=None):
"""Upload file to S3 storage"""
key_name = '%s%s' % (self.prefix, path)
buf.seek(0)
if self.is_botocore:
extra = self._headers_to_botocore_kwargs(self.HEADERS)
if headers:
extra.update(self._headers_to_botocore_kwargs(headers))
return threads.deferToThread(
self.s3_client.put_object,
Bucket=self.bucket,
Key=key_name,
Body=buf,
Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
ACL=self.POLICY,
**extra)
else:
b = self._get_boto_bucket()
k = b.new_key(key_name)
if meta:
for metakey, metavalue in six.iteritems(meta):
k.set_metadata(metakey, str(metavalue))
h = self.HEADERS.copy()
if headers:
h.update(headers)
return threads.deferToThread(
k.set_contents_from_string, buf.getvalue(),
headers=h, policy=self.POLICY)
def delete(self, combo):
"""HTTP DELETE
Invalidate a UAID (and all channels associated with it).
"""
combo = combo.strip("/")
if "/" in combo:
(uaid, chid) = combo.split("/", 2)
else:
uaid = combo
chid = None
# what is request
if not self._validate_auth(uaid):
return self._write_response(
401, 109, message="Invalid Authentication")
message = self.ap_settings.message
if chid:
# mark channel as dead
self.ap_settings.metrics.increment("updates.client.unregister",
tags=self.base_tags())
d = deferToThread(self._deleteChannel, message, uaid, chid)
d.addCallback(self._success)
d.addErrback(self._chid_not_found_err)
d.addErrback(self._response_err)
return d
# nuke uaid
d = deferToThread(self._deleteUaid, message, uaid,
self.ap_settings.router)
d.addCallback(self._success)
d.addErrback(self._uaid_not_found_err)
d.addErrback(self._response_err)
return d
def _update_user_permissions(self, user_id):
""" Updates the permissions for the indicated user.
This method downloads (or loads), validates, and saves the specified user's permission settings.
@note If a non-remote permissions endpoint is specified, the permissions will be loaded from the offline local file.
@note If the indicated user already has permission settings recorded, they will be overridden with the new settings.
@throws Throws PermissionsInvalidSchema if the dictionary defined in permission_settings does not conform to the
permission settings schema, as checked by _validate_permissions().
@throws PermissionsUserNotFound if the indicated user couldn't be located in the loaded permissions resource.
@param user_id The ID of the user that the command permissions are for.
@return Returns a deferred that will be fired with the results of the permission settings load/download. That is,
an error or the permissions object for the indicated user.
"""
# Attempt to load the user's permissions
if self.use_remote_permissions:
defer_download = threads.deferToThread(self._download_remote_permissions, user_id)
else:
defer_download = threads.deferToThread(self._load_local_permissions, user_id)
# Validate & save
defer_download.addCallback(self._validate_permissions, user_id)
defer_download.addCallback(self._save_permissions, user_id)
return defer_download
def auth(client):
@fsync_readonly
def get_file_status():
file_id = self._state.file
try:
file = self.usr0.get_node(file_id)
except errors.DoesNotExist:
return 'Dead'
return file.status
d = client.dummy_authenticate("open sesame")
d.addCallback(lambda r: client.get_root())
d.addCallbacks(
lambda r: client.make_file(request.ROOT, r, "hola"),
client.test_fail)
d.addCallback(lambda req: self._save_state("file", req.new_id))
d.addCallback(lambda _: threads.deferToThread(get_file_status))
d.addCallback(lambda status: self.assert_(status == 'Live'))
d.addCallbacks(
lambda mkfile_req: client.unlink(request.ROOT,
self._state.file),
client.test_fail)
d.addCallback(lambda _: threads.deferToThread(get_file_status))
d.addCallback(lambda status: self.assert_(status == 'Dead'))
d.addCallbacks(client.test_done, client.test_fail)
def start(self, args={}, subtask_key=None, callback=None, callback_args={}, errback=None):
"""
starts the task. This will spawn the work in a workunit thread.
"""
# only start if not already running
if self._status == STATUS_RUNNING:
return
#if this was subtask find it and execute just that subtask
if subtask_key:
logger.debug('Task - starting subtask %s' % (subtask_key))
split = subtask_key.split('.')
subtask = self.get_subtask(split)
logger.debug('Task - got subtask')
self.work_deferred = threads.deferToThread(subtask._start, args, callback, callback_args)
#else this is a normal task just execute it
else:
logger.debug('Task - starting task: %s' % self)
self.work_deferred = threads.deferToThread(self._start, args, callback, callback_args)
if errback:
self.work_deferred.addErrback(errback)
return 1
def _updateInBackground(self):
if self._updating:
return
self._updating = True
deferToThread(super(CustomLsiModel, self).add_documents, self._currentChunk[:])
self._currentChunk = []
self._updating = False
def update_schedule(self):
""" Downloads the most recent version of the schedule from the active source.
@note This method loads the schedule from the active source (either a local file or network address) and updates
the local copy using callbacks. If use_local_schedule is true, the schedule will be loaded from a local file
(specified in the configuration files). If it is false, it will be loaded from the user interface API.
@return Returns a deferred that will be called with the result of the file access (the schedule object or a
Failure).
"""
# Setup local variables
defer_download = None
# Attempt to download the schedule
if self.use_network_schedule:
defer_download = threads.deferToThread(self._download_remote_schedule)
else:
defer_download = threads.deferToThread(self._download_local_schedule)
# Add a callback to store the schedule
defer_download.addCallback(self._validate_schedule)
defer_download.addCallback(self._save_schedule)
return defer_download
def start(self, c):
if not self.isProgrammed: raise Exception ("No Programmed Sequence")
yield self.inCommunication.acquire()
yield deferToThread(self.api.resetSeqCounter)
yield deferToThread(self.api.startSingle)
self.sequenceType = 'One'
self.inCommunication.release()
def process_speech(self, plist):
phrase = ''
for phrase_plist in plist['properties']['recognition']['properties']['phrases']:
for token in phrase_plist['properties']['interpretations'][0]['properties']['tokens']:
if token['properties']['removeSpaceBefore']:
phrase = phrase[:-1]
phrase += token['properties']['text']
if not token['properties']['removeSpaceAfter']:
phrase += ' '
if phrase:
self.logger.info('[Speech Recognised] "%s"' % phrase)
if self.consumer:
self.logger.info("Sending phrase to consumer")
self.consumer(phrase)
self.consumer = None
else:
for trigger, function in self.triggers:
match = trigger.search(phrase)
if match:
fname = '%s.%s' % (function.im_class.__name__, function.__func__.__name__)
self.logger.info('Phrase matched "%s" for trigger %s' % (trigger.pattern, fname))
groups = match.groups()
args = [phrase]
if groups:
args.append(groups)
threads.deferToThread(function, *args)
def handleConnectionLost(self, dt):
log.msg('[HPFEEDS] - publishing metadata to hpfeeds')
meta = self.sessionMeta
meta['endTime'] = dt
log.msg("[HPFEEDS] - sessionMeta: " + str(meta))
threads.deferToThread(self.client.publish, HONSSHSESHCHAN, **meta)
def dataReceived(self, data):
"""
Takes "data" which we assume is json encoded
If data has a subject_id attribute, we pass that to the dispatcher
as the subject_id so it will get carried through into any
return communications and be identifiable to the client
falls back to just passing the message along...
"""
try:
address = self.guid
data = json.loads(data)
threads.deferToThread(send_signal, self.dispatcher, data)
if 'hx_subscribe' in data:
return self.dispatcher.subscribe(self.transport, data)
if 'address' in data:
address = data['address']
else:
address = self.guid
self.dispatcher.send(address, data)
except Exception, exc:
raise
self.dispatcher.send(
self.guid,
{'message': data, 'error': str(exc)}
)
def __init__(self, server, **kwargs):
AbstractBackendStore.__init__(self, server, **kwargs)
self.name = 'audio CD'
self.device_name = kwargs.get('device_name', "/dev/cdom")
threads.deferToThread(self.extractAudioCdInfo)
def _handle_event(self, event_type, args=(), respond_to=False):
"""handles event dispatch"""
for event in self.events[event_type]:
if self.config['threaded']:
threads.deferToThread(self._event, event, args, respond_to)
else:
defer.execute(self._event, event, args, respond_to)
def __CREA(self, data):
"""
Massive reading files from file storage, using threads in Twisted
"""
algorithm = self.gui.FileManager.options_frame.getCryptoAlgorithm()
key = self.gui.FileManager.options_frame.getCryptoKey()
tmp_dir = self.gui.FileManager.options_frame.tmpFolder
user_folder = self.gui.FileManager.files_folder.getUsersDir()
def defferedReadFile(user_id, name, path, file_id, servers):
new_folders = os.path.normpath(user_folder + '/' + path)
try:
os.makedirs(new_folders)
except OSError:
pass
for ip, port in servers:
src_file = os.path.normpath(user_folder + '/' + path + '/' + name)
server_ip = str(ip)
server_port = str(port)
json_file = os.path.normpath(tmp_dir + '/fsc_' + self.login + '_' + name + '_' + str(randint(0, 100000)) + '.json')
dumpConfigToJSON(json_file, "READU_FILE", user_id, file_id, src_file, key, algorithm)
self.__SendInfoToFileServer(json_file, server_ip, server_port)
evt = UpdateFileListCtrlEvent()
wx.PostEvent(self.gui, evt)
for name, path, file_id, servers in data['files_read']:
threads.deferToThread(defferedReadFile, data['user_id'], name, path, file_id, servers)
del data['files_read']
del data['user_id']
def __CLNK(self, data):
"""
Downloading file by link from another file storage
"""
tmp_dir = self.gui.FileManager.options_frame.tmpFolder
def defferedDownloadByLink(user_id, save_in, filename, file_id, key, server):
src_file = os.path.normpath(save_in + '/' + filename)
server_ip = str(server[0][0])
server_port = str(server[0][1])
json_file = os.path.normpath(tmp_dir + '/fsc_link_' + self.login + '_' + filename + '_' + str(randint(0, 100000)) + '.json')
dumpConfigToJSON(json_file, "READU_FILE", user_id, file_id, src_file, key)
self.__SendInfoToFileServer(json_file, server_ip, server_port)
wx.MessageBox("File downloading {0} has completed!".format(filename), "Message")
file_info = data.get('file_info', None)
if file_info:
file_id, filename, key, server = file_info
if file_id and filename and key and server:
dlg = wx.DirDialog(self.gui, "Choose directory for saving file", style=wx.DD_DEFAULT_STYLE)
if dlg.ShowModal() == wx.ID_OK:
save_in = dlg.GetPath()
threads.deferToThread(defferedDownloadByLink, data['user_id'], save_in, filename, file_id, key, server)
else:
wx.MessageBox("Incorrect link or file has deleted!", "Error")
del data['user_id']
del data['file_info']
else:
wx.MessageBox("Incorrect link or file has deleted!", "Error")
def afterDownload(self, result, filename, deleteFile=False):
if os.path.getsize(filename) > 0:
print>>log, "[EPGImport] afterDownload", filename
if self.source.parser == 'epg.dat':
if twisted.python.runtime.platform.supportsThreads():
print>>log, "[EPGImport] Using twisted thread for DAT file"
threads.deferToThread(self.readEpgDatFile, filename, deleteFile).addCallback(lambda ignore: self.nextImport())
else:
self.readEpgDatFile(filename, deleteFile)
return
if filename.endswith('.gz'):
self.fd = gzip.open(filename, 'rb')
else:
self.fd = open(filename, 'rb')
if twisted.python.runtime.platform.supportsThreads():
print>>log, "[EPGImport] Using twisted thread!"
threads.deferToThread(self.doThreadRead).addCallback(lambda ignore: self.nextImport())
else:
self.iterator = self.createIterator()
reactor.addReader(self)
if deleteFile:
try:
print>>log, "[EPGImport] unlink", filename
os.unlink(filename)
except Exception, e:
print>>log, "[EPGImport] warning: Could not remove '%s' intermediate" % filename, e
def dataReceived(self, data):
self.dataLogger.debug('received: %s' % upperhexstr(data))
if self.state == 'IDLE':
self.logger.warning("Discarding data in IDLE state %d bytes" % len(data))
elif self.state == 'RESPONSE_WAIT':
try:
self.input = MaraFrame.parse(data)
except FieldError:
self.logger.error("Bad package")
return
# FIXME: Hacerlos con todos los campos o con ninguno
# if self.input.command != self.output.command:
# logger.warn("Command not does not match with sent command %d" % self.input.command)
# Calcular próxima sequencia
# FIXME: Checkear que la secuencia sea == a self.output.sequence
self.logger.debug("Message OK")
self.incrementSequenceNumber()
self.pending = 0
if self.factory.defer_db_save:
deferToThread(self.saveInDatabase)
else:
self.saveInDatabase()
#MaraFrame.pretty_print(self.input, show_header=False, show_bcc=False)
self.state = 'IDLE'
def listen(self):
try:
deferToThread(self.server.serve_forever)
self.running = True
except TFTPServerConfigurationError, e:
self.logger.error('TFTP server configuration error: %s!' %
e.message)
def connectionMade(self, ip, port, honeyIP, honeyPort, sensorName):
dt = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
self.sensorName = sensorName
self.honeyIP = honeyIP
self.honeyPort = honeyPort
self.logLocation = self.cfg.get('folders', 'session_path') + "/" + self.sensorName + "/"+ ip + "/"
self.downloadFolder = self.logLocation + 'downloads/'
self.txtlog_file = self.logLocation + dt + ".log"
self.endIP = ip
self.endPort = port
self.sessionID = uuid.uuid4().hex
self.passwordTried = False
self.loginSuccess = False
self.ttyFiles = []
if self.cfg.get('txtlog', 'enabled') == 'true':
self.connectionString = '[POT ] ' + self.sensorName + ' - ' + self.honeyIP + ':' + str(self.honeyPort)
self.addConnectionString('[SSH ] Incoming Connection from ' + ip + ':' + str(port))
country = self.cname(ip)
if country != None:
self.connectionString = self.connectionString + ' - ' + self.cname(ip)
if self.cfg.get('database_mysql', 'enabled') == 'true':
self.dbLog = mysql.DBLogger()
self.dbLog.setClient(self.dbLogClient, self.cfg)
if self.cfg.get('hpfeeds', 'enabled') == 'true':
self.hpLog = hpfeeds.HPLogger()
self.hpLog.setClient(self.hpLogClient, self.cfg, self.sensorName)
if self.cfg.has_option('app_hooks', 'connection_made'):
if self.cfg.get('app_hooks', 'connection_made') != '':
cmdString = self.cfg.get('app_hooks', 'connection_made') + " CONNECTION_MADE " + dt + " " + self.endIP + " " + str(port) + " " + self.honeyIP
threads.deferToThread(self.runCommand, cmdString)
def _gotDMResult(self, feed):
plains = []
feed.entry.reverse()
hasNew = False
for a in feed.entry:
entry = doubanapi.Entry(a)
entry_id = int(entry.id)
if entry_id > self.last_dm_id:
self.last_dm_id = entry_id
hasNew = True
if entry.isRead is True:
continue
plain = "Got a doumail from %s: %s\n%s" % (
entry.authorName.decode("utf-8"),
entry.title.decode("utf-8"),
entry.alternateLink,
)
plains.append(plain)
if len(plains) > 0:
log.msg("User: %s got %s new doumail" % (self.uid, len(plains)))
conn = protocol.current_conn
for jid in self.bare_jids():
conn.send_plain(jid, "\n".join(plains))
if hasNew:
threads.deferToThread(self._deferred_write, self.short_jid, "last_dm_id", self.last_dm_id)
def calldefer():
return threads.deferToThread(test)
def get_account_by_email(self, email): return threads.deferToThread(super(DeferredGoogleDataSource, self).get_account_by_email, email)
def get_events_by_email(self, email, **kwargs):
return threads.deferToThread(super(DeferredGoogleDataSource, self).get_events_by_email, email,
**kwargs)
def cleanUp(self):
print 'clean!'
yield deferToThread(time.sleep, 5)
gc.collect()
def symbolscheck(self):
threads.deferToThread(self.JobTask)
self.timer.startLongTimer(POLLTIME)
def test_cli_lifecycle(click_runner, random_policy_label, federated_ursulas,
blockchain_ursulas, custom_filepath, custom_filepath_2,
federated):
"""
This is an end to end integration test that runs each cli call
in it's own process using only CLI character control entry points,
and a mock side channel that runs in the control process
"""
# Boring Setup Stuff
alice_config_root = custom_filepath
bob_config_root = custom_filepath_2
envvars = {'NUCYPHER_KEYRING_PASSWORD': INSECURE_DEVELOPMENT_PASSWORD}
# A side channel exists - Perhaps a dApp
side_channel = MockSideChannel()
shutil.rmtree(custom_filepath, ignore_errors=True)
shutil.rmtree(custom_filepath_2, ignore_errors=True)
"""
Scene 1: Alice Installs nucypher to a custom filepath and examines her configuration
"""
# Alice performs an installation for the first time
alice_init_args = ('alice', 'init', '--network', TEMPORARY_DOMAIN,
'--config-root', alice_config_root)
if federated:
alice_init_args += ('--federated-only', )
else:
alice_init_args += ('--provider-uri', TEST_PROVIDER_URI)
alice_init_response = click_runner.invoke(nucypher_cli,
alice_init_args,
catch_exceptions=False,
env=envvars)
assert alice_init_response.exit_code == 0
# Alice uses her configuration file to run the character "view" command
alice_configuration_file_location = os.path.join(
alice_config_root, AliceConfiguration.generate_filename())
alice_view_args = ('--json-ipc', 'alice', 'public-keys', '--config-file',
alice_configuration_file_location)
alice_view_result = click_runner.invoke(nucypher_cli,
alice_view_args,
catch_exceptions=False,
env=envvars)
assert alice_view_result.exit_code == 0
alice_view_response = json.loads(alice_view_result.output)
# Alice expresses her desire to participate in data sharing with nucypher
# by saving her public key somewhere Bob and Enrico can find it.
side_channel.save_alice_pubkey(
alice_view_response['result']['alice_verifying_key'])
"""
Scene 2: Bob installs nucypher, examines his configuration and expresses his
interest to participate in data retrieval by posting his public keys somewhere public (side-channel).
"""
bob_init_args = ('bob', 'init', '--network', TEMPORARY_DOMAIN,
'--config-root', bob_config_root)
if federated:
bob_init_args += ('--federated-only', )
else:
bob_init_args += ('--provider-uri', TEST_PROVIDER_URI)
bob_init_response = click_runner.invoke(nucypher_cli,
bob_init_args,
catch_exceptions=False,
env=envvars)
assert bob_init_response.exit_code == 0
# Alice uses her configuration file to run the character "view" command
bob_configuration_file_location = os.path.join(
bob_config_root, BobConfiguration.generate_filename())
bob_view_args = ('--json-ipc', 'bob', 'public-keys', '--config-file',
bob_configuration_file_location)
bob_view_result = click_runner.invoke(nucypher_cli,
bob_view_args,
catch_exceptions=False,
env=envvars)
assert bob_view_result.exit_code == 0
bob_view_response = json.loads(bob_view_result.output)
# Bob interacts with the sidechannel
bob_public_keys = MockSideChannel.BobPublicKeys(
bob_view_response['result']['bob_encrypting_key'],
bob_view_response['result']['bob_verifying_key'])
side_channel.save_bob_public_keys(bob_public_keys)
"""
Scene 3: Alice derives a policy keypair, and saves it's public key to a sidechannel.
"""
random_label = random_policy_label.decode() # Unicode string
derive_args = ('--mock-networking', '--json-ipc', 'alice',
'derive-policy-pubkey', '--config-file',
alice_configuration_file_location, '--label', random_label)
derive_response = click_runner.invoke(nucypher_cli,
derive_args,
catch_exceptions=False,
env=envvars)
assert derive_response.exit_code == 0
derive_response = json.loads(derive_response.output)
assert derive_response['result']['label'] == random_label
# Alice and the sidechannel: at Tinagre
policy = MockSideChannel.PolicyAndLabel(
encrypting_key=derive_response['result']['policy_encrypting_key'],
label=derive_response['result']['label'])
side_channel.save_policy(policy=policy)
"""
Scene 4: Enrico encrypts some data for some policy public key and saves it to a side channel.
"""
def enrico_encrypts():
# Fetch!
policy = side_channel.fetch_policy()
enrico_args = ('--json-ipc', 'enrico', 'encrypt',
'--policy-encrypting-key', policy.encrypting_key,
'--message', PLAINTEXT)
encrypt_result = click_runner.invoke(nucypher_cli,
enrico_args,
catch_exceptions=False,
env=envvars)
assert encrypt_result.exit_code == 0
encrypt_result = json.loads(encrypt_result.output)
encrypted_message = encrypt_result['result'][
'message_kit'] # type: str
side_channel.save_message_kit(message_kit=encrypted_message)
return encrypt_result
def _alice_decrypts(encrypt_result):
"""
alice forgot what exactly she encrypted for bob.
she decrypts it just to make sure.
"""
policy = side_channel.fetch_policy()
alice_signing_key = side_channel.fetch_alice_pubkey()
message_kit = encrypt_result['result']['message_kit']
decrypt_args = (
'--mock-networking',
'--json-ipc',
'alice',
'decrypt',
'--config-file',
alice_configuration_file_location,
'--message-kit',
message_kit,
'--label',
policy.label,
)
if federated:
decrypt_args += ('--federated-only', )
decrypt_response_fail = click_runner.invoke(nucypher_cli,
decrypt_args[0:7],
catch_exceptions=False,
env=envvars)
assert decrypt_response_fail.exit_code == 2
decrypt_response = click_runner.invoke(nucypher_cli,
decrypt_args,
catch_exceptions=False,
env=envvars)
decrypt_result = json.loads(decrypt_response.output)
for cleartext in decrypt_result['result']['cleartexts']:
assert b64decode(cleartext.encode()).decode() == PLAINTEXT
# replenish the side channel
side_channel.save_policy(policy=policy)
side_channel.save_alice_pubkey(alice_signing_key)
return encrypt_result
"""
Scene 5: Alice grants access to Bob:
We catch up with Alice later on, but before she has learned about existing Ursulas...
"""
if federated:
teacher = list(federated_ursulas)[0]
else:
teacher = list(blockchain_ursulas)[1]
teacher_uri = teacher.seed_node_metadata(as_teacher_uri=True)
# Some Ursula is running somewhere
def _run_teacher(_encrypt_result):
start_pytest_ursula_services(ursula=teacher)
return teacher_uri
def _grant(teacher_uri):
# Alice fetched Bob's public keys from the side channel
bob_keys = side_channel.fetch_bob_public_keys()
bob_encrypting_key = bob_keys.bob_encrypting_key
bob_verifying_key = bob_keys.bob_verifying_key
grant_args = ('--mock-networking', '--json-ipc', 'alice', 'grant',
'--network', TEMPORARY_DOMAIN, '--teacher-uri',
teacher_uri, '--config-file',
alice_configuration_file_location, '--m', 2, '--n', 3,
'--value', Web3.toWei(1, 'ether'), '--expiration',
(maya.now() +
datetime.timedelta(days=3)).iso8601(), '--label',
random_label, '--bob-encrypting-key', bob_encrypting_key,
'--bob-verifying-key', bob_verifying_key)
if federated:
grant_args += ('--federated-only', )
else:
grant_args += ('--provider-uri', TEST_PROVIDER_URI)
grant_result = click_runner.invoke(nucypher_cli,
grant_args,
catch_exceptions=False,
env=envvars)
assert grant_result.exit_code == 0
grant_result = json.loads(grant_result.output)
# TODO: Expand test to consider manual treasure map handing
# # Alice puts the Treasure Map somewhere Bob can get it.
# side_channel.save_treasure_map(treasure_map=grant_result['result']['treasure_map'])
return grant_result
def _bob_retrieves(_grant_result):
"""
Scene 6: Bob retrieves encrypted data from the side channel and uses nucypher to re-encrypt it
"""
# Bob interacts with a sidechannel
ciphertext_message_kit = side_channel.fetch_message_kit()
policy = side_channel.fetch_policy()
policy_encrypting_key, label = policy
alice_signing_key = side_channel.fetch_alice_pubkey()
retrieve_args = ('--mock-networking', '--json-ipc', 'bob', 'retrieve',
'--teacher-uri', teacher_uri, '--config-file',
bob_configuration_file_location, '--message-kit',
ciphertext_message_kit, '--label', label,
'--policy-encrypting-key', policy_encrypting_key,
'--alice-verifying-key', alice_signing_key)
if federated:
retrieve_args += ('--federated-only', )
retrieve_response = click_runner.invoke(nucypher_cli,
retrieve_args,
catch_exceptions=False,
env=envvars)
assert retrieve_response.exit_code == 0
retrieve_response = json.loads(retrieve_response.output)
for cleartext in retrieve_response['result']['cleartexts']:
assert b64decode(cleartext.encode()).decode() == PLAINTEXT
return
# Run the Callbacks
d = threads.deferToThread(enrico_encrypts) # scene 4
d.addCallback(_alice_decrypts) # scene 5 (uncertainty)
d.addCallback(_run_teacher) # scene 6 (preamble)
d.addCallback(_grant) # scene 7
d.addCallback(_bob_retrieves) # scene 8
yield d
def wrapper(*args, **kwargs):
from twisted.internet import threads
return threads.deferToThread(f, *args, **kwargs)
class CommandView(DefaultView):
context(ICommand)
def write_results(self, request, pid, cmd):
log.msg('Called %s got result: pid(%s) term writes=%s' %
(cmd, pid, len(cmd.write_buffer)),
system='command-view')
request.write(
json.dumps({
'status': 'ok',
'pid': pid,
'stdout': cmd.write_buffer
}))
request.finish()
def render_PUT(self, request):
""" Converts arguments into command-line counterparts and executes the omsh command.
Parameters passed as 'arg' are converted into positional arguments, others are converted into
named parameters:
PUT /bin/ls?arg=/some/path&arg=/another/path&-l&--recursive
thus translates to:
/bin/ls /some/path /another/path -l --recursive
Allows blocking (synchronous) and non-blocking operation using the 'asynchronous' parameter (any
value will trigger it). Synchronous operation requires two threads to function.
"""
def named_args_filter_and_flatten(nargs):
for name, vallist in nargs:
if name not in ('arg', 'asynchronous'):
for val in vallist:
yield name
yield val
def convert_args(args):
tokenized_args = args.get('arg', [])
return tokenized_args + list(
named_args_filter_and_flatten(args.items()))
protocol = DetachedProtocol()
protocol.interaction = get_interaction(
self.context) or request.interaction
args = convert_args(request.args)
args = filter(None, args)
cmd = self.context.cmd(protocol)
# Setting write_buffer to a list makes command save the output to the buffer too
cmd.write_buffer = []
d0 = defer.Deferred()
try:
pid = threads.blockingCallFromThread(
reactor, cmd.register, d0, args,
'%s %s' % (request.path, args))
except ArgumentParsingError, e:
raise BadRequest(str(e))
q = Queue.Queue()
def cancel_execute(_, d):
d.cancel()
def execute(q, cmd, args):
d = defer.maybeDeferred(cmd, *args)
request.notifyFinish().addBoth(cancel_execute, d)
d.addBoth(q.put)
d.chainDeferred(d0)
# cmd() may be synchronous (that are executed inline, without deferreds) or asynchronous.
# We want synchronous methods to be executed in a separate thread
dt = threads.deferToThread(execute, q, cmd, args)
request.notifyFinish().addBoth(cancel_execute, dt)
asynchronous = request.args.get('asynchronous', [])
if not asynchronous:
dt.addBoth(lambda r: threads.deferToThread(q.get, True, 300))
dt.addCallback(lambda r: reactor.callFromThread(
self.write_results, request, pid, cmd))
def handleArgumentParsingError(e, pid, cmd):
e.trap(ArgumentParsingError)
log.err(e, system='http-cmd')
request.setResponseCode(400)
request.write(str(e.value))
request.finish()
def handleQueueEmpty(e, pid, cmd):
e.trap(Queue.Empty)
msg = 'Timeout waiting for command %s (%s) to complete' % (
request.path, args)
log.msg(msg, system='http-cmd')
request.setResponseCode(504)
request.write(msg)
request.finish()
def handleUnknownError(e, pid, cmd):
e.trap(Exception)
log.err(e, system='http-cmd')
request.setResponseCode(500)
request.write(''.join(traceback.format_exc(e.value)))
request.finish()
dt.addErrback(handleArgumentParsingError, pid, cmd)
dt.addErrback(handleQueueEmpty, pid, cmd)
dt.addErrback(handleUnknownError, pid, cmd)
return NOT_DONE_YET
def process(self, msg, sender=None):
cmd = [self.print_command, msg]
return threads.deferToThread(subprocess32.call, cmd)
def _deferToThreadPool(self, f, *a, **kw):
return deferToThread(f, *a, **kw)
def submit_entries(self, batch):
"""
Large parts of this method are adapted from kippo-pyshield by jkakavas
Many thanks to their efforts. https://github.com/jkakavas/kippo-pyshield
"""
# The nonce is predefined as explained in the original script :
# trying to avoid sending the authentication key in the "clear" but
# not wanting to deal with a full digest like exchange. Using a
# fixed nonce to mix up the limited userid.
_nonceb64 = 'ElWO1arph+Jifqme6eXD8Uj+QTAmijAWxX1msbJzXDM='
log_output = ''
for attempt in self.batch:
log_output += '{0}\t{1}\t{2}\t{3}\t{4}\t{5}\n'.format(
attempt['date'], attempt['time'], attempt['timezone'],
attempt['source_ip'], attempt['user'], attempt['password'])
nonce = base64.b64decode(_nonceb64)
digest = base64.b64encode(
hmac.new('{0}{1}'.format(nonce, self.userid),
base64.b64decode(self.auth_key), hashlib.sha256).digest())
auth_header = 'credentials={0} nonce={1} userid={2}'.format(
digest, _nonceb64, self.userid)
headers = {
'X-ISC-Authorization': auth_header,
'Content-Type': 'text/plain',
'Content-Length': len(log_output)
}
log.msg(headers)
req = threads.deferToThread(
requests.request,
method='PUT',
url='https://secure.dshield.org/api/file/sshlog',
headers=headers,
timeout=10,
data=log_output)
def check_response(resp):
failed = False
response = resp.content
if resp.status_code == requests.codes.ok:
sha1_regex = re.compile(
r'<sha1checksum>([^<]+)<\/sha1checksum>')
sha1_match = sha1_regex.search(response)
if sha1_match is None:
log.err(
'dshield ERROR: Could not find sha1checksum in response'
)
failed = True
sha1_local = hashlib.sha1()
sha1_local.update(log_output)
if sha1_match.group(1) != sha1_local.hexdigest():
log.err('dshield ERROR: SHA1 Mismatch {0} {1} .'.format(
sha1_match.group(1), sha1_local.hexdigest()))
failed = True
md5_regex = re.compile(r'<md5checksum>([^<]+)<\/md5checksum>')
md5_match = md5_regex.search(response)
if md5_match is None:
log.err(
'dshield ERROR: Could not find md5checksum in response'
)
failed = True
md5_local = hashlib.md5()
md5_local.update(log_output)
if md5_match.group(1) != md5_local.hexdigest():
log.err('dshield ERROR: MD5 Mismatch {0} {1} .'.format(
md5_match.group(1), md5_local.hexdigest()))
failed = True
log.msg(
'dshield SUCCESS: Sent {0} bytes worth of data to secure.dshield.org'
.format(len(log_output)))
else:
log.err('dshield ERROR: error {0}.'.format(resp.status_code))
log.err('Response was {0}'.format(response))
failed = True
if failed:
# Something went wrong, we need to add them to batch.
reactor.callFromThread(self.transmission_error, batch)
req.addCallback(check_response)
def test_thread(self):
return threads.deferToThread(lambda : None)
def iconcheck(self):
try:
threads.deferToThread(self.JobTask)
except:
pass
self.timer.startLongTimer(30)
def stop_recording(self):
if self.recorder and self.recorder.is_alive():
self.recorder.terminate()
d = threads.deferToThread(self._flv2webm)
d.addBoth(lambda s: None)
def watch(self, count):
self.debug("[watchdog] watching (%d)" % count)
if (self.loop_call.running):
self.checks.append(deferToThread(self._check, count))
def lineReceived(self, line):
d = deferToThread(self.handle_request, line)
d.addCallbacks(self.factory.cbSucceed, self.factory.cbFailed)
def route_notification(self, notification, uaid_data):
"""Route a notification to an internal node, and store it if the node
can't deliver immediately or is no longer a valid node
"""
# Determine if they're connected at the moment
node_id = uaid_data.get("node_id")
uaid = uaid_data["uaid"]
router = self.db.router
# Node_id is present, attempt delivery.
# - Send Notification to node
# - Success: Done, return 200
# - Error (Node busy): Jump to Save notification below
# - Error (Client gone, node gone/dead): Clear node entry for user
# - Both: Done, return 503
if node_id:
result = None
try:
result = yield self._send_notification(uaid, node_id,
notification)
except (ConnectError, ConnectionClosed, ResponseFailed,
CancelledError, PotentialDataLoss) as exc:
self.metrics.increment("updates.client.host_gone")
yield deferToThread(router.clear_node,
uaid_data).addErrback(self._eat_db_err)
if isinstance(exc, ConnectionRefusedError):
# Occurs if an IP record is now used by some other node
# in AWS or if the connection timesout.
self.log.debug("Could not route message: {exc}", exc=exc)
if result and result.code == 200:
returnValue(self.delivered_response(notification))
# Save notification, node is not present or busy
# - Save notification
# - Success (older version): Done, return 202
# - Error (db error): Done, return 503
try:
yield self._save_notification(uaid_data, notification)
except ClientError as e:
log_exception = (e.response["Error"]["Code"] !=
"ProvisionedThroughputExceededException")
raise RouterException("Error saving to database",
status_code=503,
response_body="Retry Request",
log_exception=log_exception,
errno=201)
# - Lookup client again to get latest node state after save.
# - Success (node found): Notify node of new notification
# - Success: Done, return 200
# - Error (no client): Done, return 202
# - Error (no node): Clear node entry
# - Both: Done, return 202
# - Success (no node): Done, return 202
# - Error (db error): Done, return 202
# - Error (no client) : Done, return 404
try:
uaid_data = yield deferToThread(router.get_uaid, uaid)
except ClientError:
returnValue(self.stored_response(notification))
except ItemNotFound:
self.metrics.increment("updates.client.deleted")
raise RouterException("User was deleted",
status_code=410,
response_body="Invalid UAID",
log_exception=False,
errno=105)
# Verify there's a node_id in here, if not we're done
node_id = uaid_data.get("node_id")
if not node_id:
returnValue(self.stored_response(notification))
try:
result = yield self._send_notification_check(uaid, node_id)
except (ConnectError, ConnectionClosed, ResponseFailed) as exc:
self.metrics.increment("updates.client.host_gone")
if isinstance(exc, ConnectionRefusedError):
self.log.debug("Could not route message: {exc}", exc=exc)
yield deferToThread(router.clear_node,
uaid_data).addErrback(self._eat_db_err)
returnValue(self.stored_response(notification))
if result.code == 200:
returnValue(self.delivered_response(notification))
else:
ret_val = self.stored_response(notification)
returnValue(ret_val)
def blockOnThread(*args, **kwargs):
return threads.deferToThread(monitored_benchmark,
threads.blockingCallFromThread, reactor,
*args, **kwargs)
def defer_to_thread(block_method, result_method, *args):
d = threads.deferToThread(block_method, *args)
d.addCallback(result_method)
def toThread(sio):
d = threads.deferToThread(parseAvitoImage, sio, ob)
d.addCallback(saveWithPhone)
return d
def collect(self, config):
return threads.deferToThread(lambda: self.inner(config))
def afterDownload(self, result, filename, deleteFile=False):
print("[EPGImport] afterDownload", filename, file=log)
try:
if not os.path.getsize(filename):
raise Exception("File is empty")
except Exception as e:
self.downloadFail(e)
return
if self.source.parser == 'epg.dat':
if twisted.python.runtime.platform.supportsThreads():
print("[EPGImport] Using twisted thread for DAT file",
file=log)
threads.deferToThread(
self.readEpgDatFile, filename,
deleteFile).addCallback(lambda ignore: self.nextImport())
else:
self.readEpgDatFile(filename, deleteFile)
return
if filename.endswith('.gz'):
self.fd = gzip.open(filename, 'rb')
try:
# read a bit to make sure it's a gzip file
self.fd.read(10)
self.fd.seek(0, 0)
except Exception as e:
print("[EPGImport] File downloaded is not a valid gzip file",
filename,
file=log)
self.downloadFail(e)
return
elif filename.endswith('.xz') or filename.endswith('.lzma'):
try:
import lzma
except ImportError:
from backports import lzma
self.fd = lzma.open(filename, 'rb')
try:
# read a bit to make sure it's an xz file
self.fd.read(10)
self.fd.seek(0, 0)
except Exception as e:
print("[EPGImport] File downloaded is not a valid xz file",
filename,
file=log)
self.downloadFail(e)
return
else:
self.fd = open(filename, 'rb')
if deleteFile and self.source.parser != 'epg.dat':
try:
print("[EPGImport] unlink", filename, file=log)
os.unlink(filename)
except Exception as e:
print(
"[EPGImport] warning: Could not remove '%s' intermediate" %
filename,
e,
file=log)
self.channelFiles = self.source.channels.downloadables()
if not self.channelFiles:
self.afterChannelDownload(None, None)
else:
filename = random.choice(self.channelFiles)
self.channelFiles.remove(filename)
self.do_download(filename, self.afterChannelDownload,
self.channelDownloadFail)
def control(self):
if self.active_task:
msg = '%r currently busy with a task from %r' % (self,
self.active_task)
chalk.red(msg)
logger.debug(msg)
"""
First we check our task registry and check if there are any
tasks that should be running right this second.
If there are, we queue them.
Next we execute queued tasks
"""
now = utc_now()
tomorrow = now + datetime.timedelta(days=1)
time_tasks = get_task_keys(self.task_groups)
_tasks = [
t for t in TIEMPO_REGISTRY.values()
if t.group in self.task_groups and t.periodic
]
for _task in _tasks:
stop_key = '%s:schedule:%s:stop' % (resolve_group_namespace(
_task.group), _task.key)
if hasattr(_task, 'force_interval'):
expire_key = now + datetime.timedelta(
seconds=_task.force_interval)
else:
expire_key = time_tasks.get(_task.get_schedule())
# the expire key is a datetime which signifies the NEXT time this
# task would run if it was to be run right now.
#
# if there is no expire key it means that there are no tasks that
# should run right now.
if expire_key:
# if we are here, it means we have a task that could run
# right now if no other worker has already started it.
try:
# create a key whose existence prevents this task
# from executing. This will ensure that it only runs
# once across the whole network.
if REDIS.setnx(stop_key, 0):
logger.debug('running task %r because: %r', _task,
_task.get_schedule())
logger.debug('expire key: %r', expire_key)
logger.debug('now: %r', now)
logger.debug('stop key: %r', stop_key)
logger.debug('seconds: %r',
(expire_key - now).total_seconds())
# the creation of the key was successful.
# this means that previous keys whose existence
# postponed execution for this task have expired.
# We will now set the expiration on this key to expire
# at the time of the next appropriate running of this
# task
REDIS.expire(
stop_key,
int(float((expire_key - now).total_seconds())) - 1)
# queue it up
_task.soon()
logger.debug('next will be: %r', expire_key)
except ResponseError as e:
print e
except BaseException as e:
print e
try:
for g in self.task_groups:
if not self.active_task:
msg = '%r checking for work in group %r' % (self, g)
chalk.blue(msg)
logger.debug(msg)
task = REDIS.lpop(resolve_group_namespace(g))
if task:
logger.debug('RUNNING TASK on thread %r: %s' %
(self, task))
self.active_task = resolve_group_namespace(g)
threads.deferToThread(run_task, task, self)
except BaseException as e:
chalk.red('%s got an error' % self)
print traceback.format_exc()
def store(self, file):
return threads.deferToThread(self._store_in_thread, file)
def __init__(self, config, dbname):
self.logger = logging.getLogger(__name__)
self.logger.info('########################')
self.logger.info(' Loading! ')
self.logger.info('########################')
self.init_finished = False
self.init_failure = False
self.config = config
self.dbname = dbname
self.loading_stations = True
self.loading_events = True
self.root_template = load_template(self.config.template)
self.plot_template = load_template(self.config.plot_template)
#
# Initialize Classes
#
self.logger.debug(
'QueryParser(): Init DB: Load class twisted.web.resource.Resource.__init__(self)'
)
twisted.web.resource.Resource.__init__(self)
#
# Open db using Dbcentral CLASS
#
self.logger.debug(
"QueryParser(): Init DB: Create Dbcentral object with database(%s)."
% self.dbname)
self.dbcentral = Dbcentral(self.dbname, self.config.nickname,
self.config.debug, ['wfdisc', 'sitechan'])
if self.config.debug: self.dbcentral.info()
if not self.dbcentral.list():
self.logger.critical('Init DB: No databases to use! (%s)' %
self.dbname)
sys.exit(twisted.internet.reactor.stop())
self.tvals = {
"filters": '<option value="None">None</option>',
"dbname": self.dbname,
"display_arrivals": '',
"display_points": '',
"proxy_url": self.config.proxy_url,
"dbname": self.dbname,
"application_title": self.config.application_title,
}
for filter in self.config.filters:
self.tvals['filters'] += '<option value=' + filter.replace(
' ', '_') + '>'
self.tvals['filters'] += filter
self.tvals['filters'] += '</option>'
if self.config.event == 'true' and self.config.display_arrivals:
self.tvals['display_arrivals'] = 'checked="checked"'
if self.config.display_points:
self.tvals['display_points'] = 'checked="checked"'
if not self.dbcentral.list():
self.logger.critical(
'No valid databases to work with! -v or -V for more info')
return False
d = deferToThread(self._init_in_thread)
d.addCallback(self._init_finished)
d.addErrback(self._init_failed)
def iconcheck(self):
threads.deferToThread(self.JobTask)
self.timer.startLongTimer(30)
def testDeferredResult(self):
d = threads.deferToThread(lambda x, y=5: x + y, 3, y=4)
d.addCallback(self.assertEquals, 7)
return d
def wrapped(*a, **kw):
return threads.deferToThread(func, *a, **kw)
def start_instance(self, build):
if self.instance is not None:
raise ValueError('instance active')
if self.spot_instance:
return threads.deferToThread(self._request_spot_instance)
return threads.deferToThread(self._start_instance)
def process_item(self, item, spider):
return deferToThread(self._process_item, item, spider)
