def __init__(self, url):
Error.__init__(self, 302, "redirect")
self.url = url
def validate_options(self, scrapy_request_args, api_params):
url = scrapy_request_args.get("url")
start_requests = api_params.get("start_requests")
if not url and not start_requests:
raise Error('400',
"'url' is required if start_requests are disabled")
def _secretreq(self, minfo):
# unpack the request
try:
enclave_id = minfo['enclave_id']
contract_id = minfo['contract_id']
opk = minfo['opk']
signature = minfo['signature']
except KeyError as ke:
raise Error(http.BAD_REQUEST,
'missing required field {0}'.format(ke))
logger.debug('request for key for contract %s, enclave %s',
contract_id, enclave_id)
# verify the signature, that is, make sure that the request was really signed by opk
try:
opkkey = pcrypto.SIG_PublicKey(opk)
opkkey.VerifySignature(
pcrypto.string_to_byte_array(enclave_id + contract_id),
pcrypto.hex_to_byte_array(signature))
except:
logger.warn("Signature verification failed")
raise Error(http.BAD_REQUEST, 'Signature Mismatch')
# Get enclave state
try:
logger.debug('retrieve information for enclave %s', enclave_id)
enclave_info = self.__registry_helper.get_enclave_info(enclave_id)
logger.debug("enclave information retrieved: %s", enclave_info)
except Exception as err:
logger.error(
'exception occurred when getting ledger information for enclave %s; %s',
enclave_id, str(err))
raise Exception(
'could not retrieve enclave state; {0}'.format(err))
# Get contract state
try:
logger.debug('retrieve information for contract <%s>', contract_id)
contract_info = self.__registry_helper.get_contract_info(
contract_id)
logger.debug("contract_info from ledger: %s", contract_info)
except Exception as err:
logger.error(
'exception occurred when getting ledger information for contract %s; %s',
contract_id, str(err))
raise Exception(
'could not retrieve contract state; {0}'.format(err))
# make sure that the signer of this request is really the owner of the contract
try:
# make sure that the signer of this request is really the owner of the contract
# PdoContractInfo.pdo_contract_creator_pem_key is the VerifyingKey
logger.debug("Contract creator's public key: %s",
contract_info['pdo_contract_creator_pem_key'])
logger.debug("Expected public key: %s", opk)
assert contract_info['pdo_contract_creator_pem_key'] == opk
except:
logger.error(
'request to create secret did not come from the contract owner; %s != %s',
contracttxn.OriginatorID, opk)
raise Error(http.NOT_ALLOWED,
'operation not allowed for {0}'.format(opk))
# make sure the provisioning service is allowed to access contract by the checking the list of allowed provisioning services
try:
logger.debug("Contract allowed service ids: %s",
contract_info['provisioning_service_ids'])
logger.debug("Expected provisioning service id: %s", self.PSPK)
assert self.PSPK in contract_info['provisioning_service_ids']
except:
logger.error(
'This Pservice is not the list of allowed provisioning services, PSerivce ID: %s',
self.PSPK)
raise Error(http.NOT_ALLOWED,
'operation not allowed for {0}'.format(self.PSPK))
# retrieve the sealed secret
sealed_secret = self._GetContractSecret(contract_id)
logger.debug("Enclave Info: %s", str(enclave_info))
# Generate Secret for Contract Enclave, signs unsealed secret with contract enclave encryption key
esecret = self.Enclave.generate_enclave_secret(
self.SealedData,
sealed_secret,
contract_id,
opk,
json.dumps(enclave_info),
)["enclave_secret"]
logger.debug("Encrypted secret for contract %s: %s", contract_id,
esecret)
# create the response
response = dict()
response['pspk'] = self.PSPK
response['encrypted_secret'] = esecret
logger.info('created secret for contract %s and enclave %s',
contract_id, enclave_id)
return response
def trapNotFound(failure):
failure.trap(error.NodeNotFound)
raise Error(http.NOT_FOUND, "Node not found")
def trapXMPPURIParseError(failure):
failure.trap(XMPPURIParseError)
raise Error(http.BAD_REQUEST,
"Malformed XMPP URI: %s" % failure.value)
def renderQuery(self, query):
query = self.graphql.parse(query)
errors = self.graphql.validate(self.schema, query)
if errors:
raise Error(400, [e.formatted for e in errors])
def fail(url, timeout=None):
raise Error(code=404)
def _secretreq(self, minfo):
# unpack the request
try:
enclave_id = minfo['enclave_id']
contract_id = minfo['contract_id']
opk = minfo['opk']
signature = minfo['signature']
except KeyError as ke:
raise Error(http.BAD_REQUEST,
'missing required field {0}'.format(ke))
logger.debug('request for key for contract %s, enclave %s',
contract_id, enclave_id)
# verify the signature, that is, make sure that the request was really signed by opk
try:
opkkey = pcrypto.SIG_PublicKey(opk)
opkkey.VerifySignature(
pcrypto.string_to_byte_array(enclave_id + contract_id),
pcrypto.hex_to_byte_array(signature))
except:
logger.warn("Signature verification failed")
raise Error(http.BAD_REQUEST, 'Signature Mismatch')
# Get enclave state
try:
logger.debug('retrieve information for enclave %s', enclave_id)
enclave_info = self.__registry_helper.get_enclave_dict(enclave_id)
logger.debug("enclave information retrieved: %s", enclave_info)
except BaseException as err:
logger.warn(
'exception occurred when getting ledger information for enclave %s; %s',
enclave_id, str(err))
raise Error(http.BAD_REQUEST,
'could not retrieve enclave state; {0}'.format(err))
except ClientConnectException as err:
logger.warn(
'client exception occurred when getting ledger information for enclave %s; %s',
enclave_id, str(err))
raise Error(http.BAD_REQUEST,
'could not retrieve enclave state; {0}'.format(err))
# Get contract state
try:
logger.debug('retrieve information for contract <%s>', contract_id)
contract_info = self.__registry_helper.get_contract_dict(
contract_id)
logger.debug("contract_info from ledger: %s", contract_info)
except BaseException as err:
logger.warn(
'exception occurred when getting ledger information for contract %s; %s',
contract_id, str(err))
raise Error(http.BAD_REQUEST,
'could not retrieve contract state; {0}'.format(err))
except ClientConnectException as err:
logger.warn(
'client exception occurred when getting ledger information for contract %s; %s',
contract_id, str(err))
raise Error(http.BAD_REQUEST,
'could not retrieve contract state; {0}'.format(err))
# make sure that the signer of this request is really the owner of the contract
try:
# make sure that the signer of this request is really the owner of the contract
# PdoContractInfo.pdo_contract_creator_pem_key is the VerifyingKey
logger.debug("Contract creator's public key: %s",
contract_info['pdo_contract_creator_pem_key'])
logger.debug("Expected public key: %s", opk)
assert contract_info['pdo_contract_creator_pem_key'] == opk
except:
logger.error(
'request to create secret did not come from the contract owner; %s != %s',
contracttxn.OriginatorID, opk)
raise Error(http.NOT_ALLOWED,
'operation not allowed for {0}'.format(opk))
# make sure the provisioning service is allowed to access contract by the checking the list of allowed provisioning services
try:
logger.debug("Contract allowed service ids: %s",
contract_info['provisioning_service_ids'])
logger.debug("Expected provisioning service id: %s",
self.PSPK.Serialize())
assert self.PSPK.Serialize(
) in contract_info['provisioning_service_ids']
except:
logger.error(
'This Pservice is not the list of allowed provisioning services, PSerivce ID: %s',
self.PSPK.Serialize())
raise Error(
http.NOT_ALLOWED,
'operation not allowed for {0}'.format(self.PSPK.Serialize()))
# retrieve the secret
secret = self._GetContractSecret(contract_id)
# create the signature
message = secret + enclave_id + contract_id + opk
secretsig = pcrypto.byte_array_to_hex(
self.SigningKey.SignMessage(pcrypto.string_to_byte_array(message)))
# pad secret to required max size
# TODO: Eventually this requirement needs to be fixed in the crypto library itself
required_padding = 2 * pcrypto.MAX_SIG_SIZE - len(secretsig)
secretsig = secretsig + ('0' * required_padding)
enclavekey = pcrypto.PKENC_PublicKey(enclave_info['encryption_key'])
esecret = pcrypto.byte_array_to_base64(
enclavekey.EncryptMessage(
pcrypto.string_to_byte_array(secret + secretsig)))
logger.debug("Encrypted secret for contract %s: %s", contract_id,
esecret)
# create the response
response = dict()
response['pspk'] = self.PSPK.Serialize()
response['encrypted_secret'] = esecret
logger.info('created secret for contract %s and enclave %s',
contract_id, enclave_id)
return response
def __init__(self, url):
Error.__init__(self, 302, "redirect")
self.url = protect_redirect_url(unicode2bytes(url))
def test_get_bad_code(self):
return self.check_get_error(
fail(Error("999")), ProviderError,
"Unexpected HTTP 999 trying to GET "
"http://somewhe.re/rubber/chicken")
def data_acquisition(self, bucket):
request = bucket.request
content_type = request.getHeader('Content-Type')
# Read and decode request body
body = request.content.read()
bucket.body = body
# Decode data from request body
if body:
# Decode data from JSON format
if content_type.startswith('application/json'):
return json.loads(body)
# Decode data from x-www-form-urlencoded format
elif content_type.startswith('application/x-www-form-urlencoded'):
# TODO: Honor charset when receiving "application/x-www-form-urlencoded; charset=utf-8"
payload = parse_qs(body, 1)
# TODO: Decapsulate multiple values of same reading into "{name}-{N}", where N=1...
decoded = {}
for key, value in payload.items():
key = key.decode()
if type(value) is list:
decoded[key] = value[0].decode()
return decoded
# Decode data from CSV format
elif content_type.startswith('text/csv'):
if not self.metastore:
log.error(
'Generic decoding of CSV format requires metastore')
# Prepare alias for metastore table
csv_header_store = self.metastore.kotori['channel-csv-headers']
# 1. Decode CSV header like '## weight,temperature, humidity' and remember for upcoming data readings
def parse_header(channel_info, data_lines):
first_line = data_lines[0]
header_line = None
options = {}
# Regular header announcement
if first_line.startswith('## '):
header_line = first_line[3:].strip()
data_lines.pop(0)
# Convenience hack to support Open Hive CSV import
elif first_line.startswith(
'Date/Time') or first_line.startswith(
'Datum/Zeit'):
header_line = first_line
data_lines.pop(0)
# Convenience hack to support Beelogger CSV import
elif first_line.startswith('Datum,Uhrzeit'):
header_line = first_line
data_lines.pop(0)
options['rules'] = [{
'type': 'fuse',
'source': ['Datum', 'Uhrzeit'],
'target': 'time',
'join': 'T',
'suffix': 'Z'
}]
# Convenience hack to support import from http://archive.luftdaten.info/
elif first_line.startswith('sensor_id'):
header_line = first_line
data_lines.pop(0)
if header_line:
# Streamline various differences for even more convenience.
header_line = header_line.replace(';', ',')
# FIXME: Unify with ``kotori.daq.storage.influx.format_chunk()``.
date_fields = [
'Date/Time', 'Date', 'Datum/Zeit', 'timestamp'
]
for date_field in date_fields:
header_line = header_line.replace(
date_field, 'time')
header_fields = list(
map(str.strip, header_line.split(',')))
msg = u'CSV Header: fields={fields}, key={key}'.format(
fields=header_fields,
key=request.channel_identifier)
log.info(msg)
csv_header_store.update_one(
{"channel": request.channel_identifier}, {
"$set": {
"header_fields": header_fields,
"options": options
}
},
upsert=True)
message = u'Received header fields {}'.format(
header_fields)
request.messages.append({
'type': 'info',
'message': message
})
channel_info['header_fields'] = header_fields
channel_info['options'] = options
#print('header_fields, data_lines:', header_fields, data_lines)
#return header_fields, data_lines
# 2. Decode data, map to full-qualified payload container
def parse_data(channel_info):
channel_info = channel_info or {}
data_raw = body.decode().strip()
data_lines = list(map(str.strip, data_raw.split('\n')))
parse_header(channel_info, data_lines)
header_fields = channel_info.get('header_fields')
if not header_fields:
raise Error(
http.BAD_REQUEST,
response=
b'Could not process data, please supply field names '
b'via CSV header before sending readings')
#print('data_lines:', data_lines; pprint(data_lines))
data_list = []
for data_line in data_lines:
data_fields = list(
map(str.strip,
data_line.replace(';', ',').split(',')))
#print 'header_fields, data_fields:', header_fields, data_fields
data = OrderedDict(
list(zip(header_fields, data_fields)))
self.manipulate_data(data, channel_info)
data_list.append(data)
return data_list
try:
channel_info = csv_header_store.find_one(
filter={"channel": request.channel_identifier})
except Exception as ex:
log.failure(
'Could not process CSV data, unknown database error: {0}'
.format(ex))
raise Error(
http.INTERNAL_SERVER_ERROR,
response=
b'Could not process CSV data, unknown database error: {0}'
.format(ex))
return parse_data(channel_info)
else:
msg = u"Unable to handle Content-Type '{content_type}'".format(
content_type=content_type)
log.warn(msg)
raise Error(http.UNSUPPORTED_MEDIA_TYPE, response=msg)
else:
msg = u'Empty request body'
log.warn(msg)
raise Error(http.BAD_REQUEST, response=msg)
def test_get_404(self):
return self.check_get_error(
fail(Error("404")), FileNotFound,
"File was not found: 'http://somewhe.re/rubber/chicken'")
def raise_error(body):
raise Error(response.code, response=body)
def __str__(self):
return Error.__str__(self) + " " + ensure_str(self.response)
def directoryListing(self):
raise Error(NOT_ALLOWED, b"Not allowed")
def directoryListing(self):
# type: () -> None
raise Error(NOT_ALLOWED, b"Not allowed")
def __init__(self, msg):
Error.__init__(self, 403, msg)
def test_error_403(self, log_msg_mock):
exc = Error('403', 'blah_403')
result = self.resource.handle_error(exc, self.request)
self.assertEqual(self.request.code, 403)
self.assertFalse(log_msg_mock.called)
self.assertEqual(result['message'], exc.message)
def mock_fs_get(self, url, code, content=None):
self.getPage(url)
if code == 200:
self.mocker.result(succeed(content))
else:
self.mocker.result(fail(Error(str(code))))
def render_post(self, request, components, msg):
"""
Forward a signed message through the gossip network.
"""
encoding = request.getHeader('Content-Type')
data = request.content.getvalue()
try:
if encoding == 'application/json':
minfo = json2dict(data)
elif encoding == 'application/cbor':
minfo = cbor2dict(data)
else:
raise Error(
"", 'unknown message'
' encoding: {0}'.format(encoding))
typename = minfo.get('__TYPE__', '**UNSPECIFIED**')
if typename not in self.Ledger.MessageHandlerMap:
raise Error(
"", 'received request for unknown message'
' type, {0}'.format(typename))
msg = self.Ledger.MessageHandlerMap[typename][0](minfo)
except Error as e:
LOGGER.info('exception while decoding http request %s; %s',
request.path, traceback.format_exc(20))
raise Error(http.BAD_REQUEST,
'unable to decode incoming request: {0}'.format(e))
if self.Validator.Config.get("LocalValidation", True):
# determine if the message contains a valid transaction before
# we send the message to the network
# we need to start with a copy of the message due to cases
# where side effects of the validity check may impact objects
# related to msg
mymsg = copy.deepcopy(msg)
if hasattr(mymsg, 'Transaction') and mymsg.Transaction is not None:
mytxn = mymsg.Transaction
LOGGER.info(
'starting local validation '
'for txn id: %s type: %s', mytxn.Identifier,
mytxn.TransactionTypeName)
pending_block_txns = None
if self.Ledger.PendingTransactionBlock is not None:
pending_block_txns = \
self.Ledger.PendingTransactionBlock.TransactionIDs
block_id = self.Ledger.MostRecentCommittedBlockID
real_store_map = \
self.Ledger.GlobalStoreMap.get_block_store(block_id)
temp_store_map = \
global_store_manager.BlockStore(real_store_map)
if not temp_store_map:
LOGGER.info('no store map for block %s', block_id)
raise Error(
http.BAD_REQUEST, 'unable to validate enclosed'
' transaction {0}'.format(data))
pending_txns = copy.copy(self.Ledger.PendingTransactions)
pending_txn_ids = [x for x in pending_txns.iterkeys()]
# clone a copy of the ledger's message queue so we can
# temporarily play forward all locally submitted yet
# uncommitted transactions
my_queue = copy.deepcopy(self.Ledger.MessageQueue)
transaction_type = mytxn.TransactionTypeName
if transaction_type not in temp_store_map.TransactionStores:
LOGGER.info('transaction type %s not in global store map',
transaction_type)
raise Error(
http.BAD_REQUEST, 'unable to validate enclosed'
' transaction {0}'.format(data))
if pending_block_txns is not None:
pending_txn_ids = pending_block_txns + pending_txn_ids
# apply any local pending transactions
for txn_id in pending_txn_ids:
pend_txn = self.Ledger.TransactionStore[txn_id]
my_store = temp_store_map.get_transaction_store(
pend_txn.TransactionTypeName)
if pend_txn and pend_txn.is_valid(my_store):
my_pend_txn = copy.copy(pend_txn)
my_pend_txn.apply(my_store)
# apply any enqueued messages
while len(my_queue) > 0:
qmsg = my_queue.pop()
if qmsg and \
qmsg.MessageType in self.Ledger.MessageHandlerMap:
if (hasattr(qmsg, 'Transaction')
and qmsg.Transaction is not None):
my_store = temp_store_map.get_transaction_store(
qmsg.Transaction.TransactionTypeName)
if qmsg.Transaction.is_valid(my_store):
myqtxn = copy.copy(qmsg.Transaction)
myqtxn.apply(my_store)
# determine validity of the POSTed transaction against our
# new temporary state
my_store = temp_store_map.get_transaction_store(
mytxn.TransactionTypeName)
try:
mytxn.check_valid(my_store)
except InvalidTransactionError as e:
LOGGER.info(
'submitted transaction fails transaction '
'family validation check: %s; %s', request.path,
mymsg.dump())
raise Error(
http.BAD_REQUEST,
"enclosed transaction failed transaction "
"family validation check: {}".format(str(e)))
except:
LOGGER.info(
'submitted transaction is '
'not valid %s; %s; %s', request.path, mymsg.dump(),
traceback.format_exc(20))
raise Error(
http.BAD_REQUEST, "enclosed transaction is not"
" valid {}".format(data))
LOGGER.info('transaction %s is valid',
msg.Transaction.Identifier)
# and finally execute the associated method
# and send back the results
self.Ledger.handle_message(msg)
return msg
def getLoginResource(self):
raise Error(501, b"not implemented")
def __init__(self, url):
Error.__init__(self, 302, "redirect")
self.url = url
def trapNotFound(failure):
err = failure.trap(*self.errorMap.keys())
status, message = self.errorMap[err]
raise Error(status, message)
def fail_unauthorized(url, timeout=None, headers=None):
if headers != expected_headers:
raise Error(code=403)
def trapNotFound(failure):
failure.trap(StanzaError)
if not failure.value.condition == 'item-not-found':
raise failure
raise Error(http.NOT_FOUND, "Node not found")
def _missing_parameter(self, parameter):
raise Error(400, "Missing required parameter: %s" % parameter)
def _handletxnrequest(self, pathcomponents, args, testonly):
"""
Handle a transaction request. There are four types of requests:
empty path -- return a list of the committed transactions ids
txnid -- return the contents of the specified transaction
txnid and field name -- return the contents of the specified
transaction
txnid and HEAD request -- return success only if the transaction
has been committed
404 -- transaction does not exist
302 -- transaction exists but has not been committed
200 -- transaction has been committed
The request may specify additional parameters:
blockcount -- the number of blocks (newest to oldest) from which to
pull txns
Transactions are returned from oldest to newest.
"""
if len(pathcomponents) == 0:
blkcount = 0
if 'blockcount' in args:
blkcount = int(args.get('blockcount').pop(0))
txnids = []
blockids = self.Ledger.commited_block_ids(blkcount)
while blockids:
blockid = blockids.pop()
txnids.extend(self.Ledger.BlockStore[blockid].TransactionIDs)
return txnids
txnid = pathcomponents.pop(0)
if txnid not in self.Ledger.TransactionStore:
raise Error(http.NOT_FOUND,
'no such transaction {0}'.format(txnid))
txn = self.Ledger.TransactionStore[txnid]
if testonly:
if txn.Status == transaction.Status.committed:
return None
else:
raise Error(http.FOUND,
'transaction not committed {0}'.format(txnid))
tinfo = txn.dump()
tinfo['Identifier'] = txnid
tinfo['Status'] = txn.Status
if txn.Status == transaction.Status.committed:
tinfo['InBlock'] = txn.InBlock
if not pathcomponents:
return tinfo
field = pathcomponents.pop(0)
if field not in tinfo:
raise Error(http.BAD_REQUEST,
'unknown transaction field {0}'.format(field))
return tinfo[field]
def _get_active_view():
view = simple.GetActiveView()
if not view:
raise Error(404, "No view provided to WebGL resource")
return view
def parse(self, val):
# type: (bytes) -> Tuple[int, ...]
try:
return tuple(map(int, val.split(b".")))
except (TypeError, ValueError):
raise Error(BAD_REQUEST, b"Invalid version literal")
def validate_options(self, api_params):
if not api_params.get("urls"):
raise Error('400', "A list of urls to crawl must be specified.")
def __init__(self, msg):
Error.__init__(self, 403, msg)
def test_error_400(self, log_err_mock):
exc = Error('400', 'blah_400')
result = self.resource.handle_render_errors(self.request, exc)
self.assertEqual(self.request.code, 400)
self.assertFalse(log_err_mock.called)
self.assertEqual(result['message'], exc.message)
