def options_index(entity_identifier: str,
feature_name: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
if request.method == constants.HTTP_GET:
with metadata_context.get_session() as session:
query = session.query(Option).filter(
Option.entity_identifier == entity_identifier,
Option.feature_name == feature_name)
options = query.all()
return jsonify([json(option) for option in options])
request_option = request.get_json()
option_value = (request_option['value_text'] if 'value_text'
in request_option else request_option['value'])
with metadata_context.get_session() as session:
new_option = Option(
name=request_option['name'],
entity_identifier=request_option['entity_identifier'],
feature_name=request_option['feature_name'],
option_type=request_option['option_type'],
value_text=option_value,
value_number=request_option['value_number'])
session.add(new_option)
return jsonify(json(new_option)), 201
def entities_index(metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
if request.method == constants.HTTP_GET:
with metadata_context.get_session() as session:
entities = session.query(Entity).all()
return jsonify([json(entity) for entity in entities])
request_entity = request.get_json()
with metadata_context.get_session() as session:
new_entity = Entity(identifier=request_entity['identifier'],
name=request_entity['name'],
category=request_entity['category'])
session.add(new_entity)
return jsonify(json(new_entity)), 201
def entity_feature_index(entity_identifier: str,
feature_name: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
try:
with metadata_context.get_session() as session:
entity = session.query(Entity).filter(
Entity.identifier == entity_identifier).one()
if request.method == constants.HTTP_DELETE:
feature = session.query(Feature).filter(
Feature.name == feature_name).one()
entity.features.remove(feature)
session.add(entity)
return '', 204
for feature in entity.features:
if feature.name == feature_name:
return jsonify(json(feature))
response = make_response('Cannot find feature requested', 404)
return response
except NoResultFound:
response = make_response('Cannot find the entity or feature requested',
404)
return response
def feature_index(feature_name: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
"""Handler for individual feature level URI of the features endpoint.
Supports GET, PATCH, and DELETE methods for interacting."""
try:
with metadata_context.get_session() as session:
feature = session.query(Feature).filter(
Feature.name == feature_name).one()
if request.method == constants.HTTP_GET:
return jsonify(json(feature))
if request.method == constants.HTTP_DELETE:
session.delete(feature)
return '', 204
request_feature = request.get_json()
if request_feature['name'] != feature.name:
return make_response(
'Error: Request body does not match the feature referenced',
400)
handler = session.query(Handler).filter(
Handler.id == request_feature['handler']['id']).one()
feature.description = request_feature['description']
feature.uri = request_feature['uri']
feature.handler_metadata = handler
session.add(feature)
return jsonify(json(feature))
except NoResultFound:
response = make_response('Cannot find feature requested', 404)
return response
def get_entity(entity_identifier: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
try:
with metadata_context.get_session() as session:
entity = session.query(Entity).filter(
Entity.identifier == entity_identifier).one()
if request.method == constants.HTTP_GET:
return jsonify(json(entity))
if request.method == constants.HTTP_DELETE:
session.delete(entity)
return '', 204
request_entity = request.get_json()
if request_entity['identifier'] != entity.identifier:
return make_response(
'Error: Request body does not match the entity referenced',
400)
entity.identifier = request_entity['identifier']
entity.name = request_entity['name']
entity.category = request_entity['category']
session.add(entity)
return jsonify(json(entity))
except NoResultFound:
response = make_response('Cannot find the entity requested', 404)
return response
def configuration_index(configuration_name: str,
metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]):
"""
Provides a read-write endpoint for working with a single configuration option.
"""
try:
with metadata_context.get_session() as session:
configuration = session.query(Configuration).filter(
Configuration.name == configuration_name).one()
if request.method == constants.HTTP_GET:
return jsonify(json(configuration))
# Patch logic
request_configuration = request.get_json()
if request_configuration['name'] != configuration.name:
return make_response('Error: Request body does not match the configuration referenced',
400)
configuration.type = request_configuration['type']
configuration.value_text = request_configuration['value_text']
configuration.value_number = request_configuration['value_number']
session.add(configuration)
return jsonify(json(configuration))
except NoResultFound:
response = make_response(
'Cannot find configuration requested', 404)
return response
def index(metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
if request.method == constants.HTTP_GET:
with metadata_context.get_session() as session:
options = session.query(Option).all()
return jsonify([json(option) for option in options])
with metadata_context.get_session() as session:
request_option = request.get_json()
new_option = Option(name=request_option['name'],
entity_id=request_option['entity_id'],
feature_name=request_option['feature_name'],
option_type=request_option['option_type'],
value_text=request_option['value_text'],
value_number=request_option['value_number'])
session.add(new_option)
return jsonify(json(new_option)), 201
def option_index(option_name: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
with metadata_context.get_session() as session:
options = session.query(Option).filter(
Option.name == option_name).all()
if options:
return jsonify([json(option) for option in options])
response = make_response('Cannot find option requested', 404)
return response
def index(metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
"""Handler for base level URI for the features endpoint.
Supports GET and POST methods for interacting."""
if request.method == constants.HTTP_GET:
with metadata_context.get_session() as session:
features = session.query(Feature).all()
return jsonify([json(feature) for feature in features])
with metadata_context.get_session() as session:
request_feature = request.get_json()
handler = session.query(Handler).filter(
Handler.id == request_feature['handler']['id']).one()
new_feature = Feature(name=request_feature['name'],
uri=request_feature['uri'],
description=request_feature['description'],
handler_metadata=handler)
session.add(new_feature)
return jsonify(json(new_feature)), 201
def index(metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]):
if request.method == constants.HTTP_GET:
logging.info("Fetching archive metadata from sDAS database")
with metadata_context.get_session() as session:
archives = session.query(Archive).all()
return jsonify([json(archive) for archive in archives])
if len(request.files) > 0:
# Handling file upload in IPFS
logging.info("Handling archive file upload request")
file: FileStorage = request.files['dataset']
connection_string = metadata_context.get_configuration(
'archiveIpfsConnectionString')
if connection_string is None:
logging.error(
"No connection string was found, unable to upload to IPFS!")
abort(500)
symbols = request.form['company_symbols']
logging.info("Uploading archive")
archive = upload_archive(file.stream.read(),
connection_string.value,
metadata_context,
company_symbols=symbols)
else:
# Support registering existing dataset metadata within sDAS
logging.info("Registering archive in sDAS metadata database")
archive = Archive.from_meta(request.get_json())
with metadata_context.get_session() as session:
archive_count = session.query(Archive).filter(
Archive.address == archive.address).count()
if archive_count > 0:
message = f'Found possible dataset duplicate at address "{archive.address}".'
logging.warning(message)
response = make_response(message, 400)
return response
session.add(archive)
return jsonify(json(archive))
def archive_index(archive_address: str,
metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]):
try:
with metadata_context.get_session() as session:
archive = session.query(Archive).filter(
Archive.address == archive_address).one()
if archive is not None:
return jsonify(json(archive))
except NoResultFound:
message = f'Cannot find requested archive with address "{archive_address}"'
logging.error(message)
response = make_response(message, 404)
return response
def get_smtp_config(
metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]) -> dict:
"""
Retrieves the SMTP configuration from the metadata store.
Returns
-------
dict:
Dictionary object containing retrieved SMTP settings.
Raises
------
TypeError:
Thrown if the configuration object cannot be mapped to a known SMTP configuration.
"""
logging.debug("Fetching all SMTP configuration from sDAS database")
session = metadata_context.get_session()
smtp_configs: List[Configuration] = session.query(Configuration).filter(
Configuration.name.ilike('smtp%')).all()
config = {
'fromAddress': '',
'toAddress': '',
'hostname': '',
'port': 465,
'username': '',
'password': ''
}
for smtp_config in smtp_configs:
# Here's how we hack a switch statement in Python :)
if smtp_config.name == 'smtpFromAddress':
config['fromAddress'] = smtp_config.value
elif smtp_config.name == 'smtpToAddress':
config['toAddress'] = smtp_config.value
elif smtp_config.name == 'smtpHostname':
config['hostname'] = smtp_config.value
elif smtp_config.name == 'smtpPort':
config['port'] = int(smtp_config.value)
elif smtp_config.name == 'smtpUsername':
config['username'] = smtp_config.value
elif smtp_config.name == 'smtpPassword':
config['password'] = smtp_config.value
else:
raise TypeError(
f'Unknown SMTP configuration value encountered ("{smtp_config.name}")')
return config
def feature_index(company_symbol: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
with metadata_context.get_session() as session:
query = session.query(
Statistics.company_symbol, Statistics.feature_name,
func.sum(Statistics.row_count).label("row_count")).filter(
Statistics.company_symbol == company_symbol).group_by(
Statistics.company_symbol, Statistics.feature_name)
statistics = query.all()
return jsonify([{
"company_symbol": entry.company_symbol,
"feature_name": entry.feature_name,
"total_row_count": int(entry.row_count)
} for entry in statistics])
def archive_data(archive_address: str,
metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]):
with metadata_context.get_session() as session:
connection_string = session.query(Configuration).filter(
Configuration.name == 'archiveIpfsConnectionString').one_or_none()
if connection_string is None:
logging.error(
"No connection string was found, unable to upload to IPFS!")
abort(500)
try:
archive = session.query(Archive).filter(
Archive.address == archive_address).one()
return jsonify(download_archive(archive.address, connection_string.value))
except NoResultFound:
message = f'Cannot find requested archive with address "{archive_address}"'
logging.error(message)
response = make_response(message, 404)
return response
def company_index(company_symbol: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
with metadata_context.get_session() as session:
query = session.query(
Statistics.company_symbol,
func.sum(Statistics.row_count).label("row_count")).filter(
Statistics.company_symbol == company_symbol).group_by(
Statistics.company_symbol)
try:
statistics = query.one()
except NoResultFound:
response = make_response('Cannot find company requested', 404)
return response
return jsonify({
"company_symbol": statistics.company_symbol,
"total_row_count": int(statistics.row_count)
})
def entity_features_index(entity_identifier: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
try:
with metadata_context.get_session() as session:
entity = session.query(Entity).filter(
Entity.identifier == entity_identifier).one()
if request.method == constants.HTTP_GET:
return jsonify([json(feature) for feature in entity.features])
request_map = request.get_json()
feature = session.query(Feature).filter(
Feature.name == request_map['name']).one()
entity.features.append(feature)
session.add(entity)
return jsonify([json(feature) for feature in entity.features]), 201
except NoResultFound:
response = make_response('Cannot find the entity or feature requested',
404)
return response
def option_index(entity_identifier: str,
feature_name: str,
option_name: str,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
try:
with metadata_context.get_session() as session:
query = session.query(Option).filter(
Option.entity_identifier == entity_identifier,
Option.feature_name == feature_name,
Option.name == option_name)
option = query.one()
if request.method == constants.HTTP_GET:
return jsonify(json(option))
if request.method == constants.HTTP_DELETE:
session.delete(option)
return '', 204
request_option = request.get_json()
if request_option['name'] != option.name:
return make_response(
'Error: Request body does not match the option referenced',
400)
option_value = (request_option['value_text'] if 'value_text'
in request_option else request_option['value'])
entity_identifier = request_option['entity_identifier']
feature_name = request_option['feature_name']
option.option_type = request_option['option_type']
option.value_text = option_value
option.value_number = request_option['value_number']
session.add(option)
return jsonify(json(option))
except NoResultFound:
response = make_response('Cannot find option requested', 404)
return response
def register_signals(cls, metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]):
"""Registers a new event handler with the signalling factory."""
logging.debug('Registering signals...')
with metadata_context.get_session() as session:
signals: List[EventHandler] = session.query(EventHandler).all()
for signal in signals:
if not signal.is_enabled:
continue
logging.debug('Registering signal: %s', signal.name)
try:
cls.__map_signal(signal)
except KeyError as exc:
logging.warning(
"Unable to register handler '%s': %s", signal.name, exc)
continue
except ImportError as exc:
logging.warning(exc)
continue
logging.info('Registered signal: %s', signal.name)
cls.signals.append(signal)
logging.debug('Signals registered!')
def handlers_index(metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]):
"""Retrieves all handlers from data store and returns a JSON array response object."""
with metadata_context.get_session() as session:
handlers = session.query(Handler).all()
return jsonify([json(handler) for handler in handlers])
def get_configuration(
metadata_context: BaseContext = Provide[ApplicationContainer.context_factory]):
with metadata_context.get_session() as session:
configurations = session.query(Configuration).all()
return jsonify([json(configuration) for configuration in configurations])
def acquire(company_symbol,
metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
"""
Provides API function for dataset generation.
Parameters
----------
company_symbol: str
Symbol for the company to acquire data for.
Returns
-------
flask.Response:
HTTP response with dataset as a JSON payload or delimited file.
This is determined from whether a query parameter is provided
that specifies a file output formatter.
Raises
------
sqlalchemy.exc.OperationalError:
Thrown if there is an issue communicating with the metadata database.
"""
should_handle_events = metadata_context.get_feature_toggle(
FeatureToggles.event_handlers)
if should_handle_events:
logging.info("Signalling pre-acquisition event handlers")
SignalFactory.pre_acquisition.send(
company_symbol=company_symbol,
start_date=datetime.now().isoformat())
results = dict()
api_key = metadata_context.get_configuration('apiKey')
logging.debug("Creating IEX client with API Key: %s", api_key)
client = IexClient(api_key)
try:
if should_handle_events:
logging.info("Signalling pre-company event handlers")
SignalFactory.pre_company.send(
company_symbol=company_symbol,
start_date=datetime.now().isoformat())
with metadata_context.get_session() as session:
entity = session.query(Entity).filter(
Entity.identifier == company_symbol).one()
for feature in entity.features:
if should_handle_events:
logging.info("Signalling pre-feature event handlers")
SignalFactory.pre_feature.send(
company_symbol=company_symbol,
feature_name=feature.name,
start_date=datetime.now().isoformat())
feature_option = session.query(Option).filter(
Option.company_symbol == company_symbol,
Option.feature_name == feature.name).all()
option = [json(option) for option in feature_option]
logging.info('Retrieved mapped options: [%s]', (" ").join(
[json(option, True) for option in feature_option]))
# TODO: Determine if this could/should be moved into source-aware code
if feature.handler_metadata.name == "tech_indicators_handler" and not option:
logging.info(
'Adding missing option on technical indicator')
option.append({
"feature_name": feature.name,
"name": "range",
"value": "1m"
})
logging.info('Acquiring feature data')
data = client.get_feature_data(feature, entity, option)
if isinstance(data, list):
results[feature.name] = feature.get_values(data)
elif isinstance(data, dict):
results[feature.name] = [feature.get_value(data)]
logging.info("Acquired %d rows", len(results[feature.name]))
if should_handle_events:
logging.info("Signalling post-feature event handlers")
SignalFactory.post_feature.send(
company_symbol=company_symbol,
feature_name=feature.name,
feature_rows=len(results[feature.name]),
end_date=datetime.now().isoformat())
results = __format_output(results, 'json')
if should_handle_events:
logging.info("Signalling post-company event handlers")
count = reduce(lambda total, iter: total + len(iter),
results['values'], 0)
SignalFactory.post_company.send(
company_symbol=company_symbol,
data=results,
total_rows=count,
end_date=datetime.now().isoformat())
if 'format' in request.args:
try:
format_result = __format_output(results,
request.args['format'])
except Exception as exc:
response = make_response(str(exc), 400)
return response
if request.args['format'].lower() == 'file':
if should_handle_events:
logging.info("Signalling post-acquisition event handlers")
SignalFactory.post_acquisition.send(
company_symbol=company_symbol,
end_date=datetime.now().isoformat(),
message='Completed data acquisition!',
uri=request.path,
type='INFO')
return send_file(format_result,
as_attachment=True,
cache_timeout=0)
if should_handle_events:
logging.info("Signalling post-acquisition event handlers")
SignalFactory.post_acquisition.send(
company_symbol=company_symbol,
end_date=datetime.now().isoformat(),
message='Completed data acquisition!',
uri=request.path,
type='INFO')
return jsonify(results)
except NoResultFound:
response = make_response('Cannot find company', 404)
return response
def index(metadata_context: BaseContext = Provide[
ApplicationContainer.context_factory]):
with metadata_context.get_session() as session:
statistics = session.query(Statistics).all()
return jsonify([json(entry) for entry in statistics])