def __init__(self,
store=None,
data=None,
train_index=None,
prep_index=None):
"""
**Args**
store: An instance of `store.Store` or a path. If a path
Ramp will default to an `HDFPickleStore` at that path
if PyTables is installed, a `PickleStore` otherwise.
Defaults to MemoryStore.
data: a pandas DataFrame. If all data has been precomputed this
may not be required.
train_index: a pandas Index specifying the data instances to be
used in training. Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
prep_index: a pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = store if isinstance(store,
Store) else default_store(store)
self.data = data
self.train_index = train_index if train_index is not None else self.data.index if self.data is not None else None
self.prep_index = prep_index if prep_index is not None else self.data.index if self.data is not None else None
def main():
store = MemoryStore()
load(store)
for host in ('www.google.com', 'www.amazon.com'):
print('{0}: {1}'.format(
host, store.lookup(socket.gethostbyname(host), 'ipv4').cc
))
def __init__(self,
store=None,
data=None,
train_index=None,
prep_index=None,
train_once=False):
"""
Parameters:
-----------
store: string or ramp.store.Store object, default None
An instance of `ramp.store.Store` or a path. If a path, Ramp will
default to an `HDFPickleStore` at that path if PyTables is
installed, a `PickleStore` otherwise. Defaults to MemoryStore.
data: Pandas DataFrame, default None
Dataframe of data. If all data has been precomputed this
may not be required.
train_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be used in training.
Stored results will be cached against this.If not provided, the
entire index of the 'data' parameter will be used.
prep_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached
against this. If not provided, the entire index of `data`
keyword arg will be used.
train_once: boolean
If True, train and predict indexes will not be used as part of key
hashes, meaning the values from the first run with this context
will be stored permanently.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = (store if isinstance(store, Store) else
default_store(store))
self.data = data
if train_index is not None:
self.train_index = train_index
elif self.data is not None:
self.train_index = self.data.index
else:
self.train_index = None
if prep_index is not None:
self.prep_index = prep_index
elif self.data is not None:
self.prep_index = self.data.index
else:
self.prep_index = None
self.train_once = train_once
def __init__(self, store=None, data=None, train_index=None,
prep_index=None, train_once=False):
"""
Parameters:
-----------
store: string or ramp.store.Store object, default None
An instance of `ramp.store.Store` or a path. If a path, Ramp will
default to an `HDFPickleStore` at that path if PyTables is
installed, a `PickleStore` otherwise. Defaults to MemoryStore.
data: Pandas DataFrame, default None
Dataframe of data. If all data has been precomputed this
may not be required.
train_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be used in training.
Stored results will be cached against this.If not provided, the
entire index of the 'data' parameter will be used.
prep_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached
against this. If not provided, the entire index of `data`
keyword arg will be used.
train_once: boolean
If True, train and predict indexes will not be used as part of key
hashes, meaning the values from the first run with this context
will be stored permanently.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = (store if isinstance(store, Store)
else default_store(store))
self.data = data
if train_index is not None:
self.train_index = train_index
elif self.data is not None:
self.train_index = self.data.index
else:
self.train_index = None
if prep_index is not None:
self.prep_index = prep_index
elif self.data is not None:
self.prep_index = self.data.index
else:
self.prep_index = None
self.train_once = train_once
def __init__(self, store=None, data=None, train_index=None, prep_index=None):
"""
**Args**
store: An instance of `store.Store` or a path. If a path
Ramp will default to an `HDFPickleStore` at that path
if PyTables is installed, a `PickleStore` otherwise.
Defaults to MemoryStore.
data: a pandas DataFrame. If all data has been precomputed this
may not be required.
train_index: a pandas Index specifying the data instances to be
used in training. Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
prep_index: a pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = store if isinstance(store, Store) else default_store(store)
self.data = data
self.train_index = train_index if train_index is not None else self.data.index if self.data is not None else None
self.prep_index = prep_index if prep_index is not None else self.data.index if self.data is not None else None
import urltools
from flask import Flask, request
from flask.helpers import make_response
from flask.json import jsonify
from store import MemoryStore
app = Flask(__name__)
app.config['JSON_AS_ASCII'] = False # encode utf-8
store = MemoryStore()
@app.route('/stories/<page_id>')
def show_tags(page_id):
page = store.get_page(int(page_id))
if page:
return jsonify(page.to_json())
return make_response('Not Found', 404)
@app.route('/stories', methods=['POST'])
def register_url():
url_param = request.args.get('url')
if not url_param:
return make_response("url param is missing", 400) # bad request
# TODO: advanced input validation
# https://validators.readthedocs.io/en/latest/#module-validators.url
# https://github.com/django/django/blob/master/django/core/validators.py#L74
trace_activity = Activity(
label="TurnError",
name="on_turn_error Trace",
timestamp=datetime.utcnow(),
type=ActivityTypes.trace,
value=f"{error}",
value_type="https://www.botframework.com/schemas/error",
)
# Send a trace activity, which will be displayed in Bot Framework Emulator
await context.send_activity(trace_activity)
ADAPTER.on_turn_error = on_error
# Create the Bot
STORAGE = MemoryStore()
# Use BlobStore to test with Azure Blob storage.
# STORAGE = BlobStore(CONFIG.BLOB_ACCOUNT_NAME, CONFIG.BLOB_KEY, CONFIG.BLOB_CONTAINER)
DIALOG = RootDialog()
BOT = ScaleoutBot(STORAGE, DIALOG)
# Listen for incoming requests on /api/messages
async def messages(req: Request) -> Response:
# Main bot message handler.
if "application/json" in req.headers["Content-Type"]:
body = await req.json()
else:
return Response(status=415)
activity = Activity().deserialize(body)
class DataContext(object):
"""
The DataContext is the data storage environment for the Ramp analysis.
For a given store and pandas index value, Ramp will consider the data
immutable, and will not check for changes in the data.
"""
def __init__(self,
store=None,
data=None,
train_index=None,
prep_index=None,
train_once=False):
"""
Parameters:
-----------
store: string or ramp.store.Store object, default None
An instance of `ramp.store.Store` or a path. If a path, Ramp will
default to an `HDFPickleStore` at that path if PyTables is
installed, a `PickleStore` otherwise. Defaults to MemoryStore.
data: Pandas DataFrame, default None
Dataframe of data. If all data has been precomputed this
may not be required.
train_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be used in training.
Stored results will be cached against this.If not provided, the
entire index of the 'data' parameter will be used.
prep_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached
against this. If not provided, the entire index of `data`
keyword arg will be used.
train_once: boolean
If True, train and predict indexes will not be used as part of key
hashes, meaning the values from the first run with this context
will be stored permanently.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = (store if isinstance(store, Store) else
default_store(store))
self.data = data
if train_index is not None:
self.train_index = train_index
elif self.data is not None:
self.train_index = self.data.index
else:
self.train_index = None
if prep_index is not None:
self.prep_index = prep_index
elif self.data is not None:
self.prep_index = self.data.index
else:
self.prep_index = None
self.train_once = train_once
def key_on_indices(self):
return not self.train_once
def copy(self):
"""Make a shallow copy of the DataContext"""
return copy.copy(self)
def create_key(self):
"""Create hex key using MD5 algorithm."""
if not self.key_on_indices():
return ''
return md5('%s--%s' % (get_np_hashable(
self.train_index), get_np_hashable(self.prep_index))).hexdigest()
def save_context(self, name, config=None):
"""
Saves this context (specifically it's train and prep indices)
to it's store with the given name, along with the config, if provided.
"""
ctx = {
'train_index': self.train_index,
'prep_index': self.prep_index,
'config': config
}
self.store.save('context__%s' % name, ctx)
def load_context(self, name):
"""
Loads a previously saved context with given name,
assigning the stored training and prep indices
and returning any stored config.
"""
ctx = self.store.load('context__%s' % name)
self.train_index = ctx['train_index']
self.prep_index = ctx['prep_index']
return ctx['config']
class DataContext(object):
"""
All Ramp analyses require a DataContext.
A DataContext represents the environment of the
analysis. Most importantly this means for a given store
and pandas index value, Ramp will consider the data immutable --
it will not check the data again to see if it has changed.
"""
def __init__(self, store=None, data=None, train_index=None, prep_index=None):
"""
**Args**
store: An instance of `store.Store` or a path. If a path
Ramp will default to an `HDFPickleStore` at that path
if PyTables is installed, a `PickleStore` otherwise.
Defaults to MemoryStore.
data: a pandas DataFrame. If all data has been precomputed this
may not be required.
train_index: a pandas Index specifying the data instances to be
used in training. Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
prep_index: a pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = store if isinstance(store, Store) else default_store(store)
self.data = data
self.train_index = train_index if train_index is not None else self.data.index if self.data is not None else None
self.prep_index = prep_index if prep_index is not None else self.data.index if self.data is not None else None
def copy(self):
return copy.copy(self)
def create_key(self):
return md5('%s--%s' % (get_np_hashable(self.train_index),
get_np_hashable(self.prep_index))).hexdigest()
def save_context(self, name, config=None):
"""
Saves this context (specifically it's train and prep indices)
to it's store with the given
name, along with the config, if provided.
"""
ctx = {'train_index':self.train_index,
'prep_index':self.prep_index,
'config':config}
self.store.save('context__%s' % name, ctx)
def load_context(self, name):
"""
Loads a previously saved context with given name,
assigning the stored training and prep indices
and returning any stored config.
"""
ctx = self.store.load('context__%s' % name)
self.train_index = ctx['train_index']
self.prep_index = ctx['prep_index']
return ctx['config']
class DataContext(object):
"""
The DataContext is the data storage environment for the Ramp analysis.
For a given store and pandas index value, Ramp will consider the data
immutable, and will not check for changes in the data.
"""
def __init__(self, store=None, data=None, train_index=None,
prep_index=None):
"""
Parameters:
-----------
store: string or ramp.store.Store object, default None
An instance of `ramp.store.Store` or a path. If a path, Ramp will
default to an `HDFPickleStore` at that path if PyTables is
installed, a `PickleStore` otherwise. Defaults to MemoryStore.
data: Pandas DataFrame, default None
Dataframe of data. If all data has been precomputed this
may not be required.
train_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be used in training.
Stored results will be cached against this.If not provided, the
entire index of the 'data' parameter will be used.
prep_index: Pandas DataFrame Index, default None
Pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached
against this. If not provided, the entire index of `data`
keyword arg will be used.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = (store if isinstance(store, Store)
else default_store(store))
self.data = data
if train_index is not None:
self.train_index = train_index
elif self.data is not None:
self.train_index = self.data.index
else:
self.train_index = None
if prep_index is not None:
self.prep_index = prep_index
elif self.data is not None:
self.prep_index = self.data.index
else:
self.prep_index = None
def copy(self):
"""Make a shallow copy of the DataContext"""
return copy.copy(self)
def create_key(self):
"""Create hex key using MD5 algorithm."""
return md5('%s--%s' % (get_np_hashable(self.train_index),
get_np_hashable(self.prep_index))).hexdigest()
def save_context(self, name, config=None):
"""
Saves this context (specifically it's train and prep indices)
to it's store with the given name, along with the config, if provided.
"""
ctx = {'train_index': self.train_index,
'prep_index': self.prep_index,
'config': config}
self.store.save('context__%s' % name, ctx)
def load_context(self, name):
"""
Loads a previously saved context with given name,
assigning the stored training and prep indices
and returning any stored config.
"""
ctx = self.store.load('context__%s' % name)
self.train_index = ctx['train_index']
self.prep_index = ctx['prep_index']
return ctx['config']
class DataContext(object):
"""
All Ramp analyses require a DataContext.
A DataContext represents the environment of the
analysis. Most importantly this means for a given store
and pandas index value, Ramp will consider the data immutable --
it will not check the data again to see if it has changed.
"""
def __init__(self,
store=None,
data=None,
train_index=None,
prep_index=None):
"""
**Args**
store: An instance of `store.Store` or a path. If a path
Ramp will default to an `HDFPickleStore` at that path
if PyTables is installed, a `PickleStore` otherwise.
Defaults to MemoryStore.
data: a pandas DataFrame. If all data has been precomputed this
may not be required.
train_index: a pandas Index specifying the data instances to be
used in training. Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
prep_index: a pandas Index specifying the data instances to be
used in prepping ("x" values). Stored results will be cached against this.
If not provided, the entire index of `data` will be used.
"""
if store is None:
self.store = MemoryStore()
else:
self.store = store if isinstance(store,
Store) else default_store(store)
self.data = data
self.train_index = train_index if train_index is not None else self.data.index if self.data is not None else None
self.prep_index = prep_index if prep_index is not None else self.data.index if self.data is not None else None
def copy(self):
return copy.copy(self)
def create_key(self):
return md5('%s--%s' % (get_np_hashable(
self.train_index), get_np_hashable(self.prep_index))).hexdigest()
def save_context(self, name, config=None):
"""
Saves this context (specifically it's train and prep indices)
to it's store with the given
name, along with the config, if provided.
"""
ctx = {
'train_index': self.train_index,
'prep_index': self.prep_index,
'config': config
}
self.store.save('context__%s' % name, ctx)
def load_context(self, name):
"""
Loads a previously saved context with given name,
assigning the stored training and prep indices
and returning any stored config.
"""
ctx = self.store.load('context__%s' % name)
self.train_index = ctx['train_index']
self.prep_index = ctx['prep_index']
return ctx['config']
