def start(self):
"""
Translate self.get_query() into the corresponding SQL command.
The PostgreSQL's get_sql method is overloaded in order to redirect
handle queries related to pseudo tables (traceroute, bgp, ...) and craft a
customized query.
"""
query = self.get_query()
table_name = query.get_from()
if table_name in self.METHOD_MAP.keys():
if self.METHOD_MAP[table_name]:
# See manifold/gateways/tdmi/methods/*
params = None
instance = self.METHOD_MAP[table_name](query, db = self)
sql = instance.get_sql()
rows = self.selectall(sql, params)
# Does this object tweak the python dictionnary returned by selectall?
if instance.need_repack and instance.repack:
if instance.need_repack(query):
rows = [instance.repack(query, row) for row in rows]
else:
# Dummy object, like hops (hops is declared in tdmi.h) but
# do not corresponds to any
rows = list()
map(self.send, Records(rows))
self.send(LastRecord())
else:
# Update FROM clause according to postgresql aliases
self.query.object = self.get_pgsql_name(table_name)
return super(TDMIGateway, self).start()
def _on_right_done(self):
# Send records in left_results that have not been joined...
for left_record_list in self.left_map.values():
for left_record in left_record_list:
self.send(left_record)
# ... and terminates
self.send(LastRecord())
def callback_error(self, error):
"""
(Internal usage) See ManifoldGateway::receive_impl.
Args:
packet: A QUERY Packet.
error: The corresponding error message.
"""
Log.error("Error during Manifold call: %r" % error)
self.send(LastRecord())
def start(self):
"""
Propagates a START message through the FromTable Node.
"""
for record in self.records:
if not isinstance(record, Record):
record = {self.key: record}
self.send(record)
self.send(LastRecord())
def set_records(self, records):
if not isinstance(records, list):
records = [records]
self._pending_records = list()
self._records = records
self._updated = time.time()
for operator in self._operators:
for record in records:
operator.child_callback(record)
operator.child_callback(LastRecord())
def start(self):
"""
Fetch records stored in the postgresql database according to self.query
"""
# Results of the query (TODO)
rows = []
# Sending rows to parent processing node in the AST
map(self.send, Records(rows))
self.send(LastRecord())
def start(self):
# Will receive a start when executed == when the source is ready to receive records
# That's when we make the difference between different modes (cached, buffered, multicast)
# TMP CACHE DEBUG
#import pdb
#pdb.set_trace()
if self._cache_entry.has_query_in_progress():
print "query in progress"
if self._cache_entry.has_pending_records():
print "has pending records"
# Let's first return all pending records, then wait for
# set_records to receive further records
print "pending records = ", self._cache_entry._pending_records
for record in self._cache_entry._pending_records:
record.set_annotation('cache', 'buffered multicast')
self.send(record)
else:
# Query did not started, just return new incoming records
# Nothing to do, let's wait for set_records
pass
# Inform the cache entry we are interested in its records
self._cache_entry.add_operator(self)
else:
# If we reached here, we _have_ records in cache
# otherwise, we would have a cache entry and no query in progress,
# and no query ever done == INCONSISTENT
for record in self._cache_entry.get_records():
record.set_annotation('cache', 'cache')
self.send(record)
record = LastRecord()
record.set_annotation('cache', 'cache')
self.send(record)
def start(self):
assert self.query, "Cannot start gateway with no query associated"
_map_action = {
'get': self.local_query_get,
'update': self.local_query_update,
'create': self.local_query_create,
'delete': self.local_query_delete
}
table = _map_action[self.query.action](self.query)
# XXX For local namespace queries, we need to keep a dict
if table:
for t in table:
#MANDO| row = row2dict(t) if self.format == 'dict' else t.get_object()
row = row2record(t) if self.format == 'record' else t
self.send(row)
self.send(LastRecord())
def start(self):
"""
Fetch records stored in the postgresql database according to self.query
"""
sql = PostgreSQLGateway.to_sql(self.query)
Log.tmp(sql)
if self.query.get_action() in [ACTION_GET, ACTION_CREATE]:
rows = self.selectall(sql)
else:
count = self.do(sql)
if self.query.get_select():
select_query = self.query.copy()
select_query.action = ACTION_GET
if self.query.get_action() == ACTION_DELETE:
select_query.filter_by(None)
select_sql = PostgreSQLGateway.to_sql(select_query)
rows = self.selectall(select_sql)
else:
rows = list()
map(self.send, Records(rows))
self.send(LastRecord())
def start(self):
assert self.query, "Query should have been associated before start"
table = self.query.object
dialect, field_names, field_types = self.get_dialect_and_field_info(
table)
key = self.get_key(table)
filename = self.config[table]['filename']
with open(filename, 'rb') as csvfile:
reader = csv.reader(csvfile, dialect=dialect)
try:
if self.has_headers[table]:
values = reader.next()
for row in reader:
row = self.convert(row, field_names, field_types)
if not row: continue
self.send(row)
self.send(LastRecord())
except csv.Error as e:
sys.exit('file %s, line %d: %s' %
(filename, reader.line_num, e))
def exception_cb(self, error):
print 'Error during Manifold call: ', error
self.callback(LastRecord())
def start(self):
results = getattr(self, "get_%s" % self.query.object)()
map(self.send, Records(results))
self.send(LastRecord())
def start(self):
query = self.query
table_name = query.get_from()
# Need to transform manifold fields into query fields, and then back in
# callback_records
## Insert a RENAME Node above this FROM Node if necessary.
#instance = self.get_object(query.get_from())
#aliases = instance.get_aliases()
#if aliases:
# Log.warning("I don't think this properly recables everything")
# try:
# Rename(self, aliases)
# except Exception, e:
# print "EEE:", e
filters = MyPLCGateway.manifold_to_myplc_filter(query.get_where())
fields = query.get_select()
fields = list(fields)
if query.get_from() in MAP_METHOD:
method = MAP_METHOD[query.get_from()]
else:
# XXX Only admin user has an account on ple-myplc platform
# XXX Check if the user has the right to do it before the admin_execute_query
# XXX Slice or Authority Credentials !!!
if query.action == 'get' and query.get_from() == 'initscript':
method = "GetSliceInitscriptCode"
fields = False
slicename = self.get_slicename(filters)
if not slicename:
self.send(LastRecord())
return
filters = slicename
if query.action == 'update' and query.get_from() == 'initscript':
# UpdateSlice (auth, slice_id_or_name, slice_fields)
method = "UpdateSlice"
slicename = self.get_slicename(filters)
if not slicename:
self.send(LastRecord())
return
filters = slicename
params = query.get_params()
print params
fields = {'initscript_code': params['initscript_code']}
# initscript_code ???
# Either in request RSpec or using UpdateSlice method
# <node ...>
# <sliver_type name="plab-vnode">
# <planetlab:initscript name="gpolab_sirius"/>
# </sliver_type>
# </node>
if query.action == 'delete' and query.get_from() == 'initscript':
#method = "DeleteSliceTag"
# GetSliceTags (auth, slice_tag_filter, return_fields)
# slicename from slice_hrn
slicename = self.get_slicename(filters)
if not slicename:
self.send(LastRecord())
return
# slice_tag_id for initscript_code tag
filters = {'name': slicename}
fields = ['slice_tag_id', 'tagname']
plc_api = xmlrpclib.ServerProxy(API_URL, allow_none=True)
result = plc_api.GetSliceTags(self._get_auth(), filters,
fields)
# XXX How to get the result from callRemote and use it in the rest of the code?
for r in result:
if r['tagname'] == 'initscript_code':
method = 'DeleteSliceTag'
fields = False
filters = r['slice_tag_id']
if fields:
d = self._proxy.callRemote(method, self._get_auth(), filters,
fields)
else:
d = self._proxy.callRemote(method, self._get_auth(), filters)
d.addCallback(self.callback_records)
d.addErrback(self.callback_error)
def all_done(self):
"""
\brief Called when all children of the current subquery are done: we
process results stored in the parent.
"""
try:
for parent_record in self.parent_output:
# Dispatching child results
for i, child in enumerate(self.children):
relation = self.relations[i]
predicate = relation.get_predicate()
key, op, value = predicate.get_tuple()
if op == eq:
# 1..N
# Example: parent has slice_hrn, resource has a reference to slice
# PARENT CHILD
# Predicate: (slice_hrn,) == slice
# Collect in parent all child such as they have a pointer to the parent
record = Record.get_value(parent_record, key)
if not record:
record = []
if not isinstance(record,
(list, tuple, set, frozenset)):
record = [record]
if relation.get_type() in [
Relation.types.LINK_1N,
Relation.types.LINK_1N_BACKWARDS
]:
# we have a list of elements
# element = id or dict : cle simple
# = tuple or dict : cle multiple
ids = [
SubQuery.get_element_key(r, value)
for r in record
]
else:
ids = [SubQuery.get_element_key(record, value)]
if len(ids) == 1:
id, = ids
filter = Filter().filter_by(
Predicate(value, eq, id))
else:
filter = Filter().filter_by(
Predicate(value, included, ids))
#if isinstance(key, StringTypes):
# # simple key
# ids = [o[key]] if key in o else []
# #print "IDS=", ids
# #if ids and isinstance(ids[0], dict):
# # ids = map(lambda x: x[value], ids)
# # XXX we might have equality instead of IN in case of a single ID
# print "VALUE", value, "INCLUDED ids=", ids
# filter = Filter().filter_by(Predicate(value, included, ids))
#else:
# # Composite key, o[value] is a dictionary
# for field in value:
# filter = filter.filter_by(Predicate(field, included, o[value][field])) # o[value] might be multiple
parent_record[relation.get_relation_name()] = []
for child_record in self.child_results[i]:
if filter.match(child_record):
parent_record[relation.get_relation_name(
)].append(child_record)
elif op == contains:
# 1..N
# Example: parent 'slice' has a list of 'user' keys == user_hrn
# PARENT CHILD
# Predicate: user contains (user_hrn, )
# first, replace records by dictionaries. This only works for non-composite keys
if parent_record[child.query.object]:
record = parent_record[child.query.object][0]
if not isinstance(record, dict):
parent_record[child.query.object] = [{
value:
record
} for record in parent_record[
child.query.object]]
if isinstance(value, StringTypes):
for record in parent_record[child.query.object]:
# Find the corresponding record in child_results and update the one in the parent with it
for k, v in record.items():
filter = Filter().filter_by(
Predicate(value, eq, record[value]))
for r in self.child_results[i]:
if filter.match(r):
record.update(r)
else:
for record in parent_record[child.query.object]:
# Find the corresponding record in child_results and update the one in the parent with it
for k, v in record.items():
filter = Filter()
for field in value:
filter = filter.filter_by(
Predicate(field, eq,
record[field]))
for r in self.child_results[i]:
if filter.match(r):
record.update(r)
else:
raise Exception, "No link between parent and child queries"
self.send(parent_record)
self.send(LastRecord())
except Exception, e:
print "EEE", e
traceback.print_exc()
def run_children(self):
"""
Run children queries (subqueries) assuming the parent query (main query)
has successfully ended.
"""
if not self.parent_output:
# No parent record, this is useless to run children queries.
self.send(LastRecord())
return
#print "=" * 80
#print self.parent_output
#print "=" * 80
if not self.children:
# The top operator has build a SubQuery node without child queries,
# so this SubQuery operator is useless and should be replaced by
# its main query.
Log.warning(
"SubQuery::run_children: no child node. The query plan could be improved"
)
self.send(LastRecord())
return
# Inspect the first parent record to deduce which fields have already
# been fetched
first_record = self.parent_output[0]
parent_fields = set(first_record.keys())
# Optimize child queries according to the fields already retrieved thanks
# to the parent query.
useless_children = set()
for i, child in enumerate(self.children[:]):
# Test whether the current child provides relevant fields (e.g.
# fields not already fetched in the parent record). If so, reduce
# the set of queried field in order to only retrieve relevant fields.
child_fields = child.get_query().get_select()
relation = self.relations[i]
relation_name = relation.get_relation_name()
already_fetched_fields = set()
if relation_name in parent_fields:
if relation.get_type() in [
Relation.types.LINK_1N,
Relation.types.LINK_1N_BACKWARDS
]:
if relation_name in first_record and first_record[
relation_name] and len(
first_record[relation_name]) > 0:
if isinstance(first_record[relation_name][0], Record):
already_fetched_fields = set(
first_record[relation_name][0].keys())
else:
# If we do not have a dict, we have only keys, so it's like we had no field of importance...
already_fetched_fields = set()
else:
already_fetched_fields = set()
else:
if relation_name in first_record and first_record[
relation_name] and len(
first_record[relation_name]) > 0:
already_fetched_fields = set(
first_record[relation_name].keys())
else:
already_fetched_fields = set()
# XXX routerv2: we need to keep key used for subquery
key_field = relation.get_predicate().get_value()
relevant_fields = child_fields - already_fetched_fields
if not relevant_fields:
tmp = list()
for pr in self.parent_output:
tmp.extend(pr[relation_name])
self.child_results[i] = tmp # Records
useless_children.add(i)
continue
else:
relevant_fields |= frozenset([key_field]) # necessary ?
if child_fields != relevant_fields:
# XXX This seems to remove the key used for joining
self.children[i] = child.optimize_projection(
relevant_fields)
# If every children are useless, this means that we already have full records
# thanks to the parent query, so we simply forward those records.
if len(self.children) == len(useless_children):
map(self.send, self.parent_output)
self.send(LastRecord())
return
# Loop through children and inject the appropriate parent results
for i, child in enumerate(self.children):
if i in useless_children: continue
# We have two cases:
# (1) either the parent query has subquery fields (a list of child
# ids + eventually some additional information)
# (2) either the child has a backreference to the parent
# ... eventually a partial reference in case of a 1..N relationship
#
# In all cases, we will collect all identifiers to proceed to a
# single child query for efficiency purposes, unless it's not
# possible (?).
#
# We have several parent records stored in self.parent_output
#
# /!\ Can we have a mix of (1) and (2) ? For now, let's suppose NO.
# * We could expect key information to be stored in the DBGraph
# The operation to be performed is understood only be looking at the predicate
relation = self.relations[i]
predicate = relation.get_predicate()
key, op, value = predicate.get_tuple()
if op == eq:
# 1..N
# Example: parent has slice_hrn, resource has a reference to slice
if relation.get_type() == Relation.types.LINK_1N_BACKWARDS:
parent_ids = [record[key] for record in self.parent_output]
if len(parent_ids) == 1:
parent_id, = parent_ids
filter_pred = Predicate(value, eq, parent_id)
else:
filter_pred = Predicate(value, included, parent_ids)
else:
parent_ids = []
for parent_record in self.parent_output:
record = Record.get_value(parent_record, key)
if not record:
record = []
# XXX Nothing to do for the case where the list of keys in the parent is empty
if relation.get_type() in [
Relation.types.LINK_1N,
Relation.types.LINK_1N_BACKWARDS
]:
# we have a list of elements
# element = id or dict : cle simple
# = tuple or dict : cle multiple
parent_ids.extend([
self.get_element_key(r, value) for r in record
])
else:
parent_ids.append(
self.get_element_key(record, value))
#if isinstance(key, tuple):
# parent_ids = [x for record in self.parent_output if key in record for x in record[key]]
#else:
# ##### record[key] = text, dict, or list of (text, dict)
# parent_ids = [record[key] for record in self.parent_output if key in record]
#
#if parent_ids and isinstance(parent_ids[0], dict):
# parent_ids = map(lambda x: x[value], parent_ids)
if len(parent_ids) == 1:
parent_id, = parent_ids
filter_pred = Predicate(value, eq, parent_id)
else:
filter_pred = Predicate(value, included, parent_ids)
# Injecting predicate
old_child_callback = child.get_callback()
self.children[i] = child.optimize_selection(
Filter().filter_by(filter_pred))
self.children[i].set_callback(old_child_callback)
elif op == contains:
# 1..N
# Example: parent 'slice' has a list of 'user' keys == user_hrn
for slice in self.parent_output:
if not child.get_query().object in slice: continue
users = slice[key]
# users est soit une liste d'id, soit une liste de records
user_data = []
for user in users:
if isinstance(user, dict):
user_data.append(user)
else:
# have have a key
# XXX Take multiple keys into account
user_data.append({value: user})
# Let's inject user_data in the right child
child.inject(user_data, value, None)
else:
raise Exception, "No link between parent and child queries"
#print "*** before run children ***"
#self.dump()
# We make another loop since the children might have been modified in
# the previous one.
for i, child in enumerate(self.children):
if i in useless_children: continue
self.status.started(i)
for i, child in enumerate(self.children):
if i in useless_children: continue
child.start()
def all_done(self):
for record in self.key_map.values():
self.send(record)
self.send(LastRecord())
class MyPLCGateway(Gateway):
__gateway_name__ = "myplc"
#---------------------------------------------------------------------------
# Constructor
#---------------------------------------------------------------------------
def __init__(self, router, platform, query, config, user_config, user):
super(MyPLCGateway, self).__init__(router, platform, query, config,
user_config, user)
# The default xmlrpc.Proxy does not work, we need to use ReactorThread()...
# XXX Share this code among manifold
from twisted.web import xmlrpc
class Proxy(xmlrpc.Proxy):
''' See: http://twistedmatrix.com/projects/web/documentation/howto/xmlrpc.html
this is eacly like the xmlrpc.Proxy included in twisted but you can
give it a SSLContext object insted of just accepting the defaults..
'''
def setSSLClientContext(self, SSLClientContext):
self.SSLClientContext = SSLClientContext
def callRemote(self, method, *args):
def cancel(d):
factory.deferred = None
connector.disconnect()
factory = self.queryFactory(self.path, self.host, method,
self.user, self.password,
self.allowNone, args, cancel,
self.useDateTime)
#factory = xmlrpc._QueryFactory(
# self.path, self.host, method, self.user,
# self.password, self.allowNone, args)
if self.secure:
try:
self.SSLClientContext
except NameError:
print "Must Set a SSL Context"
print "use self.setSSLClientContext() first"
# Its very bad to connect to ssl without some kind of
# verfication of who your talking to
# Using the default sslcontext without verification
# Can lead to man in the middle attacks
ReactorThread().connectSSL(self.host,
self.port or 443,
factory,
self.SSLClientContext,
timeout=self.connectTimeout)
else:
ReactorThread().connectTCP(self.host,
self.port or 80,
factory,
timeout=self.connectTimeout)
return factory.deferred
self._proxy = Proxy(API_URL.encode('latin-1'), allowNone=True)
# XXX We should provide a default SSL Context...
from twisted.internet import ssl
self._proxy.setSSLClientContext(ssl.ClientContextFactory())
# ReactorThread().start_reactor()
#
# def terminate(self):
# ReactorThread().stop_reactor()
#---------------------------------------------------------------------------
# Internal methods
#---------------------------------------------------------------------------
@returns(StringTypes)
def __str__(self):
"""
Returns:
The '%s' representation of this gateway.
"""
return "<MyPLCGateway %s>" % (API_URL,
) #(self._platform_config['url'])
#---------------------------------------------------------------------------
# Methods
#---------------------------------------------------------------------------
# XXX nodes = srv.GetNodes(AUTH, {}, ['hostname', 'site_id'])
def callback_getResult(self, rows):
self.send(rows)
def callback_records(self, rows):
"""
(Internal usage) See ManifoldGateway::receive_impl.
Args:
packet: A QUERY Packet.
rows: The corresponding list of dict or Record instances.
"""
if rows is not None:
try:
iterator = iter(rows)
except TypeError, te:
print "rows = ", rows
rows = [{'initscript_code': rows}]
if isinstance(rows, basestring):
rows = [{'initscript_code': rows}]
for row in rows:
print row
self.send(Record(row))
self.send(LastRecord())
def success_cb(self, table):
print "Manifold SUCCESS", len(table)
for record in table:
self.callback(Record(record))
self.callback(LastRecord())
