def get(self, route=None, id=None):
status = {}
try:
packet = Packet()
# set packet meta attributes
if id is not None:
packet.set_meta('id', id)
# set optional user provided routing info
if route is not None:
packet.set_meta('route', route)
packet.set_meta('requestmethod', request.method)
packet.set_meta('processingtime', unicode(
datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))
status = app_globals.dfg.send(packet)
# calls into pypes core are asynchronous so we respond as such
if status['status'] == 'success':
response.status = 202
except Exception as e:
status = 'An Undefined Error Has Occurred'
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
response.content_type = 'application/json'
response.status = 500
status['error'] = str(e)
abort(500, str(e))
# return empty body on success otherwise return status object
return None if status['status'] == 'success' else json.dumps(status)
def delete(self, route, id):
status = {}
try:
packet = Packet()
# set packet meta attributes
packet.set_meta('id', id)
packet.set_meta('requestmethod', request.method)
packet.set_meta(
'processingtime',
unicode(
datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))
status = app_globals.dfg.send(packet)
# calls into pypes core are asynchronous so we respond as such
if status['status'] == 'success':
response.status = 202
except Exception as e:
status = 'An Undefined Error Has Occurred'
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
abort(500, str(e))
# return empty body on success otherwise return status object
return None if status['status'] == 'success' else json.dumps(status)
def run(self):
while True:
# get parameters outside doc loop for better performace
try:
delete_data = True if self.get_parameter(
'delete_data') == 'True' else False
except Exception as e:
log.error('Component Failed: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
# optionally send all docs without processing
for d in self.receive_all('in'):
self.send('out', d)
self.yield_ctrl()
continue # so next time we are called we continue at the top
for doc in self.receive_all('in'):
try:
data = doc.get('data')
mime = doc.get_meta('mimetype')
if data is not None:
# this adapter only handles JSON
if not mime.startswith('application/json'):
log.warn(
'Bad Content-Type, expecting application/json')
log.warn('Ignorning doc: %s' % doc.get('id'))
continue
else:
if delete_data:
doc.delete('data')
try:
doc.merge(Packet(json.loads(data)), metas=True)
except:
log.error('Unable to convert data')
log.error(traceback.print_exc())
# restore data field if if was deleted before error
if delete_data:
doc.set('data', data)
except Exception as e:
log.error('Component Failed: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.error(traceback.print_exc())
self.send('out', doc)
self.yield_ctrl()
def create(self, route=None, id=None):
status = {}
try:
content_encoding = request.headers.get('Content-Encoding', None)
content_type = request.headers.get('Content-Type', None)
content_length = request.headers.get('Content-Length', None)
log.debug('content_encoding: %s' % content_encoding)
log.debug('content_type: %s' % content_type)
log.debug('content_length: %s' % content_length)
except Exception as e:
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
abort(500, str(e))
else:
# bad content-type
if content_type == 'application/x-www-form-urlencoded':
abort(415, "Invalid or Unspecified Content-Type")
try:
packet = Packet()
# indicates a file upload
if content_type.startswith('multipart/form-data;'):
log.debug(
'found multipart form data, attempting to find source filename'
)
part = request.POST['document']
if part.filename:
fname = unicode(part.filename.lstrip(os.sep))
packet.set_meta('filename', fname)
# update content type based on filename
content_type = unicode(mimetypes.guess_type(fname)[0])
data = part.value
else:
data = request.body
# decompress if compressed
filedata = self._decompress(content_encoding, data)
# update content length since we might be decompressed now
content_length = len(filedata)
if content_length > 0:
packet.add('data', filedata)
else:
abort(400, 'Empty Request')
# set optional user provided id
if id is not None:
log.debug('id: %s' % id)
packet.set_meta('id', id)
# set optional user provided routing info
if route is not None:
log.debug('route: %s' % route)
packet.set_meta('route', route)
# set some common meta attributes on the packet
packet.set_meta('requestmethod', request.method)
packet.set_meta('contentlength', content_length)
packet.set_meta('mimetype', content_type)
packet.set_meta(
'processingtime',
unicode(datetime.datetime.utcnow().strftime(
'%Y-%m-%dT%H:%M:%SZ')))
status = app_globals.dfg.send(packet)
# calls into pypes core are asynchronous so we respond as such
if status['status'] == 'success':
response.status = 202
except Exception as e:
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
abort(500, str(e))
# return empty body on success otherwise return status object
return None if status['status'] == 'success' else json.dumps(status)
def run(self):
# Define our components entry point
while True:
# for each document waiting on our input port
for doc in self.receive_all('in'):
try:
data = doc.get('data')
mime = doc.get_meta('mimetype')
# if there is no data, move on to the next doc
if data is None:
continue
# if this is not a xml file, move on to the next doc
if mime != 'application/xml':
continue
# solr xml starts with an add tag, if this does not start
# with an add tag, move to the next doc
xml = ET.XML(data)
if xml.tag != 'add':
log.debug('Not a Solr XML document')
continue
# create a new packet for each document in the xml
for sdoc in xml.getchildren():
if sdoc.tag != 'doc':
log.warn('Unexpected tag %s, expecting doc' % \
sdoc.tag)
continue
packet = Packet()
# each field in the xml becomes a field in the packet
for sfield in sdoc.getchildren():
if sfield.tag != 'field':
log.warn('Unexpected tag %s, expecting field' \
% sfield.tag)
continue
fieldname = sfield.get('name', None)
fieldval = sfield.text
if fieldname is not None:
if not isinstance(fieldname, unicode):
fieldname = fieldname.decode('utf-8')
if not isinstance(fieldval, unicode):
fieldval = fieldval.decode('utf-8')
packet.append(fieldname, fieldval)
# send out packet along
self.send('out', packet)
except Exception as e:
log.error('Component Failed: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
#log.error(traceback.print_exc())
# yield the CPU, allowing another component to run
self.yield_ctrl()
def create(self, route=None, id=None):
status = {}
try:
content_encoding = request.headers.get('Content-Encoding', None)
content_type = request.headers.get('Content-Type', None)
content_length = request.headers.get('Content-Length', None)
log.debug('content_encoding: %s' % content_encoding)
log.debug('content_type: %s' % content_type)
log.debug('content_length: %s' % content_length)
except Exception as e:
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
abort(500, str(e))
else:
# bad content-type
if content_type == 'application/x-www-form-urlencoded':
abort(415, "Invalid or Unspecified Content-Type")
try:
packet = Packet()
# indicates a file upload
if content_type.startswith('multipart/form-data;'):
log.debug('found multipart form data, attempting to find source filename')
part = request.POST['document']
if part.filename:
fname = unicode(part.filename.lstrip(os.sep))
packet.set_meta('filename', fname)
# update content type based on filename
content_type = unicode(mimetypes.guess_type(fname)[0])
data = part.value
else:
data = request.body
# decompress if compressed
filedata = self._decompress(content_encoding, data)
# update content length since we might be decompressed now
content_length = len(filedata)
if content_length > 0:
packet.add('data', filedata)
else:
abort(400, 'Empty Request')
# set optional user provided id
if id is not None:
log.debug('id: %s' % id)
packet.set_meta('id', id)
# set optional user provided routing info
if route is not None:
log.debug('route: %s' % route)
packet.set_meta('route', route)
# set some common meta attributes on the packet
packet.set_meta('requestmethod', request.method)
packet.set_meta('contentlength', content_length)
packet.set_meta('mimetype', content_type)
packet.set_meta('processingtime', unicode(
datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))
status = app_globals.dfg.send(packet)
# calls into pypes core are asynchronous so we respond as such
if status['status'] == 'success':
response.status = 202
except Exception as e:
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
abort(500, str(e))
# return empty body on success otherwise return status object
return None if status['status'] == 'success' else json.dumps(status)
def create(self, route=None, id=None):
status = {}
try:
content_encoding = request.headers.get('Content-Encoding', None)
content_type = request.headers.get('Content-Type', None)
content_length = request.headers.get('Content-Length', None)
except Exception as e:
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
abort(500, str(e))
else:
# bad content-type
if content_type == 'application/x-www-form-urlencoded':
abort(415, "Invalid or Unspecified Content-Type")
try:
packet = Packet()
# indicates a file upload
if content_type.startswith('multipart/form-data;'):
this_file = request.POST['document']
fname = unicode(this_file.filename.lstrip(os.sep))
content_type = unicode(mimetypes.guess_type(fname)[0])
# check if file is compressed
if content_encoding == 'gzip':
gz_filedata = this_file.value
# gzip files have a header preceding the zlib stream.
# try with zlib (streams compressed on the fly) and if
# that fails, try the gzip module
try:
filedata = zlib.decompress(gz_filedata)
except:
gz_data = StringIO.StringIO(gz_filedata)
filedata = gzip.GzipFile(fileobj=gz_data).read()
else:
filedata = this_file.value
content_length = len(filedata)
if content_length > 0:
packet.add('data', filedata)
packet.set_meta('filename', fname)
else:
abort(400, 'Empty File')
else:
# request body contains data payload
request_body = request.body
if len(request_body) > 0:
packet.add('data', request_body)
else:
abort(400, 'Empty Request Body')
# set optional user provided id
if id is not None:
packet.set_meta('id', id)
# set optional user provided routing info
if route is not None:
packet.set_meta('route', route)
# set some common meta attributes on the packet
packet.set_meta('requestmethod', request.method)
packet.set_meta('contentlength', content_length)
packet.set_meta('mimetype', content_type)
packet.set_meta('processingtime', unicode(
datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))
status = app_globals.dfg.send(packet)
# calls into pypes core are asynchronous so we respond as such
if status['status'] == 'success':
response.status = 202
except Exception as e:
log.error('Controller Exception: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
log.debug(traceback.print_exc())
abort(500, str(e))
# return empty body on success otherwise return status object
return None if status['status'] == 'success' else json.dumps(status)
def run(self):
# Define our components entry point
while True:
# for each document waiting on our input port
for doc in self.receive_all('in'):
try:
data = doc.get('data')
mime = doc.get_meta('mimetype')
# if there is no data, move on to the next doc
if data is None:
continue
# if this is not a xml file, move on to the next doc
if mime != 'application/xml':
continue
# solr xml starts with an add tag, if this does not start
# with an add tag, move to the next doc
xml = ET.XML(data)
if xml.tag != 'add':
log.debug('Not a Solr XML document')
continue
# create a new packet for each document in the xml
for sdoc in xml.getchildren():
if sdoc.tag != 'doc':
log.warn('Unexpected tag %s, expecting doc' % \
sdoc.tag)
continue
packet = Packet()
# each field in the xml becomes a field in the packet
for sfield in sdoc.getchildren():
if sfield.tag != 'field':
log.warn('Unexpected tag %s, expecting field' \
% sfield.tag)
continue
fieldname = sfield.get('name', None)
fieldval = sfield.text
if fieldname is not None:
if not isinstance(fieldname, unicode):
fieldname = fieldname.decode('utf-8')
if not isinstance(fieldval, unicode):
fieldval = fieldval.decode('utf-8')
packet.append(fieldname, fieldval)
# send out packet along
self.send('out', packet)
except Exception as e:
log.error('Component Failed: %s' % self.__class__.__name__)
log.error('Reason: %s' % str(e))
#log.error(traceback.print_exc())
# yield the CPU, allowing another component to run
self.yield_ctrl()
