コード例 #1
0
ファイル: data.py プロジェクト: Big-Data/pypes
 def get(self, route=None, id=None):
     status = {}
     try:
         packet = Packet()
             
         # set packet meta attributes
         if id is not None:
             packet.set_meta('id', id)
             
         # set optional user provided routing info
         if route is not None:
             packet.set_meta('route', route)
         
         packet.set_meta('requestmethod', request.method)
         packet.set_meta('processingtime', unicode(
             datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))
         
         status = app_globals.dfg.send(packet)
         
         # calls into pypes core are asynchronous so we respond as such
         if status['status'] == 'success':
             response.status = 202
         
     except Exception as e:
         status = 'An Undefined Error Has Occurred'
         log.error('Controller Exception: %s' % self.__class__.__name__)
         log.error('Reason: %s' % str(e))                    
         log.debug(traceback.print_exc())
         response.content_type = 'application/json'
         response.status = 500
         status['error'] = str(e)
         abort(500, str(e))
             
     # return empty body on success otherwise return status object      
     return None if status['status'] == 'success' else json.dumps(status)    
コード例 #2
0
    def delete(self, route, id):
        status = {}
        try:
            packet = Packet()

            # set packet meta attributes
            packet.set_meta('id', id)
            packet.set_meta('requestmethod', request.method)
            packet.set_meta(
                'processingtime',
                unicode(
                    datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))

            status = app_globals.dfg.send(packet)

            # calls into pypes core are asynchronous so we respond as such
            if status['status'] == 'success':
                response.status = 202

        except Exception as e:
            status = 'An Undefined Error Has Occurred'
            log.error('Controller Exception: %s' % self.__class__.__name__)
            log.error('Reason: %s' % str(e))
            log.debug(traceback.print_exc())
            abort(500, str(e))

        # return empty body on success otherwise return status object
        return None if status['status'] == 'success' else json.dumps(status)
コード例 #3
0
    def run(self):
        while True:
            # get parameters outside doc loop for better performace
            try:
                delete_data = True if self.get_parameter(
                    'delete_data') == 'True' else False
            except Exception as e:
                log.error('Component Failed: %s' % self.__class__.__name__)
                log.error('Reason: %s' % str(e))

                # optionally send all docs without processing
                for d in self.receive_all('in'):
                    self.send('out', d)

                self.yield_ctrl()
                continue  # so next time we are called we continue at the top

            for doc in self.receive_all('in'):
                try:
                    data = doc.get('data')
                    mime = doc.get_meta('mimetype')

                    if data is not None:
                        # this adapter only handles JSON
                        if not mime.startswith('application/json'):
                            log.warn(
                                'Bad Content-Type, expecting application/json')
                            log.warn('Ignorning doc: %s' % doc.get('id'))
                            continue
                        else:
                            if delete_data:
                                doc.delete('data')

                            try:
                                doc.merge(Packet(json.loads(data)), metas=True)
                            except:
                                log.error('Unable to convert data')
                                log.error(traceback.print_exc())
                                # restore data field if if was deleted before error
                                if delete_data:
                                    doc.set('data', data)

                except Exception as e:
                    log.error('Component Failed: %s' % self.__class__.__name__)
                    log.error('Reason: %s' % str(e))
                    log.error(traceback.print_exc())

                self.send('out', doc)

            self.yield_ctrl()
コード例 #4
0
    def create(self, route=None, id=None):
        status = {}

        try:
            content_encoding = request.headers.get('Content-Encoding', None)
            content_type = request.headers.get('Content-Type', None)
            content_length = request.headers.get('Content-Length', None)
            log.debug('content_encoding: %s' % content_encoding)
            log.debug('content_type: %s' % content_type)
            log.debug('content_length: %s' % content_length)

        except Exception as e:
            log.error('Controller Exception: %s' % self.__class__.__name__)
            log.error('Reason: %s' % str(e))
            log.debug(traceback.print_exc())
            abort(500, str(e))

        else:
            # bad content-type
            if content_type == 'application/x-www-form-urlencoded':
                abort(415, "Invalid or Unspecified Content-Type")

            try:
                packet = Packet()

                # indicates a file upload
                if content_type.startswith('multipart/form-data;'):
                    log.debug(
                        'found multipart form data, attempting to find source filename'
                    )
                    part = request.POST['document']
                    if part.filename:
                        fname = unicode(part.filename.lstrip(os.sep))
                        packet.set_meta('filename', fname)

                        # update content type based on filename
                        content_type = unicode(mimetypes.guess_type(fname)[0])

                    data = part.value
                else:
                    data = request.body

                # decompress if compressed
                filedata = self._decompress(content_encoding, data)

                # update content length since we might be decompressed now
                content_length = len(filedata)
                if content_length > 0:
                    packet.add('data', filedata)
                else:
                    abort(400, 'Empty Request')

                # set optional user provided id
                if id is not None:
                    log.debug('id: %s' % id)
                    packet.set_meta('id', id)

                # set optional user provided routing info
                if route is not None:
                    log.debug('route: %s' % route)
                    packet.set_meta('route', route)

                # set some common meta attributes on the packet
                packet.set_meta('requestmethod', request.method)
                packet.set_meta('contentlength', content_length)
                packet.set_meta('mimetype', content_type)
                packet.set_meta(
                    'processingtime',
                    unicode(datetime.datetime.utcnow().strftime(
                        '%Y-%m-%dT%H:%M:%SZ')))

                status = app_globals.dfg.send(packet)

                # calls into pypes core are asynchronous so we respond as such
                if status['status'] == 'success':
                    response.status = 202

            except Exception as e:
                log.error('Controller Exception: %s' % self.__class__.__name__)
                log.error('Reason: %s' % str(e))
                log.debug(traceback.print_exc())
                abort(500, str(e))

        # return empty body on success otherwise return status object
        return None if status['status'] == 'success' else json.dumps(status)
コード例 #5
0
    def run(self):
        # Define our components entry point
        while True:

            # for each document waiting on our input port
            for doc in self.receive_all('in'):
                try:
                    data = doc.get('data')
                    mime = doc.get_meta('mimetype')

                    # if there is no data, move on to the next doc
                    if data is None:
                        continue

                    # if this is not a xml file, move on to the next doc
                    if mime != 'application/xml':
                        continue

                    # solr xml starts with an add tag, if this does not start
                    # with an add tag, move to the next doc
                    xml = ET.XML(data)
                    if xml.tag != 'add':
                        log.debug('Not a Solr XML document')
                        continue

                    # create a new packet for each document in the xml
                    for sdoc in xml.getchildren():
                        if sdoc.tag != 'doc':
                            log.warn('Unexpected tag %s, expecting doc' % \
                                                                    sdoc.tag)
                            continue

                        packet = Packet()
                        # each field in the xml becomes a field in the packet
                        for sfield in sdoc.getchildren():
                            if sfield.tag != 'field':
                                log.warn('Unexpected tag %s, expecting field' \
                                                                % sfield.tag)
                                continue

                            fieldname = sfield.get('name', None)
                            fieldval = sfield.text

                            if fieldname is not None:
                                if not isinstance(fieldname, unicode):
                                    fieldname = fieldname.decode('utf-8')

                                if not isinstance(fieldval, unicode):
                                    fieldval = fieldval.decode('utf-8')

                                packet.append(fieldname, fieldval)

                        # send out packet along
                        self.send('out', packet)

                except Exception as e:
                    log.error('Component Failed: %s' % self.__class__.__name__)
                    log.error('Reason: %s' % str(e))
                    #log.error(traceback.print_exc())

            # yield the CPU, allowing another component to run
            self.yield_ctrl()
コード例 #6
0
ファイル: data.py プロジェクト: Big-Data/pypes
    def create(self, route=None, id=None):
        status = {}
        
        try:
            content_encoding = request.headers.get('Content-Encoding', None)
            content_type = request.headers.get('Content-Type', None)
            content_length = request.headers.get('Content-Length', None)
            log.debug('content_encoding: %s' % content_encoding)
            log.debug('content_type: %s' % content_type)
            log.debug('content_length: %s' % content_length)
            
        except Exception as e:
            log.error('Controller Exception: %s' % self.__class__.__name__)
            log.error('Reason: %s' % str(e))                    
            log.debug(traceback.print_exc())
            abort(500, str(e))
            
        else:
            # bad content-type
            if content_type == 'application/x-www-form-urlencoded':
                abort(415, "Invalid or Unspecified Content-Type")
            
            try:
                packet = Packet()
            
                # indicates a file upload
                if content_type.startswith('multipart/form-data;'):
                    log.debug('found multipart form data, attempting to find source filename')
                    part = request.POST['document']
                    if part.filename:
                        fname = unicode(part.filename.lstrip(os.sep))
                        packet.set_meta('filename', fname)

                        # update content type based on filename
                        content_type = unicode(mimetypes.guess_type(fname)[0])
              
                    data = part.value
                else:
                    data = request.body

                
                # decompress if compressed 
                filedata = self._decompress(content_encoding, data)
 
                # update content length since we might be decompressed now
                content_length = len(filedata)
                if content_length > 0:        
                    packet.add('data', filedata)
                else:
                    abort(400, 'Empty Request')
            
                # set optional user provided id
                if id is not None:
                    log.debug('id: %s' % id)
                    packet.set_meta('id', id)
                    
                # set optional user provided routing info
                if route is not None:
                    log.debug('route: %s' % route)
                    packet.set_meta('route', route)
            
                # set some common meta attributes on the packet
                packet.set_meta('requestmethod', request.method)
                packet.set_meta('contentlength', content_length)
                packet.set_meta('mimetype', content_type)
                packet.set_meta('processingtime', unicode(
                                datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))
                
                status = app_globals.dfg.send(packet)
                
                # calls into pypes core are asynchronous so we respond as such
                if status['status'] == 'success':
                    response.status = 202
                
            except Exception as e:
                log.error('Controller Exception: %s' % self.__class__.__name__)
                log.error('Reason: %s' % str(e))                    
                log.debug(traceback.print_exc())
                abort(500, str(e))
          
        # return empty body on success otherwise return status object    
        return None if status['status'] == 'success' else json.dumps(status) 
コード例 #7
0
ファイル: data.py プロジェクト: aashish24/climatepipes1
    def create(self, route=None, id=None):
        status = {}
        
        try:
            content_encoding = request.headers.get('Content-Encoding', None)
            content_type = request.headers.get('Content-Type', None)
            content_length = request.headers.get('Content-Length', None)
            
        except Exception as e:
            log.error('Controller Exception: %s' % self.__class__.__name__)
            log.error('Reason: %s' % str(e))                    
            log.debug(traceback.print_exc())
            abort(500, str(e))
            
        else:
            # bad content-type
            if content_type == 'application/x-www-form-urlencoded':
                abort(415, "Invalid or Unspecified Content-Type")
            
            try:
                packet = Packet()
            
                # indicates a file upload
                if content_type.startswith('multipart/form-data;'):
                    this_file = request.POST['document']
                    fname = unicode(this_file.filename.lstrip(os.sep))
                    content_type = unicode(mimetypes.guess_type(fname)[0])
                
                    # check if file is compressed
                    if content_encoding == 'gzip':
                        gz_filedata = this_file.value
                        
                        # gzip files have a header preceding the zlib stream.
                        # try with zlib (streams compressed on the fly) and if
                        # that fails, try the gzip module
                        try:
                            filedata = zlib.decompress(gz_filedata)
                        except:
                            gz_data = StringIO.StringIO(gz_filedata)
                            filedata = gzip.GzipFile(fileobj=gz_data).read()
                    else:
                        filedata = this_file.value

                    content_length = len(filedata)
                    if content_length > 0:        
                        packet.add('data', filedata)
                        packet.set_meta('filename', fname)
                    else:
                        abort(400, 'Empty File')
                
                else:
                    # request body contains data payload
                    request_body = request.body
                    if len(request_body) > 0:
                        packet.add('data', request_body)
                    else:
                        abort(400, 'Empty Request Body')
            
                # set optional user provided id
                if id is not None:
                    packet.set_meta('id', id)
                    
                # set optional user provided routing info
                if route is not None:
                    packet.set_meta('route', route)
            
                # set some common meta attributes on the packet
                packet.set_meta('requestmethod', request.method)
                packet.set_meta('contentlength', content_length)
                packet.set_meta('mimetype', content_type)
                packet.set_meta('processingtime', unicode(
                                datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')))
                
                status = app_globals.dfg.send(packet)
                
                # calls into pypes core are asynchronous so we respond as such
                if status['status'] == 'success':
                    response.status = 202
                
            except Exception as e:
                log.error('Controller Exception: %s' % self.__class__.__name__)
                log.error('Reason: %s' % str(e))                    
                log.debug(traceback.print_exc())
                abort(500, str(e))
          
        # return empty body on success otherwise return status object    
        return None if status['status'] == 'success' else json.dumps(status) 
コード例 #8
0
ファイル: solrxmladapter.py プロジェクト: Big-Data/pypes
    def run(self):
        # Define our components entry point
        while True:               

            # for each document waiting on our input port
            for doc in self.receive_all('in'):
                try:
                    data = doc.get('data')
                    mime = doc.get_meta('mimetype')

                    # if there is no data, move on to the next doc
                    if data is None:
                        continue
                    
                    # if this is not a xml file, move on to the next doc
                    if mime != 'application/xml':
                        continue

                    # solr xml starts with an add tag, if this does not start
                    # with an add tag, move to the next doc
                    xml = ET.XML(data)
                    if xml.tag != 'add':
                        log.debug('Not a Solr XML document')
                        continue

                    # create a new packet for each document in the xml
                    for sdoc in xml.getchildren():
                        if sdoc.tag != 'doc':
                            log.warn('Unexpected tag %s, expecting doc' % \
                                                                    sdoc.tag)
                            continue
                
                        packet = Packet()
                        # each field in the xml becomes a field in the packet
                        for sfield in sdoc.getchildren():
                            if sfield.tag != 'field':
                                log.warn('Unexpected tag %s, expecting field' \
                                                                % sfield.tag)
                                continue

                            fieldname = sfield.get('name', None)
                            fieldval = sfield.text

                            if fieldname is not None:
                                if not isinstance(fieldname, unicode):
                                    fieldname = fieldname.decode('utf-8')

                                if not isinstance(fieldval, unicode):
                                    fieldval = fieldval.decode('utf-8')

                                packet.append(fieldname, fieldval)

                        # send out packet along
                        self.send('out', packet)

                except Exception as e:
                    log.error('Component Failed: %s' % self.__class__.__name__)
                    log.error('Reason: %s' % str(e))                    
                    #log.error(traceback.print_exc())

            # yield the CPU, allowing another component to run
            self.yield_ctrl()