Esempio n. 1
0
    def write_issue(issue):
        """Stores a single github issue as task"""

        if 'pull_request' not in issue:

            issue_tags = []
            if not ignore_labels:
                for l in issue['labels']:
                    if l['name'].lower() not in tags:
                        initial = {'uuid': std_uuid(), 'name': l['name']}
                        new_tag = db.objectmodels['tag'](initial)
                        new_tag.save()

                        tags[new_tag.name] = new_tag

                        issue_tags.append(new_tag.uuid)
                    else:
                        issue_tags.append(tags[l['name'].lower()].uuid)

            date = issue['created_at'].split('T')[0]

            initial = {
                'uuid': std_uuid(),
                'name': issue['title'],
                'notes': str(issue['state']) + "\n\n" + issue['html_url'],
                'created': date,
                'project': project_uuid
            }
            if len(issue_tags) > 0:
                initial['tags'] = issue_tags

            task = db.objectmodels['task'](initial)
            task.save()
        else:
            log('Pull request issue:', issue, lvl=debug)
Esempio n. 2
0
def xmpp(ctx):
    """Hello world"""

    check_root()

    from isomer import database
    database.initialize(ctx.obj['dbhost'], ctx.obj['dbname'])

    from isomer.schemata.component import ComponentConfigSchemaTemplate
    factory = model_factory(ComponentConfigSchemaTemplate)
    bot_config = factory.find_one({'name': 'XMPPBOT'})

    if bot_config is None:
        password = std_uuid()

        bot_config = factory({
            'nick': 'claptrap',
            'name': 'XMPPBOT',
            'componentclass': 'XMPPBot',
            'jid': 'claptrap@localhost/node',
            'password': password,
            'uuid': std_uuid()
        })
        bot_config.save()

    # log(bot_config.serializablefields(), pretty=True)
    ctx.obj['bot_config'] = bot_config
Esempio n. 3
0
    def __init__(self, *args):
        ConfigurableComponent.__init__(self, "IRCBOT", *args)
        if self.config.get('password', None) is None:
            self.config.password = std_uuid()
            self.config.save()

        self.channel = 'ircbot'

        self.fireEvent(cli_register_event('test_irc_send', cli_test_irc_send), "isomer-web")
        self.log("Started")

        self.host = self.config.host
        self.port = self.config.port

        self.hostname = gethostname()

        self.nick = self.config.nick
        self.irc_channels = self.config.channels

        # Mapping of IRC Channel -> Set of Nicks
        self.channel_map = defaultdict(set)

        # Mapping of Nick -> Set of IRC Channels
        self.nick_map = defaultdict(set)

        # Add TCPClient and IRC to the system.
        self.transport = TCPClient(channel=self.channel).register(self)
        self.protocol = IRC(channel=self.channel).register(self)

        # Keep-Alive Timer
        Timer(60, Event.create("keepalive"), persist=True).register(self)
Esempio n. 4
0
 def _scan_filesystem(self, path):
     self.log('Scanning path:', path)
     new_volume = objectmodels['volume']({'uuid': std_uuid()})
     new_volume.name = path.split('/')[-1]
     new_volume.path = path
     new_volume.save()
     self.volumes[new_volume.uuid] = new_volume
     self.fireEvent(task(filewalk, path, self), 'filemanagerworkers')
Esempio n. 5
0
def filewalk(top, component, links=None):
    data = []
    log = []
    count = 0
    handled = 0
    log.append('Started scan of %s' % top)

    file_object = objectmodels['file']

    for root, dirs, files in os.walk(top, topdown=False):
        for name in files + dirs:
            filename = os.path.join(root, name)
            try:
                stat = os.stat(filename)
            except (FileNotFoundError, PermissionError) as e:
                log.append('Access error on %s' % filename)
                continue

            hash = md5(filename.encode('utf-8')).hexdigest()

            count += 1
            if count % 1000 == 0:
                component.fireEvent(status_update(count))

            if file_object.count({'hash': hash}) > 0:
                continue

            try:
                if os.path.islink(filename):
                    if links:
                        # TODO: Handle symlinks in a nice manner
                        pass
                else:
                    # TODO: new levelled verbose
                    # TODO: allow hashing of files in a smart way
                    entry = {
                        'uuid': std_uuid(),
                        'path': root,
                        'name': name,
                        'size': stat.st_size,
                        'mtime': stat.st_mtime,
                        'hash': hash,
                        'type': 'file'
                    }
                    if os.path.isdir(filename):
                        entry['type'] = 'folder'
                    new_file = file_object(entry)

                    data.append(new_file)
                    handled += 1
            except EnvironmentError as e:
                log.append('Error during file inspection: %s' % e)

    return data, log
Esempio n. 6
0
    def __init__(self, *args):
        ConfigurableComponent.__init__(self, "XMPPBOT", *args)
        if self.config.get('password', None) is None:
            self.config.password = std_uuid()
            self.config.save()

        ClientXMPP.__init__(self, self.config.jid, self.config.password)

        self.add_event_handler("session_start", self.session_start)
        self.add_event_handler("message", self.message)
        # Discard SSL Error
        self.add_event_handler("ssl_invalid_cert", self.discard)

        # If you wanted more functionality, here's how to register plugins:
        # self.register_plugin('xep_0030') # Service Discovery
        # self.register_plugin('xep_0199') # XMPP Ping

        # Here's how to access plugins once you've registered them:
        # self['xep_0030'].add_feature('echo_demo')

        # If you are working with an OpenFire server, you will
        # need to use a different SSL version:
        # import ssl
        # self.ssl_version = ssl.PROTOCOL_SSLv3

        import ssl
        self.ssl_version = ssl.PROTOCOL_SSLv23

        self.log('Connecting bot to ejabberd')
        self.connect(use_tls=True)
        self.log('Processing ejabberd connection')
        self.process(block=False)

        self.auto_authorize = True
        self.auto_subscribe = True
        self.send_presence(pfrom='claptrap@localhost',
                           pstatus='Curiouser and curiouser!',
                           pshow='xa')

        self.fireEvent(cli_register_event('test_xmpp_send',
                                          cli_test_xmpp_send))
        self.log("Started")
Esempio n. 7
0
    def say(self, event):
        """Chat event handler for incoming events
        :param event: say-event with incoming chat message
        """

        try:
            userid = event.user.uuid
            recipient = self._get_recipient(event)
            content = self._get_content(event)

            message = objectmodels['chatmessage']({
                'timestamp': time(),
                'recipient': recipient,
                'sender': userid,
                'content': content,
                'uuid': std_uuid()
            })

            message.save()

            chat_packet = {
                'component': 'isomer.chat.host',
                'action': 'say',
                'data': message.serializablefields()
            }

            if recipient in self.chat_channels:
                for useruuid in self.users:
                    if useruuid in self.chat_channels[recipient].users:
                        self.log('User in channel', lvl=debug)
                        self.update_lastlog(useruuid, recipient)

                        self.log('Sending message', lvl=debug)
                        self.fireEvent(
                            send(useruuid, chat_packet, sendtype='user'))

        except Exception as e:
            self.log("Error: '%s' %s" % (e, type(e)), exc=True, lvl=error)
Esempio n. 8
0
    def userlogin(self, event):
        """Provides the newly authenticated user with a backlog and general
        channel status information"""

        try:
            user_uuid = event.useruuid
            user = objectmodels['user'].find_one({'uuid': user_uuid})

            if user_uuid not in self.lastlogs:
                self.log('Setting up lastlog for a new user.', lvl=debug)
                lastlog = objectmodels['chatlastlog']({
                    'owner': user_uuid,
                    'uuid': std_uuid(),
                    'channels': {}
                })
                lastlog.save()
                self.lastlogs[user_uuid] = lastlog

            self.users[user_uuid] = user
            self.user_attention[user_uuid] = None
            self._send_status(user_uuid, event.clientuuid)
        except Exception as e:
            self.log('Error during chat setup of user:', e, type(e), exc=True)
Esempio n. 9
0
    def update_from_ais(self, event):
        self.log('AIS Position update', lvl=verbose)
        mmsi = event.data['mmsi']

        vessel = objectmodels['vessel'].find_one({'mmsi': mmsi})

        if vessel is None:
            vessel = objectmodels['vessel']({'uuid': std_uuid()})
            vessel.name = 'AIS tracked ship'
            vessel.mmsi = mmsi
            vessel.sog = event.data['sog']
            vessel.cog = event.data['cog'] % 360
            vessel.true_heading = event.data['true_heading'] % 360
            vessel.source = 'AIS'

        # pprint(vessel.serializablefields())

        lat = max(-90, min(event.data['y'], 90))
        lon = max(-180, min(event.data['x'], 180))

        vessel.geojson['coordinates'] = [lat, lon]

        vessel.save()
Esempio n. 10
0
    def request_maptile_area(self, event):
        self.log("Offline caching for map area requested:", event.data)
        # constructing the template url for arcgis online

        extent = event.data['extent']
        zoom = event.data['zoom']

        layer_uuids = event.data['layers']

        tile_lists = {}

        tile_counter = 0

        for layer_uuid in layer_uuids:
            layer_object = objectmodels['layer'].find_one({'uuid': layer_uuid})
            if layer_object is None:
                self.log('Skipping non existing layer:', layer_uuid, lvl=warn)
                continue
            template = layer_object.url.split('hfoshost')[1]

            template = template.replace("{s}", choice(['a', 'b', 'c']))
            template = template.replace('{', '{{')
            template = template.replace('}', '}}')
            self.log('Template:', template, lvl=debug)

            tileUtils = TileUtils()
            tileFinder = TileFinder(tileUtils, template)
            tile_urls = tileFinder.getTileUrlsByLatLngExtent(
                extent[0], extent[1], extent[2], extent[3], zoom)
            tile_lists[layer_uuid] = tile_urls
            tile_counter += len(tile_urls)

        size = tile_counter * 15
        self.log('About to get', tile_counter, 'tiles, estimated', size, 'kB')

        if tile_counter == 0:
            response = {
                'component': 'isomer.map.maptileservice',
                'action': 'empty',
                'data': "No tiles in that area to fetch"
            }
            self.fireEvent(send(event.client.uuid, response))

            return

        if size > 500:
            uuid = std_uuid()
            request = {
                'extent': extent,
                'uuid': uuid,
                'tiles': tile_counter,
                'completed': 0,
                'size': size,
                'lists': tile_lists
            }

            msg = "Estimated size of tile offloading queue exceeded 500 kB."

            self.log(msg, lvl=warn)
            self.requests[uuid] = request

            output = copy(request)
            output.pop('lists')

            response = {
                'component': 'isomer.map.maptileservice',
                'action': 'queued',
                'data': output
            }
            self.log('output:', output, pretty=True)
            self.fireEvent(send(event.client.uuid, response))

            return
Esempio n. 11
0
def ICALImporter(ctx, filename, all, owner, calendar, create_calendar,
                 clear_calendar, dry, execfilter):
    """Calendar Importer for iCal (ics) files

    """

    log('iCal importer running')

    objectmodels = ctx.obj['db'].objectmodels

    if objectmodels['user'].count({'name': owner}) > 0:
        owner_object = objectmodels['user'].find_one({'name': owner})
    elif objectmodels['user'].count({'uuid': owner}) > 0:
        owner_object = objectmodels['user'].find_one({'uuid': owner})
    else:
        log('User unknown. Specify either uuid or name.', lvl=warn)
        return

    log('Found user')

    if objectmodels['calendar'].count({'name': calendar}) > 0:
        calendar = objectmodels['calendar'].find_one({'name': calendar})
    elif objectmodels['calendar'].count({'uuid': owner}) > 0:
        calendar = objectmodels['calendar'].find_one({'uuid': calendar})
    elif create_calendar:
        calendar = objectmodels['calendar']({
            'uuid': std_uuid(),
            'name': calendar
        })
    else:
        log('Calendar unknown and no --create-calendar specified. Specify either uuid or name of an existing calendar.',
            lvl=warn)
        return

    log('Found calendar')

    if clear_calendar is True:
        log('Clearing calendar events')
        for item in objectmodels['event'].find({'calendar': calendar.uuid}):
            item.delete()

    with open(filename, 'rb') as file_object:
        caldata = Calendar.from_ical(file_object.read())

    keys = {
        'class': 'str',
        'created': 'dt',
        'description': 'str',
        'dtstart': 'dt',
        'dtend': 'dt',
        'timestamp': 'dt',
        'modified': 'dt',
        'location': 'str',
        'status': 'str',
        'summary': 'str',
        'uid': 'str'
    }
    mapping = {'description': 'summary', 'summary': 'name'}

    imports = []

    def ical_import_filter(original, logfacilty):
        log('Passthrough filter')
        return original

    if execfilter is not None:
        import os

        textFilePath = os.path.abspath(os.path.join(os.path.curdir,
                                                    execfilter))
        textFileFolder = os.path.dirname(textFilePath)

        from importlib.machinery import SourceFileLoader

        filter_module = SourceFileLoader("importfilter",
                                         textFilePath).load_module()
        ical_import_filter = filter_module.ical_import_filter

    for event in caldata.walk():
        if event.name == 'VEVENT':
            log(event, lvl=verbose, pretty=True)
            initializer = {
                'uuid': std_uuid(),
                'calendar': calendar.uuid,
            }

            for item in keys:
                thing = event.get(item, None)
                if thing is None:
                    thing = 'NO-' + item
                else:
                    if keys[item] == 'str':
                        thing = str(thing)
                    else:
                        thing = parser.parse(str(thing.dt))
                        thing = thing.isoformat()

                if item in mapping:
                    item_assignment = mapping[item]
                else:
                    item_assignment = item

                initializer[item_assignment] = thing
            new_event = objectmodels['event'](initializer)

            new_event = ical_import_filter(new_event, log)

            imports.append(new_event)

            log(new_event, lvl=debug)

    for ev in imports:
        log(ev.summary)

    if not dry:
        log('Bulk creating events')
        objectmodels['event'].bulk_create(imports)

        calendar.save()
    else:
        log('Dry run - nothing stored.', lvl=warn)
Esempio n. 12
0
    def _register_map(self, name, target, client):
        self.log('Storing new GDAL layer ', name, lvl=verbose)

        try:
            e = xml.etree.ElementTree.parse(
                os.path.join(target, 'tilemapresource.xml')).getroot()

            bounding_box = {
                'minx': None,
                'miny': None,
                'maxx': None,
                'maxy': None
            }

            if len(e.findall('BoundingBox')) != 1:
                self.log('Irregular bounding box definitions found:',
                         target,
                         lvl=warn)
            for thing in e.findall('BoundingBox'):
                self.log('XMLBB:', thing, pretty=True, lvl=verbose)
                for key in bounding_box:
                    bounding_box[key] = float(thing.get(key))

            self.log('BOUNDING BOX:', bounding_box, pretty=True, lvl=verbose)

            zoomlevels = []
            for level in e.findall('TileSets')[0].findall('TileSet'):
                self.log("XMLTS:", level, pretty=True, lvl=verbose)
                zoomlevels.append(int(level.get('order')))

        except Exception as e:
            self.log('Problem during XML parsing:', e, type(e), exc=True)
            return

        uuid = std_uuid()

        layer = objectmodels['layer']({'uuid': uuid})
        layer.name = name
        layer.path = name
        layer.owner = client.useruuid
        layer.notes = "Imported GDAL chart"
        layer.description = "Imported GDAL chart '%s'" % name
        layer.type = 'xyz'
        layer.category = 'overlay'
        layer.layerOptions = {
            'continuousWorld':
            False,
            'opacity':
            0.75,
            'tms':
            True,
            'minZoom':
            min(zoomlevels),
            'maxZoom':
            max(zoomlevels),
            'bounds': [[bounding_box['miny'], bounding_box['minx']],
                       [bounding_box['maxy'], bounding_box['maxx']]]
        }
        layer.creation = datetime.datetime.now().isoformat()
        layer.url = 'http://hfoshost/tilecache/raster/' + name + '/{z}/{x}/{y}.png'

        layer.save()

        gdal_layers = objectmodels['layergroup'].find_one(
            {'uuid': GDAL_LAYER_UUID})

        gdal_layers.layers.append(layer.uuid)
        gdal_layers.save()

        geojson = {
            'type': 'Feature',
            'geometry': {
                'type':
                'Polygon',
                'coordinates': [[[bounding_box['minx'], bounding_box['miny']],
                                 [bounding_box['minx'], bounding_box['maxy']],
                                 [bounding_box['maxx'], bounding_box['maxy']],
                                 [bounding_box['maxx'], bounding_box['miny']]]]
            }
        }

        geoobject = objectmodels['geoobject']({'uuid': uuid})
        geoobject.name = name
        geoobject.layer = GDAL_LAYER_UUID
        geoobject.owner = client.useruuid
        geoobject.notes = "Imported GDAL chart"
        geoobject.type = "Chart"
        geoobject.opacity = 0.7
        geoobject.color = 'violet'
        geoobject.geojson = geojson
        geoobject.references = [{'layer': uuid}]

        geoobject.save()

        self.log('New GDAL layer stored:', layer._fields, lvl=debug)

        return layer
Esempio n. 13
0
    def session_confirm(self, event):
        self.log('Confirmation for session received', event.__dict__)

        session_uuid = event.data.get('session', None)
        calendar_uuid = event.data.get('calendar', None)
        # session_time = parser.parse(event.data.get('time', None))

        session = objectmodels['session'].find_one({'uuid': session_uuid})
        calendar = objectmodels['calendar'].find_one({'uuid': calendar_uuid})

        session_type = objectmodels['sessiontype'].find_one(
            {'uuid': session.sessiontype})

        # session_end = (session_time + timedelta(minutes=session_type.length)).isoformat()
        # session_time = session_time.isoformat()

        summary_data = {
            'author': [],
            'event_type': session_type.name,
            'event_id': 'ABC',
            'event_keywords': session.keywords,
            'event_topics': session.topics,
            'abstract': session.abstract,
            'keywords': "",
            'topics': "",
            'speakers': "",
            'size': 3
        }

        if len(summary_data['abstract']) > 100:
            summary_data['size'] = 4
        if len(summary_data['abstract']) > 250:
            summary_data['size'] = 5

        keyword = '<span class="label label-default">%s</span>\n'

        if ',' in summary_data['event_keywords']:
            sep = ','
        else:
            sep = ' '

        for word in summary_data['event_keywords'].split(sep):
            summary_data['keywords'] += keyword % word

        if ',' in summary_data['event_topics']:
            sep = ','
        else:
            sep = ' '

        for topic in summary_data['event_topics'].split(sep):
            summary_data['topics'] += keyword % topic

        summary_data['keywords'] = summary_data['keywords'].rstrip('\n')
        summary_data['topics'] = summary_data['topics'].rstrip('\n')

        if len(summary_data['author']) > 1:
            for speaker in summary_data['author']:
                summary_data['speakers'] += " " + speaker + ", "
            summary_data['speakers'] = summary_data['speakers'].rstrip(', ')
        elif len(summary_data['author']) == 1:
            summary_data['speakers'] = summary_data['author'][0]
        else:
            user = objectmodels['user'].find_one({'uuid': session.owner})
            summary_data['speakers'] = user.name

        summary = """<div class="event_summary">
            <h5> {speakers} - Type: {event_type}""".format(**summary_data)
        if len(summary_data['topics']) > 0:
            summary += " - <small>Topics: {topics}</small>".format(
                **summary_data)
        summary += """</h5>\n<h{size}>{abstract}</h{size}>
            <div class="row">
                <div class="col-md-10">
                    {keywords}
                </div>
                <div class="col-md-2">
                    <small>Talk ID: {event_id}</small>
                </div>
            </div>
        </div>
        """.format(**summary_data)

        initial = {
            'uuid': std_uuid(),
            'owner': event.user.uuid,
            'name': session.name,
            'calendar': calendar_uuid,
            'created': std_now(),
            'recurring': False,
            'duration': str(session_type.length),
            'category': 'session',
            'location': calendar.name,
            'summary': summary,
            'session': session.uuid
        }
        self.log(initial, pretty=True)
        session_event = objectmodels['event'](initial)

        session_event.save()

        notification = {
            'component': 'isomer.session.sessionmanager',
            'action': 'session_confirm',
            'data': {
                'uuid': session_uuid,
                'result': True
            }
        }

        self.fireEvent(send(event.client.uuid, notification))
Esempio n. 14
0
    def _update_guide(self, guide, update=False, clear=True):
        """Update a single specified guide"""

        kml_filename = os.path.join(self.cache_path, guide + '.kml')
        geojson_filename = os.path.join(self.cache_path, guide + '.geojson')

        if not os.path.exists(geojson_filename) or update:
            try:
                data = request.urlopen(
                    self.guides[guide]).read().decode('utf-8')
            except (request.URLError, request.HTTPError) as e:
                self.log('Could not get web guide data:', e, type(e), lvl=warn)
                return

            with open(kml_filename, 'w') as f:
                f.write(data)

            self._translate(kml_filename, geojson_filename)

        with open(geojson_filename, 'r') as f:
            json_data = json.loads(f.read())

        if len(json_data['features']) == 0:
            self.log('No features found!', lvl=warn)
            return

        layer = objectmodels['layer'].find_one({'name': guide})

        if clear and layer is not None:
            layer.delete()
            layer = None

        if layer is None:
            layer_uuid = std_uuid()
            layer = objectmodels['layer']({
                'uuid': layer_uuid,
                'name': guide,
                'type': 'geoobjects'
            })
            layer.save()
        else:
            layer_uuid = layer.uuid

        if clear:
            for item in objectmodels['geoobject'].find({'layer': layer_uuid}):
                self.log('Deleting old guide location', lvl=debug)
                item.delete()

        locations = []

        for item in json_data['features']:
            self.log('Adding new guide location:', item, lvl=verbose)
            location = objectmodels['geoobject']({
                'uuid':
                std_uuid(),
                'layer':
                layer_uuid,
                'geojson':
                item,
                'type':
                'Skipperguide',
                'name':
                'Guide for %s' % (item['properties']['Name'])
            })
            locations.append(location)

        self.log('Bulk inserting guide locations', lvl=debug)
        objectmodels['geoobject'].bulk_create(locations)
Esempio n. 15
0
    def put_file(self, event):
        self.log('File put event received:', event.__dict__.keys())

        volume_id = event.data.get('volume')
        raw = event.data.get('raw')
        filename = os.path.normpath(event.data.get('name'))
        path = os.path.normpath(event.data.get('path', ''))
        req = event.data.get('req', None)

        if req is None:
            self.log('Request without request id!', lvl=warn)

        if not volume_id or not raw or not filename:
            self.log('Erroneous put file request:', event.__dict__, lvl=error)
            self._notify_failure(event)
            return

        if volume_id in self.volumes:
            volume = self.volumes[volume_id]
        elif volume_id in self.volumes_lookup:
            volume = self.volumes_lookup[volume_id]
        else:
            self.log('Unknown volume for put file request specified:', volume_id, lvl=error)
            self._notify_failure(event)
            return

        if not self._check_permissions(event.user, volume.default_permissions['write']):
            self._notify_failure(event,
                                 'User ' + event.user.account.name + ' is not allowed to write to ' + volume.name)
            return

        if 'uservolume' in volume.flags:
            path = os.path.join(event.user.uuid, path)
            user_path = os.path.join(volume.path, path)
            if not os.path.exists(user_path):
                os.makedirs(user_path)

        destination = os.path.normpath(os.path.join(volume.path, path, filename))

        if not destination.startswith(volume.path):
            self.log('Client tried to write outside of volume:',
                     path, filename,
                     ' - resulting in:', destination, lvl=warn)
            self._notify_failure(event)
            return

        self.log('Writing to', destination)

        try:
            with open(destination, 'wb') as f:
                f.write(raw)
        except Exception as e:
            self.log('Error during writing:', e, type(e), lvl=error, exc=True)
            self._notify_failure(event)
            return

        uuid = std_uuid()

        fileobject = objectmodels['file']({'uuid': uuid})
        fileobject.owner = event.user.uuid
        fileobject.name = filename
        fileobject.volume = volume.uuid
        fileobject.path = path
        fileobject.type = 'file'
        fileobject.hash = md5(filename.encode('utf-8')).hexdigest()
        fileobject.size = len(raw)
        fileobject.mtime = time.time()

        fileobject.save()

        response = {
            'component': 'isomer.filemanager.manager',
            'action': 'put',
            'data': {
                'success': True,
                'req': event.data.get('req'),
                'uuid': uuid,
                'filename': filename
            }
        }

        self.fireEvent(send(event.client.uuid, response))