def write_issue(issue):
"""Stores a single github issue as task"""
if 'pull_request' not in issue:
issue_tags = []
if not ignore_labels:
for l in issue['labels']:
if l['name'].lower() not in tags:
initial = {'uuid': std_uuid(), 'name': l['name']}
new_tag = db.objectmodels['tag'](initial)
new_tag.save()
tags[new_tag.name] = new_tag
issue_tags.append(new_tag.uuid)
else:
issue_tags.append(tags[l['name'].lower()].uuid)
date = issue['created_at'].split('T')[0]
initial = {
'uuid': std_uuid(),
'name': issue['title'],
'notes': str(issue['state']) + "\n\n" + issue['html_url'],
'created': date,
'project': project_uuid
}
if len(issue_tags) > 0:
initial['tags'] = issue_tags
task = db.objectmodels['task'](initial)
task.save()
else:
log('Pull request issue:', issue, lvl=debug)
def xmpp(ctx):
"""Hello world"""
check_root()
from isomer import database
database.initialize(ctx.obj['dbhost'], ctx.obj['dbname'])
from isomer.schemata.component import ComponentConfigSchemaTemplate
factory = model_factory(ComponentConfigSchemaTemplate)
bot_config = factory.find_one({'name': 'XMPPBOT'})
if bot_config is None:
password = std_uuid()
bot_config = factory({
'nick': 'claptrap',
'name': 'XMPPBOT',
'componentclass': 'XMPPBot',
'jid': 'claptrap@localhost/node',
'password': password,
'uuid': std_uuid()
})
bot_config.save()
# log(bot_config.serializablefields(), pretty=True)
ctx.obj['bot_config'] = bot_config
def __init__(self, *args):
ConfigurableComponent.__init__(self, "IRCBOT", *args)
if self.config.get('password', None) is None:
self.config.password = std_uuid()
self.config.save()
self.channel = 'ircbot'
self.fireEvent(cli_register_event('test_irc_send', cli_test_irc_send), "isomer-web")
self.log("Started")
self.host = self.config.host
self.port = self.config.port
self.hostname = gethostname()
self.nick = self.config.nick
self.irc_channels = self.config.channels
# Mapping of IRC Channel -> Set of Nicks
self.channel_map = defaultdict(set)
# Mapping of Nick -> Set of IRC Channels
self.nick_map = defaultdict(set)
# Add TCPClient and IRC to the system.
self.transport = TCPClient(channel=self.channel).register(self)
self.protocol = IRC(channel=self.channel).register(self)
# Keep-Alive Timer
Timer(60, Event.create("keepalive"), persist=True).register(self)
def _scan_filesystem(self, path):
self.log('Scanning path:', path)
new_volume = objectmodels['volume']({'uuid': std_uuid()})
new_volume.name = path.split('/')[-1]
new_volume.path = path
new_volume.save()
self.volumes[new_volume.uuid] = new_volume
self.fireEvent(task(filewalk, path, self), 'filemanagerworkers')
def filewalk(top, component, links=None):
data = []
log = []
count = 0
handled = 0
log.append('Started scan of %s' % top)
file_object = objectmodels['file']
for root, dirs, files in os.walk(top, topdown=False):
for name in files + dirs:
filename = os.path.join(root, name)
try:
stat = os.stat(filename)
except (FileNotFoundError, PermissionError) as e:
log.append('Access error on %s' % filename)
continue
hash = md5(filename.encode('utf-8')).hexdigest()
count += 1
if count % 1000 == 0:
component.fireEvent(status_update(count))
if file_object.count({'hash': hash}) > 0:
continue
try:
if os.path.islink(filename):
if links:
# TODO: Handle symlinks in a nice manner
pass
else:
# TODO: new levelled verbose
# TODO: allow hashing of files in a smart way
entry = {
'uuid': std_uuid(),
'path': root,
'name': name,
'size': stat.st_size,
'mtime': stat.st_mtime,
'hash': hash,
'type': 'file'
}
if os.path.isdir(filename):
entry['type'] = 'folder'
new_file = file_object(entry)
data.append(new_file)
handled += 1
except EnvironmentError as e:
log.append('Error during file inspection: %s' % e)
return data, log
def __init__(self, *args):
ConfigurableComponent.__init__(self, "XMPPBOT", *args)
if self.config.get('password', None) is None:
self.config.password = std_uuid()
self.config.save()
ClientXMPP.__init__(self, self.config.jid, self.config.password)
self.add_event_handler("session_start", self.session_start)
self.add_event_handler("message", self.message)
# Discard SSL Error
self.add_event_handler("ssl_invalid_cert", self.discard)
# If you wanted more functionality, here's how to register plugins:
# self.register_plugin('xep_0030') # Service Discovery
# self.register_plugin('xep_0199') # XMPP Ping
# Here's how to access plugins once you've registered them:
# self['xep_0030'].add_feature('echo_demo')
# If you are working with an OpenFire server, you will
# need to use a different SSL version:
# import ssl
# self.ssl_version = ssl.PROTOCOL_SSLv3
import ssl
self.ssl_version = ssl.PROTOCOL_SSLv23
self.log('Connecting bot to ejabberd')
self.connect(use_tls=True)
self.log('Processing ejabberd connection')
self.process(block=False)
self.auto_authorize = True
self.auto_subscribe = True
self.send_presence(pfrom='claptrap@localhost',
pstatus='Curiouser and curiouser!',
pshow='xa')
self.fireEvent(cli_register_event('test_xmpp_send',
cli_test_xmpp_send))
self.log("Started")
def say(self, event):
"""Chat event handler for incoming events
:param event: say-event with incoming chat message
"""
try:
userid = event.user.uuid
recipient = self._get_recipient(event)
content = self._get_content(event)
message = objectmodels['chatmessage']({
'timestamp': time(),
'recipient': recipient,
'sender': userid,
'content': content,
'uuid': std_uuid()
})
message.save()
chat_packet = {
'component': 'isomer.chat.host',
'action': 'say',
'data': message.serializablefields()
}
if recipient in self.chat_channels:
for useruuid in self.users:
if useruuid in self.chat_channels[recipient].users:
self.log('User in channel', lvl=debug)
self.update_lastlog(useruuid, recipient)
self.log('Sending message', lvl=debug)
self.fireEvent(
send(useruuid, chat_packet, sendtype='user'))
except Exception as e:
self.log("Error: '%s' %s" % (e, type(e)), exc=True, lvl=error)
def userlogin(self, event):
"""Provides the newly authenticated user with a backlog and general
channel status information"""
try:
user_uuid = event.useruuid
user = objectmodels['user'].find_one({'uuid': user_uuid})
if user_uuid not in self.lastlogs:
self.log('Setting up lastlog for a new user.', lvl=debug)
lastlog = objectmodels['chatlastlog']({
'owner': user_uuid,
'uuid': std_uuid(),
'channels': {}
})
lastlog.save()
self.lastlogs[user_uuid] = lastlog
self.users[user_uuid] = user
self.user_attention[user_uuid] = None
self._send_status(user_uuid, event.clientuuid)
except Exception as e:
self.log('Error during chat setup of user:', e, type(e), exc=True)
def update_from_ais(self, event):
self.log('AIS Position update', lvl=verbose)
mmsi = event.data['mmsi']
vessel = objectmodels['vessel'].find_one({'mmsi': mmsi})
if vessel is None:
vessel = objectmodels['vessel']({'uuid': std_uuid()})
vessel.name = 'AIS tracked ship'
vessel.mmsi = mmsi
vessel.sog = event.data['sog']
vessel.cog = event.data['cog'] % 360
vessel.true_heading = event.data['true_heading'] % 360
vessel.source = 'AIS'
# pprint(vessel.serializablefields())
lat = max(-90, min(event.data['y'], 90))
lon = max(-180, min(event.data['x'], 180))
vessel.geojson['coordinates'] = [lat, lon]
vessel.save()
def request_maptile_area(self, event):
self.log("Offline caching for map area requested:", event.data)
# constructing the template url for arcgis online
extent = event.data['extent']
zoom = event.data['zoom']
layer_uuids = event.data['layers']
tile_lists = {}
tile_counter = 0
for layer_uuid in layer_uuids:
layer_object = objectmodels['layer'].find_one({'uuid': layer_uuid})
if layer_object is None:
self.log('Skipping non existing layer:', layer_uuid, lvl=warn)
continue
template = layer_object.url.split('hfoshost')[1]
template = template.replace("{s}", choice(['a', 'b', 'c']))
template = template.replace('{', '{{')
template = template.replace('}', '}}')
self.log('Template:', template, lvl=debug)
tileUtils = TileUtils()
tileFinder = TileFinder(tileUtils, template)
tile_urls = tileFinder.getTileUrlsByLatLngExtent(
extent[0], extent[1], extent[2], extent[3], zoom)
tile_lists[layer_uuid] = tile_urls
tile_counter += len(tile_urls)
size = tile_counter * 15
self.log('About to get', tile_counter, 'tiles, estimated', size, 'kB')
if tile_counter == 0:
response = {
'component': 'isomer.map.maptileservice',
'action': 'empty',
'data': "No tiles in that area to fetch"
}
self.fireEvent(send(event.client.uuid, response))
return
if size > 500:
uuid = std_uuid()
request = {
'extent': extent,
'uuid': uuid,
'tiles': tile_counter,
'completed': 0,
'size': size,
'lists': tile_lists
}
msg = "Estimated size of tile offloading queue exceeded 500 kB."
self.log(msg, lvl=warn)
self.requests[uuid] = request
output = copy(request)
output.pop('lists')
response = {
'component': 'isomer.map.maptileservice',
'action': 'queued',
'data': output
}
self.log('output:', output, pretty=True)
self.fireEvent(send(event.client.uuid, response))
return
def ICALImporter(ctx, filename, all, owner, calendar, create_calendar,
clear_calendar, dry, execfilter):
"""Calendar Importer for iCal (ics) files
"""
log('iCal importer running')
objectmodels = ctx.obj['db'].objectmodels
if objectmodels['user'].count({'name': owner}) > 0:
owner_object = objectmodels['user'].find_one({'name': owner})
elif objectmodels['user'].count({'uuid': owner}) > 0:
owner_object = objectmodels['user'].find_one({'uuid': owner})
else:
log('User unknown. Specify either uuid or name.', lvl=warn)
return
log('Found user')
if objectmodels['calendar'].count({'name': calendar}) > 0:
calendar = objectmodels['calendar'].find_one({'name': calendar})
elif objectmodels['calendar'].count({'uuid': owner}) > 0:
calendar = objectmodels['calendar'].find_one({'uuid': calendar})
elif create_calendar:
calendar = objectmodels['calendar']({
'uuid': std_uuid(),
'name': calendar
})
else:
log('Calendar unknown and no --create-calendar specified. Specify either uuid or name of an existing calendar.',
lvl=warn)
return
log('Found calendar')
if clear_calendar is True:
log('Clearing calendar events')
for item in objectmodels['event'].find({'calendar': calendar.uuid}):
item.delete()
with open(filename, 'rb') as file_object:
caldata = Calendar.from_ical(file_object.read())
keys = {
'class': 'str',
'created': 'dt',
'description': 'str',
'dtstart': 'dt',
'dtend': 'dt',
'timestamp': 'dt',
'modified': 'dt',
'location': 'str',
'status': 'str',
'summary': 'str',
'uid': 'str'
}
mapping = {'description': 'summary', 'summary': 'name'}
imports = []
def ical_import_filter(original, logfacilty):
log('Passthrough filter')
return original
if execfilter is not None:
import os
textFilePath = os.path.abspath(os.path.join(os.path.curdir,
execfilter))
textFileFolder = os.path.dirname(textFilePath)
from importlib.machinery import SourceFileLoader
filter_module = SourceFileLoader("importfilter",
textFilePath).load_module()
ical_import_filter = filter_module.ical_import_filter
for event in caldata.walk():
if event.name == 'VEVENT':
log(event, lvl=verbose, pretty=True)
initializer = {
'uuid': std_uuid(),
'calendar': calendar.uuid,
}
for item in keys:
thing = event.get(item, None)
if thing is None:
thing = 'NO-' + item
else:
if keys[item] == 'str':
thing = str(thing)
else:
thing = parser.parse(str(thing.dt))
thing = thing.isoformat()
if item in mapping:
item_assignment = mapping[item]
else:
item_assignment = item
initializer[item_assignment] = thing
new_event = objectmodels['event'](initializer)
new_event = ical_import_filter(new_event, log)
imports.append(new_event)
log(new_event, lvl=debug)
for ev in imports:
log(ev.summary)
if not dry:
log('Bulk creating events')
objectmodels['event'].bulk_create(imports)
calendar.save()
else:
log('Dry run - nothing stored.', lvl=warn)
def _register_map(self, name, target, client):
self.log('Storing new GDAL layer ', name, lvl=verbose)
try:
e = xml.etree.ElementTree.parse(
os.path.join(target, 'tilemapresource.xml')).getroot()
bounding_box = {
'minx': None,
'miny': None,
'maxx': None,
'maxy': None
}
if len(e.findall('BoundingBox')) != 1:
self.log('Irregular bounding box definitions found:',
target,
lvl=warn)
for thing in e.findall('BoundingBox'):
self.log('XMLBB:', thing, pretty=True, lvl=verbose)
for key in bounding_box:
bounding_box[key] = float(thing.get(key))
self.log('BOUNDING BOX:', bounding_box, pretty=True, lvl=verbose)
zoomlevels = []
for level in e.findall('TileSets')[0].findall('TileSet'):
self.log("XMLTS:", level, pretty=True, lvl=verbose)
zoomlevels.append(int(level.get('order')))
except Exception as e:
self.log('Problem during XML parsing:', e, type(e), exc=True)
return
uuid = std_uuid()
layer = objectmodels['layer']({'uuid': uuid})
layer.name = name
layer.path = name
layer.owner = client.useruuid
layer.notes = "Imported GDAL chart"
layer.description = "Imported GDAL chart '%s'" % name
layer.type = 'xyz'
layer.category = 'overlay'
layer.layerOptions = {
'continuousWorld':
False,
'opacity':
0.75,
'tms':
True,
'minZoom':
min(zoomlevels),
'maxZoom':
max(zoomlevels),
'bounds': [[bounding_box['miny'], bounding_box['minx']],
[bounding_box['maxy'], bounding_box['maxx']]]
}
layer.creation = datetime.datetime.now().isoformat()
layer.url = 'http://hfoshost/tilecache/raster/' + name + '/{z}/{x}/{y}.png'
layer.save()
gdal_layers = objectmodels['layergroup'].find_one(
{'uuid': GDAL_LAYER_UUID})
gdal_layers.layers.append(layer.uuid)
gdal_layers.save()
geojson = {
'type': 'Feature',
'geometry': {
'type':
'Polygon',
'coordinates': [[[bounding_box['minx'], bounding_box['miny']],
[bounding_box['minx'], bounding_box['maxy']],
[bounding_box['maxx'], bounding_box['maxy']],
[bounding_box['maxx'], bounding_box['miny']]]]
}
}
geoobject = objectmodels['geoobject']({'uuid': uuid})
geoobject.name = name
geoobject.layer = GDAL_LAYER_UUID
geoobject.owner = client.useruuid
geoobject.notes = "Imported GDAL chart"
geoobject.type = "Chart"
geoobject.opacity = 0.7
geoobject.color = 'violet'
geoobject.geojson = geojson
geoobject.references = [{'layer': uuid}]
geoobject.save()
self.log('New GDAL layer stored:', layer._fields, lvl=debug)
return layer
def session_confirm(self, event):
self.log('Confirmation for session received', event.__dict__)
session_uuid = event.data.get('session', None)
calendar_uuid = event.data.get('calendar', None)
# session_time = parser.parse(event.data.get('time', None))
session = objectmodels['session'].find_one({'uuid': session_uuid})
calendar = objectmodels['calendar'].find_one({'uuid': calendar_uuid})
session_type = objectmodels['sessiontype'].find_one(
{'uuid': session.sessiontype})
# session_end = (session_time + timedelta(minutes=session_type.length)).isoformat()
# session_time = session_time.isoformat()
summary_data = {
'author': [],
'event_type': session_type.name,
'event_id': 'ABC',
'event_keywords': session.keywords,
'event_topics': session.topics,
'abstract': session.abstract,
'keywords': "",
'topics': "",
'speakers': "",
'size': 3
}
if len(summary_data['abstract']) > 100:
summary_data['size'] = 4
if len(summary_data['abstract']) > 250:
summary_data['size'] = 5
keyword = '<span class="label label-default">%s</span>\n'
if ',' in summary_data['event_keywords']:
sep = ','
else:
sep = ' '
for word in summary_data['event_keywords'].split(sep):
summary_data['keywords'] += keyword % word
if ',' in summary_data['event_topics']:
sep = ','
else:
sep = ' '
for topic in summary_data['event_topics'].split(sep):
summary_data['topics'] += keyword % topic
summary_data['keywords'] = summary_data['keywords'].rstrip('\n')
summary_data['topics'] = summary_data['topics'].rstrip('\n')
if len(summary_data['author']) > 1:
for speaker in summary_data['author']:
summary_data['speakers'] += " " + speaker + ", "
summary_data['speakers'] = summary_data['speakers'].rstrip(', ')
elif len(summary_data['author']) == 1:
summary_data['speakers'] = summary_data['author'][0]
else:
user = objectmodels['user'].find_one({'uuid': session.owner})
summary_data['speakers'] = user.name
summary = """<div class="event_summary">
<h5> {speakers} - Type: {event_type}""".format(**summary_data)
if len(summary_data['topics']) > 0:
summary += " - <small>Topics: {topics}</small>".format(
**summary_data)
summary += """</h5>\n<h{size}>{abstract}</h{size}>
<div class="row">
<div class="col-md-10">
{keywords}
</div>
<div class="col-md-2">
<small>Talk ID: {event_id}</small>
</div>
</div>
</div>
""".format(**summary_data)
initial = {
'uuid': std_uuid(),
'owner': event.user.uuid,
'name': session.name,
'calendar': calendar_uuid,
'created': std_now(),
'recurring': False,
'duration': str(session_type.length),
'category': 'session',
'location': calendar.name,
'summary': summary,
'session': session.uuid
}
self.log(initial, pretty=True)
session_event = objectmodels['event'](initial)
session_event.save()
notification = {
'component': 'isomer.session.sessionmanager',
'action': 'session_confirm',
'data': {
'uuid': session_uuid,
'result': True
}
}
self.fireEvent(send(event.client.uuid, notification))
def _update_guide(self, guide, update=False, clear=True):
"""Update a single specified guide"""
kml_filename = os.path.join(self.cache_path, guide + '.kml')
geojson_filename = os.path.join(self.cache_path, guide + '.geojson')
if not os.path.exists(geojson_filename) or update:
try:
data = request.urlopen(
self.guides[guide]).read().decode('utf-8')
except (request.URLError, request.HTTPError) as e:
self.log('Could not get web guide data:', e, type(e), lvl=warn)
return
with open(kml_filename, 'w') as f:
f.write(data)
self._translate(kml_filename, geojson_filename)
with open(geojson_filename, 'r') as f:
json_data = json.loads(f.read())
if len(json_data['features']) == 0:
self.log('No features found!', lvl=warn)
return
layer = objectmodels['layer'].find_one({'name': guide})
if clear and layer is not None:
layer.delete()
layer = None
if layer is None:
layer_uuid = std_uuid()
layer = objectmodels['layer']({
'uuid': layer_uuid,
'name': guide,
'type': 'geoobjects'
})
layer.save()
else:
layer_uuid = layer.uuid
if clear:
for item in objectmodels['geoobject'].find({'layer': layer_uuid}):
self.log('Deleting old guide location', lvl=debug)
item.delete()
locations = []
for item in json_data['features']:
self.log('Adding new guide location:', item, lvl=verbose)
location = objectmodels['geoobject']({
'uuid':
std_uuid(),
'layer':
layer_uuid,
'geojson':
item,
'type':
'Skipperguide',
'name':
'Guide for %s' % (item['properties']['Name'])
})
locations.append(location)
self.log('Bulk inserting guide locations', lvl=debug)
objectmodels['geoobject'].bulk_create(locations)
def put_file(self, event):
self.log('File put event received:', event.__dict__.keys())
volume_id = event.data.get('volume')
raw = event.data.get('raw')
filename = os.path.normpath(event.data.get('name'))
path = os.path.normpath(event.data.get('path', ''))
req = event.data.get('req', None)
if req is None:
self.log('Request without request id!', lvl=warn)
if not volume_id or not raw or not filename:
self.log('Erroneous put file request:', event.__dict__, lvl=error)
self._notify_failure(event)
return
if volume_id in self.volumes:
volume = self.volumes[volume_id]
elif volume_id in self.volumes_lookup:
volume = self.volumes_lookup[volume_id]
else:
self.log('Unknown volume for put file request specified:', volume_id, lvl=error)
self._notify_failure(event)
return
if not self._check_permissions(event.user, volume.default_permissions['write']):
self._notify_failure(event,
'User ' + event.user.account.name + ' is not allowed to write to ' + volume.name)
return
if 'uservolume' in volume.flags:
path = os.path.join(event.user.uuid, path)
user_path = os.path.join(volume.path, path)
if not os.path.exists(user_path):
os.makedirs(user_path)
destination = os.path.normpath(os.path.join(volume.path, path, filename))
if not destination.startswith(volume.path):
self.log('Client tried to write outside of volume:',
path, filename,
' - resulting in:', destination, lvl=warn)
self._notify_failure(event)
return
self.log('Writing to', destination)
try:
with open(destination, 'wb') as f:
f.write(raw)
except Exception as e:
self.log('Error during writing:', e, type(e), lvl=error, exc=True)
self._notify_failure(event)
return
uuid = std_uuid()
fileobject = objectmodels['file']({'uuid': uuid})
fileobject.owner = event.user.uuid
fileobject.name = filename
fileobject.volume = volume.uuid
fileobject.path = path
fileobject.type = 'file'
fileobject.hash = md5(filename.encode('utf-8')).hexdigest()
fileobject.size = len(raw)
fileobject.mtime = time.time()
fileobject.save()
response = {
'component': 'isomer.filemanager.manager',
'action': 'put',
'data': {
'success': True,
'req': event.data.get('req'),
'uuid': uuid,
'filename': filename
}
}
self.fireEvent(send(event.client.uuid, response))