def started(self, *args):
logging.info("Initializing DeviceManager")
logging.info("Initializing discovery")
self.active = True
self.sample_worker = Worker(channel="sample_worker").register(self)
self.discovery_worker = Worker(workers=1, channel="discovery_worker").register(self)
self.discovery_timer = Timer(self.DISCOVERY_INTERVAL, discover(),
persist=True).register(self)
self.fire(discover())
return
class _UncappedInternetUploadWorkerPool(_WorkerPool):
"""
Worker to upload things to Internet when the destination is uncapped
"""
_worker = None
def __init__(self):
super(_UncappedInternetUploadWorkerPool, self).__init__()
# TODO let the amount of threads be configurable
self._worker = Worker(channel=self.__class__.__name__, workers=1)
def get_worker(self):
return self._worker
def register_workers(self, app):
self._worker.register(app)
class _DefaultWorkerPool(_WorkerPool):
"""
Worker to use when no more specific is defined
"""
_worker = None
def __init__(self):
super(_DefaultWorkerPool, self).__init__()
# TODO let the amount of threads be configurable
self._worker = Worker(channel=self.__class__.__name__, workers=1)
def get_worker(self):
return self._worker
def register_workers(self, app):
self._worker.register(app)
def worker(request):
worker = Worker()
def finalizer():
worker.stop()
request.addfinalizer(finalizer)
if request.config.option.verbose:
from circuits import Debugger
Debugger().register(worker)
waiter = pytest.WaitEvent(worker, "started")
worker.start()
assert waiter.wait()
return worker
def worker(request):
worker = Worker()
def finalizer():
worker.stop()
request.addfinalizer(finalizer)
if request.config.option.verbose:
from circuits import Debugger
Debugger().register(worker)
waiter = pytest.WaitEvent(worker, "started")
worker.start()
assert waiter.wait()
return worker
class _HardDriveWorkerPool(_WorkerPool):
"""
Workers to do I/O to hard drive
TODO: this can be differentiated by HD device destination
"""
_worker = None
def __init__(self):
super(_HardDriveWorkerPool, self).__init__()
# TODO let the amount of threads be configurable
self._worker = Worker(channel=self.__class__.__name__, workers=1)
def get_worker(self):
return self._worker
def register_workers(self, app):
self._worker.register(app)
def worker(request, manager):
worker = Worker().register(manager)
def finalizer():
worker.unregister()
request.addfinalizer(finalizer)
return worker
def __init__(self, application, *args, **kwargs):
self.application = application
self.multiprocess = kwargs.get('multiprocess', False)
self.multithread = kwargs.get('multithread', False)
self.worker = kwargs.get('worker', None)
super(Gateway, self).__init__(*args, **kwargs)
if self.multithread or self.multiprocess:
Worker(self.multiprocess, self.worker,
channel=self.channel).register(self)
class SenseHandler(Component):
_worker = Worker(process=True)
@handler("sense_event", priority=20)
def sense_event(self, event):
### Fire and Wait for: task()
#yield self.call(task(blocking_sense), self._worker)
### This will only print after task() is complete.
print(time_of_now(), "SENSING done. Now uploading...")
self.fire(Event.create("upload_event"))
def worker(request, manager, watcher):
worker = Worker().register(manager)
assert watcher.wait("registered")
def finalizer():
worker.unregister()
assert watcher.wait("unregistered")
request.addfinalizer(finalizer)
return worker
class SenseHandler(Component):
_worker = Worker(process=True)
@handler("SenseEvent", priority=100)
def sense_event(self, *args, **kwargs):
"hello, I got an event"
print(EventReport("SenseEvent", (str(args) + ", " + str(kwargs))))
CircuitsApp.timers["sense"] = Timer(args[0].period,
SenseEvent(args[0], args[1]),
persist=False).register(self)
self.fire(ReadEvent(args[0], args[1]))
class UploadHandler(Component):
_worker = Worker(process=True)
@handler("upload_event", priority=10)
def upload_event(self, event):
ustart = time_of_now()
print(time_of_now(), "Sensing+Uploading started")
###log.log(45, "Before upload BYTES\t" + str(get_tx_bytes()))
yield self.call(task(upload_a_packet), self._worker)
###log.log(45, "After upload BYTES\t" + str(get_tx_bytes()))
print(time_of_now(), "Sensing+Uploading IS COMPLETED. of time ",
ustart)
def __init__(self, *args, **kwargs):
"""
:param tile_path: Caching directory structure target path
:param default_tile: Used, when no tile can be cached
:param kwargs:
"""
super(MaptileLoader, self).__init__('MTL', *args, **kwargs)
for item in ['tilecache', 'rastertiles', 'rastercache']:
get_path('cache', item, ensure=True)
self.cache_path = get_path('cache', '')
self.worker = Worker(process=False, workers=2,
channel="tclworkers").register(self)
self.cancelled = []
self.requests = {}
self.fire(cli_register_event('test_maploader',
cli_test_maptile_Loader))
class UploadHandler(Component):
_worker = Worker(process=True)
@handler("UploadEvent", priority=120)
def upload_event(self, *args, **kwargs):
ustart = get_time_as_string(time.time())
print(EventReport("UploadEvent", "started"))
#yield self.call(task(upload_a_bundle), self._worker)
upload_a_bundle(args[0])
print(
EventReport("UploadEvent",
"ENDED (started at " + str(ustart) + ")"))
CircuitsApp.timers["upload"] = Timer(c["interval"]["upload"],
UploadEvent(args[0]),
persist=False).register(self)
def __init__(self,
tile_path='/var/cache/hfos',
default_tile=None,
**kwargs):
"""
:param tile_path: Caching directory structure target path
:param default_tile: Used, when no tile can be cached
:param kwargs:
"""
super(MaptileService, self).__init__('MTS', **kwargs)
self.worker = Worker(process=False, workers=2,
channel="tcworkers").register(self)
self.cache_path = tile_path
self.default_tile = default_tile
self._tiles = []
def __init__(self,
dataname,
filename,
x=0,
y=0,
width=0,
height=0,
transparency=255,
scale=(1920, 1200),
ignore_timings=True,
immediately=False):
super(GIFImage, self).__init__(dataname)
self.filename = filename
self.x = x
self.y = y
self.width = width
self.height = height
self.transparency = transparency
self.ignore_timings = ignore_timings
self.factor = 1
self.scale = scale
self.delta = 0.0
self.frames = []
self.durations = []
self.worker = Worker(process=False,
workers=2,
channel="gifimport_" + dataname).register(self)
# if transparency < 255:
# self.log('Setting alpha to %i' % transparency)
# for frame in self.frames:
# frame[0].set_alpha(transparency)
self.cur = 0
self.ptime = 0
self.playing = False
self.breakpoint = 0
self.startpoint = 0
self.reversed = False
if immediately:
self.get_frames()
def __init__(self, default_tile=None, **kwargs):
"""
:param tile_path: Caching directory structure target path
:param default_tile: Used, when no tile can be cached
:param kwargs:
"""
super(MaptileService, self).__init__('MTS', **kwargs)
self.worker = Worker(process=False, workers=2,
channel="tcworkers").register(self)
for item in ['tilecache', 'rastertiles', 'rastercache']:
get_path('cache', item, ensure=True)
self.cache_path = get_path('cache', '')
self.default_tile = default_tile
self._tiles = []
def __init__(self, settings):
super(GIFPlayer, self).__init__()
self.log('Player initializing')
pygame.display.init()
self.config = settings
self.factor = 1
self.delta = 0.0
self.frames = []
self.durations = []
self.clients = []
self.timer = None
self.worker = Worker(process=False,
workers=2,
channel="gifimport_" +
self.uniquename).register(self)
# if transparency < 255:
# self.log('Setting alpha to %i' % transparency)
# for frame in self.frames:
# frame[0].set_alpha(transparency)
self.cur = 0
self.ptime = 0
self.playing = False
self.breakpoint = 0
self.startpoint = 0
self.direction = True
self.fireEvent(cli_register_event("test_gifplayer",
cli_test_gifplayer))
if self.config['immediately']:
self.get_frames()
def __init__(self,
path="/tilecache",
tilepath="/var/cache/hfos/tilecache",
defaulttile=None,
**kwargs):
"""
:param path: Webserver path to offer cache on
:param tilepath: Caching directory structure target path
:param defaulttile: Used, when no tile can be cached
:param kwargs:
"""
super(TileCache, self).__init__(**kwargs)
self.worker = Worker(process=False, workers=2,
channel="tcworkers").register(self)
self.tilepath = tilepath
self.path = path
self.defaulttile = defaulttile
self._tilelist = []
def init(self, search_string, song_web_query_object):
Worker(process=False).register(self)
self.search_string = search_string
self.query_object = song_web_query_object
# Links to the specific page of song search service that have been used
# (Added to initial list, added to list due to low count, or accept/rejected by user.)
# This keeps from asking about previously processed links.
self.used_links = set()
# Links from a given search_link. This is in an OrderedDict, as we may have to pull
# prior to the user accept or rejecting it, due to running out of links.
self.future_links = OrderedDict()
# If youtube search is in progress, this caches calls to allow youtube so send when complete
self.youtube_wait = []
# Initial search link. Do we need to save this?
start_link = self.query_object.get_initial_url(search_string)
if not start_link:
self.fire(search_exhausted(), '*')
return
self._initialize_data(start_link)
self.fire(song_found(start_link), '*')
def __init__(self, device_id=0, targetresolution=(40, 16), *args,
**kwargs):
super(CVCamera, self).__init__(*args, **kwargs)
self.targetresolution = targetresolution
self.cam = cv2.VideoCapture(device_id)
self.cam.set(3, 320)
self.cam.set(4, 240)
self.framerate = 20 / 1000.0
self.recording = True
self.worker = Worker(process=False, workers=1,
channel="camera").register(self)
self.cam_timer = Timer(2, Event.create('takepicture')).register(
self)
self.surface = pygame.Surface(targetresolution)
self.log('Camera started')
def __init__(self, *args):
super(FileManager, self).__init__('FILES', *args)
self.file_object = objectmodels['file']
self.volumes = {}
self.volumes_lookup = {}
for item in objectmodels['volume'].find():
self.volumes[item.uuid] = item
self.volumes_lookup[item.name] = item
if not os.path.exists(item.path):
self.log('Target volume does not exist. Creating path', item.path, lvl=warn)
try:
os.makedirs(item.path)
except Exception as e:
self.log('Could not create volume path:', e, type(e), exc=True, lvl=error)
self.worker = Worker(process=False, workers=1,
channel="filemanagerworkers").register(self)
def __init__(self, bot, *args, **kwargs):
self.trigger = ' '
self.debugger = None
self.parser = ArgumentParser(description=self.__doc__, prog='')
self.subparsers = self.parser.add_subparsers(
title='User commands',
description='All available commands',
parser_class=ArgumentParser)
self.commands = {}
self.authenticated = []
self.master = {
'*': [
'spaceone',
]
}
self._master = {'*': []}
self.ignore = []
import spacebot.plugins as plugins
self.plugins = plugins
super(Commander, self).__init__(bot, *args, **kwargs)
Worker(channel=self.channel).register(self)
def main():
config = Config()
logger = setup_logging(config)
db = setup_database(config, logger)
manager = Manager()
Worker(channel="threadpool").register(manager)
if config["debug"]:
Debugger(
logger=logger,
events=config["verbose"],
).register(manager)
if config["daemon"]:
Daemon(config["pidfile"]).register(manager)
Core(config, db).register(manager)
manager.run()
def init(self, *args, **kwargs):
Worker(process=True).register(self)
def __init__(self):
super(_DefaultWorkerPool, self).__init__()
# TODO let the amount of threads be configurable
self._worker = Worker(channel=self.__class__.__name__, workers=1)
def __init__(self):
super(_UncappedInternetUploadWorkerPool, self).__init__()
# TODO let the amount of threads be configurable
self._worker = Worker(channel=self.__class__.__name__, workers=1)
def init(self, *args, **kwargs):
self.foo_count = 0
Worker(process=False).register(self)
class MaptileLoader(ConfigurableComponent):
"""
Maptile offline loader component
"""
configprops = {}
channel = 'isomer-web'
def __init__(self, *args, **kwargs):
"""
:param tile_path: Caching directory structure target path
:param default_tile: Used, when no tile can be cached
:param kwargs:
"""
super(MaptileLoader, self).__init__('MTL', *args, **kwargs)
for item in ['tilecache', 'rastertiles', 'rastercache']:
get_path('cache', item, ensure=True)
self.cache_path = get_path('cache', '')
self.worker = Worker(process=False, workers=2,
channel="tclworkers").register(self)
self.cancelled = []
self.requests = {}
self.fire(cli_register_event('test_maploader',
cli_test_maptile_Loader))
@handler("cli_test_maptile_Loader")
def cli_test_maptile_loader(self, event, *args):
self.log('Testing maptile loader')
event = request_maptile_area(
user=None,
action=None,
client=None,
data={
'layers': ['2837298e-a3e8-4a2e-8df2-f475554d2f23'],
'extent': [-22, -22, -17, -17],
'zoom': 14
})
self.fire(event)
def _split_cache_url(self, url, url_type):
filename = None
real_url = None
try:
# self.log('SPLITTING: ', url, lvl=verbose)
url = url[len('/' + url_type):].lstrip('/')
url = unquote(url)
# self.log('SPLITDONE:', url, lvl=1)
if '..' in url: # Does this need more safety checks?
self.log("Fishy url with parent path: ", url, lvl=error)
raise UrlError
split_url = url.split("/")
service = "/".join(
split_url[0:-3]) # get all but the coords as service
# self.log('SERVICE:', service, lvl=1)
x = split_url[-3]
y = split_url[-2]
z = split_url[-1].split('.')[0]
filename = os.path.join(self.cache_path, url_type, service, x,
y) + "/" + z + ".png"
real_url = "http://%s/%s/%s/%s.png" % (service, x, y, z)
except Exception as e:
self.log("ERROR (%s) in URL: %s" % (e, url), exc=True)
# raise UrlError
return filename, real_url
@handler(request_maptile_area)
def request_maptile_area(self, event):
self.log("Offline caching for map area requested:", event.data)
# constructing the template url for arcgis online
extent = event.data['extent']
zoom = event.data['zoom']
layer_uuids = event.data['layers']
tile_lists = {}
tile_counter = 0
for layer_uuid in layer_uuids:
layer_object = objectmodels['layer'].find_one({'uuid': layer_uuid})
if layer_object is None:
self.log('Skipping non existing layer:', layer_uuid, lvl=warn)
continue
template = layer_object.url.split('hfoshost')[1]
template = template.replace("{s}", choice(['a', 'b', 'c']))
template = template.replace('{', '{{')
template = template.replace('}', '}}')
self.log('Template:', template, lvl=debug)
tileUtils = TileUtils()
tileFinder = TileFinder(tileUtils, template)
tile_urls = tileFinder.getTileUrlsByLatLngExtent(
extent[0], extent[1], extent[2], extent[3], zoom)
tile_lists[layer_uuid] = tile_urls
tile_counter += len(tile_urls)
size = tile_counter * 15
self.log('About to get', tile_counter, 'tiles, estimated', size, 'kB')
if tile_counter == 0:
response = {
'component': 'isomer.map.maptileservice',
'action': 'empty',
'data': "No tiles in that area to fetch"
}
self.fireEvent(send(event.client.uuid, response))
return
if size > 500:
uuid = std_uuid()
request = {
'extent': extent,
'uuid': uuid,
'tiles': tile_counter,
'completed': 0,
'size': size,
'lists': tile_lists
}
msg = "Estimated size of tile offloading queue exceeded 500 kB."
self.log(msg, lvl=warn)
self.requests[uuid] = request
output = copy(request)
output.pop('lists')
response = {
'component': 'isomer.map.maptileservice',
'action': 'queued',
'data': output
}
self.log('output:', output, pretty=True)
self.fireEvent(send(event.client.uuid, response))
return
@handler(queue_cancel)
def queue_cancel(self, event):
self.log('Cancelling download of offline tiles')
self.worker.pool.terminate()
self.worker.unregister()
self.worker = Worker(process=False, workers=2,
channel="tclworkers").register(self)
@handler(queue_remove)
def queue_remove(self, event):
uuid = event.data
try:
queue = self.requests.pop(uuid)
queue.pop('lists')
response = {
'component': 'isomer.map.maptileservice',
'action': 'removed',
'data': queue
}
self.fire(send(event.client.uuid, response))
except ValueError:
pass
@handler(queue_trigger)
def queue_trigger(self, event):
uuid = event.data
try:
self.cancelled.remove(uuid)
except ValueError:
pass
if uuid in self.requests:
tile_lists = self.requests[uuid]['lists']
self.log('Getting tile list')
# TODO: Check if the layer is online at all
# e.g. rastercharts are not
for layer_uuid, tile_urls in tile_lists.items():
for url in tile_urls:
task = Event.create('get_tile_url',
url=url,
queue=uuid,
layer=layer_uuid,
client=event.client.uuid)
self.fire(task)
response = {
'component': 'isomer.map.maptileservice',
'action': 'acting',
'data': uuid
}
self.fireEvent(send(event.client.uuid, response))
else:
self.log('No queue request found:', uuid, lvl=warn)
def get_tile_url(self, url, queue, layer, client):
self.log('url:', url, queue, self.cancelled)
if queue in self.cancelled:
self.log('Not downloading tile due to cancellation')
return
filename, real_url = self._split_cache_url(url, 'tilecache')
if filename is None or real_url is None:
self.log('Could not get tile:', url)
return
try:
tile, log = yield self.call(task(get_tile, real_url), "tclworkers")
if log != "":
self.log("Thread error: ", log, lvl=error)
except Exception as e:
self.log("[MTS]", e, type(e))
tile = None
tile_path = os.path.dirname(filename)
if tile:
try:
os.makedirs(tile_path)
except OSError as e:
if e.errno != errno.EEXIST:
self.log("Couldn't create path: %s (%s)" % (e, type(e)))
self.log("Caching tile.", lvl=verbose)
try:
with open(filename, "wb") as tile_file:
try:
tile_file.write(bytes(tile))
except Exception as e:
self.log("Writing error: %s" % str([type(e), e]))
except Exception as e:
self.log("Open error: %s" % str([type(e), e]))
return
self.log("Tile stored.", lvl=verbose)
self.requests[queue]['lists'][layer].remove(url)
self.requests[queue]['completed'] += 1
completed = self.requests[queue]['completed']
if completed % 10 == 0:
self.log('Sending progress report:', completed, '/',
self.requests[queue]['tiles'])
progress_report = {
'component': 'isomer.map.maptileservice',
'action': 'offline_loader_progress',
'data': {
'queue': queue,
'completed': completed
}
}
self.fire(send(client, progress_report))
else:
self.log("Got no tile: %s" % real_url)
def queue_cancel(self, event):
self.log('Cancelling download of offline tiles')
self.worker.pool.terminate()
self.worker.unregister()
self.worker = Worker(process=False, workers=2,
channel="tclworkers").register(self)
