def start(self):
self.sin = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sout = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sin.bind((self.inIP, self.inPort))
self.sin.setblocking(False)
r = Timer(1, self.recv)
r.name = "RTP Receiver"
r.start()
t = Timer(1, self.trans)
t.name = "RTP Transmitter"
t.start()
def _run_nanny(self):
"""Runs the nanny. It will check the channel's in-progress queue at the
configured frequency, and add any in-progress items to an internal
queue, which the :class:`~kadabra.agent.NannyThread`\s will listen to
and attempt to republish metrics from."""
try:
self.logger.debug("Running nanny")
in_progress = self.channel.in_progress(self.query_limit)
if len(in_progress) == 0:
self.logger.debug("No metrics found in progress.")
for metrics in in_progress:
if metrics.serialized_at is not None:
should_republish = _should_republish(metrics,
self.threshold_seconds, self.logger)
else:
self.logger.warn("Metrics is missing serialized_at, "
"something is wrong with the channel. "
"Attempting to republish anyway")
should_republish = True
if should_republish:
self.queue.put(metrics)
except:
self.logger.warn("Encountered exception trying to get "
"in-progress metrics", exc_info=1)
finally:
timer = Timer(self.frequency_seconds, self._run_nanny)
timer.name = "KadabraNanny"
self.timer = timer
timer.start()
def callback(self, request):
call_id = request.headers['Call-ID']
if request.method == "INVITE":
if call_id in self.calls:
return #Raise Error
if self.callCallback == None:
message = self.sip.genBusy(request)
self.sip.out.sendto(message.encode('utf8'), (self.server, self.port))
else:
sess_id = None
while sess_id == None:
proposed = random.randint(1, 100000)
if not proposed in self.session_ids:
self.session_ids.append(proposed)
sess_id = proposed
self.calls[call_id] = VoIPCall(self, request, sess_id, self.myIP, self.rtpPortLow, self.rtpPortHigh)
try:
t = Timer(1, self.callCallback, [self.calls[call_id]])
t.name = "Phone Call: "+call_id
t.start()
except Exception as e:
message = self.sip.genBusy(request)
self.sip.out.sendto(message.encode('utf8'), (self.server, self.port))
raise e
elif request.method == "BYE":
if not call_id in self.calls:
return
self.calls[call_id].bye()
def process(self, data):
if not data.args:
msg = "Please specify a time (in seconds) and a message in the following format: !remind <time> <msg>."
self.reply(data, msg)
return
try:
wait = int(data.args[0])
except ValueError:
msg = "The time must be given as an integer, in seconds."
self.reply(data, msg)
return
message = ' '.join(data.args[1:])
if not message:
msg = "What message do you want me to give you when time is up?"
self.reply(data, msg)
return
end = time.localtime(time.time() + wait)
end_time = time.strftime("%b %d %H:%M:%S", end)
end_time_with_timezone = time.strftime("%b %d %H:%M:%S %Z", end)
msg = 'Set reminder for "{0}" in {1} seconds (ends {2}).'
msg = msg.format(message, wait, end_time_with_timezone)
self.reply(data, msg)
t_reminder = Timer(wait, self.reply, args=(data, message))
t_reminder.name = "reminder " + end_time
t_reminder.daemon = True
t_reminder.start()
def registerSensorCleanup(self): """Register scheduled job to clean up sensors that have not been updated for a while""" Application().registerScheduledTask(self.cleanupSensors, hours=12) # every 12th hour tmr = Timer(600, self.cleanupSensors) # run a first time after 10 minutes tmr.daemon = True tmr.name = 'Sensor cleanup' tmr.start()
def defer(self, t, run_on_cancel, func, *args):
event = Timer(t, self.run_action, kwargs={'func': func, 'args': args})
event.name = '%s deferring %s' % (event.name, func.__name__)
event.start()
with worker_lock:
self.events[event.ident] = Event(event, run_on_cancel)
return event.ident
def program_next_poll(
self,
interval: float,
method: Callable[..., None],
times: int = None,
args: Tuple = None,
kwargs: Mapping = None,
) -> None:
if times is not None and times <= 0:
return
t = Timer(
interval=interval,
function=self.poller,
kwargs={
"interval": interval,
"method": method,
"times": times,
"args": args,
"kwargs": kwargs,
},
)
self.current_timers.append(t) # save the timer to be able to kill it
t.name = f"Poller thread for {type(method.__self__).__name__}"
t.daemon = True # so it is not locking on exit
t.start()
def get(self, rent_time=60, filter_func=lambda dev: True, timeout=0):
""" allocate reousrce from pool
Args：
time (int): in sec. how long to rent. if time exceed the rent time, the pool will force release the device
filter_func (function): used to filter device, so you can get the exact device you want, for example, 'lambda dev: dev.id == 1', you can get the device with id attribute is 1
timeout (int): in sec. how long do you want to wait, when there is no resource available, default 0 sec
Returns:
Device: return a device object. if timeout, return None
"""
assert timeout >= 0, "timeout can't be negtive"
assert rent_time > 0, "rent time can't less than zero"
start_time = time.time()
while True:
device = list(filter(filter_func, self.__available_devices))
if len(device) != 0:
dev = device[0]
self.__available_devices.remove(dev)
self.__unavailable_devices.append(dev)
def force_free():
self._free(dev)
timer = Timer(rent_time, force_free)
timer.name = 'devpool-free-timer'
timer.start()
return Device(dev, self, rent_time=rent_time)
else:
wait_time = time.time() - start_time
if wait_time >= timeout:
return None
def update_prices(self, app):
price_timer = Timer(
60.0,
self.update_prices,
kwargs={'app': app}
)
price_timer.name = 'price_timer'
price_timer.daemon = True
price_timer.start()
conn = database.get_db(app)
db = conn.cursor()
for unit in app.config['units']:
self.log.info('fetching price for {}'.format(unit))
streamer_price = self.streamer.get_price(unit)
if streamer_price is None:
price = self.standard.get_price(unit)
self.log.warn('price streamer offline!')
else:
price = streamer_price
db.execute(
"INSERT INTO prices (unit, price) VALUES (%s,%s)", (
unit,
price
)
)
self.log.info('{} price set to {}'.format(unit, price))
conn.commit()
conn.close()
def _iter(self, con, evt, i, prev=0):
result = self._check_iteration(evt, i)
if not result:
try:
if i <= 1:
self.automated_run.plot_panel.counts = 1
else:
self.automated_run.plot_panel.counts += 1
except AttributeError:
pass
if not self._iter_hook(con, i):
evt.set()
return
ot = time.time()
p = self.period_ms * 0.001
t = Timer(max(0, p - prev), self._iter, args=(con, evt, i + 1,
time.time() - ot))
t.name = 'iter_{}'.format(i + 1)
t.start()
else:
if result == 'cancel':
self.canceled = True
elif result == 'terminate':
self.terminated = True
#self.debug('no more iter')
evt.set()
def defer(self, t, run_on_cancel, func, *args):
event = Timer(t, self.run_action, kwargs={'func': func, 'args': args})
event.name = '%s deferring %s' % (event.name, func.__name__)
event.start()
with worker_lock:
self.events[event.ident] = Event(event, run_on_cancel)
return event.ident
def registerSensorCleanup(self):
Application().registerScheduledTask(self.cleanupSensors,
hours=12) # every 12th hour
t = Timer(10, self.cleanupSensors) # run a first time after 10 minutes
t.daemon = True
t.name = 'Sensor cleanup'
t.start()
def start(self):
self.s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.out = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.s.bind((self.myIP, self.myPort))
register = self.register()
t = Timer(1, self.recv)
t.name = "SIP Recieve"
t.start()
def _dumper():
def _dump():
task.kill()
dump_signal(None, None)
timer = Timer(10.0, _dump)
timer.name = "MainThreadHearthbeatCheck"
timer.daemon = True
timer.start()
GLib.idle_add(timer.cancel)
def startservice(self):
# Start a thread that waits for inputs
receiver_thread = Thread(target=self.server_loop, name='ReceiverThread')
receiver_thread.start()
# Start a thread with the server which will start a thread for each request
main_thread = Thread(target=self.handle_messages, name='MainThread')
main_thread.start()
# Start a thread that waits for inputs
input_thread = Thread(target=self.get_user_input_from_shell, name='InputThread')
input_thread.start()
# Start a thread that pings all neighbors
ping_thread = Timer(LIVENESSTIMEOUT, self.ping_neighbor)
ping_thread.name = 'PingThread'
ping_thread.start()
# Start a thread that goes through the nascentset and cleans expired ones
nascent_thread = Timer(NASCENTTIMEOUT, self.clean_nascent)
nascent_thread.name = 'NascentThread'
nascent_thread.start()
return self
def _dumper():
def _dump():
task.kill()
dump_signal(None, None)
timer = Timer(10.0, _dump)
timer.name = "MainThreadHearthbeatCheck"
timer.daemon = True
timer.start()
GLib.idle_add(timer.cancel)
def take_observation(self,
observation,
headers=None,
filename=None,
*args,
**kwargs):
"""Take an observation
Gathers various header information, sets the file path, and calls
`take_exposure`. Also creates a `threading.Event` object and a
`threading.Timer` object. The timer calls `process_exposure` after the
set amount of time is expired (`observation.exptime + self.readout_time`).
Note:
If a `filename` is passed in it can either be a full path that includes
the extension, or the basename of the file, in which case the directory
path and extension will be added to the `filename` for output
Args:
observation (~pocs.scheduler.observation.Observation): Object
describing the observation
headers (dict): Header data to be saved along with the file
filename (str, optional): Filename for saving, defaults to ISOT time stamp
**kwargs (dict): Optional keyword arguments (`exptime`)
Returns:
threading.Event: An event to be set when the image is done processing
"""
# To be used for marking when exposure is complete (see `process_exposure`)
observation_event = Event()
exptime, file_path, image_id, metadata = self._setup_observation(
observation, headers, filename, **kwargs)
exposure_event = self.take_exposure(seconds=exptime,
filename=file_path)
# Add most recent exposure to list
if self.is_primary:
if 'POINTING' in headers:
observation.pointing_images[image_id] = file_path.replace(
'.cr2', '.fits')
else:
observation.exposure_list[image_id] = file_path.replace(
'.cr2', '.fits')
# Process the image after a set amount of time
wait_time = exptime + self.readout_time
t = Timer(wait_time, self.process_exposure,
(metadata, observation_event))
t.name = f'{self.name}Thread'
t.start()
return observation_event
def take_observation(self, observation, headers=None, **kwargs):
camera_event = Event()
if headers is None:
headers = {}
start_time = headers.get('start_time', current_time(flatten=True))
filename = "solved.{}".format(self.file_extension)
file_path = "{}/pocs/tests/data/{}".format(os.getenv('POCS'), filename)
image_id = '{}_{}_{}'.format(
self.config['name'],
self.uid,
start_time
)
self.logger.debug("image_id: {}".format(image_id))
sequence_id = '{}_{}_{}'.format(
self.config['name'],
self.uid,
observation.seq_time
)
# Camera metadata
metadata = {
'camera_name': self.name,
'camera_uid': self.uid,
'field_name': observation.field.field_name,
'file_path': file_path,
'filter': self.filter_type,
'image_id': image_id,
'is_primary': self.is_primary,
'sequence_id': sequence_id,
'start_time': start_time,
}
metadata.update(headers)
exp_time = kwargs.get('exp_time', observation.exp_time.value)
exp_time = 5
self.logger.debug("Trimming camera simulator exposure to 5 s")
self.take_exposure(seconds=exp_time, filename=file_path)
# Add most recent exposure to list
observation.exposure_list[image_id] = file_path.replace('.cr2', '.fits')
# Process the image after a set amount of time
wait_time = exp_time + self.readout_time
t = Timer(wait_time, self.process_exposure, (metadata, camera_event,))
t.name = '{}Thread'.format(self.name)
t.start()
return camera_event
def startservice(self):
# Start a thread that waits for inputs
receiver_thread = Thread(target=self.server_loop,
name='ReceiverThread')
receiver_thread.start()
# Start a thread with the server which will start a thread for each request
main_thread = Thread(target=self.handle_messages, name='MainThread')
main_thread.start()
# Start a thread that waits for inputs
input_thread = Thread(target=self.get_user_input_from_shell,
name='InputThread')
input_thread.start()
# Start a thread that pings all neighbors
ping_thread = Timer(LIVENESSTIMEOUT, self.ping_neighbor)
ping_thread.name = 'PingThread'
ping_thread.start()
# Start a thread that goes through the nascentset and cleans expired ones
nascent_thread = Timer(NASCENTTIMEOUT, self.clean_nascent)
nascent_thread.name = 'NascentThread'
nascent_thread.start()
return self
def _queue_script(self, delay, name, commands):
self._logger.debug("Queuing script: %s", name)
self._plugin_manager.send_plugin_message(
self._identifier,
dict(notification_success=("Queuing script: {0:s}".format(name))))
thread = Timer(delay, self._run_script, [name, commands])
thread.name = name
thread.start()
self._queue.append(thread)
def take_observation(self, observation, headers=None, **kwargs):
camera_event = Event()
if headers is None:
headers = {}
start_time = headers.get('start_time', current_time(flatten=True))
filename = "solved.{}".format(self.file_extension)
file_path = "{}/pocs/tests/data/{}".format(os.getenv('POCS'), filename)
image_id = '{}_{}_{}'.format(self.config['name'], self.uid, start_time)
self.logger.debug("image_id: {}".format(image_id))
sequence_id = '{}_{}_{}'.format(self.config['name'], self.uid,
observation.seq_time)
# Camera metadata
metadata = {
'camera_name': self.name,
'camera_uid': self.uid,
'field_name': observation.field.field_name,
'file_path': file_path,
'filter': self.filter_type,
'image_id': image_id,
'is_primary': self.is_primary,
'sequence_id': sequence_id,
'start_time': start_time,
}
metadata.update(headers)
exp_time = kwargs.get('exp_time', observation.exp_time.value)
exp_time = 5
self.logger.debug("Trimming camera simulator exposure to 5 s")
self.take_exposure(seconds=exp_time, filename=file_path)
# Add most recent exposure to list
observation.exposure_list[image_id] = file_path.replace(
'.cr2', '.fits')
# Process the image after a set amount of time
wait_time = exp_time + self.readout_time
t = Timer(wait_time, self.process_exposure, (
metadata,
camera_event,
))
t.name = '{}Thread'.format(self.name)
t.start()
return camera_event
def start_upload_timer():
# We're using an timer in case the user adds more files to any of the subfolders after the threshold is met
# This will give the user a buffer of x seconds to add more files. Timer is reset every time the
upload_thread = list(filter(lambda t: (t.name == 'upload_timer'), threading.enumerate()))
if upload_thread: #'upload_timer' in [t.name == 'upload_timer' for t in threading.enumerate()]:
print("more files added, resetting upload timer")
upload_thread[0].cancel()
else:
print("files exceed threshold, starting upload timer")
upload_timer_wait_time = 5.0
print(f"waiting for {upload_timer_wait_time} seconds to upload files")
# upload_timer = Timer(upload_timer_wait_time, infer_images) #, args=[files_to_upload], name="upload_timer")
upload_timer = Timer(upload_timer_wait_time, infer_images_async) #, args=[files_to_upload], name="upload_timer")
upload_timer.name = 'upload_timer'
upload_timer.start()
def wait_terminate(proc, timeout):
"""
:param Popen proc:
:param int timeout: in seconds. Kill process if it times out
:return Any: retcode
"""
def _t():
proc.kill()
t = Timer(timeout, _t)
t.name = "Waiter-{}".format(proc)
t.start()
retcode = proc.wait()
t.cancel()
return retcode
def reset_training_timer():
train_timer.cancel()
# check if upload thread is running, if so wait for 25 seconds
for i in range(0, 5):
upload_thread = filter(lambda t: (t.name == 'upload_timer'),
threading.enumerate())
if upload_thread: #threading.active_count() > 1:
print("waiting for uploads to complete")
time.sleep(5)
train_timer = Timer(
5.0,
train_model,
args=[ds_ids[dataset_name]['_id'], config['model']['action']])
train_timer.name = "train_timer"
train_timer.start()
def startservice(self):
# Start a thread that waits for inputs
receiver_thread = Thread(target=self.server_loop, name='ReceiverThread')
receiver_thread.start()
# Start a thread with the server which will start a thread for each request
main_thread = Thread(target=self.handle_messages, name='MainThread')
main_thread.start()
# Start a thread that waits for inputs
#input_thread = Thread(target=self.get_user_input_from_shell, name='InputThread')
#input_thread.start()
# Start a thread that pings neighbors
timer_thread = Timer(ACKTIMEOUT/5, self.periodic)
timer_thread.name = 'PeriodicThread'
timer_thread.start()
return self
def start(self):
"""Start the nanny by starting up each
:class:`~kadabra.agent.NannyThread`."""
self.threads = []
for i in range(self.num_threads):
name = "KadabraNannyThread-%s" % str(i)
nanny_thread = NannyThread(self.channel, self.publisher,
self.queue, self.logger)
nanny_thread.name = name
self.threads.append(nanny_thread)
nanny_thread.start()
timer = Timer(self.frequency_seconds, self._run_nanny)
timer.name = "KadabraNanny"
self.timer = timer
timer.start()
def post_worker_init(worker):
process = Process(worker.pid)
def mem_monitor():
rss = process.memory_info().rss / 1000 / 1000 # [MB]
if rss > MAX_RSS:
sig = SIGINT
worker.log.info(
"Workers RSS > Max RSS ({:.2f} MB > {:.2f} MB)".format(
rss, MAX_RSS))
worker.log.info("Suicide with signal {}".format(sig))
kill(worker.pid, sig)
t = Timer(MEM_CHECK_INTERVAL, mem_monitor)
t.name = "WorkerMemoryMonitorTimer"
t.daemon = True
t.start()
def run(self):
for twt in self.only_one:
twt.interrupt()
del self.only_one[:]
self.only_one.append(self)
# Save obj creation time
self.writer = keyboard.Controller()
self.mouseListener = mouse.Listener(on_click=self.on_click)
self.mouseListener.name = f'{id(self)}-mouselistener'
# Set a timeout
t = Timer(self.timeout, self.interrupt)
t.name = f'{id(self)}-timeout'
t.start()
# Mouselistener thread
self.mouseListener.start()
while True:
time.sleep(0.2)
if self._stop_thread:
break
def input_with_timeout(x): # function for inputting with a time limit
def time_up():
print("\nTime up! Enter any key to continue.")
t = Timer(10, time_up) # x is amount of time in seconds
t.name = "countdown timer"
t.start()
try:
answer = input("Enter key: ")
t.cancel()
except Exception:
print('pass\n')
answer = None
# if answer != True:
# t.cancel() # time_up will not execute(so, no skip)
return answer
def _add_recent_search_safe(self, search):
"""
Add text element to recent searches.
"""
if len(search) < self.MIN_RECENT_SEARCH_KEY_LEN:
return
if search not in self._search_completion_model_set:
def _prepend():
self._search_completion_model.prepend((search,))
self._search_completion_model_set.add(search)
GLib.idle_add(_prepend)
with self._search_writeback_mutex:
if self._search_writeback_thread is None:
task = Timer(15.0, self._store_searches_thread)
task.name = "StoreRecentSearches"
task.daemon = True
self._search_writeback_thread = task
task.start()
def _run_batched_receiver(self):
"""Run the batched receiver. This will grab all metrics from the
queue, move them to the in progress queue, attempt to publish them with
one call to the publisher, and, if successful, mark each one as
complete. It will run at the interval specified by the
``publishing_interval``."""
try:
self.logger.debug("Running batched receiver")
batch = self.channel.receive_batch(self.max_batch_size)
self.logger.debug("%s metrics received" % len(batch))
self.publisher.publish(batch)
self.channel.complete(batch)
except:
self.logger.warn("Batched receiver runner encountered exception",\
exc_info=1)
finally:
timer = Timer(self.publishing_interval, self._run_batched_receiver)
timer.name = "KadabraBatchedReceiverRunner"
self.timer = timer
timer.start()
def paint(self, canvas):
g15theme.G15Page.paint(self, canvas)
wait = self._frame_wait
size = self._screen.driver.get_size()
if self._playing != None:
# Process may have been killed
if not self._playing._playing():
self._stop()
dir = sorted(os.listdir(self._playing.temp_dir), reverse=True)
if len(dir) > 1:
dir = dir[1:]
file = os.path.join(self._playing.temp_dir, dir[0])
self._surface = g15cairo.load_surface_from_file(file)
for path in dir:
file = os.path.join(self._playing.temp_dir, path)
os.remove(file)
else:
wait = 0.1
if self._surface != None:
target_size = (float(size[0]),
float(size[0]) * (float(self._aspect[1])) /
float(self._aspect[0]))
sx = float(target_size[0]) / float(self._surface.get_width())
sy = float(target_size[1]) / float(self._surface.get_height())
canvas.save()
canvas.translate((size[0] - target_size[0]) / 2.0,
(size[1] - target_size[1]) / 2.0)
canvas.scale(sx, sy)
canvas.set_source_surface(self._surface)
canvas.paint()
canvas.restore()
if self._playing != None:
timer = Timer(wait, self.redraw)
timer.name = "VideoRedrawTimer"
timer.setDaemon(True)
timer.start()
def _iter(self, con, evt, i, prev=0):
if not self._check_iteration(evt, i):
if not self._iter_hook(con, i):
evt.set()
return
ot = time.time()
p = self.period_ms * 0.001
p -= prev
p = max(0, p)
#self.debug('period {} {} {}'.format(p,prev, self.period_ms))
t = Timer(p, self._iter, args=(con, evt, i + 1,
time.time() - ot))
t.name = 'iter_{}'.format(i + 1)
t.start()
else:
#self.debug('no more iter')
evt.set()
def _add_recent_search_safe(self, search):
"""
Add text element to recent searches.
"""
if len(search) < self.MIN_RECENT_SEARCH_KEY_LEN:
return
if search not in self._search_completion_model_set:
def _prepend():
self._search_completion_model.prepend((search, ))
self._search_completion_model_set.add(search)
GLib.idle_add(_prepend)
with self._search_writeback_mutex:
if self._search_writeback_thread is None:
task = Timer(15.0, self._store_searches_thread)
task.name = "StoreRecentSearches"
task.daemon = True
self._search_writeback_thread = task
task.start()
def paint(self, canvas):
g15theme.G15Page.paint(self, canvas)
wait = self._frame_wait
size = self._screen.driver.get_size()
if self._playing != None:
# Process may have been killed
if not self._playing._playing():
self._stop()
dir = sorted(os.listdir(self._playing.temp_dir), reverse=True)
if len(dir) > 1:
dir = dir[1:]
file = os.path.join(self._playing.temp_dir, dir[0])
self._surface = g15cairo.load_surface_from_file(file)
for path in dir:
file = os.path.join(self._playing.temp_dir, path)
os.remove(file)
else:
wait = 0.1
if self._surface != None:
target_size = ( float(size[0]), float(size[0]) * (float(self._aspect[1]) ) / float(self._aspect[0]) )
sx = float(target_size[0]) / float(self._surface.get_width())
sy = float(target_size[1]) / float(self._surface.get_height())
canvas.save()
canvas.translate((size[0] - target_size[0]) / 2.0,(size[1] - target_size[1]) / 2.0)
canvas.scale(sx, sy)
canvas.set_source_surface(self._surface)
canvas.paint()
canvas.restore()
if self._playing != None:
timer = Timer(wait, self.redraw)
timer.name = "VideoRedrawTimer"
timer.setDaemon(True)
timer.start()
def _run_nanny(self):
"""Runs the nanny. It will check the channel's in-progress queue at the
configured frequency, and attempt to republish the in-progress metrics
as a batch."""
try:
self.logger.debug("Running batched nanny")
in_progress = self.channel.in_progress(self.max_batch_size)
if len(in_progress) == 0:
self.logger.debug("No metrics found in progress.")
batch = [m for m in in_progress if _should_republish(m,
self.threshold_seconds, self.logger)]
self.publisher.publish(batch)
self.channel.complete(batch)
except:
self.logger.warn("Batched nanny encountered exception", exc_info=1)
finally:
timer = Timer(self.frequency_seconds, self._run_nanny)
timer.name = "KadabraBatchedNanny"
self.timer = timer
timer.start()
def start_training_timer():
train_thread = list(
filter(lambda t: (t.name == 'train_timer'), threading.enumerate()))
if train_thread:
print("resetting training timer")
train_thread[0].cancel()
else:
print("starting training timer")
train_timer = Timer(
10.0,
train_model,
args=[ds_ids[dataset_name]['_id'], config['model']['action']])
train_timer.name = "train_timer"
# check if upload thread is running, if so wait for 30 seconds
# for i in range(0,3):
# upload_thread = filter(lambda t: (t.name == 'upload_timer'), threading.enumerate())
# if upload_thread: #threading.active_count() > 1:
# print("waiting for uploads to complete")
# time.sleep(10)
# else:
# break
# train_timer.start()
wait_uploads_thread = Thread(target=wait_for_uploads, args=[train_timer])
wait_uploads_thread.start()
def take_observation(self, observation, headers=None, filename=None, **kwargs):
"""Take an observation
Gathers various header information, sets the file path, and calls `take_exposure`. Also creates a
`threading.Event` object and a `threading.Timer` object. The timer calls `process_exposure` after the
set amount of time is expired (`observation.exp_time + self.readout_time`).
Note:
If a `filename` is passed in it can either be a full path that includes the extension,
or the basename of the file, in which case the directory path and extension will be added
to the `filename` for output
Args:
observation (~pocs.scheduler.observation.Observation): Object describing the observation
headers (dict): Header data to be saved along with the file
filename (str, optional): Filename for saving, defaults to ISOT time stamp
**kwargs (dict): Optional keyword arguments (`exp_time`)
Returns:
threading.Event: An event to be set when the image is done processing
"""
# To be used for marking when exposure is complete (see `process_exposure`)
camera_event = Event()
if headers is None:
headers = {}
start_time = headers.get('start_time', current_time(flatten=True))
# Get the filename
image_dir = "{}/fields/{}/{}/{}/".format(
self.config['directories']['images'],
observation.field.field_name,
self.uid,
observation.seq_time,
)
# Get full file path
if filename is None:
file_path = "{}/{}.{}".format(image_dir, start_time, self.file_extension)
else:
# Add extension
if '.' not in filename:
filename = '{}.{}'.format(filename, self.file_extension)
# Add directory
if '/' not in filename:
filename = '{}/{}'.format(image_dir, filename)
file_path = filename
image_id = '{}_{}_{}'.format(
self.config['name'],
self.uid,
start_time
)
self.logger.debug("image_id: {}".format(image_id))
sequence_id = '{}_{}_{}'.format(
self.config['name'],
self.uid,
observation.seq_time
)
# Camera metadata
metadata = {
'camera_name': self.name,
'camera_uid': self.uid,
'field_name': observation.field.field_name,
'file_path': file_path,
'filter': self.filter_type,
'image_id': image_id,
'is_primary': self.is_primary,
'sequence_id': sequence_id,
'start_time': start_time,
}
metadata.update(headers)
exp_time = kwargs.get('exp_time', observation.exp_time.value)
proc = self.take_exposure(seconds=exp_time, filename=file_path)
# Add most recent exposure to list
observation.exposure_list[image_id] = file_path.replace('.cr2', '.fits')
# Process the image after a set amount of time
wait_time = exp_time + self.readout_time
t = Timer(wait_time, self.process_exposure, (metadata, camera_event, proc))
t.name = '{}Thread'.format(self.name)
t.start()
return camera_event
def credit(app, log):
"""
This runs every minute and calculates the total liquidity on order (rank 1) and
each users proportion of it.
:param log:
:param rpc:
:param app:
:return:
"""
# Set the timer going again
credit_timer = Timer(
60.0,
credit,
kwargs={'app': app, 'log': log}
)
credit_timer.name = 'credit_timer'
credit_timer.daemon = True
credit_timer.start()
log_output = False
# reload the config
config.load(app, log, app.config['config_dir'], log_output)
# calculate the credit time
credit_time = int(time.time())
conn = database.get_db(app)
db = conn.cursor()
# Get all the orders from the database.
db.execute("SELECT * FROM orders WHERE credited=0")
all_orders = db.fetchall()
if len(all_orders) > 0:
log_output = True
log.info('Start credit')
# store the credit time in the info table
db.execute("UPDATE info SET value=%s WHERE key=%s", (
credit_time,
'last_credit_time'
))
# set up for some stats
# build the blank meta stats object
meta = {'last-credit-time': credit_time,
'number-of-users-active': 0,
'number-of-orders': 0}
db.execute("SELECT value FROM info WHERE key=%s", ('next_payout_time',))
meta['next-payout-time'] = int(db.fetchone()[0])
db.execute("SELECT COUNT(id) FROM users")
meta['number-of-users'] = int(db.fetchone()[0])
# create a list of active users
active_users = []
# de-duplicate the orders
deduped_orders = deduplicate_orders(all_orders, db)
# calculate the liquidity totals
totals = get_total_liquidity(app, deduped_orders)
# We've calculated the totals so submit them as liquidity_info
Thread(
target=liquidity_info,
kwargs={'app': app, 'totals': totals, 'log': log}
).start()
# calculate the round rewards based on percentages of target and ratios of side and
# rank
rewards = calculate_reward(app, totals)
# parse the orders
for order in deduped_orders:
# save some stats
meta['number-of-orders'] += 1
if order[1] not in active_users:
meta['number-of-users-active'] += 1
active_users.append(order[1])
# calculate the details
reward, percentage = calculate_order_reward(order, totals, rewards)
# and save to the database
db.execute(
"INSERT INTO credits (time,key,exchange,unit,rank,side,order_id,provided,"
"percentage,reward,paid) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(credit_time, order[1], order[8], order[9], order[2], order[5], order[0],
order[4], (percentage * 100), reward, 0)
)
# update the original order too to indicate that it has been credited
db.execute("UPDATE orders SET credited=%s WHERE id=%s", (1, order[0]))
# write the stats to the database
stats_config = {}
for ex in app.config['exchanges']:
stats_config[ex] = {}
for unit in app.config['{}.units'.format(ex)]:
stats_config[ex][unit] = {
'target': app.config['{}.{}.target'.format(ex, unit)],
'reward': app.config['{}.{}.reward'.format(ex, unit)]
}
for side in ['ask', 'bid']:
stats_config[ex][unit][side] = {
'ratio': app.config['{}.{}.{}.ratio'.format(
ex,
unit,
side
)]
}
for rank in app.config['{}.{}.{}.ranks'.format(ex, unit, side)]:
stats_config[ex][unit][side][rank] = {
'ratio': app.config['{}.{}.{}.{}.ratio'.format(
ex,
unit,
side,
rank
)]
}
db.execute("INSERT INTO stats (time,meta,totals,rewards,config) VALUES (%s,%s,%s,"
"%s,%s)",
(credit_time, json.dumps(meta), json.dumps(totals), json.dumps(rewards),
json.dumps(stats_config)))
conn.commit()
conn.close()
if log_output:
log.info('End credit')
return
def _setup_timer():
timer = Timer(3600, _gc_collect)
timer.name = "HourlyGarbageCollector"
timer.daemon = True
timer.start()
db_client.query('SELECT TOP(bytes,ASN,{}) INTO {} FROM {} WHERE time > now() - 1d'
''.format(_TOP_TALKERS_COUNT, _TOP_TALKERS_MEASUREMENT, _TRAFFIC_MEASUREMENT),
database=_INFLUX_DATABASE)
except (exceptions.InfluxDBClientError, exceptions.InfluxDBServerError) as db_error:
logger.error('Could not update : {}'.format(db_error))
logger.error('Exiting...')
# Canceling Timer thread
timer_obj.cancel()
# Sending KeyboardInterrupt to main thread
interrupt_main()
if __name__ == '__main__':
try:
# Starting inotify watcher thread
watcher = Thread(target=watch_prefix_file, name='prefix_file_watcher', args=(bytes(_PMACCT_DATA, 'utf-8'),))
watcher.start()
while main_thread().is_alive():
current_time = datetime.today()
# Setting timer for clear_top_talkers_measurement function to 3am each day
replaced_time = current_time.replace(hour=3, minute=0, second=0, microsecond=0)
run_time = replaced_time + timedelta(1)
timer_obj = Timer((run_time - current_time).total_seconds(), clear_top_talkers_measurement)
timer_obj.name = 'clear_timer'
timer_obj.start()
timer_obj.join()
logger.debug("Cleared top talkers at {}".format(run_time))
except KeyboardInterrupt:
logger.error("Main thread received KeyboardInterrupt signal and closed")
exit(1)
def _setup_timer():
timer = Timer(3600, _gc_collect)
timer.name = "HourlyGarbageCollector"
timer.daemon = True
timer.start()
def set_timer():
t = Timer(1.0, unset)
t.name = "UnsetScrollingTimer"
t.daemon = True
return t
def pay(app, log):
"""
Pay all users who have a balance greater than the minimum payout
:param log:
:param rpc:
:param app:
:return:
"""
log.info('payout started')
# get the credit details from the database
conn = database.get_db(app)
db = conn.cursor()
db.execute("SELECT c.id,c.key,c.reward,u.address FROM credits AS c INNER JOIN "
"users AS u on u.key=c.key WHERE c.paid=0")
rewards = db.fetchall()
# Calculate the total credit for each unique address
user_rewards = {}
for reward in rewards:
if reward[3] not in user_rewards:
user_rewards[reward[3]] = 0.00
user_rewards[reward[3]] += float(reward[2])
# remove those which don't meet the minimum payout threshold
# and round to 6dp
user_payouts = user_rewards.copy()
for address in user_rewards:
if user_rewards[address] < float(app.config['pool.minimum_payout']):
del(user_payouts[address])
continue
user_payouts[address] = round(float(user_payouts[address]), 6)
if not user_payouts:
log.info('no-one to payout to: %s', user_rewards)
timer_time = 86400.0
else:
# SendMany from nud. Report any error to log output
try:
# get an rpc connection
rpc = get_rpc(app, log)
rpc.sendmany("", user_payouts)
log.info('payout successful: \'%s\'', json.dumps(user_payouts))
# mark credits to paid addresses as paid
for reward in rewards:
if reward[3] in user_payouts:
db.execute('UPDATE credits SET paid=1 WHERE id=%s', (reward[0],))
# set the timer for the next payout
timer_time = 86400.0
except JSONRPCException as e:
log.error('Payout failed - %s: \'%s\'', e.message, json.dumps(user_payouts))
timer_time = 120.0
except (socket.error, CannotSendRequest, ValueError):
log.error('Payout failed - no connection with nud: \'%s\'', json.dumps(
user_payouts))
timer_time = 120.0
# reset timer
payout_timer = Timer(timer_time, pay,
kwargs={'app': app, 'log': log})
payout_timer.name = 'payout_timer'
payout_timer.daemon = True
payout_timer.start()
# update the next payout time
db.execute('UPDATE info SET value=%s WHERE key=%s', (int(time.time() + timer_time),
'next_payout_time'))
conn.commit()
conn.close()
if 'poloniex' in app.config['exchanges']:
wrappers['poloniex'] = src.exchanges.Poloniex()
if 'test_exchange' in app.config['exchanges']:
wrappers['test_exchange'] = src.exchanges.TestExchange()
# save the start time of the server for reporting up-time
app.config['start_time'] = time.time()
# set up a price fetcher object
pf = PriceFetcher(app, log)
# Set the timer for credits
log.info('running credit timer')
credit_timer = Timer(60.0, credit.credit,
kwargs={'app': app, 'log': log})
credit_timer.name = 'credit_timer'
credit_timer.daemon = True
credit_timer.start()
# Set the timer for payouts
log.info('running payout timer')
conn = database.get_db(app)
db = conn.cursor()
db.execute("SELECT value FROM info WHERE key = %s", ('next_payout_time',))
next_payout_time = int(db.fetchone()[0])
if next_payout_time == 0:
payout_time = 86400
db.execute('UPDATE info SET value=%s WHERE key=%s', (int(time.time() + payout_time),
'next_payout_time'))
else:
payout_time = int(next_payout_time - int(time.time()))
