def test_multiprocess_func_does_something():
q = Queue()
multiprocess(lambda q, x: q.push(x), [(q, 1), (q, 2)])
arr = []
while not q.empty():
arr.append(q.pop())
assert (set([1, 2]) == set(arr))
def func(q: mp.Queue, pause: mp.Event
): # finish_table True will not read exit until all done
print(f'Process {os.getpid()}')
idle = True # A flag for whether player is actively playing
starttime = time.time()
while True: # global process loop
try: # non-blocking Queue get handler
val = q.get(block=False) # get from queue
# do when got from queue
if val == 'exit': # exit code for process
break
if type(val) == float:
print('time', val)
flag = False
idle = False
for item in val: # player loop
print(item, time.time() - starttime) # message event
# message delay
endtime = time.time() + 1
while pause.is_set():
continue
while time.time() < endtime:
if not q.empty(): # if new item in queue
flag = True # set exit flag
break
if flag: # play exit condition
idle = True
break
idle = True
except Exception: # handle Queue.Empty
pass # Do nothing if no input
def run(self, func, arglist):
# TODO: Generalise this a bit maybe?
# We know len(arglist) == self.workers in pretty much all cases
pool = []
queue = Queue()
for idx, split in enumerate(arglist):
proc = Process(target=self._eval, args=(idx, func, split, queue))
pool.append(proc)
proc.start()
for proc in pool:
proc.join()
yss = [[]] * len(arglist)
while not queue.empty():
idx, ys = queue.get()
yss[idx] = ys
ret = [y for ys in yss for y in ys]
#from code import interact
#interact(local=locals())
return ret
class DataStreamRecorder(Process):
def __init__(self,
name,
data_sampler_method,
cache_path=None,
save_every=50):
""" Initializes a DataStreamRecorder
Parameters
----------
name : string
User-friendly identifier for this data stream
data_sampler_method : function
Method to call to retrieve data
"""
Process.__init__(self)
self._data_sampler_method = data_sampler_method
self._has_set_sampler_params = False
self._recording = False
self._name = name
self._cmds_q = Queue()
self._data_qs = [Queue()]
self._ok_q = None
self._tokens_q = None
self._save_every = save_every
self._cache_path = cache_path
self._saving_cache = cache_path is not None
if self._saving_cache:
self._save_path = os.path.join(cache_path, self.name)
if not os.path.exists(self._save_path):
os.makedirs(self._save_path)
self._start_data_segment = 0
self._cur_data_segment = 0
self._saving_ps = []
def run(self):
setproctitle('python.DataStreamRecorder.{0}'.format(self._name))
try:
logging.debug("Starting data recording on {0}".format(self.name))
self._tokens_q.put(("return", self.name))
while True:
if not self._cmds_q.empty():
cmd = self._cmds_q.get()
if cmd[0] == 'stop':
break
elif cmd[0] == 'pause':
self._recording = False
if self._saving_cache:
self._save_cache(self._cur_data_segment)
self._cur_data_segment += 1
self._data_qs.append(Queue())
elif cmd[0] == 'reset_data_segment':
self._start_data_segment = self._cur_data_segment
elif cmd[0] == 'resume':
self._recording = True
elif cmd[0] == 'save':
self._save_data(cmd[1], cmd[2], cmd[3])
elif cmd[0] == 'params':
self._args = cmd[1]
self._kwargs = cmd[2]
if self._recording and not self._ok_q.empty():
timestamp = self._ok_q.get()
self._tokens_q.put(("take", self.name))
data = self._data_sampler_method(*self._args,
**self._kwargs)
cur_data_q = self._data_qs[self._cur_data_segment]
if self._saving_cache and cur_data_q.qsize(
) == self._save_every:
self._save_cache(self._cur_data_segment)
cur_data_q = Queue()
self._data_qs.append(cur_data_q)
self._cur_data_segment += 1
cur_data_q.put((timestamp, data))
self._tokens_q.put(("return", self.name))
except KeyboardInterrupt:
logging.debug("Shutting down data streamer on {0}".format(
self.name))
sys.exit(0)
def _extract_q(self, i):
q = self._data_qs[i]
vals = []
while q.qsize() > 0:
vals.append(q.get())
self._data_qs[i] = None
del q
return vals
def _save_data(self, path, cb, concat):
if not os.path.exists(path):
os.makedirs(path)
target_filename = os.path.join(path, "{0}.jb".format(self.name))
if self._saving_cache:
while True in [p.is_alive() for p in self._saving_ps]:
sleep(1e-3)
p = Process(target=_caches_to_file,
args=(self._save_path, self._start_data_segment,
self._cur_data_segment, target_filename, cb,
concat))
p.start()
self._start_data_segment = self._cur_data_segment
else:
data = self._extract_q(0)
p = Process(target=_dump, args=(data, target_filename, cb))
p.start()
def _save_cache(self, i):
if not self._save_cache:
raise Exception(
"Cannot save cache if no cache path was specified.")
logging.debug("Saving cache for {0} block {1}".format(
self.name, self._cur_data_segment))
data = self._extract_q(i)
p = Process(target=_dump_cache,
args=(data,
os.path.join(self._save_path, "{0}.jb".format(
self._cur_data_segment)), self.name,
self._cur_data_segment))
p.start()
self._saving_ps.append(p)
def _start_recording(self, *args, **kwargs):
""" Starts recording
Parameters
----------
*args : any
Ordinary args used for calling the specified data sampler method
**kwargs : any
Keyword args used for calling the specified data sampler method
"""
while not self._cmds_q.empty():
self._cmds_q.get_nowait()
while not self._data_qs[self._cur_data_segment].empty():
self._data_qs[self._cur_data_segment].get_nowait()
self._args = args
self._kwargs = kwargs
self._recording = True
self.start()
@property
def name(self):
return self._name
def _set_qs(self, ok_q, tokens_q):
self._ok_q = ok_q
self._tokens_q = tokens_q
def _flush(self):
""" Returns a list of all current data """
if self._recording:
raise Exception("Cannot flush data queue while recording!")
if self._saving_cache:
logging.warn(
"Flush when using cache means unsaved data will be lost and not returned!"
)
self._cmds_q.put(("reset_data_segment", ))
else:
data = self._extract_q(0)
return data
def save_data(self, path, cb=_NULL, concat=True):
if self._recording:
raise Exception("Cannot save data while recording!")
self._cmds_q.put(("save", path, cb, concat))
def _stop(self):
""" Stops recording. Returns all recorded data and their timestamps. Destroys recorder process."""
self._pause()
self._cmds_q.put(("stop", ))
try:
self._recorder.terminate()
except Exception:
pass
self._recording = False
def _pause(self):
""" Pauses recording """
self._cmds_q.put(("pause", ))
self._recording = False
def _resume(self):
""" Resumes recording """
self._cmds_q.put(("resume", ))
self._recording = True
def change_data_sampler_params(self, *args, **kwargs):
""" Chanes args and kwargs for data sampler method
Parameters
----------
*args : any
Ordinary args used for calling the specified data sampler method
**kwargs : any
Keyword args used for calling the specified data sampler method
"""
self._cmds_q.put(('params', args, kwargs))
class BaseAggregator(object):
"""Base class for the data aggregator interface. This class is used
alongside the BaseListener class to spawn an aggregator process that
combines data from multiple crawl processes. The BaseAggregator class
manages the child listener process.
Parameters
----------
manager_params : dict
TaskManager configuration parameters
browser_params : list of dict
List of browser configuration dictionaries"""
__metaclass__ = abc.ABCMeta
def __init__(self, manager_params, browser_params):
self.manager_params = manager_params
self.browser_params = browser_params
self.listener_address = None
self.listener_process = None
self.status_queue = Queue()
self.shutdown_queue = Queue()
self._last_status = None
self._last_status_received = None
self.logger = logging.getLogger('openwpm')
@abc.abstractmethod
def save_configuration(self, openwpm_version, browser_version):
"""Save configuration details to the database"""
@abc.abstractmethod
def get_next_visit_id(self):
"""Return a unique visit ID to be used as a key for a single visit"""
@abc.abstractmethod
def get_next_crawl_id(self):
"""Return a unique crawl ID used as a key for a browser instance"""
def get_most_recent_status(self):
"""Return the most recent queue size sent from the listener process"""
# Block until we receive the first status update
if self._last_status is None:
return self.get_status()
# Drain status queue until we receive most recent update
while not self.status_queue.empty():
self._last_status = self.status_queue.get()
self._last_status_received = time.time()
# Check last status signal
if (time.time() - self._last_status_received) > STATUS_TIMEOUT:
raise RuntimeError(
"No status update from DataAggregator listener process "
"for %d seconds." % (time.time() - self._last_status_received))
return self._last_status
def get_status(self):
"""Get listener process status. If the status queue is empty, block."""
try:
self._last_status = self.status_queue.get(block=True,
timeout=STATUS_TIMEOUT)
self._last_status_received = time.time()
except queue.Empty:
raise RuntimeError(
"No status update from DataAggregator listener process "
"for %d seconds." % (time.time() - self._last_status_received))
return self._last_status
def launch(self, listener_process_runner, *args):
"""Launch the aggregator listener process"""
args = (self.manager_params, self.status_queue,
self.shutdown_queue) + args
self.listener_process = Process(target=listener_process_runner,
args=args)
self.listener_process.daemon = True
self.listener_process.start()
self.listener_address = self.status_queue.get()
def shutdown(self):
""" Terminate the aggregator listener process"""
self.logger.debug(
"Sending the shutdown signal to the %s listener process..." %
type(self).__name__)
self.shutdown_queue.put(SHUTDOWN_SIGNAL)
start_time = time.time()
self.listener_process.join(300)
self.logger.debug("%s took %s seconds to close." %
(type(self).__name__, str(time.time() - start_time)))
self.listener_address = None
self.listener_process = None
class BaseAggregator(object):
"""Base class for the data aggregator interface. This class is used
alongside the BaseListener class to spawn an aggregator process that
combines data from multiple crawl processes. The BaseAggregator class
manages the child listener process.
Parameters
----------
manager_params : dict
TaskManager configuration parameters
browser_params : list of dict
List of browser configuration dictionaries"""
__metaclass__ = abc.ABCMeta
def __init__(self, manager_params, browser_params):
self.manager_params = manager_params
self.browser_params = browser_params
self.logger = loggingclient(*manager_params['logger_address'])
self.listener_address = None
self.listener_process = None
self.status_queue = Queue()
self.shutdown_queue = Queue()
self._last_status = None
self._last_status_received = None
@abc.abstractmethod
def save_configuration(self, openwpm_version, browser_version):
"""Save configuration details to the database"""
@abc.abstractmethod
def get_next_visit_id(self):
"""Return a unique visit ID to be used as a key for a single visit"""
@abc.abstractmethod
def get_next_crawl_id(self):
"""Return a unique crawl ID used as a key for a browser instance"""
def get_most_recent_status(self):
"""Return the most recent queue size sent from the listener process"""
# Block until we receive the first status update
if self._last_status is None:
return self.get_status()
# Drain status queue until we receive most recent update
while not self.status_queue.empty():
self._last_status = self.status_queue.get()
self._last_status_received = time.time()
# Check last status signal
if (time.time() - self._last_status_received) > STATUS_TIMEOUT:
raise RuntimeError(
"No status update from DataAggregator listener process "
"for %d seconds." % (time.time() - self._last_status_received)
)
return self._last_status
def get_status(self):
"""Get listener process status. If the status queue is empty, block."""
try:
self._last_status = self.status_queue.get(
block=True, timeout=STATUS_TIMEOUT)
self._last_status_received = time.time()
except queue.Empty:
raise RuntimeError(
"No status update from DataAggregator listener process "
"for %d seconds." % (time.time() - self._last_status_received)
)
return self._last_status
def launch(self, listener_process_runner, *args):
"""Launch the aggregator listener process"""
args = (self.manager_params, self.status_queue,
self.shutdown_queue) + args
self.listener_process = Process(
target=listener_process_runner,
args=args
)
self.listener_process.daemon = True
self.listener_process.start()
self.listener_address = self.status_queue.get()
def shutdown(self):
""" Terminate the aggregator listener process"""
self.logger.debug(
"Sending the shutdown signal to the %s listener process..." %
type(self).__name__
)
self.shutdown_queue.put(SHUTDOWN_SIGNAL)
start_time = time.time()
self.listener_process.join(300)
self.logger.debug(
"%s took %s seconds to close." % (
type(self).__name__,
str(time.time() - start_time)
)
)
self.listener_address = None
self.listener_process = None
class StorageControllerHandle:
"""This class contains all methods relevant for the TaskManager
to interact with the StorageController
"""
def __init__(
self,
structured_storage: StructuredStorageProvider,
unstructured_storage: Optional[UnstructuredStorageProvider],
) -> None:
self.listener_address: Optional[Tuple[str, int]] = None
self.listener_process: Optional[Process] = None
self.status_queue = Queue()
self.completion_queue = Queue()
self.shutdown_queue = Queue()
self._last_status = None
self._last_status_received: Optional[float] = None
self.logger = logging.getLogger("openwpm")
self.storage_controller = StorageController(
structured_storage,
unstructured_storage,
status_queue=self.status_queue,
completion_queue=self.completion_queue,
shutdown_queue=self.shutdown_queue,
)
def get_next_visit_id(self) -> VisitId:
"""Generate visit id as randomly generated positive integer less than 2^53.
Parquet can support integers up to 64 bits, but Javascript can only
represent integers up to 53 bits:
https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Number/MAX_SAFE_INTEGER
Thus, we cap these values at 53 bits.
"""
return VisitId(random.getrandbits(53))
def get_next_browser_id(self) -> BrowserId:
"""Generate crawl id as randomly generated positive 32bit integer
Note: Parquet's partitioned dataset reader only supports integer
partition columns up to 32 bits.
"""
return BrowserId(random.getrandbits(32))
def save_configuration(
self,
manager_params: ManagerParamsInternal,
browser_params: List[BrowserParamsInternal],
openwpm_version: str,
browser_version: str,
) -> None:
assert self.listener_address is not None
sock = DataSocket(self.listener_address)
task_id = random.getrandbits(32)
sock.store_record(
TableName("task"),
INVALID_VISIT_ID,
{
"task_id": task_id,
"manager_params": manager_params.to_json(),
"openwpm_version": openwpm_version,
"browser_version": browser_version,
},
)
# Record browser details for each browser
for browser_param in browser_params:
sock.store_record(
TableName("crawl"),
INVALID_VISIT_ID,
{
"browser_id": browser_param.browser_id,
"task_id": task_id,
"browser_params": browser_param.to_json(),
},
)
sock.finalize_visit_id(INVALID_VISIT_ID, success=True)
def launch(self) -> None:
"""Starts the storage controller"""
self.storage_controller = Process(
name="StorageController",
target=StorageController.run,
args=(self.storage_controller, ),
)
self.storage_controller.daemon = True
self.storage_controller.start()
self.listener_address = self.status_queue.get()
def get_new_completed_visits(self) -> List[Tuple[int, bool]]:
"""
Returns a list of all visit ids that have been processed since
the last time the method was called and whether or not they
ran successfully.
This method will return an empty list in case no visit ids have
been processed since the last time this method was called
"""
finished_visit_ids = list()
while not self.completion_queue.empty():
finished_visit_ids.append(self.completion_queue.get())
return finished_visit_ids
def shutdown(self, relaxed: bool = True) -> None:
"""Terminate the storage controller process"""
assert isinstance(self.storage_controller, Process)
self.logger.debug(
"Sending the shutdown signal to the Storage Controller...")
self.shutdown_queue.put((SHUTDOWN_SIGNAL, relaxed))
start_time = time.time()
self.storage_controller.join(300)
self.logger.debug("%s took %s seconds to close." %
(type(self).__name__, str(time.time() - start_time)))
def get_most_recent_status(self) -> int:
"""Return the most recent queue size sent from the Storage Controller process"""
# Block until we receive the first status update
if self._last_status is None:
return self.get_status()
# Drain status queue until we receive most recent update
while not self.status_queue.empty():
self._last_status = self.status_queue.get()
self._last_status_received = time.time()
# Check last status signal
if (time.time() - self._last_status_received) > STATUS_TIMEOUT:
raise RuntimeError(
"No status update from the storage controller process "
"for %d seconds." % (time.time() - self._last_status_received))
return self._last_status
def get_status(self) -> int:
"""Get listener process status. If the status queue is empty, block."""
try:
self._last_status = self.status_queue.get(block=True,
timeout=STATUS_TIMEOUT)
self._last_status_received = time.time()
except queue.Empty:
assert self._last_status_received is not None
raise RuntimeError(
"No status update from the storage controller process "
"for %d seconds." % (time.time() - self._last_status_received))
assert isinstance(self._last_status, int)
return self._last_status
class BaseAggregator:
"""Base class for the data aggregator interface. This class is used
alongside the BaseListener class to spawn an aggregator process that
combines data from multiple crawl processes. The BaseAggregator class
manages the child listener process.
Parameters
----------
manager_params : ManagerParamsInternal
TaskManager configuration parameters
browser_params : list of BrowserParamsInternal
List of browser configuration class<BrowserParams>"""
__metaclass__ = abc.ABCMeta
def __init__(
self,
manager_params: ManagerParamsInternal,
browser_params: List[BrowserParamsInternal],
):
self.manager_params = manager_params
self.browser_params = browser_params
self.listener_address = None
self.listener_process = None
self.status_queue = Queue()
self.completion_queue = Queue()
self.shutdown_queue = Queue()
self._last_status = None
self._last_status_received = None
self.logger = logging.getLogger("openwpm")
@abc.abstractmethod
def save_configuration(self, openwpm_version, browser_version):
"""Save configuration details to the database"""
@abc.abstractmethod
def get_next_visit_id(self):
"""Return a unique visit ID to be used as a key for a single visit"""
@abc.abstractmethod
def get_next_browser_id(self):
"""Return a unique crawl ID used as a key for a browser instance"""
def get_most_recent_status(self):
"""Return the most recent queue size sent from the listener process"""
# Block until we receive the first status update
if self._last_status is None:
return self.get_status()
# Drain status queue until we receive most recent update
while not self.status_queue.empty():
self._last_status = self.status_queue.get()
self._last_status_received = time.time()
# Check last status signal
if (time.time() - self._last_status_received) > STATUS_TIMEOUT:
raise RuntimeError(
"No status update from DataAggregator listener process "
"for %d seconds." % (time.time() - self._last_status_received))
return self._last_status
def get_status(self):
"""Get listener process status. If the status queue is empty, block."""
try:
self._last_status = self.status_queue.get(block=True,
timeout=STATUS_TIMEOUT)
self._last_status_received = time.time()
except queue.Empty:
raise RuntimeError(
"No status update from DataAggregator listener process "
"for %d seconds." % (time.time() - self._last_status_received))
return self._last_status
def get_new_completed_visits(self) -> List[Tuple[int, bool]]:
"""
Returns a list of all visit ids that have been processed since
the last time the method was called and whether or not they
have been interrupted.
This method will return an empty list in case no visit ids have
been processed since the last time this method was called
"""
finished_visit_ids = list()
while not self.completion_queue.empty():
finished_visit_ids.append(self.completion_queue.get())
return finished_visit_ids
def launch(self, listener_process_runner, *args):
"""Launch the aggregator listener process"""
args = ((self.status_queue, self.completion_queue,
self.shutdown_queue), ) + args
self.listener_process = Process(target=listener_process_runner,
args=args)
self.listener_process.daemon = True
self.listener_process.start()
self.listener_address = self.status_queue.get()
def shutdown(self, relaxed: bool = True):
""" Terminate the aggregator listener process"""
self.logger.debug(
"Sending the shutdown signal to the %s listener process..." %
type(self).__name__)
self.shutdown_queue.put((SHUTDOWN_SIGNAL, relaxed))
start_time = time.time()
self.listener_process.join(300)
self.logger.debug("%s took %s seconds to close." %
(type(self).__name__, str(time.time() - start_time)))
self.listener_address = None
self.listener_process = None
class MRClient:
map = StandardOperation("map")
filter = StandardOperation("filter")
reduce = StandardOperation("reduce")
flatmap = StandardOperation("flatmap")
def __init__(self, num_cores):
self.num_cores = num_cores
self.names = set()
self.global_queue = Queue()
self.pool = []
self.channels = []
self.__terminated = False
atexit.register(self.__del__)
queues = [Queue() for _ in range(num_cores)]
for i in range(num_cores):
pipe_master, pipe_slave = Pipe()
state = Value('i', 0)
process = MRServer(i, queues, pipe_slave, state, self.global_queue)
self.channels.append(Channel(queues[i], pipe_master, state))
self.pool.append(process)
process.start()
def wait(self, queue=True):
while not self.idle(queue):
sleep(0.01)
def idle(self, queue=True):
if queue:
return self.global_queue.empty() and all(map(lambda x: x.state.value==0 and x.queue.empty(), self.channels))
else:
return all(map(lambda x: x.state.value==0, self.channels))
def _register_name(self, name):
i = 0
while 1:
name = "{name}{i}".format(name=name, i=i)
if name in self.names:
i += 1
else:
self.names.add(name)
return name
@contextmanager
def acquire(self, queue=True):
"""
Wait until the processes are idle
"""
self.wait(queue)
yield
def _send(self, item):
"""Send commands to processes"""
for channel in self.channels:
channel.pipe.send(item)
def _recv(self):
"""
Receive info from processes.
"""
for channel in self.channels:
yield channel.pipe.recv()
def distribute(self, name, data):
"""
Distribute the data to processes.
Parameters
----------
name: str
Store the data by a key `name`.
data: Iterable
the data to be distributed.
"""
name = self._register_name(name)
if isinstance(data, dict):
data = data.items()
i = 0
n = self.num_cores
with self.acquire():
for batch in bufferize(data, CONFIG.BUFFER_SIZE):
self.channels[i % n].queue.put(batch)
i += 1
self._send({'action': "add_dataset", 'name': name})
return Distributed(self, name)
def copy(self, dataset):
"""A shallow copy of the current dataset"""
name = self._register_name(dataset.name)
with self.acquire():
self._send({'action': 'copy', 'src': dataset.name, 'dest': name})
return Distributed(self, name)
def reduce2(self, func, dataset, inplace=True):
"""
Step 1: Reduce in seperate processes;
Step 2: Partition;
Step 3: Reduce in seperate processes again.
"""
data = dataset.reduce(func, inplace=inplace) \
.partition() \
.reduce(func,inplace=True)
return data
def partition(self, dataset, by=None):
"""
Redistribute objects to different processes according to
their key.
Parameters
----------
by: object -> Union[str, num]
a function that maps object to a key,
this key decides which process to put
the data on.
"""
with self.acquire():
self._send({'action': 'partition', 'name': dataset.name, 'by': by})
with self.acquire(queue=False):
self._send({'action': "add_dataset", 'name': dataset.name})
return dataset
def merge(self, data):
"""
Merges multiple datasets to a new one.
"""
new_name = "/".join(["merge"] + [d.name for d in data])
name = self._register_name(new_name)
self._send({'action': 'merge',
'src': [d.name for d in data],
'dest': name})
return Distributed(self, name)
def count(self, dataset):
"""
Returns
-------
The length of the dataset
"""
n = 0
with self.acquire():
self._send({'action': 'count', 'name': dataset.name})
for c in self._recv():
n += c
return n
def remove(self, dataset):
"""
Remove the data from processes.
"""
with self.acquire():
self._send({'action': 'remove_dataset', 'name': dataset.name})
self.names.remove(dataset.name)
def exists(self, dataset):
"""
Returns
-------
Whether the data exists on the processes, or
it it removed already.
"""
return dataset.name in self.names
def terminate(self):
"""
Terminate the processes.
"""
with self.acquire():
self._send({'action': 'terminate'})
self.__terminated = True
def collect(self, dataset, remove=False):
"""
Collect the data from processes to the client.
"""
with self.acquire():
self._send({'action': 'collect', 'name': dataset.name})
data = []
for item in robust_recv(self.global_queue, retries=3):
data.extend(item)
if remove:
self.remove(dataset)
return data
def __del__(self):
if not self.__terminated:
self.terminate()
class AmpDecorator(Amplifier):
"""This class 'decorates' the Low-Level Amplifier classes with
Network-Marker and Save-To-File functionality.
You use it by decorating (not as in Python-Decorator, but in the GoF
sense) the low level amplifier class you want to use::
import libmushu
from libmushu.ampdecorator import AmpDecorator
from libmushu.driver.randomamp import RandomAmp
amp = Ampdecorator(RandomAmp)
Waring: The network marker timings on Windows have a resolution of
10ms-15ms. On Linux the resolution is 1us. This is due to
limitations of Python's time.time method, or rather a Windows
specific issue.
There exists currently no precise timer, providing times which are
comparable between two running processes on Windows. The performance
counter provided on Windows, has a much better resolution but is
relative to the processes start time and it drifts (1s per 100s), so
it is only precise for a relatively short amount of time.
If a higher precision is needed one has to replace the time.time
calls with something which provides a better precision. For example
one could create a third process which provides times or regularly
synchronize both processes with the clock synchronization algorithm
as described here:
http://en.wikipedia.org/wiki/Network_Time_Protocol
Alternatively one could use `timeGetTime` from Windows' Multi Media
library, which is tunable via `timeBeginPeriod` and provides a
precision of 1-2ms. Apparently this is the way Chrome and many
others do it.::
from __future__ import division
from ctypes import windll
import time
timeBeginPeriod = windll.winmm.timeBeginPeriod
timeEndPeriod = windll.winmm.timeEndPeriod
timeGetTime = windll.winmm.timeGetTime
if __name__ == '__main__':
# wrap the code that needs high precision in timeBegin- and
# timeEndPeriod with the same parameter. The parameter is
# the interval in ms you want as precision. Usually the
# minimum value allowed is 1 (best).
timeBeginPeriod(1)
times = []
t_start = time.time()
while time.time() < (time.time() + 1):
times.append(timeGetTime())
times = sorted(list(set(times)))
print(1000 / len(times))
timeEndPeriod(1)
"""
def __init__(self, ampcls):
self.amp = ampcls()
self.write_to_file = False
@property
def presets(self):
return self.amp.presets
def start(self, filename=None):
# prepare files for writing
self.write_to_file = False
if filename is not None:
self.write_to_file = True
filename_marker = filename + '.marker'
filename_eeg = filename + '.eeg'
filename_meta = filename + '.meta'
for filename in filename_marker, filename_eeg, filename_meta:
if os.path.exists(filename):
logger.error('A file "%s" already exists, aborting.' %
filename)
raise Exception
self.fh_eeg = open(filename_eeg, 'wb')
self.fh_marker = open(filename_marker, 'w')
self.fh_meta = open(filename_meta, 'w')
# write meta data
meta = {
'Channels': self.amp.get_channels(),
'Sampling Frequency': self.amp.get_sampling_frequency(),
'Amp': str(self.amp)
}
json.dump(meta, self.fh_meta, indent=4)
# start the marker server
self.marker_queue = Queue()
self.tcp_reader_running = Event()
self.tcp_reader_running.set()
tcp_reader_ready = Event()
self.tcp_reader = Process(target=marker_reader,
args=(self.marker_queue,
self.tcp_reader_running,
tcp_reader_ready))
self.tcp_reader.start()
logger.debug('Waiting for marker server to become ready...')
tcp_reader_ready.wait()
logger.debug('Marker server is ready.')
# zero the sample counter
self.received_samples = 0
# start the amp
self.amp.start()
def stop(self):
# stop the amp
self.amp.stop()
# stop the marker server
self.tcp_reader_running.clear()
logger.debug('Waiting for marker server process to stop...')
self.tcp_reader.join()
logger.debug('Marker server process stopped.')
# close the files
if self.write_to_file:
logger.debug('Closing files.')
for fh in self.fh_eeg, self.fh_marker, self.fh_meta:
fh.close()
def configure(self, **kwargs):
self.amp.configure(**kwargs)
def get_data(self):
"""Get data from the amplifier.
This method is supposed to get called as fast as possible (i.e
hundreds of times per seconds) and returns the data and the
markers.
Returns
-------
data : 2darray
a numpy array (time, channels) of the EEG data
markers : list of (float, str)
a list of markers. Each element is a tuple of timestamp and
string. The timestamp is the time in ms relative to the
onset of the block of data. Note that negative values are
*allowed* as well as values bigger than the length of the
block of data returned. That is to be interpreted as a
marker from the last block and a marker for a future block
respectively.
"""
# get data and marker from underlying amp
data, marker = self.amp.get_data()
t = time.time()
# length in sec of the new block according to #samples and fs
block_duration = len(data) / self.amp.get_sampling_frequency()
# abs time of start of the block
t0 = t - block_duration
# duration of all blocks in ms except the current one
duration = 1000 * self.received_samples / self.amp.get_sampling_frequency(
)
# merge markers
tcp_marker = []
while not self.marker_queue.empty():
m = self.marker_queue.get()
m[0] = (m[0] - t0) * 1000
tcp_marker.append(m)
marker = sorted(marker + tcp_marker)
# save data to files
if self.write_to_file:
for m in marker:
self.fh_marker.write("%f %s\n" % (duration + m[0], m[1]))
self.fh_eeg.write(struct.pack("f" * data.size, *data.flatten()))
self.received_samples += len(data)
if len(data) == 0 and len(marker) > 0:
logger.error(
'Received marker but no data. This is an error, the amp should block on get_data until data is available. Marker timestamps will be unreliable.'
)
return data, marker
def get_channels(self):
return self.amp.get_channels()
def get_sampling_frequency(self):
return self.amp.get_sampling_frequency()
