def __init__(self, TAlgorithm, datahandler, Broker, Portfolio, name):
"""Initialize a new Stock Simulation
Parameters:
TAlgorithm - A Class which extends TradingAlgorithm. Will be run
as Trader in this simulation. Look at TradingAlgorithm
for information about what this class should contain
DataHandler - An instance of Class extending DataHandler. Look
at DataHandler class about how this classes should look like
Broker - A Class which will extend the Broker Class. Look at the
Broker class about how this classes should look like
Portfolio - a Reference to the Portfolio class
string name - name of the simulation. has no use yet
"""
self.name = name
self.game_end = Signal()
self.data = datahandler
start_port = defaultdict(lambda: 0)
start_port['EUR'] = 50000
self.broker = Broker(self)
self.talgo_port = Portfolio(start_port)
self.talgo = TAlgorithm(self, self.broker, self.talgo_port)
self.date = None
def __init__(self,
market,
limit_storage_time=timedelta(days=10),
lag_time=timedelta(milliseconds=500)):
"""Initialize the broker
Parameters:
Market market - a market instance
timedelta limit_storage_time - a time span which limit
orders will be saved maximum
timdelta lag_time - the time which it will take to broker
to actually set a market order
"""
log.debug("Initiating Broker Object")
self.market = market
self.market.data.tick.registerObserver(self.market_tick)
self.market.data.time_change.registerObserver(self.market_tick)
self._orders = {}
self._limit_storage_time = limit_storage_time
self._market_order_delay = lag_time
self.order_fill = Signal()
self.order_delete = Signal()
self._newest_order_id = -1
def inference(self, input_, orders=None, *args, **kwargs):
# Sanity check
if not isinstance(input_, Signal):
raise TypeError('!! Input must be an instance of Signal')
if self.order_lock is not None: orders = self.order_lock
if orders is not None: orders = self._check_orders(orders)
# Calculate
y = np.zeros_like(input_)
if orders is None: pool = self.indices_full
else:
pool = []
for order in orders:
pool += list(
self.kernels.get_homogeneous_indices(
order, self.memory_depth[order - 1], symmetric=False))
for lags in pool:
# lags = (\tau_1, \tau_2, \cdots, \tau_k)
# prod = h_k(\tau_1, \cdots, \tau_k) * \prod_{i=1}^k x[n-\tau_i]
prod = self.kernels[lags]
if prod == 0: continue
for lag in lags:
prod *= self._delay(input_, lag)
y += prod
output = Signal(y)
output.__array_finalize__(input_)
return output
def __init__(self, copy_from=None, FS=96000):
self.tempdir = None
self.tempdir_keep = False
self.backward_euler = False
self.solver = None
self.build_verbose = False
self.module_id = None
self.module_name = None
self.module_desc = None
self.c_datatype = "double"
self.table_datatype = "double" #"float"
self.pre_filter = None
self.post_filter = None
self.code_generator = None
self.plugindef = None
self.build_script = None
self.resample = True
self.FS = FS
self.use_sim = SIM_C
self.sig = Signal()
self._clear()
self._rmtree = shutil.rmtree
self.subcircuits = {}
if copy_from is not None:
self.backward_euler = copy_from.backward_euler
self.solver = copy_from.solver
self.FS = copy_from.FS
self.S = copy_from.S
self.V = copy_from.V
def __init__(self, directory, fxcodes, time = None):
"""Initialize the Handler.
Parameters:
string directory - the directory which the .csv files lay in
list fxcodes - a list of strings which represent currency
pairs, e.g. ['EURUSD', 'USDJPY']. the Handler will look
into the directory provided by `directory`, if it can
find files called `fxcode`.csv (where fxcode is an
item in fxcodes), e.g. EURUSD.csv
datetime time - the time at which the simulation should
start at, if not provided, it just uses the first time
available
"""
log.debug("Initiating CSVForexTicksHandler Object")
self._directory = directory
self._data = defaultdict(None)
self.fxcodes = fxcodes
self.data_available = True
self.time_change = Signal()
self.tick = Signal()
self._comb_index = None
self._load_files(self._directory, self.fxcodes)
if None == time:
self._time = self._comb_index[0]
else:
self._time = time
self._start_time = self._time
def testCantConnectMethodTwice(self):
"""Actually testing an implementation limitation..."""
receiver = Receiver()
signal = Signal()
signal.connect(receiver.slot)
signal.connect(receiver.slot)
self.assertEqual(len(signal), 1)
def setUp(self):
self.sig_function = Signal()
self.sig_method = Signal()
self.sig_function.connect(onSignalFunction)
global signal_calls
signal_calls = []
def setUp(self):
self.sig_function = Signal()
self.sig_method = Signal()
self.sig_function.connect(onSignalFunction)
global signal_calls
signal_calls = []
def __init__(self):
"""
"""
self.__label = "Initial renaming of new images .... "
self.startSignal = Signal()
self.refreshSignal = Signal()
self.finishSignal = Signal()
self.NbrJobsSignal = Signal()
def testDisconnect(self):
receiver = Receiver()
signal = Signal()
signal.connect(receiver.slot)
signal.disconnect(receiver.slot)
signal()
self.assertEqual(receiver.count, 0)
self.assertEqual(len(signal), 0)
def __init__(self):
"""
"""
self.__label = "Un moment..."
self.startSignal = Signal()
self.refreshSignal = Signal()
self.finishSignal = Signal()
self.NbrJobsSignal = Signal()
def testTwoSlots(self):
receiver1 = Receiver()
receiver2 = Receiver()
signal = Signal()
signal.connect(receiver1.slot)
signal.connect(receiver2.slot)
signal()
self.assertEqual(receiver1.count, 1)
self.assertEqual(receiver2.count, 1)
def __init__(self):
self.states = ["INIT", "RUN_COMMANDS"]
self.state = self.states[0]
self.num_frames = 0
self.added_cmd_idx = 0
self.signal_capacity = 15
self.signal = Signal(self.signal_capacity)
self.commands = Commands()
self.unlock_frame_num = 0
def __init__(self, profileID=None, nameAndDecInfo=None, profileData=None):
self.profileID = profileID
self.nameAndDecInfo = nameAndDecInfo
self.originalProfileData = profileData
self.groupsBySettingID = None
self.remoteSvc = sm.RemoteSvc(REMOTE_SERVICE_NAME)
self.on_profile_saved = Signal()
self.on_groups_added = Signal()
self.on_group_changed = Signal()
self.on_profile_value_changed = Signal()
def inference(self, input_, **kwargs):
if not self.nn.built:
raise AssertionError('!! Model has not been built yet')
mlp_input = self._gen_mlp_input(input_)
tfinput = TFData(mlp_input)
output = self.nn.predict(tfinput, **kwargs).flatten()
output = Signal(output)
output.__array_finalize__(input_)
return output
def import_data():
"""Import your data here"""
# Import your data as np array
np_x = None
np_y = None
# Wrap your data up into a Signal
fs = 1000 # sampling frequency
x = Signal(np_x, fs=fs)
y = Signal(np_y, fs=fs)
# Return
return x, y
def inference(self, input_, orders=None, *args, **kwargs):
# Sanity check
if not isinstance(input_, Signal):
raise TypeError('!! Input must be an instance of Signal')
y = np.zeros_like(input_)
orders = range(self.degree + 1) if orders is None else [orders]
for n in orders:
y += self.G_n(n, input_)
output = Signal(y)
output.__array_finalize__(input_)
return output
def G_n(self, n, x):
# Sanity check
if n < 0:
raise ValueError(
'!! degree of any Wiener operator must be non-negative')
if n == 0: y_n = self.kernels[()] * np.ones_like(x)
else:
y_n = np.zeros_like(x)
for i in range(n // 2 + 1):
y_n += self.G_n_i(n, i, x)
output = Signal(y_n)
output.__array_finalize__(x)
return output
def separate_interp(self, input_, alphas=None, max_order=4, verbose=False):
# Sanity check
if not isinstance(input_, Signal):
raise TypeError('!! Input must be an instance of Signal')
if alphas is None:
alphas = []
for order in range(max_order):
alphas.append((-1)**order * (1 + order * 0.25))
if verbose: print('>> alphas = {}'.format(alphas))
N = len(alphas)
# Generate matrix R
R = np.zeros(shape=(N, input_.size))
for i in range(N):
R[i] = self(alphas[i] * input_)
# Generate matrix A
A = np.zeros(shape=(N, N))
for r in range(N):
for c in range(N):
A[r, c] = alphas[r]**(c + 1)
inv_A = np.linalg.inv(A)
if verbose:
print('>> ||inv(A)|| = {:.4f}'.format(np.linalg.norm(inv_A)))
# Separate
results = []
yn = np.matmul(inv_A, R)
for order in range(N):
results.append(Signal(yn[order]))
results[order].__array_finalize__(input_)
return results
def __init__(self, settingID, groupID, value=0.0):
self.settingInfo = structures.SETTING_OBJECT_BY_SETTINGID[settingID]
self.settingType = self.settingInfo.valueType
self.settingID = settingID
self.groupID = groupID
self.value = value
self.on_profile_value_changed = Signal()
def __init__(self,
itemID,
solarsystemID,
typeID,
ownerID,
structureServices=None,
profileID=None,
currentSchedule=0,
nextSchedule=0,
state=None,
unanchoring=None,
fuelExpires=None):
self.structureInfo = KeyVal(itemID=itemID,
solarsystemID=solarsystemID,
typeID=typeID,
ownerID=ownerID,
structureServices=structureServices,
profileID=profileID,
currentSchedule=currentSchedule,
nextSchedule=nextSchedule,
state=state,
unanchoring=unanchoring,
fuelExpires=fuelExpires)
self.systemAndStructureName = None
self.requiredHours = None
self.on_structure_state_changed = Signal()
def response(self, input_, **kwargs):
# Calculate
y = np.zeros_like(input_)
pool = self.kernels.params.keys()
for lags in pool:
assert isinstance(lags, tuple)
# lags = (\tau_1, \tau_2, \cdots, \tau_k)
# prod = h_k(\tau_1, \cdots, \tau_k) * \prod_{i=1}^k x[n-\tau_i]
prod = self.kernels[lags]
for lag in lags:
prod *= self._delay(input_, lag)
y += prod
output = Signal(y)
output.__array_finalize__(input_)
return output
def build_vbi_signals(price_volume_df, percent_change_price, percent_change_volume, transaction_currency, counter_currency,
source, resample_period):
# build signals
all_buy_signals = []
first_cross_buy_signals = []
valid_in_previous_step = False
for row in price_volume_df.itertuples():
timestamp = row.Index.timestamp()
price = row.price
volume = row.volume
avg_price = row.average_price
avg_volume = row.average_volume
# check whether to generate a buy signal:
if price > (1 + percent_change_price)*avg_price and volume > (1 + percent_change_volume)*avg_volume:
signal = Signal("RSI", 1, Horizon.any, 3, 3, price, 0, timestamp, 0, transaction_currency, counter_currency,
source, resample_period)
all_buy_signals.append(signal)
if not valid_in_previous_step:
valid_in_previous_step = True
first_cross_buy_signals.append(signal)
# print(timestamp, price, avg_price, volume, avg_volume)
else:
valid_in_previous_step = False
return all_buy_signals, first_cross_buy_signals
def get_signals(signal_type,
transaction_currency,
start_time,
end_time,
counter_currency="BTC"):
signal_query = """ SELECT trend, horizon, strength_value, strength_max, price, price_change, timestamp, rsi_value
FROM signal_signal
WHERE signal_signal.signal=%s AND
transaction_currency=%s AND
counter_currency=%s AND
timestamp >= %s AND
timestamp <= %s AND
source = 0
ORDER BY timestamp;"""
counter_currency_id = CounterCurrency[counter_currency].value
cursor = dbc.execute(signal_query,
params=(signal_type.value, transaction_currency,
counter_currency_id, start_time, end_time))
signals = []
for (trend, horizon, strength_value, strength_max, price, price_change,
timestamp, rsi_value) in cursor:
signals.append(
Signal(signal_type, trend, horizon, strength_value, strength_max,
price / 1E8, price_change, timestamp, rsi_value,
transaction_currency, CounterCurrency[counter_currency]))
return signals
def get_signals_rsi(transaction_currency,
start_time,
end_time,
rsi_overbought,
rsi_oversold,
counter_currency="BTC"):
rsi_signal_query = """ SELECT trend, horizon, strength_value, strength_max, price, price_change, timestamp, rsi_value
FROM signal_signal
WHERE signal_signal.signal=%s AND
transaction_currency=%s AND
counter_currency=%s AND
timestamp >= %s AND
timestamp <= %s AND
(rsi_value > %s OR
rsi_value < %s) AND
source = 0
ORDER BY timestamp;"""
counter_currency_id = CounterCurrency[counter_currency].value
cursor = dbc.execute(rsi_signal_query,
params=("RSI", transaction_currency,
counter_currency_id, start_time, end_time,
rsi_overbought, rsi_oversold))
signals = []
for (trend, horizon, strength_value, strength_max, price, price_change,
timestamp, rsi_value) in cursor:
signals.append(
Signal(SignalType.RSI, trend, horizon, strength_value,
strength_max, price / 1E8, price_change, timestamp,
rsi_value, transaction_currency,
CounterCurrency[counter_currency]))
return signals
def __init__(self, TAlgorithm, datahandler, Broker, Portfolio, name): """Initialize a new Stock Simulation Parameters: TAlgorithm - A Class which extends TradingAlgorithm. Will be run as Trader in this simulation. Look at TradingAlgorithm for information about what this class should contain DataHandler - An instance of Class extending DataHandler. Look at DataHandler class about how this classes should look like Broker - A Class which will extend the Broker Class. Look at the Broker class about how this classes should look like Portfolio - a Reference to the Portfolio class string name - name of the simulation. has no use yet """ self.name = name self.game_end = Signal() self.data = datahandler start_port = defaultdict(lambda: 0) start_port['EUR'] = 50000 self.broker = Broker(self) self.talgo_port = Portfolio(start_port) self.talgo = TAlgorithm( self, self.broker, self.talgo_port ) self.date = None
def spectrogram(signal, binWidth, overlap):
"""
Generates the spectrogram of an input signal.
@param signal The input signal object
@return specs The values of the spectrogram
@return f The frequency spectrum
@return t The time domain
"""
try:
signal.name = signal.name
except AttributeError as e:
print('AttributeError: input is not a Signal object')
f = np.linspace(0, binWidth // 2 * signal.sampleRate // binWidth,
binWidth // 2)
t = np.linspace(0, signal.length / signal.sampleRate,
signal.length // binWidth * overlap)
starts = np.arange(0, signal.length, binWidth // overlap)
starts = np.append(starts, signal.length)
specs = np.zeros((binWidth // 2, np.shape(t)[0]))
for step in range(np.shape(starts)[0] - overlap - 1):
subsignal = Signal(sampleRate=signal.sampleRate,
length=starts[step + overlap] - starts[step],
values=signal.values[starts[step]:starts[step +
overlap]])
specs[:, step] = G_xx(subsignal)
return specs, f, t
def _connect(self, callback, args = (), kwargs = {}, once = False,
weak = False, pos = -1):
"""
Internal connect function. Always set once to True because InProgress
will be emited only once.
"""
return Signal._connect(self, callback, args, kwargs, True, weak, pos)
def run(self):
for i, price_data in self.prices_df.iterrows():
signals_now = self.reindexed_signals_df[
self.reindexed_signals_df['timestamp'] ==
price_data['timestamp']]
signals_now = Signal.pandas_to_objects_list(signals_now)
self.notify_listeners(price_data, signals_now)
self.broadcast_ended()
class StreamProcesser:
def __init__(self):
self.states = ["INIT", "RUN_COMMANDS"]
self.state = self.states[0]
self.num_frames = 0
self.added_cmd_idx = 0
self.signal_capacity = 15
self.signal = Signal(self.signal_capacity)
self.commands = Commands()
self.unlock_frame_num = 0
def add_frame_freq(self, freq):
self.num_frames += 1
self.signal.push_note(freq)
if self.num_frames <= FRAMES_PER_FFT:
return
if self.state == self.states[0]:
self.add_freq_init(freq)
else:
self.add_freq_worker(freq)
def add_freq_init(self, freq):
if self.signal.is_correct(
) and self.unlock_frame_num < self.num_frames:
add_cmd_result = self.commands.add_command(
time_history=self.signal.history,
label=COMMANDS[self.added_cmd_idx]['label'],
triger_func=COMMANDS[self.added_cmd_idx]['triggers'])
if add_cmd_result:
self.unlock_frame_num = self.num_frames + self.signal_capacity + 1
self.added_cmd_idx += 1
if len(COMMANDS) == self.added_cmd_idx:
self.state = self.states[-1]
def add_freq_worker(self, freq):
self.commands.trigger_command(self.signal.history,
payload={
"message":
"signal_received: {}".format(
self.signal.get_corrected_note())
})
def inference(self, input_, orders=None, *args, **kwargs):
if not isinstance(input_, Signal):
raise TypeError('!! Input must be an instance of Signal')
# Update Phi
self._update_Phi_naive(input_)
# Calculate output
y = self.coefs[()] * np.ones_like(input_)
pool = (self.coefs.get_indices(symmetric=False)
if orders is None else self.coefs.get_homogeneous_indices(
orders, self.memory_depth[orders + 1], symmetric=False))
for indices in pool:
y_ = self.coefs[indices] * np.ones_like(input_)
for index in indices:
y_ *= self.Phi[index]
y += y_
output = Signal(y)
output.__array_finalize__(input_)
return output
def G_n_i(self, n, i, x):
y_i = np.zeros_like(x)
# multiplicity is n - i
indices_pool = self.kernels.get_homogeneous_indices(
n - i, self.memory_depth[n - i - 1], symmetric=False)
for indices in indices_pool:
assert isinstance(indices, list) or isinstance(indices, tuple)
# Determine indices
lags = indices + indices[slice(n - 2 * i, n - i)]
x_lags = indices[slice(n - 2 * i)]
prod = self.kernels[lags]
if prod == 0: continue
for lag in x_lags:
prod *= self._delay(x, lag)
y_i += prod
output = Signal(y_i * self._get_coef(n, i))
output.__array_finalize__(x)
return output
def __init__(self, typeID):
self.typeID = typeID
self.radius = evetypes.GetRadius(typeID)
self.movingOffset = None
self.cameraMatrixes = None
self._opacity = 0.0
self.blueprintColor = None
self.ballpark = sm.GetService('michelle').GetBallpark()
self.model_valid = self._LoadModel(':variant?placement', display=True)
self.model_invalid = self._LoadModel(':variant?forbiddenplacement',
display=False)
self.on_location_updated = Signal()
self.UpdateModel()
def __init__(self, market, limit_storage_time=timedelta(days=10), lag_time=timedelta(milliseconds=500)):
"""Initialize the broker
Parameters:
Market market - a market instance
timedelta limit_storage_time - a time span which limit
orders will be saved maximum
timdelta lag_time - the time which it will take to broker
to actually set a market order
"""
log.debug("Initiating Broker Object")
self.market = market
self.market.data.tick.registerObserver(self.market_tick)
self.market.data.time_change.registerObserver(self.market_tick)
self._orders = {}
self._limit_storage_time = limit_storage_time
self._market_order_delay = lag_time
self.order_fill = Signal()
self.order_delete = Signal()
self._newest_order_id = -1
def test_signals_conv():
from convolutions import convolve_signals
from signals import Signal
test_track = os.path.join(ESPACES_PROJECT, "dev", "ir", "data", "crash.wav")
args = {"normalize": False, "mono": False}
sig = Signal(test_track, **args)
file_name = os.path.basename(sig.location)
assert sig.data.shape[0] == sig.length
assert sig.data.shape[1] == sig.n_chan
assert sig.length > sig.n_chan
args = {"normalize": False, "mono": True}
sigmono = Signal(test_track, **args)
assert sigmono.n_chan == 1
with tempfile.NamedTemporaryFile(suffix='.mp3') as output_file:
sigmono.write(output_file.name)
convolve_signals(sig,sig,mode='full',kind='ss')
def test_get_signal_dimensions():
s_1 = Signal(known_intruder_1)
assert s_1.get_signal_dimensions() == (8, 11)
s_2 = Signal("----")
assert s_2.get_signal_dimensions() == (1, 4)
def __init__(self):
sm.RegisterNotify(self)
self.remoteSvc = sm.RemoteSvc(REMOTE_SERVICE_NAME)
self.objectCashingSvc = sm.GetService('objectCaching')
self.on_new_group_created = Signal()
self.on_group_selected = Signal()
self.on_groupmembers_changed = Signal()
self.on_groupmembers_removed = Signal()
self.on_group_deleted = Signal()
self.on_group_updated = Signal()
self.on_groups_reload = Signal()
self.membersByGroupID = {}
self.groupByGroupID = None
self.publicGroupsByGroupID = {}
self.myGroupsCacheInvalidated = False
self.currentSearchResults = None
def parse(self, s, debug=False):
self.clear()
lines = s.strip().split("\n")
if debug:
print "Parsing message"
print lines
header = lines[0]
parts = header.split(" ")
if parts[0] != "BO_":
print "Aborting: Expected \"BO_\" at beginning of message. Got \"" + parts[
0] + "\"."
return
self.ID = int(parts[1])
self.name = parts[2][:-1]
self.DLC = int(parts[3])
# TODO: Evaluate whatever is the meaning of "Vector__XXX"
for line in lines[1:]:
self.signals.append(Signal(line, debug))
def __init__(self, abortable=False, frame=0):
"""
:param abortable: see the :attr:`~kaa.InProgress.abortable` property.
(Default: False)
:type abortable: bool
"""
super(InProgress, self).__init__()
self._exception_signal = Signal()
self._finished = False
self._finished_event = threading.Event()
self._exception = None
self._unhandled_exception = None
# TODO: make progress a property so we can document it.
self.progress = None
self.abortable = abortable
# Stack frame for the caller who is creating us, for debugging.
self._stack = traceback.extract_stack()[:frame-2]
self._name = 'owner=%s:%d:%s()' % self._stack[-1][:3]
def test_get_signal_edge_cases():
# Test that empty strings cant be created for Signal class
with pytest.raises(Exception, match="Signal string can not be empty") as exec_info:
signal = Signal("")
# Test that empty strings cant be created for subclasses of Signal class
with pytest.raises(Exception, match="Signal string can not be empty") as exec_info:
signal = RadarSignal("")
with pytest.raises(Exception, match="Signal string can not be empty") as exec_info:
signal = IntruderSignal("")
# Test that the constructor does not take none as argument
with pytest.raises(Exception, match="Signal string can not be empty") as exec_info:
signal = RadarSignal(None)
def sys97(x):
assert isinstance(x, Signal)
N = x.size
v1 = np.zeros_like(x)
v2 = np.zeros_like(x)
for n in range(N):
x_1 = x[n - 1] if n - 1 >= 0 else 0
x_2 = x[n - 2] if n - 2 >= 0 else 0
v1_1 = v1[n - 1] if n - 1 >= 0 else 0
v1_2 = v1[n - 2] if n - 2 >= 0 else 0
v1[n] = 1.2 * v1_1 - 0.6 * v1_2 + 0.5 * x_1
v2_1 = v2[n - 1] if n - 1 >= 0 else 0
v2_2 = v2[n - 2] if n - 2 >= 0 else 0
v2_3 = v2[n - 3] if n - 3 >= 0 else 0
v2[n] = 1.8 * v2_1 - 1.1 * v2_2 + 0.2 * v2_3 + 0.1 * (x_1 + x_2)
y = (v1 + 0.8 * v2 * v2 - 0.6 * v1 * v1 * v2) * np.sin((v1 + v2) / 5)
output = Signal(y, fs=x.fs)
return output
class InProgress(Signal, Object):
"""
InProgress objects are returned from functions that require more time to
complete (because they are either blocked on some resource, are executing
in a thread, or perhaps simply because they yielded control back to the main
loop as a form of cooperative time slicing).
InProgress subclasses :class:`~kaa.Signal`, which means InProgress objects are
themselves signals. Callbacks connected to an InProgress receive a single
argument containing the result of the asynchronously executed task.
If the asynchronous task raises an exception, the
:attr:`~kaa.InProgress.exception` member, which is a separate signal, is
emitted instead.
"""
__kaasignals__ = {
'abort':
'''
Emitted when abort() is called.
.. describe:: def callback(exc)
:param exc: an exception object the InProgress was aborted with.
:type exc: InProgressAborted
If the task cannot be aborted, the callback can return False, which
will cause an exception to be raised by abort().
'''
}
def __init__(self, abortable=False, frame=0):
"""
:param abortable: see the :attr:`~kaa.InProgress.abortable` property.
(Default: False)
:type abortable: bool
"""
super(InProgress, self).__init__()
self._exception_signal = Signal()
self._finished = False
self._finished_event = threading.Event()
self._exception = None
self._unhandled_exception = None
# TODO: make progress a property so we can document it.
self.progress = None
self.abortable = abortable
# Stack frame for the caller who is creating us, for debugging.
self._stack = traceback.extract_stack()[:frame-2]
self._name = 'owner=%s:%d:%s()' % self._stack[-1][:3]
def __repr__(self):
return '<%s object at 0x%08x, %s>' % (self.__class__.__name__, id(self), self._name)
def __inprogress__(self):
"""
We subclass Signal which implements this method, but as we are already
an InProgress, we simply return self.
"""
return self
@property
def exception(self):
"""
A :class:`~kaa.Signal` emitted when the asynchronous task this InProgress
represents has raised an exception.
Callbacks connected to this signal receive three arguments: exception class,
exception instance, traceback.
"""
return self._exception_signal
@property
def finished(self):
"""
True if the InProgress is finished.
"""
return self._finished
@property
def result(self):
"""
The result the InProgress was finished with. If an exception was thrown
to the InProgress, accessing this property will raise that exception.
"""
if not self._finished:
raise RuntimeError('operation not finished')
if self._exception:
self._unhandled_exception = None
if self._exception[2]:
# We have the traceback, so we can raise using it.
exc_type, exc_value, exc_tb_or_stack = self._exception
raise exc_type, exc_value, exc_tb_or_stack
else:
# No traceback, so construct an AsyncException based on the
# stack.
raise self._exception[1]
return self._result
@property
def failed(self):
"""
True if an exception was thrown to the InProgress, False if it was
finished without error or if it is not yet finished.
"""
return bool(self._exception)
# XXX: is this property sane after all? Maybe there is no such case where
# an IP can be just ignored upon abort(). kaa.delay() turned out not to be
# an example after all.
@property
def abortable(self):
"""
True if the asynchronous task this InProgress represents can be
aborted by a call to :meth:`~kaa.InProgress.abort`.
Normally :meth:`~kaa.InProgress.abort` will fail if there are no
callbacks attached to the :attr:`~kaa.InProgress.signals.abort` signal.
This property may be explicitly set to ``True``, in which case
:meth:`~kaa.InProgress.abort` will succeed regardless. An InProgress is
therefore abortable if the ``abortable`` property has been explicitly
set to True, if if there are callbacks connected to the
:attr:`~kaa.InProgress.signals.abort` signal.
This is useful when constructing an InProgress object that corresponds
to an asynchronous task that can be safely aborted with no explicit action.
"""
return self._abortable or self.signals['abort'].count() > 0
@abortable.setter
def abortable(self, abortable):
self._abortable = abortable
def finish(self, result):
"""
This method should be called when the owner (creator) of the InProgress is
finished successfully (with no exception).
Any callbacks connected to the InProgress will then be emitted with the
result passed to this method.
If *result* is an unfinished InProgress, then instead of finishing, we
wait for the result to finish.
:param result: the result of the completed asynchronous task. (This can
be thought of as the return value of the task if it had
been executed synchronously.)
:return: This method returns self, which makes it convenient to prime InProgress
objects with a finished value. e.g. ``return InProgress().finish(42)``
"""
if self._finished:
raise RuntimeError('%s already finished' % self)
if isinstance(result, InProgress) and result is not self:
# we are still not finished, wait for this new InProgress
self.waitfor(result)
return self
# store result
self._finished = True
self._result = result
self._exception = None
# Wake any threads waiting on us
self._finished_event.set()
# emit signal
self.emit_when_handled(result)
# cleanup
self.disconnect_all()
self._exception_signal.disconnect_all()
self.signals['abort'].disconnect_all()
return self
def throw(self, type, value, tb, aborted=False):
"""
This method should be called when the owner (creator) of the InProgress is
finished because it raised an exception.
Any callbacks connected to the :attr:`~kaa.InProgress.exception` signal will
then be emitted with the arguments passed to this method.
The parameters correspond to sys.exc_info().
:param type: the class of the exception
:param value: the instance of the exception
:param tb: the traceback object representing where the exception took place
"""
# This function must deal with a tricky problem. See:
# http://mail.python.org/pipermail/python-dev/2005-September/056091.html
#
# Ideally, we want to store the traceback object so we can defer the
# exception handling until some later time. The problem is that by
# storing the traceback, we create some ridiculously deep circular
# references.
#
# The way we deal with this is to pass along the traceback object to
# any handler that can handle the exception immediately, and then
# discard the traceback. A stringified formatted traceback is attached
# to the exception in the formatted_traceback attribute.
#
# The above URL suggests a possible non-trivial workaround: create a
# custom traceback object in C code that preserves the parts of the
# stack frames needed for printing tracebacks, but discarding objects
# that would create circular references. This might be a TODO.
self._finished = True
self._exception = type, value, tb
self._unhandled_exception = True
stack = traceback.extract_tb(tb)
# Attach a stringified traceback to the exception object. Right now,
# this is the best we can do for asynchronous handlers.
trace = ''.join(traceback.format_exception(*self._exception)).strip()
value.formatted_traceback = trace
# Wake any threads waiting on us. We've initialized _exception with
# the traceback object, so any threads that access the result property
# between now and the end of this function will have an opportunity to
# get the live traceback.
self._finished_event.set()
if self._exception_signal.count() == 0:
# There are no exception handlers, so we know we will end up
# queuing the traceback in the exception signal. Set it to None
# to prevent that.
tb = None
if self._exception_signal.emit_when_handled(type, value, tb) == False:
# A handler has acknowledged handling this exception by returning
# False. So we won't log it.
self._unhandled_exception = None
if isinstance(value, InProgressAborted):
if not aborted:
# An InProgress we were waiting on has been aborted, so we
# abort too.
self.signals['abort'].emit(value)
self._unhandled_exception = None
if self._unhandled_exception:
# This exception was not handled synchronously, so we set up a
# weakref object with a finalize callback to a function that
# logs the exception. We could do this in __del__, except that
# the gc refuses to collect objects with a destructor. The weakref
# kludge lets us accomplish the same thing without actually using
# __del__.
#
# If the exception is passed back via result property, then it is
# considered handled, and it will not be logged.
cb = Callback(InProgress._log_exception, trace, value, self._stack)
self._unhandled_exception = _weakref.ref(self, cb)
# Remove traceback from stored exception. If any waiting threads
# haven't gotten it by now, it's too late.
if not isinstance(value, AsyncExceptionBase):
value = AsyncException(value, stack)
self._exception = value.__class__, value, None
# cleanup
self.disconnect_all()
self._exception_signal.disconnect_all()
self.signals['abort'].disconnect_all()
# We return False here so that if we've received a thrown exception
# from another InProgress we're waiting on, we essentially inherit
# the exception from it and indicate to it that we'll handle it
# from here on. (Otherwise the linked InProgress would figure
# nobody handled it and would dump out an unhandled async exception.)
return False
@classmethod
def _log_exception(cls, weakref, trace, exc, create_stack):
"""
Callback to log unhandled exceptions.
"""
if isinstance(exc, (KeyboardInterrupt, SystemExit)):
# We have an unhandled asynchronous SystemExit or KeyboardInterrupt
# exception. Rather than logging it, we reraise it in the main
# loop so that the main loop exception handler can act
# appropriately.
def reraise():
raise exc
return main.signals['step'].connect_once(reraise)
log.error('Unhandled %s exception:\n%s', cls.__name__, trace)
if log.level <= logging.INFO:
# Asynchronous exceptions create a bit of a problem in that while you
# know where the exception came from, you don't easily know where it
# was going. Here we dump the stack obtained in the constructor,
# so it's possible to find out which caller didn't properly catch
# the exception.
create_tb = ''.join(traceback.format_list(create_stack))
log.info('Create-stack for InProgress from preceding exception:\n%s', create_tb)
def abort(self, exc=None):
"""
Aborts the asynchronous task this InProgress represents.
:param exc: optional exception object with which to abort the InProgress; if
None is given, a general InProgressAborted exception will
be used.
:type exc: InProgressAborted
Not all such tasks can be aborted. If aborting is not supported, or if
the InProgress is already finished, a RuntimeError exception is raised.
If a coroutine is aborted, the CoroutineInProgress object returned by
the coroutine will be finished with InProgressAborted, while the underlying
generator used by the coroutine will have the standard GeneratorExit
raised inside it.
"""
if self.finished:
raise RuntimeError('InProgress is already finished.')
if exc is None:
exc = InProgressAborted('InProgress task aborted by abort()')
elif not isinstance(exc, InProgressAborted):
raise ValueError('Exception must be instance of InProgressAborted (or subclass thereof)')
if not self.abortable or self.signals['abort'].emit(exc) == False:
raise RuntimeError('%s cannot be aborted.' % self)
self.throw(exc.__class__, exc, None, aborted=True)
def timeout(self, timeout, callback=None, abort=False):
"""
Create a new InProgress object linked to this one that will throw
a TimeoutException if this object is not finished by the given timeout.
:param callback: called (with no additional arguments) just prior
to TimeoutException
:return: a new :class:`~kaa.InProgress` object that is subject to the timeout
If the original InProgress finishes before the timeout, the new InProgress
(returned by this method) is finished with the result of the original.
If a timeout does occur, the original InProgress object is not affected:
it is not finished with the TimeoutException, nor is it aborted. If you
want to abort the original task you must do it explicitly::
@kaa.coroutine()
def read_from_socket(sock):
try:
data = yield sock.read().timeout(3)
except TimeoutException, (msg, inprogress):
print 'Error:', msg
inprogress.abort()
"""
async = InProgress()
def trigger():
self.disconnect(async.finish)
self._exception_signal.disconnect(async.throw)
if not async._finished:
if callback:
callback()
msg = 'InProgress timed out after %.02f seconds' % timeout
async.throw(TimeoutException, TimeoutException(msg, self), None)
if abort:
self.abort()
async.waitfor(self)
OneShotTimer(trigger).start(timeout)
return async
def testDisconnectWeakSlot(self):
receiver = Receiver()
signal = Signal()
signal.connect(receiver.slot)
del receiver
self.assertEqual(len(signal), 0)
class SignalTestCase_listeners(TestCase):
def setUp(self):
self.baseSignal = Signal()
def tearDown(self):
self.baseSignal = None
def test_default_length_zero(self):
self.assertEqual(0, self.baseSignal.get_length())
def test_listener_is_added(self):
self.baseSignal.add(self.listenerOne)
self.assertEqual(1, self.baseSignal.get_length())
def test_add_listener_is_callable(self):
self.assertRaises(ValueError, self.baseSignal.add, 10)
def test_listener_is_added_once(self):
self.baseSignal.add(self.listenerOne)
self.assertEqual(1, self.baseSignal.get_length())
def test_add_once_listener_is_callable(self):
self.assertRaises(ValueError, self.baseSignal.addOnce, 10)
def test_listener_is_removed(self):
self.baseSignal.add(self.listenerOne)
self.baseSignal.remove(self.listenerOne)
self.assertEqual(0, self.baseSignal.get_length())
def test_all_listeners_are_removed(self):
self.baseSignal.add(self.listenerOne)
self.baseSignal.add(self.listenerTwo)
self.baseSignal.removeAll()
self.assertEqual(0, self.baseSignal.get_length())
def listenerOne(self):
pass
def listenerTwo(self):
pass
def testOneSlot(self):
receiver = Receiver()
signal = Signal()
signal.connect(receiver.slot)
signal()
self.assertEqual(receiver.count, 1)
class Market:
"""Main Class. Runs the Simulation as a whole"""
def __init__(self, TAlgorithm,
datahandler, Broker, Portfolio, name):
"""Initialize a new Stock Simulation
Parameters:
TAlgorithm - A Class which extends TradingAlgorithm. Will be run
as Trader in this simulation. Look at TradingAlgorithm
for information about what this class should contain
DataHandler - An instance of Class extending DataHandler. Look
at DataHandler class about how this classes should look like
Broker - A Class which will extend the Broker Class. Look at the
Broker class about how this classes should look like
Portfolio - a Reference to the Portfolio class
string name - name of the simulation. has no use yet
"""
self.name = name
self.game_end = Signal()
self.data = datahandler
start_port = defaultdict(lambda: 0)
start_port['EUR'] = 50000
self.broker = Broker(self)
self.talgo_port = Portfolio(start_port)
self.talgo = TAlgorithm(
self, self.broker, self.talgo_port )
self.date = None
# self.database_path = directory + name + '.db'
# self.conn = lite.connect(database_path)
def run(self, data_frequency):
"""Starts and runs the simulation. Will forward in time
`data_frequency` seconds. The broker will check each tick in
that time (check if limits are reached etc.). the trading
algorithm only gets a notification at the end of the forward
that means, `data_frequency` being 10 would mean, the
talgorithm would get notificated about new data every 10
seconds. he could then load all ticks of the last 10 secs
Parameters:
number data_frequency - the frequency in which the
talgorithm gets new data and is able to take action
upon that data
"""
log.info("Starting simulation %s", self.name)
time_intervall = timedelta(seconds=data_frequency)
while True == self.data.data_available:
self.data.update_current_time(time_intervall)
if self.data.get_current_time().date() != self.date:
self.date = self.data.get_current_time().date()
log.info("Today is the %s. Your Portfolio is worth "
"%f€", self.date,
self.talgo_port.get_portfolio_value_in_eur(self.data))
log.info("Game ended, no more Data available")
self.game_end.trigger()
print(self.talgo_port.get_portfolio())
log.info("In EUR your portfolio is worth %f",
self.talgo_port.get_portfolio_value_in_eur(self.data))
class TickBroker(Broker):
""" Broker handling the Order, which are represented by python
Dictionaries
For TAlgos, if you pass an order to TickBroker.order_xchange,
ALL ORDERS dicts have to include keys:
string fxcode, int `amount`, `kind` ('buy'/'sell'),
`type` ('market'/'limit')
LIMIT ORDERS need to contain
number `limit`, datetime `expires`
"""
def __init__(self, market, limit_storage_time=timedelta(days=10), lag_time=timedelta(milliseconds=500)):
"""Initialize the broker
Parameters:
Market market - a market instance
timedelta limit_storage_time - a time span which limit
orders will be saved maximum
timdelta lag_time - the time which it will take to broker
to actually set a market order
"""
log.debug("Initiating Broker Object")
self.market = market
self.market.data.tick.registerObserver(self.market_tick)
self.market.data.time_change.registerObserver(self.market_tick)
self._orders = {}
self._limit_storage_time = limit_storage_time
self._market_order_delay = lag_time
self.order_fill = Signal()
self.order_delete = Signal()
self._newest_order_id = -1
def get_max_limit_storage_time(self):
return self._limit_storage_time
def _make_transaction_xchange(self, portfolio, curr1, amount1, curr2, amount2):
"""Makes the actuall transaction in the portfolio
ParametersL
Portfolio portfolio - a portfolio instance on which the
transaction will happen
string curr1,2 - the currency codes (e.g. 'EUR') on which
the holdings should change
numer amount1,2 - amount`i` is the number which will be
added to your holdings of curr`i`
Normally either curr1 or curr2 will be negative and the
other one positive, because this is normally a transaction
from one currency into another
"""
try:
portfolio.transact(curr1, amount1)
portfolio.transact(curr2, amount2)
except InDebt as e:
log.info("You are in debt, you ow %f %s", e.amount, e.code)
def _fill_order_xchange(self, order):
"""Fill an order. Check if it is sell or buy, calculate the
price for the given amount that should be bought/sold
and then make the transaction
Parameters:
dict order - a dicitonary representing an order
"""
tick = self.market.data.get_current_tick(order["fxcode"])
curr = self._get_currencies_from_fxcode(order["fxcode"])
if order["kind"] == "buy":
if order["amount"] == 0:
order["amount"] = order["portfolio"].get_amount(curr[1]) / tick["ask"]
self._make_transaction_xchange(
order["portfolio"], curr[0], order["amount"], curr[1], -1 * order["amount"] * tick["ask"]
)
elif order["kind"] == "sell":
if order["amount"] == 0:
order["amount"] = order["portfolio"].get_amount(curr[0])
self._make_transaction_xchange(
order["portfolio"], curr[0], -1 * order["amount"], curr[1], order["amount"] * tick["bid"]
)
else:
raise ValueError("order needs to be either sellout or an" "amount has to be set")
self.order_fill.trigger(order)
def _get_currencies_from_fxcode(self, fxcode):
if fxcode == "EURUSD":
return ("EUR", "USD")
else:
raise CurrencyNotForTrade(
"Currency %s isn't implemented "
" in the Broker Class yet. No information to resolve "
" the fxcode to the single currencies" % fxcode
)
def market_tick(self, market):
"""called on every new tick"""
self._check_open_orders()
def order_xchange(self, order, portfolio):
"""Make an Limit or Market order. Return the `order_id` which
the TAlgo can use to recognize the order later.
This method will only append the order to self._orders,
the actuall execution is done in _check_open_orders later
"""
self._newest_order_id += 1
order.update({"sector": "xchange", "portfolio": portfolio})
order = self._check_order(order)
if order["type"] == "market":
order["execute_time"] = self.market.data.get_current_time() + self._market_order_delay
self._orders[self._newest_order_id] = order
return self._newest_order_id
def _check_open_orders(self):
"""Run through all orders in self._orders and check if they
expired or need to be executed. limit orders will be
executed when `limit` is reachend and deleted if `expires`
date is reached. Market orders will be executed after the
market order delay (simulates delay of your internet
connection) passed by. If order is executed or deleted, it
extends the order with the order-id so that the TAlgo
will recieve it in the event trigger and then deletes it
"""
for order_id, order in list(self._orders.items()):
if order["type"] == "limit":
if order["expires"] < self.market.data.get_time():
order["id"] = order_id
self.order_delete.trigger(order)
del self._orders[order_id]
continue
if (
order["kind"] == "buy"
and self.market.data.get_current_tick(order["fxcode"])["bid"][0] >= order["limit"]
or order["kind"] == "sell"
and self.market.data.get_current_tick(order["fxcode"])["ask"][0] <= order["limit"]
):
order["id"] = order_id
self._fill_order_xchange(order)
del self._orders[order_id]
else:
if order["execute_time"] <= (self.market.data.get_current_time()):
order["id"] = order_id
self._fill_order_xchange(order)
del self._orders[order_id]
def _check_order(self, order):
"""Checks a Order dict if it is a valid order"""
if order["type"] not in ("limit", "market"):
raise ValueError("Order key `type` must be set to either " "'market' or 'limit'")
if order["kind"] not in ("buy", "sell"):
raise ValueError("Order key `kind` must be set to either " "'buy' or 'sell'")
if order["type"] == "limit":
if order["expires"] == None:
order["expires"] = self.market.data.get_current_time() + self._orders_storage_time
elif (
order["expires"] > (self._orders_storage_time + self.market.data.get_current_time())
or order["expires"] < 0
):
raise ValueError(
"Expires should be smaller then or equal " " to %s and not-negative, you passed '%s' by.",
self._orders_storage_time,
expires,
)
if not isinstance(order["limit"], Number):
raise ValueError("If you pass by a limit order you have" "to set a limit")
return order
def setUp(self):
self.signal = Signal()
class ModelCopySelected(object):
"""
Implemantation MVC de la procedure copySelected
"""
def __init__(self):
"""
"""
self.__label = "Un moment..."
self.startSignal = Signal()
self.refreshSignal = Signal()
self.finishSignal = Signal()
self.NbrJobsSignal = Signal()
def start(self, lstFiles):
"""
Lance les calculs
@param lstFiles: list of files to process
"""
self.startSignal.emit(self.__label, max(1, len(lstFiles)))
if config.Filigrane:
filigrane = Signature(config.FiligraneSource)
else:
filigrane = None
SelectedDir = os.path.join(config.DefaultRepository, config.SelectedDirectory)
self.refreshSignal.emit(-1, "copie des fichiers existants")
if not os.path.isdir(SelectedDir): fileutils.mkdir(SelectedDir)
#####first of all : copy the subfolders into the day folder to help mixing the files
for day in os.listdir(SelectedDir):
for File in os.listdir(os.path.join(SelectedDir, day)):
if File.find(config.PagePrefix) == 0:
if os.path.isdir(os.path.join(SelectedDir, day, File)):
for strImageFile in os.listdir(os.path.join(SelectedDir, day, File)):
src = os.path.join(SelectedDir, day, File, strImageFile)
dst = os.path.join(SelectedDir, day, strImageFile)
if os.path.isfile(src) and not os.path.exists(dst):
shutil.move(src, dst)
if (os.path.isdir(src)) and (os.path.split(src)[1] in [config.ScaledImages["Suffix"], config.Thumbnails["Suffix"]]):
shutil.rmtree(src)
#######then copy the selected files to their folders###########################
globalCount = 0
for File in lstFiles:
dest = os.path.join(SelectedDir, File)
src = os.path.join(config.DefaultRepository, File)
destdir = os.path.dirname(dest)
self.refreshSignal.emit(globalCount, File)
globalCount += 1
if not os.path.isdir(destdir):
fileutils.makedir(destdir)
if not os.path.exists(dest):
if filigrane:
image = Image.open(src)
filigrane.substract(image).save(dest, quality=config.FiligraneQuality, optimize=config.FiligraneOptimize, progressive=config.FiligraneOptimize)
else:
shutil.copy(src, dest)
try:
os.chmod(dest, config.DefaultFileMode)
except OSError:
logger.warning("In ModelCopySelected: unable to chmod %s", dest)
else :
logger.info("In ModelCopySelected: %s already exists", dest)
######copy the comments of the directory to the Selected directory
AlreadyDone = []
for File in lstFiles:
directory = os.path.split(File)[0]
if directory in AlreadyDone:
continue
else:
AlreadyDone.append(directory)
dst = os.path.join(SelectedDir, directory, config.CommentFile)
src = os.path.join(config.DefaultRepository, directory, config.CommentFile)
if os.path.isfile(src):
shutil.copy(src, dst)
self.finishSignal.emit()
class ModelProcessSelected(object):
"""
Implemantation MVC de la procedure processSelected
"""
def __init__(self):
"""
"""
self.__label = "Un moment..."
self.startSignal = Signal()
self.refreshSignal = Signal()
self.finishSignal = Signal()
self.NbrJobsSignal = Signal()
def start(self, lstFiles):
"""
Lance les calculs pour "processSelected"
i.e.
@param lstFiles: list of files to process
"""
def splitIntoPages(pathday, globalCount):
"""Split a directory (pathday) into pages of 20 images (see config.NbrPerPage)
@param pathday:
@param globalCount:
@return: the number of images for current page
"""
logger.debug("In splitIntoPages %s %s", pathday, globalCount)
files = []
for i in os.listdir(pathday):
if os.path.splitext(i)[1] in config.Extensions:files.append(i)
files.sort()
if len(files) > config.NbrPerPage:
pages = 1 + (len(files) - 1) / config.NbrPerPage
for i in range(1, pages + 1):
folder = os.path.join(pathday, config.PagePrefix + str(i))
fileutils.mkdir(folder)
for j in range(len(files)):
i = 1 + (j) / config.NbrPerPage
filename = os.path.join(pathday, config.PagePrefix + str(i), files[j])
self.refreshSignal.emit(globalCount, files[j])
globalCount += 1
shutil.move(os.path.join(pathday, files[j]), filename)
scaleImage(filename, filigrane)
else:
for j in files:
self.refreshSignal.emit(globalCount, j)
globalCount += 1
scaleImage(os.path.join(pathday, j), filigrane)
return globalCount
def arrangeOneFile(dirname, filename):
"""
@param dirname:
@param filename:
"""
try:
timetuple = time.strptime(filename[:19], "%Y-%m-%d_%Hh%Mm%S")
suffix = filename[19:]
except ValueError:
try:
timetuple = time.strptime(filename[:11], "%Y-%m-%d_")
suffix = filename[11:]
except ValueError:
logger.warning("Unable to handle such file: %s" % filename)
return
daydir = os.path.join(SelectedDir, time.strftime("%Y-%m-%d", timetuple))
os.mkdir(daydir)
shutil.move(os.path.join(dirname, filename), os.path.join(daydir, time.strftime("%Hh%Mm%S", timetuple) + suffix))
logger.debug("In Process Selected" + " ".join(lstFiles))
self.startSignal.emit(self.__label, max(1, len(lstFiles)))
if config.Filigrane:
filigrane = Signature(config.FiligraneSource)
else:
filigrane = None
SelectedDir = os.path.join(config.DefaultRepository, config.SelectedDirectory)
self.refreshSignal.emit(-1, "copie des fichiers existants")
if not os.path.isdir(SelectedDir):
fileutils.mkdir(SelectedDir)
#####first of all : copy the subfolders into the day folder to help mixing the files
AlsoProcess = 0
for day in os.listdir(SelectedDir):
#if SingleDir : revert to a foldered structure
DayOrFile = os.path.join(SelectedDir, day)
if os.path.isfile(DayOrFile):
arrangeOneFile(SelectedDir, day)
AlsoProcess += 1
#end SingleDir normalization
elif os.path.isdir(DayOrFile):
if day in [config.ScaledImages["Suffix"], config.Thumbnails["Suffix"]]:
fileutils.recursive_delete(DayOrFile)
elif day.find(config.PagePrefix) == 0: #subpages in SIngleDir mode that need to be flatten
for File in os.listdir(DayOrFile):
if os.path.isfile(os.path.join(DayOrFile, File)):
arrangeOneFile(DayOrFile, File)
AlsoProcess += 1
# elif os.path.isdir(os.path.join(DayOrFile,File)) and File in [config.ScaledImages["Suffix"],config.Thumbnails["Suffix"]]:
# recursive_delete(os.path.join(DayOrFile,File))
fileutils.recursive_delete(DayOrFile)
else:
for File in os.listdir(DayOrFile):
if File.find(config.PagePrefix) == 0:
if os.path.isdir(os.path.join(SelectedDir, day, File)):
for strImageFile in os.listdir(os.path.join(SelectedDir, day, File)):
src = os.path.join(SelectedDir, day, File, strImageFile)
dst = os.path.join(SelectedDir, day, strImageFile)
if os.path.isfile(src) and not os.path.exists(dst):
shutil.move(src, dst)
AlsoProcess += 1
if (os.path.isdir(src)) and (os.path.split(src)[1] in [config.ScaledImages["Suffix"], config.Thumbnails["Suffix"]]):
shutil.rmtree(src)
else:
if os.path.splitext(File)[1] in config.Extensions:
AlsoProcess += 1
#######then copy the selected files to their folders###########################
for File in lstFiles:
dest = os.path.join(SelectedDir, File)
src = os.path.join(config.DefaultRepository, File)
destdir = os.path.dirname(dest)
if not os.path.isdir(destdir):
fileutils.makedir(destdir)
if not os.path.exists(dest):
logger.info("copie de %s " % File)
shutil.copy(src, dest)
try:
os.chmod(dest, config.DefaultFileMode)
except OSError:
logger.warning("Unable to chmod %s" % dest)
AlsoProcess += 1
else :
logger.warning("%s existe déja" % dest)
if AlsoProcess > 0:self.NbrJobsSignal.emit(AlsoProcess)
######copy the comments of the directory to the Selected directory
AlreadyDone = []
for File in lstFiles:
directory = os.path.split(File)[0]
if directory in AlreadyDone:
continue
else:
AlreadyDone.append(directory)
dst = os.path.join(SelectedDir, directory, config.CommentFile)
src = os.path.join(config.DefaultRepository, directory, config.CommentFile)
if os.path.isfile(src):
shutil.copy(src, dst)
########finaly recreate the structure with pages or make a single page ########################
logger.debug("in ModelProcessSelected, SelectedDir= %s", SelectedDir)
dirs = [ i for i in os.listdir(SelectedDir) if os.path.isdir(os.path.join(SelectedDir, i))]
dirs.sort()
if config.ExportSingleDir: #SingleDir
#first move all files to the root
for day in dirs:
daydir = os.path.join(SelectedDir, day)
for filename in os.listdir(daydir):
try:
timetuple = time.strptime(day[:10] + "_" + filename[:8], "%Y-%m-%d_%Hh%Mm%S")
suffix = filename[8:]
except ValueError:
try:
timetuple = time.strptime(day[:10], "%Y-%m-%d")
suffix = filename
except ValueError:
logger.info("Unable to handle dir: %s\t file: %s" , day, filename)
continue
src = os.path.join(daydir, filename)
dst = os.path.join(SelectedDir, time.strftime("%Y-%m-%d_%Hh%Mm%S", timetuple) + suffix)
shutil.move(src, dst)
fileutils.recursive_delete(daydir)
splitIntoPages(SelectedDir, 0)
else: #Multidir
logger.debug("in Multidir, dirs= " + " ".join(dirs))
globalCount = 0
for day in dirs:
globalCount = splitIntoPages(os.path.join(SelectedDir, day), globalCount)
self.finishSignal.emit()
def testSignalWithArguments(self):
receiver = ReceiverWithArg()
signal = Signal()
signal.connect(receiver.slot)
signal(123)
self.assertEquals(receiver.count, 123)
def setUp(self):
self.baseSignal = Signal()
from __future__ import division
v = 340.29
sampling_rate = 44100
carrier_freq = 12000
up_sample_rate = 20
import numpy
from math import sin, cos, pi, sqrt
from signals import Signal, LPF
beacon0 = Signal.get_variable("beacon0")[0]
beacon1 = Signal.get_variable("beacon1")[0]
beacon2 = Signal.get_variable("beacon2")[0]
beacon3 = Signal.get_variable("beacon3")[0]
beacon4 = Signal.get_variable("beacon4")[0]
beacon5 = Signal.get_variable("beacon5")[0]
beacon = [beacon0, beacon1, beacon2, beacon3, beacon4, beacon5]
def demodulate_signal(signal):
"""
Demodulate the signal using complex demodulation.
"""
# Demodulate the signal using cosine and sine bases
demod_real_base = [cos(2 * pi * carrier_freq * i / sampling_rate)
for i in range(1, len(signal) + 1)]
demod_imaginary_base = [sin(2 * pi * carrier_freq * i / sampling_rate)
for i in range(1, len(signal) + 1)]
# Multiply the bases to the signal received
demod_real = [demod_real_base[i] * signal[i] for i in range(len(signal))]
def testSignalLen(self):
receiver = Receiver()
signal = Signal()
self.assertEqual(len(signal), 0)
signal.connect(receiver.slot)
self.assertEqual(len(signal), 1)
class SignalTest(unittest.TestCase):
def setUp(self):
self.signal = Signal()
def test_basic_construction_works(self):
assert self.signal.add_once(simple_listener_returning_true) is None
assert self.signal.num_listeners is 1
def test_add_once_then_add_raises(self):
self.signal = Signal()
self.signal.add_once(simple_listener_returning_true)
self.assertRaises(RuntimeError, self.signal.add, simple_listener_returning_true)
def test_dispatch_no_listeners(self):
self.signal.dispatch()
def test_no_param_dispatch(self):
self.signal.add(simple_listener_returning_true)
self.signal.dispatch()
self.assertRaises(TypeError, self.signal.dispatch, 1)
self.assertRaises(TypeError, self.signal.dispatch, {'someKeyWord': 1})
def test_no_param_dispatch_multiple_times(self):
self.signal.add(simple_listener_returning_true)
self.signal.dispatch()
self.signal.dispatch()
self.signal.dispatch()
def test_add_once_only_fires_once(self):
def mylistener(f):
f.callcount = f.callcount + 1
mylistener.callcount = 0
self.signal.add_once(mylistener)
self.signal.dispatch(mylistener)
self.signal.dispatch(mylistener)
self.assertEqual(mylistener.callcount, 1)
def test_removed_listener_doesnt_fire(self):
def mylistener(f):
f.callcount = f.callcount + 1
mylistener.callcount = 0
self.signal.add(mylistener)
self.signal.remove(mylistener)
self.signal.dispatch(mylistener)
self.assertEqual(mylistener.callcount, 0)
def test_listener_added_twice_doesnt_duplicate(self):
self.signal.add(simple_listener_returning_false)
self.signal.add(simple_listener_returning_false)
self.assertEqual(self.signal.num_listeners, 1)
self.signal.dispatch()
def test_adding_null_doesnt_add_listener(self):
self.assertRaises(ValueError, self.signal.add, None)
class TestSignal(unittest.TestCase):
def setUp(self):
self.sig_function = Signal()
self.sig_method = Signal()
self.sig_function.connect(onSignalFunction)
global signal_calls
signal_calls = []
def test_signal_function(self):
self.sig_function('sender1', 'data1')
self.assertEqual(1, len(signal_calls))
self.assertEqual( 'function', signal_calls[0][0] )
self.assertEqual( 'sender1', signal_calls[0][1] )
self.assertEqual( 'data1', signal_calls[0][2] )
def test_signal_method(self):
a = A()
self.sig_method.connect(a.onSignalMethod)
self.sig_method('sender1', 'data1')
self.assertEqual(1, len(signal_calls))
self.assertEqual( 'A::onSignalMethod', signal_calls[0][0] )
self.assertEqual( 'sender1', signal_calls[0][1] )
self.assertEqual( 'data1', signal_calls[0][2] )
del a
self.sig_method('sender1', 'data1')
self.assertEqual(1, len(signal_calls))
b = B(self.sig_method)
self.sig_method('sender2', 'data2')
self.assertEqual(2, len(signal_calls))
self.assertEqual( 'B::onSignalMethod', signal_calls[1][0] )
self.assertEqual( 'sender2', signal_calls[1][1] )
self.assertEqual( 'data2', signal_calls[1][2] )
del b
self.sig_method('sender3', 'data3')
self.assertEqual(2, len(signal_calls))
def test_sender_signal_method(self):
a = A()
self.sig_method.connect(a.onSignalMethod, 'sender1')
self.sig_method('sender1', 'data1')
self.assertEqual(1, len(signal_calls))
self.assertEqual( 'A::onSignalMethod', signal_calls[0][0] )
self.assertEqual( 'sender1', signal_calls[0][1] )
self.assertEqual( 'data1', signal_calls[0][2] )
self.sig_method('sender2', 'data2')
self.assertEqual(1, len(signal_calls))
self.sig_method('sender1', 'data2')
self.assertEqual(2, len(signal_calls))
del a
self.sig_method('sender1', 'data3')
self.assertEqual(2, len(signal_calls))
def test_sender_signal_function(self):
def on_signal_func(sender, data):
signal_calls.append( ( 'function', sender, data ) )
self.sig_method.connect( on_signal_func, 'sender1' )
self.sig_method('sender1', 'data1')
self.assertEqual(1, len(signal_calls))
self.assertEqual( 'function', signal_calls[0][0] )
self.assertEqual( 'sender1', signal_calls[0][1] )
self.assertEqual( 'data1', signal_calls[0][2] )
self.sig_method('sender2', 'data2')
self.assertEqual(1, len(signal_calls))
def test_add_once_then_add_raises(self):
self.signal = Signal()
self.signal.add_once(simple_listener_returning_true)
self.assertRaises(RuntimeError, self.signal.add, simple_listener_returning_true)
class ModelRangeTout(object):
"""Implemantation MVC de la procedure rangeTout
moves all the JPEG files to a directory named from
their day and with the name according to the time"""
def __init__(self):
"""
"""
self.__label = "Initial renaming of new images .... "
self.startSignal = Signal()
self.refreshSignal = Signal()
self.finishSignal = Signal()
self.NbrJobsSignal = Signal()
def start(self, rootDir):
""" Lance les calculs
@param rootDir: top level directory to start processing
@return: 2tuple containing the list of all images and the start-index
@rtype: (list,integer)
"""
config.DefaultRepository = rootDir
AllJpegs = fileutils.findFiles(rootDir)
AllFilesToProcess = []
AllreadyDone = []
NewFiles = []
uid = os.getuid()
gid = os.getgid()
for i in AllJpegs:
if i.find(config.TrashDirectory) == 0: continue
if i.find(config.SelectedDirectory) == 0: continue
try:
a = int(i[:4])
m = int(i[5:7])
j = int(i[8:10])
if (a >= 0000) and (m <= 12) and (j <= 31) and (i[4] in ["-", "_", "."]) and (i[7] in ["-", "_"]):
AllreadyDone.append(i)
else:
AllFilesToProcess.append(i)
except ValueError:
AllFilesToProcess.append(i)
AllFilesToProcess.sort()
NumFiles = len(AllFilesToProcess)
self.startSignal.emit(self.__label, NumFiles)
for h in range(NumFiles):
i = AllFilesToProcess[h]
self.refreshSignal.emit(h, i)
myPhoto = Photo(i, dontCache=True)
data = myPhoto.readExif()
try:
datei, heurei = data["Heure"].split()
date = re.sub(":", "-", datei)
heurej = re.sub(":", "h", heurei, 1)
model = data["Modele"].split(",")[-1]
heure = unicode2ascii("%s-%s.jpg" % (re.sub(":", "m", heurej, 1), re.sub("/", "", re.sub(" ", "_", model))))
except ValueError:
date = time.strftime("%Y-%m-%d", time.gmtime(os.path.getctime(os.path.join(rootDir, i))))
heure = unicode2ascii("%s-%s.jpg" % (time.strftime("%Hh%Mm%S", time.gmtime(os.path.getctime(os.path.join(rootDir, i)))), re.sub("/", "-", re.sub(" ", "_", os.path.splitext(i)[0]))))
if not (os.path.isdir(os.path.join(rootDir, date))) :
fileutils.mkdir(os.path.join(rootDir, date))
# strImageFile = os.path.join(rootDir, date, heure)
ToProcess = os.path.join(date, heure)
bSkipFile = False
for strImageFile in fileutils.list_files_in_named_dir(rootDir, date, heure):
logger.warning("%s -x-> %s", i, strImageFile)
existing = Photo(strImageFile, dontCache=True)
try:
existing.readExif()
originalName = existing.exif["Exif.Photo.UserComment"]
except:
logger.error("in ModelRangeTout: reading Exif for %s", i)
else:
if "human_value" in dir(originalName):
originalName = originalName.human_value
if os.path.basename(originalName) == os.path.basename(i):
logger.info("File already in repository, leaving as it is")
bSkipFile = True
continue #to next file, i.e. leave the existing one
if bSkipFile:
continue
else:
strImageFile = os.path.join(rootDir, date, heure)
if os.path.isfile(strImageFile):
s = 0
for j in os.listdir(os.path.join(rootDir, date)):
if j.find(heure[:-4]) == 0:s += 1
ToProcess = os.path.join(date, heure[:-4] + "-%s.jpg" % s)
strImageFile = os.path.join(rootDir, ToProcess)
shutil.move(os.path.join(rootDir, i), strImageFile)
try:
os.chown(strImageFile, uid, gid)
os.chmod(strImageFile, config.DefaultFileMode)
except OSError:
logger.warning("in ModelRangeTout: unable to chown ot chmod %s" , strImageFile)
myPhoto = Photo(strImageFile, dontCache=True)
# Save the old image name in exif tag
myPhoto.storeOriginalName(i)
if config.AutoRotate:
myPhoto.autorotate()
AllreadyDone.append(ToProcess)
NewFiles.append(ToProcess)
AllreadyDone.sort()
self.finishSignal.emit()
if len(NewFiles) > 0:
FirstImage = min(NewFiles)
return AllreadyDone, AllreadyDone.index(FirstImage)
else:
return AllreadyDone, 0
