def __init__(self, dict_map, embedding_dim, name, config, padding_idx=None,
pretrained_embedding_file=None, mode=EmbeddingProcessType.FLAT,
dropout=0, init_type=InitType.XAVIER_UNIFORM, low=0, high=1,
mean=0, std=1, activation_type=ActivationType.NONE,
fan_mode=FAN_MODE.FAN_IN, negative_slope=0):
super(Embedding, self).__init__()
self.logger = Logger(config)
self.dropout = torch.nn.Dropout(p=dropout)
self.mode = mode
if self.mode == EmbeddingProcessType.FLAT:
self.embedding = torch.nn.Embedding(
len(dict_map), embedding_dim, padding_idx=padding_idx)
else:
self.embedding = torch.nn.EmbeddingBag(
len(dict_map), embedding_dim, mode=mode)
embedding_lookup_table = init_tensor(
tensor=torch.empty(len(dict_map), embedding_dim),
init_type=init_type, low=low, high=high, mean=mean, std=std,
activation_type=activation_type, fan_mode=fan_mode,
negative_slope=negative_slope)
if pretrained_embedding_file is not None and \
pretrained_embedding_file != "":
self.load_pretrained_embedding(
embedding_lookup_table, dict_map, embedding_dim, name,
pretrained_embedding_file)
if padding_idx is not None:
embedding_lookup_table[padding_idx] = 0.0
self.embedding.weight.data.copy_(embedding_lookup_table)
def create(self,
table,
columns,
types,
primary_key_index=(),
is_ifnotexists=True):
"""
Create table in the database
:param table: Table name
:param columns: Column array
:param types: Type array
:param is_ifnotexists: Create table if not exists keyword
"""
file_path = os.path.join(self.file_directory, table + ".csv")
columns = [e.split(' ')[0] for e in columns]
if len(columns) != len(types):
return False
self.lock.acquire()
if os.path.isfile(file_path):
Logger.info(self.__class__.__name__,
"File (%s) has been created already." % file_path)
else:
with open(file_path, 'w+') as csvfile:
csvfile.write(','.join(["\"" + e + "\""
for e in columns]) + '\n')
self.lock.release()
return True
def __init__(self, datahandler, symbol_list=None):
self._datahandler = datahandler
self._selected_symbol_list = symbol_list
self._symbol_list = []
self._baseseries_list = datahandler.baseseries_list
self._date_list = datahandler.date_list
self._logger = Logger()
def connect(self, url,
on_message_handler=None,
on_open_handler=None,
on_close_handler=None,
on_error_handler=None):
"""
:param url: Url link
:param on_message_handler: Message handler which take the message as
the first argument
:param on_open_handler: Socket open handler which take the socket as
the first argument
:param on_close_handler: Socket close handler which take the socket as
the first argument
:param on_error_handler: Socket error handler which take the socket as
the first argument and the error as the second
argument
"""
Logger.info(self.__class__.__name__, "Connecting to socket <%s>..." % self.id)
if on_message_handler is not None:
self.on_message_handlers.append(on_message_handler)
if on_open_handler is not None:
self.on_open_handlers.append(on_open_handler)
if on_close_handler is not None:
self.on_close_handlers.append(on_close_handler)
if on_error_handler is not None:
self.on_error_handlers.append(on_error_handler)
if not self._connected:
self.wst = threading.Thread(target=lambda: self.run(url, 80))
self.wst.start()
return self.wst
def __init__(self, pipes, float_sizes, shapes, flags, n, learners):
"""
:param pipes: One pipe for each agent
:param float_sizes: Array which stores for each weight its size in bytes
:param shapes: Array which stores for each weight its shape
:param warm_start: if the first epochs synchronous should be used
:param epochs: number of epochs for which synchronous is used
:param flags: flags set by the user
:param n: n of n-softsync.
:param learners: number of agents
"""
super().__init__(pipes, float_sizes, shapes)
assert (learners % n == 0) #otherwise this algorithm won't finish!!!
self.learners = learners
self.num = learners // n
self.flags = flags
self.timing = self.flags.time_program
self.qlist = []
self.counter = 0
logger.state("softsync with number of learners", learners)
self.Random = Random(time.time())
self.shuffle = flags.shuffle
self.step_in_phase = [0] * learners
self.steps_for_learner = [0] * learners
self.min = 0
self.mode = 2
self.holdback = []
self.max_staleness = 0
self.gen = self._pipes_generator()
def __init__(self, element, parent):
''' The init method, used to create a new preparation object which can
be ensured on the host system.'''
# Store information about the preparation object
self.description = element.get('description')
self.test = element.get('test')
self.commands = element.get('commands')
self.result = element.get('result')
# Store the system for running commands
self.parent = parent
# Check if we're in chroot
if self.parent.user != 'chroot':
# Initialise the logger
self.logger = Logger(self.parent.environment['WANDER'],
self.parent.stage[1], '.')
else:
# Initialise the logger
self.logger = Logger('/', self.parent.stage[1])
# Note that we've started the check
Output.log(Output.PENDING, self.description)
def learn(self, result, delta, learning_rate):
super(BiasedNeuralLayer, self).learn(result, delta, learning_rate)
Logger.debug("delta: " + str(delta))
Logger.debug("biases: " + str(self.biases))
tmp = -(learning_rate * delta)
self.biases = tmp + np.atleast_2d(self.biases)
self.biases[self.biases < 0] = 0
def get_order_book_worker(self, instmt):
"""
Get order book worker
:param instmt: Instrument
"""
while True:
ExchGwQuoine.last_query_time_lock.acquire()
if datetime.now() - ExchGwQuoine.last_query_time < timedelta(
seconds=ExchGwQuoine.waiting_seconds):
ExchGwQuoine.last_query_time_lock.release()
time.sleep(random.uniform(0, 1))
else:
ExchGwQuoine.last_query_time = datetime.now()
try:
l2_depth = self.api_socket.get_order_book(instmt)
if l2_depth is not None and l2_depth.is_diff(
instmt.get_l2_depth()):
instmt.set_prev_l2_depth(instmt.get_l2_depth())
instmt.set_l2_depth(l2_depth)
instmt.incr_order_book_id()
self.insert_order_book(instmt)
except Exception as e:
Logger.error(self.__class__.__name__,
"Error in order book: %s" % e)
ExchGwQuoine.last_query_time_lock.release()
def __init__(self, sys, clk, user_id, level, num_load_mshrs,
num_parallel_stores, cache_size, line_size, latency,
logger_on, parent_component_id, child_component_id):
super().__init__("L" + str(level) + " Cache " + str(user_id), clk, sys,
parent_component_id, child_component_id)
self.level = level
self.num_load_mshrs = num_load_mshrs
self.num_parallel_stores = num_parallel_stores
self.load_stall_queue = []
self.store_stall_queue = []
self.load_mshr_bank = MSHRBank(self.num_load_mshrs)
self.logger = Logger(self.name, logger_on, self.sys)
# Cache Configuration
self.tlb_size = 32
self.cache_size = cache_size
self.line_size = line_size
self.max_size = self.cache_size / self.line_size
self.latency = latency
self.accesses = []
self.cache = [0, 0]
self.load_queue = []
self.store_queue = []
self.offset_bits = int(math.log2(self.line_size))
self.word_size = 64
self.byte_addressable = True
def __init__(self, setting):
try:
self.logger = Logger(setting.saveDirPath)
self.fm = FileManager(setting.saveDirPath, setting.downloadDirPath,
self.logger)
self.driver = IDriver(setting.chromeDriverPath)
self.setting = setting
except:
raise ('[error] setting is not valid')
def __init__(self, file_path):
"""
CSVLoader loads fund's nav or index's value from a csv file.
:param file_path:
"""
self._file_path = file_path
self._symbol2data = dict()
self._symbol2status = dict()
self._logger = Logger()
self._read()
def train_test(self, train_X, train_Y, test_X):
self.results = []
rez = np.array([0.0] * len(test_X))
l = Logger(len(self.models), 1, tag='BigModel')
for i, m in enumerate(self.models):
m_rez, m_batches = m.train_test(train_X, train_Y, test_X)
rez += self.weights[i] * m_rez
self.results.append((m_rez, m_batches))
l.step()
return rez
def distr_owners(self,map_otn):
"""Randomly distribute cells to owners"""
if(sum(map_otn.items())>self.x*self.y):
Logger.warning("Board.distr_owners called, \
but the board is too small to allocate all the cells!")
cells=set(self.cells)
otc={}
for owner in map_otn:
s=self.rng.sample(cells, map_otn[owner])
cells.difference_update(s)
otc[owner]=s
for c in s: c.owner=owner
def get_db_data(tags=['train1', 'train2']):
records = auctions.find({'tag': {'$in': tags}})
l = Logger(records.count(),20000, tag=str(tags))
X = []
Y = []
IDs = []
for record in records:
l.step()
xr = [record.get(k, -1) for k in columns]# removed Nones
X.append(xr)
Y.append(record.get(Ycol))
IDs.append(record['SalesID'])
return (X, Y, IDs)
def get_delta(self, out_data, last_delta, last_weights):
"""calculate delta for layer before"""
#Logger.DEBUG = True
Logger.debug("Get delta: ")
Logger.debug("out: " + str(np.shape(out_data)))
Logger.debug("last_delta: " + str(np.shape(last_delta)))
Logger.debug("last_weights: " + str(np.shape(last_weights)))
dot = np.dot(last_weights.T, last_delta)
Logger.debug("dot shape: " + str(np.shape(dot)))
delta = dot * self.activation_deriv(out_data)
#Logger.debug("delta shape: " + str(np.shape(delta)))
#Logger.DEBUG = False
return delta
class ExpenseReport(object):
MANDATORY_FIELDS = ['amount','user','date','category','reimburseable']
def __init__(self):
self.logger = Logger('REPORT')
@staticmethod
def create(inputmsg, state):
# create an expense
report = ExpenseReport()
report.log('Creating report')
# check if all fields are present
if (not ExpenseReport._is_valid_input(inputmsg)):
state.error('Illegal input')
return None
for key in inputmsg.keys():
report.__setattr__(key, inputmsg[key])
return report
@staticmethod
def _is_valid_input(msg):
l = map(lambda x: x in msg.keys(), ExpenseReport.MANDATORY_FIELDS)
x = reduce(lambda x,y: x&y, l)
return x
################################
#### INSTANCE METHODS ####
################################
def store(self, db,state):
if (state.is_valid()):
self.log('Persisting report')
db.write_record(self)
def log(self, msg):
self.logger.write(msg)
################################
#### INSTANCE METHODS ####
################################
def __str__(self):
msg = 'Expense Report printer:\n'
for key in self.__dict__.keys():
msg += ' %s: %s\n' % (key, self.__dict__[key])
msg += '-------------------------'
return msg
class Expenses:
def __init__(self):
self.logger = Logger("WEB")
def GET(self):
web.header('Access-Control-Allow-Origin', '*')
return 'Listing expenses not yet implemented'
def POST(self, resource_id=None):
web.header('Access-Control-Allow-Origin', '*')
jsondata = web.data()
self.logger.write(jsondata)
resp = api.API().add_expense(jsondata)
return resp
def __init__(self, dataset, weights=None, norms=None,
keep_invalids=False, minvalid=None, clean_weights=True,
logger=None, loglevel=None, zerofill=False, **kwargs):
# Logger
Logger.__init__(self, logger=logger, loglevel=loglevel,
**dict_filter(kwargs, 'log_'))
# Input shape
if isinstance(dataset, (list, tuple)):
dataset = list(dataset)
self.map = len(dataset)
else:
dataset = [dataset]
self.map = 0
self.ndataset = self.nd = len(dataset)
self.dataset = dataset
# Other inits
self.data = []
self.nt = None
weights = self.remap(weights, reshape=True)
norms = self.remap(norms, reshape=True)
if self.ndataset==1 and norms[0] is None: norms = [False]
self._invalids = []
self.masked = False
# Loop on datasets
for idata,data in enumerate(dataset):
# Create the Data instance and pack array
dd = Data(data, norm=norms[idata], weights=weights[idata],
keep_invalids=keep_invalids, minvalid=minvalid, clean_weights=clean_weights,
zerofill=zerofill)
self.data.append(dd)
self._invalids.append(dd.invalids)
self.masked |= dd.masked
# Check nt
if self.nt is None:
self.nt = dd.nt
elif self.nt != dd.nt:
self.error('Time dimension of variable %i must have length %i (not %i)'%(idata, self.nt, dd.nt))
# Merge
self.stacked_data = npy.asfortranarray(npy.vstack([d.packed_data for d in self.data]))
self.splits = npy.cumsum([d.packed_data.shape[0] for d in self.data[:-1]])
self.stacked_weights = npy.hstack([d.packed_weights for d in self.data])
self.ns = self.stacked_data.shape[0]
self.ntv = (self.stacked_data!=default_missing_value).any(axis=0).sum()
def __init__(self, in_size, out_size, activation_fn=None, activation_fn_deriv=None):
Logger.debug("create Neural Layer: \n" + \
" in_size: " + str(in_size) + "\n" + \
" out_size: " + str(out_size))
# initialize weight matrix
self.weights = np.random.uniform(-0.001, 0.001, (out_size, in_size)).astype(np.float64)
# set biases if not already set (by child class for example)
if not hasattr(self, 'biases'):
self.biases = np.zeros(out_size).astype(np.float64)
# set activation function and derivate
self.activation = activation_fn or NeuralLayer.activation_linear
self.activation_deriv = activation_fn_deriv or NeuralLayer.activation_linear_deriv
# set size and input size
self.size = out_size
self.in_size = in_size
def train_test(self, Xtrain, Ytrain, Xtest):
Xtrain = np.array(Xtrain)
Ytrain = np.array(Ytrain)
Xtest = np.array(Xtest)
tasks = []
task_id = 0
results = []
l = Logger(len(self.split_keys) * len(self.model_params), \
len(self.split_keys), tag='SplitModel')
for key in self.split_keys:
mask_train = Xtrain[:,self.index] == key
mask_test = Xtest[:,self.index] == key
for model_key, model_type, model_params in self.model_params:
task = (
task_id,
model_type,
model_params,
Xtrain[mask_train],
Ytrain[mask_train],
Xtest[mask_test],
model_key,
key,
)
results.append(tt(task))
print (task_id, model_key, key)
tasks.append((task_id, model_key, key))
l.step()
task_id += 1
tasks = {t[0]: t[1:] for t in tasks}
result_batches = [np.array([0.0] * len(Xtest))\
for i in range(len(self.model_params))]
for result_set in results:
task_id, Ytask = result_set
task = tasks[task_id]
model_key = task[-2]
mask_test = Xtest[:,self.index] == task[-1]
result_batches[model_key][mask_test] += Ytask
Ytest = np.array([0.0] * len(Xtest))
for (i, batch) in enumerate(result_batches):
Ytest += batch * self.weights[i]
if self.bias:
Ytest += self.bias
return (Ytest, result_batches)
def parse(self, utterance):
'''
Parses and returns result.
Returns:
RobotCommand
'''
rc = self.parser.parse(utterance)
self.parse_buffer = Logger.get_buffer()
return rc
def _set_world_internal(self, world_objects, robot):
'''
Sets the parser's world, robot, parser maybe regenerates.
This is so we can capture the log for reporting (if desired).
Args:
world_objects ([WorldObject])
robot ([Robot])
'''
self.parser.set_world(world_objects, robot)
self.start_buffer = Logger.get_buffer()
def ground(self, grounding_query):
'''
Grounds the provided expression with probabilities over objs.
Args:
grounding_query (str)
Returns:
{str: float}: Map of obj : P(obj).
'''
gprobs = self.parser.ground(grounding_query)
self.parse_buffer = Logger.get_buffer()
return gprobs
def describe(self, grab_buffer=True):
'''
Describes all objects in the world.
Args:
grab_buffer (bool, optional): Whether to grab the log buffer
and save it as self.parse_buffer. Defaults to True.
Returns:
{str: str}: Map of object names to their description.
'''
desc = self.parser.describe()
if grab_buffer:
self.parse_buffer = Logger.get_buffer()
return desc
def learn(self, result, delta, learning_rate):
#raw_input("press Enter")
# apply learning rule
#Logger.DEBUG = True
Logger.debug("NeuralLayer:learn")
Logger.debug("result: " + str(np.shape(result)))
Logger.debug("delta: " + str(np.shape(delta)))# + "\nresult shape:" + str(np.shape(result)))
delta_weights = learning_rate * np.outer(delta, result)
#Logger.debug("delta weights shape:" + str(np.shape(delta_weights)))
#Logger.log(str(delta_weights))
self.weights += delta_weights
def feed(self, input_data):
# calculate activation of layer for given inputs
#Logger.DEBUG = True
Logger.debug("NeuralLayer:feed")
Logger.debug("input: " + str(np.shape(input_data)))
dot = np.dot(self.weights, input_data)
result = self.activation(dot + np.atleast_2d(self.biases).T)
Logger.debug("output: " + str(np.shape(result)))
#Logger.debug("weights: " + str(np.shape(self.weights)))
#Logger.debug("dot shape: " + str(np.shape(dot)))
#Logger.DEBUG = False
return result
def __init__(self, env, id=None, db=None, messageid=None, row=None):
self.env = env
self.db = db
self.log = Logger(env)
if id is not None:
self.resource = Resource('mailarchive', str(id), None)
self._fetch_mail(id)
elif messageid is not None:
self._fetch_mail_by_messageid(messageid)
self.resource = Resource('mailarchive', self.id, None)
elif row is not None:
self._fetch_mail_by_row(row)
self.resource = Resource('mailarchive', self.id, None)
else:
self.messageid = ''
self.subject = ''
self.utcdate = 0
self.localdate = ''
self.zoneoffset = 0
self.body = ''
def setUpClass(cls):
Logger.init_log()
cls.db_client = FileClient(dir=path)
cls.db_client.connect()
def __init__(self, data, weights=None, norm=None, keep_invalids=False,
minvalid=None, clean_weights=True,
logger=None, loglevel=None, zerofill=False, **kwargs):
# Logger
Logger.__init__(self, logger=logger, loglevel=loglevel, **dict_filter(kwargs, 'log_'))
# Guess data type and copy
if cdms2_isVariable(data):
self.array_type = 'MV2'
self.array_mod = MV2
data = data.clone()
elif npy.ma.isMA(data):
self.array_type = 'numpy.ma'
self.array_mod = numpy.ma
data = data.copy()
else:
self.array_type = 'numpy'
data = data.copy()
self.array_mod = numpy
self.data = data
self.dtype = data.dtype
data = data.astype('d')
# Shape
self.shape = data.shape
self.ndim = data.ndim
self.nt = self.shape[0]
self.nstot = data.size/self.nt
self.nsdim = data.ndim-1
# Check time axis
if cdms2_isVariable(data) and data.getTime() is not None:
order = data.getOrder()
if not order.startswith('t'):
warn('Time axis is not the first axis of input variable (order="%s")'%order)
# Weights ?
if weights is None or weights is False:
if False and weights is not False and data.ndim == 3 and \
cdms2_isVariable(data) and \
'x' in data.getOrder() and 'y' in data.getOrder():
import cdutil# FIXME: WARNING FALSE
weights = cdutil.area_weights(data[0]).data.astype('d') # Geographic weights
elif self.nstot==1:
weights = npy.ones(1)
else:
weights = npy.ones(self.shape[1:])
elif npy.ma.isMA(weights):
weights = weight.astype('d').filled(0.)
else:
weights = npy.asarray(weights, dtype='d')
if data.ndim>1 and self.shape[1:] != weights.shape:
self.error('Weights must be of shape %s (instead of %s)'
%(self.shape[1:], weights.shape))
# Store some info
# - time
if not cdms2_isVariable(data):
self.taxis = data.shape[0]
else:
self.taxis = data.getAxis(0)
# - others axes and attributes
if cdms2_isVariable(data): # cdms -> ids
self.saxes = data.getAxisList()[1:]
self.id = data.id
self.atts = {}
for att in data.listattributes():
self.atts[att] = data.attributes[att]
self.grid = data.getGrid()
data = data.asma()
else: # numpy -> length
self.saxes = data.shape[1:]
self.id = None
self.atts = None
self.grid = None
# - missing value
if npy.ma.isMA(data):
self.missing_value = data.get_fill_value()
else:
self.missing_value = 1.e20
# - special cases
for att in 'long_name', 'units':
if hasattr(data, att):
setattr(self, att, data.attributes[att])
# Masking nans
nans = npy.isnan(data)
if nans.any():
self.warning("Masking %i NaNs"%nans.sum())
if self.array_type == 'numpy':
self.array_type = 'numpy.ma'
self.array_mod = numpy.ma
data = npy.ma.array(data, mask=nans, copy=False)
else:
data[nans] = npy.ma.masked
self.data = data
# Mask (1 means good)
# - real good values
bmask = npy.ma.getmaskarray(data)
good = 1-bmask.astype('l')
# - first from data (integrate) => 1D
count = npy.atleast_1d(good.sum(axis=0))
del good
# - now remove channels where weight is zero
if clean_weights:
count[npy.atleast_1d(weights==0.)] = 0
# - check number of valid data along time
minvalid = kwargs.pop('nvalid', minvalid)
if minvalid is not None and minvalid < 0:
minvalid = -int(round(npy.clip(minvalid, -100., 0)*self.nt/100))
minvalid = npy.clip(int(minvalid), 1, self.nt) if minvalid is not None else 1
count[count<minvalid] = 0 # <minvalid -> 0
count = npy.clip(count, 0, 1)
# - save as 0/1
self.ns = long(count.sum())
self.compress = count.size != self.ns
self.good = count>0 # points in space where there are enough data in time
self.minvalid = self.nvalid = minvalid
# Scale unpacked data
if not self.good.any():
self.warning('No valid data')
self.norm = 1.
self.mean = 0
else:
# - mean
self.mean = data.mean(axis=0)
# - normalisation factor
if norm is True or norm is None:
norm = self.data.std() # Standard norm
elif norm is not False:
if norm <0: # Relative norm, else strict norm
norm = abs(norm)*self.data.std()
else:
norm = 1.
self.norm = norm
# - apply
self.scale(data)
# Fill data
# - fill with missing value or mean (0.) where possible
if minvalid != self.nt:
# invalids = bmask & self.good # invalids = masked data that will be analyzed
# data[invalids] = 0. if zerofill else default_missing_value
# data[invalids] = default_missing_value
data[:, ~self.good] = default_missing_value
if keep_invalids:
self.invalids = bmask & self.good # invalids = masked data that will be analyzed
else:
self.invalids = None
#del invalids
else:
self.invalids = None
# - finally fill with missing values at zero
if npy.ma.isMA(data):
data_num = data.filled(default_missing_value)
else:
data_num = data
# Pack
# - data
self.packed_data = self.core_pack(data_num, force2d=True)
self.masked = npy.isclose(self.packed_data, default_missing_value).any()
# - weights
self.packed_weights = self.core_pack(weights)
def __init__(self):
self.logger = Logger("WEB")
files.sort()
pprint(files)
cfiles = glob.glob(os.path.join(path, multiplefilepath + '*_initialcentroids*'))
cfiles.sort()
plotexperiments = []
# points = np.zeros(()) #Use for data seen so far experiment
numexperiments = 10
for x in xrange(numexperiments): # experiments
if x != 3:
continue
fileoutpostfix = '_mrjobmultiple_%s_%si' % (x, maxiterations)
logger = Logger.function_logger(str(x), fileoutpostfix, logging.DEBUG, logging.INFO, logging.DEBUG)
initialMeans = np.loadtxt(cfiles[x]) #separate initial file, #Deprecated
initialcentroidsinputfile = cfiles[x]
filenameinitcentroids = os.path.basename(initialcentroidsinputfile)
oldCentroids = initialMeans #separate initial centroids
numberfiles = 10
for count in xrange(numberfiles): #files
mrinputfile = files[count]
# Use for current data set experiment
points = np.loadtxt(mrinputfile)
def __init__(self):
self.logger = Logger('REPORT')
def main():
global args
args = parser.parse_args()
# fix random seeds
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
np.random.seed(args.seed)
# CNN
if args.verbose:
print('Architecture: {}'.format(args.arch))
model = models.__dict__[args.arch](sobel=args.sobel)
fd = int(model.top_layer.weight.size()[1])
model.top_layer = None
model.features = torch.nn.DataParallel(model.features)
model.cuda()
cudnn.benchmark = True
# create optimizer
optimizer = torch.optim.SGD(
filter(lambda x: x.requires_grad, model.parameters()),
lr=args.lr,
momentum=args.momentum,
weight_decay=10**args.wd,
)
# define loss function
criterion = nn.CrossEntropyLoss().cuda()
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
# remove top_layer parameters from checkpoint
for key in checkpoint['state_dict']:
if 'top_layer' in key:
del checkpoint['state_dict'][key]
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
# creating checkpoint repo
exp_check = os.path.join(args.exp, 'checkpoints')
if not os.path.isdir(exp_check):
os.makedirs(exp_check)
# creating cluster assignments log
cluster_log = Logger(os.path.join(args.exp, 'clusters'))
# preprocessing of data
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
tra = [transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize]
# load the data
end = time.time()
dataset = datasets.ImageFolder(args.data, transform=transforms.Compose(tra))
if args.verbose: print('Load dataset: {0:.2f} s'.format(time.time() - end))
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=args.batch,
num_workers=args.workers,
pin_memory=True)
# clustering algorithm to use
deepcluster = clustering.__dict__[args.clustering](args.nmb_cluster)
# training convnet with DeepCluster
for epoch in range(args.start_epoch, args.epochs):
end = time.time()
# remove head
model.top_layer = None
model.classifier = nn.Sequential(*list(model.classifier.children())[:-1])
# get the features for the whole dataset
features = compute_features(dataloader, model, len(dataset))
# cluster the features
clustering_loss = deepcluster.cluster(features, verbose=args.verbose)
# assign pseudo-labels
train_dataset = clustering.cluster_assign(deepcluster.images_lists,
dataset.imgs)
# uniformely sample per target
sampler = UnifLabelSampler(int(args.reassign * len(train_dataset)),
deepcluster.images_lists)
train_dataloader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch,
num_workers=args.workers,
sampler=sampler,
pin_memory=True,
)
# set last fully connected layer
mlp = list(model.classifier.children())
mlp.append(nn.ReLU(inplace=True).cuda())
model.classifier = nn.Sequential(*mlp)
model.top_layer = nn.Linear(fd, len(deepcluster.images_lists))
model.top_layer.weight.data.normal_(0, 0.01)
model.top_layer.bias.data.zero_()
model.top_layer.cuda()
# train network with clusters as pseudo-labels
end = time.time()
loss = train(train_dataloader, model, criterion, optimizer, epoch)
# print log
if args.verbose:
print('###### Epoch [{0}] ###### \n'
'Time: {1:.3f} s\n'
'Clustering loss: {2:.3f} \n'
'ConvNet loss: {3:.3f}'
.format(epoch, time.time() - end, clustering_loss, loss))
try:
nmi = normalized_mutual_info_score(
clustering.arrange_clustering(deepcluster.images_lists),
clustering.arrange_clustering(cluster_log.data[-1])
)
print('NMI against previous assignment: {0:.3f}'.format(nmi))
except IndexError:
pass
print('####################### \n')
# save running checkpoint
torch.save({'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer' : optimizer.state_dict()},
os.path.join(args.exp, 'checkpoint.pth.tar'))
# save cluster assignments
cluster_log.log(deepcluster.images_lists)
class Mail(object):
"""model for the Mail."""
id_is_valid = staticmethod(lambda num: 0 < int(num) <= 1L << 31)
def __init__(self, env, id=None, db=None, messageid=None, row=None):
self.env = env
self.db = db
self.log = Logger(env)
if id is not None:
self.resource = Resource('mailarchive', str(id), None)
self._fetch_mail(id)
elif messageid is not None:
self._fetch_mail_by_messageid(messageid)
self.resource = Resource('mailarchive', self.id, None)
elif row is not None:
self._fetch_mail_by_row(row)
self.resource = Resource('mailarchive', self.id, None)
else:
self.messageid = ''
self.subject = ''
self.utcdate = 0
self.localdate = ''
self.zoneoffset = 0
self.body = ''
def __eq__(self, other):
if isinstance(other, Mail):
return self.messageid == other.messageid
return super.__eq__(other)
def _get_db(self):
if self.db:
return self.db
else:
return self.env.get_db_cnx()
def _get_db_for_write(self):
if self.db:
return (self.db, False)
else:
return (self.env.get_db_cnx(), True)
def get_sanitized_fromaddr(self):
return self.fromaddr.replace('@',
self.env.config.get('mailarchive',
'replaceat', '@'))
def get_fromtext(self):
return get_author(self.fromname, self.fromaddr)
def get_category(self):
yearmonth = time.strftime("%Y%m", time.gmtime(self.utcdate))
category = self.mlid + yearmonth
return category.encode('utf-8')
def get_plain_body(self):
return self._sanitize(self.env, self.body)
def get_html_body(self, req):
# for HTML Mail
if self.body.lstrip().startswith('<'):
return Markup(self.body)
contentlines = self.body.splitlines()
htmllines = ['',]
#customize!
#http://d.hatena.ne.jp/ohgui/20090604/1244114483
wikimode = req.args.get('wikimode', 'on')
for line in contentlines:
if self.env.config.get('mailarchive', 'wikiview',' enabled') == 'enabled' and wikimode == 'on':
htmllines.append(wiki_to_oneliner(line, self.env, self.db, False, False, req))
else:
htmllines.append(Markup(Markup().escape(line).replace(' ',' ')))
content = Markup('<br/>').join(htmllines)
return content
def _sanitize(self, env, text):
return text.replace('@', env.config.get('mailarchive', 'replaceat','_at_') )
def _fetch_mail(self, id):
row = None
if self.id_is_valid(id):
db = self._get_db()
cursor = db.cursor()
cursor.execute(SELECT_FROM_MAILARC + " WHERE id=%s", (id,))
row = cursor.fetchone()
if not row:
raise ResourceNotFound('Mail %s does not exist.' % id,
'Invalid Mail Number')
self._fetch_mail_by_row(row)
def _fetch_mail_by_messageid(self, messageid):
row = None
db = self._get_db()
cursor = db.cursor()
cursor.execute(SELECT_FROM_MAILARC + " WHERE messageid=%s",
(messageid,))
row = cursor.fetchone()
if not row:
raise ResourceNotFound('Mail messageid %s does not exist.' % messageid,
'Invalid Mail messageid Number')
self._fetch_mail_by_row(row)
def _fetch_mail_by_row(self, row):
self.id = row[0]
self.messageid = row[1]
self.utcdate = row[2]
self.zoneoffset = row[3]
self.subject = row[4]
self.fromname = row[5]
self.fromaddr = row[6]
self.header =row[7]
self.body = row[8]
self.thread_root = row[9]
self.thread_parent = row[10]
self.zone = self._to_zone(self.zoneoffset)
self.localdate = self._to_localdate(self.utcdate, self.zoneoffset)
def _to_localdate(self, utcdate, zoneoffset):
return time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(utcdate + zoneoffset))
def _to_zone(self, zoneoffset):
#zone and date
zone = ''
if zoneoffset == '':
zoneoffset = 0
if zoneoffset > 0:
zone = ' +' + time.strftime('%H%M', time.gmtime(zoneoffset))
elif zoneoffset < 0:
zone = ' -' + time.strftime('%H%M', time.gmtime(-1 * zoneoffset))
return zone
def get_href(self, req):
return req.href.mailarchive(self.id)
def get_subject(self):
if is_empty(self.subject):
return '(no subject)'
else:
return self.subject
def get_senddate(self):
return self.localdate + self.zone
def get_thread_root(self):
if self.thread_root == '':
return self
try:
root_mail = Mail(self.env, messageid=self.thread_root)
except ResourceNotFound:
return self
#self.thread_rootはオリジナル版だと親のメールになってしまっている。
#互換性維持のため、ルートではない場合は自力で探しにいくロジックを走らす
if root_mail.thread_root == '':
return root_mail
else:
if self.thread_parent != '':
root_id = MailFinder.find_root_id(self.env, self.messageid)
return Mail(self.env, messageid=root_id)
def get_thread_parent_id(self):
if self.thread_parent != '':
return self.thread_parent.split(' ')[0]
return None
def get_thread_parent(self):
if self.thread_parent != '':
return Mail(self.env, db=self.db, messageid=self.get_thread_parent_id())
return self
def get_children(self, desc=False, cached_mails=None):
if cached_mails:
self.log.debug("[%s] mail's threads is cached." % self.id)
return [x for x in cached_mails if x.get_thread_parent_id() == self.messageid]
db = self._get_db()
cursor = db.cursor()
sql = SELECT_FROM_MAILARC + " WHERE threadparent LIKE %s ORDER BY utcdate"
if desc:
sql += " DESC"
cursor.execute(sql, ('%s %%' % self.messageid,))
children = []
for row in cursor:
child_mail = Mail(self.env, row=row, db=self.db)
children.append(child_mail)
return children
def get_thread_mails(self, desc=False):
root = self.get_thread_root()
db = self._get_db()
cursor = db.cursor()
sql = SELECT_FROM_MAILARC + " WHERE threadroot = %s ORDER BY utcdate"
if desc:
sql += " DESC"
cursor.execute(sql, (root.messageid,))
mails = []
for row in cursor:
mails.append(Mail(self.env, row=row, db=self.db))
return mails
def has_children(self, cached_mails=None):
rtn = len(self.get_children(cached_mails=cached_mails)) > 0
return rtn
def get_related_tickets(self, req):
db = self._get_db()
return get_related_tickets(self.env, req, db, self.id)
def has_attachments(self, req):
attachment = MailArchiveAttachment(self.env, self.id)
return attachment.has_attachments(req)
def populate(self, author, msg, mlid):
"""Populate the mail with 'suitable' values from a message"""
if 'message-id' not in msg:
raise 'Illegal Format Mail!'
self.is_new_mail = False
self.mlid = mlid
self._parse_messageid(msg)
self._parse_date(msg)
self._parse_subject(msg)
if msg.is_multipart():
self._parse_multipart(author, msg)
else:
self._parse_body(msg)
ref_messageid = self._parse_reference(msg)
self._make_thread(ref_messageid)
def update_or_save(self):
if self.messageid is None or self.messageid == '':
raise "Can't save mail to database."
db, has_tran = self._get_db_for_write()
cursor = db.cursor()
yearmonth = time.strftime("%Y%m", time.gmtime(self.utcdate))
category = self.mlid + yearmonth
cursor.execute("SELECT category, mlid, yearmonth, count FROM mailarc_category WHERE category=%s",
(category.encode('utf-8'),))
row = cursor.fetchone()
count = 0
if row:
count = row[3]
pass
else:
cursor.execute("INSERT INTO mailarc_category (category, mlid, yearmonth, count) VALUES(%s, %s, %s, %s)",
(category.encode('utf-8'),
self.mlid.encode('utf-8'),
yearmonth,
0))
if self.is_new_mail:
count = count + 1
cursor.execute("UPDATE mailarc_category SET count=%s WHERE category=%s",
(count, category.encode('utf-8')))
# insert or update mailarc
#self.log.debug(
# "VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)" %(str(id),
# category.encode('utf-8'),
# messageid,
# utcdate,
# zoneoffset,
# subject.encode('utf-8'), fromname.encode('utf-8'),
# fromaddr.encode('utf-8'),'','',
# thread_root,thread_parent))
cursor.execute("DELETE FROM mailarc where messageid=%s",
(self.messageid,))
cursor.execute("INSERT INTO mailarc ("
"id, category, messageid, utcdate, zoneoffset, subject,"
"fromname, fromaddr, header, text, threadroot, threadparent) "
"VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)",
(str(self.id),
category.encode('utf-8'),
self.messageid,
self.utcdate,
self.zoneoffset,
self.subject.encode('utf-8'), self.fromname.encode('utf-8'),
self.fromaddr.encode('utf-8'), '', self.body.encode('utf-8'),
self.thread_root, self.thread_parent))
if has_tran:
db.commit()
def _parse_messageid(self, msg):
self.messageid = msg['message-id'].strip('<>')
#check messageid is unique
self.log.debug("Creating new mailarc '%s'" % 'mailarc')
db = self._get_db()
cursor = db.cursor()
cursor.execute("SELECT id from mailarc WHERE messageid=%s", (self.messageid,))
row = cursor.fetchone()
id = None
if row:
id = row[0]
if id == None or id == "":
# why? get_last_id return 0 at first.
#id = db.get_last_id(cursor, 'mailarc')
self.is_new_mail = True
cursor.execute("SELECT Max(id)+1 as id from mailarc")
row = cursor.fetchone()
if row and row[0] != None:
id = row[0]
else:
id = 1
self.id = int(id) # Because id might be 'n.0', int() is called.
def _parse_date(self, msg):
if 'date' in msg:
datetuple_tz = email.Utils.parsedate_tz(msg['date'])
localdate = calendar.timegm(datetuple_tz[:9]) #toDB
zoneoffset = datetuple_tz[9] # toDB
utcdate = localdate - zoneoffset # toDB
#make zone ( +HHMM or -HHMM
zone = ''
if zoneoffset > 0:
zone = '+' + time.strftime('%H%M', time.gmtime(zoneoffset))
elif zoneoffset < 0:
zone = '-' + time.strftime('%H%M', time.gmtime(-1 * zoneoffset))
#self.log.debug( time.strftime("%y/%m/%d %H:%M:%S %z",datetuple_tz[:9]))
self.log.debug(time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(utcdate)))
self.log.debug(time.strftime("%Y/%m/%d %H:%M:%S", time.gmtime(localdate)))
self.log.debug(zone)
fromname, fromaddr = email.Utils.parseaddr(msg['from'])
self.fromname = self._decode_to_unicode(fromname)
self.fromaddr = self._decode_to_unicode(fromaddr)
self.zone = zone
self.utcdate = utcdate
self.zoneoffset = zoneoffset
self.localdate = self._to_localdate(utcdate, zoneoffset)
self.log.info(' ' + self.localdate + ' ' + zone +' '+ fromaddr)
def _parse_subject(self, msg):
if 'subject' in msg:
self.subject = self._decode_to_unicode(msg['subject'])
def _parse_reference(self, msg):
# make thread infomations
ref_messageid = ''
if 'in-reply-to' in msg:
ref_messageid = ref_messageid + msg['In-Reply-To'] + ' '
self.log.debug('In-Reply-To:%s' % ref_messageid )
if 'references' in msg:
ref_messageid = ref_messageid + msg['References'] + ' '
m = re.findall(r'<(.+?)>', ref_messageid)
ref_messageid = ''
for text in m:
ref_messageid = ref_messageid + "'%s'," % text
ref_messageid = ref_messageid.strip(',')
self.log.debug('RefMessage-ID:%s' % ref_messageid)
return ref_messageid
def _parse_multipart(self, author, msg):
body = ''
# delete all attachement at message-id
Attachment.delete_all(self.env, 'mailarchive', self.id, self.db)
for part in msg.walk():
content_type = part.get_content_type()
self.log.debug('Content-Type:' + content_type)
file_counter = 1
if content_type == 'multipart/mixed':
pass
elif content_type == 'text/html' and self._is_file(part) == False:
if body != '':
body += "\n------------------------------\n\n"
body = part.get_payload(decode=True)
charset = part.get_content_charset()
self.log.debug('charset:' + str(charset))
# Todo:need try
if charset != None:
body = self._to_unicode(body, charset)
elif content_type == 'text/plain' and self._is_file(part) == False:
#body = part.get_payload(decode=True)
if body != '':
body += "\n------------------------------\n\n"
current_body = part.get_payload(decode=True)
charset = part.get_content_charset()
self.log.debug('charset:' + str(charset))
# Todo:need try
if charset != None:
#body = self._to_unicode(body, charset)
body += self._to_unicode(current_body, charset)
else:
body += current_body
elif part.get_payload(decode=True) == None:
pass
# file attachment
else:
self.log.debug(part.get_content_type())
# get filename
# Applications should really sanitize the given filename so that an
# email message can't be used to overwrite important files
filename = self._get_filename(part)
if not filename:
import mimetypes
ext = mimetypes.guess_extension(part.get_content_type())
if not ext:
# Use a generic bag-of-bits extension
ext = '.bin'
filename = 'part-%03d%s' % (file_counter, ext)
file_counter += 1
self.log.debug("filename:" + filename.encode(OUTPUT_ENCODING))
# make attachment
tmp = os.tmpfile()
tempsize = len(part.get_payload(decode=True))
tmp.write(part.get_payload(decode=True))
tmp.flush()
tmp.seek(0,0)
attachment = Attachment(self.env, 'mailarchive', self.id)
attachment.description = '' # req.args.get('description', '')
attachment.author = author #req.args.get('author', '')
attachment.ipnr = '127.0.0.1'
try:
attachment.insert(filename,
tmp, tempsize, None, self.db)
except Exception, e:
try:
ext = filename.split('.')[-1]
if ext == filename:
ext = '.bin'
else:
ext = '.' + ext
filename = 'part-%03d%s' % (file_counter, ext)
file_counter += 1
attachment.description += ', Original FileName: %s' % filename
attachment.insert(filename,
tmp, tempsize, None, self.db)
self.log.warn('As name is too long, the attached file is renamed : ' + filename)
except Exception, e:
self.log.error('Exception at attach file of Message-ID:' + self.messageid)
traceback.print_exc(e)
tmp.close()
class Watcher(object):
exchanges = {}
def __init__(self, settings):
self.set_settings(settings)
self.L = Logger(settings)
self.L.log('Setup watcher for %s' % (self.exchange_names), 'info')
self.load_exchanges(settings)
def set_settings(self, settings):
self.trade_threshold = settings['trade_threshold']
if self.trade_threshold <= 0:
raise Error('settings variable trade_threshold must be above 0!')
self.exchange_names = settings['exchanges']
self.poll_interval = settings['poll_interval']
def load_exchanges(self, settings):
c_name = '%sExchange'
modules = zip(self.exchange_names, [__import__('src.exchanges', fromlist=[str(c_name%e)]) for e in self.exchange_names])
exchange_classes = [(e, getattr(module, c_name % e)) for e, module in modules]
for name, klass in exchange_classes:
self.exchanges[name] = klass(settings)
self.L.log('Loaded exchanges %s' % self.exchanges, 'info')
def find_trade(self):
buys = [(name, exch.buy_price()) for name, exch in self.exchanges.iteritems()]
sells = [(name, exch.sell_price()) for name, exch in self.exchanges.iteritems()]
#find the minimum buy and the max sell price
min_buy = min(buys, key = lambda x: x[1])
max_sell = max(sells, key = lambda x : x[1])
if max_sell[1] - min_buy[1] > self.trade_threshold:
self.L.log('Possible Trade opportunity:', 'info')
self.L.log('Buy from %s @ %s and sell to %s @ %s' % (min_buy + max_sell), 'info')
else:
self.L.log('No trading opportunity', 'info')
self.L.log('Min buy from %s @ %s | Max sell to %s @ %s' % (min_buy + max_sell), 'info')
def watch(self):
while True:
self.find_trade()
time.sleep(self.poll_interval)
def __init__(self, settings):
self.set_settings(settings)
self.L = Logger(settings)
self.L.log('Setup watcher for %s' % (self.exchange_names), 'info')
self.load_exchanges(settings)
# Delete a row from the table
self.db_client.delete(
table_name,
"k=2")
# Fetch the whole table
row = self.db_client.select(table=table_name)
self.assertEqual(len(row), 2)
self.assertEqual(row[0][0], 1)
self.assertEqual(row[0][1], "20161026")
self.assertEqual(row[0][2], "10:00:00.000000")
self.assertEqual(row[0][3], 'AbC')
self.assertEqual(row[0][4], 10.3)
self.assertEqual(row[1][0], 3)
self.assertEqual(row[1][1], "20161026")
self.assertEqual(row[1][2], "10:00:02.000000")
self.assertEqual(row[1][3], 'Efgh')
self.assertEqual(row[1][4], 10.5)
# Delete remaining rows from the table
self.db_client.delete(table_name)
# Fetch the whole table
row = self.db_client.select(table=table_name)
self.assertEqual(len(row), 0)
if __name__ == '__main__':
Logger.init_log()
unittest.main()
