def find_layers(pkts, pcap): packet = OrderedDict() count = 1 try: for p in pkts: header = {"Buffer": {"timestamp": datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f'), "packetnumber": count, "pcapfile": pcap}} packet.update(header) counter = 0 while True: layer = p.getlayer(counter) if (layer != None): i = int(counter) x = p[0][i].fields t = exclude_layers(x, layer.name) s = rename_layer(t, layer.name) v = '{"' + layer.name.replace('.','_') + '[' + str(i) + ']' + '":' + str(s) + '}' s = eval(v) packet.update(s) else: break counter += 1 count += 1 yield packet packet.clear() except Exception as e: print RED + 'Error within packet: ' + str(count) + ', on layer: ' + layer.name + ' . The full error was: ' + str(e) + END
def test_clear(self): pairs = [("c", 1), ("b", 2), ("a", 3), ("d", 4), ("e", 5), ("f", 6)] shuffle(pairs) od = OrderedDict(pairs) self.assertEqual(len(od), len(pairs)) od.clear() self.assertEqual(len(od), 0)
class OrderedDict(object): def __init__(self, eq_func=None, hash_func=None): self.contents = PyOrderedDict() self.eq_func = eq_func or operator.eq self.hash_func = hash_func or hash def __getitem__(self, key): return self.contents[self._key(key)] def __setitem__(self, key, value): self.contents[self._key(key)] = value def __delitem__(self, key): del self.contents[self._key(key)] def __contains__(self, key): return self._key(key) in self.contents def __len__(self): return len(self.contents) def _key(self, key): return DictKey(self, key) def keys(self): return [k.key for k in self.contents.keys()] def values(self): return self.contents.values() def iteritems(self): for k, v in self.contents.iteritems(): yield k.key, v def get(self, key, default): return self.contents.get(self._key(key), default) def pop(self, key, default=MARKER): if default is MARKER: return self.contents.pop(self._key(key)) else: return self.contents.pop(self._key(key), default) def popitem(self): if not self: raise KeyError k, v = self.contents.popitem() return k.key, v def update(self, d): self.contents.update(d.contents) def clear(self): self.contents.clear() def copy(self): d = OrderedDict(self.eq_func, self.hash_func) for k, v in self.iteritems(): d[k] = v return d
def find_layers(pkts, pcap, pcap_id, streamid): packet = OrderedDict() count = 1 pcap_id = pcap_id.encode('utf-8') streamid = streamid.encode('utf-8') try: for p in pkts: header = {"Buffer": {"timestamp": datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f'), "packetnumber": count, "PCAP ID": pcap_id, "pcapfile": pcap, "StreamID": streamid}} packet.update(header) counter = 0 while True: layer = p.getlayer(counter) if layer != None: i = int(counter) x = p[0][i].fields t = exclude_layers(x, layer.name) s = rename_layer(t, layer.name) v = '{"' + layer.name.replace('.', '_') + '[' + str(i) + ']' + '":' + str(s) + '}' s = eval(v) try: del s['HTTP[3]'] del s['HTTP[5]'] except KeyError: pass packet.update(s) else: break counter += 1 count += 1 yield packet packet.clear() except Exception as e: error_logging(str(e), 'PacketParser') pass
class Tweak(object): # {{{ def __init__(self, name, doc, var_names, defaults, custom): translate = _ self.name = translate(name) self.doc = doc.strip() self.doc = ' ' + self.doc self.var_names = var_names if self.var_names: self.doc = u"%s: %s\n\n%s"%(_('ID'), self.var_names[0], format_doc(self.doc)) self.default_values = OrderedDict() for x in var_names: self.default_values[x] = defaults[x] self.custom_values = OrderedDict() for x in var_names: if x in custom: self.custom_values[x] = custom[x] def __str__(self): ans = ['#: ' + self.name] for line in self.doc.splitlines(): if line: ans.append('# ' + line) for key, val in iteritems(self.default_values): val = self.custom_values.get(key, val) ans.append(u'%s = %r'%(key, val)) ans = '\n'.join(ans) if isinstance(ans, unicode_type): ans = ans.encode('utf-8') return ans @property def sort_key(self): return 0 if self.is_customized else 1 @property def is_customized(self): for x, val in iteritems(self.default_values): cval = self.custom_values.get(x, val) if normalize_tweak(cval) != normalize_tweak(val): return True return False @property def edit_text(self): from pprint import pformat ans = ['# %s'%self.name] for x, val in iteritems(self.default_values): val = self.custom_values.get(x, val) if isinstance(val, (list, tuple, dict, set, frozenset)): ans.append(u'%s = %s' % (x, pformat(val))) else: ans.append(u'%s = %r'%(x, val)) return '\n\n'.join(ans) def restore_to_default(self): self.custom_values.clear() def update(self, varmap): self.custom_values.update(varmap)
def get_available_skins(selected=None): """Returns a dictionary of skin name --> directory where "templates" and "media" subdirectories can be found. selected is a name of preferred skin if it's None, then information about all skins will be returned otherwise, only data about selected and default skins will be returned selected skin is guaranteed to be the first item in the dictionary """ skins = OrderedDict() extra_skins_dir = getattr(django_settings, 'ASKBOT_EXTRA_SKINS_DIR', None) if extra_skins_dir: skins.update(get_skins_from_dir(extra_skins_dir)) if 'default' in skins: raise ValueError('"default" is not an acceptable name for a custom skin') if selected in skins: selected_dir = skins[selected] skins.clear() skins[selected] = selected_dir elif selected == 'default': skins = OrderedDict() elif selected: raise ValueError( 'skin ' + str(selected) + \ ' not found, please check ASKBOT_EXTRA_SKINS_DIR setting ' + \ 'or in the corresponding directory' ) #insert default as a last item skins['default'] = askbot.get_install_directory() return skins
def main(): base_path = os.path.dirname(__file__) xml_file = os.path.join(base_path, "Survey-4.xml") csv_file = os.path.join(base_path, "Survey-4.csv") csv_headers_map = {"PIVA": "CUSTOMER_ID", "Email_Aziendale_Cliente": "ADDRESS", "Codice_Ticket": "SURVEY_ID"} csv_headers_required = ["CUSTOMER_ID", "ADDRESS", "SURVEY_ID", "LANGUAGE"] csv_headers_meta = None try: xml_tree = ET.parse(xml_file) except ET.ParseError as e: print(str.format("[#] parser error: {}", e)) sys.exit(1) csv_headers = OrderedDict() for item in csv_headers_required: csv_headers[item] = None print(str.format("[*] csv_headers: {}", csv_headers)) print(str.format("[*] csv_headers.key(): {}", csv_headers.keys())) print(str.format("[*] csv_headers.values(): {}", csv_headers.values())) print(str.format("[*] len(csv_headers): {}", len(csv_headers))) csv_headers.clear() print(str.format("[*] len(csv_headers): {}", len(csv_headers)))
def packet_summary(pkts, pcap): packet = OrderedDict() count = 1 try: for p in pkts: p_header = {"Buffer": {"timestamp": datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f'), "packetnumber": count, "pcapfile": pcap, "packet_length": p.len}} packet.update(p_header) if p.haslayer(Ether): p_ether = {"Ethernet": {"ether_src": p[Ether].src, "ether_dst": p[Ether].dst}} packet.update(p_ether) if p.haslayer(IP): p_ip = {"IP": {"ip_src": p[IP].src, "ip_dst": p[IP].dst, "ip_ttl": p[IP].ttl}} packet.update(p_ip) if p.haslayer(TCP): p_tcp = {"TCP": {"tcp_sport": p[TCP].sport, "tcp_dport": p[TCP].dport, "tcp_flags": p[TCP].flags}} packet.update(p_tcp) if p.haslayer(UDP): p_udp = {"UDP": {"udp_sport": p[UDP].sport, "udp_dport": p[UDP].dport, "udp_len": p[UDP].len}} packet.update(p_udp) count += 1 yield packet packet.clear() except Exception, e: print e pass
class SubModel: def __init__(self, params=None, save_path=(), shared=False, copy_shared=False): self.params = OrderedDict() if params is None else params # string (param identifier) -> parameter self.save_path = save_path self.shared = shared self.copy_shared = copy_shared def save_sub_model(self, d, *args): self.get_sub_dict(d).update(args + (("param_keys", list(self.params.keys())),)) return list(self.params.values()) def load_sub_model(self, d, *args, load_path=None): d = self.get_sub_dict(d, load_path=load_path) param_keys = d.get("param_keys", ()) assert len(param_keys) <= len(args), "%s loaded values: expected %d, got %d" % ("/".join(self.save_path), len(param_keys), len(args)) self.params.clear() self.params.update(zip(param_keys, args)) return d def get_sub_dict(self, d, load_path=None): for element in load_path or self.save_path: d = d.setdefault(element, OrderedDict()) return d def params_str(self): return "/".join(self.save_path) + (": " if self.save_path else "") + ", ".join(self.params.keys()) def invalidate_caches(self): for model in self.sub_models(): model.invalidate_caches() def sub_models(self): return ()
class AliasGenerator(object): def __init__(self, size=200): self.MAX_CACHE = size # TODO: Make configurable self.cache = OrderedDict() def generate_alias(self, str_value): """ Generate an alias for the given string. :param str_value: the given string :return: a 24 character alias string """ alias = self.get_alias(str_value) if not alias: h = hashlib.md5() h.update(str_value.encode('utf-8')) alias = h.hexdigest() self.cache[str_value] = alias if len(self.cache) > self.MAX_CACHE: self.cache.popitem(last=False) # FIFO else: alias = alias return alias def needs_alias(self, str_value): return len(str_value) > MAX_NAME_LEN or len(RE_CLEANSE.findall(str_value)) > 0 def get_alias(self, str_value): return self.cache.get(str_value) def clear_aliases(self): self.cache.clear()
def test_clear(self): pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)] shuffle(pairs) od = OrderedDict(pairs) self.assertEqual(len(od), len(pairs)) od.clear() self.assertEqual(len(od), 0)
class Records(object): def __init__(self): self._data = OrderedDict() self._stamps = {} def __iter__(self): return self._data.iteritems() def __contains__(self, key): return key in self._data def keys(self): return self._data.keys() def update(self, data): self._data.update(data) self._stamps.update(dict.fromkeys(data.keys(), time())) def pluck(self, t): return OrderedDict(filter(lambda item:self._stamps[item[0]]>=t, self._data.items())) def clear(self): self._data.clear() self._stamps.clear() def get(self, key): return self._data[key]
class LRUCache(Cache): """ Implements a simple Least Recently Used Cache Very simple using collections.OrderedDict. The size can be changed during run-time """ def __init__(self, size_limit): super(LRUCache, self).__init__() self._size_limit = size_limit self._cache = OrderedDict() @property def count(self): return len(self._cache), 0 @property def size(self): return self.size_limit, 0 @property def size_limit(self): return self._size_limit @size_limit.setter def size_limit(self, new_size): if new_size < self.size_limit: self._check_size_limit() self._size_limit = new_size def __iter__(self): return iter(self._cache) def __reversed__(self): return reversed(self._cache) def __getitem__(self, item): obj = self._cache.pop(item) self._cache[item] = obj return obj def __setitem__(self, key, value, **kwargs): self._cache[key] = value self._check_size_limit() def _check_size_limit(self): while len(self._cache) > self.size_limit: self._cache.popitem(last=False) def __contains__(self, item): return item in self._cache def clear(self): self._cache.clear() def __len__(self): return len(self._cache)
class LRUCacheDict(object): """ A dictionary-like object, supporting LRU caching semantics. """ def __init__(self, max_size=1024, expiration=15*60): self.max_size = max_size self.expiration = expiration self.__values = {} self.__expire_times = OrderedDict() self.__access_times = OrderedDict() def size(self): return len(self.__values) def clear(self): self.__values.clear() self.__expire_times.clear() self.__access_times.clear() def has_key(self, key): return self.__values.has_key(key) def __setitem__(self, key, value): t = int(time()) self.__delete__(key) self.__values[key] = value self.__access_times[key] = t self.__expire_times[key] = t + self.expiration self.cleanup() def __getitem__(self, key): t = int(time()) del self.__access_times[key] self.__access_times[key] = t self.cleanup() return self.__values[key] def __delete__(self, key): if self.__values.has_key(key): del self.__values[key] del self.__expire_times[key] del self.__access_times[key] def cleanup(self): if self.expiration is None: return None t = int(time()) #Delete expired for k in self.__expire_times.iterkeys(): if self.__expire_times[k] < t: self.__delete__(k) else: break #If we have more than self.max_size items, delete the oldest while (len(self.__values) > self.max_size): for k in self.__access_times.iterkeys(): self.__delete__(k) break
class Error_manager(): def __init__(self): self.objec_error_dict = OrderedDict() self.Lines_With_Errors = [] def log_error(self, object_name, verbose = True): self.objec_error_dict[object_name] = format_exc() if verbose: print '\n-- Error logged' def log_emLine_error(self, CodeName, Label): #Case for the first error log for a given object if Label not in self.objec_error_dict: LinesList = [[],[]] #Case the objects has more lines with an error else: LinesList = self.objec_error_dict[CodeName] LinesList[0].append([Label, format_exc()]) self.objec_error_dict[CodeName] = LinesList def display_errors(self, error_type = 'general', extra_verbose = False): if error_type == 'general': if len(self.objec_error_dict) != 0: print '\n-These objects treatment produced a script error:\n' print self.objec_error_dict.keys(),'\n' for i in range(len(self.objec_error_dict)): print '--Object', self.objec_error_dict.keys()[i], print self.objec_error_dict.values() #We empty the dictionary as we don not need it anymore self.objec_error_dict.clear() elif error_type == 'emLine measurement': if len(self.objec_error_dict) != 0: print '\n-These objects treatment produced a script error:\n' print self.objec_error_dict.keys(),'\n' for i in range(len(self.objec_error_dict)): print '--Object', self.objec_error_dict.keys()[i] print self.objec_error_dict.values()[i][0], '\n' if extra_verbose == True: for i in range(len(self.objec_error_dict)): print '--Object', self.objec_error_dict.keys()[i] print '--Label', self.objec_error_dict.values()[i][0], '\n' print '--Errors', self.objec_error_dict.values()[i][1], '\n' return ' '
class TagBase(): def __init__(self, out_file): self.items = OrderedDict() self.out_file = out_file def add_items(self, item_pairs): for p in item_pairs: self.items[p[0]] = p[1] # this function is called after ALL tags are added from ALL ini files # default behavior is to clear added tags here # If certain tag type required to set default value, it can override this function def reserve_in_default_list(self): self.items.clear() def Print(self): for t in self.items: sys.stdout.write('\t%s := %s\n' % (t, self.items[t])) print '' def commit(self): lines = [] lines += self.out_header() for it in self.items: lines += self.gen_tag_format(it, self.items[it]) lines += self.out_tailer() fp = open(self.out_file, 'a+') for l in lines: fp.write('%s\n' % l) fp.close() def isYesTag(self, tag): return tag.lower() == 'yes' def out_header(self): return [] def out_tailer(self): return [] def isStringInOutFile(self, text): if not self.out_file in out_text_appear_once_record: out_text_appear_once_record[self.out_file] = [] in_record = text in out_text_appear_once_record[self.out_file] out_text_appear_once_record[self.out_file].append(text) return in_record def out_text_appear_once(self, text): if self.isStringInOutFile(text): return [] else: return [text] def remove_text_from_out(self, text): if path.exists(self.out_file): lines = open(self.out_file).readlines() lines.remove(text+'\n') open(self.out_file, 'w').writelines(lines)
class Classifier(object): """Parent factory for all classifier classes.""" __metaclass__ = Factory def __init__(self): super(Classifier, self).__init__() self._pp = None # setup preprocessor in the train method self.model = None self._clf = None self._classes = OrderedDict() def description(self): return def setParameters(self, params): assert isinstance(params, dict) self._clf.set_params(**params) def setupPreProcessor(self, features): self._pp = PreProcessor(features) def normalize(self, features): return self._pp(features) @property def classes(self): return self._classes def classByName(self, name): for class_ in self.classes.itervalues(): if class_.name == name: return class_ raise RuntimeError("No classes named %s" %name) def setClasses(self, classes, parent, panel): """Update the dictionary of the class definition and the list of the context menu actions.""" self._classes.clear() self._classes.update(classes) @classmethod def classifiers(cls): return cls._classes.keys() def parameterWidget(self, parent): return QtWidgets.QWidget(self) def train(self, features): raise NotImplementedError def predict(self, features): return [UnClassified]*features.shape[0] def saveToHdf(self, file_, feature_names): raise NotImplementedError
class Reaction(object): """docstring for Reaction""" def __init__(self, arg, **kwargs): self.id = arg self.name = kwargs.get('name', self.id) self.reversible = kwargs.get('reversible', None) self.participants = OrderedDict() self.lower_bound = -1e4 self.upper_bound = 1e4 self.default_bounds = (self.lower_bound, self.upper_bound) self.objective_coefficient = 0.0 self.flux_value = None self.notes = {} self.genes = [] self.lp_var = None def __str__(self): reactants = [] products = [] for p in self.participants: if self.participants[p] < 0: reactants.append(' %03.1f %s ' % (abs(self.participants[p]), p.name)) else: products.append(' %03.1f %s ' % (abs(self.participants[p]), p.name)) if self.reversible: arrow = '<===>' else: arrow = '====>' return self.id + ' : ' + '+'.join(reactants) + ' ' + arrow + ' ' \ + '+'.join(products) + '\n' def __repr__(self): return self.id def clear_participants(self): """docstring for clear_participants""" for metabolite in self.participants: metabolite.participations.pop(self.id) self.participants.clear() def remove_participant(self, metabolite): """docstring for remove_participant""" metabolite.participations.pop(self.id) self.participants.pop(metabolite) def add_participant(self, metabolite, stoichiometry): """docstring for add_participant""" self.participants[metabolite] = stoichiometry metabolite.participations[self.id] = stoichiometry def set_default_bounds(self): """docstring for set_default_bounds""" self.default_bounds = (self.lower_bound, self.upper_bound) def reset_bounds(self): """docstring for reset_bounds""" self.lower_bound = self.default_bounds[0] self.upper_bound = self.default_bounds[1]
class Recording(object): """Recording Class is used for managing the recording video process.""" #----------------------------------------------------------------------# # Devices Class Constructor # #----------------------------------------------------------------------# def __init__(self): """Devices Class Constructor.""" # Creates a dictionary for managering multiple cameras. self.__videos = OrderedDict() self.__fourcc = cv2.VideoWriter_fourcc(*"WMV2") #----------------------------------------------------------------------# # Public Class Methods # #----------------------------------------------------------------------# def AddVideo(self, index, fps=30.0, size=(640, 480)): """Add a new video in the Python dictionary.""" print index key = str(index) if self.__videos.has_key(key): return False self.__videos[key] = cv2.VideoWriter(index, self.__fourcc, fps, size) return True def DelVideo(self, index): """Remove a video from the Python dictionary.""" key = str(index) if not self.__videos.has_key(key): return False self.__videos[key].release() del self.__videos[key] return True def Write(self, images): """Writes the next video frame.""" if type(images).__module__ == np.__name__: images = [images] for key, image in zip(self.__videos, images): self.__videos[key].write(image) def Release(self): """Closes all videos.""" for key in self.__videos: self.__videos[key].release() self.__videos.clear() #----------------------------------------------------------------------# # Class Methods # #----------------------------------------------------------------------# def __repr__(self): """Get a object representation in a string format.""" return "Framework.RecordingVideos.Recording object."
class Tweak(object): # {{{ def __init__(self, name, doc, var_names, defaults, custom): translate = _ self.name = translate(name) self.doc = doc.strip() if self.doc: self.doc = translate(self.doc) self.var_names = var_names if self.var_names: self.doc = u"%s: %s\n\n%s"%(_('ID'), self.var_names[0], self.doc) self.default_values = OrderedDict() for x in var_names: self.default_values[x] = defaults[x] self.custom_values = OrderedDict() for x in var_names: if x in custom: self.custom_values[x] = custom[x] def __str__(self): ans = ['#: ' + self.name] for line in self.doc.splitlines(): if line: ans.append('# ' + line) for key, val in self.default_values.iteritems(): val = self.custom_values.get(key, val) ans.append('%s = %r'%(key, val)) ans = '\n'.join(ans) if isinstance(ans, unicode): ans = ans.encode('utf-8') return ans def __cmp__(self, other): return -1 * cmp(self.is_customized, getattr(other, 'is_customized', False)) @property def is_customized(self): for x, val in self.default_values.iteritems(): if self.custom_values.get(x, val) != val: return True return False @property def edit_text(self): ans = ['# %s'%self.name] for x, val in self.default_values.iteritems(): val = self.custom_values.get(x, val) ans.append('%s = %r'%(x, val)) return '\n\n'.join(ans) def restore_to_default(self): self.custom_values.clear() def update(self, varmap): self.custom_values.update(varmap)
def clear(self): """ Remove all keys. See: help(OrderedDict.clear) >>> T = pykov.Matrix({('A','B'): .3, ('A','A'): .7, ('B','A'): 1.}) >>> T.clear() >>> T {} """ OrderedDict.clear(self)
def getOutlinks(pmid, preferPmc=False): """ use NCBI eutils to get outlinks for a pmid as a dict provider -> url """ logging.debug("%s: Getting outlink from pubmed" % (pmid)) url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&id=%s&retmode=llinks&cmd=llinks" % pmid #logging.debug("getting %s" % url) #try: #except #logging.info(traceback.format_exc()) #logging.info("Exception when downloading") #return None userAgent = 'User-Agent: Mozilla ([email protected], http://text.soe.ucsc.edu)' #html = urllib2.urlopen(req) html = maxCommon.retryHttpRequest(url, userAgent=userAgent) if html==None: return None outlinks = OrderedDict() provider = False fullText = False isPmc = False for line in html: if line.find("<ObjUrl>") != -1: url="" fullText=False origPublisher=False if line.find("Attribute") != -1: attribute=stripTag(line) if attribute.lower()=="full-text online" or attribute=="full-text pdf": fullText=True if line.find("<NameAbbr>") != -1 and fullText and origPublisher: db = stripTag(line) outlinks[db]=url if line.find("publishers/providers")!=-1: origPublisher=True if line.find("<Provider>") != -1: provider=True if line.find("</Provider>") != -1: provider=False if line.find("<DbFrom>") != -1: db = stripTag(line) if line.find("<Url>") != -1 and not provider: url = line url = stripTag(url).replace("&", "&") # XX strange! url = stripTag(url).replace("<", "<") url = stripTag(url).replace(">", ">") if "www.ncbi.nlm.nih.gov" in url and "/pmc/" in url and preferPmc: # override all other links outlinks.clear() outlinks["pmc"] = url logging.debug("%s: Found outlinks %s" % (pmid, str(outlinks))) return outlinks
class FuncCollection(object): '''A selection of functions ''' def __init__(self, funcitems=None): self.funcs = OrderedDict() if funcitems: if not isinstance(funcitems, list): funcitems = [funcitems] for item in funcitems: self.append(item) def append(self, item, attr_path='nodeid'): # self.funcs[tosymbol(attrgetter(attr_path)(item))] = item self.funcs[attrgetter(attr_path)(item)] = item def addtests(self, test_set): for item in test_set._items: self.append(item) def remove(self, item): self.funcs.pop(item.nodeid, None) def removetests(self, test_set): for item in test_set._items: self.remove(item) def clear(self): self.funcs.clear() def keys(self): return self.funcs.keys() def values(self): return self.funcs.values() def __len__(self): return len(self.funcs) def __getitem__(self, key): if isinstance(key, int): return self.enumitems()[key][1] if isinstance(key, (int, slice)): return list(map(itemgetter(1), self.enumitems()[key])) return self.funcs[key] def __dir__(self): return dirinfo(self) def items(self): return self.funcs.items() def enumitems(self, items=None): if not items: items = self.funcs.values() return [(i, node) for i, node in enumerate(items)]
class memoize_mask(object): ''' Decorator. Caches wktToMask keyed to model_id and the WKT string ''' def __init__(self, func, maxsize=50): ''' Args: func: the function to wrap maxsize (int): Max size of cache (in MB) ''' self.hits = self.misses = 0 self.func = func self.maxsize = maxsize self.cache = OrderedDict() def __call__(self, *args): nc, fname, wkt, varname = args # If we have no key, automatic cache miss if (not hasattr(nc, 'model_id')): log.debug('Cache MISS (attribute \'model_id\' not found)') return self.func(*args) # Set key to model_id and wkt polygon key = (nc.model_id, wkt) log.debug('Checking cache for key %s', key) with cache_lock: try: result = self.cache[key] self.cache.move_to_end(key) # record recent use of this key self.hits += 1 log.debug('Cache HIT') return result except KeyError: pass log.debug('Cache MISS') result = self.func(*args) with cache_lock: self.cache[key] = result self.misses += 1 while getsize(self.cache) > self.maxsize * 1024 * 1024: # convert to MB self.cache.popitem(0) # Purge least recently used cache entry return result def cache_clear(self): with cache_lock: self.cache.clear() self.hits = 0 self.misses = 0
def clear(self): """ clear all keys and keys in key_time_map """ with self.lock: try: OrderedDict.clear(self) self.key_time_map.clear() return True except KeyError: return False
class CaselessDict(OrderedDict): """Case-insensitive dict. Values can be set and retrieved using keys of any case. Note that when iterating, the original case is returned for each key.""" def __init__(self, *args): OrderedDict.__init__(self, {}) ## requirement for the empty {} here seems to be a python bug? self.keyMap = OrderedDict([(k.lower(), k) for k in OrderedDict.keys(self)]) if len(args) == 0: return elif len(args) == 1 and isinstance(args[0], dict): for k in args[0]: self[k] = args[0][k] else: raise Exception("CaselessDict may only be instantiated with a single dict.") #def keys(self): #return self.keyMap.values() def __setitem__(self, key, val): kl = key.lower() if kl in self.keyMap: OrderedDict.__setitem__(self, self.keyMap[kl], val) else: OrderedDict.__setitem__(self, key, val) self.keyMap[kl] = key def __getitem__(self, key): kl = key.lower() if kl not in self.keyMap: raise KeyError(key) return OrderedDict.__getitem__(self, self.keyMap[kl]) def __contains__(self, key): return key.lower() in self.keyMap def update(self, d): for k, v in d.iteritems(): self[k] = v def copy(self): return CaselessDict(OrderedDict.copy(self)) def __delitem__(self, key): kl = key.lower() if kl not in self.keyMap: raise KeyError(key) OrderedDict.__delitem__(self, self.keyMap[kl]) del self.keyMap[kl] def __deepcopy__(self, memo): raise Exception("deepcopy not implemented") def clear(self): OrderedDict.clear(self) self.keyMap.clear()
def item_assignment(demand, max_capa): ''' Demand is the list of the requested capacities that need to be fitted in to bins. Each requested capacity is not bigger than 10 In this simulation, there are 4 bin with different maximum capacity. We will see how the demand will fit these 4 bins in BFD. ''' # Sorting the demand list values = sorted(demand, reverse = True) bins = [] for i in range(4): bins.append(Bin(max_capa)) for index, bin in enumerate(bins): print "Index: %s, bin: %s" %(index+1, bin) print "\nThe list of demands: %s \n" % values from collections import OrderedDict fittable_bins = OrderedDict() for item in values[:]: # Starts fitting each requested capa to bin print ("\n%s Starting a new fitting %s" %(('*'*30), ('*'*30))) print ("\nThe requested capacity is going to be checked: %s" %item) for index, bin in enumerate(bins): print "\n******** BEFORE FITTING" print "Bin: %s, Value:%s" %(index+1, bin) if bin.capacity + item <= bin.max_capa: # Calculate the current capa of each bin current_capa = bin.capacity + item # Store the fittable to the fittable_bins fittable_bins[bin] = (current_capa, index) if fittable_bins: print "\nThe fittable_bins:" for bin, value in fittable_bins.items(): print "Bin: %s, Value:%s" %(value[1]+1, bin) else: print "\nThere is NO fittable_bins" if fittable_bins: max_current_capa = max(fittable_bins.values(), key=lambda x:x[0])[0] for bin, value in fittable_bins.items(): if value[0] == max_current_capa: bin.bin_append(item) values.remove(item) fittable_bins.clear() print "\n******** AFTER FITTING" print "Bin: %s, Value:%s" %(value[1]+1, bin) print "\nThe demaining requested capacities: %s \n" %values break return bins
class DeviceManager: """Handles creation and destruction of local device drivers and controller RPC clients.""" def __init__(self, ddb, virtual_devices=dict()): self.ddb = ddb self.virtual_devices = virtual_devices self.active_devices = OrderedDict() def get_device_db(self): """Returns the full contents of the device database.""" return self.ddb.get_device_db() def get_desc(self, name): desc = self.ddb.get(name) while isinstance(desc, str): # alias desc = self.ddb.get(desc) return desc def get(self, name): """Get the device driver or controller client corresponding to a device database entry.""" if name in self.virtual_devices: return self.virtual_devices[name] if name in self.active_devices: return self.active_devices[name] else: try: desc = self.get_desc(name) except Exception as e: raise DeviceError("Failed to get description of device '{}'" .format(name)) from e try: dev = _create_device(desc, self) except Exception as e: raise DeviceError("Failed to create device '{}'" .format(name)) from e self.active_devices[name] = dev return dev def close_devices(self): """Closes all active devices, in the opposite order as they were requested.""" for dev in reversed(list(self.active_devices.values())): try: if isinstance(dev, (Client, BestEffortClient)): dev.close_rpc() elif hasattr(dev, "close"): dev.close() except Exception as e: logger.warning("Exception %r when closing device %r", e, dev) self.active_devices.clear()
class DualPriorityQueueHeap(): def __init__(self): self.strings = OrderedDict() self.next_index=-1 def _get_next_index(self): self.next_index += 1 return self.next_index def enqueue(self, val_a, val_b, value): '''This method accepts three parameters - a string, priority value A, and priority value B, where the priority values are real numbers (see above). The string is inserted into the priority queue with the given priority values A and B''' ix = self._get_next_index() self.strings[ix] = ((val_a, val_b, value)) def _dequeue(self, index): ''' Adds entries from enqueued strings to priority queue based on provided index. ''' heap = [] for key, value in self.strings.iteritems(): heappush(heap, (value[index], key)) item = heappop(heap) return self.strings.pop(item[1])[2] def dequeue_a(self): '''This method removes and returns the string from the priority queue with the highest priority A value. If two entries have the same A priority, return whichever was enqueued first.''' return self._dequeue(0) def dequeue_b(self): '''This method removes and returns the string from the priority queue with the highest priority B value. If two entries have the same B priority, return whichever was enqueued first.''' return self._dequeue(1) def count(self): '''This method returns the number of strings in the queue.''' return len(self.strings) def clear(self): '''This removes all entries from the priority queue.''' self.strings.clear() def dequeue_first(self): '''This removes and returns the string from the priority queue that was enqueued first, ignoring priority.''' return self.strings.popitem(last=False)[1][2]
class LRUCache: # @param capacity, an integer def __init__(self, capacity=None): self.capacity = capacity self.__cache = OrderedDict() @property def lru(self): return list(self.__cache.keys()) @property def length(self): return len(self.__cache) def clear(self): self.__cache.clear() def __len__(self): return self.length def __contains__(self, item): return item in self.__cache def __setitem__(self, key, value): self.set(key, value) def __delitem__(self, key): del self.__cache[key] def __getitem__(self, key): return self.get(key) def get(self, key, default=None): value = self.__cache.get(key) if value: del self.__cache[key] self.__cache[key] = value return value return default def set(self, key, value): if self.__cache.get(key): del self.__cache[key] self.__cache[key] = value else: self.__cache[key] = value # Check, if the cache is full and we have to remove old items # If the queue is of unlimited size, self.capacity is NaN and # x > NaN is always False in Python and the cache won't be cleared. if self.capacity is not None and self.length > self.capacity: self.__cache.popitem(last=False)
class ErrorHandler: """ Global configurable application exception handler. For "basic" errors (I/O errors, keyboard interrupt, etc.) just the error message is printed as there's generally no need to confuse the user with a complete stack trace when it's just a missing file. Other exceptions, however, are logged with the usual full stack trace. The configuration can be augmented with other exception classes that should be handled specially by treating the instance as a dictionary mapping exception classes to :class:`ErrorAction` tuples (or any 2-tuple, which will be converted to an :class:`ErrorAction`). For example:: >>> from pibootctl.term import ErrorAction, ErrorHandler >>> import sys >>> sys.excepthook = ErrorHandler() >>> sys.excepthook[KeyboardInterrupt] (None, 1) >>> sys.excepthook[SystemExit] (None, <function ErrorHandler.exc_value at 0x7f6178915e18>) >>> sys.excepthook[ValueError] = (sys.excepthook.exc_message, 3) >>> sys.excepthook[Exception] = ("An error occurred", 1) >>> raise ValueError("foo is not an integer") foo is not an integer Note the lack of a traceback in the output; if the example were a script it would also have exited with return code 3. """ def __init__(self): self._config = OrderedDict([ # Exception type, (handler method, exit code) (SystemExit, (None, self.exc_value)), (KeyboardInterrupt, (None, 2)), (argparse.ArgumentError, (self.syntax_error, 2)), ]) @staticmethod def exc_message(exc_type, exc_value, exc_tb): """ Extracts the message associated with the exception (by calling :class:`str` on the exception instance). The result is returned as a one-element list containing the message. """ return [str(exc_value)] @staticmethod def exc_value(exc_type, exc_value, exc_tb): """ Returns the first argument of the exception instance. In the case of :exc:`SystemExit` this is the expected return code of the script. """ return exc_value.args[0] @staticmethod def syntax_error(exc_type, exc_value, exc_tb): """ Returns the message associated with the exception, and an additional line suggested the user try the ``--help`` option. This should be used in response to exceptions indicating the user made an error in their command line. """ return ErrorHandler.exc_message(exc_type, exc_value, exc_tb) + [ _('Try the --help option for more information.'), ] def clear(self): """ Remove all pre-defined error handlers. """ self._config.clear() def __len__(self): return len(self._config) def __contains__(self, key): return key in self._config def __getitem__(self, key): return self._config[key] def __setitem__(self, key, value): self._config[key] = ErrorAction(*value) def __delitem__(self, key): del self._config[key] def __call__(self, exc_type, exc_value, exc_tb): for exc_class, (message, value) in self._config.items(): if issubclass(exc_type, exc_class): if callable(message): message = message(exc_type, exc_value, exc_tb) if callable(value): value = value(exc_type, exc_value, exc_tb) if message is not None: for line in message: print(line, file=sys.stderr) sys.stderr.flush() raise SystemExit(value) # Otherwise, log the stack trace and the exception into the log # file for debugging purposes for line in traceback.format_exception(exc_type, exc_value, exc_tb): for msg in line.rstrip().split('\n'): print(msg, file=sys.stderr) sys.stderr.flush() raise SystemExit(1)
class RecordWriter(object): def __init__(self, ofile, maxresultrows=None): self._maxresultrows = 50000 if maxresultrows is None else maxresultrows self._ofile = ofile self._fieldnames = None self._buffer = StringIO() self._writer = csv.writer(self._buffer, dialect=CsvDialect) self._writerow = self._writer.writerow self._finished = False self._flushed = False self._inspector = OrderedDict() self._chunk_count = 0 self._record_count = 0 self._total_record_count = 0 @property def is_flushed(self): return self._flushed @is_flushed.setter def is_flushed(self, value): self._flushed = True if value else False @property def ofile(self): return self._ofile @ofile.setter def ofile(self, value): self._ofile = value def flush(self, finished=None, partial=None): assert finished is None or isinstance(finished, bool) assert partial is None or isinstance(partial, bool) assert not (finished is None and partial is None) assert finished is None or partial is None self._ensure_validity() def write_message(self, message_type, message_text, *args, **kwargs): self._ensure_validity() self._inspector.setdefault('messages', []).append( (message_type, message_text.format(*args, **kwargs))) def write_record(self, record): self._ensure_validity() self._write_record(record) def write_records(self, records): self._ensure_validity() write_record = self._write_record for record in records: write_record(record) def _clear(self): self._buffer.seek(0) self._buffer.truncate() self._inspector.clear() self._record_count = 0 self._flushed = False def _ensure_validity(self): if self._finished is True: assert self._record_count == 0 and len(self._inspector) == 0 raise RuntimeError('I/O operation on closed record writer') def _write_record(self, record): fieldnames = self._fieldnames if fieldnames is None: self._fieldnames = fieldnames = list(record.keys()) value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)), fieldnames) self._writerow(list(chain.from_iterable(value_list))) get_value = record.get values = [] for fieldname in fieldnames: value = get_value(fieldname, None) if value is None: values += (None, None) continue value_t = type(value) if issubclass(value_t, (list, tuple)): if len(value) == 0: values += (None, None) continue if len(value) > 1: value_list = value sv = '' mv = '$' for value in value_list: if value is None: sv += '\n' mv += '$;$' continue value_t = type(value) if value_t is not bytes: if value_t is bool: value = str(value.real) elif value_t is six.text_type: value = value elif value_t is int or value_t is int or value_t is float or value_t is complex: value = str(value) elif issubclass(value_t, (dict, list, tuple)): value = str(''.join( RecordWriter._iterencode_json(value, 0))) else: value = repr(value).encode( 'utf-8', errors='backslashreplace') sv += value + '\n' mv += value.replace('$', '$$') + '$;$' values += (sv[:-1], mv[:-2]) continue value = value[0] value_t = type(value) if value_t is bool: values += (str(value.real), None) continue if value_t is bytes: values += (value, None) continue if value_t is six.text_type: if six.PY2: value = value.encode('utf-8') values += (value, None) continue if value_t is int or value_t is int or value_t is float or value_t is complex: values += (str(value), None) continue if issubclass(value_t, dict): values += (str(''.join(RecordWriter._iterencode_json(value, 0))), None) continue values += (repr(value), None) self._writerow(values) self._record_count += 1 if self._record_count >= self._maxresultrows: self.flush(partial=True) try: # noinspection PyUnresolvedReferences from _json import make_encoder except ImportError: # We may be running under PyPy 2.5 which does not include the _json module _iterencode_json = JSONEncoder(separators=(',', ':')).iterencode else: # Creating _iterencode_json this way yields a two-fold performance improvement on Python 2.7.9 and 2.7.10 from json.encoder import encode_basestring_ascii @staticmethod def _default(o): raise TypeError(repr(o) + ' is not JSON serializable') _iterencode_json = make_encoder( {}, # markers (for detecting circular references) _default, # object_encoder encode_basestring_ascii, # string_encoder None, # indent ':', ',', # separators False, # sort_keys False, # skip_keys True # allow_nan ) del make_encoder
class Strange(object): __strategy_id = 1001 __account_id = 10001 def __init__(self, limit_order): self.__bar_list = [] self.__pred = Prediciton() self.__pred.init_model(1,'../test/model/') self.__limit_order = OrderedDict() # 委托单队列 self.long_volumes = OrderedDict() #当前多头持仓情况 self.short_volumes = OrderedDict() #当前空头持仓情况 # 当日行情结束后,进行导出,用于分析 self.__history_limit_order = OrderedDict() # 历史委托单 self.__history_limit_volumes = OrderedDict() # 历史成交记录 self.__working_limit_order = limit_order # 交易记录存储 self.__trader_record = OrderedDict() # 暂时将账户和合约初始化放在策略内初始化 # 账户初始化 self.__account = Account() self.__account.set_account_id(self.__account_id) self.__account.set_init_cash(DEFAULT_CASH) #期货合约初始化 self.__instrument = Instrument() self.__instrument.set_fee(0.00005) self.__instrument.set_short_margin_ratio(0.07) self.__instrument.set_long_margin_ratio(0.07) self.__instrument.set_price_tick(1) self.__instrument.set_min_limit_order_volume(15) self.__instrument.set_instrument_id('ag') self.__instrument.set_instrument_name('白银') def __reset(self): self.__limit_order.clear() self.__history_limit_order.clear() self.__history_limit_order.clear() self.__working_limit_order.clear() def record(self): self.__account.dump() record = DailyRecord() record.set_close_profit(self.__account.close_profit()) record.set_position_profit(self.__account.position_profit()) record.set_commssion(self.__account.commission()) record.set_limit_volume(self.__history_limit_volumes) record.set_limit_order(self.__history_limit_order) record.set_mktime(self.__bar_list[-1].mktime()) MLog.write().info('mkdate:%d, available_cash:%f,profit:%f'%(record.mktime(),self.__account.available_cash(), record.all_profit())) self.__trader_record[record.mktime()] = record self.__account.reset() self.__reset() def calc_result(self): max_profit = 0.0 total_profit = 0.0 base_value = 0.0 for key,value in self.__trader_record.items(): total_profit += value.all_profit() starting_cash = self.__account.starting_cash() base_value = (int(abs(total_profit) / starting_cash) + 1) * self.__account.starting_cash() MLog.write().info('all:%f, base_value:%f',total_profit,base_value) total_profit = 0.0 df = pd.DataFrame(columns = ['mktime','interests','value','retrace','chg','log_chg','profit']) # 计算每日权益,每日净值, 涨跌幅,回撤,日对数收益 last_value = 0.0 for mktime,daily_record in self.__trader_record.items(): daily_record.set_base_value(base_value) daily_record.set_last_value(last_value) daily_record.set_max_profit(max_profit) daily_record.set_last_profit(total_profit) daily_record.calc_result() daily_record.dump() df = df.append(pd.DataFrame({'mktime':mktime, 'interests':daily_record.interests(), 'value':daily_record.value(), 'retrace': daily_record.retrace(), 'chg':daily_record.chg(),'log_chg':daily_record.log_chg(), 'profit':daily_record.all_profit()},index=['mktime']), ignore_index=True) total_profit += daily_record.all_profit() if max_profit < total_profit: max_profit = total_profit last_value = daily_record.value() base_value += daily_record.all_profit() chg_mean = np.mean(df['log_chg']) chg_std = np.std(df['log_chg']) summary_record = SummaryRecord() summary_record.set_chg_mean(chg_mean) summary_record.set_chg_std(chg_std) summary_record.set_trade_count(df.shape[0]) summary_record.set_final_value(df['value'].values[-1]) summary_record.set_max_retr(np.min(df['retrace'])) summary_record.calc_record(df[df['chg'] > 0.00].shape[0]) summary_record.dump() df.to_csv('result_strange.csv', encoding = 'utf-8') def save(self): ''' filename = 'strange' + '.csv' fieldnames = ['mktime','interests','value','retrace','chg','log_chg','profit',] with open(filename, 'w') as csvfile: writer = csv.DictWriter(csvfile, fieldnames=fieldnames) writer.writeheader() for mktime,daily_record in self.__trader_record.items(): writer.writerow({'mktime':mktime,'interests':daily_record.interests(),'value':daily_record.value(),'retrace':daily_record.retrace(), 'chg':daily_record.chg(),'log_chg':daily_record.log_chg(),'profit':daily_record.all_profit()}) ''' # close, high, low open 下一分钟收盘价 def __on_fc_single(self): for bar in self.__bar_list: if self.__bar_list.index(bar) == 0: x = np.array([bar.close_price(), bar.high_price(), bar.low_price(), bar.open_price(),bar.volume(), bar.last_buy_price(),bar.last_buy_vol(),bar.last_sell_price(),bar.last_sell_vol()]) else: t = np.array([bar.close_price(), bar.high_price(), bar.low_price(), bar.open_price(),bar.volume(), bar.last_buy_price(),bar.last_buy_vol(),bar.last_sell_price(),bar.last_sell_vol()]) x = np.vstack((x,t)) close_price = self.__pred.new_signal(x) return close_price # pred = Prediciton() # price = pred.signal(x) # return price # rand = int(np.std(x) * 1000 % 2) # if rand == 0: # return np.mean(x) + np.std(x) # else: # return np.mean(x) # 多头成本控制 def __is_long_order(self, order_price, close_price, next_price): long_fee = (order_price * self.__instrument.min_limit_order_volume() * self.__instrument.fee()) * 2 # 多头建仓手续费 long_profit = abs(order_price - next_price) * self.__instrument.min_limit_order_volume() # 多头最小利润 if long_profit > long_fee: MLog.write().debug('long_profit:%f more than long_fee:%f allow open long'%(long_profit, long_fee)) else: MLog.write().debug('long_profit:%f less than long_fee:%f not open long'%(long_profit, long_fee)) return 1 if long_profit > long_fee else 0 # 空头成本控制 def __is_short_order(self, order_price, close_price, next_price): short_fee = (order_price * self.__instrument.min_limit_order_volume() * self.__instrument.fee()) * 2 short_profit = abs(order_price - next_price) * self.__instrument.min_limit_order_volume() # 空头最小利润 if short_profit > short_fee: MLog.write().debug('short_profit:%f more than short_fee:%f allow open short'%(short_profit, short_fee)) else: MLog.write().debug('short_profit:%f less than short_fee:%f not open short'%(short_profit, short_fee)) return 1 if short_profit > short_fee else 0 def cancel_order(self): for oid, order in self.__limit_order.items(): del self.__working_limit_order[oid] del self.__limit_order[oid] order.set_status(OrderStatus.cacelled) self.__history_limit_order[oid] = order self.__account.canncel_order_cash(order.margin(), order.fee()) # 多头操作 def __long_operation(self, next_price, last_bar, amount): self.cancel_order() #是否持有多头仓 if len(self.long_volumes) == 0: if self.__is_long_order(last_bar.last_sell_price(), last_bar.close_price(), next_price) == 1: order = self.__long_open(last_bar.last_sell_price(), amount,last_bar.current_time()) order.dump() self.__account.insert_order_cash(order.margin(), order.fee()) self.__working_limit_order[order.order_id()] = order self.__limit_order[order.order_id()] = order self.__account.dump() else: # 已经持有多头仓 for k,v in self.long_volumes.items(): MLog.write().debug("hold long volumens:%d price:%f close_price:%f" %(v.amount(),v.limit_price(), last_bar.close_price())) #是否有空头仓 if len(self.short_volumes) > 0: MLog.write().debug("long singal start close short volume") for k,v in self.short_volumes.items(): order = self.__short_close(last_bar.last_sell_price(), v.amount(),last_bar.current_time()) order.set_hold_volume_id(v.trader_id()) self.__account.insert_order_cash(order.margin(), order.fee()) self.__working_limit_order[order.order_id()] = order self.__limit_order[order.order_id()] = order order.dump() self.__account.dump() # 空头操作 def __short_operation(self, next_price, last_bar, amount): self.cancel_order() # 是否持有空头仓 if len(self.short_volumes) == 0: if self.__is_short_order(last_bar.last_buy_price(), last_bar.close_price(), next_price) == 1: order = self.__short_open(last_bar.last_buy_price(), amount, last_bar.current_time()) order.dump() self.__account.insert_order_cash(order.margin(), order.fee()) self.__working_limit_order[order.order_id()] = order self.__limit_order[order.order_id()] = order self.__account.dump() else: for k,v in self.short_volumes.items(): MLog.write().debug("hold short volumens:%d price:%f close_price:%f" %(v.amount(),v.limit_price(),last_bar.close_price())) # 是否持有多头仓 if len(self.long_volumes) > 0: MLog.write().debug("short singal start close long volume") for k,v in self.long_volumes.items(): order = self.__long_close(last_bar.last_buy_price(), v.amount(), last_bar.current_time()) order.set_hold_volume_id(v.trader_id()) self.__account.insert_order_cash(order.margin(), order.fee()) self.__working_limit_order[order.order_id()] = order self.__limit_order[order.order_id()] = order order.dump() self.__account.dump() # 多头开仓 def __long_open(self, order_price, amount, bar_time): return self.__create_limit_price(order_price, amount, CombOffset.open, Direction.buy_direction,bar_time) # 多头平仓 def __long_close(self, order_price, amount, bar_time): return self.__create_limit_price(order_price, amount, CombOffset.close, Direction.sell_direction,bar_time) # 空头开仓 def __short_open(self, order_price, amount, bar_time): return self.__create_limit_price(order_price, amount, CombOffset.open, Direction.sell_direction,bar_time) # 空头平仓 def __short_close(self, order_price, amount, bar_time): return self.__create_limit_price(order_price, amount, CombOffset.close, Direction.buy_direction,bar_time) # 创建限价单 def __create_limit_price(self, order_price, amount, off_flag, direction,bar_time): # pdb.set_trace() order = Order() order.create_order_id() order.set_create_time(bar_time) order.set_account_id(self.__account_id) order.set_symbol(self.__instrument.instrument_id()) order.set_comb_offset_flag(off_flag) order.set_order_price_type(OrderPrice.limit_price) order.set_limit_price(order_price) order.set_direction(direction) order.set_amount(amount) order.set_strategy_id(self.__strategy_id) order.set_min_volume(self.__instrument.min_limit_order_volume()) if direction == Direction.buy_direction: order.set_margin(self.__instrument.long_margin_ratio()) else: order.set_margin(self.__instrument.short_margin_ratio()) order.set_fee(self.__instrument.fee()) return order def on_volume(self, vol, order): # 从委托队列中删除 # pdb.set_trace() del self.__limit_order[vol.order_id()] self.__history_limit_volumes[vol.trader_id()] = vol self.__history_limit_order[order.order_id()] = order vol.dump() # 判断是开仓还是平仓 if vol.comb_offset_flag() == CombOffset.open: # 开仓 if vol.direction() == Direction.buy_direction: # 多头仓 # self.long_volumes[vol.symbol()] = vol self.long_volumes[vol.trader_id()] = vol else: # 空头仓 # self.short_volumes[vol.symbol()] = vol self.short_volumes[vol.trader_id()] = vol self.__account.open_cash(order.margin(),vol.margin(), order.fee(), vol.fee()) else: # 平仓 # pdb.set_trace() MLog.write().debug("close position account:") if vol.direction() == Direction.buy_direction: # 平空头仓 if self.short_volumes.has_key(order.hold_volume_id()): v = self.short_volumes[order.hold_volume_id()] # self.__account.set_close_profit(v,vol) del self.short_volumes[order.hold_volume_id()] else: # 平多头仓 if self.long_volumes.has_key(order.hold_volume_id()): v = self.long_volumes[order.hold_volume_id()] # self.__account.set_close_profit(v,vol) del self.long_volumes[order.hold_volume_id()] self.__account.close_cash(v,vol,order.fee()) self.__account.dump() def on_order(self, order): if order.status() == OrderStatus.all_traded: MLog.write().debug('order is all traded') self.__limit_order[order.order_id()] = order self.__history_limit_order[order.order_id()] = order def on_bar(self, bar): self.__bar_list.append(bar) # 休盘的最后两分钟平仓 if len(self.__bar_list) < 5: return if len(self.__bar_list) > 5: self.__bar_list.pop(0) next_price = self.__on_fc_single() close_price = self.__bar_list[-1].close_price() # 距行情结束还有5分钟则全部平仓 if bar.current_time() > time.mktime(datetime.datetime(bar.mktime() / 10000,bar.mktime() / 100 % 100, bar.mktime() % 100,14,55,00).timetuple()): MLog.write().debug("time out start close position") if len(self.short_volumes) > 0: for k,v in self.short_volumes.items(): order = self.__short_close(bar.last_sell_price(), v.amount(),bar.current_time()) order.set_hold_volume_id(v.trader_id()) self.__account.insert_order_cash(order.margin(), order.fee()) self.__working_limit_order[order.order_id()] = order self.__limit_order[order.order_id()] = order order.dump() if len(self.long_volumes) > 0: for k,v in self.long_volumes.items(): order = self.__long_close(bar.last_buy_price(), v.amount(), bar.current_time()) order.set_hold_volume_id(v.trader_id()) self.__account.insert_order_cash(order.margin(), order.fee()) self.__working_limit_order[order.order_id()] = order self.__limit_order[order.order_id()] = order order.dump() return # pdb.set_trace() # 多仓,空仓判断 if close_price < next_price: # 多仓信号 MLog.write().debug("long singal start") self.__long_operation(next_price, self.__bar_list[-1], DEFAULT_AMOUNT) elif close_price > next_price: # 空仓信号 MLog.write().debug("short singal start") self.__short_operation(next_price, self.__bar_list[-1], DEFAULT_AMOUNT)
class GridSearch: def __init__(self, algorithm, param_grid, score_function, n_times=5, k_folds=10, n_top=50, bootstrap=True, evaluation_class=None, visualization=False): self.algorithm = algorithm self.param_grid = param_grid self.score_function = score_function self.n_times = n_times self.k_folds = k_folds self.bootstrap = bootstrap self.n_top = n_top self.best_model_dict = OrderedDict() self.evaluation_class = evaluation_class self.visualization = visualization def fit(self, x, y, thres): """ This function runs the main part of the Grid Search :param x: numpy array - Includes the train data :param y: numpy array - Includes the actual value of each data sample :param thres: float - The cut-off boundary of the machine learning algorithm """ values_list, key_list = self.parameters_to_grid(thres) # must be true unchecked = True # number of models num_of_models = len(list(product(*values_list))) # iterate per model model_id = 1 cut_off_boundary = 0 x_train_list, y_train_list, x_test_list, y_test_list = list(), list( ), list(), list() for tuples in list(product(*values_list)): # extract model parameters and cut off boundary parameters_dict, cut_off_boundary = self.extract_models_parameters( tuples, key_list) model = ModelTuningResults(model_id, parameters_dict, cut_off_boundary, self.n_times, self.k_folds) model_id += 1 # n-times for i in range(0, self.n_times, 1): if not self.bootstrap: x_train_list, y_train_list, x_test_list, y_test_list = k_fold_sample_data_set( x, y, self.k_folds) # k-fold for j in range(0, self.k_folds, 1): # split data set in train and test set if self.bootstrap: x_train, y_train, x_test, y_test = random_sample_data_set( x, y, self.k_folds) else: x_train = x_train_list[j] y_train = y_train_list[j] x_test = x_test_list[j] y_test = y_test_list[j] if unchecked: unchecked = self.calculate_grid_time( x, y, num_of_models, parameters_dict) # fit model clf = self.algorithm.set_params(**parameters_dict).fit( x_train, y_train) # predict predicted_probabilities = clf.predict_proba(x_test) predicted_labels = [ 0 if r[0] > cut_off_boundary else 1 for r in predicted_probabilities ] # calculate results model.add_results( calculate_metrics(y_test, predicted_labels, predicted_probabilities, self.score_function, self.evaluation_class)) self.calculate_best_models(model) write_to_file(self.best_model_dict, cut_off_boundary, self.visualization) def calculate_best_models(self, new_model): """ This function keeps and updates the list with the best models according to the selected scoring function :param new_model: ModelTuningResults - A newly created model from the Grid search """ self.best_model_dict[new_model.get_model_id()] = [ new_model.calculate_model_score(), new_model ] temp = sorted(self.best_model_dict.items(), key=lambda x: x[1], reverse=True) self.best_model_dict.clear() for result in temp: self.best_model_dict[result[0]] = result[1] if len(self.best_model_dict) > self.n_top: self.best_model_dict.popitem() def calculate_grid_time(self, x, y, num_of_models, parameters_dict): """ Makes an approximation of the time that is needed to complete the whole Grid Search procedure :param x: numpy array - Includes the train data :param y: numpy array - Includes the actual value of each data sample :param num_of_models: integer - The number of models that selected for evaluation :param parameters_dict: dict - The parameters of a model :return: boolean - Always false in order to executed only once """ start = time() x_train, y_train, x_test, y_test = random_sample_data_set( x, y, self.k_folds) # check param compatibility with the selected model self.check_param_compatibility(self.algorithm, parameters_dict, x_train, y_train) end = time() print("Number of different models: " + str(num_of_models)) print("The procedure needs " + str(( (end - start) * num_of_models * self.n_times * self.k_folds) / 60) + " minutes") return False def parameters_to_grid(self, thres): """ :param thres: float - The cut-off boundary of the machine learning algorithm :return: """ values_list, key_list = list(), list() for key, value_list in self.param_grid.iteritems(): key_list.append(key) values_list.append(value_list) values_list.append(thres) key_list.append('threshold') return values_list, key_list @staticmethod def extract_models_parameters(tuples, key_list): """ :param tuples: :param key_list: :return: """ parameters_dict = dict() for i, t in enumerate(tuples): parameters_dict[key_list[i]] = t cut_off_boundary = parameters_dict['threshold'] del parameters_dict['threshold'] return parameters_dict, cut_off_boundary @staticmethod def check_param_compatibility(algorithm, parameters_dict, x_train, y_train): """ This function checks if a random set of parameters are acceptable from the selected ML algorithm :param algorithm: sklearn model - The selected machine learning algorithm :param parameters_dict: dict - The parameters of a model :param x_train: numpy array - Includes the train data :param y_train: numpy array - Includes the actual value of each data sample :return: """ try: algorithm.set_params(**parameters_dict).fit(x_train, y_train) except Exception as e: print("Error Message: " + e.message) exit()
class Combobox(ttk.Combobox): """ A ttk.Combobox that accepts a list of (key, label) pair of options. """ def __init__(self, master=None, **kw): self._keyvarcb = None self._txtvarcb = None self._choices = OrderedDict() self.__options = options = { 'keyvariable': None, 'textvariable': None, } # remove custom options from kw before initializating for k, v in options.items(): options[k] = kw.pop(k, v) key = 'values' if key in kw: values = kw[key] self.choices = values values = self.__choices2tkvalues(self.choices) kw[key] = values # only normal/disabled supported # normal is converted to readonly key = 'state' if key in kw: if kw[key] not in ('readonly', 'disabled'): kw[key] = 'readonly' else: kw[key] = 'readonly' ttk.Combobox.__init__(self, master, **kw) self.__config_keyvar(options['keyvariable']) self.configure(textvariable=options['textvariable']) def __config_keyvar(self, var): if var is None: return def on_keyvar_changed(varname, elementname, mode): nvalue = self.__options['keyvariable'].get() if nvalue in self._choices: self.current(nvalue) keyvar = self.__options['keyvariable'] if self._keyvarcb is not None: keyvar.trace_vdelete('w', self._keyvarcb) self.__options['keyvariable'] = var self._keyvarcb = var.trace(mode="w", callback=on_keyvar_changed) # A textvariable is needed if keyvariable is set txtvar = self.cget('textvariable') if not txtvar: var = tk.StringVar() self.configure(textvariable=var) else: self.__config_txtvar(txtvar) def __config_txtvar(self, var): keyvar = self.__options['keyvariable'] if var is None or keyvar is None: return def on_txtvar_changed(varname, elementname, mode): ckey = self.current() keyvar.set(ckey) txtvar = self.cget('textvariable') if txtvar and self._txtvarcb is not None: txtvar.trace_vdelete('w', self._txtvarcb) self.__options['textvariable'] = var self._txtvarcb = var.trace(mode="w", callback=on_txtvar_changed) def configure(self, cnf=None, **kw): args = tk._cnfmerge((cnf, kw)) key = 'values' if key in args: self.choices = args[key] args[key] = self.__choices2tkvalues(self.choices) key = 'keyvariable' if key in args: value = args.pop(key) self.__config_keyvar(value) return key = 'textvariable' if key in args: value = args[key] self.__config_txtvar(value) # only readonly and disabled supported key = 'state' if key in args: if args[key] not in ('readonly', 'disabled'): args[key] = 'readonly' return ttk.Combobox.configure(self, args) config = configure def cget(self, key): option = 'values' if key == option: return json.dumps(self.choices) option = 'keyvariable' if key == option: return self.__options[key] option = 'textvariable' if key == option: return self.__options[key] return ttk.Combobox.cget(self, key) def __obj2choices(self, values): """ - json list of key, value pairs: Example: [["A", "Option 1 Label"], ["B", "Option 2 Label"]] - space separated string with a list of options: Example: "Option1 Opt2 Opt3" will be converted to a key, value pair of the following form: (("Option1", "Option1), ("Opt2", "Opt2"), ("Opt3", "Opt3")) - an iterable of key, value pairs """ choices = values # choices from string if type(values) == type(''): obj = None # try json string try: obj = json.loads(values) except ValueError: obj = values if type(obj) == type([]): choices = obj else: choices = obj.split() return choices def __choices2tkvalues(self, choices): """choices: iterable of key, value pairs""" values = [] for k, v in choices: values.append(v) return values @property def choices(self): return tuple(self._choices.items()) @choices.setter def choices(self, values): values = self.__obj2choices(values) self._choices.clear() for idx, v in enumerate(values): key = value = v if type(v) != type('') and len(v) == 2: key, value = v self._choices[key] = value def current(self, key=None): idx = ttk.Combobox.current(self, None) index = -1 ckey = None # find the key of the current selection, # or if a key was given, its index for i, k in enumerate(self._choices.keys()): if key is None and idx == i: ckey = k break elif key == k: index = i break if key is None: # return current key return ckey return ttk.Combobox.current(self, index) def set(self, key): value = self._choices[key] return ttk.Combobox.set(self, value) def get(self): return self.current()
class CardTable(object): card_id = '' table_def = None # type: TableDef @staticmethod def from_bdf(card): raise NotImplementedError @staticmethod def to_bdf(data): raise NotImplementedError def __init__(self, h5n, parent): self._h5n = h5n self._parent = parent self._table_def = deepcopy(self.table_def) self._table_def.set_h5f(self._h5n.h5f) self.data = OrderedDict() if self.card_id is not None: if self.card_id == '': self.__class__.card_id = self.__class__.__name__ self._h5n.register_card_table(self) def write_data(self, cards, domain): if self.from_bdf is CardTable.from_bdf: self._table_def.not_implemented() raise NotImplementedError self._table_def.write_data(cards, domain, self.from_bdf) def finalize(self): pass def read(self): if self.from_bdf is CardTable.from_bdf: # if not implemented, then bail now so the table isn't created in the h5 file return self.data.clear() # main table should ALWAYS be identity, regardless of inconsistencies in MSC spec self.data['identity'] = self._table_def.read() self.data[self._table_def.table_id.lower()] = self.data['identity'] # set to actual table id for convenience def _get_subtable_data(subtable): assert subtable.table_id not in self.data, subtable.table_id self.data[subtable.table_id.lower()] = subtable.read() for _ in subtable.subtables: _get_subtable_data(_) for subtable in self._table_def.subtables: _get_subtable_data(subtable) def __getattr__(self, item): if len(self.data) == 0: self.read() try: return self.data[item] except KeyError: raise AttributeError('Attribute %s not found on CardTable.' % (item))
#initialize Zones=OrderedDict() Zone=OrderedDict() Par=OrderedDict() Pars=OrderedDict() SubPars=OrderedDict() SubPar=OrderedDict() Params=OrderedDict() Param=OrderedDict() Flags=OrderedDict() Flag=OrderedDict() DeviceTypes=OrderedDict() DeviceType=OrderedDict() Zones.clear() Zone.clear() Par.clear() Pars.clear() SubPar.clear() SubPars.clear() Params.clear() Param.clear() Flags.clear() Flag.clear() DeviceTypes=[] DeviceType=[] CurrDevices=None CurrRequired=None CurrRD_ID=None
class LRUCache(abc.MutableMapping, Generic[K, V]): def __init__(self, capacity=None): self.capacity = capacity self.cache = OrderedDict() # type: OrderedDict[K, V] @property def lru(self) -> List[K]: return list(self.cache.keys()) @property def length(self) -> int: return len(self.cache) def clear(self) -> None: self.cache.clear() def __len__(self) -> int: return self.length def __contains__(self, key: object) -> bool: return key in self.cache def __setitem__(self, key: K, value: V) -> None: self.set(key, value) def __delitem__(self, key: K) -> None: del self.cache[key] def __getitem__(self, key) -> V: value = self.get(key) if value is None: raise KeyError(key) return value def __iter__(self) -> Iterator[K]: return iter(self.cache) def get(self, key: K, default: D = None) -> Optional[Union[V, D]]: value = self.cache.get(key) if value is not None: # Put the key back to the front of the ordered dict by # re-insertig it del self.cache[key] self.cache[key] = value return value return default def set(self, key: K, value: V): if self.cache.get(key): del self.cache[key] self.cache[key] = value else: self.cache[key] = value # Check, if the cache is full and we have to remove old items # If the queue is of unlimited size, self.capacity is NaN and # x > NaN is always False in Python and the cache won't be cleared. if self.capacity is not None and self.length > self.capacity: self.cache.popitem(last=False)
class Map(object): def __init__(self, runtime, task_id, tag=0): assert type(tag) != bool self._runtime = runtime self._task_id = runtime.get_task_id(task_id) self._args = list() self._region_args = OrderedDict() self._output_region_args = OrderedDict() self._output_reqs = OrderedDict() self._promoted_storages = OrderedDict() self._next_region_idx = 0 self._next_output_region_idx = 0 self._projections = list() self._future_args = list() self._future_map_args = list() self._tag = tag def __del__(self): self._region_args.clear() self._output_region_args.clear() self._output_reqs.clear() self._promoted_storages.clear() self._projections.clear() self._future_args.clear() self._future_map_args.clear() def _add_region_arg(self, region_arg, field_id): if region_arg not in self._region_args: idx = self._next_region_idx self._next_region_idx += 1 self._region_args[region_arg] = ([field_id], idx) return idx else: (fields, idx) = self._region_args[region_arg] if field_id not in fields: fields.append(field_id) return idx def _add_output_region_arg(self, storage, ipart, field_id): if (storage, ipart) not in self._output_region_args: idx = self._next_output_region_idx self._next_output_region_idx += 1 self._output_region_args[(storage, ipart)] = (set([field_id]), idx) return idx else: (fields, idx) = self._output_region_args[(storage, ipart)] if field_id not in fields: fields.add(field_id) return idx def add_scalar_arg(self, value, dtype): self._args.append(ScalarArg(value, dtype)) def add_dtype_arg(self, dtype): self._args.append(DtypeArg(dtype)) def add_region_arg(self, storage, proj, perm, tag, flags): # For empty fields we override the permission with no access # to avoid bogus uninitilaized access warnings from the runtime if storage.dtype.itemsize == 0: perm = Permission.NO_ACCESS region_idx = self._add_region_arg( RegionArg(storage.region, proj, perm, tag, flags), storage.field_id, ) self._args.append( AccessorArg( region_idx, storage.field_id, storage.dtype, )) def add_no_access(self, storage, proj, tag=0, flags=0): self.add_region_arg(storage, proj, Permission.NO_ACCESS, tag, flags) def add_input(self, storage, proj, tag=0, flags=0): self.add_region_arg(storage, proj, Permission.READ, tag, flags) def add_output(self, storage, proj, tag=0, flags=0): self.add_region_arg(storage, proj, Permission.WRITE, tag, flags) def add_inout(self, storage, proj, tag=0, flags=0): self.add_region_arg(storage, proj, Permission.READ_WRITE, tag, flags) def add_output_only(self, storage): ipart = storage.primary_ipart if storage.storage.fixed else None output_region_idx = self._add_output_region_arg( storage.storage, ipart, storage.field_id) self._args.append( AccessorArg( output_region_idx, storage.field_id, storage.dtype, )) def add_future(self, future): self._future_args.append(future) def add_future_map(self, future_map): self._future_map_args.append(future_map) def build_task(self, launch_domain, argbuf): for arg in self._args: arg.pack(argbuf) task = IndexTask( self._task_id, launch_domain, self._runtime.empty_argmap, argbuf.get_string(), argbuf.get_size(), mapper=self._runtime.mapper_id, tag=self._tag, ) for region_arg, (fields, _) in self._region_args.items(): region_arg.proj.add(self._runtime, task, region_arg, fields) for future in self._future_args: task.add_future(future) for future_map in self._future_map_args: task.add_point_future(ArgumentMap(future_map=future_map)) for (storage, ipart), (fields, _) in self._output_region_args.items(): output_req = self._runtime.create_output_region(storage, list(fields), ipart=ipart) if not storage.fixed: self._output_reqs[storage] = output_req task.add_output(output_req) return task def build_single_task(self, argbuf): assert len(self._output_region_args) == 0 for arg in self._args: arg.pack(argbuf) task = Task( self._task_id, argbuf.get_string(), argbuf.get_size(), mapper=self._runtime.mapper_id, tag=self._tag, ) for region_arg, (fields, _) in self._region_args.items(): region_arg.proj.add_single(task, region_arg, fields) for future in self._future_args: task.add_future(future) if len(self._region_args) == 0: task.set_local_function(True) return task def execute(self, launch_domain): # Note that we should hold a reference to this buffer # until we launch a task, otherwise the Python GC will # collect the Python object holding the buffer, which # in turn will deallocate the C side buffer. argbuf = PandasBufferBuilder() task = self.build_task(launch_domain, argbuf) result = self._runtime.dispatch(task) for storage, output_req in self._output_reqs.items(): region = output_req.get_region() ipart = output_req.get_partition().index_partition self._promoted_storages[storage] = storage.promote(region, ipart) return result def execute_single(self): argbuf = PandasBufferBuilder() return self._runtime.dispatch(self.build_single_task(argbuf)) def promote_output_storage(self, storage): assert not storage.fixed assert storage in self._promoted_storages return self._promoted_storages[storage]
class DirectoryInformation(prebase.ProtoM21Object): ''' returns information about a directory in a Corpus. Called from corpus.corpora.Corpus only tested with CoreCorpus so far. ''' def __init__(self, dirName='', dirTitle='', isComposer=True, corpusObject=None): self.directoryName = dirName self.directoryTitle = dirTitle self.isComposer = isComposer self.works = OrderedDict() self.corpusObject = corpusObject self.findWorks() def _reprInternal(self): return str(self.directoryName) def findWorks(self): ''' populate other information about the directory such as files and filenames. >>> di = corpus.work.DirectoryInformation('schoenberg', ... corpusObject=corpus.corpora.CoreCorpus()) >>> di.findWorks() OrderedDict([(...'opus19', CorpusWork(title='Opus 19', files=[CorpusFile(path='schoenberg/opus19/movement2.mxl', title='Movement 2', filename='movement2.mxl', format='musicxml', ext='.mxl'), CorpusFile(path='schoenberg/opus19/movement6.mxl', title='Movement 6', filename='movement6.mxl', format='musicxml', ext='.mxl')], virtual=False))]) ''' self.works.clear() works = self.corpusObject.getComposer(self.directoryName) # TODO: this should be renamed since not all are composers for path in works: # split by the composer dir to get relative path # environLocal.printDebug(['dir composer', composerDirectory, path]) junk, fileStub = path.as_posix().split(self.directoryName) if fileStub.startswith(os.sep): fileStub = fileStub[len(os.sep):] # break into file components fileComponents = fileStub.split(os.sep) # the first is either a directory for containing components # or a top-level name m21Format, ext = common.findFormatExtFile(fileComponents[-1]) if ext is None: # environLocal.printDebug([ # 'file that does not seem to have an extension', # ext, path]) continue # if not a file w/ ext, we will get None for format if m21Format is None: workStub = fileComponents[0] else: # remove the extension workStub = fileComponents[0].replace(ext, '') # create list location if not already added if workStub not in self.works: title = common.spaceCamelCase(workStub).title() self.works[workStub] = CorpusWork(title=title, files=[], virtual=False) # last component is name m21Format, ext = common.findFormatExtFile(fileComponents[-1]) # all path parts after corpus corpusPath = os.path.join(str(self.directoryName), fileStub) corpusFileName = fileComponents[-1] # all after title = None # this works but takes a long time! # title = converter.parse(path).metadata.title # TODO: get from RichMetadataBundle! if title is None: title = common.spaceCamelCase(fileComponents[-1].replace( ext, '')) title = title.title() fileTuple = CorpusFile(path=corpusPath, title=title, filename=corpusFileName, format=m21Format, ext=ext) self.works[workStub].files.append(fileTuple) # add this path return self.works
class CoverSettingsWidget(QWidget): changed = pyqtSignal() def __init__(self, mi=None, prefs=None, parent=None, for_global_prefs=False): QWidget.__init__(self, parent) self.ignore_changed = False self.for_global_prefs = for_global_prefs self.l = l = QHBoxLayout(self) l.setContentsMargins(0, 0, 0, 0) self.setLayout(l) self.settings_tabs = st = QTabWidget(self) l.addWidget(st) self.preview_label = la = Preview(self) l.addWidget(la) if prefs is None: prefs = cprefs self.original_prefs = prefs self.mi = mi or self.default_mi() self.colors_page = cp = QWidget(st) st.addTab(cp, _('&Colors')) cp.l = l = QGridLayout() cp.setLayout(l) if for_global_prefs: msg = _('When generating covers, a color scheme for the cover is chosen at random from the' ' color schemes below. You can prevent an individual scheme from being selected by' ' unchecking it. The preview on the right shows the currently selected color scheme.') else: msg = _('Choose a color scheme to be used for this generated cover.') + '<p>' + _( 'In normal cover generation, the color scheme is chosen at random from the list of color schemes below. You' ' can prevent an individual color scheme from being chosen by unchecking it here.') cp.la = la = QLabel('<p>' + msg) la.setWordWrap(True) l.addWidget(la, 0, 0, 1, -1) self.colors_list = cl = QListWidget(cp) l.addWidget(cl, 1, 0, 1, -1) self.colors_map = OrderedDict() self.ncs = ncs = QPushButton(QIcon(I('plus.png')), _('&New color scheme'), cp) ncs.clicked.connect(self.create_color_scheme) l.addWidget(ncs) self.ecs = ecs = QPushButton(QIcon(I('format-fill-color.png')), _('&Edit color scheme'), cp) ecs.clicked.connect(self.edit_color_scheme) l.addWidget(ecs, l.rowCount()-1, 1) self.rcs = rcs = QPushButton(QIcon(I('minus.png')), _('&Remove color scheme'), cp) rcs.clicked.connect(self.remove_color_scheme) l.addWidget(rcs, l.rowCount()-1, 2) self.styles_page = sp = QWidget(st) st.addTab(sp, _('&Styles')) sp.l = l = QVBoxLayout() sp.setLayout(l) if for_global_prefs: msg = _('When generating covers, a style for the cover is chosen at random from the' ' styles below. You can prevent an individual style from being selected by' ' unchecking it. The preview on the right shows the currently selected style.') else: msg = _('Choose a style to be used for this generated cover.') + '<p>' + _( 'In normal cover generation, the style is chosen at random from the list of styles below. You' ' can prevent an individual style from being chosen by unchecking it here.') sp.la = la = QLabel('<p>' + msg) la.setWordWrap(True) l.addWidget(la) self.styles_list = sl = QListWidget(sp) l.addWidget(sl) self.style_map = OrderedDict() self.font_page = fp = QWidget(st) st.addTab(fp, _('&Fonts and sizes')) fp.l = l = QFormLayout() fp.setLayout(l) fp.f = [] def add_hline(): f = QFrame() fp.f.append(f) f.setFrameShape(f.HLine) l.addRow(f) for x, label, size_label in ( ('title', _('&Title font family:'), _('&Title font size:')), ('subtitle', _('&Subtitle font family'), _('&Subtitle font size:')), ('footer', _('&Footer font family'), _('&Footer font size')), ): attr = '%s_font_family' % x ff = FontFamilyChooser(fp) setattr(self, attr, ff) l.addRow(label, ff) ff.family_changed.connect(self.emit_changed) attr = '%s_font_size' % x fs = QSpinBox(fp) setattr(self, attr, fs) fs.setMinimum(8), fs.setMaximum(200), fs.setSuffix(' px') fs.setValue(prefs[attr]) fs.valueChanged.connect(self.emit_changed) l.addRow(size_label, fs) add_hline() self.changed_timer = t = QTimer(self) t.setSingleShot(True), t.setInterval(500), t.timeout.connect(self.emit_changed) def create_sz(label): ans = QSpinBox(self) ans.setSuffix(' px'), ans.setMinimum(100), ans.setMaximum(10000) l.addRow(label, ans) ans.valueChanged.connect(self.changed_timer.start) return ans self.cover_width = create_sz(_('Cover &width:')) self.cover_height = create_sz(_('Cover &height:')) fp.cla = la = QLabel(_( 'Note that the preview to the side is of fixed aspect ratio, so changing the cover' ' width above will not have any effect. If you change the height, you should also change the width nevertheless' ' as it will be used in actual cover generation.')) la.setWordWrap(True) l.addRow(la) self.templates_page = tp = QWidget(st) st.addTab(tp, _('&Text')) tp.l = l = QVBoxLayout() tp.setLayout(l) tp.la = la = QLabel(_( 'The text on the generated cover is taken from the metadata of the book.' ' This is controlled via templates. You can use the <b>, <i> and <br> tags' ' in the templates for bold, italic and line breaks, respectively. The' ' default templates use the title, series and authors. You can change them to use' ' whatever metadata you like.')) la.setWordWrap(True), la.setTextFormat(Qt.PlainText) l.addWidget(la) def create_template_widget(title, which, button): attr = which + '_template' heading = QLabel('<h2>' + title) setattr(tp, attr + '_heading', heading) l.addWidget(heading) la = QLabel() setattr(self, attr, la) l.addWidget(la), la.setTextFormat(Qt.PlainText), la.setStyleSheet('QLabel {font-family: monospace}') la.setWordWrap(True) b = QPushButton(button) b.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) connect_lambda(b.clicked, self, lambda self: self.change_template(which)) setattr(self, attr + '_button', b) l.addWidget(b) if which != 'footer': f = QFrame(tp) setattr(tp, attr + '_sep', f), f.setFrameShape(QFrame.HLine) l.addWidget(f) l.addSpacing(10) create_template_widget(_('The title template'), 'title', _('Change the &title template')) create_template_widget(_('The sub-title template'), 'subtitle', _('Change the &sub-title template')) create_template_widget(_('The footer template'), 'footer', _('Change the &footer template')) l.addStretch(2) self.apply_prefs(prefs) self.changed.connect(self.update_preview) self.styles_list.itemSelectionChanged.connect(self.update_preview) self.colors_list.itemSelectionChanged.connect(self.update_preview) self.update_preview() def __enter__(self): self.ignore_changed = True def __exit__(self, *args): self.ignore_changed = False def emit_changed(self): if not self.ignore_changed: self.changed.emit() def apply_prefs(self, prefs): with self: self._apply_prefs(prefs) def _apply_prefs(self, prefs): for x in ('title', 'subtitle', 'footer'): attr = '%s_font_family' % x getattr(self, attr).font_family = prefs[attr] attr = '%s_font_size' % x getattr(self, attr).setValue(prefs[attr]) for x in ('title', 'subtitle', 'footer'): x += '_template' getattr(self, x).setText(prefs[x]) for x in ('width', 'height'): x = 'cover_' + x getattr(self, x).setValue(prefs[x]) color_themes = prefs['color_themes'].copy() color_themes.update(default_color_themes) disabled = set(prefs['disabled_color_themes']) self.colors_list.clear() self.colors_map = {} for name in sorted(color_themes, key=sort_key): self.colors_map[name] = li = QListWidgetItem(name, self.colors_list) li.setFlags(li.flags() | Qt.ItemIsUserCheckable) li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked) li.setData(Qt.UserRole, color_themes[name]) lu = prefs.get('last_used_colors') if not self.for_global_prefs and lu in self.colors_map and self.colors_map[lu].checkState() == Qt.Checked: self.colors_map[lu].setSelected(True) else: for name, li in iteritems(self.colors_map): if li.checkState() == Qt.Checked: li.setSelected(True) break else: next(itervalues(self.colors_map)).setSelected(True) disabled = set(prefs['disabled_styles']) self.styles_list.clear() self.style_map.clear() for name in sorted(all_styles(), key=sort_key): self.style_map[name] = li = QListWidgetItem(name, self.styles_list) li.setFlags(li.flags() | Qt.ItemIsUserCheckable) li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked) lu = prefs.get('last_used_style') if not self.for_global_prefs and lu in self.style_map and self.style_map[lu].checkState() == Qt.Checked: self.style_map[lu].setSelected(True) else: for name, li in iteritems(self.style_map): if li.checkState() == Qt.Checked: li.setSelected(True) break else: next(itervalues(self.style_map)).setSelected(True) @property def current_colors(self): for name, li in iteritems(self.colors_map): if li.isSelected(): return name @property def disabled_colors(self): for name, li in iteritems(self.colors_map): if li.checkState() == Qt.Unchecked: yield name @property def custom_colors(self): ans = {} for name, li in iteritems(self.colors_map): if name.startswith('#'): ans[name] = li.data(Qt.UserRole) return ans @property def current_style(self): for name, li in iteritems(self.style_map): if li.isSelected(): return name @property def disabled_styles(self): for name, li in iteritems(self.style_map): if li.checkState() == Qt.Unchecked: yield name @property def current_prefs(self): prefs = {k:self.original_prefs[k] for k in self.original_prefs.defaults} for x in ('title', 'subtitle', 'footer'): attr = '%s_font_family' % x prefs[attr] = getattr(self, attr).font_family attr = '%s_font_size' % x prefs[attr] = getattr(self, attr).value() prefs['color_themes'] = self.custom_colors prefs['disabled_styles'] = list(self.disabled_styles) prefs['disabled_color_themes'] = list(self.disabled_colors) for x in ('title', 'subtitle', 'footer'): x += '_template' prefs[x] = getattr(self, x).text() for x in ('width', 'height'): x = 'cover_' + x prefs[x] = getattr(self, x).value() return prefs def insert_scheme(self, name, li): with self: self.colors_list.insertItem(0, li) cm = OrderedDict() cm[name] = li for k, v in iteritems(self.colors_map): cm[k] = v self.colors_map = cm li.setSelected(True) for i in range(1, self.colors_list.count()): self.colors_list.item(i).setSelected(False) def create_color_scheme(self): scheme = self.colors_map[self.current_colors].data(Qt.UserRole) d = CreateColorScheme('#' + _('My Color Scheme'), scheme, set(self.colors_map), parent=self) if d.exec_() == d.Accepted: name, scheme = d.data li = QListWidgetItem(name) li.setData(Qt.UserRole, scheme), li.setFlags(li.flags() | Qt.ItemIsUserCheckable), li.setCheckState(Qt.Checked) self.insert_scheme(name, li) self.emit_changed() self.original_prefs['color_themes'] = self.current_prefs['color_themes'] def edit_color_scheme(self): cs = self.current_colors if cs is None or not cs.startswith('#'): return error_dialog(self, _('Cannot edit'), _( 'Cannot edit a builtin color scheme. Create a new' ' color scheme instead.'), show=True) li = self.colors_map[cs] d = CreateColorScheme(cs, li.data(Qt.UserRole), set(self.colors_map), edit_scheme=True, parent=self) if d.exec_() == d.Accepted: name, scheme = d.data li.setText(name) li.setData(Qt.UserRole, scheme) if name != cs: self.colors_map.pop(cs, None) self.insert_scheme(name, li) self.emit_changed() self.original_prefs['color_themes'] = self.current_prefs['color_themes'] def remove_color_scheme(self): cs = self.current_colors if cs is None or not cs.startswith('#'): return error_dialog(self, _('Cannot remove'), _( 'Cannot remove a builtin color scheme.'), show=True) for i in range(self.colors_list.count()): item = self.colors_list.item(i) if item.isSelected(): with self: del self.colors_map[item.text()] self.colors_list.takeItem(i) i = i % self.colors_list.count() self.colors_list.item(i).setSelected(True) self.emit_changed() self.original_prefs['color_themes'] = self.current_prefs['color_themes'] return def change_template(self, which): from calibre.gui2.dialogs.template_dialog import TemplateDialog from calibre.gui2.ui import get_gui gui = get_gui() if gui is None: from calibre.ebooks.metadata.book.base import field_metadata else: field_metadata = gui.current_db.new_api.field_metadata attr = which + '_template' templ = getattr(self, attr).text() d = TemplateDialog(self, templ, mi=self.mi, fm=field_metadata) if d.exec_() == d.Accepted: templ = d.rule[1] getattr(self, attr).setText(templ) self.emit_changed() @property def prefs_for_rendering(self): prefs = self.current_prefs prefs = override_prefs(prefs, override_style=self.current_style, override_color_theme=self.current_colors) return prefs def update_preview(self): if self.ignore_changed: return dpr = getattr(self, 'devicePixelRatioF', self.devicePixelRatio)() w, h = int(dpr * self.preview_label.sizeHint().width()), int(dpr * self.preview_label.sizeHint().height()) prefs = self.prefs_for_rendering hr = h / prefs['cover_height'] for x in ('title', 'subtitle', 'footer'): attr = '%s_font_size' % x prefs[attr] = int(prefs[attr] * hr) prefs['cover_width'], prefs['cover_height'] = w, h img = generate_cover(self.mi, prefs=prefs, as_qimage=True) img.setDevicePixelRatio(dpr) self.preview_label.setPixmap(QPixmap.fromImage(img)) def default_mi(self): from calibre.ebooks.metadata.book.base import Metadata mi = Metadata(_('A sample book'), [_('Author One'), _('Author Two')]) mi.series = _('A series of samples') mi.series_index = 4 mi.tags = [_('Tag One'), _('Tag Two')] mi.publisher = _('Some publisher') mi.rating = 4 mi.identifiers = {'isbn':'123456789', 'url': 'https://calibre-ebook.com'} mi.languages = ['eng', 'fra'] mi.pubdate = mi.timestamp = now() return mi def restore_defaults(self): defaults = self.original_prefs.defaults.copy() # Dont delete custom color themes when restoring defaults defaults['color_themes'] = self.custom_colors self.apply_prefs(defaults) self.update_preview() def save_state(self): self.original_prefs.set('last_used_colors', self.current_colors) self.original_prefs.set('last_used_style', self.current_style) def save_as_prefs(self): with self.original_prefs: for k, v in iteritems(self.current_prefs): self.original_prefs[k] = v
class SystemLayoutInput: """ Geometry file. Contains 2-D mapping of geometry. This approach to specifying core layout has been deprecated in favor of the :py:class:`armi.reactor.blueprints.gridBlueprints.GridBlueprints` and ``systems:`` section of :py:class:`armi.reactor.blueprints.Blueprints` """ _GEOM_FILE_EXTENSION = ".xml" ROOT_TAG = INP_SYSTEMS def __init__(self): self.fName = None self.modifiedFileName = None self.geomType: str self.symmetry = None self.assemTypeByIndices = OrderedDict() self.eqPathInput = {} self.eqPathsHaveBeenModified = False self.maxRings = 0 def __repr__(self): return "<Geometry file {0}>".format(self.fName) def readGeomFromFile(self, fName): """ Read the 2-d geometry input file. See Also -------- fromReactor : Build SystemLayoutInput from a Reactor object. """ self.fName = os.path.expandvars(fName) with open(self.fName) as stream: self.readGeomFromStream(stream) def readGeomFromStream(self, stream): """ Read geometry info from a stream. This populates the object with info from any source. Notes ----- There are two formats of geometry: yaml and xml. This tries xml first (legacy), and if it fails it tries yaml. """ # Warn the user that this feature is schedule for deletion. warn = "XML Geom Files are scheduled to be removed from ARMI, please use blueprint files." runLog.important(warn) try: self._readXml(stream) except ET.ParseError: stream.seek(0) self._readYaml(stream) def toGridBlueprints(self, name: str = "core"): """ Migrate old-style SystemLayoutInput to new GridBlueprint. Returns a list of GridBlueprint objects. There will at least be one entry, containing the main core layout. If equilibrium fuel paths are specified, it will occupy the second element. """ # TODO: After moving SystemLayoutInput out of geometry.py, we may be able to # move this back out to top-level without causing blueprint import order issues. from armi.reactor.blueprints.gridBlueprint import GridBlueprint geom = self.geomType symmetry = self.symmetry bounds = None if self.geomType == geometry.GeomType.RZT: # We need a grid in order to go from whats in the input to indices, and to # be able to provide grid bounds to the blueprint. rztGrid = grids.ThetaRZGrid.fromGeom(self) theta, r, _ = rztGrid.getBounds() bounds = {"theta": theta.tolist(), "r": r.tolist()} gridContents = dict() for indices, spec in self.assemTypeByIndices.items(): if self.geomType == geometry.GeomType.HEX: i, j = grids.HexGrid.getIndicesFromRingAndPos(*indices) elif self.geomType == geometry.GeomType.RZT: i, j, _ = rztGrid.indicesOfBounds(*indices[0:4]) else: i, j = indices gridContents[(i, j)] = spec bp = GridBlueprint( name=name, gridContents=gridContents, geom=geom, symmetry=symmetry, gridBounds=bounds, ) bps = [bp] if any(val != (None, None) for val in self.eqPathInput.values()): # We could probably just copy eqPathInput, but we don't want to preserve # (None, None) entries. eqPathContents = dict() for idx, eqPath in self.eqPathInput.items(): if eqPath == (None, None): continue if self.geomType == geometry.GeomType.HEX: i, j = grids.HexGrid.getIndicesFromRingAndPos(*idx) elif self.geomType == geometry.GeomType.RZT: i, j, _ = rztGrid.indicesOfBounds(*idx[0:4]) else: i, j = idx eqPathContents[i, j] = copy(self.eqPathInput[idx]) pathBp = GridBlueprint( name=name + "EqPath", gridContents=eqPathContents, geom=geom, symmetry=symmetry, gridBounds=bounds, ) bps.append(pathBp) return bps def _readXml(self, stream): tree = ET.parse(stream) root = tree.getroot() self._getGeomTypeAndSymmetryFromXml(root) self.assemTypeByIndices.clear() for assemblyNode in root: aType = str(assemblyNode.attrib["name"]) eqPathIndex, eqPathCycle = None, None if self.geomType == geometry.GeomType.CARTESIAN: indices = x, y = tuple( int(assemblyNode.attrib[key]) for key in LOC_CARTESIAN ) self.maxRings = max(x + 1, y + 1, self.maxRings) elif self.geomType == geometry.GeomType.RZT: indices = tuple( float(assemblyNode.attrib[key]) for key in LOC_RZ ) + tuple(int(assemblyNode.attrib[key]) for key in MESH_RZ) else: # assume hex geom. indices = ring, _pos = tuple( int(assemblyNode.attrib[key]) for key in LOC_HEX ) self.maxRings = max(ring, self.maxRings) if INP_FUEL_PATH in assemblyNode.attrib: # equilibrium geometry info. eqPathIndex = int(assemblyNode.attrib[INP_FUEL_PATH]) eqPathCycle = int(assemblyNode.attrib[INP_FUEL_CYCLE]) self.assemTypeByIndices[indices] = aType self.eqPathInput[indices] = (eqPathIndex, eqPathCycle) def _readYaml(self, stream): """ Read geometry from yaml. Notes ----- This is intended to replace the XML format as we converge on consistent inputs. """ yaml = YAML() tree = yaml.load(stream) tree = INPUT_SCHEMA(tree) self.assemTypeByIndices.clear() for _systemName, system in tree[INP_SYSTEMS].items(): # no need to check for valid since the schema handled that. self.geomType = geometry.GeomType.fromStr(system[INP_GEOM]) self.symmetry = geometry.SymmetryType.fromStr(system[INP_SYMMETRY]) if INP_DISCRETES in system: self._read_yaml_discretes(system) elif INP_LATTICE in system: self._read_yaml_lattice(system) def _read_yaml_discretes(self, system): for discrete in system[INP_DISCRETES]: location = discrete[INP_LOCATION] indices = tuple(location[k] for k in LOC_KEYS[str(self.geomType)]) if self.geomType == geometry.GeomType.CARTESIAN: x, y = indices self.maxRings = max(x + 1, y + 1, self.maxRings) elif self.geomType == geometry.GeomType.RZT: pass else: # assume hex geom. x, y = indices self.maxRings = max(x, self.maxRings) self.assemTypeByIndices[indices] = discrete[INP_SPEC] self.eqPathInput[indices] = ( discrete.get(INP_FUEL_CYCLE), discrete.get(INP_FUEL_PATH), ) def _read_yaml_lattice(self, system): """Read a ascii map string into this object.""" mapTxt = system[INP_LATTICE] if ( self.geomType == geometry.GeomType.HEX and self.symmetry.domain == geometry.DomainType.THIRD_CORE ): asciimap = asciimaps.AsciiMapHexThirdFlatsUp() asciimap.readAscii(mapTxt) for (i, j), spec in asciimap.items(): if spec == "-": # skip whitespace placeholders continue ring, pos = grids.HexGrid.indicesToRingPos(i, j) self.assemTypeByIndices[(ring, pos)] = spec self.maxRings = max(ring, self.maxRings) else: raise ValueError( f"ASCII map reading from geom/domain: {self.geomType}/" f"{self.symmetry.domain} not supported." ) def modifyEqPaths(self, modifiedPaths): """ Modifies the geometry object by updating the equilibrium path indices and equilibrium path cycles. Parameters ---------- modifiedPaths : dict, required This is a dictionary that contains the indices that are mapped to the eqPathIndex and eqPathCycle. modifiedPath[indices] = (eqPathIndex, eqPathCycle) """ runLog.important("Modifying the equilibrium paths on {}".format(self)) self.eqPathsHaveBeenModified = True self.eqPathInput.update(modifiedPaths) def _getModifiedFileName(self, originalFileName, suffix): """ Generates the modified geometry file name based on the requested suffix """ originalFileName = originalFileName.split(self._GEOM_FILE_EXTENSION)[0] suffix = suffix.split(self._GEOM_FILE_EXTENSION)[0] self.modifiedFileName = originalFileName + suffix + self._GEOM_FILE_EXTENSION def writeGeom(self, outputFileName, suffix=""): """ Write data out as a geometry xml file Parameters ---------- outputFileName : str Geometry file name suffix : str Added suffix to the geometry output file name """ if suffix: self._getModifiedFileName(outputFileName, suffix) outputFileName = self.modifiedFileName runLog.important("Writing reactor geometry file as {}".format(outputFileName)) root = ET.Element( INP_SYSTEMS, attrib={ INP_GEOM: str(self.geomType), INP_SYMMETRY: str(self.symmetry), }, ) tree = ET.ElementTree(root) # start at ring 1 pos 1 and go out for targetIndices in sorted(list(self.assemTypeByIndices)): ring, pos = targetIndices assembly = ET.SubElement(root, "assembly") assembly.set("ring", str(ring)) assembly.set("pos", str(pos)) fuelPath, fuelCycle = self.eqPathInput.get((ring, pos), (None, None)) if fuelPath is not None: # set the equilibrium shuffling info if it exists assembly.set(INP_FUEL_PATH, str(fuelPath)) assembly.set(INP_FUEL_CYCLE, str(fuelCycle)) aType = self.assemTypeByIndices[targetIndices] assembly.set("name", aType) # note: This is ugly and one-line, but that's ok # since we're transitioning. tree.write(outputFileName) def _writeYaml(self, stream, sysName="core"): """ Write the system layout in YAML format. These can eventually either define a type/specifier mapping and then point to a grid definition full of specifiers to be populated, or they can just be a list of "singles" which define indices:specifier. Note ---- So far, only singles are implemented (analogous to old XML format) """ geomData = [] for indices in sorted(list(self.assemTypeByIndices)): specifier = self.assemTypeByIndices[indices] fuelPath, fuelCycle = self.eqPathInput.get(indices, (None, None)) keys = LOC_KEYS[str(self.geomType)] dataPoint = {INP_LOCATION: dict(zip(keys, indices)), INP_SPEC: specifier} if fuelPath is not None: dataPoint.update({INP_FUEL_PATH: fuelPath, INP_FUEL_CYCLE: fuelCycle}) geomData.append(dataPoint) yaml = YAML() yaml.default_flow_style = False yaml.indent(mapping=2, sequence=4, offset=2) fullData = { INP_SYSTEMS: { sysName: { INP_GEOM: str(self.geomType), INP_SYMMETRY: str(self.symmetry), INP_DISCRETES: geomData, } } } fullData = INPUT_SCHEMA(fullData) # validate on the way out yaml.dump(fullData, stream) def _writeAsciiMap(self): """Generate an ASCII map representation. Warning ------- This only works for HexGrid. """ lattice = {} for ring, pos in sorted(list(self.assemTypeByIndices)): specifier = self.assemTypeByIndices[(ring, pos)] i, j = grids.HexGrid.getIndicesFromRingAndPos(ring, pos) lattice[i, j] = specifier geomMap = asciimaps.AsciiMapHexThirdFlatsUp() geomMap.asciiLabelByIndices = lattice geomMap.gridContentsToAscii() geomMap.writeAscii(sys.stdout) def growToFullCore(self): """ Convert geometry input to full core. Notes ----- This only works for Hex 1/3rd core geometry inputs. """ if self.symmetry.domain == geometry.DomainType.FULL_CORE: # already full core from geometry file. No need to copy symmetry over. runLog.important( "Detected that full core geometry already exists. Cannot expand." ) return elif ( self.symmetry.domain != geometry.DomainType.THIRD_CORE or self.symmetry.boundary != geometry.BoundaryType.PERIODIC ): raise ValueError( "Cannot convert shape `{}` to full core, must be {}".format( self.symmetry.domain, str( geometry.SymmetryType( geometry.DomainType.THIRD_CORE, geometry.BoundaryType.PERIODIC, ) ), ), ) grid = grids.HexGrid.fromPitch(1.0) grid._symmetry: str = str(self.symmetry) # need to cast to a list because we will modify during iteration for (ring, pos), specifierID in list(self.assemTypeByIndices.items()): indices = grids.HexGrid.getIndicesFromRingAndPos(ring, pos) for symmetricI, symmetricJ in grid.getSymmetricEquivalents(indices): symmetricRingPos = grids.HexGrid.indicesToRingPos( symmetricI, symmetricJ ) self.assemTypeByIndices[symmetricRingPos] = specifierID self.symmetry = geometry.SymmetryType( geometry.DomainType.FULL_CORE, geometry.BoundaryType.NO_SYMMETRY, ) def _getGeomTypeAndSymmetryFromXml(self, root): """Read the geometry type and symmetry.""" try: self.geomType = geometry.GeomType.fromStr( str(root.attrib[INP_GEOM]).lower() ) except ValueError: # will not execute if the geom was specified as thetarz, cartesian or anything else specific runLog.warning( "Could not find geometry type. Assuming hex geometry with third core periodic symmetry." ) self.geomType = geometry.GeomType.HEX self.symmetry = geometry.SymmetryType( geometry.DomainType.THIRD_CORE, geometry.BoundaryType.PERIODIC, ) else: inputString = str(root.attrib[INP_SYMMETRY]).lower() self.symmetry = geometry.SymmetryType.fromStr(inputString) @classmethod def fromReactor(cls, reactor): """ Build SystemLayoutInput object based on the current state of a Reactor. See Also -------- readGeomFromFile : Builds a SystemLayoutInput from an XML file. """ geom = cls() runLog.info("Reading core map from {}".format(reactor)) geom.geomType = str(reactor.core.geomType) geom.symmetry = reactor.core.symmetry bp = reactor.blueprints assemDesigns = bp.assemDesigns if bp else () for assembly in reactor.core: aType = assembly.getType() if aType in assemDesigns: aType = assemDesigns[aType].specifier (x, _y) = indices = reactor.core.spatialGrid.getRingPos( assembly.spatialLocator.getCompleteIndices() ) geom.maxRings = max(x, geom.maxRings) geom.assemTypeByIndices[indices] = aType return geom @classmethod def loadFromCs(cls, cs): """Function to load Geoemtry based on supplied ``CaseSettings``.""" if not cs["geomFile"]: return None with directoryChangers.DirectoryChanger(cs.inputDirectory): geom = cls() geom.readGeomFromFile(cs["geomFile"]) return geom
class DotMap(MutableMapping, OrderedDict): def __init__(self, *args, **kwargs): self._map = OrderedDict() self._dynamic = True if kwargs: if '_dynamic' in kwargs: self._dynamic = kwargs['_dynamic'] if args: d = args[0] if isinstance(d, dict): for k, v in self.__call_items(d): if isinstance(v, dict): v = DotMap(v, _dynamic=self._dynamic) if type(v) is list: l = [] for i in v: n = i if type(i) is dict: n = DotMap(i, _dynamic=self._dynamic) l.append(n) v = l self._map[k] = v if kwargs: for k, v in self.__call_items(kwargs): if k is not '_dynamic': self._map[k] = v def __call_items(self, obj): if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')): return obj.iteritems() else: return obj.items() def items(self): return self.iteritems() def iteritems(self): return self.__call_items(self._map) def __iter__(self): return self._map.__iter__() def next(self): return self._map.next() def __setitem__(self, k, v): self._map[k] = v def __getitem__(self, k): if k not in self._map and self._dynamic and k != '_ipython_canary_method_should_not_exist_': # automatically extend to new DotMap self[k] = DotMap() return self._map[k] def __setattr__(self, k, v): if k in { '_map', '_dynamic', '_ipython_canary_method_should_not_exist_' }: super(DotMap, self).__setattr__(k, v) else: self[k] = v def __getattr__(self, k): if k == { '_map', '_dynamic', '_ipython_canary_method_should_not_exist_' }: super(DotMap, self).__getattr__(k) else: return self[k] def __delattr__(self, key): return self._map.__delitem__(key) def __contains__(self, k): return self._map.__contains__(k) def __str__(self): items = [] for k, v in self.__call_items(self._map): # bizarre recursive assignment situation (why someone would do this is beyond me) if id(v) == id(self): items.append('{0}=DotMap(...)'.format(k)) else: items.append('{0}={1}'.format(k, repr(v))) joined = ', '.join(items) out = '{0}({1})'.format(self.__class__.__name__, joined) return out def __repr__(self): return str(self) def toDict(self): d = {} for k, v in self.items(): if type(v) is DotMap: # bizarre recursive assignment support if id(v) == id(self): v = d else: v = v.toDict() elif type(v) is list: l = [] for i in v: n = i if type(i) is DotMap: n = i.toDict() l.append(n) v = l d[k] = v return d def pprint(self, pformat='dict'): if pformat == 'json': print(dumps(self.toDict(), indent=4, sort_keys=True)) else: pprint(self.toDict()) def empty(self): return (not any(self)) # proper dict subclassing def values(self): return self._map.values() # ipython support def __dir__(self): return self.keys() @classmethod def parseOther(self, other): if type(other) is DotMap: return other._map else: return other def __cmp__(self, other): other = DotMap.parseOther(other) return self._map.__cmp__(other) def __eq__(self, other): other = DotMap.parseOther(other) if not isinstance(other, dict): return False return self._map.__eq__(other) def __ge__(self, other): other = DotMap.parseOther(other) return self._map.__ge__(other) def __gt__(self, other): other = DotMap.parseOther(other) return self._map.__gt__(other) def __le__(self, other): other = DotMap.parseOther(other) return self._map.__le__(other) def __lt__(self, other): other = DotMap.parseOther(other) return self._map.__lt__(other) def __ne__(self, other): other = DotMap.parseOther(other) return self._map.__ne__(other) def __delitem__(self, key): return self._map.__delitem__(key) def __len__(self): return self._map.__len__() def clear(self): self._map.clear() def copy(self): return DotMap(self) def __copy__(self): return self.copy() def __deepcopy__(self, memo=None): return self.copy() def get(self, key, default=None): return self._map.get(key, default) def has_key(self, key): return key in self._map def iterkeys(self): return self._map.iterkeys() def itervalues(self): return self._map.itervalues() def keys(self): return self._map.keys() def pop(self, key, default=None): return self._map.pop(key, default) def popitem(self): return self._map.popitem() def setdefault(self, key, default=None): self._map.setdefault(key, default) def update(self, *args, **kwargs): if len(args) != 0: self._map.update(*args) self._map.update(kwargs) def viewitems(self): return self._map.viewitems() def viewkeys(self): return self._map.viewkeys() def viewvalues(self): return self._map.viewvalues() @classmethod def fromkeys(cls, seq, value=None): d = DotMap() d._map = OrderedDict.fromkeys(seq, value) return d def __getstate__(self): return self.__dict__ def __setstate__(self, d): self.__dict__.update(d) # bannerStr def _getListStr(self, items): out = '[' mid = '' for i in items: mid += ' {}\n'.format(i) if mid != '': mid = '\n' + mid out += mid out += ']' return out def _getValueStr(self, k, v): outV = v multiLine = len(str(v).split('\n')) > 1 if multiLine: # push to next line outV = '\n' + v if type(v) is list: outV = self._getListStr(v) out = '{} {}'.format(k, outV) return out def _getSubMapDotList(self, pre, name, subMap): outList = [] if pre == '': pre = name else: pre = '{}.{}'.format(pre, name) def stamp(pre, k, v): valStr = self._getValueStr(k, v) return '{}.{}'.format(pre, valStr) for k, v in subMap.items(): if isinstance(v, DotMap) and v != DotMap(): subList = self._getSubMapDotList(pre, k, v) outList.extend(subList) else: outList.append(stamp(pre, k, v)) return outList def _getSubMapStr(self, name, subMap): outList = ['== {} =='.format(name)] for k, v in subMap.items(): if isinstance(v, DotMap) and v != DotMap(): # break down to dots subList = self._getSubMapDotList('', k, v) # add the divit # subList = ['> {}'.format(i) for i in subList] outList.extend(subList) else: out = self._getValueStr(k, v) # out = '> {}'.format(out) out = '{}'.format(out) outList.append(out) finalOut = '\n'.join(outList) return finalOut def bannerStr(self): lines = [] previous = None for k, v in self.items(): if previous == 'DotMap': lines.append('-') out = '' if isinstance(v, DotMap): name = k subMap = v out = self._getSubMapStr(name, subMap) lines.append(out) previous = 'DotMap' else: out = self._getValueStr(k, v) lines.append(out) previous = 'other' lines.append('--') s = '\n'.join(lines) return s
class CellAnalyzer(LoggerObject): def __init__(self, timeholder, position, create_images, binning_factor, detect_objects): super(CellAnalyzer, self).__init__() self.timeholder = timeholder self.P = position self.bCreateImages = create_images self.iBinningFactor = binning_factor self.detect_objects = detect_objects self._iT = None self._channel_registry = OrderedDict() def initTimepoint(self, iT): self._channel_registry.clear() self._iT = iT self.timeholder.initTimePoint(iT) # XXX rename it to "add_channel" def register_channel(self, channel): self._channel_registry[channel.NAME] = channel def get_channel_names(self): return self._channel_registry.keys() def get_channel(self, name): return self._channel_registry[str(name)] @property def proc_channels(self): """Return processing channels i.e the dict contains no virtual or pseudo channels. """ pchannels = OrderedDict() for name, channel in self._channel_registry.iteritems(): if not channel.is_virtual(): pchannels[name] = channel return pchannels @property def virtual_channels(self): """Return a dict that contains all virtual channels.""" channels = dict() for name, channel in self._channel_registry.iteritems(): if channel.is_virtual(): channels[name] = channel return channels def process(self, apply=True, extract_features=True): """Perform the segmentation and feature extraction.""" channels = sorted(self._channel_registry.values()) primary_channel = None cnames = [c.NAME for c in channels] for channel in channels: self.timeholder.prepare_raw_image(channel) if self.detect_objects: if channel.NAME == PrimaryChannel.NAME: self.timeholder.apply_segmentation(channel) primary_channel = channel elif channel.NAME == SecondaryChannel.NAME: self.timeholder.apply_segmentation(channel, primary_channel) secondary_channel = channel elif channel.NAME == TertiaryChannel.NAME: if SecondaryChannel.NAME not in cnames: raise RuntimeError(("Tertiary channel requiers a " "secondary channel")) self.timeholder.apply_segmentation(channel, primary_channel, secondary_channel) elif channel.NAME == MergedChannel.NAME: channel.meta_image = primary_channel.meta_image self.timeholder.apply_segmentation(channel, self._channel_registry) else: raise ValueError("Channel with name '%s' not supported." % channel.NAME) if extract_features: self.timeholder.apply_features(channel) if apply: # want apply also the pseudo channels for channel in sorted(self._channel_registry.values()): self.timeholder.apply_channel(channel) def purge(self, features=None): for channel in self._channel_registry.values(): if not features is None and channel.strChannelId in features: channelFeatures = features[channel.strChannelId] else: channelFeatures = None channel.purge(features=channelFeatures) def exportLabelImages(self, pathOut, compression='LZW'): # no segmentaion in virtual channels --> no label images for channel in self.proc_channels.itervalues(): channel_id = channel.strChannelId for region, container in channel.containers.iteritems(): outdir = join(pathOut, channel_id, region) makedirs(outdir) fname = join(outdir, 'P%s_T%05d.tif' %(self.P, self._iT)) container.exportLabelImage(fname, compression) def getImageSize(self, name): oChannel = self._channel_registry[name] w = oChannel.meta_image.width h = oChannel.meta_image.height return (w,h) def render(self, strPathOut, dctRenderInfo=None, strFileSuffix='.jpg', strCompression='98', writeToDisc=True, images=None): lstImages = [] if not images is None: lstImages += images if dctRenderInfo is None: for name, oChannel in self._channel_registry.iteritems(): for strRegion, oContainer in oChannel.containers.iteritems(): strHexColor, fAlpha = oChannel.dctAreaRendering[strRegion] imgRaw = oChannel.meta_image.image imgCon = ccore.Image(imgRaw.width, imgRaw.height) ccore.drawContour(oContainer.getBinary(), imgCon, 255, False) lstImages.append((imgRaw, strHexColor, 1.0)) lstImages.append((imgCon, strHexColor, fAlpha)) else: for channel_name, dctChannelInfo in dctRenderInfo.iteritems(): if channel_name in self._channel_registry: oChannel = self._channel_registry[channel_name] if 'raw' in dctChannelInfo: strHexColor, fAlpha = dctChannelInfo['raw'] # special casing for virtual channel to mix # raw images together if oChannel.is_virtual(): lstImages.extend(oChannel.meta_images(fAlpha)) else: lstImages.append((oChannel.meta_image.image, strHexColor, 1.0)) if 'contours' in dctChannelInfo: # transform the old dict-style to the new tuple-style, # which allows multiple definitions for one region if isinstance(dctChannelInfo['contours'], dict): lstContourInfos = [(k,)+v for k,v in dctChannelInfo['contours'].iteritems()] else: lstContourInfos = dctChannelInfo['contours'] for tplData in lstContourInfos: strRegion, strNameOrColor, fAlpha, bShowLabels = tplData[:4] # draw contours only if region is present if oChannel.has_region(strRegion): if len(tplData) > 4: bThickContours = tplData[4] else: bThickContours = False imgRaw = oChannel.meta_image.image if strNameOrColor == 'class_label': oContainer = oChannel.containers[strRegion] oRegion = oChannel.get_region(strRegion) dctLabels = {} dctColors = {} for iObjId, oObj in oRegion.iteritems(): iLabel = oObj.iLabel if not iLabel is None: if not iLabel in dctLabels: dctLabels[iLabel] = [] dctLabels[iLabel].append(iObjId) dctColors[iLabel] = oObj.strHexColor imgCon2 = ccore.Image(imgRaw.width, imgRaw.height) for iLabel, lstObjIds in dctLabels.iteritems(): imgCon = ccore.Image(imgRaw.width, imgRaw.height) # Flip this and use drawContours with fill option enables to get black background oContainer.drawContoursByIds(lstObjIds, 255, imgCon, bThickContours, False) # oContainer.drawContoursByIds(lstObjIds, 255, imgCon, bThickContours, True) lstImages.append((imgCon, dctColors[iLabel], fAlpha)) if isinstance(bShowLabels, bool) and bShowLabels: oContainer.drawTextsByIds(lstObjIds, [str(iLabel)]*len(lstObjIds), imgCon2) lstImages.append((imgCon2, '#FFFFFF', 1.0)) else: oContainer = oChannel.containers[strRegion] oRegion = oChannel.get_region(strRegion) lstObjIds = oRegion.keys() imgCon = ccore.Image(imgRaw.width, imgRaw.height) if not strNameOrColor is None: oContainer.drawContoursByIds(lstObjIds, 255, imgCon, bThickContours, False) else: strNameOrColor = '#FFFFFF' lstImages.append((imgCon, strNameOrColor, fAlpha)) if bShowLabels: imgCon2 = ccore.Image(imgRaw.width, imgRaw.height) oContainer.drawLabelsByIds(lstObjIds, imgCon2) lstImages.append((imgCon2, '#FFFFFF', 1.0)) if len(lstImages) > 0: imgRgb = ccore.makeRGBImage([x[0].getView() for x in lstImages], [ccore.RGBValue(*hex2rgb(x[1])) for x in lstImages], [x[2] for x in lstImages]) if writeToDisc: strFilePath = join(strPathOut, "P%s_T%05d%s" %(self.P, self._iT, strFileSuffix)) makedirs(strPathOut) ccore.writeImage(imgRgb, strFilePath, strCompression) self.logger.debug("* rendered image written '%s'" % strFilePath) else: strFilePath = '' return imgRgb, strFilePath def collectObjects(self, plate_id, P, sample_readers, oLearner, byTime=True): self.logger.debug('* collecting samples...') self.process(apply = False, extract_features = False) chname = oLearner.name region = oLearner.regions oChannel = self._channel_registry[chname] oContainer = oChannel.get_container(region) objects = oContainer.getObjects() # key = class_label, value = list of samples object_lookup = {} object_ids = set() for reader in sample_readers: if (byTime and P == reader.position() and self._iT in reader): coords = reader[self._iT] elif (not byTime and P in reader): coords = reader[P] else: coords = None if coords is not None: for data in coords: label = data['iClassLabel'] if (label in oLearner.class_names and 0 <= data['iPosX'] < oContainer.width and 0 <= data['iPosY'] < oContainer.height): center1 = ccore.Diff2D(data['iPosX'], data['iPosY']) # test for obj_id "under" annotated pixel first obj_id = oContainer.img_labels[center1] # if not background: valid obj_id found if obj_id > 0: object_ids.add(obj_id) try: object_lookup[label].extend([obj_id]) except KeyError: object_lookup[label] = [obj_id] # otherwise try to find nearest object in a search # radius of 30 pixel (compatibility with CellCounter) else: dists = [] for obj_id, obj in objects.iteritems(): diff = obj.oCenterAbs - center1 dist_sq = diff.squaredMagnitude() # limit to 30 pixel radius if dist_sq < 900: dists.append((obj_id, dist_sq)) if len(dists) > 0: dists.sort(lambda a,b: cmp(a[1], b[1])) obj_id = dists[0][0] object_ids.add(obj_id) try: object_lookup[label].extend([obj_id]) except KeyError: object_lookup[label] = [obj_id] objects_del = set(objects.keys()) - object_ids for obj_id in objects_del: oContainer.delObject(obj_id) # calculate features of sub channels first # want invoke time_holder for hdf5 if oChannel.is_virtual(): for mchannel, _ in oChannel.sub_channels(): self.timeholder.apply_features(mchannel) self.timeholder.apply_features(oChannel) training_set = self.annotate(object_lookup, oLearner, oContainer, oChannel.get_region(region)) if oChannel.is_virtual(): images = {} for s_ch, reg in oChannel.sub_channels(): rid = "%s_%s" %(s_ch.NAME, reg) self.draw_annotation_images(plate_id, training_set, s_ch.get_container(reg), oLearner, rid) images.update(self.write_annotation_images(s_ch, reg, oLearner)) else: self.draw_annotation_images(plate_id, training_set, oContainer, oLearner) images = self.write_annotation_images(oChannel, region, oLearner) if training_set: oLearner.set_training_data(training_set) return images def write_annotation_images(self, channel, region, learner): cnt = channel.get_container(region) images = {} if isinstance(region, tuple): region = "-".join(region) name = join(learner.controls_dir, "P%s_T%05d_C%s_R%s.jpg" %(self.P, self._iT, learner.color_channel, region)) cnt.exportRGB(name, "90") images["%s_%s" %(channel.NAME.lower(), region)] = cnt.img_rgb return images def draw_annotation_images(self, plate, training_set, container, learner, rid=""): cldir = dict([(cname, join(learner.samples_dir, cname)) \ for cname in learner.class_names.values()]) # create dir per class name for dir_ in cldir.values(): makedirs(dir_) for obj in training_set.itervalues(): rgb_value = ccore.RGBValue(*hex2rgb(obj.strHexColor)) file_ = 'PL%s___P%s___T%05d___X%04d___Y%04d' \ %(plate, self.P, self._iT, obj.oCenterAbs[0], obj.oCenterAbs[1]) obj.file = file_ file_ = join(cldir[obj.strClassName], '%s___%s.png' %(file_, rid+"_%s")) container.exportObject(obj.iId, file_ %"img", file_ %"msk") container.markObjects([obj.iId], rgb_value, False, True) ccore.drawFilledCircle(ccore.Diff2D(*obj.oCenterAbs), 3, container.img_rgb, rgb_value) def annotate(self, sample_objects, learner, container, region): """Annotate predefined class labels to picked samples.""" training_set = ObjectHolder(region.name) training_set.feature_names = region.feature_names for class_label, object_ids in sample_objects.iteritems(): class_name = learner.class_names[class_label] hex_color = learner.hexcolors[class_name] for obj_id in object_ids: obj = region[obj_id] obj.iLabel = class_label obj.strClassName = class_name obj.strHexColor = hex_color training_set[obj_id] = obj return training_set def classify_objects(self, predictor): channel = self._channel_registry[predictor.name] holder = channel.get_region(predictor.regions) try: signal_idx = holder.feature_names.index('n2_avg') roisize_idx = holder.feature_names.index('roisize') has_basic_features = True except ValueError: has_basic_features = False for label, obj in holder.iteritems(): if obj.aFeatures.size != len(holder.feature_names): msg = ('Incomplete feature set found (%d/%d): skipping sample ' 'object label %s' %(obj.aFeatures.size, len(holder.feature_names), label)) self.logger.warning(msg) else: label, probs = predictor.predict(obj.aFeatures, holder.feature_names) obj.iLabel = label obj.dctProb = probs obj.strClassName = predictor.class_names[label] obj.strHexColor = predictor.hexcolors[obj.strClassName] if has_basic_features: obj.roisize = obj.aFeatures[roisize_idx] obj.signal = obj.aFeatures[signal_idx]
class SubsectionGradeFactory(object): """ Factory for Subsection Grades. """ def __init__(self, student, course=None, course_structure=None, course_data=None): self.student = student self.course_data = course_data or CourseData(student, course=course, structure=course_structure) self._cached_subsection_grades = None self._unsaved_subsection_grades = OrderedDict() def create(self, subsection, read_only=False, force_calculate=False): """ Returns the SubsectionGrade object for the student and subsection. If read_only is True, doesn't save any updates to the grades. force_calculate - If true, will cause this function to return a `CreateSubsectionGrade` object if no cached grade currently exists, even if the assume_zero_if_absent flag is enabled for the course. """ self._log_event( log.debug, u"create, read_only: {0}, subsection: {1}".format(read_only, subsection.location), subsection, ) subsection_grade = self._get_bulk_cached_grade(subsection) if not subsection_grade: if assume_zero_if_absent(self.course_data.course_key) and not force_calculate: subsection_grade = ZeroSubsectionGrade(subsection, self.course_data) else: subsection_grade = CreateSubsectionGrade( subsection, self.course_data.structure, self._submissions_scores, self._csm_scores, ) if should_persist_grades(self.course_data.course_key): if read_only: self._unsaved_subsection_grades[subsection_grade.location] = subsection_grade else: grade_model = subsection_grade.update_or_create_model(self.student) self._update_saved_subsection_grade(subsection.location, grade_model) return subsection_grade def bulk_create_unsaved(self): """ Bulk creates all the unsaved subsection_grades to this point. """ CreateSubsectionGrade.bulk_create_models( self.student, list(self._unsaved_subsection_grades.values()), self.course_data.course_key ) self._unsaved_subsection_grades.clear() def update(self, subsection, only_if_higher=None, score_deleted=False, force_update_subsections=False, persist_grade=True): """ Updates the SubsectionGrade object for the student and subsection. """ self._log_event(log.debug, u"update, subsection: {}".format(subsection.location), subsection) calculated_grade = CreateSubsectionGrade( subsection, self.course_data.structure, self._submissions_scores, self._csm_scores, ) if persist_grade and should_persist_grades(self.course_data.course_key): if only_if_higher: try: grade_model = PersistentSubsectionGrade.read_grade(self.student.id, subsection.location) except PersistentSubsectionGrade.DoesNotExist: pass else: orig_subsection_grade = ReadSubsectionGrade(subsection, grade_model, self) if not is_score_higher_or_equal( orig_subsection_grade.graded_total.earned, orig_subsection_grade.graded_total.possible, calculated_grade.graded_total.earned, calculated_grade.graded_total.possible, treat_undefined_as_zero=True, ): return orig_subsection_grade grade_model = calculated_grade.update_or_create_model( self.student, score_deleted, force_update_subsections ) self._update_saved_subsection_grade(subsection.location, grade_model) return calculated_grade @lazy def _csm_scores(self): """ Lazily queries and returns all the scores stored in the user state (in CSM) for the course, while caching the result. """ scorable_locations = [block_key for block_key in self.course_data.structure if possibly_scored(block_key)] return ScoresClient.create_for_locations(self.course_data.course_key, self.student.id, scorable_locations) @lazy def _submissions_scores(self): """ Lazily queries and returns the scores stored by the Submissions API for the course, while caching the result. """ anonymous_user_id = anonymous_id_for_user(self.student, self.course_data.course_key) return submissions_api.get_scores(str(self.course_data.course_key), anonymous_user_id) def _get_bulk_cached_grade(self, subsection): """ Returns the student's SubsectionGrade for the subsection, while caching the results of a bulk retrieval for the course, for future access of other subsections. Returns None if not found. """ if should_persist_grades(self.course_data.course_key): saved_subsection_grades = self._get_bulk_cached_subsection_grades() grade = saved_subsection_grades.get(subsection.location) if grade: return ReadSubsectionGrade(subsection, grade, self) def _get_bulk_cached_subsection_grades(self): """ Returns and caches (for future access) the results of a bulk retrieval of all subsection grades in the course. """ if self._cached_subsection_grades is None: self._cached_subsection_grades = { record.full_usage_key: record for record in PersistentSubsectionGrade.bulk_read_grades(self.student.id, self.course_data.course_key) } return self._cached_subsection_grades def _update_saved_subsection_grade(self, subsection_usage_key, subsection_model): """ Updates (or adds) the subsection grade for the given subsection usage key in the local cache, iff the cache is populated. """ if self._cached_subsection_grades is not None: self._cached_subsection_grades[subsection_usage_key] = subsection_model def _log_event(self, log_func, log_statement, subsection): """ Logs the given statement, for this instance. """ log_func(u"Grades: SGF.{}, course: {}, version: {}, edit: {}, user: {}".format( log_statement, self.course_data.course_key, getattr(subsection, 'course_version', None), getattr(subsection, 'subtree_edited_on', None), self.student.id, ))
class Cache: """ A python class which used ordered dictionary (OrderedDict) to implement the LRU cache. Each entry of the dictinoary will be a key/value pair. The search would be by key. LRU Cache will have its maximum size defined at initiation. When adding new keys that cause the capacity to be exceed the size defined at initialtion, the oldest items will be removed to make room. The newly added items/last accessed items will be moved to the back of the dictonary (most recently used) and the elements at the begining will correspond to lest recently used. The element at the begining of the dictionary will be the one to discard when the cache if full (least recently used) Methods: get(key) - Get the value corresponding to key put(key,value) - Insert key/value into the cache delKey(key) - Delete the key reset() - Clear the cache dumpCache() - Print the cache contents """ def __init__(self, size, verbose=False): """ Initialize new cache object with the size passed. Args: size (int): The max size of the cache. """ self.printDebug = verbose self.sizeOfCache = size self.cacheLRU = OrderedDict() def get(self, key): """ Returns the value corresponding to the key provided. If the key does not exit, it returns None else it moves the key to the end of the dictionary using move_to_end (method of OrderedDict) Args: key (int): The key to lookup the value Returns: value (int): The value corresponding to the key if found else None """ if self.sizeOfCache == 0: if self.printDebug: print("Zero capacity cache and get request. Raise exception.") raise Exception('Cache is not defined') else: if key not in self.cacheLRU: return None else: self.cacheLRU.move_to_end(key) return self.cacheLRU[key] def put(self, key, value): """ Inserts the key/value pair to the cache. If the cache is already full (max capacity), it will remove the element at the head of the dict (least recently used) using popitem and insert the key/value pair at the end of the dict (most recently used). If the key already exists, it will not fail but mark the item as recently used (move it to the back of the dict) Returns: Nothing or exception if cache capacity is 0 """ if key in self.cacheLRU: if self.printDebug: print( "Key {} already exists in cache. Update this and mark as recently used" .format(key)) self.cacheLRU[key] = value self.cacheLRU.move_to_end(key) else: if self.sizeOfCache == 0: if self.printDebug: print( "Zero capacity cache and put request. Raise exception." ) raise Exception('Cache is not defined') else: if len(self.cacheLRU) >= self.sizeOfCache: outKey, outVal = self.cacheLRU.popitem(last=False) if self.printDebug: print("Cache at capacity of {}. Removing LRU key {}". format(self.sizeOfCache, outKey)) self.cacheLRU[key] = value self.cacheLRU.move_to_end(key) def delKey(self, key): """ Delete the key from the cache if it exists. If the key does not exist nothing happens. The delete is treated as a cache hit and the key is moved to the end (most recently used) and then removed. Returns: Nothing or exception if cache capacity is 0 """ if self.sizeOfCache == 0: if self.printDebug: print("Zero capacity cache and del request. Raise exception.") raise Exception('Cache is not defined') else: if key in self.cacheLRU: self.cacheLRU.move_to_end(key) self.cacheLRU.popitem(last=True) else: if self.printDebug: print( "Key {} to delete does not exist in the cache. Doing nothing.\n" .format(self.sizeOfCache)) def reset(self): """ Reset the cache Returns: Nothing or exception if cache capacity is 0 """ if self.sizeOfCache == 0: if self.printDebug: print( "Zero capacity cache and reset request. Raise exception.") raise Exception('Cache is not defined') else: self.cacheLRU.clear() def dumpCache(self): """ Print the contents of the cache Returns: Nothing """ print(self.cacheLRU)
class Graph(object): """This graph class stores the nodes and will evaluate the graph on request. Simple example: gx = Graph("newGraph") t = sumNode.SumNode("newMathNode") # first create an instance of the node gx.addNode(t) # adds a node to the graph gx.getNode("newMathNode") # gets node by name if t in gx: print t.name # check to see if newNode is in the graph """ addedNode = vortexEvent.VortexSignal() removedNode = vortexEvent.VortexSignal() addedEdge = vortexEvent.VortexSignal() deletedEdge = vortexEvent.VortexSignal() def __init__(self, name=""): """ :param name: str, the name of the graph """ self._edges = OrderedDict() self._name = name self._nodes = OrderedDict() def __repr__(self): return "{}{}".format(self.__class__.__name__, self.__dict__) def __len__(self): """Returns the length of the nodes in the graph :return: int, the length of nodes in this graph """ return len(self._nodes) def __eq__(self, other): return isinstance( other, Graph ) and self._nodes == other.nodes and self._edges == other.edges def __contains__(self, node): """Returns a bool if the node is in the graph :param node: BaseNode instance :return:bool """ try: return node in self._nodes.values() except TypeError: return False def get(self, fullPath): """Returns the node/edge/plug based on the fullPath rg. "testNode|output" would the output plug :param fullPath: str :return: Node,Plug,edge """ for node in self._nodes.values(): if node.fullPath() == fullPath: return node for plug in node.plugs.values(): if plug.fullPath() == fullPath: return plug for edge in self._edges.values(): if edge.fullPath() == fullPath: return edge def addNode(self, node, **kwargs): """Adds a Node instance to the graph this will also add the node to the graph class instance as a attribute which can be accessed by graph.node :param node: Node instance, the node to add :param kwargs: plugName=plugValue, the kwargs sets the input plugs value. :return Node instance """ if self.hasNode(node): return node.name = self.generateUniqueName(node) self._nodes[node.name] = node for plugName, plugValue in kwargs.iteritems(): plug = node.getPlug(plugName) if plug.isInput(): plug.value = plugValue for plug in node.plugs.values(): plug.connectionAdded.connect(self.addEdge) plug.connectionRemoved.connect(self.deleteEdge) self.addedNode.emit(node) return node @property def nodes(self): """Returns all the nodes in the graph :return: OrderedDict """ return self._nodes @nodes.setter def nodes(self, newNodes): """Empties and sets the nodes dict :param newNodes: dict """ self._nodes = newNodes def hasNode(self, node): """Checks the graph for the given node _name :param node: node instance :return: bool """ return node in self._nodes.values() def deleteNode(self, node): """Removes a node from the graph :param node: the node instance to delete """ if isinstance(node, str): node = self.getNode(node) node.disconnectAll() del self._nodes[node.name] self.removedNode.emit(node) def getNode(self, nodeName): """Returns a node based on the name or empty list :param nodeName: the name of the node to get :return:Node instance """ return self._nodes.get(nodeName) @property def edges(self): """Returns a dict of the graph edges :return: Dict """ return self._edges def getEdge(self, edgeName): """Returns a edge if the edge name is in edges :param edgeName: str, th name of the edge :return: """ return self._edges.get(edgeName) def addEdge(self, edge): """Adds the edge to the graph :param edge: Edge """ print "trying to add edge" if isinstance(edge, str): edge = self.getEdge(edge) if edge not in self._edges.values(): self._edges[edge.name] = edge print "creating edge" self.addedEdge.emit(edge) def deleteEdge(self, edge): """Removes the edge from the graph :param edge: Edge """ if edge in self._edges.values(): tmpEdge = edge del self._edges[edge.name] self.deletedEdge.emit(tmpEdge) def generateUniqueName(self, node): """Create a unique name for the node in the graph, on node creation a digit is appended , eg nodeName00, nodeName01 :param node: node Instance :return: str, returns the new node name as a string """ value = "%0{}d".format(0) uIndex = 0 currentIndex = [int(i) for i in node.name if i.isdigit()] if currentIndex: name = node.name.split(str(sum(currentIndex)))[0] else: name = node.name while name in self._nodes: name = name + value % uIndex uIndex += 1 return name def clear(self): """Clears all the nodes and the edges from the graph :return: None """ self._nodes.clear() self._edges.clear() def allLeaves(self): """Returns all the leaf nodes in the graph, a Leaf node is any node that has on connections :return: list(Node) """ leafNodes = [] for node in self._nodes.values(): if not all(plug.connections for plug in node.inputs()): leafNodes.append(node) return leafNodes def serializeGraph(self): """Creates a python dict from the graph, each node,plug and edge gets serialized :return: """ logger.debug("serializing graph") serializedGraph = { "name": self._name, "version": "1.0.0", "nodes": OrderedDict(), "edges": dict(), "moduleName": inspect.getmodulename(__file__) } logger.debug(serializedGraph) for node in self._nodes.values(): serializedGraph["nodes"][node.name] = node.serialize() for edge in self._edges.values(): serializedGraph["edges"][edge.name] = edge.serialize() return serializedGraph @classmethod def loadGraph(cls, graphData): """load a vortex graph dict creates and returns a graph object :param graphData: dict :return: Graph() """ graph = cls(name=graphData.get("name")) for node in graphData["nodes"].values(): moduleName = node.get("moduleName") nodeName = node.get("name") if moduleName == "baseNode": newNode = baseNode.BaseNode(name=nodeName) else: newNode = allNodes.getNode( node.get("moduleName"))(name=nodeName) for plugName, values in node.get("plugs").iteritems(): plug = newNode.getPlug(plugName=plugName) if plug: plug.value = values.get("value") continue newNode.addPlugByType(ioType=values.get("io"), name=plugName, value=values.get("value")) graph.addNode(newNode) for edge in graphData["edges"].values(): inputPlug = graph.getNode(edge["input"][1]).getPlug( edge["input"][0]) outputPlug = graph.getNode(edge["output"][1]).getPlug( edge["output"][0]) graph.addEdge( baseEdge.Edge(name=edge["name"], inputPlug=inputPlug, outputPlug=outputPlug)) return graph
class BaseSelectedFilters(object): __slots__ = ('__filters', '_selectedFilters') def __init__(self): self._selectedFilters = [] self.__filters = OrderedDict() def changeFilter(self, filterName, model): filters = model.getFilters() self._selectedFilters = [] for filterModel in filters: if filterName == filterModel.getName(): filterModel.setIsSelected(not filterModel.getIsSelected()) if filterModel.getIsSelected(): self._selectedFilters.append(filterModel.getName()) filters.invalidate() if not self._selectedFilters: model.setIsEnabled(False) else: model.setIsEnabled(True) def initFilters(self, model): filters = model.getFilters() filters.clear() for filterName in self.__filters: filterModel = self._createSubFilterModel(filterName) filters.addViewModel(filterModel) filters.invalidate() def resetFilters(self, model): for filterModel in model.getFilters(): filterModel.setIsSelected(False) self._selectedFilters = [] model.setIsEnabled(False) model.getFilters().invalidate() def getItems(self, items, model): filteredItems = self._getFilteredItems(items) model.setSelectedFilterCount(len(filteredItems)) model.setTotalFilterCount(len(items)) return filteredItems def checkItem(self, item): result = True for idx, filterName in enumerate(self._selectedFilters): filterData = self.__filters[filterName] if idx == 0 and filterData.operator is operator.or_: result = False result = filterData.operator( self.__filters[filterName].method(item), result) return result def clear(self): self.__filters = None self._selectedFilters = None return def _getFilteredItems(self, items): result = tuple((item for item in items if self.checkItem(item))) return result @staticmethod def _createSubFilterModel(filterName): filterModel = SubFilterModel() filterModel.setName(filterName) filterModel.setIsSelected(False) return filterModel def _addFilter(self, filterName, filterMethod, op=operator.and_): self.__filters[filterName] = _FilterData(filterMethod, op) def _delFilter(self, filterName): del self.__filters[filterName] def _clearFilters(self): self.__filters.clear()
class BufferPoolManager: """ Create a buffer pool manager that maps hot :class:`Document` s of a :class:`DocumentArrayMemmap` to a memory buffer. This helps keep access to memory-loaded :class:`Document` instances synced with :class:`DocumentArrayMemmap` values. The memory buffer has a fixed size and uses an LRU strategy to empty spots when full. """ def __init__(self, pool_size: int = 1000): self.pool_size = pool_size self.doc_map = OrderedDict() # dam_idx: (buffer_idx, version) self.buffer = [] self._empty = [] def add_or_update(self, idx: str, doc: 'Document') -> Optional[Tuple[str, 'Document']]: """ Adds a document to the buffer pool or updates it if it already exists :param idx: index :param doc: document :return: returns a couple of ID and :class:`Document` if there's a document to persist """ result = None # if document is already in buffer, update it if idx in self.doc_map: self.buffer[self.doc_map[idx][0]] = doc self.doc_map.move_to_end(idx) # else, if len is less than the size, append to buffer elif len(self.buffer) < self.pool_size: self.doc_map[idx] = (len(self.buffer), doc.version) self.doc_map.move_to_end(idx) self.buffer.append(doc) # else, if buffer has empty spots, allocate them elif self._empty: empty_idx = self._empty.pop() self.doc_map[idx] = (empty_idx, doc.version) self.buffer[empty_idx] = doc self.doc_map.move_to_end(idx) # else, choose a spot to free and use it with LRU strategy else: # the least recently used item is the first item in doc_map dam_idx, (buffer_idx, version) = self.doc_map.popitem(last=False) if version != self.buffer[buffer_idx].version: result = dam_idx, self.buffer[buffer_idx] self.doc_map[idx] = (buffer_idx, doc.version) self.doc_map.move_to_end(idx) self.buffer[buffer_idx] = doc return result def delete_if_exists(self, key): """ Adds a document to the buffer pool or updates it if it already exists :param key: document key """ if key in self: del self[key] def docs_to_flush(self) -> List[Tuple[str, 'Document']]: """ Persists the updated documents in disk :return: returns a list of documents to be flushed """ result = [] for dam_idx, (buffer_idx, version) in self.doc_map.items(): doc = self.buffer[buffer_idx] if version != doc.version: result.append((dam_idx, self.buffer[buffer_idx])) return result def clear(self): """ Clears the memory buffer """ self.doc_map.clear() self.buffer = [] def __getitem__(self, key: str): if isinstance(key, str): doc = self.buffer[self.doc_map[key][0]] self.doc_map.move_to_end(key) return doc else: raise TypeError(f'`key` must be str, but receiving {key!r}') def __delitem__(self, key): buffer_idx, _ = self.doc_map.pop(key) self._empty.append(buffer_idx) def __contains__(self, key): return key in self.doc_map
class SimpleClock(Checkpointable, Configurable): def __init__(self, A, skip_load=None, **kwargs): if skip_load is None: skip_load = A.pull('skip_load', False) ckpt = None if not skip_load: ckpt = A.pull('_load-ckpt', None) super().__init__(A, **kwargs) self.ticks = 0 self.alerts = OrderedDict() self._info = None self._skip_load = None if ckpt is not None: self.load_checkpoint(ckpt) def prep(self, info=None): if info is None: info = self._info for idx, (name, alert) in enumerate(self.alerts.items()): alert.prep(idx, info=info) def sort_alerts(self, start_with=None, end_with=None, strict=True): order = OrderedDict() if start_with is not None: for name in start_with: if name in self.alerts: order[name] = self.alerts[name] elif strict: raise AlertNotFoundError(name) for name, alert in self.alerts.items(): if end_with is None or name not in end_with: order[name] = alert if end_with is not None: for name in end_with: if name in self.alerts: order[name] = self.alerts[name] elif strict: raise AlertNotFoundError(name) self.alerts.clear() self.alerts.update(order) def get_info(self): return self._info def set_info(self, info): self._info = info def register_alert_fn(self, name, check=None, activate=None): self.register_alert(name, CustomAlert(check=check, activate=activate)) def register_alert(self, name, alert, **unused): if name is None: name = f'{alert}#{id(alert)}' self.alerts[name] = alert def checkpoint(self, path, ident='clock'): path = Path(path) / f'{ident}.json' states = {} for name, alert in self.alerts.items(): if isinstance(alert, Savable): states[name] = alert.state_dict() save_json({'time': self.get_time(), 'alert_states': states}, path) def load_checkpoint(self, path, ident='clock'): path = Path(path) / f'{ident}.json' if path.is_file(): data = load_json(path) self.ticks = data['time'] states = data['alert_states'] for name, state in states.items(): if name in self.alerts: self.alerts[name].load_state_dict(state) def clear(self): self.alerts.clear() self._info = None def _call_alert(self, name, alert, info=None): if info is None: info = self._info return alert.activate(self.ticks, info) def get_time(self): return self.ticks def __len__(self): return self.get_time() def set_time(self, ticks): self.ticks = ticks def tick(self, info=None): self.ticks += 1 for name, alert in self.alerts.items(): if alert.check(self.ticks, info): self._call_alert(name, alert, info=info) def step(self, info=None, n=None): if n is None: n = 1 for _ in range(n): self.tick(info=info)
class LRUCacheDict(object): """ A dictionary-like object, supporting LRU caching semantics. >>> d = LRUCacheDict(max_size=3, expiration=3) >>> d['foo'] = 'bar' >>> d['foo'] 'bar' >>> import time >>> time.sleep(4) # 4 seconds > 3 second cache expiry of d >>> d['foo'] Traceback (most recent call last): ... KeyError: 'foo' >>> d['a'] = 'A' >>> d['b'] = 'B' >>> d['c'] = 'C' >>> d['d'] = 'D' >>> d['a'] # Should return value error, since we exceeded the max cache size Traceback (most recent call last): ... KeyError: 'a' By default, this cache will only expire items whenever you poke it - all methods on this class will result in a cleanup. If the thread_clear option is specified, a background thread will clean it up every thread_clear_min_check seconds. If this class must be used in a multithreaded environment, the option concurrent should be set to true. Note that the cache will always be concurrent if a background cleanup thread is used. """ def __init__(self, max_size=1024, expiration=15 * 60): self.max_size = max_size self.expiration = expiration self.__values = {} self.__expire_times = OrderedDict() self.__access_times = OrderedDict() def size(self): return len(self.__values) def clear(self): """ Clears the dict. >>> d = LRUCacheDict(max_size=3, expiration=1) >>> d['foo'] = 'bar' >>> d['foo'] 'bar' >>> d.clear() >>> d['foo'] Traceback (most recent call last): ... KeyError: 'foo' """ self.__values.clear() self.__expire_times.clear() self.__access_times.clear() def __contains__(self, key): return key in self def has_key(self, key): """ This method should almost NEVER be used. The reason is that between the time has_key is called, and the key is accessed, the key might vanish. You should ALWAYS use a try: ... except KeyError: ... block. >>> d = LRUCacheDict(max_size=3, expiration=1) >>> d['foo'] = 'bar' >>> d['foo'] 'bar' >>> import time >>> if d.has_key('foo'): ... time.sleep(2) #Oops, the key 'foo' is gone! ... d['foo'] Traceback (most recent call last): ... KeyError: 'foo' """ return key in self.__values def __setitem__(self, key, value): t = int(time.time()) self.__delete__(key) self.__values[key] = value self.__access_times[key] = t self.__expire_times[key] = t + self.expiration self.cleanup() def __getitem__(self, key): del self.__access_times[key] self.__access_times[key] = int(time.time()) self.cleanup() return self.__values[key] def __delete__(self, key): if key in self.__values: del self.__values[key] del self.__expire_times[key] del self.__access_times[key] def cleanup(self): if self.expiration is None: return None key_need_deleted = [] for k, expire_time in self.__expire_times.items(): if expire_time < int(time.time()): key_need_deleted.append(k) else: break for key in key_need_deleted: self.__delete__(key) # If we have more than self.max_size items, delete the oldest key_need_deleted = [] delete_size = len(self.__values) - self.max_size for k in self.__access_times: if delete_size > 0: key_need_deleted.append(k) delete_size -= 1 else: break for key in key_need_deleted: self.__delete__(key)
class Scope: def __init__(self, parent=None, name=None): self.children = OrderedDict() self.index = {} self.records = OrderedDict() if name == None: self.name = self.full_name = "" else: self.name = name self.full_name = parent.full_name + name + "/" def get_scope(self, name, unique=True): if not unique: index = self.index.get(name, 0) self.index[name] = index + 1 name = name + f'_{index}' if name not in self.children: sub_scope = Scope(self, name) self.children[name] = sub_scope else: sub_scope = self.children[name] assert type( sub_scope) == Scope, f"Name {name} is a Var: {sub_scope}" return sub_scope def make_var(self, shape, dtype, init, name, unique): if not unique: index = self.index.get(name, 0) self.index[name] = index + 1 name = name + f'_{index}' if name in self.children: var = self.children[name] assert type(var) == core.Var, f"Name {name} exist: {var}" assert ( shape is None or var.shape == shape ) and var.dtype == dtype, f"Shape or dtype not match {var} != {dtype}{shape}" return var else: full_name = self.full_name + name if type(init) != core.Var: if callable(init): var = init(shape, dtype) if type(var) != core.Var: var = array(var) else: assert init != None var = array(init) else: var = init var.stop_fuse() self.children[name] = var var.name(full_name) return var def clean_index(self): self.index.clear() def clean(self): self.children.clear() self.records.clear() self.index.clear()
class _StatsSection(object): """ Not intended to be directly instantiated This is the type for the SolverStats.sections values The method parameter descriptions are the same as for those the parent with the same method name Parameters ---------- name : string key for the parent sections dictionary will also be used as the header parent : `class` : SolverStats The container for all the sections Attributes ---------- name : string key for the parent sections dictionary will also be used as the header parent : `class` : SolverStats The container for all the sections header : string Used as heading for section in report counts : OrderedDict The integer type statistics for the stats section timings : OrderedDict The timing type statistics for the stats section Expected to contain float values representing values in seconds messages : OrderedDict Text type output to be reported total_time : float Total time for processing in the section Can be None, meaning that section timing percentages will be reported """ def __init__(self, name, parent): self.parent = parent self.header = str(name) self.name = name self.counts = OrderedDict() self.timings = OrderedDict() self.messages = OrderedDict() self.total_time = None def add_count(self, key, value): """ Add value to count. If key does not already exist in section then it is created with this value. If key already exists it is increased by the given value value is expected to be an integer """ if not isinstance(value, int): try: value = int(value) except: raise TypeError("value is expected to be an integer") if key in self.counts: self.counts[key] += value else: self.counts[key] = value def add_timing(self, key, value): """ Add value to timing. If key does not already exist in section then it is created with this value. If key already exists it is increased by the give value value is expected to be a float, and given in seconds. """ if not isinstance(value, float): try: value = float(value) except: raise TypeError("value is expected to be a float") if key in self.timings: self.timings[key] += value else: self.timings[key] = value def add_message(self, key, value, sep=";"): """ Add value to message. If key does not already exist in section then it is created with this value. If key already exists the value is added to the message The value will be converted to a string """ value = str(value) if key in self.messages: if sep is not None: try: value = sep + value except: TypeError( "It is not possible to concatenate the value with " "the given seperator") self.messages[key] += value else: self.messages[key] = value def report(self, output=sys.stdout): """ Report the counts, timings and messages for this section. Note the percentage of the section and solver total times will be given if the parent and or section total_time is set """ if self.header: output.write("\n{}\n{}\n".format(self.header, ("-" * len(self.header)))) # TODO: Make the timings and counts ouput in a table format # Generally make more pretty # Report timings try: ttt = self.parent.total_time except: ttt = None tt = self.total_time output.write("### Timings:\n") for key, value in self.timings.items(): l = " - {}\t{}\n".format(_format_time(value, tt, ttt), key) output.write(l) if tt is not None: output.write(" - {}\t{} total time\n".format( _format_time(tt), self.name)) # Report counts output.write("### Counts:\n") for key, value in self.counts.items(): l = " - {}\t{}\n".format(value, key) output.write(l) # Report messages output.write("### Messages:\n") for key, value in self.messages.items(): l = " - {}:\t{}\n".format(key, value) output.write(l) def clear(self): """ Clear counts, timings and messages from this section """ self.counts.clear() self.timings.clear() self.messages.clear() self.total_time = None
class JdParser(object): def __init__(self): self.degreedic = set( line.strip() for line in codecs.open('./data/degrees.txt','rb','utf-8')) # 载入学历词库 self.majordic =set( line.strip() for line in codecs.open('./data/majordic.txt','rb','utf-8')) # 载入专业词库 self.citydic = set( line.strip() for line in codecs.open("./data/citydic.txt",'rb','utf-8')) # 载入城市词库 self.firmnames =set( line.strip() for line in codecs.open('./data/firm.txt','rb','utf-8')) # 载入公司缩写名库 self.jobdic = set(line.strip() for line in codecs.open('./data/jobposition.txt','rb','utf-8') ) # 载入招聘职位名库 self.skills = set( line.strip() for line in codecs.open('./data/skills.txt','rb','utf-8')) # self.wordlisttf = pickle.load(open('./data/wordlist.pkl')) # 出现频率最高的2000个单词 # self.w2vdict = json.load(open('./data/word2vec_50.json')) # 2000个词的word2vector self.clf = Grocery("jdclf") # 句子分类器,分为demand,duty,other self.clf.load() self.SEX = re.compile(u"性别不限|性别|男|女") self.AGE = re.compile(u"\d+周?岁|年龄") self.DEGREE = re.compile(u"(全日制)?(初中|高中|中专|大专|专科|大学专科|中职|本科|大学本科|硕士|研究生|博士|博士后)(.?以上)?") self.MAJOR = re.compile(u"\S+(相关专业|专业优先|及其.专业|[类等]专业[优先]?)") self.EXP = re.compile(u"工作经验:|工作经[历验]|工作年限|年.{0,4}经[历验]|经[历验].{1,6}年") self.PUB_TIME = re.compile(u"(\d+)(天前发布)") self.INCNAME = re.compile(u"\S+(有限公司|酒店|银行|集团|研究中心|研究所|学校|旅行社|分?公司|研发中心|技术部|事.部|招聘)") self.NOT_INC = re.compile(u"职位|描述|收藏|推荐|地址|邮箱|主页|介绍|欢迎|加入|要求|简介|险一金|奖金|包吃住|社区|厂房|人员|职责") self.INCTAG = re.compile(u"大公司|五百强|全球500强|小公司|成长型公司|创业公司|私有经济|集体经济|集团|外企|已上市|稳定性高|平均年龄\d+岁|妹纸多|学历高|福利待遇好|晋升机会大|民营公司|民营企业|互联网|创业型|国企|央企") self.JOBNAME = re.compile(u'\S*(研发工程师|工程师|经理|助理|顾问|前台|秘书|主管|研究员|实习生|操作员|专员|教学人员|技术人员|管理员|业务员|公关|程序员|教师|老师|培训生|\ 文员|研究员|策划|主任|总监|设计师|分析师|架构师|摄影师|编辑|BD|游戏UI|Android(开发)?|PHP(开发)?|Python(开发)?|.?(急招|急聘|初级|中级|高级|方向).?[\s)】\)])|\ |行政人事|网店设计|客服|会计|电话销售|外贸跟单|web前端|游戏UI|后.开发|产品运营|商业数据分析') self.START_DEMAND = re.compile(u"(岗位要求|应聘条件|任职要求|岗位资格|任职资格|岗位条件|工作要求|任职条件|人员条件|职位.求|职位条件|职位描述|岗位资格|职位资格|具备条件)[::\s]\ |如果你.{0,10}[::\s]|我们希望你.{0,12}[::\s]|(要求|条件)[::\s]|你需要?具备什么.+[?\?::\s]|任职资格[::\s]") self.DEMAND = re.compile(u"熟悉|熟练|具有|善于|懂得|掌握|具备|能够|优先|不少于|不超过|至少|团队.作|良好的|工作经验|开发经验|实习经历|能力强|富有|以上学历|经验|喜欢|\ 较强的.{2,8}能力|相关专业|相关学历|者优先|精通|了解|及以上|技术全面|.强的责任心|[能有]独立|英文流利") self.DUTY = re.compile(u"跟进|协助|负责|配合|其他工作|领导交办的|对.+提供|审核|参与|提出|跟踪|报告|为.+提出|日常.+工作|指导|对.+进行|为.+提供|跟进|拓展|运营|用户|客户|协调|拟写|通过|协同|完成|沟通|需求|秘书.{2,5}翻译") self.START_DUTY = re.compile(u"(岗位职责|岗位描述|职位描述|职责描述|任职描述|职位职责|工作职责|工作职能|职位职能|工作内容|实习内容|职位内容)[::\s]|做这样的事[::\s]|职责.{0,5}[::\s]") self.PAY = re.compile(u"薪酬|待遇|月薪|薪资|年薪|底薪|\d+k|\d+万|\d+元|工资|报酬|薪水|福利") self.BENEFIT = re.compile(u"周休|补助|补贴|假日|餐补|提成|交通补助|食宿|加班工资|期权|年假|领导|扁平化|管理|氛围|空间|休假|月假|带薪|全休|晋升|培训|舒适的|旅游|奖励|过节费|五险一金|奖金|\ |弹性工作|氛围|成长空间|实训|培训|高薪|前景|旅游|活动|分红") self.SPLIT_JD = re.compile(u"岗位[【(]?[一二三四五六七八九][】)][::\s]|(^招聘岗位\S+|岗位\d|岗位[一二三四五六])[::\s]") self.CLEAR_NUM = re.compile(u"^\d[\.: :。、]|^[\((【]?\d[\))】\.]") self.CLEAR_COLO = re.compile(u"^[\s\.。)(【】,,]|[。;,\.;,]$|^\d[\.]") self.SKILL = re.compile(u"精通|了解|熟练|熟悉|掌握|懂得|优先|具备|具有|者优先|擅长|善于|较强的.{2,6}能力|良好的|有.+经验|能力|极强的") jieba.load_userdict('./data/majordic.txt') jieba.load_userdict('./data/skills.txt') jieba.load_userdict('./data/firm.txt') jieba.load_userdict('./data/degrees.txt') jieba.load_userdict('./data/benefits.txt') self.jdStr = "" self.linelist = [] self.lineindex = defaultdict(int) self.result = OrderedDict() # 分句,预处理 def preprocess(self,jdstr): self.result.clear() jdstr = re.sub(u"[【】◆ \u25cf\u25c6\u2605]","",jdstr.decode('utf-8')) self.linelist = [ line.strip() for line in jdstr.split('\n') if len(line)>1 ] self.jdStr = '\n'.join(self.linelist) for line in self.linelist: # print self.clf.predict(line),'\t',line self.lineindex[re.sub(u"[\s ]+"," ",line)] = 0 def line2vec(self,line): vec = np.zeros(50) cnt = 1 for word in jieba.cut(line): if word in self.w2vdict: vec += self.w2vdict[word] cnt += 1 vec = vec/cnt return vec # 抽取性别要求 def regular_sex(self): """ 不限:0 男:1 女:2 """ res = set() sex = '0' for line in self.linelist: if self.clf.predict(line)=='demand' or self.DEMAND.search(line): findsex = self.SEX.search(line) if findsex: getsex = re.search(u"性别不限|男|女",line.replace(u"男女不限",u"性别不限")) if getsex: res.add(getsex.group()) break if res: sexstr = '/'.join(res) if re.search(u"男",sexstr): sex = '1' elif re.search(u"女",sexstr): sex = '2' self.result['sex'] = sex # 抽取年龄要求 def regular_age(self): res = [] for line in self.linelist: if re.search(u'\d{2}后',line):continue findage = self.AGE.search(line) if findage: findage = re.search(u"(\d{2}[\--到])\d{2}岁|(至少|不低于|不超过|不高于|不大于|大概|大约)?\d+岁|\d+岁(以上|左右|以下)?",line) if findage: agestr = findage.group() age = re.findall(u'\d{2}',agestr) age = map(int,age) if len(age)>=2: res = (age[0],age[1]) elif len(age)==1: if re.search(u'以上|不低于|至少|大于',line): res = (age[0],100) elif re.search(u"小于|低于|不超过|不得?高于|以下|不大于",line): res = (0,age[0]) elif re.search(u"左右|大约|大概",line): res = (age[0]-3,age[0]+3) break if len(res)<1: res = (0,100) self.result['age'] = res return res # 抽取专业要求 def regular_major(self): res = [] for line in self.linelist: findmajor = re.search(u"专业要求[::\s]",line) if findmajor: items = self.clean_line(line[findmajor.span()[1]:]).split() items = filter(lambda x: x not in self.degreedic and not re.search(u"薪酬|经验|元|\d+|月",x),items) res.append(' / '.join(items)) break if not res: for line in self.linelist: if re.search(u"专业.限|.限专业",line) and not re.search(u"专业优先",line): res.append(u"专业不限") break else: findmajor = self.MAJOR.search(line) if findmajor: majoritem = re.split(u'[\s,,;; ]',findmajor.group()) for item in majoritem: if re.search(u'学历|年龄|岁|学校|公司|性格|具有|具备|能够|经验|有|毕业|性别|男|女',item):continue if self.BENEFIT.search(line):continue if re.search(u"专业",item) and len(item)<3:continue res.append(self.clean_line(item)) break if not res: for majorword in jieba.cut(line): if majorword in self.majordic or majorword[:-2] in self.majordic: res.append(majorword) if re.search(u"[等及类]?相关专业",self.jdStr) and len(res)==1: res[0]+=u"等相关专业" if not res: res.append(u"专业不限") self.result['major'] = res # 抽取学历要求 def regular_degree(self): """ 抽查学历信息,先整找关键字,而后再切词,用词典匹配 """ degree=[u'小学',u'初中',u'中专',u'中技',u'高中',u'专科',u'大专',u'本科',u'硕士',u'博士',u'博士后'] res = set() for line in self.linelist: finddegree = re.search(u"学历要求[::\s]",line) if finddegree: items =self.clean_line(line[finddegree.span()[1]:]).split() items = filter(lambda x: not re.search(u"薪酬|经验|元|月|年|\d+",x),items) res.add(' / '.join(items)) break if not res: for line in self.linelist: if re.search(u"学历不限|学历要求不限|不限学历",line): res.add(u"学历不限") break else: finddegree = self.DEGREE.search(line) if finddegree: res.add(finddegree.group()) break # 如果没有匹配到学历的要求信息,就整个文本切词后匹配查找 if len(res)==0: for word in jieba.cut(self.jdStr): if word in self.degreedic: res.add(word) res = list(res) if len(res)==1 and re.search(u'[及或]?以上',res[0]): tmp = res[0][:2] if tmp==u'全日': tmp = u'本科' elif tmp==u'研究': tmp=u'硕士' if tmp in degree: idx = degree.index(tmp) res = degree[idx:] self.result['degree'] = ' / '.join(res) # 抽取工作经验年限要求 def regular_exp(self): cnyear = u"[一二三四五六七八九半两十]年" jdStr = self.jdStr res = [] if re.search(u'经[历验]不限|不限[经验]',jdStr): res = (0,100) return res for line in self.linelist: findexp = self.EXP.search(line) if findexp: exp = re.search(u"(\d[\--])?\d{1,2}年|(不少于)?半\d?年|经[历验]\d[\--]\d{1,2}年",line) if not exp: exp = re.search(cnyear,line) if exp: expstr = exp.group() # 缩小查找范围 expstr = expstr.replace(u"两",u"2").replace(u"十",u"10").replace(u"半",u"0.5").replace(u"三",u"3").replace(u"一",u'1') year = re.findall(u"\d{1,2}",expstr) if not year: continue year = map(int,year) if len(year)>1: res = (year[0],year[-1]) elif len(year)==1: if re.search(u'以上|不低于',line): res = (year[0],100) elif re.search(u"不超过|不高于|以下",line): res = (0,year[0]) elif re.search(u"大概|大约|左右",line): if year[0]-3<0.001: minyear = 0 res = (minyear,year[0]+3) break if not res: res = (0,100) self.result["exp"] = tuple(res) return res def regular_jobtag(self): """ 有关职位标签信息 """ res = [] job_tag = re.search(u"应届生|全职|兼职|实习生|应届毕业生|社招|急招|急聘",self.jdStr) if job_tag: res.append(job_tag.group()) job_tag = re.search(u"招聘人数[::]?|招聘[::\s]|人数[::\s]",self.jdStr) if job_tag: jdstr = self.jdStr[job_tag.span()[1]:] for line in jdstr.split(): if len(line.strip())<1:continue else: num = re.search(u"(\d+\-)?\d+人?|若干|\d+位",line) if num: res.append(u"招聘人数:"+num.group()) break job_tag = re.search(u"(职能类别|职位标签)[:: ]?",self.jdStr) if job_tag: jdstr = self.jdStr[job_tag.span()[1]:] for line in jdstr.split('\n'): if len(line.strip())<3:continue else: res.append("职业标签:"+line.strip()) break if len(line)>25:break # 根据产品部需求专门切割出包含经验的句子等有关职位标注信息,句子进行更精细化切割 linelist = [ line for line in re.split(u"[,。;\s]",self.jdStr) if 5<len(line)<15 ] for line in linelist: if re.search(u"经验",line) and not re.search(u"月薪|地点|日期",line): if re.search(u"\d+k|[。?)\)\]]",line):continue res.append(self.clean_line(line)) break self.result["job_tag"] = res return res # 清除句子前的数字和标点符合 def clean_line(self,line): line = self.CLEAR_NUM.sub("",line.strip()) line = self.CLEAR_COLO.sub("",line) return line # 抽取工作地点 def regular_workplace(self): res = set() jdstr = self.jdStr pos = list(re.finditer(u"(工作地.|上班地.|实习地.|地址|地点)[::\s]",jdstr)) if pos: jdstr = jdstr[pos[0].span()[1]:] for line in jdstr.split(): if len(line.strip())<2:continue if len(line)<26: res.add(line.strip().replace(":","").replace(":","")) else: for city in jieba.cut(line): if city in self.citydic and city[:-1] not in res: res.add(city) break if not res: for city in jieba.cut(jdstr): if city in self.citydic and city[:-1] not in res and u"国" not in city: res.add(city) break self.result["workplace"] = " / ".join(res) return res # 抽取证书获奖情况等其他要求 def regular_cert(self): res = set() linelist = [line for line in re.split(u"[\s ,。;,]",self.jdStr) if len(line)>3] for line in linelist: findcert = re.search(u"(\S+证书|CET-\d|普通话|英语|口语|.语|日文|雅思|托福|托业)(至少)?(通过)?[\d一二三四五六七八九]级[及或]?(以上)?|(英语)?CET-\d级?(以上)?|职业资格|律师证|会计证",line) if findcert: res.add(findcert.group()) else: findcert = re.search(u"有(.+证)书?",line) if findcert: res.add(findcert.group(1)) else: findcert = re.search(u"有.+资格",line) if findcert: res.add(findcert.group()) self.result['cert'] = re.sub(u"[或及以上]","",' / '.join(res)) if self.result['cert']: self.result['cert'] = self.result['cert'].split(' / ') else: self.result['cert'] = [] # 利用技能词库抽取相关技能 def regular_skill(self,num=6): res = [] for line in self.linelist: if self.DEMAND.search(line) or self.clf.predict(line)=='demand': for word in jieba.cut(line): word = strQ2B(word).lower() if word in self.skills: res.append(word) sorted_words = [w[0] for w in Counter(res).most_common(2*num)] for word in jieba.cut(self.result['job_name']): word = strQ2B(word).lower() if word in self.skills and word not in sorted_words: sorted_words.insert(0,word) after_top3 = sorted_words[3:] np.random.shuffle(after_top3) keywords = sorted_words[:3]+after_top3[:num-3] keywords.sort() self.result['skill'] = keywords # 抽取岗位职责 def regular_duty(self): res = [] jdStr = self.jdStr pos = list(self.START_DUTY.finditer(jdStr)) if len(pos)>0: linelist =[ re.sub("[\s ]+"," ",line) for line in jdStr[pos[-1].span()[1]:].split("\n") if len(line)>1] for i in xrange(len(linelist)): line = linelist[i] print 'dutyline',line if self.START_DUTY.search(line) or self.lineindex[line]==1 or re.search(u".年来|谢谢|请在|公司介绍|举报|收藏|岗位职责|介绍|描述|[\-=\.]{3,}",line):continue if re.search(u"要求[::\s]?|岗位要求",line) and len(line)<6:break if re.match(u"\d{1,2}|\u25cf|[\uff0d(\(\-\+]|[a-z][\.、\s]",line.strip()) or self.DUTY.search(line) or self.clf.predict(line)=='duty': res.append(line.strip()) print 'dutyline',line elif i<len(linelist)-1 and self.clf.predict(linelist[i+1])=='duty': res.append(line) else: break if not res: for line in self.linelist: if re.search(u"粉丝团",line) and len(line)<12:continue if self.DUTY.search(line) and self.clf.predict(line) == "duty": if self.lineindex[line]!=1: res.append(line) self.result["duty"] ="\n".join(res) for line in res: self.lineindex[line]=1 return res # 抽取岗位要求 def regular_demand(self): res = [] jdStr =self.jdStr pos = list(self.START_DEMAND.finditer(jdStr)) if len(pos)>0: tmppos = pos[-1].span()[0] if re.search(u"具有|具备",jdStr[tmppos-5:tmppos+5]) or re.search(u"证书|证",jdStr[tmppos:tmppos+8]): pos.pop() if pos: linelist =[ re.sub("[\s ]+"," ",line) for line in jdStr[pos[-1].span()[1]:].split("\n") if len(line)>2] else: linelist = [] for i in xrange(len(linelist)): line = linelist[i] if self.START_DEMAND.search(linelist[i]) or re.search(u"谢谢|请在|公司介绍|举报|收藏|\d+k?元|加分",line):continue if re.match(u"\d{1,2}|\u25cf|[\uff0d(\(\-\+]|[a-z][\.、\s]",line) or self.DEMAND.search(line) or self.clf.predict(line)=='demand': res.append(line) elif i<len(linelist)-1 and self.clf.predict(linelist[i+1])=='demand': res.append(line) else: break if not res: for line in self.linelist: if self.lineindex[line]==1 or len(line.split())>6:continue # 如果该句已经被处理过,就不再重复显示 if self.clf.predict(line)=='demand' or self.DEMAND.search(line): res.append(line.strip()) self.result['demand'] = '\n'.join(res) for line in res: self.lineindex[line]=1 return res # 招聘的职位名 def regular_jobname(self): res = set() jdStr = self.jdStr findpos = re.search(u"(招聘岗位|招聘职位|职位名称|岗位名称|岗位[一二三四五六七八九])[:、:\s ]",jdStr) # if not findpos: # findpos = re.search(u"(职位类别|职位职能)[::\s ]",jdStr) if findpos: pos = findpos.span()[1] linelist = jdStr[pos:].split("\n") for line in linelist: if len(line)<2:continue if len(line)>=2 and len(line)<20: if re.search(u"职位描述|查看|地址|工作|分享|举报|下一条|时间|福利|待遇|周末|双休",line):continue res.add(re.sub(u"聘请|高薪诚聘|诚聘|[,。、\d!]+","",line.strip())) break # 如果没有匹配到招聘的具体职位信息,就切词后到职位列表去匹配 if not res: for line in self.linelist: if re.search(u"招聘|高薪|诚聘",line):continue if len(line)<6 and not re.search(u'岗位|岗位内容|工作内容|职责|任职|资格',line) and self.clf.predict(line)=='job_name': res.add(line) break findPos = self.JOBNAME.search(line) if findPos and len(findPos.group())<20 and not re.match(u'\d',findPos.group()): jobname = findPos.group() res.add(re.sub(u"聘请|高薪诚聘|诚聘|急.|[,。、!]+","",jobname)) break # res.add(re.sub(u"\(.+\)|(.+)|【.+】|[,。、\s\d]+|聘请|高薪诚聘|诚聘|急招|","",line.strip())) if not res: for line in self.linelist: for word in jieba.cut(line.lower()): if word in self.jobdic: res.add(word) self.result["job_name"] = " / ".join(res) return res if not res: tag = re.search(u"实习生|兼职",self.jdStr) if tag: res.add(tag.group()) self.result["job_name"] = strQ2B(" / ".join(res)).lower() return res # 薪酬 def regular_pay(self): paystr = "" lagoup = re.search(u"(\d+[kK][-——]\d+[kK])|(\d{3,5}-\d{3,5}元?/[月日天])|(\d{3,5}-\d{3,5}元)|((\d+[-~]\d+)万.[年月])|底薪\d+(-\d+)?元?|\d{3,5}元(左右|以上)?|年薪\d+万?元(左右|以上)?",self.jdStr) # 针对拉勾网,没有待遇等关键字符 if lagoup: paystr = lagoup.group() else: findpay = self.PAY.search(self.jdStr) if findpay: pos = findpay.span()[1] jdstr = self.jdStr[max(0,pos-5):min(pos+10,len(self.jdStr))] if re.search(u"面议",jdstr): paystr = u"面议" else: findpay = re.findall(u"\d{3,7}",jdstr) paystr = "-".join(findpay) paystr = paystr.replace(u'k','000').replace(u'K','000').replace(u"万","0000") pay = re.findall('\d+',paystr) pay = map(int,pay) if len(pay)>1: res = tuple((pay[0],pay[-1])) elif len(pay)==1: res = (pay[0],0) else: res = (0,0) self.result['pay'] = res return res # 抽取薪资福利 def regular_benefits(self): res = [] jdStr = self.jdStr findpos =list(re.finditer(u"薪酬福利[::\s]|(福利|待遇)\s?[::】]",jdStr)) if not findpos: findpos =list(re.finditer(u"(晋升制度|工作环境|职位诱惑|你会获得什么)\s?[?\?::】]",jdStr)) if findpos: pos = findpos[-1].span()[1] linelist = jdStr[pos:].split('\n') for line in linelist: if len(line.strip())<3:continue if re.match(ur"[((]?\d+",line) or self.BENEFIT.search(line): res.append(line.strip()) self.lineindex[line.strip()] = 1 else: break if not res: for line in jdStr.split(): if len(line)>1 and re.search(u"带薪|双休|股票期权|五险一金|发展空间|福利|诱惑|休假|薪酬|补助|年假|弹性工作",line): if re.search(u"福利|待遇|诱惑",line) and len(line.strip())<6:continue res.append(line.strip()) if len(res)==1 and re.search(u"险一金",res[0]) and not re.search(u"[,、]",res[0]): res[0] = self.clean_line(' '.join(jieba.cut(res[0]))) self.result["benefits"] ="\n".join(res) return res
class CachedStore(BaseStore): """ A thread-safe cached persistent store of the subscriber database. Prerequisite: persistent_store need to be thread safe """ def __init__(self, persistent_store, cache_capacity=512, loop=None): self._lock = threading.Lock() self._cache = OrderedDict() self._cache_capacity = cache_capacity self._persistent_store = persistent_store self._on_ready = OnDataReady(loop=loop) def add_subscriber(self, subscriber_data): """ Method that adds the subscriber. """ sid = SIDUtils.to_str(subscriber_data.sid) with self._lock: if sid in self._cache: raise DuplicateSubscriberError(sid) self._persistent_store.add_subscriber(subscriber_data) self._cache_put(sid, subscriber_data) self._on_ready.add_subscriber(subscriber_data) @contextmanager def edit_subscriber(self, subscriber_id): """ Context manager to modify the subscriber data. """ with self._lock: if subscriber_id in self._cache: data = self._cache_get(subscriber_id) subscriber_data = copy.deepcopy(data) else: subscriber_data = \ self._persistent_store.get_subscriber_data(subscriber_id) yield subscriber_data self._persistent_store.update_subscriber(subscriber_data) self._cache_put(subscriber_id, subscriber_data) def delete_subscriber(self, subscriber_id): """ Method that deletes a subscriber, if present. """ with self._lock: if subscriber_id in self._cache: del self._cache[subscriber_id] self._persistent_store.delete_subscriber(subscriber_id) def delete_all_subscribers(self): """ Method that removes all the subscribers from the store """ with self._lock: self._cache_clear() self._persistent_store.delete_all_subscribers() def resync(self, subscribers): """ Method that should resync the store with the mentioned list of subscribers. The resync leaves the current state of subscribers intact. Args: subscribers - list of subscribers to be in the store. """ with self._lock: self._cache_clear() self._persistent_store.resync(subscribers) self._on_ready.resync(subscribers) def get_subscriber_data(self, subscriber_id): """ Method that returns the subscriber data for the subscriber. """ with self._lock: if subscriber_id in self._cache: return self._cache_get(subscriber_id) else: subscriber_data = \ self._persistent_store.get_subscriber_data(subscriber_id) self._cache_put(subscriber_id, subscriber_data) return subscriber_data def list_subscribers(self): """ Method that returns the list of subscribers stored. Note: this method is not cached since it's easier to get the whole list from persistent store """ return self._persistent_store.list_subscribers() async def on_ready(self): return await self._on_ready.event.wait() def _cache_get(self, k): """ Get from the LRU cache. Move the last hit entry to the end. """ self._cache.move_to_end(k) return self._cache[k] def _cache_put(self, k, v): """ Put to the LRU cache. Evict the first item if full. """ if self._cache_capacity == len(self._cache): self._cache.popitem(last=False) self._cache[k] = v def _cache_list(self): return list(self._cache.keys()) def _cache_clear(self): self._cache.clear()
class LRU(object): """ A dictionary-like object, supporting LRU caching semantics. >>> d = LRU(max_size=3) >>> d['foo'] = 'bar' >>> d['foo'] 'bar' >>> d['a'] = 'A' >>> d['b'] = 'B' >>> d['c'] = 'C' >>> d['d'] = 'D' >>> d['a'] # Should return value error, since we exceeded the max cache size Traceback (most recent call last): ... KeyError: 'a' """ def __init__(self, max_size=1024): self.max_size = max_size self.__access_times = OrderedDict() def __len__(self): return len(self.__access_times) def clear(self): """ Clears the dict. >>> d = LRU(max_size=3) >>> d['foo'] = 'bar' >>> d['foo'] 'bar' >>> d.clear() >>> d['foo'] Traceback (most recent call last): ... KeyError: 'foo' """ self.__access_times.clear() def __contains__(self, key): """ This method should almost NEVER be used. The reason is that between the time has_key is called, and the key is accessed, the key might vanish. You should ALWAYS use a try: ... except KeyError: ... block. >>> d = LRU(max_size=3) >>> d['foo'] = 'bar' >>> d['foo'] 'bar' >>> 'foo' in d True >>> 'goo' in d False """ return self.__access_times.has_key(key) def __setitem__(self, key, value): self.__access_times[key] = value self.cleanup() def __getitem__(self, key): return self.__access_times[key] def __delete__(self, key): del self.__access_times[key] def cleanup(self): #If we have more than self.max_size items, delete the oldest while (len(self.__access_times) > self.max_size): for k in self.__access_times.iterkeys(): self.__delete__(k) break
class Cache(object): __slots__ = ( "cache", "max_entries", "name", "keylen", "sequence", "thread", "metrics", ) def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False): if lru: cache_type = TreeCache if tree else dict self.cache = LruCache(max_size=max_entries, keylen=keylen, cache_type=cache_type) self.max_entries = None else: self.cache = OrderedDict() self.max_entries = max_entries self.name = name self.keylen = keylen self.sequence = 0 self.thread = None self.metrics = register_cache(name, self.cache) def check_thread(self): expected_thread = self.thread if expected_thread is None: self.thread = threading.current_thread() else: if expected_thread is not threading.current_thread(): raise ValueError( "Cache objects can only be accessed from the main thread") def get(self, key, default=_CacheSentinel): val = self.cache.get(key, _CacheSentinel) if val is not _CacheSentinel: self.metrics.inc_hits() return val self.metrics.inc_misses() if default is _CacheSentinel: raise KeyError() else: return default def update(self, sequence, key, value): self.check_thread() if self.sequence == sequence: # Only update the cache if the caches sequence number matches the # number that the cache had before the SELECT was started (SYN-369) self.prefill(key, value) def prefill(self, key, value): if self.max_entries is not None: while len(self.cache) >= self.max_entries: self.cache.popitem(last=False) self.cache[key] = value def invalidate(self, key): self.check_thread() if not isinstance(key, tuple): raise TypeError("The cache key must be a tuple not %r" % (type(key), )) # Increment the sequence number so that any SELECT statements that # raced with the INSERT don't update the cache (SYN-369) self.sequence += 1 self.cache.pop(key, None) def invalidate_many(self, key): self.check_thread() if not isinstance(key, tuple): raise TypeError("The cache key must be a tuple not %r" % (type(key), )) self.sequence += 1 self.cache.del_multi(key) def invalidate_all(self): self.check_thread() self.sequence += 1 self.cache.clear()
gen_max_file = open('gen_max.txt', 'w') converge_time = [] run_time = [] rs = time.time() g = 0 # for g in range(0, generations): while True: print('The generation number is: ', g + 1) file.write('\nThe generation number is:' + str(g + 1)) dicts = OrderedDict(sorted(dicts.items(), key=lambda x: x[1])) cs = time.time() new_pop = mating_pool(list(dicts.keys()), neurons, dicts, sample_weights) final_pop = mutate_pool(new_pop, neurons, number_of_mutate_node) ce = time.time() converge_time.append(ce - cs) dicts.clear() fit_result = multiprocess(pool, final_pop, x_train, y_train) prob_pop = probability(fit_result) dicts = {final_pop[i]: [fit_result[i], prob_pop[i]] for i in range(0, len(final_pop))} print('The mean of fitness is:', np.mean(fit_result)) print('The best of fitness is:', np.max(fit_result)) file.write('\nThe mean of fitness is:' + str(np.mean(fit_result))) file.write('\nThe best of fitness is:' + str(np.max(fit_result))) file.flush() gen_fit_file.write('\n' + str(fit_result)) gen_fit_file.flush() gen_mean_file.write('\n' + str(np.mean(fit_result))) gen_mean_file.flush() gen_max_file.write('\n' + str(np.max(fit_result))) gen_max_file.flush()
class BacktestingEngine(object): """ CTA回测引擎 函数接口和策略引擎保持一样, 从而实现同一套代码从回测到实盘。 """ TICK_MODE = 'tick' BAR_MODE = 'bar' #---------------------------------------------------------------------- def __init__(self): """Constructor""" # 本地停止单编号计数 self.stopOrderCount = 0 # stopOrderID = STOPORDERPREFIX + str(stopOrderCount) # 本地停止单字典 # key为stopOrderID,value为stopOrder对象 self.stopOrderDict = {} # 停止单撤销后不会从本字典中删除 self.workingStopOrderDict = {} # 停止单撤销后会从本字典中删除 # 引擎类型为回测 self.engineType = ENGINETYPE_BACKTESTING # 回测相关 self.strategy = None # 回测策略 self.mode = self.BAR_MODE # 回测模式,默认为K线 self.startDate = '' self.initDays = 0 self.endDate = '' self.slippage = 0 # 回测时假设的滑点 self.rate = 0 # 回测时假设的佣金比例(适用于百分比佣金) self.size = 1 # 合约大小,默认为1 self.priceTick = 0 # 价格最小变动 self.dbClient = None # 数据库客户端 self.dbCursor = None # 数据库指针 #self.historyData = [] # 历史数据的列表,回测用 self.initData = [] # 初始化用的数据 #self.backtestingData = [] # 回测用的数据 self.dbName = '' # 回测数据库名 self.symbol = '' # 回测集合名 self.dataStartDate = None # 回测数据开始日期,datetime对象 self.dataEndDate = None # 回测数据结束日期,datetime对象 self.strategyStartDate = None # 策略启动日期(即前面的数据用于初始化),datetime对象 self.limitOrderDict = OrderedDict() # 限价单字典 self.workingLimitOrderDict = OrderedDict() # 活动限价单字典,用于进行撮合用 self.limitOrderCount = 0 # 限价单编号 self.tradeCount = 0 # 成交编号 self.tradeDict = OrderedDict() # 成交字典 self.logList = [] # 日志记录 # 当前最新数据,用于模拟成交用 self.tick = None self.bar = None self.dt = None # 最新的时间 #---------------------------------------------------------------------- def setStartDate(self, startDate='20100416', initDays=10): """设置回测的启动日期""" self.startDate = startDate self.initDays = initDays self.dataStartDate = datetime.strptime(startDate, '%Y%m%d') initTimeDelta = timedelta(initDays) self.strategyStartDate = self.dataStartDate + initTimeDelta #---------------------------------------------------------------------- def setEndDate(self, endDate=''): """设置回测的结束日期""" self.endDate = endDate if endDate: self.dataEndDate = datetime.strptime(endDate, '%Y%m%d') # 若不修改时间则会导致不包含dataEndDate当天数据 self.dataEndDate = self.dataEndDate.replace(hour=23, minute=59) #---------------------------------------------------------------------- def setBacktestingMode(self, mode): """设置回测模式""" self.mode = mode #---------------------------------------------------------------------- def setDatabase(self, dbName, symbol): """设置历史数据所用的数据库""" self.dbName = dbName self.symbol = symbol #---------------------------------------------------------------------- def loadHistoryData(self): """载入历史数据""" host, port, logging = loadMongoSetting() self.dbClient = pymongo.MongoClient(host, port) collection = self.dbClient[self.dbName][self.symbol] self.output(u'开始载入数据') # 首先根据回测模式,确认要使用的数据类 if self.mode == self.BAR_MODE: dataClass = CtaBarData func = self.newBar else: dataClass = CtaTickData func = self.newTick # 载入初始化需要用的数据 flt = { 'datetime': { '$gte': self.dataStartDate, '$lt': self.strategyStartDate } } initCursor = collection.find(flt) # 将数据从查询指针中读取出,并生成列表 self.initData = [] # 清空initData列表 for d in initCursor: data = dataClass() data.__dict__ = d self.initData.append(data) # 载入回测数据 if not self.dataEndDate: flt = {'datetime': {'$gte': self.strategyStartDate}} # 数据过滤条件 else: flt = { 'datetime': { '$gte': self.strategyStartDate, '$lte': self.dataEndDate } } self.dbCursor = collection.find(flt) self.output(u'载入完成,数据量:%s' % (initCursor.count() + self.dbCursor.count())) #---------------------------------------------------------------------- def runBacktesting(self): """运行回测""" # 载入历史数据 self.loadHistoryData() # 首先根据回测模式,确认要使用的数据类 if self.mode == self.BAR_MODE: dataClass = CtaBarData func = self.newBar else: dataClass = CtaTickData func = self.newTick self.output(u'开始回测') self.strategy.inited = True self.strategy.onInit() self.output(u'策略初始化完成') self.strategy.trading = True self.strategy.onStart() self.output(u'策略启动完成') self.output(u'开始回放数据') for d in self.dbCursor: data = dataClass() data.__dict__ = d func(data) self.output(u'数据回放结束') #---------------------------------------------------------------------- def newBar(self, bar): """新的K线""" self.bar = bar self.dt = bar.datetime self.crossLimitOrder() # 先撮合限价单 self.crossStopOrder() # 再撮合停止单 self.strategy.onBar(bar) # 推送K线到策略中 #---------------------------------------------------------------------- def newTick(self, tick): """新的Tick""" self.tick = tick self.dt = tick.datetime self.crossLimitOrder() self.crossStopOrder() self.strategy.onTick(tick) #---------------------------------------------------------------------- def initStrategy(self, strategyClass, setting=None): """ 初始化策略 setting是策略的参数设置,如果使用类中写好的默认设置则可以不传该参数 """ self.strategy = strategyClass(self, setting) self.strategy.name = self.strategy.className #---------------------------------------------------------------------- def sendOrder(self, vtSymbol, orderType, price, volume, strategy): """发单""" self.limitOrderCount += 1 orderID = str(self.limitOrderCount) order = VtOrderData() order.vtSymbol = vtSymbol order.price = self.roundToPriceTick(price) order.totalVolume = volume order.status = STATUS_NOTTRADED # 刚提交尚未成交 order.orderID = orderID order.vtOrderID = orderID order.orderTime = str(self.dt) # CTA委托类型映射 if orderType == CTAORDER_BUY: order.direction = DIRECTION_LONG order.offset = OFFSET_OPEN elif orderType == CTAORDER_SELL: order.direction = DIRECTION_SHORT order.offset = OFFSET_CLOSE elif orderType == CTAORDER_SHORT: order.direction = DIRECTION_SHORT order.offset = OFFSET_OPEN elif orderType == CTAORDER_COVER: order.direction = DIRECTION_LONG order.offset = OFFSET_CLOSE # 保存到限价单字典中 self.workingLimitOrderDict[orderID] = order self.limitOrderDict[orderID] = order return orderID #---------------------------------------------------------------------- def cancelOrder(self, vtOrderID): """撤单""" if vtOrderID in self.workingLimitOrderDict: order = self.workingLimitOrderDict[vtOrderID] order.status = STATUS_CANCELLED order.cancelTime = str(self.dt) del self.workingLimitOrderDict[vtOrderID] #---------------------------------------------------------------------- def sendStopOrder(self, vtSymbol, orderType, price, volume, strategy): """发停止单(本地实现)""" self.stopOrderCount += 1 stopOrderID = STOPORDERPREFIX + str(self.stopOrderCount) so = StopOrder() so.vtSymbol = vtSymbol so.price = self.roundToPriceTick(price) so.volume = volume so.strategy = strategy so.stopOrderID = stopOrderID so.status = STOPORDER_WAITING if orderType == CTAORDER_BUY: so.direction = DIRECTION_LONG so.offset = OFFSET_OPEN elif orderType == CTAORDER_SELL: so.direction = DIRECTION_SHORT so.offset = OFFSET_CLOSE elif orderType == CTAORDER_SHORT: so.direction = DIRECTION_SHORT so.offset = OFFSET_OPEN elif orderType == CTAORDER_COVER: so.direction = DIRECTION_LONG so.offset = OFFSET_CLOSE # 保存stopOrder对象到字典中 self.stopOrderDict[stopOrderID] = so self.workingStopOrderDict[stopOrderID] = so return stopOrderID #---------------------------------------------------------------------- def cancelStopOrder(self, stopOrderID): """撤销停止单""" # 检查停止单是否存在 if stopOrderID in self.workingStopOrderDict: so = self.workingStopOrderDict[stopOrderID] so.status = STOPORDER_CANCELLED del self.workingStopOrderDict[stopOrderID] #---------------------------------------------------------------------- def crossLimitOrder(self): """基于最新数据撮合限价单""" # 先确定会撮合成交的价格 if self.mode == self.BAR_MODE: buyCrossPrice = self.bar.low # 若买入方向限价单价格高于该价格,则会成交 sellCrossPrice = self.bar.high # 若卖出方向限价单价格低于该价格,则会成交 buyBestCrossPrice = self.bar.open # 在当前时间点前发出的买入委托可能的最优成交价 sellBestCrossPrice = self.bar.open # 在当前时间点前发出的卖出委托可能的最优成交价 else: buyCrossPrice = self.tick.askPrice1 sellCrossPrice = self.tick.bidPrice1 buyBestCrossPrice = self.tick.askPrice1 sellBestCrossPrice = self.tick.bidPrice1 # 遍历限价单字典中的所有限价单 for orderID, order in self.workingLimitOrderDict.items(): # 判断是否会成交 buyCross = (order.direction == DIRECTION_LONG and order.price >= buyCrossPrice and buyCrossPrice > 0 ) # 国内的tick行情在涨停时askPrice1为0,此时买无法成交 sellCross = (order.direction == DIRECTION_SHORT and order.price <= sellCrossPrice and sellCrossPrice > 0 ) # 国内的tick行情在跌停时bidPrice1为0,此时卖无法成交 # 如果发生了成交 if buyCross or sellCross: # 推送成交数据 self.tradeCount += 1 # 成交编号自增1 tradeID = str(self.tradeCount) trade = VtTradeData() trade.vtSymbol = order.vtSymbol trade.tradeID = tradeID trade.vtTradeID = tradeID trade.orderID = order.orderID trade.vtOrderID = order.orderID trade.direction = order.direction trade.offset = order.offset # 以买入为例: # 1. 假设当根K线的OHLC分别为:100, 125, 90, 110 # 2. 假设在上一根K线结束(也是当前K线开始)的时刻,策略发出的委托为限价105 # 3. 则在实际中的成交价会是100而不是105,因为委托发出时市场的最优价格是100 if buyCross: trade.price = min(order.price, buyBestCrossPrice) self.strategy.pos += order.totalVolume else: trade.price = max(order.price, sellBestCrossPrice) self.strategy.pos -= order.totalVolume trade.volume = order.totalVolume trade.tradeTime = str(self.dt) trade.dt = self.dt self.strategy.onTrade(trade) self.tradeDict[tradeID] = trade # 推送委托数据 order.tradedVolume = order.totalVolume order.status = STATUS_ALLTRADED self.strategy.onOrder(order) # 从字典中删除该限价单 del self.workingLimitOrderDict[orderID] #---------------------------------------------------------------------- def crossStopOrder(self): """基于最新数据撮合停止单""" # 先确定会撮合成交的价格,这里和限价单规则相反 if self.mode == self.BAR_MODE: buyCrossPrice = self.bar.high # 若买入方向停止单价格低于该价格,则会成交 sellCrossPrice = self.bar.low # 若卖出方向限价单价格高于该价格,则会成交 bestCrossPrice = self.bar.open # 最优成交价,买入停止单不能低于,卖出停止单不能高于 else: buyCrossPrice = self.tick.lastPrice sellCrossPrice = self.tick.lastPrice bestCrossPrice = self.tick.lastPrice # 遍历停止单字典中的所有停止单 for stopOrderID, so in self.workingStopOrderDict.items(): # 判断是否会成交 buyCross = so.direction == DIRECTION_LONG and so.price <= buyCrossPrice sellCross = so.direction == DIRECTION_SHORT and so.price >= sellCrossPrice # 如果发生了成交 if buyCross or sellCross: # 推送成交数据 self.tradeCount += 1 # 成交编号自增1 tradeID = str(self.tradeCount) trade = VtTradeData() trade.vtSymbol = so.vtSymbol trade.tradeID = tradeID trade.vtTradeID = tradeID if buyCross: self.strategy.pos += so.volume trade.price = max(bestCrossPrice, so.price) else: self.strategy.pos -= so.volume trade.price = min(bestCrossPrice, so.price) self.limitOrderCount += 1 orderID = str(self.limitOrderCount) trade.orderID = orderID trade.vtOrderID = orderID trade.direction = so.direction trade.offset = so.offset trade.volume = so.volume trade.tradeTime = str(self.dt) trade.dt = self.dt self.strategy.onTrade(trade) self.tradeDict[tradeID] = trade # 推送委托数据 so.status = STOPORDER_TRIGGERED order = VtOrderData() order.vtSymbol = so.vtSymbol order.symbol = so.vtSymbol order.orderID = orderID order.vtOrderID = orderID order.direction = so.direction order.offset = so.offset order.price = so.price order.totalVolume = so.volume order.tradedVolume = so.volume order.status = STATUS_ALLTRADED order.orderTime = trade.tradeTime self.strategy.onOrder(order) self.limitOrderDict[orderID] = order # 从字典中删除该限价单 if stopOrderID in self.workingStopOrderDict: del self.workingStopOrderDict[stopOrderID] #---------------------------------------------------------------------- def insertData(self, dbName, collectionName, data): """考虑到回测中不允许向数据库插入数据,防止实盘交易中的一些代码出错""" pass #---------------------------------------------------------------------- def loadBar(self, dbName, collectionName, startDate): """直接返回初始化数据列表中的Bar""" return self.initData #---------------------------------------------------------------------- def loadTick(self, dbName, collectionName, startDate): """直接返回初始化数据列表中的Tick""" return self.initData #---------------------------------------------------------------------- def writeCtaLog(self, content): """记录日志""" log = str(self.dt) + ' ' + content self.logList.append(log) #---------------------------------------------------------------------- def output(self, content): """输出内容""" print(str(datetime.now()) + "\t" + content) #---------------------------------------------------------------------- def calculateBacktestingResult(self): """ 计算回测结果 """ self.output(u'计算回测结果') # 首先基于回测后的成交记录,计算每笔交易的盈亏 resultList = [] # 交易结果列表 longTrade = [] # 未平仓的多头交易 shortTrade = [] # 未平仓的空头交易 tradeTimeList = [] # 每笔成交时间戳 posList = [0] # 每笔成交后的持仓情况 for trade in self.tradeDict.values(): # 多头交易 if trade.direction == DIRECTION_LONG: # 如果尚无空头交易 if not shortTrade: longTrade.append(trade) # 当前多头交易为平空 else: while True: entryTrade = shortTrade[0] exitTrade = trade # 清算开平仓交易 closedVolume = min(exitTrade.volume, entryTrade.volume) result = TradingResult(entryTrade.price, entryTrade.dt, exitTrade.price, exitTrade.dt, -closedVolume, self.rate, self.slippage, self.size) resultList.append(result) posList.extend([-1, 0]) tradeTimeList.extend([result.entryDt, result.exitDt]) # 计算未清算部分 entryTrade.volume -= closedVolume exitTrade.volume -= closedVolume # 如果开仓交易已经全部清算,则从列表中移除 if not entryTrade.volume: shortTrade.pop(0) # 如果平仓交易已经全部清算,则退出循环 if not exitTrade.volume: break # 如果平仓交易未全部清算, if exitTrade.volume: # 且开仓交易已经全部清算完,则平仓交易剩余的部分 # 等于新的反向开仓交易,添加到队列中 if not shortTrade: longTrade.append(exitTrade) break # 如果开仓交易还有剩余,则进入下一轮循环 else: pass # 空头交易 else: # 如果尚无多头交易 if not longTrade: shortTrade.append(trade) # 当前空头交易为平多 else: while True: entryTrade = longTrade[0] exitTrade = trade # 清算开平仓交易 closedVolume = min(exitTrade.volume, entryTrade.volume) result = TradingResult(entryTrade.price, entryTrade.dt, exitTrade.price, exitTrade.dt, closedVolume, self.rate, self.slippage, self.size) resultList.append(result) posList.extend([1, 0]) tradeTimeList.extend([result.entryDt, result.exitDt]) # 计算未清算部分 entryTrade.volume -= closedVolume exitTrade.volume -= closedVolume # 如果开仓交易已经全部清算,则从列表中移除 if not entryTrade.volume: longTrade.pop(0) # 如果平仓交易已经全部清算,则退出循环 if not exitTrade.volume: break # 如果平仓交易未全部清算, if exitTrade.volume: # 且开仓交易已经全部清算完,则平仓交易剩余的部分 # 等于新的反向开仓交易,添加到队列中 if not longTrade: shortTrade.append(exitTrade) break # 如果开仓交易还有剩余,则进入下一轮循环 else: pass # 检查是否有交易 if not resultList: self.output(u'无交易结果') return {} # 然后基于每笔交易的结果,我们可以计算具体的盈亏曲线和最大回撤等 capital = 0 # 资金 maxCapital = 0 # 资金最高净值 drawdown = 0 # 回撤 totalResult = 0 # 总成交数量 totalTurnover = 0 # 总成交金额(合约面值) totalCommission = 0 # 总手续费 totalSlippage = 0 # 总滑点 timeList = [] # 时间序列 pnlList = [] # 每笔盈亏序列 capitalList = [] # 盈亏汇总的时间序列 drawdownList = [] # 回撤的时间序列 winningResult = 0 # 盈利次数 losingResult = 0 # 亏损次数 totalWinning = 0 # 总盈利金额 totalLosing = 0 # 总亏损金额 for result in resultList: capital += result.pnl maxCapital = max(capital, maxCapital) drawdown = capital - maxCapital pnlList.append(result.pnl) timeList.append(result.exitDt) # 交易的时间戳使用平仓时间 capitalList.append(capital) drawdownList.append(drawdown) totalResult += 1 totalTurnover += result.turnover totalCommission += result.commission totalSlippage += result.slippage if result.pnl >= 0: winningResult += 1 totalWinning += result.pnl else: losingResult += 1 totalLosing += result.pnl # 计算盈亏相关数据 winningRate = winningResult / totalResult * 100 # 胜率 averageWinning = 0 # 这里把数据都初始化为0 averageLosing = 0 profitLossRatio = 0 if winningResult: averageWinning = totalWinning / winningResult # 平均每笔盈利 if losingResult: averageLosing = totalLosing / losingResult # 平均每笔亏损 if averageLosing: profitLossRatio = -averageWinning / averageLosing # 盈亏比 # 返回回测结果 d = {} d['capital'] = capital d['maxCapital'] = maxCapital d['drawdown'] = drawdown d['totalResult'] = totalResult d['totalTurnover'] = totalTurnover d['totalCommission'] = totalCommission d['totalSlippage'] = totalSlippage d['timeList'] = timeList d['pnlList'] = pnlList d['capitalList'] = capitalList d['drawdownList'] = drawdownList d['winningRate'] = winningRate d['averageWinning'] = averageWinning d['averageLosing'] = averageLosing d['profitLossRatio'] = profitLossRatio d['posList'] = posList d['tradeTimeList'] = tradeTimeList return d #---------------------------------------------------------------------- def showBacktestingResult(self): """显示回测结果""" d = self.calculateBacktestingResult() # 输出 self.output('-' * 30) self.output(u'第一笔交易:\t%s' % d['timeList'][0]) self.output(u'最后一笔交易:\t%s' % d['timeList'][-1]) self.output(u'总交易次数:\t%s' % formatNumber(d['totalResult'])) self.output(u'总盈亏:\t%s' % formatNumber(d['capital'])) self.output(u'最大回撤: \t%s' % formatNumber(min(d['drawdownList']))) self.output(u'平均每笔盈利:\t%s' % formatNumber(d['capital'] / d['totalResult'])) self.output(u'平均每笔滑点:\t%s' % formatNumber(d['totalSlippage'] / d['totalResult'])) self.output(u'平均每笔佣金:\t%s' % formatNumber(d['totalCommission'] / d['totalResult'])) self.output(u'胜率\t\t%s%%' % formatNumber(d['winningRate'])) self.output(u'盈利交易平均值\t%s' % formatNumber(d['averageWinning'])) self.output(u'亏损交易平均值\t%s' % formatNumber(d['averageLosing'])) self.output(u'盈亏比:\t%s' % formatNumber(d['profitLossRatio'])) # 绘图 import matplotlib.pyplot as plt import numpy as np try: import seaborn as sns # 如果安装了seaborn则设置为白色风格 sns.set_style('whitegrid') except ImportError: pass pCapital = plt.subplot(4, 1, 1) pCapital.set_ylabel("capital") pCapital.plot(d['capitalList'], color='r', lw=0.8) pDD = plt.subplot(4, 1, 2) pDD.set_ylabel("DD") pDD.bar(range(len(d['drawdownList'])), d['drawdownList'], color='g') pPnl = plt.subplot(4, 1, 3) pPnl.set_ylabel("pnl") pPnl.hist(d['pnlList'], bins=50, color='c') pPos = plt.subplot(4, 1, 4) pPos.set_ylabel("Position") if d['posList'][-1] == 0: del d['posList'][-1] tradeTimeIndex = [ item.strftime("%m/%d %H:%M:%S") for item in d['tradeTimeList'] ] xindex = np.arange(0, len(tradeTimeIndex), np.int(len(tradeTimeIndex) / 10)) tradeTimeIndex = map(lambda i: tradeTimeIndex[i], xindex) pPos.plot(d['posList'], color='k', drawstyle='steps-pre') pPos.set_ylim(-1.2, 1.2) plt.sca(pPos) plt.tight_layout() plt.xticks(xindex, tradeTimeIndex, rotation=30) # 旋转15 plt.show() #---------------------------------------------------------------------- def putStrategyEvent(self, name): """发送策略更新事件,回测中忽略""" pass #---------------------------------------------------------------------- def setSlippage(self, slippage): """设置滑点点数""" self.slippage = slippage #---------------------------------------------------------------------- def setSize(self, size): """设置合约大小""" self.size = size #---------------------------------------------------------------------- def setRate(self, rate): """设置佣金比例""" self.rate = rate #---------------------------------------------------------------------- def setPriceTick(self, priceTick): """设置价格最小变动""" self.priceTick = priceTick #---------------------------------------------------------------------- def runOptimization(self, strategyClass, optimizationSetting): """优化参数""" # 获取优化设置 settingList = optimizationSetting.generateSetting() targetName = optimizationSetting.optimizeTarget # 检查参数设置问题 if not settingList or not targetName: self.output(u'优化设置有问题,请检查') # 遍历优化 resultList = [] for setting in settingList: self.clearBacktestingResult() self.output('-' * 30) self.output('setting: %s' % str(setting)) self.initStrategy(strategyClass, setting) self.runBacktesting() d = self.calculateBacktestingResult() try: targetValue = d[targetName] except KeyError: targetValue = 0 resultList.append(([str(setting)], targetValue)) # 显示结果 resultList.sort(reverse=True, key=lambda result: result[1]) self.output('-' * 30) self.output(u'优化结果:') for result in resultList: self.output(u'%s: %s' % (result[0], result[1])) return result #---------------------------------------------------------------------- def clearBacktestingResult(self): """清空之前回测的结果""" # 清空限价单相关 self.limitOrderCount = 0 self.limitOrderDict.clear() self.workingLimitOrderDict.clear() # 清空停止单相关 self.stopOrderCount = 0 self.stopOrderDict.clear() self.workingStopOrderDict.clear() # 清空成交相关 self.tradeCount = 0 self.tradeDict.clear() #---------------------------------------------------------------------- def runParallelOptimization(self, strategyClass, optimizationSetting): """并行优化参数""" # 获取优化设置 settingList = optimizationSetting.generateSetting() targetName = optimizationSetting.optimizeTarget # 检查参数设置问题 if not settingList or not targetName: self.output(u'优化设置有问题,请检查') # 多进程优化,启动一个对应CPU核心数量的进程池 pool = multiprocessing.Pool(multiprocessing.cpu_count()) l = [] for setting in settingList: l.append( pool.apply_async(optimize, (strategyClass, setting, targetName, self.mode, self.startDate, self.initDays, self.endDate, self.slippage, self.rate, self.size, self.dbName, self.symbol))) pool.close() pool.join() # 显示结果 resultList = [res.get() for res in l] resultList.sort(reverse=True, key=lambda result: result[1]) self.output('-' * 30) self.output(u'优化结果:') for result in resultList: self.output(u'%s: %s' % (result[0], result[1])) #---------------------------------------------------------------------- def roundToPriceTick(self, price): """取整价格到合约最小价格变动""" if not self.priceTick: return price newPrice = round(price / self.priceTick, 0) * self.priceTick return newPrice
class DownloadManager(ThreadManager): """""" def __init__(self): """""" ThreadManager.__init__(self) self.active_downloads = {} self.queue_downloads = OrderedDict() self.complete_downloads = {} self.stopped_downloads = {} self.global_slots = Slots() def reorder_queue(self, id_order_list): """""" ordered_items_dict = OrderedDict() for id_item in id_order_list: try: ordered_items_dict[id_item] = self.queue_downloads[id_item] except KeyError: pass if len(self.queue_downloads) == len(ordered_items_dict): self.queue_downloads.clear() self.queue_downloads.update(ordered_items_dict) else: logger.warning("reorder_queue failed") def start_all(self, id_order_list): """""" self.queue_downloads.update(self.stopped_downloads) self.stopped_downloads.clear() self.reorder_queue(id_order_list) self.next_download() def stop_all(self): """""" for id_item, download_item in self.active_downloads.iteritems(): self.stop_thread(id_item) for id_item, download_item in self.queue_downloads.items(): self.stopped_downloads[id_item] = download_item del self.queue_downloads[id_item] def start_download(self, id_item): """""" try: download_item = self.stopped_downloads.pop(id_item) except KeyError: return False else: self.queue_downloads[id_item] = download_item self.next_download() return True def stop_download(self, id_item): """""" try: download_item = self.active_downloads[id_item] except KeyError: try: download_item = self.queue_downloads.pop(id_item) self.stopped_downloads[id_item] = download_item except KeyError: return False else: return True else: # active self.stop_thread(id_item) return True def delete_download(self, id_items_list, remove_file=False): """""" for id_item in id_items_list: th = None try: download_item = self.active_downloads.pop(id_item) except KeyError: try: download_item = self.stopped_downloads.pop(id_item) except KeyError: try: download_item = self.queue_downloads.pop(id_item) except KeyError: try: download_item = self.complete_downloads.pop( id_item) except KeyError: raise else: # active th = self.get_thread(id_item) self.stop_thread(id_item) self.delete_thread(id_item) self.global_slots.remove_slot() self.next_download() if remove_file: threading.Thread(group=None, target=self.remove_file, name=None, args=(download_item, th)).start() def remove_file(self, download_item, th): """""" if th is not None: th.join() try: os.remove(os.path.join(download_item.path, download_item.name)) except Exception as err: logger.warning(err) def update_active_downloads(self): """ This may change the active_downloads dict, you should get a dict copy before calling this method. """ for id_item, download_item in self.active_downloads.items(): th = self.thread_downloads[download_item.id] self.update_download_item(th, download_item) old_status = download_item.status if old_status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED, cons.STATUS_ERROR): if old_status == cons.STATUS_STOPPED: self.stopped_downloads[id_item] = download_item elif old_status == cons.STATUS_FINISHED: self.complete_downloads[id_item] = download_item else: #status == cons.STATUS_ERROR download_item.fail_count += 1 if download_item.fail_count > conf.get_retries_limit(): download_item.status = cons.STATUS_STOPPED self.stopped_downloads[id_item] = download_item else: download_item.status = cons.STATUS_QUEUE self.queue_downloads[id_item] = download_item del self.thread_downloads[id_item] del self.active_downloads[id_item] self.global_slots.remove_slot() self.next_download() if old_status == cons.STATUS_FINISHED: events.download_complete.emit(download_item) if not self.active_downloads and old_status != cons.STATUS_STOPPED: events.all_downloads_complete.emit() if th.limit_exceeded and self.active_downloads and old_status == cons.STATUS_ERROR: events.limit_exceeded.emit() def update_download_item(self, th, download_item): #th = self.thread_downloads[download_item.id] if th.error_flag and th.stop_flag: #fix on stopped failed download status = cons.STATUS_STOPPED else: status = th.status #get before any other attr download_item.status = status download_item.chunks, download_item.size_complete = th.get_chunk_n_size( ) download_item.name = th.file_name download_item.status_msg = th.status_msg download_item.size = th.size_file download_item.start_time = th.start_time download_item.size_resume = th.size_tmp download_item.can_resume = th.can_resume download_item.is_premium = th.is_premium download_item.video_quality = th.video_quality download_item.calc_stats() def add_to_downloader(self, download_item): self.queue_downloads[download_item.id] = download_item def next_download(self): for download_item in self.queue_downloads.values(): self.download_starter(download_item) if not self.global_slots.available_slot(): break def download_starter(self, download_item): """""" if not self.global_slots.available_slot(): return elif not self.is_host_slot_available( download_item.host ): # and host_accounts.get_account(download_item.host) is None: return else: download_item.fail_count = 0 self.global_slots.add_slot() self.create_thread(download_item) #threadmanager self.active_downloads[download_item.id] = download_item del self.queue_downloads[download_item.id] def is_host_slot_available(self, host): """""" count = 0 host_slots = plugins_config.get_plugin_item(host).get_slots_limit() if host_slots > 0: # -1 or 0 means unlimited slots for download_item in self.active_downloads.itervalues(): if host == download_item.host: count += 1 if count >= host_slots: return False return True def new_slot_limit(self, new_limit): """""" current_limit = self.global_slots.get_limit() self.global_slots.set_limit(new_limit) if new_limit > current_limit: self.next_download()