Ejemplo n.º 1
0
def find_layers(pkts, pcap):
  packet = OrderedDict()
  count = 1
  try:
    for p in pkts:
      header = {"Buffer": {"timestamp": datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f'), "packetnumber": count, "pcapfile": pcap}}
      packet.update(header)  
      counter = 0
      while True:
        layer = p.getlayer(counter)
        if (layer != None):
          i = int(counter)
          x = p[0][i].fields
          t = exclude_layers(x, layer.name)
          s = rename_layer(t, layer.name)
          v = '{"' + layer.name.replace('.','_') + '[' + str(i) + ']' + '":' + str(s) + '}'
          s = eval(v)
          packet.update(s)
        else:
          break
        counter += 1
      count += 1
      yield packet
      packet.clear()
  except Exception as e:
    print RED + 'Error within packet: ' + str(count) + ', on layer: ' + layer.name + ' . The full error was: ' + str(e) + END
Ejemplo n.º 2
0
 def test_clear(self):
     pairs = [("c", 1), ("b", 2), ("a", 3), ("d", 4), ("e", 5), ("f", 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     self.assertEqual(len(od), len(pairs))
     od.clear()
     self.assertEqual(len(od), 0)
Ejemplo n.º 3
0
class OrderedDict(object):
    def __init__(self, eq_func=None, hash_func=None):
        self.contents = PyOrderedDict()
        self.eq_func = eq_func or operator.eq
        self.hash_func = hash_func or hash

    def __getitem__(self, key):
        return self.contents[self._key(key)]

    def __setitem__(self, key, value):
        self.contents[self._key(key)] = value

    def __delitem__(self, key):
        del self.contents[self._key(key)]

    def __contains__(self, key):
        return self._key(key) in self.contents

    def __len__(self):
        return len(self.contents)

    def _key(self, key):
        return DictKey(self, key)

    def keys(self):
        return [k.key for k in self.contents.keys()]

    def values(self):
        return self.contents.values()

    def iteritems(self):
        for k, v in self.contents.iteritems():
            yield k.key, v

    def get(self, key, default):
        return self.contents.get(self._key(key), default)

    def pop(self, key, default=MARKER):
        if default is MARKER:
            return self.contents.pop(self._key(key))
        else:
            return self.contents.pop(self._key(key), default)

    def popitem(self):
        if not self:
            raise KeyError
        k, v = self.contents.popitem()
        return k.key, v

    def update(self, d):
        self.contents.update(d.contents)

    def clear(self):
        self.contents.clear()

    def copy(self):
        d = OrderedDict(self.eq_func, self.hash_func)
        for k, v in self.iteritems():
            d[k] = v
        return d
Ejemplo n.º 4
0
def find_layers(pkts, pcap, pcap_id, streamid):
    packet = OrderedDict()
    count = 1
    pcap_id = pcap_id.encode('utf-8')
    streamid = streamid.encode('utf-8')
    try:
        for p in pkts:
            header = {"Buffer": {"timestamp": datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f'),
                                 "packetnumber": count, "PCAP ID": pcap_id, "pcapfile": pcap, "StreamID": streamid}}
            packet.update(header)
            counter = 0
            while True:
                layer = p.getlayer(counter)
                if layer != None:
                    i = int(counter)
                    x = p[0][i].fields
                    t = exclude_layers(x, layer.name)
                    s = rename_layer(t, layer.name)
                    v = '{"' + layer.name.replace('.', '_') + '[' + str(i) + ']' + '":' + str(s) + '}'
                    s = eval(v)
                    try:
                        del s['HTTP[3]']
                        del s['HTTP[5]']
                    except KeyError:
                        pass
                    packet.update(s)
                else:
                    break
                counter += 1
            count += 1
            yield packet
            packet.clear()
    except Exception as e:
        error_logging(str(e), 'PacketParser')
        pass
Ejemplo n.º 5
0
class Tweak(object):  # {{{

    def __init__(self, name, doc, var_names, defaults, custom):
        translate = _
        self.name = translate(name)
        self.doc = doc.strip()
        self.doc = ' ' + self.doc
        self.var_names = var_names
        if self.var_names:
            self.doc = u"%s: %s\n\n%s"%(_('ID'), self.var_names[0], format_doc(self.doc))
        self.default_values = OrderedDict()
        for x in var_names:
            self.default_values[x] = defaults[x]
        self.custom_values = OrderedDict()
        for x in var_names:
            if x in custom:
                self.custom_values[x] = custom[x]

    def __str__(self):
        ans = ['#: ' + self.name]
        for line in self.doc.splitlines():
            if line:
                ans.append('# ' + line)
        for key, val in iteritems(self.default_values):
            val = self.custom_values.get(key, val)
            ans.append(u'%s = %r'%(key, val))
        ans = '\n'.join(ans)
        if isinstance(ans, unicode_type):
            ans = ans.encode('utf-8')
        return ans

    @property
    def sort_key(self):
        return 0 if self.is_customized else 1

    @property
    def is_customized(self):
        for x, val in iteritems(self.default_values):
            cval = self.custom_values.get(x, val)
            if normalize_tweak(cval) != normalize_tweak(val):
                return True
        return False

    @property
    def edit_text(self):
        from pprint import pformat
        ans = ['# %s'%self.name]
        for x, val in iteritems(self.default_values):
            val = self.custom_values.get(x, val)
            if isinstance(val, (list, tuple, dict, set, frozenset)):
                ans.append(u'%s = %s' % (x, pformat(val)))
            else:
                ans.append(u'%s = %r'%(x, val))
        return '\n\n'.join(ans)

    def restore_to_default(self):
        self.custom_values.clear()

    def update(self, varmap):
        self.custom_values.update(varmap)
Ejemplo n.º 6
0
def get_available_skins(selected=None):
    """Returns a dictionary of skin name --> directory where
    "templates" and "media" subdirectories can be found.

    selected is a name of preferred skin
    if it's None, then information about all skins will be returned
    otherwise, only data about selected and default skins
    will be returned

    selected skin is guaranteed to be the first item in the dictionary
    """
    skins = OrderedDict()
    extra_skins_dir = getattr(django_settings, 'ASKBOT_EXTRA_SKINS_DIR', None)
    if extra_skins_dir:
        skins.update(get_skins_from_dir(extra_skins_dir))

    if 'default' in skins:
        raise ValueError('"default" is not an acceptable name for a custom skin')

    if selected in skins:
        selected_dir = skins[selected]
        skins.clear()
        skins[selected] = selected_dir
    elif selected == 'default':
        skins = OrderedDict()
    elif selected:
        raise ValueError(
            'skin ' + str(selected) + \
            ' not found, please check ASKBOT_EXTRA_SKINS_DIR setting ' + \
            'or in the corresponding directory'
        )

    #insert default as a last item
    skins['default'] = askbot.get_install_directory()
    return skins
Ejemplo n.º 7
0
def main():
    base_path = os.path.dirname(__file__)
    xml_file = os.path.join(base_path, "Survey-4.xml")
    csv_file = os.path.join(base_path, "Survey-4.csv")

    csv_headers_map = {"PIVA": "CUSTOMER_ID", "Email_Aziendale_Cliente": "ADDRESS", "Codice_Ticket": "SURVEY_ID"}

    csv_headers_required = ["CUSTOMER_ID", "ADDRESS", "SURVEY_ID", "LANGUAGE"]
    csv_headers_meta = None

    try:
        xml_tree = ET.parse(xml_file)
    except ET.ParseError as e:
        print(str.format("[#] parser error: {}", e))
        sys.exit(1)

    csv_headers = OrderedDict()
    for item in csv_headers_required:
        csv_headers[item] = None

    print(str.format("[*] csv_headers: {}", csv_headers))
    print(str.format("[*] csv_headers.key(): {}", csv_headers.keys()))
    print(str.format("[*] csv_headers.values(): {}", csv_headers.values()))
    print(str.format("[*] len(csv_headers): {}", len(csv_headers)))
    csv_headers.clear()
    print(str.format("[*] len(csv_headers): {}", len(csv_headers)))
Ejemplo n.º 8
0
def packet_summary(pkts, pcap):
  packet = OrderedDict()
  count = 1
  try:
    for p in pkts:
      p_header = {"Buffer": {"timestamp": datetime.datetime.fromtimestamp(p.time).strftime('%Y-%m-%d %H:%M:%S.%f'), "packetnumber": count, "pcapfile": pcap, "packet_length": p.len}}
      packet.update(p_header)
      if p.haslayer(Ether):
        p_ether = {"Ethernet": {"ether_src": p[Ether].src, "ether_dst": p[Ether].dst}}
        packet.update(p_ether)
      if p.haslayer(IP):
        p_ip = {"IP": {"ip_src": p[IP].src, "ip_dst": p[IP].dst, "ip_ttl": p[IP].ttl}}
        packet.update(p_ip)
      if p.haslayer(TCP):
        p_tcp = {"TCP": {"tcp_sport": p[TCP].sport, "tcp_dport": p[TCP].dport, "tcp_flags": p[TCP].flags}}
        packet.update(p_tcp)
      if p.haslayer(UDP):
        p_udp = {"UDP": {"udp_sport": p[UDP].sport, "udp_dport": p[UDP].dport, "udp_len": p[UDP].len}}
        packet.update(p_udp)
      count += 1
      yield packet
      packet.clear()
  except Exception, e:
    print e
    pass
Ejemplo n.º 9
0
class SubModel:
    def __init__(self, params=None, save_path=(), shared=False, copy_shared=False):
        self.params = OrderedDict() if params is None else params  # string (param identifier) -> parameter
        self.save_path = save_path
        self.shared = shared
        self.copy_shared = copy_shared

    def save_sub_model(self, d, *args):
        self.get_sub_dict(d).update(args + (("param_keys", list(self.params.keys())),))
        return list(self.params.values())

    def load_sub_model(self, d, *args, load_path=None):
        d = self.get_sub_dict(d, load_path=load_path)
        param_keys = d.get("param_keys", ())
        assert len(param_keys) <= len(args), "%s loaded values: expected %d, got %d" % ("/".join(self.save_path),
                                                                                        len(param_keys), len(args))
        self.params.clear()
        self.params.update(zip(param_keys, args))
        return d

    def get_sub_dict(self, d, load_path=None):
        for element in load_path or self.save_path:
            d = d.setdefault(element, OrderedDict())
        return d

    def params_str(self):
        return "/".join(self.save_path) + (": " if self.save_path else "") + ", ".join(self.params.keys())

    def invalidate_caches(self):
        for model in self.sub_models():
            model.invalidate_caches()

    def sub_models(self):
        return ()
Ejemplo n.º 10
0
class AliasGenerator(object):
    def __init__(self, size=200):
        self.MAX_CACHE = size  # TODO: Make configurable
        self.cache = OrderedDict()

    def generate_alias(self, str_value):
        """
        Generate an alias for the given string.
        :param str_value: the given string
        :return: a 24 character alias string
        """
        alias = self.get_alias(str_value)
        if not alias:
            h = hashlib.md5()
            h.update(str_value.encode('utf-8'))
            alias = h.hexdigest()
            self.cache[str_value] = alias
            if len(self.cache) > self.MAX_CACHE:
                self.cache.popitem(last=False)  # FIFO
        else:
            alias = alias
        return alias

    def needs_alias(self, str_value):
        return len(str_value) > MAX_NAME_LEN or len(RE_CLEANSE.findall(str_value)) > 0

    def get_alias(self, str_value):
        return self.cache.get(str_value)

    def clear_aliases(self):
        self.cache.clear()
 def test_clear(self):
     pairs = [('c', 1), ('b', 2), ('a', 3), ('d', 4), ('e', 5), ('f', 6)]
     shuffle(pairs)
     od = OrderedDict(pairs)
     self.assertEqual(len(od), len(pairs))
     od.clear()
     self.assertEqual(len(od), 0)
Ejemplo n.º 12
0
class Records(object):
    def __init__(self):
        self._data      = OrderedDict()
        self._stamps    = {}
        
    def __iter__(self):
        return self._data.iteritems()
    
    def __contains__(self, key):
        return key in self._data
    
    def keys(self):
        return self._data.keys()
        
    def update(self, data):
        self._data.update(data)
        self._stamps.update(dict.fromkeys(data.keys(), time()))
        
    def pluck(self, t):
        return OrderedDict(filter(lambda item:self._stamps[item[0]]>=t, self._data.items()))
    
    def clear(self):
        self._data.clear()
        self._stamps.clear()
        
    def get(self, key):
        return self._data[key]
Ejemplo n.º 13
0
class LRUCache(Cache):
    """
    Implements a simple Least Recently Used Cache

    Very simple using collections.OrderedDict. The size can be changed during
    run-time
    """

    def __init__(self, size_limit):
        super(LRUCache, self).__init__()
        self._size_limit = size_limit
        self._cache = OrderedDict()

    @property
    def count(self):
        return len(self._cache), 0

    @property
    def size(self):
        return self.size_limit, 0

    @property
    def size_limit(self):
        return self._size_limit

    @size_limit.setter
    def size_limit(self, new_size):
        if new_size < self.size_limit:
            self._check_size_limit()

        self._size_limit = new_size

    def __iter__(self):
        return iter(self._cache)

    def __reversed__(self):
        return reversed(self._cache)

    def __getitem__(self, item):
        obj = self._cache.pop(item)
        self._cache[item] = obj
        return obj

    def __setitem__(self, key, value, **kwargs):
        self._cache[key] = value
        self._check_size_limit()

    def _check_size_limit(self):
        while len(self._cache) > self.size_limit:
            self._cache.popitem(last=False)

    def __contains__(self, item):
        return item in self._cache

    def clear(self):
        self._cache.clear()

    def __len__(self):
        return len(self._cache)
Ejemplo n.º 14
0
class LRUCacheDict(object):
    """ A dictionary-like object, supporting LRU caching semantics.
    """
    def __init__(self, max_size=1024, expiration=15*60):
        self.max_size = max_size
        self.expiration = expiration

        self.__values = {}
        self.__expire_times = OrderedDict()
        self.__access_times = OrderedDict()

    def size(self):
        return len(self.__values)

    def clear(self):
        self.__values.clear()
        self.__expire_times.clear()
        self.__access_times.clear()

    def has_key(self, key):
        return self.__values.has_key(key)

    def __setitem__(self, key, value):
        t = int(time())
        self.__delete__(key)
        self.__values[key] = value
        self.__access_times[key] = t
        self.__expire_times[key] = t + self.expiration
        self.cleanup()

    def __getitem__(self, key):
        t = int(time())
        del self.__access_times[key]
        self.__access_times[key] = t
        self.cleanup()
        return self.__values[key]

    def __delete__(self, key):
        if self.__values.has_key(key):
            del self.__values[key]
            del self.__expire_times[key]
            del self.__access_times[key]

    def cleanup(self):
        if self.expiration is None:
            return None
        t = int(time())
        #Delete expired
        for k in self.__expire_times.iterkeys():
            if self.__expire_times[k] < t:
                self.__delete__(k)
            else:
                break

        #If we have more than self.max_size items, delete the oldest
        while (len(self.__values) > self.max_size):
            for k in self.__access_times.iterkeys():
                self.__delete__(k)
                break
Ejemplo n.º 15
0
class Error_manager():
    
    def __init__(self):
        
        self.objec_error_dict   = OrderedDict()
        self.Lines_With_Errors  = []
        
    def log_error(self, object_name, verbose = True):
        
        self.objec_error_dict[object_name] = format_exc()
        
        if verbose:
            print '\n-- Error logged'
    
    def log_emLine_error(self, CodeName, Label):
        
        #Case for the first error log for a given object 
        if Label not in self.objec_error_dict:
            LinesList   = [[],[]]
        
        #Case the objects has more lines with an error
        else:
            LinesList   = self.objec_error_dict[CodeName] 

        LinesList[0].append([Label, format_exc()])
                
        self.objec_error_dict[CodeName] = LinesList
        
    def display_errors(self, error_type = 'general', extra_verbose = False):
        
        if error_type == 'general':
            
            if len(self.objec_error_dict) != 0:
                print '\n-These objects treatment produced a script error:\n'
                print self.objec_error_dict.keys(),'\n'
                for i in range(len(self.objec_error_dict)):
                    print '--Object', self.objec_error_dict.keys()[i],
                    print self.objec_error_dict.values()
                    
                #We empty the dictionary as we don not need it anymore
                self.objec_error_dict.clear()
        
        elif error_type == 'emLine measurement':
            
            if len(self.objec_error_dict) != 0:
                print '\n-These objects treatment produced a script error:\n'
                print self.objec_error_dict.keys(),'\n'
                
                for i in range(len(self.objec_error_dict)):
                    print '--Object', self.objec_error_dict.keys()[i] 
                    print self.objec_error_dict.values()[i][0], '\n'
                    
                if extra_verbose == True:
                    for i in range(len(self.objec_error_dict)):
                        print '--Object',   self.objec_error_dict.keys()[i] 
                        print '--Label',    self.objec_error_dict.values()[i][0], '\n'
                        print '--Errors',   self.objec_error_dict.values()[i][1], '\n'  
                        
        return ' '
Ejemplo n.º 16
0
class TagBase():
    def __init__(self, out_file):
        self.items = OrderedDict()
        self.out_file = out_file

    def add_items(self, item_pairs):
        for p in item_pairs:
            self.items[p[0]] = p[1]

    # this function is called after ALL tags are added from ALL ini files
    # default behavior is to clear added tags here
    # If certain tag type required to set default value, it can override this function
    def reserve_in_default_list(self):
        self.items.clear()

    def Print(self):
        for t in self.items:
            sys.stdout.write('\t%s := %s\n' % (t, self.items[t]))
        print ''

    def commit(self):
        lines = []
        lines += self.out_header()
        for it in self.items:
            lines += self.gen_tag_format(it, self.items[it])
        lines += self.out_tailer()

        fp = open(self.out_file, 'a+')
        for l in lines:
            fp.write('%s\n' % l)
        fp.close()

    def isYesTag(self, tag):
        return tag.lower() == 'yes'

    def out_header(self):
        return []

    def out_tailer(self):
        return []

    def isStringInOutFile(self, text):
        if not self.out_file in out_text_appear_once_record:
            out_text_appear_once_record[self.out_file] = []
        in_record = text in out_text_appear_once_record[self.out_file]
        out_text_appear_once_record[self.out_file].append(text)
        return in_record

    def out_text_appear_once(self, text):
        if self.isStringInOutFile(text):
            return []
        else:
            return [text]

    def remove_text_from_out(self, text):
        if path.exists(self.out_file):
            lines = open(self.out_file).readlines()
            lines.remove(text+'\n')
            open(self.out_file, 'w').writelines(lines)
Ejemplo n.º 17
0
class Classifier(object):
    """Parent factory for all classifier classes."""

    __metaclass__ = Factory

    def __init__(self):
        super(Classifier, self).__init__()
        self._pp = None # setup preprocessor in the train method
        self.model = None
        self._clf = None
        self._classes = OrderedDict()

    def description(self):
        return

    def setParameters(self, params):
        assert isinstance(params, dict)
        self._clf.set_params(**params)

    def setupPreProcessor(self, features):
        self._pp = PreProcessor(features)

    def normalize(self, features):
        return self._pp(features)

    @property
    def classes(self):
        return self._classes

    def classByName(self, name):
        for class_ in self.classes.itervalues():
            if class_.name == name:
                return class_

        raise RuntimeError("No classes named %s" %name)

    def setClasses(self, classes, parent, panel):
        """Update the dictionary of the class definition and the list of
        the context menu actions."""

        self._classes.clear()
        self._classes.update(classes)

    @classmethod
    def classifiers(cls):
        return cls._classes.keys()

    def parameterWidget(self, parent):
        return QtWidgets.QWidget(self)

    def train(self, features):
        raise NotImplementedError

    def predict(self, features):
        return [UnClassified]*features.shape[0]

    def saveToHdf(self, file_, feature_names):
        raise NotImplementedError
Ejemplo n.º 18
0
class Reaction(object):
    """docstring for Reaction"""
    def __init__(self, arg, **kwargs):
        self.id                     = arg
        self.name                   = kwargs.get('name', self.id)
        self.reversible             = kwargs.get('reversible', None)
        self.participants           = OrderedDict()
        self.lower_bound            = -1e4
        self.upper_bound            = 1e4
        self.default_bounds         = (self.lower_bound, self.upper_bound)
        self.objective_coefficient  = 0.0
        self.flux_value             = None
        self.notes                  = {}
        self.genes                  = []
        self.lp_var                 = None

    def __str__(self):
        reactants = []
        products  = []
        for p in self.participants:
            if self.participants[p] < 0:
                reactants.append(' %03.1f %s ' % (abs(self.participants[p]), p.name))
            else:
                products.append(' %03.1f %s ' % (abs(self.participants[p]), p.name))
        if self.reversible:
            arrow = '<===>'
        else:
            arrow = '====>'
        return self.id  + ' : ' + '+'.join(reactants) + '   ' + arrow + '   ' \
                        + '+'.join(products) + '\n'

    def __repr__(self):
        return self.id

    def clear_participants(self):
        """docstring for clear_participants"""
        for metabolite in self.participants:
            metabolite.participations.pop(self.id)
        self.participants.clear()

    def remove_participant(self, metabolite):
        """docstring for remove_participant"""
        metabolite.participations.pop(self.id)
        self.participants.pop(metabolite)

    def add_participant(self, metabolite, stoichiometry):
        """docstring for add_participant"""
        self.participants[metabolite]      = stoichiometry
        metabolite.participations[self.id] = stoichiometry

    def set_default_bounds(self):
        """docstring for set_default_bounds"""
        self.default_bounds = (self.lower_bound, self.upper_bound)

    def reset_bounds(self):
        """docstring for reset_bounds"""
        self.lower_bound = self.default_bounds[0]
        self.upper_bound = self.default_bounds[1]
Ejemplo n.º 19
0
class Recording(object):
    """Recording Class is used for managing the recording video process."""

    #----------------------------------------------------------------------#
    #                      Devices Class Constructor                       #
    #----------------------------------------------------------------------#
    def __init__(self):
        """Devices Class Constructor."""
        # Creates a dictionary for managering multiple cameras.
        self.__videos = OrderedDict()
        self.__fourcc = cv2.VideoWriter_fourcc(*"WMV2")

    #----------------------------------------------------------------------#
    #                         Public Class Methods                         #
    #----------------------------------------------------------------------#
    def AddVideo(self, index, fps=30.0, size=(640, 480)):
        """Add a new video in the Python dictionary."""
        print index
        key = str(index)
        if self.__videos.has_key(key):
            return False

        self.__videos[key] = cv2.VideoWriter(index, self.__fourcc, fps, size)

        return True

    def DelVideo(self, index):
        """Remove a video from the Python dictionary."""
        key = str(index)
        if not self.__videos.has_key(key):
            return False

        self.__videos[key].release()
        del self.__videos[key]

        return True

    def Write(self, images):
        """Writes the next video frame."""
        if type(images).__module__ == np.__name__:
            images = [images]

        for key, image in zip(self.__videos, images):
            self.__videos[key].write(image)

    def Release(self):
        """Closes all videos."""
        for key in self.__videos:
            self.__videos[key].release()
        self.__videos.clear()

    #----------------------------------------------------------------------#
    #                            Class Methods                             #
    #----------------------------------------------------------------------#
    def __repr__(self):
        """Get a object representation in a string format."""
        return "Framework.RecordingVideos.Recording object."
Ejemplo n.º 20
0
class Tweak(object):  # {{{

    def __init__(self, name, doc, var_names, defaults, custom):
        translate = _
        self.name = translate(name)
        self.doc = doc.strip()
        if self.doc:
            self.doc = translate(self.doc)
        self.var_names = var_names
        if self.var_names:
            self.doc = u"%s: %s\n\n%s"%(_('ID'), self.var_names[0], self.doc)
        self.default_values = OrderedDict()
        for x in var_names:
            self.default_values[x] = defaults[x]
        self.custom_values = OrderedDict()
        for x in var_names:
            if x in custom:
                self.custom_values[x] = custom[x]

    def __str__(self):
        ans = ['#: ' + self.name]
        for line in self.doc.splitlines():
            if line:
                ans.append('# ' + line)
        for key, val in self.default_values.iteritems():
            val = self.custom_values.get(key, val)
            ans.append('%s = %r'%(key, val))
        ans = '\n'.join(ans)
        if isinstance(ans, unicode):
            ans = ans.encode('utf-8')
        return ans

    def __cmp__(self, other):
        return -1 * cmp(self.is_customized,
                            getattr(other, 'is_customized', False))

    @property
    def is_customized(self):
        for x, val in self.default_values.iteritems():
            if self.custom_values.get(x, val) != val:
                return True
        return False

    @property
    def edit_text(self):
        ans = ['# %s'%self.name]
        for x, val in self.default_values.iteritems():
            val = self.custom_values.get(x, val)
            ans.append('%s = %r'%(x, val))
        return '\n\n'.join(ans)

    def restore_to_default(self):
        self.custom_values.clear()

    def update(self, varmap):
        self.custom_values.update(varmap)
Ejemplo n.º 21
0
 def clear(self):
     """
     Remove all keys.
     See: help(OrderedDict.clear)
     >>> T = pykov.Matrix({('A','B'): .3, ('A','A'): .7, ('B','A'): 1.})
     >>> T.clear()
     >>> T
     {}
     """
     OrderedDict.clear(self)
Ejemplo n.º 22
0
def getOutlinks(pmid, preferPmc=False):
    """ use NCBI eutils to get outlinks for a pmid as a dict provider -> url  """
    logging.debug("%s: Getting outlink from pubmed" % (pmid))
    url = "http://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?dbfrom=pubmed&id=%s&retmode=llinks&cmd=llinks" % pmid
    #logging.debug("getting %s" % url)
    #try:
    #except
    #logging.info(traceback.format_exc())
    #logging.info("Exception when downloading")
    #return None
    userAgent = 'User-Agent: Mozilla ([email protected], http://text.soe.ucsc.edu)'

    #html = urllib2.urlopen(req)
    html = maxCommon.retryHttpRequest(url, userAgent=userAgent)

    if html==None:
        return None

    outlinks = OrderedDict()
    provider = False
    fullText = False
    isPmc = False

    for line in html:
        if line.find("<ObjUrl>") != -1:
            url=""
            fullText=False
            origPublisher=False
        if line.find("Attribute") != -1:
            attribute=stripTag(line)
            if attribute.lower()=="full-text online" or attribute=="full-text pdf":
                fullText=True
        if line.find("<NameAbbr>") != -1 and fullText and origPublisher:
            db = stripTag(line)
            outlinks[db]=url
        if line.find("publishers/providers")!=-1:
            origPublisher=True
        if line.find("<Provider>") != -1:
            provider=True
        if line.find("</Provider>") != -1:
            provider=False
        if line.find("<DbFrom>") != -1:
            db = stripTag(line)
        if line.find("<Url>") != -1 and not provider:
            url = line
            url = stripTag(url).replace("&amp;", "&") # XX strange!
            url = stripTag(url).replace("&lt;", "<")
            url = stripTag(url).replace("&gt;", ">")
            if "www.ncbi.nlm.nih.gov" in url and "/pmc/" in url and preferPmc:
                # override all other links
                outlinks.clear()
                outlinks["pmc"] = url

    logging.debug("%s: Found outlinks %s" % (pmid, str(outlinks)))
    return outlinks
Ejemplo n.º 23
0
class FuncCollection(object):
    '''A selection of functions
    '''
    def __init__(self, funcitems=None):
        self.funcs = OrderedDict()
        if funcitems:
            if not isinstance(funcitems, list):
                funcitems = [funcitems]
            for item in funcitems:
                self.append(item)

    def append(self, item, attr_path='nodeid'):
        # self.funcs[tosymbol(attrgetter(attr_path)(item))] = item
        self.funcs[attrgetter(attr_path)(item)] = item

    def addtests(self, test_set):
        for item in test_set._items:
            self.append(item)

    def remove(self, item):
        self.funcs.pop(item.nodeid, None)

    def removetests(self, test_set):
        for item in test_set._items:
            self.remove(item)

    def clear(self):
        self.funcs.clear()

    def keys(self):
        return self.funcs.keys()

    def values(self):
        return self.funcs.values()

    def __len__(self):
        return len(self.funcs)

    def __getitem__(self, key):
        if isinstance(key, int):
            return self.enumitems()[key][1]
        if isinstance(key, (int, slice)):
            return list(map(itemgetter(1), self.enumitems()[key]))
        return self.funcs[key]

    def __dir__(self):
        return dirinfo(self)

    def items(self):
        return self.funcs.items()

    def enumitems(self, items=None):
        if not items:
            items = self.funcs.values()
        return [(i, node) for i, node in enumerate(items)]
Ejemplo n.º 24
0
class memoize_mask(object):
    '''
    Decorator. Caches wktToMask keyed to model_id and the WKT string
    '''
    def __init__(self, func, maxsize=50):
        '''
        Args:
            func: the function to wrap
            maxsize (int): Max size of cache (in MB)
        '''

        self.hits = self.misses = 0
        self.func = func
        self.maxsize = maxsize
        self.cache = OrderedDict()

    def __call__(self, *args):

        nc, fname, wkt, varname = args
        # If we have no key, automatic cache miss
        if (not hasattr(nc, 'model_id')):
            log.debug('Cache MISS (attribute \'model_id\' not found)')
            return self.func(*args)

        # Set key to model_id and wkt polygon
        key = (nc.model_id, wkt)
        log.debug('Checking cache for key %s', key)

        with cache_lock:
            try:
                result = self.cache[key]
                self.cache.move_to_end(key) # record recent use of this key
                self.hits += 1
                log.debug('Cache HIT')
                return result
            except KeyError:
                pass

        log.debug('Cache MISS')
        result = self.func(*args)

        with cache_lock:
            self.cache[key] = result
            self.misses += 1
            while getsize(self.cache) > self.maxsize * 1024 * 1024: # convert to MB
                self.cache.popitem(0) # Purge least recently used cache entry

        return result

    def cache_clear(self):
        with cache_lock:
            self.cache.clear()
            self.hits = 0
            self.misses = 0
Ejemplo n.º 25
0
 def clear(self):
     """ 
         clear all keys and keys in key_time_map
     """
     with self.lock:
         try:
             OrderedDict.clear(self)
             self.key_time_map.clear()
             return True
         except KeyError:
             return False
Ejemplo n.º 26
0
class CaselessDict(OrderedDict):
    """Case-insensitive dict. Values can be set and retrieved using keys of any case.
    Note that when iterating, the original case is returned for each key."""
    def __init__(self, *args):
        OrderedDict.__init__(self, {}) ## requirement for the empty {} here seems to be a python bug?
        self.keyMap = OrderedDict([(k.lower(), k) for k in OrderedDict.keys(self)])
        if len(args) == 0:
            return
        elif len(args) == 1 and isinstance(args[0], dict):
            for k in args[0]:
                self[k] = args[0][k]
        else:
            raise Exception("CaselessDict may only be instantiated with a single dict.")
        
    #def keys(self):
        #return self.keyMap.values()
    
    def __setitem__(self, key, val):
        kl = key.lower()
        if kl in self.keyMap:
            OrderedDict.__setitem__(self, self.keyMap[kl], val)
        else:
            OrderedDict.__setitem__(self, key, val)
            self.keyMap[kl] = key
            
    def __getitem__(self, key):
        kl = key.lower()
        if kl not in self.keyMap:
            raise KeyError(key)
        return OrderedDict.__getitem__(self, self.keyMap[kl])
        
    def __contains__(self, key):
        return key.lower() in self.keyMap
    
    def update(self, d):
        for k, v in d.iteritems():
            self[k] = v
            
    def copy(self):
        return CaselessDict(OrderedDict.copy(self))
        
    def __delitem__(self, key):
        kl = key.lower()
        if kl not in self.keyMap:
            raise KeyError(key)
        OrderedDict.__delitem__(self, self.keyMap[kl])
        del self.keyMap[kl]
            
    def __deepcopy__(self, memo):
        raise Exception("deepcopy not implemented")

    def clear(self):
        OrderedDict.clear(self)
        self.keyMap.clear()
def item_assignment(demand, max_capa):
    '''
    Demand is the list of the requested capacities that need to be fitted in to bins. Each requested capacity is not bigger than 10
    In this simulation, there are 4 bin with different maximum capacity.
    We will see how the demand will fit these 4 bins in BFD.
    '''
    
    # Sorting the demand list
    values = sorted(demand, reverse = True)
    bins = []
    for i in range(4):
        bins.append(Bin(max_capa))
    for index, bin in enumerate(bins):
        print "Index: %s, bin: %s" %(index+1, bin)
    print "\nThe list of demands: %s \n" % values

    from collections import OrderedDict
    fittable_bins = OrderedDict()

    for item in values[:]:
        
        # Starts fitting each requested capa to bin
        print ("\n%s Starting a new fitting %s" %(('*'*30), ('*'*30)))
        print ("\nThe requested capacity is going to be checked: %s" %item)

        for index, bin in enumerate(bins):
            print "\n******** BEFORE FITTING"
            print "Bin: %s, Value:%s" %(index+1, bin)
            if bin.capacity + item <= bin.max_capa:
                # Calculate the current capa of each bin
                current_capa = bin.capacity + item
                # Store the fittable to the fittable_bins
                fittable_bins[bin] = (current_capa, index)

        if fittable_bins:
            print "\nThe fittable_bins:"
            for bin, value in fittable_bins.items():
                print "Bin: %s, Value:%s" %(value[1]+1, bin)
        else:
            print "\nThere is NO fittable_bins"

        if fittable_bins:
            max_current_capa = max(fittable_bins.values(), key=lambda x:x[0])[0]
            for bin, value in fittable_bins.items():
                if value[0] == max_current_capa:
                    bin.bin_append(item)
                    values.remove(item)
                    fittable_bins.clear()
                    print "\n******** AFTER FITTING"
                    print "Bin: %s, Value:%s" %(value[1]+1, bin)
                    print "\nThe demaining requested capacities: %s \n" %values
                    break
    return bins
Ejemplo n.º 28
0
class DeviceManager:
    """Handles creation and destruction of local device drivers and controller
    RPC clients."""
    def __init__(self, ddb, virtual_devices=dict()):
        self.ddb = ddb
        self.virtual_devices = virtual_devices
        self.active_devices = OrderedDict()

    def get_device_db(self):
        """Returns the full contents of the device database."""
        return self.ddb.get_device_db()

    def get_desc(self, name):
        desc = self.ddb.get(name)
        while isinstance(desc, str):
            # alias
            desc = self.ddb.get(desc)
        return desc

    def get(self, name):
        """Get the device driver or controller client corresponding to a
        device database entry."""
        if name in self.virtual_devices:
            return self.virtual_devices[name]
        if name in self.active_devices:
            return self.active_devices[name]
        else:
            try:
                desc = self.get_desc(name)
            except Exception as e:
                raise DeviceError("Failed to get description of device '{}'"
                                  .format(name)) from e
            try:
                dev = _create_device(desc, self)
            except Exception as e:
                raise DeviceError("Failed to create device '{}'"
                                  .format(name)) from e
            self.active_devices[name] = dev
            return dev

    def close_devices(self):
        """Closes all active devices, in the opposite order as they were
        requested."""
        for dev in reversed(list(self.active_devices.values())):
            try:
                if isinstance(dev, (Client, BestEffortClient)):
                    dev.close_rpc()
                elif hasattr(dev, "close"):
                    dev.close()
            except Exception as e:
                logger.warning("Exception %r when closing device %r", e, dev)
        self.active_devices.clear()
class DualPriorityQueueHeap():
    def __init__(self):
        self.strings = OrderedDict()
        self.next_index=-1
        
    def _get_next_index(self):
        self.next_index += 1
        return self.next_index
        
    def enqueue(self, val_a, val_b, value):
        '''This method accepts three parameters - a string, priority value A,
           and priority value B, where the priority values are real numbers 
           (see above). The string is inserted into the priority queue with the
           given priority values A and B'''
        ix = self._get_next_index()
        self.strings[ix] = ((val_a, val_b, value))

    def _dequeue(self, index):
        ''' Adds entries from enqueued strings to priority queue based on 
            provided index.
        '''
        heap = []
        for key, value in self.strings.iteritems():
            heappush(heap, (value[index], key))
        item = heappop(heap)
        return self.strings.pop(item[1])[2]        
        
    def dequeue_a(self):
        '''This method removes and returns the string from the priority queue 
           with the highest priority A value. If two entries have the same A 
           priority, return whichever was enqueued first.'''
        return self._dequeue(0)
        
    def dequeue_b(self):
        '''This method removes and returns the string from the priority queue 
           with the highest priority B value. If two entries have the same B 
           priority, return whichever was enqueued first.'''
        return self._dequeue(1)
        
    def count(self):
        '''This method returns the number of strings in the queue.'''
        return len(self.strings)
        
    def clear(self):
        '''This removes all entries from the priority queue.'''
        self.strings.clear()
        
    def dequeue_first(self):
        '''This removes and returns the string from the priority queue that 
           was enqueued first, ignoring priority.'''
        return self.strings.popitem(last=False)[1][2]
Ejemplo n.º 30
0
class LRUCache:
    # @param capacity, an integer
    def __init__(self, capacity=None):
        self.capacity = capacity
        self.__cache = OrderedDict()

    @property
    def lru(self):
        return list(self.__cache.keys())

    @property
    def length(self):
        return len(self.__cache)

    def clear(self):
        self.__cache.clear()

    def __len__(self):
        return self.length

    def __contains__(self, item):
        return item in self.__cache

    def __setitem__(self, key, value):
        self.set(key, value)

    def __delitem__(self, key):
        del self.__cache[key]

    def __getitem__(self, key):
        return self.get(key)

    def get(self, key, default=None):
        value = self.__cache.get(key)
        if value:
            del self.__cache[key]
            self.__cache[key] = value
            return value
        return default

    def set(self, key, value):
        if self.__cache.get(key):
            del self.__cache[key]
            self.__cache[key] = value
        else:
            self.__cache[key] = value
            # Check, if the cache is full and we have to remove old items
            # If the queue is of unlimited size, self.capacity is NaN and
            # x > NaN is always False in Python and the cache won't be cleared.
            if self.capacity is not None and self.length > self.capacity:
                self.__cache.popitem(last=False)
Ejemplo n.º 31
0
class ErrorHandler:
    """
    Global configurable application exception handler. For "basic" errors (I/O
    errors, keyboard interrupt, etc.) just the error message is printed as
    there's generally no need to confuse the user with a complete stack trace
    when it's just a missing file. Other exceptions, however, are logged with
    the usual full stack trace.

    The configuration can be augmented with other exception classes that should
    be handled specially by treating the instance as a dictionary mapping
    exception classes to :class:`ErrorAction` tuples (or any 2-tuple, which
    will be converted to an :class:`ErrorAction`).

    For example::

        >>> from pibootctl.term import ErrorAction, ErrorHandler
        >>> import sys
        >>> sys.excepthook = ErrorHandler()
        >>> sys.excepthook[KeyboardInterrupt]
        (None, 1)
        >>> sys.excepthook[SystemExit]
        (None, <function ErrorHandler.exc_value at 0x7f6178915e18>)
        >>> sys.excepthook[ValueError] = (sys.excepthook.exc_message, 3)
        >>> sys.excepthook[Exception] = ("An error occurred", 1)
        >>> raise ValueError("foo is not an integer")
        foo is not an integer

    Note the lack of a traceback in the output; if the example were a script
    it would also have exited with return code 3.
    """
    def __init__(self):
        self._config = OrderedDict([
            # Exception type,        (handler method, exit code)
            (SystemExit,             (None, self.exc_value)),
            (KeyboardInterrupt,      (None, 2)),
            (argparse.ArgumentError, (self.syntax_error, 2)),
        ])

    @staticmethod
    def exc_message(exc_type, exc_value, exc_tb):
        """
        Extracts the message associated with the exception (by calling
        :class:`str` on the exception instance). The result is returned as a
        one-element list containing the message.
        """
        return [str(exc_value)]

    @staticmethod
    def exc_value(exc_type, exc_value, exc_tb):
        """
        Returns the first argument of the exception instance. In the case of
        :exc:`SystemExit` this is the expected return code of the script.
        """
        return exc_value.args[0]

    @staticmethod
    def syntax_error(exc_type, exc_value, exc_tb):
        """
        Returns the message associated with the exception, and an additional
        line suggested the user try the ``--help`` option. This should be used
        in response to exceptions indicating the user made an error in their
        command line.
        """
        return ErrorHandler.exc_message(exc_type, exc_value, exc_tb) + [
            _('Try the --help option for more information.'),
        ]

    def clear(self):
        """
        Remove all pre-defined error handlers.
        """
        self._config.clear()

    def __len__(self):
        return len(self._config)

    def __contains__(self, key):
        return key in self._config

    def __getitem__(self, key):
        return self._config[key]

    def __setitem__(self, key, value):
        self._config[key] = ErrorAction(*value)

    def __delitem__(self, key):
        del self._config[key]

    def __call__(self, exc_type, exc_value, exc_tb):
        for exc_class, (message, value) in self._config.items():
            if issubclass(exc_type, exc_class):
                if callable(message):
                    message = message(exc_type, exc_value, exc_tb)
                if callable(value):
                    value = value(exc_type, exc_value, exc_tb)
                if message is not None:
                    for line in message:
                        print(line, file=sys.stderr)
                    sys.stderr.flush()
                raise SystemExit(value)
        # Otherwise, log the stack trace and the exception into the log
        # file for debugging purposes
        for line in traceback.format_exception(exc_type, exc_value, exc_tb):
            for msg in line.rstrip().split('\n'):
                print(msg, file=sys.stderr)
        sys.stderr.flush()
        raise SystemExit(1)
Ejemplo n.º 32
0
class RecordWriter(object):
    def __init__(self, ofile, maxresultrows=None):
        self._maxresultrows = 50000 if maxresultrows is None else maxresultrows

        self._ofile = ofile
        self._fieldnames = None
        self._buffer = StringIO()

        self._writer = csv.writer(self._buffer, dialect=CsvDialect)
        self._writerow = self._writer.writerow
        self._finished = False
        self._flushed = False

        self._inspector = OrderedDict()
        self._chunk_count = 0
        self._record_count = 0
        self._total_record_count = 0

    @property
    def is_flushed(self):
        return self._flushed

    @is_flushed.setter
    def is_flushed(self, value):
        self._flushed = True if value else False

    @property
    def ofile(self):
        return self._ofile

    @ofile.setter
    def ofile(self, value):
        self._ofile = value

    def flush(self, finished=None, partial=None):
        assert finished is None or isinstance(finished, bool)
        assert partial is None or isinstance(partial, bool)
        assert not (finished is None and partial is None)
        assert finished is None or partial is None
        self._ensure_validity()

    def write_message(self, message_type, message_text, *args, **kwargs):
        self._ensure_validity()
        self._inspector.setdefault('messages', []).append(
            (message_type, message_text.format(*args, **kwargs)))

    def write_record(self, record):
        self._ensure_validity()
        self._write_record(record)

    def write_records(self, records):
        self._ensure_validity()
        write_record = self._write_record
        for record in records:
            write_record(record)

    def _clear(self):
        self._buffer.seek(0)
        self._buffer.truncate()
        self._inspector.clear()
        self._record_count = 0
        self._flushed = False

    def _ensure_validity(self):
        if self._finished is True:
            assert self._record_count == 0 and len(self._inspector) == 0
            raise RuntimeError('I/O operation on closed record writer')

    def _write_record(self, record):

        fieldnames = self._fieldnames

        if fieldnames is None:
            self._fieldnames = fieldnames = list(record.keys())
            value_list = imap(lambda fn: (str(fn), str('__mv_') + str(fn)),
                              fieldnames)
            self._writerow(list(chain.from_iterable(value_list)))

        get_value = record.get
        values = []

        for fieldname in fieldnames:
            value = get_value(fieldname, None)

            if value is None:
                values += (None, None)
                continue

            value_t = type(value)

            if issubclass(value_t, (list, tuple)):

                if len(value) == 0:
                    values += (None, None)
                    continue

                if len(value) > 1:
                    value_list = value
                    sv = ''
                    mv = '$'

                    for value in value_list:

                        if value is None:
                            sv += '\n'
                            mv += '$;$'
                            continue

                        value_t = type(value)

                        if value_t is not bytes:

                            if value_t is bool:
                                value = str(value.real)
                            elif value_t is six.text_type:
                                value = value
                            elif value_t is int or value_t is int or value_t is float or value_t is complex:
                                value = str(value)
                            elif issubclass(value_t, (dict, list, tuple)):
                                value = str(''.join(
                                    RecordWriter._iterencode_json(value, 0)))
                            else:
                                value = repr(value).encode(
                                    'utf-8', errors='backslashreplace')

                        sv += value + '\n'
                        mv += value.replace('$', '$$') + '$;$'

                    values += (sv[:-1], mv[:-2])
                    continue

                value = value[0]
                value_t = type(value)

            if value_t is bool:
                values += (str(value.real), None)
                continue

            if value_t is bytes:
                values += (value, None)
                continue

            if value_t is six.text_type:
                if six.PY2:
                    value = value.encode('utf-8')
                values += (value, None)
                continue

            if value_t is int or value_t is int or value_t is float or value_t is complex:
                values += (str(value), None)
                continue

            if issubclass(value_t, dict):
                values += (str(''.join(RecordWriter._iterencode_json(value,
                                                                     0))),
                           None)
                continue

            values += (repr(value), None)

        self._writerow(values)
        self._record_count += 1

        if self._record_count >= self._maxresultrows:
            self.flush(partial=True)

    try:
        # noinspection PyUnresolvedReferences
        from _json import make_encoder
    except ImportError:
        # We may be running under PyPy 2.5 which does not include the _json module
        _iterencode_json = JSONEncoder(separators=(',', ':')).iterencode
    else:
        # Creating _iterencode_json this way yields a two-fold performance improvement on Python 2.7.9 and 2.7.10
        from json.encoder import encode_basestring_ascii

        @staticmethod
        def _default(o):
            raise TypeError(repr(o) + ' is not JSON serializable')

        _iterencode_json = make_encoder(
            {},  # markers (for detecting circular references)
            _default,  # object_encoder
            encode_basestring_ascii,  # string_encoder
            None,  # indent
            ':',
            ',',  # separators
            False,  # sort_keys
            False,  # skip_keys
            True  # allow_nan
        )

        del make_encoder
Ejemplo n.º 33
0
class Strange(object):
    __strategy_id = 1001
    __account_id = 10001
    
    
    def __init__(self, limit_order):
        
        
        self.__bar_list = []


        self.__pred = Prediciton()
        self.__pred.init_model(1,'../test/model/')
        self.__limit_order = OrderedDict() # 委托单队列
        self.long_volumes = OrderedDict() #当前多头持仓情况
        self.short_volumes = OrderedDict() #当前空头持仓情况

        # 当日行情结束后,进行导出,用于分析

        self.__history_limit_order = OrderedDict() # 历史委托单
        self.__history_limit_volumes = OrderedDict() # 历史成交记录

        self.__working_limit_order = limit_order



        # 交易记录存储
        self.__trader_record = OrderedDict()
        
        
        # 暂时将账户和合约初始化放在策略内初始化

        # 账户初始化
        self.__account = Account()
        self.__account.set_account_id(self.__account_id)
        self.__account.set_init_cash(DEFAULT_CASH)

        #期货合约初始化 
        self.__instrument = Instrument()
        self.__instrument.set_fee(0.00005)
        self.__instrument.set_short_margin_ratio(0.07)
        self.__instrument.set_long_margin_ratio(0.07)
        self.__instrument.set_price_tick(1)
        self.__instrument.set_min_limit_order_volume(15)
        self.__instrument.set_instrument_id('ag')
        self.__instrument.set_instrument_name('白银')



    def __reset(self):
        self.__limit_order.clear()
        self.__history_limit_order.clear()
        self.__history_limit_order.clear()
        self.__working_limit_order.clear()

    def record(self):
        self.__account.dump()
        record = DailyRecord()
        record.set_close_profit(self.__account.close_profit())
        record.set_position_profit(self.__account.position_profit())
        record.set_commssion(self.__account.commission())
        record.set_limit_volume(self.__history_limit_volumes)
        record.set_limit_order(self.__history_limit_order)
        
        
        record.set_mktime(self.__bar_list[-1].mktime())

        MLog.write().info('mkdate:%d, available_cash:%f,profit:%f'%(record.mktime(),self.__account.available_cash(), record.all_profit()))
        self.__trader_record[record.mktime()] = record
        self.__account.reset()
        self.__reset()

    def calc_result(self):
        max_profit = 0.0
        total_profit = 0.0
        base_value = 0.0
        for key,value in self.__trader_record.items():
            total_profit += value.all_profit()

        starting_cash = self.__account.starting_cash()
        base_value = (int(abs(total_profit) / starting_cash) + 1) * self.__account.starting_cash()
        MLog.write().info('all:%f, base_value:%f',total_profit,base_value)
        total_profit = 0.0

        df = pd.DataFrame(columns = ['mktime','interests','value','retrace','chg','log_chg','profit'])
        # 计算每日权益,每日净值, 涨跌幅,回撤,日对数收益
        last_value = 0.0
        for mktime,daily_record in self.__trader_record.items():
            daily_record.set_base_value(base_value)
            daily_record.set_last_value(last_value)
            daily_record.set_max_profit(max_profit)
            daily_record.set_last_profit(total_profit)
            daily_record.calc_result()
            daily_record.dump()
            df = df.append(pd.DataFrame({'mktime':mktime, 'interests':daily_record.interests(),
                                         'value':daily_record.value(), 'retrace': daily_record.retrace(),
                                         'chg':daily_record.chg(),'log_chg':daily_record.log_chg(),
                                         'profit':daily_record.all_profit()},index=['mktime']), ignore_index=True)

            total_profit += daily_record.all_profit()
            if max_profit < total_profit:
                max_profit = total_profit
            last_value = daily_record.value()
            base_value += daily_record.all_profit()
            
        chg_mean = np.mean(df['log_chg'])
        chg_std = np.std(df['log_chg'])

        summary_record = SummaryRecord()
        summary_record.set_chg_mean(chg_mean)
        summary_record.set_chg_std(chg_std)
        summary_record.set_trade_count(df.shape[0])
        summary_record.set_final_value(df['value'].values[-1])
        summary_record.set_max_retr(np.min(df['retrace']))


        summary_record.calc_record(df[df['chg'] > 0.00].shape[0])
        summary_record.dump()
        df.to_csv('result_strange.csv', encoding = 'utf-8')


    def save(self):
        '''
        filename = 'strange' + '.csv'
        fieldnames = ['mktime','interests','value','retrace','chg','log_chg','profit',]
        with open(filename, 'w') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
            writer.writeheader()
            for mktime,daily_record in self.__trader_record.items():
                writer.writerow({'mktime':mktime,'interests':daily_record.interests(),'value':daily_record.value(),'retrace':daily_record.retrace(),
                                 'chg':daily_record.chg(),'log_chg':daily_record.log_chg(),'profit':daily_record.all_profit()})
        '''


    # close, high, low open  下一分钟收盘价  
    def __on_fc_single(self):
        for bar in self.__bar_list:
            if self.__bar_list.index(bar) == 0:
                x = np.array([bar.close_price(), bar.high_price(), bar.low_price(), bar.open_price(),bar.volume(),
                             bar.last_buy_price(),bar.last_buy_vol(),bar.last_sell_price(),bar.last_sell_vol()])
            else:
                t = np.array([bar.close_price(), bar.high_price(), bar.low_price(), bar.open_price(),bar.volume(),
                             bar.last_buy_price(),bar.last_buy_vol(),bar.last_sell_price(),bar.last_sell_vol()])
                x = np.vstack((x,t))

        close_price = self.__pred.new_signal(x)
        return close_price
        # pred = Prediciton()
        # price = pred.signal(x)
        # return  price
        # rand =  int(np.std(x) * 1000  % 2)
        # if rand == 0:
        #    return np.mean(x) + np.std(x)
        # else:
        #    return np.mean(x)


    # 多头成本控制
    def __is_long_order(self, order_price, close_price, next_price):
        long_fee = (order_price * self.__instrument.min_limit_order_volume() * self.__instrument.fee()) * 2  # 多头建仓手续费
        long_profit = abs(order_price - next_price) * self.__instrument.min_limit_order_volume() # 多头最小利润
        
        if long_profit > long_fee:
            MLog.write().debug('long_profit:%f more than long_fee:%f allow open long'%(long_profit, long_fee))
        else:
            MLog.write().debug('long_profit:%f less than long_fee:%f not open long'%(long_profit, long_fee))
        return 1 if long_profit > long_fee else 0
    
    # 空头成本控制
    def __is_short_order(self, order_price, close_price, next_price):
        short_fee = (order_price * self.__instrument.min_limit_order_volume() * self.__instrument.fee()) * 2
        short_profit = abs(order_price - next_price) * self.__instrument.min_limit_order_volume() # 空头最小利润
        
        if short_profit > short_fee:
            MLog.write().debug('short_profit:%f more than short_fee:%f allow  open short'%(short_profit, short_fee))
        else:
            MLog.write().debug('short_profit:%f less than short_fee:%f not open short'%(short_profit, short_fee))
        return 1 if short_profit > short_fee else 0

    def cancel_order(self):
        for oid, order in self.__limit_order.items():
            del self.__working_limit_order[oid]
            del self.__limit_order[oid]
            order.set_status(OrderStatus.cacelled)
            self.__history_limit_order[oid] = order
            self.__account.canncel_order_cash(order.margin(), order.fee())
    
    
    # 多头操作
    def __long_operation(self, next_price, last_bar, amount):
        self.cancel_order()
        #是否持有多头仓
        if len(self.long_volumes) == 0:
            if self.__is_long_order(last_bar.last_sell_price(), last_bar.close_price(), next_price) == 1:
                order = self.__long_open(last_bar.last_sell_price(), amount,last_bar.current_time())
                order.dump()
                self.__account.insert_order_cash(order.margin(), order.fee())
                self.__working_limit_order[order.order_id()] = order
                self.__limit_order[order.order_id()] = order
                self.__account.dump()
        else: # 已经持有多头仓
            for k,v in self.long_volumes.items():
                MLog.write().debug("hold long volumens:%d price:%f close_price:%f" %(v.amount(),v.limit_price(), last_bar.close_price()))
        #是否有空头仓
        if len(self.short_volumes) > 0:
            MLog.write().debug("long singal start close short volume")
            for k,v in self.short_volumes.items():
                order = self.__short_close(last_bar.last_sell_price(), v.amount(),last_bar.current_time())
                order.set_hold_volume_id(v.trader_id())
                self.__account.insert_order_cash(order.margin(), order.fee())
                self.__working_limit_order[order.order_id()] = order
                self.__limit_order[order.order_id()] = order
                order.dump()
                self.__account.dump()

   # 空头操作
    def __short_operation(self, next_price, last_bar, amount):
        self.cancel_order()
        # 是否持有空头仓
        if len(self.short_volumes) == 0:
            if self.__is_short_order(last_bar.last_buy_price(), last_bar.close_price(), next_price) == 1:
                order = self.__short_open(last_bar.last_buy_price(), amount, last_bar.current_time())
                order.dump()
                self.__account.insert_order_cash(order.margin(), order.fee())
                self.__working_limit_order[order.order_id()] = order
                self.__limit_order[order.order_id()] = order
                self.__account.dump()
        else:
            for k,v in self.short_volumes.items():
                MLog.write().debug("hold short volumens:%d price:%f close_price:%f" %(v.amount(),v.limit_price(),last_bar.close_price()))
        # 是否持有多头仓
        if len(self.long_volumes) > 0: 
            MLog.write().debug("short singal start close long volume")
            for k,v in self.long_volumes.items():
                order = self.__long_close(last_bar.last_buy_price(), v.amount(), last_bar.current_time())
                order.set_hold_volume_id(v.trader_id())
                self.__account.insert_order_cash(order.margin(), order.fee())
                self.__working_limit_order[order.order_id()] = order
                self.__limit_order[order.order_id()] = order
                order.dump()
                self.__account.dump()
    # 多头开仓
    def __long_open(self, order_price, amount, bar_time):
        return self.__create_limit_price(order_price, amount, CombOffset.open, Direction.buy_direction,bar_time)

    # 多头平仓
    def __long_close(self, order_price, amount, bar_time):
        return self.__create_limit_price(order_price, amount, CombOffset.close, Direction.sell_direction,bar_time)
    
    # 空头开仓
    def __short_open(self, order_price, amount, bar_time):
        return self.__create_limit_price(order_price, amount, CombOffset.open, Direction.sell_direction,bar_time)

    # 空头平仓
    def __short_close(self, order_price, amount, bar_time):
        return self.__create_limit_price(order_price, amount, CombOffset.close, Direction.buy_direction,bar_time)

    # 创建限价单
    def __create_limit_price(self, order_price, amount, off_flag, direction,bar_time):
        # pdb.set_trace()
        order = Order()
        order.create_order_id()
        order.set_create_time(bar_time)
        order.set_account_id(self.__account_id)
        order.set_symbol(self.__instrument.instrument_id())
        order.set_comb_offset_flag(off_flag)
        order.set_order_price_type(OrderPrice.limit_price)
        order.set_limit_price(order_price)
        order.set_direction(direction)
        order.set_amount(amount)
        order.set_strategy_id(self.__strategy_id)
        order.set_min_volume(self.__instrument.min_limit_order_volume())
        if direction == Direction.buy_direction:
            order.set_margin(self.__instrument.long_margin_ratio())
        else:
            order.set_margin(self.__instrument.short_margin_ratio())

        order.set_fee(self.__instrument.fee())
        return order


    def on_volume(self, vol, order):
        # 从委托队列中删除
        # pdb.set_trace()
        del self.__limit_order[vol.order_id()]
        self.__history_limit_volumes[vol.trader_id()] = vol
        self.__history_limit_order[order.order_id()] = order
        vol.dump()
        # 判断是开仓还是平仓
        if vol.comb_offset_flag() == CombOffset.open: # 开仓
            if vol.direction() == Direction.buy_direction: # 多头仓
                # self.long_volumes[vol.symbol()] = vol
                self.long_volumes[vol.trader_id()] = vol
            else: # 空头仓
               # self.short_volumes[vol.symbol()] = vol
                self.short_volumes[vol.trader_id()] = vol
            self.__account.open_cash(order.margin(),vol.margin(), order.fee(), vol.fee())
        else: # 平仓
            # pdb.set_trace()
            MLog.write().debug("close position account:")
            if vol.direction() == Direction.buy_direction: # 平空头仓
                if self.short_volumes.has_key(order.hold_volume_id()):
                    v = self.short_volumes[order.hold_volume_id()]
                    # self.__account.set_close_profit(v,vol)
                    del self.short_volumes[order.hold_volume_id()]
            else: # 平多头仓
                if self.long_volumes.has_key(order.hold_volume_id()):
                    v = self.long_volumes[order.hold_volume_id()]
                    # self.__account.set_close_profit(v,vol)
                    del self.long_volumes[order.hold_volume_id()]
            self.__account.close_cash(v,vol,order.fee())
        self.__account.dump()

    def on_order(self, order):
        if order.status() == OrderStatus.all_traded:
            MLog.write().debug('order is all traded')
        self.__limit_order[order.order_id()] = order
        self.__history_limit_order[order.order_id()] = order
    
    def on_bar(self, bar):
        self.__bar_list.append(bar)       

        # 休盘的最后两分钟平仓
        if len(self.__bar_list) < 5:
            return
        if len(self.__bar_list) > 5:
            self.__bar_list.pop(0)
        
        next_price = self.__on_fc_single()
        close_price = self.__bar_list[-1].close_price()

        # 距行情结束还有5分钟则全部平仓
        if bar.current_time() > time.mktime(datetime.datetime(bar.mktime() / 10000,bar.mktime() / 100 % 100, bar.mktime() % 100,14,55,00).timetuple()):
            MLog.write().debug("time out start close position")
            if len(self.short_volumes) > 0:
                for k,v in self.short_volumes.items():
                    order = self.__short_close(bar.last_sell_price(), v.amount(),bar.current_time())
                    order.set_hold_volume_id(v.trader_id())
                    self.__account.insert_order_cash(order.margin(), order.fee())
                    self.__working_limit_order[order.order_id()] = order
                    self.__limit_order[order.order_id()] = order
                    order.dump()
            if len(self.long_volumes) > 0: 
                for k,v in self.long_volumes.items():
                    order = self.__long_close(bar.last_buy_price(), v.amount(), bar.current_time())
                    order.set_hold_volume_id(v.trader_id())
                    self.__account.insert_order_cash(order.margin(), order.fee())
                    self.__working_limit_order[order.order_id()] = order
                    self.__limit_order[order.order_id()] = order
                    order.dump()
            return 

        # pdb.set_trace()
        # 多仓,空仓判断
        if close_price < next_price: # 多仓信号
            MLog.write().debug("long singal start")
            self.__long_operation(next_price, self.__bar_list[-1], DEFAULT_AMOUNT)
        elif close_price > next_price: # 空仓信号
            MLog.write().debug("short singal start")
            self.__short_operation(next_price, self.__bar_list[-1], DEFAULT_AMOUNT)
Ejemplo n.º 34
0
class GridSearch:
    def __init__(self,
                 algorithm,
                 param_grid,
                 score_function,
                 n_times=5,
                 k_folds=10,
                 n_top=50,
                 bootstrap=True,
                 evaluation_class=None,
                 visualization=False):

        self.algorithm = algorithm
        self.param_grid = param_grid
        self.score_function = score_function
        self.n_times = n_times
        self.k_folds = k_folds
        self.bootstrap = bootstrap
        self.n_top = n_top
        self.best_model_dict = OrderedDict()
        self.evaluation_class = evaluation_class
        self.visualization = visualization

    def fit(self, x, y, thres):
        """
        This function runs the main part of the Grid Search
        :param x: numpy array
            - Includes the train data
        :param y: numpy array
            - Includes the actual value of each data sample
        :param thres: float
            - The cut-off boundary of the machine learning algorithm
        """
        values_list, key_list = self.parameters_to_grid(thres)
        # must be true
        unchecked = True
        # number of models
        num_of_models = len(list(product(*values_list)))
        # iterate per model
        model_id = 1
        cut_off_boundary = 0
        x_train_list, y_train_list, x_test_list, y_test_list = list(), list(
        ), list(), list()
        for tuples in list(product(*values_list)):
            # extract model parameters and cut off boundary
            parameters_dict, cut_off_boundary = self.extract_models_parameters(
                tuples, key_list)
            model = ModelTuningResults(model_id, parameters_dict,
                                       cut_off_boundary, self.n_times,
                                       self.k_folds)
            model_id += 1
            # n-times
            for i in range(0, self.n_times, 1):
                if not self.bootstrap:
                    x_train_list, y_train_list, x_test_list, y_test_list = k_fold_sample_data_set(
                        x, y, self.k_folds)
                # k-fold
                for j in range(0, self.k_folds, 1):
                    # split data set in train and test set
                    if self.bootstrap:
                        x_train, y_train, x_test, y_test = random_sample_data_set(
                            x, y, self.k_folds)
                    else:
                        x_train = x_train_list[j]
                        y_train = y_train_list[j]
                        x_test = x_test_list[j]
                        y_test = y_test_list[j]
                    if unchecked:
                        unchecked = self.calculate_grid_time(
                            x, y, num_of_models, parameters_dict)
                    # fit model
                    clf = self.algorithm.set_params(**parameters_dict).fit(
                        x_train, y_train)
                    # predict
                    predicted_probabilities = clf.predict_proba(x_test)
                    predicted_labels = [
                        0 if r[0] > cut_off_boundary else 1
                        for r in predicted_probabilities
                    ]
                    # calculate results
                    model.add_results(
                        calculate_metrics(y_test, predicted_labels,
                                          predicted_probabilities,
                                          self.score_function,
                                          self.evaluation_class))
            self.calculate_best_models(model)
        write_to_file(self.best_model_dict, cut_off_boundary,
                      self.visualization)

    def calculate_best_models(self, new_model):
        """
        This function keeps and updates the list with the best models according to the selected scoring function
        :param new_model: ModelTuningResults
            - A newly created model from the Grid search
        """
        self.best_model_dict[new_model.get_model_id()] = [
            new_model.calculate_model_score(), new_model
        ]
        temp = sorted(self.best_model_dict.items(),
                      key=lambda x: x[1],
                      reverse=True)
        self.best_model_dict.clear()
        for result in temp:
            self.best_model_dict[result[0]] = result[1]
        if len(self.best_model_dict) > self.n_top:
            self.best_model_dict.popitem()

    def calculate_grid_time(self, x, y, num_of_models, parameters_dict):
        """
        Makes an approximation of the time that is needed to complete the whole Grid Search procedure
        :param x: numpy array
            - Includes the train data
        :param y: numpy array
            - Includes the actual value of each data sample
        :param num_of_models: integer
            - The number of models that selected for evaluation
        :param parameters_dict: dict
            - The parameters of a model
        :return: boolean
            - Always false in order to executed only once
        """
        start = time()
        x_train, y_train, x_test, y_test = random_sample_data_set(
            x, y, self.k_folds)
        # check param compatibility with the selected model
        self.check_param_compatibility(self.algorithm, parameters_dict,
                                       x_train, y_train)
        end = time()
        print("Number of different models: " + str(num_of_models))
        print("The procedure needs " + str((
            (end - start) * num_of_models * self.n_times * self.k_folds) /
                                           60) + " minutes")
        return False

    def parameters_to_grid(self, thres):
        """

        :param thres: float
            - The cut-off boundary of the machine learning algorithm
        :return:
        """
        values_list, key_list = list(), list()
        for key, value_list in self.param_grid.iteritems():
            key_list.append(key)
            values_list.append(value_list)
        values_list.append(thres)
        key_list.append('threshold')
        return values_list, key_list

    @staticmethod
    def extract_models_parameters(tuples, key_list):
        """

        :param tuples:
        :param key_list:
        :return:
        """
        parameters_dict = dict()
        for i, t in enumerate(tuples):
            parameters_dict[key_list[i]] = t
        cut_off_boundary = parameters_dict['threshold']
        del parameters_dict['threshold']
        return parameters_dict, cut_off_boundary

    @staticmethod
    def check_param_compatibility(algorithm, parameters_dict, x_train,
                                  y_train):
        """
        This function checks if a random set of parameters are acceptable from the selected ML algorithm
        :param algorithm: sklearn model
            - The selected machine learning algorithm
        :param parameters_dict: dict
            - The parameters of a model
        :param x_train: numpy array
            - Includes the train data
        :param y_train: numpy array
            - Includes the actual value of each data sample
        :return:
        """
        try:
            algorithm.set_params(**parameters_dict).fit(x_train, y_train)
        except Exception as e:
            print("Error Message: " + e.message)
            exit()
Ejemplo n.º 35
0
class Combobox(ttk.Combobox):
    """
    A ttk.Combobox that accepts a list of (key, label) pair of options.
    """
    def __init__(self, master=None, **kw):
        self._keyvarcb = None
        self._txtvarcb = None
        self._choices = OrderedDict()
        
        self.__options = options = {
            'keyvariable': None,
            'textvariable': None,
        }
        # remove custom options from kw before initializating
        for k, v in options.items():
            options[k] = kw.pop(k, v)
        
        key = 'values'
        if key in kw:
            values = kw[key]
            self.choices = values
            values = self.__choices2tkvalues(self.choices)
            kw[key] = values
        
        # only normal/disabled supported
        # normal is converted to readonly
        key = 'state'
        if key in kw:
            if kw[key] not in ('readonly', 'disabled'):
                kw[key] = 'readonly'
        else:
            kw[key] = 'readonly'
            
        ttk.Combobox.__init__(self, master, **kw)
        
        self.__config_keyvar(options['keyvariable'])
        self.configure(textvariable=options['textvariable'])
    
    def __config_keyvar(self, var):
        if var is None:
            return
        
        def on_keyvar_changed(varname, elementname, mode):
            nvalue = self.__options['keyvariable'].get()
            if nvalue in self._choices:
                self.current(nvalue)
        
        keyvar = self.__options['keyvariable']
        if self._keyvarcb is not None:
            keyvar.trace_vdelete('w', self._keyvarcb)
        self.__options['keyvariable'] = var
        self._keyvarcb = var.trace(mode="w", callback=on_keyvar_changed)
        
        # A textvariable is needed if keyvariable is set
        txtvar = self.cget('textvariable')
        if not txtvar:
            var = tk.StringVar()
            self.configure(textvariable=var)
        else:
            self.__config_txtvar(txtvar)
        
    def __config_txtvar(self, var):
        keyvar = self.__options['keyvariable']
        if var is None or keyvar is None:
            return
        
        def on_txtvar_changed(varname, elementname, mode):
            ckey = self.current()
            keyvar.set(ckey)
        
        txtvar = self.cget('textvariable')
        if txtvar and self._txtvarcb is not None:
            txtvar.trace_vdelete('w', self._txtvarcb)
        self.__options['textvariable'] = var
        self._txtvarcb = var.trace(mode="w", callback=on_txtvar_changed)
            
    def configure(self, cnf=None, **kw):
        args = tk._cnfmerge((cnf, kw))
        key = 'values'
        if key in args:
            self.choices = args[key]
            args[key] = self.__choices2tkvalues(self.choices)
        key = 'keyvariable'
        if key in args:
            value = args.pop(key)
            self.__config_keyvar(value)
            return
        key = 'textvariable'
        if key in args:
            value = args[key]
            self.__config_txtvar(value)
        
        # only readonly and disabled supported
        key = 'state'
        if key in args:
            if args[key] not in ('readonly', 'disabled'):
                args[key] = 'readonly'
        return ttk.Combobox.configure(self, args)

    config = configure

    def cget(self, key):
        option = 'values'
        if key == option:
            return json.dumps(self.choices)
        option = 'keyvariable'
        if key == option:
            return self.__options[key]
        option = 'textvariable'
        if key == option:
            return self.__options[key]
        return ttk.Combobox.cget(self, key)
    
    def __obj2choices(self, values):
        """
        - json list of key, value pairs:
            Example: [["A", "Option 1 Label"], ["B", "Option 2 Label"]]
        - space separated string with a list of options:
            Example: "Option1 Opt2 Opt3"
            will be converted to a key, value pair of the following form:
            (("Option1", "Option1), ("Opt2", "Opt2"), ("Opt3", "Opt3"))
        - an iterable of key, value pairs
        """
        choices = values
        # choices from string
        if type(values) == type(''):
            obj = None
            # try json string
            try:
                obj = json.loads(values)
            except ValueError:
                obj = values
            if type(obj) == type([]):
                choices = obj
            else:
                choices = obj.split()
        return choices
        
    def __choices2tkvalues(self, choices):
        """choices: iterable of key, value pairs"""
        values = []
        for k, v in choices:
            values.append(v)
        return values
    
    @property
    def choices(self):
        return tuple(self._choices.items())
    
    @choices.setter
    def choices(self, values):
        values = self.__obj2choices(values)
        self._choices.clear()
        for idx, v in enumerate(values):
            key = value = v
            if type(v) != type('') and len(v) == 2:
                key, value = v
            self._choices[key] = value
    
    def current(self, key=None):
        idx = ttk.Combobox.current(self, None)
        index = -1
        ckey = None
        # find the key of the current selection,
        # or if a key was given, its index
        for i, k in enumerate(self._choices.keys()):
            if key is None and idx == i:
                ckey = k
                break
            elif key == k:
                index = i
                break
        if key is None:
            # return current key
            return ckey
        return ttk.Combobox.current(self, index)
    
    def set(self, key):
        value = self._choices[key]
        return ttk.Combobox.set(self, value)
        
    def get(self):
        return self.current()
Ejemplo n.º 36
0
class CardTable(object):
    card_id = ''
    table_def = None  # type: TableDef

    @staticmethod
    def from_bdf(card):
        raise NotImplementedError

    @staticmethod
    def to_bdf(data):
        raise NotImplementedError

    def __init__(self, h5n, parent):
        self._h5n = h5n
        self._parent = parent
        self._table_def = deepcopy(self.table_def)
        self._table_def.set_h5f(self._h5n.h5f)
        self.data = OrderedDict()

        if self.card_id is not None:
            if self.card_id == '':
                self.__class__.card_id = self.__class__.__name__

            self._h5n.register_card_table(self)

    def write_data(self, cards, domain):
        if self.from_bdf is CardTable.from_bdf:
            self._table_def.not_implemented()
            raise NotImplementedError
        self._table_def.write_data(cards, domain, self.from_bdf)

    def finalize(self):
        pass

    def read(self):
        if self.from_bdf is CardTable.from_bdf:
            # if not implemented, then bail now so the table isn't created in the h5 file
            return

        self.data.clear()
        # main table should ALWAYS be identity, regardless of inconsistencies in MSC spec
        self.data['identity'] = self._table_def.read()
        self.data[self._table_def.table_id.lower()] = self.data['identity']  # set to actual table id for convenience

        def _get_subtable_data(subtable):
            assert subtable.table_id not in self.data, subtable.table_id
            self.data[subtable.table_id.lower()] = subtable.read()

            for _ in subtable.subtables:
                _get_subtable_data(_)

        for subtable in self._table_def.subtables:
            _get_subtable_data(subtable)

    def __getattr__(self, item):
        if len(self.data) == 0:
            self.read()
        try:
            return self.data[item]
        except KeyError:
            raise AttributeError('Attribute %s not found on CardTable.' % (item))
 #initialize
 Zones=OrderedDict()
 Zone=OrderedDict()
 Par=OrderedDict()
 Pars=OrderedDict()
 SubPars=OrderedDict()
 SubPar=OrderedDict()
 Params=OrderedDict()
 Param=OrderedDict()
 Flags=OrderedDict()
 Flag=OrderedDict()
 DeviceTypes=OrderedDict()
 DeviceType=OrderedDict()
 
 
 Zones.clear()
 Zone.clear()
 Par.clear()
 Pars.clear()
 SubPar.clear()
 SubPars.clear()
 Params.clear()
 Param.clear()
 Flags.clear()
 Flag.clear()
 DeviceTypes=[]
 DeviceType=[]
 
 CurrDevices=None
 CurrRequired=None
 CurrRD_ID=None
Ejemplo n.º 38
0
class LRUCache(abc.MutableMapping, Generic[K, V]):
    def __init__(self, capacity=None):
        self.capacity = capacity
        self.cache = OrderedDict()  # type: OrderedDict[K, V]

    @property
    def lru(self) -> List[K]:
        return list(self.cache.keys())

    @property
    def length(self) -> int:
        return len(self.cache)

    def clear(self) -> None:
        self.cache.clear()

    def __len__(self) -> int:
        return self.length

    def __contains__(self, key: object) -> bool:
        return key in self.cache

    def __setitem__(self, key: K, value: V) -> None:
        self.set(key, value)

    def __delitem__(self, key: K) -> None:
        del self.cache[key]

    def __getitem__(self, key) -> V:
        value = self.get(key)
        if value is None:
            raise KeyError(key)

        return value

    def __iter__(self) -> Iterator[K]:
        return iter(self.cache)

    def get(self, key: K, default: D = None) -> Optional[Union[V, D]]:
        value = self.cache.get(key)

        if value is not None:
            # Put the key back to the front of the ordered dict by
            # re-insertig it
            del self.cache[key]
            self.cache[key] = value
            return value

        return default

    def set(self, key: K, value: V):
        if self.cache.get(key):
            del self.cache[key]
            self.cache[key] = value
        else:
            self.cache[key] = value
            # Check, if the cache is full and we have to remove old items
            # If the queue is of unlimited size, self.capacity is NaN and
            # x > NaN is always False in Python and the cache won't be cleared.
            if self.capacity is not None and self.length > self.capacity:
                self.cache.popitem(last=False)
Ejemplo n.º 39
0
class Map(object):
    def __init__(self, runtime, task_id, tag=0):
        assert type(tag) != bool
        self._runtime = runtime
        self._task_id = runtime.get_task_id(task_id)
        self._args = list()
        self._region_args = OrderedDict()
        self._output_region_args = OrderedDict()
        self._output_reqs = OrderedDict()
        self._promoted_storages = OrderedDict()
        self._next_region_idx = 0
        self._next_output_region_idx = 0
        self._projections = list()
        self._future_args = list()
        self._future_map_args = list()
        self._tag = tag

    def __del__(self):
        self._region_args.clear()
        self._output_region_args.clear()
        self._output_reqs.clear()
        self._promoted_storages.clear()
        self._projections.clear()
        self._future_args.clear()
        self._future_map_args.clear()

    def _add_region_arg(self, region_arg, field_id):
        if region_arg not in self._region_args:
            idx = self._next_region_idx
            self._next_region_idx += 1
            self._region_args[region_arg] = ([field_id], idx)
            return idx
        else:
            (fields, idx) = self._region_args[region_arg]
            if field_id not in fields:
                fields.append(field_id)
            return idx

    def _add_output_region_arg(self, storage, ipart, field_id):
        if (storage, ipart) not in self._output_region_args:
            idx = self._next_output_region_idx
            self._next_output_region_idx += 1
            self._output_region_args[(storage, ipart)] = (set([field_id]), idx)
            return idx
        else:
            (fields, idx) = self._output_region_args[(storage, ipart)]
            if field_id not in fields:
                fields.add(field_id)
            return idx

    def add_scalar_arg(self, value, dtype):
        self._args.append(ScalarArg(value, dtype))

    def add_dtype_arg(self, dtype):
        self._args.append(DtypeArg(dtype))

    def add_region_arg(self, storage, proj, perm, tag, flags):
        # For empty fields we override the permission with no access
        # to avoid bogus uninitilaized access warnings from the runtime
        if storage.dtype.itemsize == 0:
            perm = Permission.NO_ACCESS
        region_idx = self._add_region_arg(
            RegionArg(storage.region, proj, perm, tag, flags),
            storage.field_id,
        )

        self._args.append(
            AccessorArg(
                region_idx,
                storage.field_id,
                storage.dtype,
            ))

    def add_no_access(self, storage, proj, tag=0, flags=0):
        self.add_region_arg(storage, proj, Permission.NO_ACCESS, tag, flags)

    def add_input(self, storage, proj, tag=0, flags=0):
        self.add_region_arg(storage, proj, Permission.READ, tag, flags)

    def add_output(self, storage, proj, tag=0, flags=0):
        self.add_region_arg(storage, proj, Permission.WRITE, tag, flags)

    def add_inout(self, storage, proj, tag=0, flags=0):
        self.add_region_arg(storage, proj, Permission.READ_WRITE, tag, flags)

    def add_output_only(self, storage):
        ipart = storage.primary_ipart if storage.storage.fixed else None
        output_region_idx = self._add_output_region_arg(
            storage.storage, ipart, storage.field_id)
        self._args.append(
            AccessorArg(
                output_region_idx,
                storage.field_id,
                storage.dtype,
            ))

    def add_future(self, future):
        self._future_args.append(future)

    def add_future_map(self, future_map):
        self._future_map_args.append(future_map)

    def build_task(self, launch_domain, argbuf):
        for arg in self._args:
            arg.pack(argbuf)
        task = IndexTask(
            self._task_id,
            launch_domain,
            self._runtime.empty_argmap,
            argbuf.get_string(),
            argbuf.get_size(),
            mapper=self._runtime.mapper_id,
            tag=self._tag,
        )

        for region_arg, (fields, _) in self._region_args.items():
            region_arg.proj.add(self._runtime, task, region_arg, fields)
        for future in self._future_args:
            task.add_future(future)
        for future_map in self._future_map_args:
            task.add_point_future(ArgumentMap(future_map=future_map))
        for (storage, ipart), (fields, _) in self._output_region_args.items():
            output_req = self._runtime.create_output_region(storage,
                                                            list(fields),
                                                            ipart=ipart)
            if not storage.fixed:
                self._output_reqs[storage] = output_req
            task.add_output(output_req)
        return task

    def build_single_task(self, argbuf):
        assert len(self._output_region_args) == 0
        for arg in self._args:
            arg.pack(argbuf)
        task = Task(
            self._task_id,
            argbuf.get_string(),
            argbuf.get_size(),
            mapper=self._runtime.mapper_id,
            tag=self._tag,
        )
        for region_arg, (fields, _) in self._region_args.items():
            region_arg.proj.add_single(task, region_arg, fields)
        for future in self._future_args:
            task.add_future(future)
        if len(self._region_args) == 0:
            task.set_local_function(True)
        return task

    def execute(self, launch_domain):
        # Note that we should hold a reference to this buffer
        # until we launch a task, otherwise the Python GC will
        # collect the Python object holding the buffer, which
        # in turn will deallocate the C side buffer.
        argbuf = PandasBufferBuilder()
        task = self.build_task(launch_domain, argbuf)
        result = self._runtime.dispatch(task)
        for storage, output_req in self._output_reqs.items():
            region = output_req.get_region()
            ipart = output_req.get_partition().index_partition
            self._promoted_storages[storage] = storage.promote(region, ipart)
        return result

    def execute_single(self):
        argbuf = PandasBufferBuilder()
        return self._runtime.dispatch(self.build_single_task(argbuf))

    def promote_output_storage(self, storage):
        assert not storage.fixed
        assert storage in self._promoted_storages
        return self._promoted_storages[storage]
Ejemplo n.º 40
0
class DirectoryInformation(prebase.ProtoM21Object):
    '''
    returns information about a directory in a Corpus.  Called from corpus.corpora.Corpus

    only tested with CoreCorpus so far.
    '''
    def __init__(self,
                 dirName='',
                 dirTitle='',
                 isComposer=True,
                 corpusObject=None):
        self.directoryName = dirName
        self.directoryTitle = dirTitle
        self.isComposer = isComposer
        self.works = OrderedDict()

        self.corpusObject = corpusObject

        self.findWorks()

    def _reprInternal(self):
        return str(self.directoryName)

    def findWorks(self):
        '''
        populate other information about the directory such as
        files and filenames.


        >>> di = corpus.work.DirectoryInformation('schoenberg',
        ...             corpusObject=corpus.corpora.CoreCorpus())
        >>> di.findWorks()
        OrderedDict([(...'opus19', CorpusWork(title='Opus 19',
                                       files=[CorpusFile(path='schoenberg/opus19/movement2.mxl',
                                                         title='Movement 2',
                                                         filename='movement2.mxl',
                                                         format='musicxml',
                                                         ext='.mxl'),
                                              CorpusFile(path='schoenberg/opus19/movement6.mxl',
                                                         title='Movement 6',
                                                         filename='movement6.mxl',
                                                         format='musicxml',
                                                         ext='.mxl')],
                                        virtual=False))])
        '''
        self.works.clear()
        works = self.corpusObject.getComposer(self.directoryName)
        # TODO: this should be renamed since not all are composers
        for path in works:
            # split by the composer dir to get relative path
            # environLocal.printDebug(['dir composer', composerDirectory, path])
            junk, fileStub = path.as_posix().split(self.directoryName)
            if fileStub.startswith(os.sep):
                fileStub = fileStub[len(os.sep):]
            # break into file components
            fileComponents = fileStub.split(os.sep)
            # the first is either a directory for containing components
            # or a top-level name
            m21Format, ext = common.findFormatExtFile(fileComponents[-1])
            if ext is None:
                # environLocal.printDebug([
                #    'file that does not seem to have an extension',
                #    ext, path])
                continue
            # if not a file w/ ext, we will get None for format
            if m21Format is None:
                workStub = fileComponents[0]
            else:  # remove the extension
                workStub = fileComponents[0].replace(ext, '')
            # create list location if not already added
            if workStub not in self.works:
                title = common.spaceCamelCase(workStub).title()
                self.works[workStub] = CorpusWork(title=title,
                                                  files=[],
                                                  virtual=False)
            # last component is name
            m21Format, ext = common.findFormatExtFile(fileComponents[-1])
            # all path parts after corpus
            corpusPath = os.path.join(str(self.directoryName), fileStub)
            corpusFileName = fileComponents[-1]  # all after
            title = None
            # this works but takes a long time!
            # title = converter.parse(path).metadata.title
            # TODO: get from RichMetadataBundle!
            if title is None:
                title = common.spaceCamelCase(fileComponents[-1].replace(
                    ext, ''))
                title = title.title()
            fileTuple = CorpusFile(path=corpusPath,
                                   title=title,
                                   filename=corpusFileName,
                                   format=m21Format,
                                   ext=ext)
            self.works[workStub].files.append(fileTuple)
            # add this path
        return self.works
Ejemplo n.º 41
0
class CoverSettingsWidget(QWidget):

    changed = pyqtSignal()

    def __init__(self, mi=None, prefs=None, parent=None, for_global_prefs=False):
        QWidget.__init__(self, parent)
        self.ignore_changed = False
        self.for_global_prefs = for_global_prefs

        self.l = l = QHBoxLayout(self)
        l.setContentsMargins(0, 0, 0, 0)
        self.setLayout(l)
        self.settings_tabs = st = QTabWidget(self)
        l.addWidget(st)
        self.preview_label = la = Preview(self)
        l.addWidget(la)

        if prefs is None:
            prefs = cprefs
        self.original_prefs = prefs
        self.mi = mi or self.default_mi()

        self.colors_page = cp = QWidget(st)
        st.addTab(cp, _('&Colors'))
        cp.l = l = QGridLayout()
        cp.setLayout(l)
        if for_global_prefs:
            msg = _('When generating covers, a color scheme for the cover is chosen at random from the'
                    ' color schemes below. You can prevent an individual scheme from being selected by'
                    ' unchecking it. The preview on the right shows the currently selected color scheme.')
        else:
            msg = _('Choose a color scheme to be used for this generated cover.') + '<p>' + _(
                'In normal cover generation, the color scheme is chosen at random from the list of color schemes below. You'
                ' can prevent an individual color scheme from being chosen by unchecking it here.')
        cp.la = la = QLabel('<p>' + msg)
        la.setWordWrap(True)
        l.addWidget(la, 0, 0, 1, -1)
        self.colors_list = cl = QListWidget(cp)
        l.addWidget(cl, 1, 0, 1, -1)
        self.colors_map = OrderedDict()
        self.ncs = ncs = QPushButton(QIcon(I('plus.png')), _('&New color scheme'), cp)
        ncs.clicked.connect(self.create_color_scheme)
        l.addWidget(ncs)
        self.ecs = ecs = QPushButton(QIcon(I('format-fill-color.png')), _('&Edit color scheme'), cp)
        ecs.clicked.connect(self.edit_color_scheme)
        l.addWidget(ecs, l.rowCount()-1, 1)
        self.rcs = rcs = QPushButton(QIcon(I('minus.png')), _('&Remove color scheme'), cp)
        rcs.clicked.connect(self.remove_color_scheme)
        l.addWidget(rcs, l.rowCount()-1, 2)

        self.styles_page = sp = QWidget(st)
        st.addTab(sp, _('&Styles'))
        sp.l = l = QVBoxLayout()
        sp.setLayout(l)
        if for_global_prefs:
            msg = _('When generating covers, a style for the cover is chosen at random from the'
                    ' styles below. You can prevent an individual style from being selected by'
                    ' unchecking it. The preview on the right shows the currently selected style.')
        else:
            msg = _('Choose a style to be used for this generated cover.') + '<p>' + _(
                'In normal cover generation, the style is chosen at random from the list of styles below. You'
                ' can prevent an individual style from being chosen by unchecking it here.')
        sp.la = la = QLabel('<p>' + msg)
        la.setWordWrap(True)
        l.addWidget(la)
        self.styles_list = sl = QListWidget(sp)
        l.addWidget(sl)
        self.style_map = OrderedDict()

        self.font_page = fp = QWidget(st)
        st.addTab(fp, _('&Fonts and sizes'))
        fp.l = l = QFormLayout()
        fp.setLayout(l)
        fp.f = []

        def add_hline():
            f = QFrame()
            fp.f.append(f)
            f.setFrameShape(f.HLine)
            l.addRow(f)

        for x, label, size_label in (
                ('title', _('&Title font family:'), _('&Title font size:')),
                ('subtitle', _('&Subtitle font family'), _('&Subtitle font size:')),
                ('footer', _('&Footer font family'), _('&Footer font size')),
        ):
            attr = '%s_font_family' % x
            ff = FontFamilyChooser(fp)
            setattr(self, attr, ff)
            l.addRow(label, ff)
            ff.family_changed.connect(self.emit_changed)
            attr = '%s_font_size' % x
            fs = QSpinBox(fp)
            setattr(self, attr, fs)
            fs.setMinimum(8), fs.setMaximum(200), fs.setSuffix(' px')
            fs.setValue(prefs[attr])
            fs.valueChanged.connect(self.emit_changed)
            l.addRow(size_label, fs)
            add_hline()
        self.changed_timer = t = QTimer(self)
        t.setSingleShot(True), t.setInterval(500), t.timeout.connect(self.emit_changed)

        def create_sz(label):
            ans = QSpinBox(self)
            ans.setSuffix(' px'), ans.setMinimum(100), ans.setMaximum(10000)
            l.addRow(label, ans)
            ans.valueChanged.connect(self.changed_timer.start)
            return ans

        self.cover_width = create_sz(_('Cover &width:'))
        self.cover_height = create_sz(_('Cover &height:'))
        fp.cla = la = QLabel(_(
            'Note that the preview to the side is of fixed aspect ratio, so changing the cover'
            ' width above will not have any effect. If you change the height, you should also change the width nevertheless'
            ' as it will be used in actual cover generation.'))
        la.setWordWrap(True)
        l.addRow(la)

        self.templates_page = tp = QWidget(st)
        st.addTab(tp, _('&Text'))
        tp.l = l = QVBoxLayout()
        tp.setLayout(l)
        tp.la = la = QLabel(_(
            'The text on the generated cover is taken from the metadata of the book.'
            ' This is controlled via templates. You can use the <b>, <i> and <br> tags'
            ' in the templates for bold, italic and line breaks, respectively. The'
            ' default templates use the title, series and authors. You can change them to use'
            ' whatever metadata you like.'))
        la.setWordWrap(True), la.setTextFormat(Qt.PlainText)
        l.addWidget(la)

        def create_template_widget(title, which, button):
            attr = which + '_template'
            heading = QLabel('<h2>' + title)
            setattr(tp, attr + '_heading', heading)
            l.addWidget(heading)
            la = QLabel()
            setattr(self, attr, la)
            l.addWidget(la), la.setTextFormat(Qt.PlainText), la.setStyleSheet('QLabel {font-family: monospace}')
            la.setWordWrap(True)
            b = QPushButton(button)
            b.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed)
            connect_lambda(b.clicked, self, lambda self: self.change_template(which))
            setattr(self, attr + '_button', b)
            l.addWidget(b)
            if which != 'footer':
                f = QFrame(tp)
                setattr(tp, attr + '_sep', f), f.setFrameShape(QFrame.HLine)
                l.addWidget(f)
            l.addSpacing(10)

        create_template_widget(_('The title template'), 'title', _('Change the &title template'))
        create_template_widget(_('The sub-title template'), 'subtitle', _('Change the &sub-title template'))
        create_template_widget(_('The footer template'), 'footer', _('Change the &footer template'))
        l.addStretch(2)

        self.apply_prefs(prefs)
        self.changed.connect(self.update_preview)
        self.styles_list.itemSelectionChanged.connect(self.update_preview)
        self.colors_list.itemSelectionChanged.connect(self.update_preview)
        self.update_preview()

    def __enter__(self):
        self.ignore_changed = True

    def __exit__(self, *args):
        self.ignore_changed = False

    def emit_changed(self):
        if not self.ignore_changed:
            self.changed.emit()

    def apply_prefs(self, prefs):
        with self:
            self._apply_prefs(prefs)

    def _apply_prefs(self, prefs):
        for x in ('title', 'subtitle', 'footer'):
            attr = '%s_font_family' % x
            getattr(self, attr).font_family = prefs[attr]
            attr = '%s_font_size' % x
            getattr(self, attr).setValue(prefs[attr])

        for x in ('title', 'subtitle', 'footer'):
            x += '_template'
            getattr(self, x).setText(prefs[x])

        for x in ('width', 'height'):
            x = 'cover_' + x
            getattr(self, x).setValue(prefs[x])

        color_themes = prefs['color_themes'].copy()
        color_themes.update(default_color_themes)
        disabled = set(prefs['disabled_color_themes'])
        self.colors_list.clear()
        self.colors_map = {}
        for name in sorted(color_themes, key=sort_key):
            self.colors_map[name] = li = QListWidgetItem(name, self.colors_list)
            li.setFlags(li.flags() | Qt.ItemIsUserCheckable)
            li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked)
            li.setData(Qt.UserRole, color_themes[name])
        lu = prefs.get('last_used_colors')
        if not self.for_global_prefs and lu in self.colors_map and self.colors_map[lu].checkState() == Qt.Checked:
            self.colors_map[lu].setSelected(True)
        else:
            for name, li in iteritems(self.colors_map):
                if li.checkState() == Qt.Checked:
                    li.setSelected(True)
                    break
            else:
                next(itervalues(self.colors_map)).setSelected(True)

        disabled = set(prefs['disabled_styles'])
        self.styles_list.clear()
        self.style_map.clear()
        for name in sorted(all_styles(), key=sort_key):
            self.style_map[name] = li = QListWidgetItem(name, self.styles_list)
            li.setFlags(li.flags() | Qt.ItemIsUserCheckable)
            li.setCheckState(Qt.Unchecked if name in disabled else Qt.Checked)
        lu = prefs.get('last_used_style')
        if not self.for_global_prefs and lu in self.style_map and self.style_map[lu].checkState() == Qt.Checked:
            self.style_map[lu].setSelected(True)
        else:
            for name, li in iteritems(self.style_map):
                if li.checkState() == Qt.Checked:
                    li.setSelected(True)
                    break
            else:
                next(itervalues(self.style_map)).setSelected(True)

    @property
    def current_colors(self):
        for name, li in iteritems(self.colors_map):
            if li.isSelected():
                return name

    @property
    def disabled_colors(self):
        for name, li in iteritems(self.colors_map):
            if li.checkState() == Qt.Unchecked:
                yield name

    @property
    def custom_colors(self):
        ans = {}
        for name, li in iteritems(self.colors_map):
            if name.startswith('#'):
                ans[name] = li.data(Qt.UserRole)
        return ans

    @property
    def current_style(self):
        for name, li in iteritems(self.style_map):
            if li.isSelected():
                return name

    @property
    def disabled_styles(self):
        for name, li in iteritems(self.style_map):
            if li.checkState() == Qt.Unchecked:
                yield name

    @property
    def current_prefs(self):
        prefs = {k:self.original_prefs[k] for k in self.original_prefs.defaults}
        for x in ('title', 'subtitle', 'footer'):
            attr = '%s_font_family' % x
            prefs[attr] = getattr(self, attr).font_family
            attr = '%s_font_size' % x
            prefs[attr] = getattr(self, attr).value()
        prefs['color_themes'] = self.custom_colors
        prefs['disabled_styles'] = list(self.disabled_styles)
        prefs['disabled_color_themes'] = list(self.disabled_colors)
        for x in ('title', 'subtitle', 'footer'):
            x += '_template'
            prefs[x] = getattr(self, x).text()
        for x in ('width', 'height'):
            x = 'cover_' + x
            prefs[x] = getattr(self, x).value()
        return prefs

    def insert_scheme(self, name, li):
        with self:
            self.colors_list.insertItem(0, li)
            cm = OrderedDict()
            cm[name] = li
            for k, v in iteritems(self.colors_map):
                cm[k] = v
            self.colors_map = cm
            li.setSelected(True)
            for i in range(1, self.colors_list.count()):
                self.colors_list.item(i).setSelected(False)

    def create_color_scheme(self):
        scheme = self.colors_map[self.current_colors].data(Qt.UserRole)
        d = CreateColorScheme('#' + _('My Color Scheme'), scheme, set(self.colors_map), parent=self)
        if d.exec_() == d.Accepted:
            name, scheme = d.data
            li = QListWidgetItem(name)
            li.setData(Qt.UserRole, scheme), li.setFlags(li.flags() | Qt.ItemIsUserCheckable), li.setCheckState(Qt.Checked)
            self.insert_scheme(name, li)
            self.emit_changed()
            self.original_prefs['color_themes'] = self.current_prefs['color_themes']

    def edit_color_scheme(self):
        cs = self.current_colors
        if cs is None or not cs.startswith('#'):
            return error_dialog(self, _('Cannot edit'), _(
                'Cannot edit a builtin color scheme. Create a new'
                ' color scheme instead.'), show=True)
        li = self.colors_map[cs]
        d = CreateColorScheme(cs, li.data(Qt.UserRole), set(self.colors_map), edit_scheme=True, parent=self)
        if d.exec_() == d.Accepted:
            name, scheme = d.data
            li.setText(name)
            li.setData(Qt.UserRole, scheme)
            if name != cs:
                self.colors_map.pop(cs, None)
                self.insert_scheme(name, li)
            self.emit_changed()
            self.original_prefs['color_themes'] = self.current_prefs['color_themes']

    def remove_color_scheme(self):
        cs = self.current_colors
        if cs is None or not cs.startswith('#'):
            return error_dialog(self, _('Cannot remove'), _(
                'Cannot remove a builtin color scheme.'), show=True)
        for i in range(self.colors_list.count()):
            item = self.colors_list.item(i)
            if item.isSelected():
                with self:
                    del self.colors_map[item.text()]
                    self.colors_list.takeItem(i)
                    i = i % self.colors_list.count()
                    self.colors_list.item(i).setSelected(True)
                self.emit_changed()
                self.original_prefs['color_themes'] = self.current_prefs['color_themes']
                return

    def change_template(self, which):
        from calibre.gui2.dialogs.template_dialog import TemplateDialog
        from calibre.gui2.ui import get_gui
        gui = get_gui()
        if gui is None:
            from calibre.ebooks.metadata.book.base import field_metadata
        else:
            field_metadata = gui.current_db.new_api.field_metadata
        attr = which + '_template'
        templ = getattr(self, attr).text()
        d = TemplateDialog(self, templ, mi=self.mi, fm=field_metadata)
        if d.exec_() == d.Accepted:
            templ = d.rule[1]
            getattr(self, attr).setText(templ)
            self.emit_changed()

    @property
    def prefs_for_rendering(self):
        prefs = self.current_prefs
        prefs = override_prefs(prefs, override_style=self.current_style, override_color_theme=self.current_colors)
        return prefs

    def update_preview(self):
        if self.ignore_changed:
            return
        dpr = getattr(self, 'devicePixelRatioF', self.devicePixelRatio)()
        w, h = int(dpr * self.preview_label.sizeHint().width()), int(dpr * self.preview_label.sizeHint().height())
        prefs = self.prefs_for_rendering
        hr = h / prefs['cover_height']
        for x in ('title', 'subtitle', 'footer'):
            attr = '%s_font_size' % x
            prefs[attr] = int(prefs[attr] * hr)
        prefs['cover_width'], prefs['cover_height'] = w, h
        img = generate_cover(self.mi, prefs=prefs, as_qimage=True)
        img.setDevicePixelRatio(dpr)
        self.preview_label.setPixmap(QPixmap.fromImage(img))

    def default_mi(self):
        from calibre.ebooks.metadata.book.base import Metadata
        mi = Metadata(_('A sample book'), [_('Author One'), _('Author Two')])
        mi.series = _('A series of samples')
        mi.series_index = 4
        mi.tags = [_('Tag One'), _('Tag Two')]
        mi.publisher = _('Some publisher')
        mi.rating = 4
        mi.identifiers = {'isbn':'123456789', 'url': 'https://calibre-ebook.com'}
        mi.languages = ['eng', 'fra']
        mi.pubdate = mi.timestamp = now()
        return mi

    def restore_defaults(self):
        defaults = self.original_prefs.defaults.copy()
        # Dont delete custom color themes when restoring defaults
        defaults['color_themes'] = self.custom_colors
        self.apply_prefs(defaults)
        self.update_preview()

    def save_state(self):
        self.original_prefs.set('last_used_colors', self.current_colors)
        self.original_prefs.set('last_used_style', self.current_style)

    def save_as_prefs(self):
        with self.original_prefs:
            for k, v in iteritems(self.current_prefs):
                self.original_prefs[k] = v
Ejemplo n.º 42
0
class SystemLayoutInput:
    """
    Geometry file. Contains 2-D mapping of geometry.

    This approach to specifying core layout has been deprecated in favor of the
    :py:class:`armi.reactor.blueprints.gridBlueprints.GridBlueprints` and ``systems:``
    section of :py:class:`armi.reactor.blueprints.Blueprints`
    """

    _GEOM_FILE_EXTENSION = ".xml"
    ROOT_TAG = INP_SYSTEMS

    def __init__(self):
        self.fName = None
        self.modifiedFileName = None
        self.geomType: str
        self.symmetry = None
        self.assemTypeByIndices = OrderedDict()
        self.eqPathInput = {}
        self.eqPathsHaveBeenModified = False
        self.maxRings = 0

    def __repr__(self):
        return "<Geometry file {0}>".format(self.fName)

    def readGeomFromFile(self, fName):
        """
        Read the 2-d geometry input file.

        See Also
        --------
        fromReactor : Build SystemLayoutInput from a Reactor object.
        """
        self.fName = os.path.expandvars(fName)
        with open(self.fName) as stream:
            self.readGeomFromStream(stream)

    def readGeomFromStream(self, stream):
        """
        Read geometry info from a stream.

        This populates the object with info from any source.

        Notes
        -----
        There are two formats of geometry: yaml and xml. This tries
        xml first (legacy), and if it fails it tries yaml.
        """
        # Warn the user that this feature is schedule for deletion.
        warn = "XML Geom Files are scheduled to be removed from ARMI, please use blueprint files."
        runLog.important(warn)

        try:
            self._readXml(stream)
        except ET.ParseError:
            stream.seek(0)
            self._readYaml(stream)

    def toGridBlueprints(self, name: str = "core"):
        """
        Migrate old-style SystemLayoutInput to new GridBlueprint.

        Returns a list of GridBlueprint objects. There will at least be one entry,
        containing the main core layout. If equilibrium fuel paths are specified, it
        will occupy the second element.
        """
        # TODO: After moving SystemLayoutInput out of geometry.py, we may be able to
        # move this back out to top-level without causing blueprint import order issues.
        from armi.reactor.blueprints.gridBlueprint import GridBlueprint

        geom = self.geomType
        symmetry = self.symmetry

        bounds = None

        if self.geomType == geometry.GeomType.RZT:
            # We need a grid in order to go from whats in the input to indices, and to
            # be able to provide grid bounds to the blueprint.
            rztGrid = grids.ThetaRZGrid.fromGeom(self)
            theta, r, _ = rztGrid.getBounds()
            bounds = {"theta": theta.tolist(), "r": r.tolist()}

        gridContents = dict()
        for indices, spec in self.assemTypeByIndices.items():
            if self.geomType == geometry.GeomType.HEX:
                i, j = grids.HexGrid.getIndicesFromRingAndPos(*indices)
            elif self.geomType == geometry.GeomType.RZT:
                i, j, _ = rztGrid.indicesOfBounds(*indices[0:4])
            else:
                i, j = indices
            gridContents[(i, j)] = spec

        bp = GridBlueprint(
            name=name,
            gridContents=gridContents,
            geom=geom,
            symmetry=symmetry,
            gridBounds=bounds,
        )

        bps = [bp]

        if any(val != (None, None) for val in self.eqPathInput.values()):
            # We could probably just copy eqPathInput, but we don't want to preserve
            # (None, None) entries.
            eqPathContents = dict()
            for idx, eqPath in self.eqPathInput.items():
                if eqPath == (None, None):
                    continue
                if self.geomType == geometry.GeomType.HEX:
                    i, j = grids.HexGrid.getIndicesFromRingAndPos(*idx)
                elif self.geomType == geometry.GeomType.RZT:
                    i, j, _ = rztGrid.indicesOfBounds(*idx[0:4])
                else:
                    i, j = idx
                eqPathContents[i, j] = copy(self.eqPathInput[idx])

            pathBp = GridBlueprint(
                name=name + "EqPath",
                gridContents=eqPathContents,
                geom=geom,
                symmetry=symmetry,
                gridBounds=bounds,
            )

            bps.append(pathBp)

        return bps

    def _readXml(self, stream):
        tree = ET.parse(stream)
        root = tree.getroot()
        self._getGeomTypeAndSymmetryFromXml(root)
        self.assemTypeByIndices.clear()

        for assemblyNode in root:
            aType = str(assemblyNode.attrib["name"])
            eqPathIndex, eqPathCycle = None, None

            if self.geomType == geometry.GeomType.CARTESIAN:
                indices = x, y = tuple(
                    int(assemblyNode.attrib[key]) for key in LOC_CARTESIAN
                )
                self.maxRings = max(x + 1, y + 1, self.maxRings)
            elif self.geomType == geometry.GeomType.RZT:
                indices = tuple(
                    float(assemblyNode.attrib[key]) for key in LOC_RZ
                ) + tuple(int(assemblyNode.attrib[key]) for key in MESH_RZ)
            else:
                # assume hex geom.
                indices = ring, _pos = tuple(
                    int(assemblyNode.attrib[key]) for key in LOC_HEX
                )
                self.maxRings = max(ring, self.maxRings)

                if INP_FUEL_PATH in assemblyNode.attrib:
                    # equilibrium geometry info.
                    eqPathIndex = int(assemblyNode.attrib[INP_FUEL_PATH])
                    eqPathCycle = int(assemblyNode.attrib[INP_FUEL_CYCLE])

            self.assemTypeByIndices[indices] = aType
            self.eqPathInput[indices] = (eqPathIndex, eqPathCycle)

    def _readYaml(self, stream):
        """
        Read geometry from yaml.

        Notes
        -----
        This is intended to replace the XML format as we converge on
        consistent inputs.
        """
        yaml = YAML()
        tree = yaml.load(stream)
        tree = INPUT_SCHEMA(tree)
        self.assemTypeByIndices.clear()
        for _systemName, system in tree[INP_SYSTEMS].items():
            # no need to check for valid since the schema handled that.
            self.geomType = geometry.GeomType.fromStr(system[INP_GEOM])
            self.symmetry = geometry.SymmetryType.fromStr(system[INP_SYMMETRY])
            if INP_DISCRETES in system:
                self._read_yaml_discretes(system)
            elif INP_LATTICE in system:
                self._read_yaml_lattice(system)

    def _read_yaml_discretes(self, system):
        for discrete in system[INP_DISCRETES]:
            location = discrete[INP_LOCATION]
            indices = tuple(location[k] for k in LOC_KEYS[str(self.geomType)])
            if self.geomType == geometry.GeomType.CARTESIAN:
                x, y = indices
                self.maxRings = max(x + 1, y + 1, self.maxRings)
            elif self.geomType == geometry.GeomType.RZT:
                pass
            else:
                # assume hex geom.
                x, y = indices
                self.maxRings = max(x, self.maxRings)

            self.assemTypeByIndices[indices] = discrete[INP_SPEC]
            self.eqPathInput[indices] = (
                discrete.get(INP_FUEL_CYCLE),
                discrete.get(INP_FUEL_PATH),
            )

    def _read_yaml_lattice(self, system):
        """Read a ascii map string into this object."""
        mapTxt = system[INP_LATTICE]
        if (
            self.geomType == geometry.GeomType.HEX
            and self.symmetry.domain == geometry.DomainType.THIRD_CORE
        ):
            asciimap = asciimaps.AsciiMapHexThirdFlatsUp()
            asciimap.readAscii(mapTxt)
            for (i, j), spec in asciimap.items():
                if spec == "-":
                    # skip whitespace placeholders
                    continue
                ring, pos = grids.HexGrid.indicesToRingPos(i, j)
                self.assemTypeByIndices[(ring, pos)] = spec
                self.maxRings = max(ring, self.maxRings)
        else:
            raise ValueError(
                f"ASCII map reading from geom/domain: {self.geomType}/"
                f"{self.symmetry.domain} not supported."
            )

    def modifyEqPaths(self, modifiedPaths):
        """
        Modifies the geometry object by updating the equilibrium path indices and equilibrium path cycles.

        Parameters
        ----------
        modifiedPaths : dict, required
            This is a dictionary that contains the indices that are mapped to the
            eqPathIndex and eqPathCycle.  modifiedPath[indices] = (eqPathIndex,
            eqPathCycle)
        """
        runLog.important("Modifying the equilibrium paths on {}".format(self))
        self.eqPathsHaveBeenModified = True
        self.eqPathInput.update(modifiedPaths)

    def _getModifiedFileName(self, originalFileName, suffix):
        """
        Generates the modified geometry file name based on the requested suffix
        """
        originalFileName = originalFileName.split(self._GEOM_FILE_EXTENSION)[0]
        suffix = suffix.split(self._GEOM_FILE_EXTENSION)[0]
        self.modifiedFileName = originalFileName + suffix + self._GEOM_FILE_EXTENSION

    def writeGeom(self, outputFileName, suffix=""):
        """
        Write data out as a geometry xml file

        Parameters
        ----------
        outputFileName : str
            Geometry file name

        suffix : str
            Added suffix to the geometry output file name

        """
        if suffix:
            self._getModifiedFileName(outputFileName, suffix)
            outputFileName = self.modifiedFileName

        runLog.important("Writing reactor geometry file as {}".format(outputFileName))
        root = ET.Element(
            INP_SYSTEMS,
            attrib={
                INP_GEOM: str(self.geomType),
                INP_SYMMETRY: str(self.symmetry),
            },
        )
        tree = ET.ElementTree(root)
        # start at ring 1 pos 1 and go out
        for targetIndices in sorted(list(self.assemTypeByIndices)):
            ring, pos = targetIndices
            assembly = ET.SubElement(root, "assembly")
            assembly.set("ring", str(ring))
            assembly.set("pos", str(pos))
            fuelPath, fuelCycle = self.eqPathInput.get((ring, pos), (None, None))
            if fuelPath is not None:
                # set the equilibrium shuffling info if it exists
                assembly.set(INP_FUEL_PATH, str(fuelPath))
                assembly.set(INP_FUEL_CYCLE, str(fuelCycle))

            aType = self.assemTypeByIndices[targetIndices]
            assembly.set("name", aType)
        # note: This is ugly and one-line, but that's ok
        # since we're transitioning.
        tree.write(outputFileName)

    def _writeYaml(self, stream, sysName="core"):
        """
        Write the system layout in YAML format.

        These can eventually either define a type/specifier mapping and then point
        to a grid definition full of specifiers to be populated, or
        they can just be a list of "singles" which define indices:specifier.

        Note
        ----
        So far, only singles are implemented (analogous to old XML format)
        """
        geomData = []
        for indices in sorted(list(self.assemTypeByIndices)):
            specifier = self.assemTypeByIndices[indices]
            fuelPath, fuelCycle = self.eqPathInput.get(indices, (None, None))
            keys = LOC_KEYS[str(self.geomType)]
            dataPoint = {INP_LOCATION: dict(zip(keys, indices)), INP_SPEC: specifier}
            if fuelPath is not None:
                dataPoint.update({INP_FUEL_PATH: fuelPath, INP_FUEL_CYCLE: fuelCycle})
            geomData.append(dataPoint)

        yaml = YAML()
        yaml.default_flow_style = False
        yaml.indent(mapping=2, sequence=4, offset=2)
        fullData = {
            INP_SYSTEMS: {
                sysName: {
                    INP_GEOM: str(self.geomType),
                    INP_SYMMETRY: str(self.symmetry),
                    INP_DISCRETES: geomData,
                }
            }
        }
        fullData = INPUT_SCHEMA(fullData)  # validate on the way out
        yaml.dump(fullData, stream)

    def _writeAsciiMap(self):
        """Generate an ASCII map representation.

        Warning
        -------
        This only works for HexGrid.
        """
        lattice = {}
        for ring, pos in sorted(list(self.assemTypeByIndices)):
            specifier = self.assemTypeByIndices[(ring, pos)]
            i, j = grids.HexGrid.getIndicesFromRingAndPos(ring, pos)
            lattice[i, j] = specifier

        geomMap = asciimaps.AsciiMapHexThirdFlatsUp()
        geomMap.asciiLabelByIndices = lattice
        geomMap.gridContentsToAscii()
        geomMap.writeAscii(sys.stdout)

    def growToFullCore(self):
        """
        Convert geometry input to full core.

        Notes
        -----
        This only works for Hex 1/3rd core geometry inputs.
        """
        if self.symmetry.domain == geometry.DomainType.FULL_CORE:
            # already full core from geometry file. No need to copy symmetry over.
            runLog.important(
                "Detected that full core geometry already exists. Cannot expand."
            )
            return
        elif (
            self.symmetry.domain != geometry.DomainType.THIRD_CORE
            or self.symmetry.boundary != geometry.BoundaryType.PERIODIC
        ):
            raise ValueError(
                "Cannot convert shape `{}` to full core, must be {}".format(
                    self.symmetry.domain,
                    str(
                        geometry.SymmetryType(
                            geometry.DomainType.THIRD_CORE,
                            geometry.BoundaryType.PERIODIC,
                        )
                    ),
                ),
            )

        grid = grids.HexGrid.fromPitch(1.0)
        grid._symmetry: str = str(self.symmetry)

        # need to cast to a list because we will modify during iteration
        for (ring, pos), specifierID in list(self.assemTypeByIndices.items()):
            indices = grids.HexGrid.getIndicesFromRingAndPos(ring, pos)
            for symmetricI, symmetricJ in grid.getSymmetricEquivalents(indices):
                symmetricRingPos = grids.HexGrid.indicesToRingPos(
                    symmetricI, symmetricJ
                )
                self.assemTypeByIndices[symmetricRingPos] = specifierID

        self.symmetry = geometry.SymmetryType(
            geometry.DomainType.FULL_CORE,
            geometry.BoundaryType.NO_SYMMETRY,
        )

    def _getGeomTypeAndSymmetryFromXml(self, root):
        """Read the geometry type and symmetry."""
        try:
            self.geomType = geometry.GeomType.fromStr(
                str(root.attrib[INP_GEOM]).lower()
            )
        except ValueError:
            # will not execute if the geom was specified as thetarz, cartesian or anything else specific
            runLog.warning(
                "Could not find geometry type. Assuming hex geometry with third core periodic symmetry."
            )
            self.geomType = geometry.GeomType.HEX
            self.symmetry = geometry.SymmetryType(
                geometry.DomainType.THIRD_CORE,
                geometry.BoundaryType.PERIODIC,
            )
        else:
            inputString = str(root.attrib[INP_SYMMETRY]).lower()
            self.symmetry = geometry.SymmetryType.fromStr(inputString)

    @classmethod
    def fromReactor(cls, reactor):
        """
        Build SystemLayoutInput object based on the current state of a Reactor.

        See Also
        --------
        readGeomFromFile : Builds a SystemLayoutInput from an XML file.
        """
        geom = cls()
        runLog.info("Reading core map from {}".format(reactor))
        geom.geomType = str(reactor.core.geomType)
        geom.symmetry = reactor.core.symmetry

        bp = reactor.blueprints
        assemDesigns = bp.assemDesigns if bp else ()

        for assembly in reactor.core:
            aType = assembly.getType()

            if aType in assemDesigns:
                aType = assemDesigns[aType].specifier

            (x, _y) = indices = reactor.core.spatialGrid.getRingPos(
                assembly.spatialLocator.getCompleteIndices()
            )
            geom.maxRings = max(x, geom.maxRings)

            geom.assemTypeByIndices[indices] = aType

        return geom

    @classmethod
    def loadFromCs(cls, cs):
        """Function to load Geoemtry based on supplied ``CaseSettings``."""
        if not cs["geomFile"]:
            return None

        with directoryChangers.DirectoryChanger(cs.inputDirectory):
            geom = cls()
            geom.readGeomFromFile(cs["geomFile"])
            return geom
Ejemplo n.º 43
0
class DotMap(MutableMapping, OrderedDict):
    def __init__(self, *args, **kwargs):
        self._map = OrderedDict()
        self._dynamic = True
        if kwargs:
            if '_dynamic' in kwargs:
                self._dynamic = kwargs['_dynamic']
        if args:
            d = args[0]
            if isinstance(d, dict):
                for k, v in self.__call_items(d):
                    if isinstance(v, dict):
                        v = DotMap(v, _dynamic=self._dynamic)
                    if type(v) is list:
                        l = []
                        for i in v:
                            n = i
                            if type(i) is dict:
                                n = DotMap(i, _dynamic=self._dynamic)
                            l.append(n)
                        v = l
                    self._map[k] = v
        if kwargs:
            for k, v in self.__call_items(kwargs):
                if k is not '_dynamic':
                    self._map[k] = v

    def __call_items(self, obj):
        if hasattr(obj, 'iteritems') and ismethod(getattr(obj, 'iteritems')):
            return obj.iteritems()
        else:
            return obj.items()

    def items(self):
        return self.iteritems()

    def iteritems(self):
        return self.__call_items(self._map)

    def __iter__(self):
        return self._map.__iter__()

    def next(self):
        return self._map.next()

    def __setitem__(self, k, v):
        self._map[k] = v

    def __getitem__(self, k):
        if k not in self._map and self._dynamic and k != '_ipython_canary_method_should_not_exist_':
            # automatically extend to new DotMap
            self[k] = DotMap()
        return self._map[k]

    def __setattr__(self, k, v):
        if k in {
                '_map', '_dynamic', '_ipython_canary_method_should_not_exist_'
        }:
            super(DotMap, self).__setattr__(k, v)
        else:
            self[k] = v

    def __getattr__(self, k):
        if k == {
                '_map', '_dynamic', '_ipython_canary_method_should_not_exist_'
        }:
            super(DotMap, self).__getattr__(k)
        else:
            return self[k]

    def __delattr__(self, key):
        return self._map.__delitem__(key)

    def __contains__(self, k):
        return self._map.__contains__(k)

    def __str__(self):
        items = []
        for k, v in self.__call_items(self._map):
            # bizarre recursive assignment situation (why someone would do this is beyond me)
            if id(v) == id(self):
                items.append('{0}=DotMap(...)'.format(k))
            else:
                items.append('{0}={1}'.format(k, repr(v)))
        joined = ', '.join(items)
        out = '{0}({1})'.format(self.__class__.__name__, joined)
        return out

    def __repr__(self):
        return str(self)

    def toDict(self):
        d = {}
        for k, v in self.items():
            if type(v) is DotMap:
                # bizarre recursive assignment support
                if id(v) == id(self):
                    v = d
                else:
                    v = v.toDict()
            elif type(v) is list:
                l = []
                for i in v:
                    n = i
                    if type(i) is DotMap:
                        n = i.toDict()
                    l.append(n)
                v = l
            d[k] = v
        return d

    def pprint(self, pformat='dict'):
        if pformat == 'json':
            print(dumps(self.toDict(), indent=4, sort_keys=True))
        else:
            pprint(self.toDict())

    def empty(self):
        return (not any(self))

    # proper dict subclassing
    def values(self):
        return self._map.values()

    # ipython support
    def __dir__(self):
        return self.keys()

    @classmethod
    def parseOther(self, other):
        if type(other) is DotMap:
            return other._map
        else:
            return other

    def __cmp__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__cmp__(other)

    def __eq__(self, other):
        other = DotMap.parseOther(other)
        if not isinstance(other, dict):
            return False
        return self._map.__eq__(other)

    def __ge__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__ge__(other)

    def __gt__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__gt__(other)

    def __le__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__le__(other)

    def __lt__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__lt__(other)

    def __ne__(self, other):
        other = DotMap.parseOther(other)
        return self._map.__ne__(other)

    def __delitem__(self, key):
        return self._map.__delitem__(key)

    def __len__(self):
        return self._map.__len__()

    def clear(self):
        self._map.clear()

    def copy(self):
        return DotMap(self)

    def __copy__(self):
        return self.copy()

    def __deepcopy__(self, memo=None):
        return self.copy()

    def get(self, key, default=None):
        return self._map.get(key, default)

    def has_key(self, key):
        return key in self._map

    def iterkeys(self):
        return self._map.iterkeys()

    def itervalues(self):
        return self._map.itervalues()

    def keys(self):
        return self._map.keys()

    def pop(self, key, default=None):
        return self._map.pop(key, default)

    def popitem(self):
        return self._map.popitem()

    def setdefault(self, key, default=None):
        self._map.setdefault(key, default)

    def update(self, *args, **kwargs):
        if len(args) != 0:
            self._map.update(*args)
        self._map.update(kwargs)

    def viewitems(self):
        return self._map.viewitems()

    def viewkeys(self):
        return self._map.viewkeys()

    def viewvalues(self):
        return self._map.viewvalues()

    @classmethod
    def fromkeys(cls, seq, value=None):
        d = DotMap()
        d._map = OrderedDict.fromkeys(seq, value)
        return d

    def __getstate__(self):
        return self.__dict__

    def __setstate__(self, d):
        self.__dict__.update(d)

    # bannerStr
    def _getListStr(self, items):
        out = '['
        mid = ''
        for i in items:
            mid += '  {}\n'.format(i)
        if mid != '':
            mid = '\n' + mid
        out += mid
        out += ']'
        return out

    def _getValueStr(self, k, v):
        outV = v
        multiLine = len(str(v).split('\n')) > 1
        if multiLine:
            # push to next line
            outV = '\n' + v
        if type(v) is list:
            outV = self._getListStr(v)
        out = '{} {}'.format(k, outV)
        return out

    def _getSubMapDotList(self, pre, name, subMap):
        outList = []
        if pre == '':
            pre = name
        else:
            pre = '{}.{}'.format(pre, name)

        def stamp(pre, k, v):
            valStr = self._getValueStr(k, v)
            return '{}.{}'.format(pre, valStr)

        for k, v in subMap.items():
            if isinstance(v, DotMap) and v != DotMap():
                subList = self._getSubMapDotList(pre, k, v)
                outList.extend(subList)
            else:
                outList.append(stamp(pre, k, v))
        return outList

    def _getSubMapStr(self, name, subMap):
        outList = ['== {} =='.format(name)]
        for k, v in subMap.items():
            if isinstance(v, DotMap) and v != DotMap():
                # break down to dots
                subList = self._getSubMapDotList('', k, v)
                # add the divit
                # subList = ['> {}'.format(i) for i in subList]
                outList.extend(subList)
            else:
                out = self._getValueStr(k, v)
                # out = '> {}'.format(out)
                out = '{}'.format(out)
                outList.append(out)
        finalOut = '\n'.join(outList)
        return finalOut

    def bannerStr(self):
        lines = []
        previous = None
        for k, v in self.items():
            if previous == 'DotMap':
                lines.append('-')
            out = ''
            if isinstance(v, DotMap):
                name = k
                subMap = v
                out = self._getSubMapStr(name, subMap)
                lines.append(out)
                previous = 'DotMap'
            else:
                out = self._getValueStr(k, v)
                lines.append(out)
                previous = 'other'
        lines.append('--')
        s = '\n'.join(lines)
        return s
Ejemplo n.º 44
0
class CellAnalyzer(LoggerObject):

    def __init__(self, timeholder, position, create_images, binning_factor,
                 detect_objects):
        super(CellAnalyzer, self).__init__()

        self.timeholder = timeholder
        self.P = position
        self.bCreateImages = create_images
        self.iBinningFactor = binning_factor
        self.detect_objects = detect_objects

        self._iT = None
        self._channel_registry = OrderedDict()

    def initTimepoint(self, iT):
        self._channel_registry.clear()
        self._iT = iT
        self.timeholder.initTimePoint(iT)

    # XXX rename it to "add_channel"
    def register_channel(self, channel):
        self._channel_registry[channel.NAME] = channel

    def get_channel_names(self):
        return self._channel_registry.keys()

    def get_channel(self, name):
        return self._channel_registry[str(name)]

    @property
    def proc_channels(self):
        """Return processing channels i.e the dict contains no virtual or
        pseudo channels.
        """
        pchannels = OrderedDict()
        for name, channel in self._channel_registry.iteritems():
            if not channel.is_virtual():
                pchannels[name] =  channel
        return pchannels

    @property
    def virtual_channels(self):
        """Return a dict that contains all virtual channels."""
        channels = dict()
        for name, channel in self._channel_registry.iteritems():
            if channel.is_virtual():
                channels[name] =  channel
        return channels

    def process(self, apply=True, extract_features=True):
        """Perform the segmentation and feature extraction."""
        channels = sorted(self._channel_registry.values())
        primary_channel = None

        cnames = [c.NAME for c in channels]
        for channel in channels:
            self.timeholder.prepare_raw_image(channel)
            if self.detect_objects:
                if channel.NAME == PrimaryChannel.NAME:
                    self.timeholder.apply_segmentation(channel)
                    primary_channel = channel
                elif channel.NAME == SecondaryChannel.NAME:
                    self.timeholder.apply_segmentation(channel, primary_channel)
                    secondary_channel = channel
                elif channel.NAME == TertiaryChannel.NAME:
                    if SecondaryChannel.NAME not in cnames:
                        raise RuntimeError(("Tertiary channel requiers a "
                                            "secondary channel"))
                    self.timeholder.apply_segmentation(channel,
                                                       primary_channel,
                                                       secondary_channel)
                elif channel.NAME == MergedChannel.NAME:
                    channel.meta_image = primary_channel.meta_image
                    self.timeholder.apply_segmentation(channel, self._channel_registry)
                else:
                    raise ValueError("Channel with name '%s' not supported." % channel.NAME)

                if extract_features:
                    self.timeholder.apply_features(channel)

        if apply:
            # want apply also the pseudo channels
            for channel in sorted(self._channel_registry.values()):
                self.timeholder.apply_channel(channel)

    def purge(self, features=None):
        for channel in self._channel_registry.values():
            if not features is None and channel.strChannelId in features:
                channelFeatures = features[channel.strChannelId]
            else:
                channelFeatures = None
            channel.purge(features=channelFeatures)

    def exportLabelImages(self, pathOut, compression='LZW'):
        # no segmentaion in virtual channels --> no label images
        for channel in self.proc_channels.itervalues():
            channel_id = channel.strChannelId
            for region, container in channel.containers.iteritems():
                outdir = join(pathOut, channel_id, region)
                makedirs(outdir)
                fname = join(outdir, 'P%s_T%05d.tif' %(self.P, self._iT))
                container.exportLabelImage(fname, compression)

    def getImageSize(self, name):
        oChannel = self._channel_registry[name]
        w = oChannel.meta_image.width
        h = oChannel.meta_image.height
        return (w,h)

    def render(self, strPathOut, dctRenderInfo=None,
               strFileSuffix='.jpg', strCompression='98', writeToDisc=True,
               images=None):

        lstImages = []
        if not images is None:
            lstImages += images

        if dctRenderInfo is None:
            for name, oChannel in self._channel_registry.iteritems():
                for strRegion, oContainer in oChannel.containers.iteritems():
                    strHexColor, fAlpha = oChannel.dctAreaRendering[strRegion]
                    imgRaw = oChannel.meta_image.image
                    imgCon = ccore.Image(imgRaw.width, imgRaw.height)
                    ccore.drawContour(oContainer.getBinary(), imgCon, 255, False)
                    lstImages.append((imgRaw, strHexColor, 1.0))
                    lstImages.append((imgCon, strHexColor, fAlpha))
        else:
            for channel_name, dctChannelInfo in dctRenderInfo.iteritems():
                if channel_name in self._channel_registry:
                    oChannel = self._channel_registry[channel_name]
                    if 'raw' in dctChannelInfo:
                        strHexColor, fAlpha = dctChannelInfo['raw']
                        # special casing for virtual channel to mix
                        # raw images together
                        if oChannel.is_virtual():
                            lstImages.extend(oChannel.meta_images(fAlpha))
                        else:
                            lstImages.append((oChannel.meta_image.image, strHexColor, 1.0))

                    if 'contours' in dctChannelInfo:
                        # transform the old dict-style to the new tuple-style,
                        # which allows multiple definitions for one region
                        if isinstance(dctChannelInfo['contours'], dict):
                            lstContourInfos = [(k,)+v
                                               for k,v in dctChannelInfo['contours'].iteritems()]
                        else:
                            lstContourInfos = dctChannelInfo['contours']

                        for tplData in lstContourInfos:
                            strRegion, strNameOrColor, fAlpha, bShowLabels = tplData[:4]
                            # draw contours only if region is present
                            if oChannel.has_region(strRegion):
                                if len(tplData) > 4:
                                    bThickContours = tplData[4]
                                else:
                                    bThickContours = False
                                imgRaw = oChannel.meta_image.image
                                if strNameOrColor == 'class_label':
                                    oContainer = oChannel.containers[strRegion]
                                    oRegion = oChannel.get_region(strRegion)
                                    dctLabels = {}
                                    dctColors = {}
                                    for iObjId, oObj in oRegion.iteritems():
                                        iLabel = oObj.iLabel
                                        if not iLabel is None:
                                            if not iLabel in dctLabels:
                                                dctLabels[iLabel] = []
                                            dctLabels[iLabel].append(iObjId)
                                            dctColors[iLabel] = oObj.strHexColor
                                    imgCon2 = ccore.Image(imgRaw.width, imgRaw.height)
                                    for iLabel, lstObjIds in dctLabels.iteritems():
                                        imgCon = ccore.Image(imgRaw.width, imgRaw.height)
                                        # Flip this and use drawContours with fill option enables to get black background
                                        oContainer.drawContoursByIds(lstObjIds, 255, imgCon, bThickContours, False)
#                                        oContainer.drawContoursByIds(lstObjIds, 255, imgCon, bThickContours, True)
                                        lstImages.append((imgCon, dctColors[iLabel], fAlpha))

                                        if isinstance(bShowLabels, bool) and bShowLabels:
                                            oContainer.drawTextsByIds(lstObjIds, [str(iLabel)]*len(lstObjIds), imgCon2)
                                    lstImages.append((imgCon2, '#FFFFFF', 1.0))

                                else:
                                    oContainer = oChannel.containers[strRegion]
                                    oRegion = oChannel.get_region(strRegion)
                                    lstObjIds = oRegion.keys()
                                    imgCon = ccore.Image(imgRaw.width, imgRaw.height)
                                    if not strNameOrColor is None:
                                        oContainer.drawContoursByIds(lstObjIds, 255, imgCon, bThickContours, False)
                                    else:
                                        strNameOrColor = '#FFFFFF'
                                    lstImages.append((imgCon, strNameOrColor, fAlpha))
                                    if bShowLabels:
                                        imgCon2 = ccore.Image(imgRaw.width, imgRaw.height)
                                        oContainer.drawLabelsByIds(lstObjIds, imgCon2)
                                        lstImages.append((imgCon2, '#FFFFFF', 1.0))

        if len(lstImages) > 0:
            imgRgb = ccore.makeRGBImage([x[0].getView() for x in lstImages],
                                        [ccore.RGBValue(*hex2rgb(x[1])) for x in lstImages],
                                        [x[2] for x in lstImages])

            if writeToDisc:
                strFilePath = join(strPathOut, "P%s_T%05d%s"
                                   %(self.P, self._iT, strFileSuffix))
                makedirs(strPathOut)
                ccore.writeImage(imgRgb, strFilePath, strCompression)
                self.logger.debug("* rendered image written '%s'" % strFilePath)
            else:
                strFilePath = ''
            return imgRgb, strFilePath

    def collectObjects(self, plate_id, P, sample_readers, oLearner, byTime=True):
        self.logger.debug('* collecting samples...')
        self.process(apply = False, extract_features = False)

        chname = oLearner.name
        region = oLearner.regions

        oChannel = self._channel_registry[chname]
        oContainer = oChannel.get_container(region)
        objects = oContainer.getObjects()
        # key = class_label, value = list of samples
        object_lookup = {}
        object_ids = set()

        for reader in sample_readers:
            if (byTime and P == reader.position() and self._iT in reader):
                coords = reader[self._iT]
            elif (not byTime and P in reader):
                coords = reader[P]
            else:
                coords = None

            if coords is not None:
                for data in coords:
                    label = data['iClassLabel']
                    if (label in oLearner.class_names and
                        0 <= data['iPosX'] < oContainer.width and
                        0 <= data['iPosY'] < oContainer.height):

                        center1 = ccore.Diff2D(data['iPosX'], data['iPosY'])
                        # test for obj_id "under" annotated pixel first
                        obj_id = oContainer.img_labels[center1]

                        # if not background: valid obj_id found
                        if obj_id > 0:
                            object_ids.add(obj_id)
                            try:
                                object_lookup[label].extend([obj_id])
                            except KeyError:
                                object_lookup[label] = [obj_id]
                        # otherwise try to find nearest object in a search
                        # radius of 30 pixel (compatibility with CellCounter)
                        else:
                            dists = []
                            for obj_id, obj in objects.iteritems():
                                diff = obj.oCenterAbs - center1
                                dist_sq = diff.squaredMagnitude()
                                # limit to 30 pixel radius
                                if dist_sq < 900:
                                    dists.append((obj_id, dist_sq))
                            if len(dists) > 0:
                                dists.sort(lambda a,b: cmp(a[1], b[1]))
                                obj_id = dists[0][0]
                                object_ids.add(obj_id)
                                try:
                                    object_lookup[label].extend([obj_id])
                                except KeyError:
                                    object_lookup[label] = [obj_id]

        objects_del = set(objects.keys()) - object_ids
        for obj_id in objects_del:
            oContainer.delObject(obj_id)

        # calculate features of sub channels first
        # want invoke time_holder for hdf5
        if oChannel.is_virtual():
            for mchannel, _ in oChannel.sub_channels():
                self.timeholder.apply_features(mchannel)

        self.timeholder.apply_features(oChannel)
        training_set = self.annotate(object_lookup,
                                     oLearner, oContainer,
                                     oChannel.get_region(region))

        if oChannel.is_virtual():
            images = {}
            for s_ch, reg in oChannel.sub_channels():
                rid = "%s_%s" %(s_ch.NAME, reg)
                self.draw_annotation_images(plate_id, training_set,
                                   s_ch.get_container(reg), oLearner, rid)
                images.update(self.write_annotation_images(s_ch, reg, oLearner))

        else:
            self.draw_annotation_images(plate_id, training_set, oContainer, oLearner)
            images = self.write_annotation_images(oChannel, region, oLearner)

        if training_set:
            oLearner.set_training_data(training_set)

        return images

    def write_annotation_images(self, channel, region, learner):
        cnt = channel.get_container(region)
        images = {}
        if isinstance(region, tuple):
            region = "-".join(region)
        name = join(learner.controls_dir, "P%s_T%05d_C%s_R%s.jpg"
                    %(self.P, self._iT, learner.color_channel, region))
        cnt.exportRGB(name, "90")
        images["%s_%s" %(channel.NAME.lower(), region)] = cnt.img_rgb
        return images

    def draw_annotation_images(self, plate, training_set, container, learner, rid=""):
        cldir = dict([(cname, join(learner.samples_dir, cname)) \
                          for cname in learner.class_names.values()])
        # create dir per class name
        for dir_ in cldir.values():
            makedirs(dir_)

        for obj in training_set.itervalues():
            rgb_value = ccore.RGBValue(*hex2rgb(obj.strHexColor))

            file_ = 'PL%s___P%s___T%05d___X%04d___Y%04d' \
                %(plate, self.P, self._iT,
                  obj.oCenterAbs[0], obj.oCenterAbs[1])
            obj.file = file_
            file_ = join(cldir[obj.strClassName],
                         '%s___%s.png' %(file_, rid+"_%s"))
            container.exportObject(obj.iId, file_ %"img", file_ %"msk")
            container.markObjects([obj.iId], rgb_value, False, True)
            ccore.drawFilledCircle(ccore.Diff2D(*obj.oCenterAbs),
                                   3, container.img_rgb, rgb_value)

    def annotate(self, sample_objects, learner, container, region):
        """Annotate predefined class labels to picked samples."""

        training_set = ObjectHolder(region.name)
        training_set.feature_names = region.feature_names

        for class_label, object_ids in sample_objects.iteritems():
            class_name = learner.class_names[class_label]
            hex_color = learner.hexcolors[class_name]

            for obj_id in object_ids:
                obj = region[obj_id]
                obj.iLabel = class_label
                obj.strClassName = class_name
                obj.strHexColor = hex_color
                training_set[obj_id] = obj
        return training_set

    def classify_objects(self, predictor):
        channel = self._channel_registry[predictor.name]
        holder = channel.get_region(predictor.regions)

        try:
            signal_idx = holder.feature_names.index('n2_avg')
            roisize_idx = holder.feature_names.index('roisize')
            has_basic_features = True
        except ValueError:
            has_basic_features = False

        for label, obj in holder.iteritems():
            if obj.aFeatures.size != len(holder.feature_names):
                msg = ('Incomplete feature set found (%d/%d): skipping sample '
                       'object label %s'
                       %(obj.aFeatures.size, len(holder.feature_names), label))
                self.logger.warning(msg)
            else:
                label, probs = predictor.predict(obj.aFeatures, holder.feature_names)
                obj.iLabel = label
                obj.dctProb = probs
                obj.strClassName = predictor.class_names[label]
                obj.strHexColor = predictor.hexcolors[obj.strClassName]
                if has_basic_features:
                    obj.roisize = obj.aFeatures[roisize_idx]
                    obj.signal = obj.aFeatures[signal_idx]
Ejemplo n.º 45
0
class SubsectionGradeFactory(object):
    """
    Factory for Subsection Grades.
    """
    def __init__(self, student, course=None, course_structure=None, course_data=None):
        self.student = student
        self.course_data = course_data or CourseData(student, course=course, structure=course_structure)

        self._cached_subsection_grades = None
        self._unsaved_subsection_grades = OrderedDict()

    def create(self, subsection, read_only=False, force_calculate=False):
        """
        Returns the SubsectionGrade object for the student and subsection.

        If read_only is True, doesn't save any updates to the grades.
        force_calculate - If true, will cause this function to return a `CreateSubsectionGrade` object if no cached
        grade currently exists, even if the assume_zero_if_absent flag is enabled for the course.
        """
        self._log_event(
            log.debug, u"create, read_only: {0}, subsection: {1}".format(read_only, subsection.location), subsection,
        )

        subsection_grade = self._get_bulk_cached_grade(subsection)
        if not subsection_grade:
            if assume_zero_if_absent(self.course_data.course_key) and not force_calculate:
                subsection_grade = ZeroSubsectionGrade(subsection, self.course_data)
            else:
                subsection_grade = CreateSubsectionGrade(
                    subsection, self.course_data.structure, self._submissions_scores, self._csm_scores,
                )
                if should_persist_grades(self.course_data.course_key):
                    if read_only:
                        self._unsaved_subsection_grades[subsection_grade.location] = subsection_grade
                    else:
                        grade_model = subsection_grade.update_or_create_model(self.student)
                        self._update_saved_subsection_grade(subsection.location, grade_model)
        return subsection_grade

    def bulk_create_unsaved(self):
        """
        Bulk creates all the unsaved subsection_grades to this point.
        """
        CreateSubsectionGrade.bulk_create_models(
            self.student, list(self._unsaved_subsection_grades.values()), self.course_data.course_key
        )
        self._unsaved_subsection_grades.clear()

    def update(self, subsection, only_if_higher=None, score_deleted=False, force_update_subsections=False, persist_grade=True):
        """
        Updates the SubsectionGrade object for the student and subsection.
        """
        self._log_event(log.debug, u"update, subsection: {}".format(subsection.location), subsection)

        calculated_grade = CreateSubsectionGrade(
            subsection, self.course_data.structure, self._submissions_scores, self._csm_scores,
        )

        if persist_grade and should_persist_grades(self.course_data.course_key):
            if only_if_higher:
                try:
                    grade_model = PersistentSubsectionGrade.read_grade(self.student.id, subsection.location)
                except PersistentSubsectionGrade.DoesNotExist:
                    pass
                else:
                    orig_subsection_grade = ReadSubsectionGrade(subsection, grade_model, self)
                    if not is_score_higher_or_equal(
                        orig_subsection_grade.graded_total.earned,
                        orig_subsection_grade.graded_total.possible,
                        calculated_grade.graded_total.earned,
                        calculated_grade.graded_total.possible,
                        treat_undefined_as_zero=True,
                    ):
                        return orig_subsection_grade

            grade_model = calculated_grade.update_or_create_model(
                self.student,
                score_deleted,
                force_update_subsections
            )
            self._update_saved_subsection_grade(subsection.location, grade_model)

        return calculated_grade

    @lazy
    def _csm_scores(self):
        """
        Lazily queries and returns all the scores stored in the user
        state (in CSM) for the course, while caching the result.
        """
        scorable_locations = [block_key for block_key in self.course_data.structure if possibly_scored(block_key)]
        return ScoresClient.create_for_locations(self.course_data.course_key, self.student.id, scorable_locations)

    @lazy
    def _submissions_scores(self):
        """
        Lazily queries and returns the scores stored by the
        Submissions API for the course, while caching the result.
        """
        anonymous_user_id = anonymous_id_for_user(self.student, self.course_data.course_key)
        return submissions_api.get_scores(str(self.course_data.course_key), anonymous_user_id)

    def _get_bulk_cached_grade(self, subsection):
        """
        Returns the student's SubsectionGrade for the subsection,
        while caching the results of a bulk retrieval for the
        course, for future access of other subsections.
        Returns None if not found.
        """
        if should_persist_grades(self.course_data.course_key):
            saved_subsection_grades = self._get_bulk_cached_subsection_grades()
            grade = saved_subsection_grades.get(subsection.location)
            if grade:
                return ReadSubsectionGrade(subsection, grade, self)

    def _get_bulk_cached_subsection_grades(self):
        """
        Returns and caches (for future access) the results of
        a bulk retrieval of all subsection grades in the course.
        """
        if self._cached_subsection_grades is None:
            self._cached_subsection_grades = {
                record.full_usage_key: record
                for record in PersistentSubsectionGrade.bulk_read_grades(self.student.id, self.course_data.course_key)
            }
        return self._cached_subsection_grades

    def _update_saved_subsection_grade(self, subsection_usage_key, subsection_model):
        """
        Updates (or adds) the subsection grade for the given
        subsection usage key in the local cache, iff the cache
        is populated.
        """
        if self._cached_subsection_grades is not None:
            self._cached_subsection_grades[subsection_usage_key] = subsection_model

    def _log_event(self, log_func, log_statement, subsection):
        """
        Logs the given statement, for this instance.
        """
        log_func(u"Grades: SGF.{}, course: {}, version: {}, edit: {}, user: {}".format(
            log_statement,
            self.course_data.course_key,
            getattr(subsection, 'course_version', None),
            getattr(subsection, 'subtree_edited_on', None),
            self.student.id,
        ))
Ejemplo n.º 46
0
class Cache:
    """
    A python class which used ordered dictionary (OrderedDict) to implement the LRU cache.
    Each entry of the dictinoary will be a key/value pair. The search would be 
    by key. LRU Cache will have its maximum size defined at initiation. When adding 
    new keys that cause the capacity to be exceed the size defined at initialtion, 
    the oldest items will be removed to make room. The newly added items/last accessed 
    items will be moved to the back of the dictonary (most recently used) and the elements 
    at the begining will correspond to lest recently used. The element at the begining of the 
    dictionary will be the one to discard when the cache if full (least recently used)
    
    Methods:
        get(key) - Get the value corresponding to key
        put(key,value) - Insert key/value into the cache
        delKey(key) - Delete the key
        reset() - Clear the cache
        dumpCache() - Print the cache contents
    """
    def __init__(self, size, verbose=False):
        """
        Initialize new cache object with the size passed.
        Args:
            size (int): The max size of the cache.
        """
        self.printDebug = verbose
        self.sizeOfCache = size
        self.cacheLRU = OrderedDict()

    def get(self, key):
        """
        Returns the value corresponding to the key provided. 
        If the key does not exit, it returns None else it moves
        the key to the end of the dictionary using move_to_end 
        (method of OrderedDict)
        Args:
            key (int): The key to lookup the value
        Returns:
            value (int): The value corresponding to the key if found else None
        """
        if self.sizeOfCache == 0:
            if self.printDebug:
                print("Zero capacity cache and get request. Raise exception.")
            raise Exception('Cache is not defined')
        else:
            if key not in self.cacheLRU:
                return None
            else:
                self.cacheLRU.move_to_end(key)
                return self.cacheLRU[key]

    def put(self, key, value):
        """
        Inserts the key/value pair to the cache. If the cache is 
        already full (max capacity), it will remove the element at
        the head of the dict (least recently used) using popitem and 
        insert the key/value pair at the end of the dict (most recently used).
        If the key already exists, it will not fail but mark the item 
        as recently used (move it to the back of the dict)
        Returns:
            Nothing or exception if cache capacity is 0
        """
        if key in self.cacheLRU:
            if self.printDebug:
                print(
                    "Key {} already exists in cache. Update this and mark as recently used"
                    .format(key))
            self.cacheLRU[key] = value
            self.cacheLRU.move_to_end(key)
        else:
            if self.sizeOfCache == 0:
                if self.printDebug:
                    print(
                        "Zero capacity cache and put request. Raise exception."
                    )
                raise Exception('Cache is not defined')
            else:
                if len(self.cacheLRU) >= self.sizeOfCache:
                    outKey, outVal = self.cacheLRU.popitem(last=False)
                    if self.printDebug:
                        print("Cache at capacity of {}. Removing LRU key {}".
                              format(self.sizeOfCache, outKey))
                self.cacheLRU[key] = value
                self.cacheLRU.move_to_end(key)

    def delKey(self, key):
        """
        Delete the key from the cache if it exists. If the key does not exist
        nothing happens. The delete is treated as a cache hit and the key is moved
        to the end (most recently used) and then removed. 
        Returns:
            Nothing or exception if cache capacity is 0
        """
        if self.sizeOfCache == 0:
            if self.printDebug:
                print("Zero capacity cache and del request. Raise exception.")
            raise Exception('Cache is not defined')
        else:
            if key in self.cacheLRU:
                self.cacheLRU.move_to_end(key)
                self.cacheLRU.popitem(last=True)
            else:
                if self.printDebug:
                    print(
                        "Key {} to delete does not exist in the cache. Doing nothing.\n"
                        .format(self.sizeOfCache))

    def reset(self):
        """
        Reset the cache
        Returns:
            Nothing or exception if cache capacity is 0
        """
        if self.sizeOfCache == 0:
            if self.printDebug:
                print(
                    "Zero capacity cache and reset request. Raise exception.")
            raise Exception('Cache is not defined')
        else:
            self.cacheLRU.clear()

    def dumpCache(self):
        """
        Print the contents of the cache
        Returns:
            Nothing
        """
        print(self.cacheLRU)
Ejemplo n.º 47
0
class Graph(object):
    """This graph class stores the nodes and will evaluate the graph on request.
    Simple example:
    gx = Graph("newGraph")
    t = sumNode.SumNode("newMathNode") # first create an instance of the node
    gx.addNode(t) # adds a node to the graph
    gx.getNode("newMathNode") # gets node by name
    if t in gx:
        print t.name # check to see if newNode is in the graph
    """
    addedNode = vortexEvent.VortexSignal()
    removedNode = vortexEvent.VortexSignal()
    addedEdge = vortexEvent.VortexSignal()
    deletedEdge = vortexEvent.VortexSignal()

    def __init__(self, name=""):
        """
        :param name: str, the name of the graph
        """
        self._edges = OrderedDict()
        self._name = name
        self._nodes = OrderedDict()

    def __repr__(self):
        return "{}{}".format(self.__class__.__name__, self.__dict__)

    def __len__(self):
        """Returns the length of the nodes in the graph
        :return: int, the length of nodes in this graph
        """
        return len(self._nodes)

    def __eq__(self, other):
        return isinstance(
            other, Graph
        ) and self._nodes == other.nodes and self._edges == other.edges

    def __contains__(self, node):
        """Returns a bool if the node is in the graph
        :param node: BaseNode instance
        :return:bool
        """
        try:
            return node in self._nodes.values()
        except TypeError:
            return False

    def get(self, fullPath):
        """Returns the node/edge/plug based on the fullPath rg. "testNode|output" would  the output plug
        :param fullPath: str
        :return: Node,Plug,edge
        """
        for node in self._nodes.values():
            if node.fullPath() == fullPath:
                return node
            for plug in node.plugs.values():
                if plug.fullPath() == fullPath:
                    return plug
        for edge in self._edges.values():
            if edge.fullPath() == fullPath:
                return edge

    def addNode(self, node, **kwargs):
        """Adds a Node instance to the graph this will also add the node to the graph class instance as a attribute
        which can be accessed by graph.node
        :param node: Node instance, the node to add
        :param kwargs: plugName=plugValue, the kwargs sets the input plugs value.
        :return Node instance
        """

        if self.hasNode(node):
            return
        node.name = self.generateUniqueName(node)
        self._nodes[node.name] = node
        for plugName, plugValue in kwargs.iteritems():
            plug = node.getPlug(plugName)
            if plug.isInput():
                plug.value = plugValue
        for plug in node.plugs.values():
            plug.connectionAdded.connect(self.addEdge)
            plug.connectionRemoved.connect(self.deleteEdge)
        self.addedNode.emit(node)
        return node

    @property
    def nodes(self):
        """Returns all the nodes in the graph
        :return: OrderedDict
        """
        return self._nodes

    @nodes.setter
    def nodes(self, newNodes):
        """Empties and sets the nodes dict
        :param newNodes: dict
        """
        self._nodes = newNodes

    def hasNode(self, node):
        """Checks the graph for the given node _name
        :param node: node instance
        :return: bool
        """
        return node in self._nodes.values()

    def deleteNode(self, node):
        """Removes a node from the graph
        :param node: the node instance to delete
        """
        if isinstance(node, str):
            node = self.getNode(node)
        node.disconnectAll()
        del self._nodes[node.name]
        self.removedNode.emit(node)

    def getNode(self, nodeName):
        """Returns a node based on the name or empty list
        :param nodeName: the name of the node to get
        :return:Node instance
        """
        return self._nodes.get(nodeName)

    @property
    def edges(self):
        """Returns a dict of the graph edges
        :return: Dict
        """
        return self._edges

    def getEdge(self, edgeName):
        """Returns a edge if the edge name is in edges
        :param edgeName: str, th name of the edge
        :return:
        """
        return self._edges.get(edgeName)

    def addEdge(self, edge):
        """Adds the edge to the graph
        :param edge: Edge
        """
        print "trying to add edge"
        if isinstance(edge, str):
            edge = self.getEdge(edge)

        if edge not in self._edges.values():
            self._edges[edge.name] = edge
            print "creating edge"
            self.addedEdge.emit(edge)

    def deleteEdge(self, edge):
        """Removes the edge from the graph
        :param edge: Edge
        """
        if edge in self._edges.values():
            tmpEdge = edge
            del self._edges[edge.name]
            self.deletedEdge.emit(tmpEdge)

    def generateUniqueName(self, node):
        """Create a unique name for the node in the graph, on node creation a digit is appended , eg nodeName00, nodeName01
        :param node: node Instance
        :return: str, returns the new node name as a string
        """
        value = "%0{}d".format(0)
        uIndex = 0
        currentIndex = [int(i) for i in node.name if i.isdigit()]
        if currentIndex:
            name = node.name.split(str(sum(currentIndex)))[0]
        else:
            name = node.name
        while name in self._nodes:
            name = name + value % uIndex
            uIndex += 1
        return name

    def clear(self):
        """Clears all the nodes and the edges from the graph
        :return: None
        """
        self._nodes.clear()
        self._edges.clear()

    def allLeaves(self):
        """Returns all the leaf nodes in the graph, a Leaf node is any node that has on connections
        :return: list(Node)
        """
        leafNodes = []

        for node in self._nodes.values():
            if not all(plug.connections for plug in node.inputs()):
                leafNodes.append(node)

        return leafNodes

    def serializeGraph(self):
        """Creates a python dict from the graph, each node,plug and edge gets serialized
        :return:
        """
        logger.debug("serializing graph")
        serializedGraph = {
            "name": self._name,
            "version": "1.0.0",
            "nodes": OrderedDict(),
            "edges": dict(),
            "moduleName": inspect.getmodulename(__file__)
        }
        logger.debug(serializedGraph)
        for node in self._nodes.values():
            serializedGraph["nodes"][node.name] = node.serialize()
        for edge in self._edges.values():
            serializedGraph["edges"][edge.name] = edge.serialize()

        return serializedGraph

    @classmethod
    def loadGraph(cls, graphData):
        """load a vortex graph dict creates and returns a graph object
        :param graphData: dict
        :return: Graph()
        """
        graph = cls(name=graphData.get("name"))
        for node in graphData["nodes"].values():
            moduleName = node.get("moduleName")
            nodeName = node.get("name")

            if moduleName == "baseNode":
                newNode = baseNode.BaseNode(name=nodeName)
            else:
                newNode = allNodes.getNode(
                    node.get("moduleName"))(name=nodeName)
            for plugName, values in node.get("plugs").iteritems():
                plug = newNode.getPlug(plugName=plugName)
                if plug:
                    plug.value = values.get("value")
                    continue
                newNode.addPlugByType(ioType=values.get("io"),
                                      name=plugName,
                                      value=values.get("value"))
            graph.addNode(newNode)

        for edge in graphData["edges"].values():
            inputPlug = graph.getNode(edge["input"][1]).getPlug(
                edge["input"][0])
            outputPlug = graph.getNode(edge["output"][1]).getPlug(
                edge["output"][0])
            graph.addEdge(
                baseEdge.Edge(name=edge["name"],
                              inputPlug=inputPlug,
                              outputPlug=outputPlug))

        return graph
class BaseSelectedFilters(object):
    __slots__ = ('__filters', '_selectedFilters')

    def __init__(self):
        self._selectedFilters = []
        self.__filters = OrderedDict()

    def changeFilter(self, filterName, model):
        filters = model.getFilters()
        self._selectedFilters = []
        for filterModel in filters:
            if filterName == filterModel.getName():
                filterModel.setIsSelected(not filterModel.getIsSelected())
            if filterModel.getIsSelected():
                self._selectedFilters.append(filterModel.getName())

        filters.invalidate()
        if not self._selectedFilters:
            model.setIsEnabled(False)
        else:
            model.setIsEnabled(True)

    def initFilters(self, model):
        filters = model.getFilters()
        filters.clear()
        for filterName in self.__filters:
            filterModel = self._createSubFilterModel(filterName)
            filters.addViewModel(filterModel)

        filters.invalidate()

    def resetFilters(self, model):
        for filterModel in model.getFilters():
            filterModel.setIsSelected(False)

        self._selectedFilters = []
        model.setIsEnabled(False)
        model.getFilters().invalidate()

    def getItems(self, items, model):
        filteredItems = self._getFilteredItems(items)
        model.setSelectedFilterCount(len(filteredItems))
        model.setTotalFilterCount(len(items))
        return filteredItems

    def checkItem(self, item):
        result = True
        for idx, filterName in enumerate(self._selectedFilters):
            filterData = self.__filters[filterName]
            if idx == 0 and filterData.operator is operator.or_:
                result = False
            result = filterData.operator(
                self.__filters[filterName].method(item), result)

        return result

    def clear(self):
        self.__filters = None
        self._selectedFilters = None
        return

    def _getFilteredItems(self, items):
        result = tuple((item for item in items if self.checkItem(item)))
        return result

    @staticmethod
    def _createSubFilterModel(filterName):
        filterModel = SubFilterModel()
        filterModel.setName(filterName)
        filterModel.setIsSelected(False)
        return filterModel

    def _addFilter(self, filterName, filterMethod, op=operator.and_):
        self.__filters[filterName] = _FilterData(filterMethod, op)

    def _delFilter(self, filterName):
        del self.__filters[filterName]

    def _clearFilters(self):
        self.__filters.clear()
Ejemplo n.º 49
0
class BufferPoolManager:
    """
    Create a buffer pool manager that maps hot :class:`Document` s of a :class:`DocumentArrayMemmap` to a memory buffer.

    This helps keep access to memory-loaded :class:`Document` instances synced with :class:`DocumentArrayMemmap` values.
    The memory buffer has a fixed size and uses an LRU strategy to empty spots when full.
    """
    def __init__(self, pool_size: int = 1000):
        self.pool_size = pool_size
        self.doc_map = OrderedDict()  # dam_idx: (buffer_idx, version)
        self.buffer = []
        self._empty = []

    def add_or_update(self, idx: str,
                      doc: 'Document') -> Optional[Tuple[str, 'Document']]:
        """
        Adds a document to the buffer pool or updates it if it already exists

        :param idx: index
        :param doc: document

        :return: returns a couple of ID and :class:`Document` if there's a document to persist
        """
        result = None
        # if document is already in buffer, update it
        if idx in self.doc_map:
            self.buffer[self.doc_map[idx][0]] = doc
            self.doc_map.move_to_end(idx)
        # else, if len is less than the size, append to buffer
        elif len(self.buffer) < self.pool_size:
            self.doc_map[idx] = (len(self.buffer), doc.version)
            self.doc_map.move_to_end(idx)
            self.buffer.append(doc)
        # else, if buffer has empty spots, allocate them
        elif self._empty:
            empty_idx = self._empty.pop()
            self.doc_map[idx] = (empty_idx, doc.version)
            self.buffer[empty_idx] = doc
            self.doc_map.move_to_end(idx)
        # else, choose a spot to free and use it with LRU strategy
        else:
            # the least recently used item is the first item in doc_map
            dam_idx, (buffer_idx, version) = self.doc_map.popitem(last=False)
            if version != self.buffer[buffer_idx].version:
                result = dam_idx, self.buffer[buffer_idx]
            self.doc_map[idx] = (buffer_idx, doc.version)
            self.doc_map.move_to_end(idx)
            self.buffer[buffer_idx] = doc

        return result

    def delete_if_exists(self, key):
        """
        Adds a document to the buffer pool or updates it if it already exists

        :param key: document key
        """
        if key in self:
            del self[key]

    def docs_to_flush(self) -> List[Tuple[str, 'Document']]:
        """
        Persists the updated documents in disk

        :return: returns a list of documents to be flushed
        """
        result = []
        for dam_idx, (buffer_idx, version) in self.doc_map.items():
            doc = self.buffer[buffer_idx]
            if version != doc.version:
                result.append((dam_idx, self.buffer[buffer_idx]))
        return result

    def clear(self):
        """
        Clears the memory buffer
        """
        self.doc_map.clear()
        self.buffer = []

    def __getitem__(self, key: str):
        if isinstance(key, str):
            doc = self.buffer[self.doc_map[key][0]]
            self.doc_map.move_to_end(key)
            return doc
        else:
            raise TypeError(f'`key` must be str, but receiving {key!r}')

    def __delitem__(self, key):
        buffer_idx, _ = self.doc_map.pop(key)
        self._empty.append(buffer_idx)

    def __contains__(self, key):
        return key in self.doc_map
Ejemplo n.º 50
0
class SimpleClock(Checkpointable, Configurable):
    def __init__(self, A, skip_load=None, **kwargs):

        if skip_load is None:
            skip_load = A.pull('skip_load', False)

        ckpt = None
        if not skip_load:
            ckpt = A.pull('_load-ckpt', None)

        super().__init__(A, **kwargs)
        self.ticks = 0
        self.alerts = OrderedDict()
        self._info = None

        self._skip_load = None

        if ckpt is not None:
            self.load_checkpoint(ckpt)

    def prep(self, info=None):
        if info is None:
            info = self._info
        for idx, (name, alert) in enumerate(self.alerts.items()):
            alert.prep(idx, info=info)

    def sort_alerts(self, start_with=None, end_with=None, strict=True):
        order = OrderedDict()

        if start_with is not None:
            for name in start_with:
                if name in self.alerts:
                    order[name] = self.alerts[name]
                elif strict:
                    raise AlertNotFoundError(name)

        for name, alert in self.alerts.items():
            if end_with is None or name not in end_with:
                order[name] = alert

        if end_with is not None:
            for name in end_with:
                if name in self.alerts:
                    order[name] = self.alerts[name]
                elif strict:
                    raise AlertNotFoundError(name)

        self.alerts.clear()
        self.alerts.update(order)

    def get_info(self):
        return self._info

    def set_info(self, info):
        self._info = info

    def register_alert_fn(self, name, check=None, activate=None):
        self.register_alert(name, CustomAlert(check=check, activate=activate))

    def register_alert(self, name, alert, **unused):
        if name is None:
            name = f'{alert}#{id(alert)}'
        self.alerts[name] = alert

    def checkpoint(self, path, ident='clock'):

        path = Path(path) / f'{ident}.json'

        states = {}
        for name, alert in self.alerts.items():
            if isinstance(alert, Savable):
                states[name] = alert.state_dict()

        save_json({'time': self.get_time(), 'alert_states': states}, path)

    def load_checkpoint(self, path, ident='clock'):

        path = Path(path) / f'{ident}.json'

        if path.is_file():
            data = load_json(path)

            self.ticks = data['time']

            states = data['alert_states']
            for name, state in states.items():
                if name in self.alerts:
                    self.alerts[name].load_state_dict(state)

    def clear(self):
        self.alerts.clear()
        self._info = None

    def _call_alert(self, name, alert, info=None):
        if info is None:
            info = self._info
        return alert.activate(self.ticks, info)

    def get_time(self):
        return self.ticks

    def __len__(self):
        return self.get_time()

    def set_time(self, ticks):
        self.ticks = ticks

    def tick(self, info=None):
        self.ticks += 1
        for name, alert in self.alerts.items():
            if alert.check(self.ticks, info):
                self._call_alert(name, alert, info=info)

    def step(self, info=None, n=None):
        if n is None:
            n = 1
        for _ in range(n):
            self.tick(info=info)
Ejemplo n.º 51
0
class LRUCacheDict(object):
    """ A dictionary-like object, supporting LRU caching semantics.

    >>> d = LRUCacheDict(max_size=3, expiration=3)
    >>> d['foo'] = 'bar'
    >>> d['foo']
    'bar'
    >>> import time
    >>> time.sleep(4) # 4 seconds > 3 second cache expiry of d
    >>> d['foo']
    Traceback (most recent call last):
        ...
    KeyError: 'foo'
    >>> d['a'] = 'A'
    >>> d['b'] = 'B'
    >>> d['c'] = 'C'
    >>> d['d'] = 'D'
    >>> d['a'] # Should return value error, since we exceeded the max cache size
    Traceback (most recent call last):
        ...
    KeyError: 'a'

    By default, this cache will only expire items whenever you poke it - all methods on
    this class will result in a cleanup. If the thread_clear option is specified, a background
    thread will clean it up every thread_clear_min_check seconds.

    If this class must be used in a multithreaded environment, the option concurrent should be
    set to true. Note that the cache will always be concurrent if a background cleanup thread
    is used.
    """

    def __init__(self, max_size=1024, expiration=15 * 60):
        self.max_size = max_size
        self.expiration = expiration

        self.__values = {}
        self.__expire_times = OrderedDict()
        self.__access_times = OrderedDict()

    def size(self):
        return len(self.__values)

    def clear(self):
        """
        Clears the dict.

        >>> d = LRUCacheDict(max_size=3, expiration=1)
        >>> d['foo'] = 'bar'
        >>> d['foo']
        'bar'
        >>> d.clear()
        >>> d['foo']
        Traceback (most recent call last):
        ...
        KeyError: 'foo'
        """
        self.__values.clear()
        self.__expire_times.clear()
        self.__access_times.clear()

    def __contains__(self, key):
        return key in self

    def has_key(self, key):
        """
        This method should almost NEVER be used. The reason is that between the time
        has_key is called, and the key is accessed, the key might vanish.

        You should ALWAYS use a try: ... except KeyError: ... block.

        >>> d = LRUCacheDict(max_size=3, expiration=1)
        >>> d['foo'] = 'bar'
        >>> d['foo']
        'bar'
        >>> import time
        >>> if d.has_key('foo'):
        ...    time.sleep(2) #Oops, the key 'foo' is gone!
        ...    d['foo']
        Traceback (most recent call last):
        ...
        KeyError: 'foo'
        """
        return key in self.__values

    def __setitem__(self, key, value):
        t = int(time.time())
        self.__delete__(key)
        self.__values[key] = value
        self.__access_times[key] = t
        self.__expire_times[key] = t + self.expiration
        self.cleanup()

    def __getitem__(self, key):
        del self.__access_times[key]
        self.__access_times[key] = int(time.time())
        self.cleanup()
        return self.__values[key]

    def __delete__(self, key):
        if key in self.__values:
            del self.__values[key]
            del self.__expire_times[key]
            del self.__access_times[key]

    def cleanup(self):
        if self.expiration is None:
            return None

        key_need_deleted = []
        for k, expire_time in self.__expire_times.items():
            if expire_time < int(time.time()):
                key_need_deleted.append(k)
            else:
                break

        for key in key_need_deleted:
            self.__delete__(key)

        # If we have more than self.max_size items, delete the oldest
        key_need_deleted = []
        delete_size = len(self.__values) - self.max_size
        for k in self.__access_times:
            if delete_size > 0:
                key_need_deleted.append(k)
                delete_size -= 1
            else:
                break

        for key in key_need_deleted:
            self.__delete__(key)
Ejemplo n.º 52
0
class Scope:
    def __init__(self, parent=None, name=None):
        self.children = OrderedDict()
        self.index = {}
        self.records = OrderedDict()
        if name == None:
            self.name = self.full_name = ""
        else:
            self.name = name
            self.full_name = parent.full_name + name + "/"

    def get_scope(self, name, unique=True):
        if not unique:
            index = self.index.get(name, 0)
            self.index[name] = index + 1
            name = name + f'_{index}'
        if name not in self.children:
            sub_scope = Scope(self, name)
            self.children[name] = sub_scope
        else:
            sub_scope = self.children[name]
            assert type(
                sub_scope) == Scope, f"Name {name} is a Var: {sub_scope}"
        return sub_scope

    def make_var(self, shape, dtype, init, name, unique):
        if not unique:
            index = self.index.get(name, 0)
            self.index[name] = index + 1
            name = name + f'_{index}'
        if name in self.children:
            var = self.children[name]
            assert type(var) == core.Var, f"Name {name} exist: {var}"
            assert (
                shape is None or var.shape == shape
            ) and var.dtype == dtype, f"Shape or dtype not match {var} != {dtype}{shape}"
            return var
        else:
            full_name = self.full_name + name
            if type(init) != core.Var:
                if callable(init):
                    var = init(shape, dtype)
                    if type(var) != core.Var:
                        var = array(var)
                else:
                    assert init != None
                    var = array(init)
            else:
                var = init
            var.stop_fuse()
            self.children[name] = var
            var.name(full_name)
            return var

    def clean_index(self):
        self.index.clear()

    def clean(self):
        self.children.clear()
        self.records.clear()
        self.index.clear()
Ejemplo n.º 53
0
class _StatsSection(object):
    """
    Not intended to be directly instantiated
    This is the type for the SolverStats.sections values
    
    The method parameter descriptions are the same as for those the parent 
    with the same method name
    
    Parameters
    ----------
    name : string
        key for the parent sections dictionary
        will also be used as the header
    
    parent : `class` :  SolverStats
        The container for all the sections
        
    Attributes
    ----------
    name : string
        key for the parent sections dictionary
        will also be used as the header
    
    parent : `class` :  SolverStats
        The container for all the sections
        
    header : string
        Used as heading for section in report
        
    counts : OrderedDict
        The integer type statistics for the stats section
        
    timings : OrderedDict
        The timing type statistics for the stats section
        Expected to contain float values representing values in seconds
        
    messages : OrderedDict
        Text type output to be reported
    
    total_time : float
        Total time for processing in the section
        Can be None, meaning that section timing percentages will be reported
    """
    def __init__(self, name, parent):
        self.parent = parent
        self.header = str(name)
        self.name = name
        self.counts = OrderedDict()
        self.timings = OrderedDict()
        self.messages = OrderedDict()
        self.total_time = None

    def add_count(self, key, value):
        """
        Add value to count. If key does not already exist in section then
        it is created with this value.
        If key already exists it is increased by the given value
        value is expected to be an integer
        """
        if not isinstance(value, int):
            try:
                value = int(value)
            except:
                raise TypeError("value is expected to be an integer")

        if key in self.counts:
            self.counts[key] += value
        else:
            self.counts[key] = value

    def add_timing(self, key, value):
        """
        Add value to timing. If key does not already exist in section then
        it is created with this value.
        If key already exists it is increased by the give value
        value is expected to be a float, and given in seconds.
        """
        if not isinstance(value, float):
            try:
                value = float(value)
            except:
                raise TypeError("value is expected to be a float")

        if key in self.timings:
            self.timings[key] += value
        else:
            self.timings[key] = value

    def add_message(self, key, value, sep=";"):
        """
        Add value to message. If key does not already exist in section then
        it is created with this value.
        If key already exists the value is added to the message
        The value will be converted to a string
        """
        value = str(value)

        if key in self.messages:
            if sep is not None:
                try:
                    value = sep + value
                except:
                    TypeError(
                        "It is not possible to concatenate the value with "
                        "the given seperator")
            self.messages[key] += value
        else:
            self.messages[key] = value

    def report(self, output=sys.stdout):
        """
        Report the counts, timings and messages for this section.
        Note the percentage of the section and solver total times will be
        given if the parent and or section total_time is set
        """
        if self.header:
            output.write("\n{}\n{}\n".format(self.header,
                                             ("-" * len(self.header))))

        # TODO: Make the timings and counts ouput in a table format
        #       Generally make more pretty

        # Report timings
        try:
            ttt = self.parent.total_time
        except:
            ttt = None

        tt = self.total_time

        output.write("### Timings:\n")
        for key, value in self.timings.items():
            l = " - {}\t{}\n".format(_format_time(value, tt, ttt), key)
            output.write(l)
        if tt is not None:
            output.write(" - {}\t{} total time\n".format(
                _format_time(tt), self.name))

        # Report counts
        output.write("### Counts:\n")
        for key, value in self.counts.items():
            l = " - {}\t{}\n".format(value, key)
            output.write(l)

        # Report messages
        output.write("### Messages:\n")
        for key, value in self.messages.items():
            l = " - {}:\t{}\n".format(key, value)
            output.write(l)

    def clear(self):
        """
        Clear counts, timings and messages from this section
        """
        self.counts.clear()
        self.timings.clear()
        self.messages.clear()
        self.total_time = None
Ejemplo n.º 54
0
class JdParser(object):

    def __init__(self):
        self.degreedic = set( line.strip() for line in codecs.open('./data/degrees.txt','rb','utf-8')) # 载入学历词库
        self.majordic =set( line.strip() for line in codecs.open('./data/majordic.txt','rb','utf-8')) # 载入专业词库
        self.citydic = set( line.strip() for line in codecs.open("./data/citydic.txt",'rb','utf-8'))   # 载入城市词库
        self.firmnames =set( line.strip() for line in codecs.open('./data/firm.txt','rb','utf-8'))    # 载入公司缩写名库
        self.jobdic = set(line.strip() for line in codecs.open('./data/jobposition.txt','rb','utf-8') ) # 载入招聘职位名库
        self.skills = set( line.strip() for line in codecs.open('./data/skills.txt','rb','utf-8'))
#        self.wordlisttf = pickle.load(open('./data/wordlist.pkl'))  # 出现频率最高的2000个单词
        # self.w2vdict = json.load(open('./data/word2vec_50.json')) # 2000个词的word2vector
        self.clf = Grocery("jdclf")        # 句子分类器,分为demand,duty,other
        self.clf.load()
        
        self.SEX = re.compile(u"性别不限|性别|男|女")
        self.AGE = re.compile(u"\d+周?岁|年龄")
        self.DEGREE = re.compile(u"(全日制)?(初中|高中|中专|大专|专科|大学专科|中职|本科|大学本科|硕士|研究生|博士|博士后)(.?以上)?")
        self.MAJOR = re.compile(u"\S+(相关专业|专业优先|及其.专业|[类等]专业[优先]?)")
        self.EXP = re.compile(u"工作经验:|工作经[历验]|工作年限|年.{0,4}经[历验]|经[历验].{1,6}年")
        self.PUB_TIME = re.compile(u"(\d+)(天前发布)")
        
        self.INCNAME = re.compile(u"\S+(有限公司|酒店|银行|集团|研究中心|研究所|学校|旅行社|分?公司|研发中心|技术部|事.部|招聘)") 
        self.NOT_INC = re.compile(u"职位|描述|收藏|推荐|地址|邮箱|主页|介绍|欢迎|加入|要求|简介|险一金|奖金|包吃住|社区|厂房|人员|职责") 
        self.INCTAG = re.compile(u"大公司|五百强|全球500强|小公司|成长型公司|创业公司|私有经济|集体经济|集团|外企|已上市|稳定性高|平均年龄\d+岁|妹纸多|学历高|福利待遇好|晋升机会大|民营公司|民营企业|互联网|创业型|国企|央企")

        self.JOBNAME = re.compile(u'\S*(研发工程师|工程师|经理|助理|顾问|前台|秘书|主管|研究员|实习生|操作员|专员|教学人员|技术人员|管理员|业务员|公关|程序员|教师|老师|培训生|\
                                  文员|研究员|策划|主任|总监|设计师|分析师|架构师|摄影师|编辑|BD|游戏UI|Android(开发)?|PHP(开发)?|Python(开发)?|.?(急招|急聘|初级|中级|高级|方向).?[\s)】\)])|\
                                  |行政人事|网店设计|客服|会计|电话销售|外贸跟单|web前端|游戏UI|后.开发|产品运营|商业数据分析')

        self.START_DEMAND = re.compile(u"(岗位要求|应聘条件|任职要求|岗位资格|任职资格|岗位条件|工作要求|任职条件|人员条件|职位.求|职位条件|职位描述|岗位资格|职位资格|具备条件)[::\s]\
                                       |如果你.{0,10}[::\s]|我们希望你.{0,12}[::\s]|(要求|条件)[::\s]|你需要?具备什么.+[?\?::\s]|任职资格[::\s]")
        self.DEMAND = re.compile(u"熟悉|熟练|具有|善于|懂得|掌握|具备|能够|优先|不少于|不超过|至少|团队.作|良好的|工作经验|开发经验|实习经历|能力强|富有|以上学历|经验|喜欢|\
                                 较强的.{2,8}能力|相关专业|相关学历|者优先|精通|了解|及以上|技术全面|.强的责任心|[能有]独立|英文流利")

        self.DUTY = re.compile(u"跟进|协助|负责|配合|其他工作|领导交办的|对.+提供|审核|参与|提出|跟踪|报告|为.+提出|日常.+工作|指导|对.+进行|为.+提供|跟进|拓展|运营|用户|客户|协调|拟写|通过|协同|完成|沟通|需求|秘书.{2,5}翻译")
        self.START_DUTY = re.compile(u"(岗位职责|岗位描述|职位描述|职责描述|任职描述|职位职责|工作职责|工作职能|职位职能|工作内容|实习内容|职位内容)[::\s]|做这样的事[::\s]|职责.{0,5}[::\s]")
        self.PAY = re.compile(u"薪酬|待遇|月薪|薪资|年薪|底薪|\d+k|\d+万|\d+元|工资|报酬|薪水|福利")
        self.BENEFIT = re.compile(u"周休|补助|补贴|假日|餐补|提成|交通补助|食宿|加班工资|期权|年假|领导|扁平化|管理|氛围|空间|休假|月假|带薪|全休|晋升|培训|舒适的|旅游|奖励|过节费|五险一金|奖金|\
        |弹性工作|氛围|成长空间|实训|培训|高薪|前景|旅游|活动|分红")
        
        self.SPLIT_JD = re.compile(u"岗位[【(]?[一二三四五六七八九][】)][::\s]|(^招聘岗位\S+|岗位\d|岗位[一二三四五六])[::\s]")
        self.CLEAR_NUM = re.compile(u"^\d[\.: :。、]|^[\((【]?\d[\))】\.]")
        self.CLEAR_COLO = re.compile(u"^[\s\.。)(【】,,]|[。;,\.;,]$|^\d[\.]")
        self.SKILL = re.compile(u"精通|了解|熟练|熟悉|掌握|懂得|优先|具备|具有|者优先|擅长|善于|较强的.{2,6}能力|良好的|有.+经验|能力|极强的")
        
        jieba.load_userdict('./data/majordic.txt')
        jieba.load_userdict('./data/skills.txt')
        jieba.load_userdict('./data/firm.txt')
        jieba.load_userdict('./data/degrees.txt')
        jieba.load_userdict('./data/benefits.txt')


        self.jdStr = ""
        self.linelist = []
        self.lineindex = defaultdict(int)
        self.result = OrderedDict() 

    
    # 分句,预处理
    def preprocess(self,jdstr):
        self.result.clear()
        jdstr = re.sub(u"[【】◆ \u25cf\u25c6\u2605]","",jdstr.decode('utf-8'))
        self.linelist = [ line.strip() for line in jdstr.split('\n') if len(line)>1 ]
        self.jdStr = '\n'.join(self.linelist)
        for line in self.linelist:
          #  print self.clf.predict(line),'\t',line
            self.lineindex[re.sub(u"[\s ]+"," ",line)] = 0

    def line2vec(self,line):
        vec = np.zeros(50)
        cnt = 1
        for word in jieba.cut(line):
            if word in self.w2vdict:
                vec += self.w2vdict[word] 
                cnt += 1
        vec = vec/cnt
        return vec

    # 抽取性别要求
    def regular_sex(self):
        """
        不限:0
        男:1
        女:2
        """
        res = set()
        sex = '0'
        for line in self.linelist:
            if self.clf.predict(line)=='demand' or self.DEMAND.search(line):
                findsex = self.SEX.search(line)
                if findsex:
                    getsex = re.search(u"性别不限|男|女",line.replace(u"男女不限",u"性别不限"))
                    if getsex:
                        res.add(getsex.group())
                        break
        if res:
            sexstr = '/'.join(res)
            if re.search(u"男",sexstr):
                sex = '1'
            elif re.search(u"女",sexstr):
                sex = '2'

        self.result['sex'] = sex


    # 抽取年龄要求
    def regular_age(self):
        res = []
        for line in self.linelist:
            if re.search(u'\d{2}后',line):continue
            findage = self.AGE.search(line)
            if findage:
                findage = re.search(u"(\d{2}[\--到])\d{2}岁|(至少|不低于|不超过|不高于|不大于|大概|大约)?\d+岁|\d+岁(以上|左右|以下)?",line)
                if findage:
                    agestr = findage.group()
                    age = re.findall(u'\d{2}',agestr)
                    age = map(int,age)
                    if len(age)>=2:
                        res = (age[0],age[1])
                    elif len(age)==1:
                        if re.search(u'以上|不低于|至少|大于',line):
                            res = (age[0],100)
                        elif re.search(u"小于|低于|不超过|不得?高于|以下|不大于",line):
                            res = (0,age[0])
                        elif re.search(u"左右|大约|大概",line):
                            res = (age[0]-3,age[0]+3)
                    break
        if len(res)<1:
            res = (0,100)
        self.result['age'] = res
        return res



    # 抽取专业要求
    def regular_major(self):
        res = []
        
        for line in self.linelist:
            findmajor = re.search(u"专业要求[::\s]",line)
            if findmajor:
                items = self.clean_line(line[findmajor.span()[1]:]).split()
                items = filter(lambda x: x not in self.degreedic and not re.search(u"薪酬|经验|元|\d+|月",x),items)
                res.append(' / '.join(items))
                break

        if not res:
            for line in self.linelist:
                if re.search(u"专业.限|.限专业",line) and not re.search(u"专业优先",line):
                    res.append(u"专业不限")
                    break
                else:
                    findmajor = self.MAJOR.search(line)
                    if findmajor:
                        majoritem = re.split(u'[\s,,;; ]',findmajor.group())
                        for item in majoritem:
                            if re.search(u'学历|年龄|岁|学校|公司|性格|具有|具备|能够|经验|有|毕业|性别|男|女',item):continue
                            if self.BENEFIT.search(line):continue
                            if re.search(u"专业",item) and len(item)<3:continue
                            res.append(self.clean_line(item))
                        break
                        if not res:
                            for majorword in jieba.cut(line):
                                if majorword in self.majordic or majorword[:-2] in self.majordic:
                                    res.append(majorword)

                            if re.search(u"[等及类]?相关专业",self.jdStr) and len(res)==1:
                                res[0]+=u"等相关专业"
        if not res:
            res.append(u"专业不限")

        self.result['major'] = res
        

    # 抽取学历要求
    def regular_degree(self):
        """
        抽查学历信息,先整找关键字,而后再切词,用词典匹配
        """
        degree=[u'小学',u'初中',u'中专',u'中技',u'高中',u'专科',u'大专',u'本科',u'硕士',u'博士',u'博士后']
        res = set()
        for line in self.linelist:
            finddegree = re.search(u"学历要求[::\s]",line)
            if finddegree:
                items =self.clean_line(line[finddegree.span()[1]:]).split()
                items = filter(lambda x: not re.search(u"薪酬|经验|元|月|年|\d+",x),items)
                res.add(' / '.join(items))
                break

        if not res:
            for line in self.linelist:
                if re.search(u"学历不限|学历要求不限|不限学历",line):
                    res.add(u"学历不限")
                    break
                else:
                    finddegree = self.DEGREE.search(line)
                    if finddegree:
                        res.add(finddegree.group())
                        break

        # 如果没有匹配到学历的要求信息,就整个文本切词后匹配查找
        if len(res)==0:
            for word in jieba.cut(self.jdStr):
                if word in self.degreedic:
                    res.add(word)
        res = list(res)
        if len(res)==1 and re.search(u'[及或]?以上',res[0]):
            tmp = res[0][:2]
            if tmp==u'全日':
                tmp = u'本科'
            elif tmp==u'研究':
                tmp=u'硕士'
            if tmp in degree:
                idx = degree.index(tmp)
                res = degree[idx:]
        
        self.result['degree'] = ' / '.join(res)



   
    # 抽取工作经验年限要求    
    def regular_exp(self):
        
        cnyear = u"[一二三四五六七八九半两十]年"
        jdStr = self.jdStr

        res = []
        if re.search(u'经[历验]不限|不限[经验]',jdStr):
            res = (0,100)
            return res
        
        for line in self.linelist:
            findexp = self.EXP.search(line)
            if findexp:
                exp = re.search(u"(\d[\--])?\d{1,2}年|(不少于)?半\d?年|经[历验]\d[\--]\d{1,2}年",line)
                if not exp:
                    exp = re.search(cnyear,line)
                if exp:
                    expstr = exp.group() # 缩小查找范围
                    expstr = expstr.replace(u"两",u"2").replace(u"十",u"10").replace(u"半",u"0.5").replace(u"三",u"3").replace(u"一",u'1')
                    year = re.findall(u"\d{1,2}",expstr)
                    if not year:
                        continue
                    year = map(int,year)
                    if len(year)>1:
                        res = (year[0],year[-1])

                    elif len(year)==1:
                        if re.search(u'以上|不低于',line):
                            res = (year[0],100)
                        elif re.search(u"不超过|不高于|以下",line):
                            res = (0,year[0])
                        elif re.search(u"大概|大约|左右",line):
                            if year[0]-3<0.001:
                                minyear = 0
                            res = (minyear,year[0]+3)
                    break
        if not res:
            res = (0,100)
        self.result["exp"] = tuple(res)
        return res

    def regular_jobtag(self):
        """
        有关职位标签信息
        """
        res = []
        job_tag = re.search(u"应届生|全职|兼职|实习生|应届毕业生|社招|急招|急聘",self.jdStr)
        if job_tag:
            res.append(job_tag.group())
        
        job_tag = re.search(u"招聘人数[::]?|招聘[::\s]|人数[::\s]",self.jdStr)
        if job_tag:
            jdstr = self.jdStr[job_tag.span()[1]:]
            for line in jdstr.split():
                if len(line.strip())<1:continue
                else:
                    num = re.search(u"(\d+\-)?\d+人?|若干|\d+位",line)
                    if num:
                        res.append(u"招聘人数:"+num.group())
                    break

        job_tag = re.search(u"(职能类别|职位标签)[:: ]?",self.jdStr)
        if job_tag:
            jdstr = self.jdStr[job_tag.span()[1]:]
            for line in jdstr.split('\n'):
                if len(line.strip())<3:continue
                else:
                    res.append("职业标签:"+line.strip())
                    break
                if len(line)>25:break

        #  根据产品部需求专门切割出包含经验的句子等有关职位标注信息,句子进行更精细化切割
        linelist = [ line for line in re.split(u"[,。;\s]",self.jdStr) if 5<len(line)<15 ]
        for line in linelist:
            if re.search(u"经验",line) and not re.search(u"月薪|地点|日期",line):
                if re.search(u"\d+k|[。?)\)\]]",line):continue
                res.append(self.clean_line(line))
                break

        self.result["job_tag"] = res
        return res

    # 清除句子前的数字和标点符合
    def clean_line(self,line):
        line = self.CLEAR_NUM.sub("",line.strip())
        line = self.CLEAR_COLO.sub("",line)
        return line


    # 抽取工作地点
    def regular_workplace(self):
        res = set() 
        jdstr = self.jdStr
        pos = list(re.finditer(u"(工作地.|上班地.|实习地.|地址|地点)[::\s]",jdstr))
        if pos:
            jdstr = jdstr[pos[0].span()[1]:]

            for line in jdstr.split():
                if len(line.strip())<2:continue
                if len(line)<26:
                    res.add(line.strip().replace(":","").replace(":",""))
                else:
                    for city in jieba.cut(line):
                        if city in self.citydic and city[:-1] not in res:
                            res.add(city)
                break
        if not res: 
            for city in jieba.cut(jdstr):
                if city in self.citydic and city[:-1] not in res and u"国" not in city:
                    res.add(city)
                    break
        self.result["workplace"] = " / ".join(res)
        return res




    # 抽取证书获奖情况等其他要求
    def regular_cert(self):
        res = set()
        linelist = [line for line in re.split(u"[\s ,。;,]",self.jdStr) if len(line)>3]
        for line in linelist:
            findcert = re.search(u"(\S+证书|CET-\d|普通话|英语|口语|.语|日文|雅思|托福|托业)(至少)?(通过)?[\d一二三四五六七八九]级[及或]?(以上)?|(英语)?CET-\d级?(以上)?|职业资格|律师证|会计证",line)
            if findcert:
                res.add(findcert.group())
            else:
                findcert = re.search(u"有(.+证)书?",line)
                if findcert:
                    res.add(findcert.group(1))
                else:
                    findcert = re.search(u"有.+资格",line)
                    if findcert:
                        res.add(findcert.group())

        self.result['cert'] = re.sub(u"[或及以上]","",' / '.join(res))
        if self.result['cert']:
            self.result['cert'] = self.result['cert'].split(' / ')
        else:
            self.result['cert'] = []

    
    # 利用技能词库抽取相关技能
    def regular_skill(self,num=6):
        res = []
        for line in self.linelist:
            if self.DEMAND.search(line) or self.clf.predict(line)=='demand':
                for word in jieba.cut(line):
                    word = strQ2B(word).lower()
                    if word in self.skills:
                        res.append(word)

        sorted_words = [w[0] for w in Counter(res).most_common(2*num)]
        
        for word in jieba.cut(self.result['job_name']):
            word = strQ2B(word).lower()
            if word in self.skills and word not in sorted_words:
                sorted_words.insert(0,word)

        after_top3 = sorted_words[3:]
        np.random.shuffle(after_top3)
        keywords = sorted_words[:3]+after_top3[:num-3]
        keywords.sort()
        self.result['skill'] = keywords


    # 抽取岗位职责
    def regular_duty(self):
        res = []
        jdStr = self.jdStr
        pos = list(self.START_DUTY.finditer(jdStr))
        if len(pos)>0:
            linelist =[ re.sub("[\s ]+"," ",line) for line in jdStr[pos[-1].span()[1]:].split("\n") if len(line)>1]
            for i in xrange(len(linelist)):
                line = linelist[i]
                print 'dutyline',line
                if self.START_DUTY.search(line) or self.lineindex[line]==1 or re.search(u".年来|谢谢|请在|公司介绍|举报|收藏|岗位职责|介绍|描述|[\-=\.]{3,}",line):continue
                if re.search(u"要求[::\s]?|岗位要求",line) and len(line)<6:break
                if re.match(u"\d{1,2}|\u25cf|[\uff0d(\(\-\+]|[a-z][\.、\s]",line.strip()) or self.DUTY.search(line) or self.clf.predict(line)=='duty':
                    res.append(line.strip())
                    print 'dutyline',line
                elif i<len(linelist)-1 and self.clf.predict(linelist[i+1])=='duty':
                    res.append(line)
                else:
                    break
        if not res:
            for line in self.linelist:
                if re.search(u"粉丝团",line) and len(line)<12:continue
                if self.DUTY.search(line) and self.clf.predict(line) == "duty":
                    if self.lineindex[line]!=1:
                        res.append(line)

        self.result["duty"] ="\n".join(res)
        for line in res:
            self.lineindex[line]=1

        return res


    # 抽取岗位要求
    def regular_demand(self):
        res = []
        jdStr =self.jdStr
        pos = list(self.START_DEMAND.finditer(jdStr))
        if len(pos)>0:
            tmppos = pos[-1].span()[0]
            if re.search(u"具有|具备",jdStr[tmppos-5:tmppos+5]) or re.search(u"证书|证",jdStr[tmppos:tmppos+8]):
                pos.pop()
            if pos:
                linelist =[ re.sub("[\s ]+"," ",line) for line in jdStr[pos[-1].span()[1]:].split("\n") if len(line)>2]
            else:
                linelist = []
            for i in xrange(len(linelist)):
                line = linelist[i]
                if self.START_DEMAND.search(linelist[i]) or re.search(u"谢谢|请在|公司介绍|举报|收藏|\d+k?元|加分",line):continue
                if re.match(u"\d{1,2}|\u25cf|[\uff0d(\(\-\+]|[a-z][\.、\s]",line) or self.DEMAND.search(line) or self.clf.predict(line)=='demand':
                    res.append(line)
                elif i<len(linelist)-1 and self.clf.predict(linelist[i+1])=='demand':
                    res.append(line)
                else:
                    break
        if not res:
            for line in self.linelist:
                if self.lineindex[line]==1 or len(line.split())>6:continue # 如果该句已经被处理过,就不再重复显示
                if self.clf.predict(line)=='demand' or self.DEMAND.search(line):
                    res.append(line.strip())
                    
        self.result['demand'] = '\n'.join(res)
        for line in res:
            self.lineindex[line]=1

        return res
    
    # 招聘的职位名
    def regular_jobname(self):
        res = set()
        jdStr = self.jdStr
        findpos = re.search(u"(招聘岗位|招聘职位|职位名称|岗位名称|岗位[一二三四五六七八九])[:、:\s ]",jdStr)
#        if not findpos:
#            findpos = re.search(u"(职位类别|职位职能)[::\s ]",jdStr)

        if findpos:
            pos = findpos.span()[1]    
            linelist = jdStr[pos:].split("\n")
            for line in linelist:
                if len(line)<2:continue
                if len(line)>=2 and len(line)<20:
                    if re.search(u"职位描述|查看|地址|工作|分享|举报|下一条|时间|福利|待遇|周末|双休",line):continue
                    res.add(re.sub(u"聘请|高薪诚聘|诚聘|[,。、\d!]+","",line.strip()))
                    break

        # 如果没有匹配到招聘的具体职位信息,就切词后到职位列表去匹配
        if not res:
            for line in self.linelist:
                if re.search(u"招聘|高薪|诚聘",line):continue
                if len(line)<6 and not re.search(u'岗位|岗位内容|工作内容|职责|任职|资格',line) and self.clf.predict(line)=='job_name':
                    res.add(line)
                    break
                findPos = self.JOBNAME.search(line)
                if findPos and len(findPos.group())<20 and not re.match(u'\d',findPos.group()):
                    jobname = findPos.group()
                    res.add(re.sub(u"聘请|高薪诚聘|诚聘|急.|[,。、!]+","",jobname))
                    break
                #   res.add(re.sub(u"\(.+\)|(.+)|【.+】|[,。、\s\d]+|聘请|高薪诚聘|诚聘|急招|","",line.strip()))

        if not res:
            for line in self.linelist:
                for word in jieba.cut(line.lower()):
                    if word in self.jobdic:            
                        res.add(word)
                        self.result["job_name"] = " / ".join(res)
                        return res
        if not res:
            tag = re.search(u"实习生|兼职",self.jdStr)
            if tag:
                res.add(tag.group())
        self.result["job_name"] = strQ2B(" / ".join(res)).lower()
        return res
    
    # 薪酬
    def regular_pay(self):
        paystr = ""
        lagoup =  re.search(u"(\d+[kK][-——]\d+[kK])|(\d{3,5}-\d{3,5}元?/[月日天])|(\d{3,5}-\d{3,5}元)|((\d+[-~]\d+)万.[年月])|底薪\d+(-\d+)?元?|\d{3,5}元(左右|以上)?|年薪\d+万?元(左右|以上)?",self.jdStr) # 针对拉勾网,没有待遇等关键字符
        if lagoup:
            paystr = lagoup.group()
        else:
            findpay = self.PAY.search(self.jdStr)
            if findpay:
                pos = findpay.span()[1]
                jdstr = self.jdStr[max(0,pos-5):min(pos+10,len(self.jdStr))]
                if re.search(u"面议",jdstr):
                    paystr = u"面议"
                else:
                    findpay = re.findall(u"\d{3,7}",jdstr)
                    paystr = "-".join(findpay)

        paystr = paystr.replace(u'k','000').replace(u'K','000').replace(u"万","0000")
        pay = re.findall('\d+',paystr)
        pay = map(int,pay)
        if len(pay)>1:
            res = tuple((pay[0],pay[-1]))
        elif len(pay)==1:
            res = (pay[0],0)
        else:
            res = (0,0)
        self.result['pay'] = res
        return res


    # 抽取薪资福利
    def regular_benefits(self):
        res = []
        jdStr = self.jdStr 
        findpos =list(re.finditer(u"薪酬福利[::\s]|(福利|待遇)\s?[::】]",jdStr))
        if not findpos:
            findpos =list(re.finditer(u"(晋升制度|工作环境|职位诱惑|你会获得什么)\s?[?\?::】]",jdStr))
        if findpos:
            pos = findpos[-1].span()[1]
            linelist = jdStr[pos:].split('\n')
            for line in linelist:
                if len(line.strip())<3:continue
                if re.match(ur"[((]?\d+",line) or self.BENEFIT.search(line):
                    res.append(line.strip())
                    self.lineindex[line.strip()] = 1
                else:
                    break
        
        if not res:
            for line in jdStr.split():
                if len(line)>1 and re.search(u"带薪|双休|股票期权|五险一金|发展空间|福利|诱惑|休假|薪酬|补助|年假|弹性工作",line):
                    if re.search(u"福利|待遇|诱惑",line) and len(line.strip())<6:continue
                    res.append(line.strip())

        if len(res)==1 and re.search(u"险一金",res[0]) and not re.search(u"[,、]",res[0]):
            res[0] = self.clean_line(' '.join(jieba.cut(res[0])))

        self.result["benefits"] ="\n".join(res)
        return res
Ejemplo n.º 55
0
class CachedStore(BaseStore):
    """
    A thread-safe cached persistent store of the subscriber database.
    Prerequisite: persistent_store need to be thread safe
    """
    def __init__(self, persistent_store, cache_capacity=512, loop=None):
        self._lock = threading.Lock()
        self._cache = OrderedDict()
        self._cache_capacity = cache_capacity
        self._persistent_store = persistent_store
        self._on_ready = OnDataReady(loop=loop)

    def add_subscriber(self, subscriber_data):
        """
        Method that adds the subscriber.
        """
        sid = SIDUtils.to_str(subscriber_data.sid)
        with self._lock:
            if sid in self._cache:
                raise DuplicateSubscriberError(sid)

            self._persistent_store.add_subscriber(subscriber_data)
            self._cache_put(sid, subscriber_data)
        self._on_ready.add_subscriber(subscriber_data)

    @contextmanager
    def edit_subscriber(self, subscriber_id):
        """
        Context manager to modify the subscriber data.
        """
        with self._lock:
            if subscriber_id in self._cache:
                data = self._cache_get(subscriber_id)
                subscriber_data = copy.deepcopy(data)
            else:
                subscriber_data = \
                    self._persistent_store.get_subscriber_data(subscriber_id)
            yield subscriber_data
            self._persistent_store.update_subscriber(subscriber_data)
            self._cache_put(subscriber_id, subscriber_data)

    def delete_subscriber(self, subscriber_id):
        """
        Method that deletes a subscriber, if present.
        """
        with self._lock:
            if subscriber_id in self._cache:
                del self._cache[subscriber_id]

            self._persistent_store.delete_subscriber(subscriber_id)

    def delete_all_subscribers(self):
        """
        Method that removes all the subscribers from the store
        """
        with self._lock:
            self._cache_clear()
            self._persistent_store.delete_all_subscribers()

    def resync(self, subscribers):
        """
        Method that should resync the store with the mentioned list of
        subscribers. The resync leaves the current state of subscribers
        intact.

        Args:
            subscribers - list of subscribers to be in the store.
        """
        with self._lock:
            self._cache_clear()
            self._persistent_store.resync(subscribers)
        self._on_ready.resync(subscribers)

    def get_subscriber_data(self, subscriber_id):
        """
        Method that returns the subscriber data for the subscriber.
        """
        with self._lock:
            if subscriber_id in self._cache:
                return self._cache_get(subscriber_id)
            else:
                subscriber_data = \
                    self._persistent_store.get_subscriber_data(subscriber_id)
                self._cache_put(subscriber_id, subscriber_data)
                return subscriber_data

    def list_subscribers(self):
        """
        Method that returns the list of subscribers stored.
        Note: this method is not cached since it's easier to get the whole list
        from persistent store
        """
        return self._persistent_store.list_subscribers()

    async def on_ready(self):
        return await self._on_ready.event.wait()

    def _cache_get(self, k):
        """
        Get from the LRU cache. Move the last hit entry to the end.
        """
        self._cache.move_to_end(k)
        return self._cache[k]

    def _cache_put(self, k, v):
        """
        Put to the LRU cache. Evict the first item if full.
        """
        if self._cache_capacity == len(self._cache):
            self._cache.popitem(last=False)
        self._cache[k] = v

    def _cache_list(self):
        return list(self._cache.keys())

    def _cache_clear(self):
        self._cache.clear()
Ejemplo n.º 56
0
class LRU(object):
    """ A dictionary-like object, supporting LRU caching semantics.

    >>> d = LRU(max_size=3)
    >>> d['foo'] = 'bar'
    >>> d['foo']
    'bar'
    >>> d['a'] = 'A'
    >>> d['b'] = 'B'
    >>> d['c'] = 'C'
    >>> d['d'] = 'D'
    >>> d['a'] # Should return value error, since we exceeded the max cache size
    Traceback (most recent call last):
        ...
    KeyError: 'a'
    """
    def __init__(self, max_size=1024):
        self.max_size = max_size
        self.__access_times = OrderedDict()

    def __len__(self):
        return len(self.__access_times)

    def clear(self):
        """
        Clears the dict.

        >>> d = LRU(max_size=3)
        >>> d['foo'] = 'bar'
        >>> d['foo']
        'bar'
        >>> d.clear()
        >>> d['foo']
        Traceback (most recent call last):
        ...
        KeyError: 'foo'
        """
        self.__access_times.clear()

    def __contains__(self, key):
        """
        This method should almost NEVER be used. The reason is that between the time
        has_key is called, and the key is accessed, the key might vanish.

        You should ALWAYS use a try: ... except KeyError: ... block.

        >>> d = LRU(max_size=3)
        >>> d['foo'] = 'bar'
        >>> d['foo']
        'bar'
        >>> 'foo' in d
        True
        >>> 'goo' in d
        False

        """
        return self.__access_times.has_key(key)

    def __setitem__(self, key, value):
        self.__access_times[key] = value
        self.cleanup()

    def __getitem__(self, key):
        return self.__access_times[key]

    def __delete__(self, key):
        del self.__access_times[key]

    def cleanup(self):
        #If we have more than self.max_size items, delete the oldest
        while (len(self.__access_times) > self.max_size):
            for k in self.__access_times.iterkeys():
                self.__delete__(k)
                break
Ejemplo n.º 57
0
class Cache(object):
    __slots__ = (
        "cache",
        "max_entries",
        "name",
        "keylen",
        "sequence",
        "thread",
        "metrics",
    )

    def __init__(self, name, max_entries=1000, keylen=1, lru=True, tree=False):
        if lru:
            cache_type = TreeCache if tree else dict
            self.cache = LruCache(max_size=max_entries,
                                  keylen=keylen,
                                  cache_type=cache_type)
            self.max_entries = None
        else:
            self.cache = OrderedDict()
            self.max_entries = max_entries

        self.name = name
        self.keylen = keylen
        self.sequence = 0
        self.thread = None
        self.metrics = register_cache(name, self.cache)

    def check_thread(self):
        expected_thread = self.thread
        if expected_thread is None:
            self.thread = threading.current_thread()
        else:
            if expected_thread is not threading.current_thread():
                raise ValueError(
                    "Cache objects can only be accessed from the main thread")

    def get(self, key, default=_CacheSentinel):
        val = self.cache.get(key, _CacheSentinel)
        if val is not _CacheSentinel:
            self.metrics.inc_hits()
            return val

        self.metrics.inc_misses()

        if default is _CacheSentinel:
            raise KeyError()
        else:
            return default

    def update(self, sequence, key, value):
        self.check_thread()
        if self.sequence == sequence:
            # Only update the cache if the caches sequence number matches the
            # number that the cache had before the SELECT was started (SYN-369)
            self.prefill(key, value)

    def prefill(self, key, value):
        if self.max_entries is not None:
            while len(self.cache) >= self.max_entries:
                self.cache.popitem(last=False)

        self.cache[key] = value

    def invalidate(self, key):
        self.check_thread()
        if not isinstance(key, tuple):
            raise TypeError("The cache key must be a tuple not %r" %
                            (type(key), ))

        # Increment the sequence number so that any SELECT statements that
        # raced with the INSERT don't update the cache (SYN-369)
        self.sequence += 1
        self.cache.pop(key, None)

    def invalidate_many(self, key):
        self.check_thread()
        if not isinstance(key, tuple):
            raise TypeError("The cache key must be a tuple not %r" %
                            (type(key), ))
        self.sequence += 1
        self.cache.del_multi(key)

    def invalidate_all(self):
        self.check_thread()
        self.sequence += 1
        self.cache.clear()
Ejemplo n.º 58
0
gen_max_file = open('gen_max.txt', 'w')
converge_time = []
run_time = []
rs = time.time()
g = 0
# for g in range(0, generations):
while True:
    print('The generation number is: ', g + 1)
    file.write('\nThe generation number is:' + str(g + 1))
    dicts = OrderedDict(sorted(dicts.items(), key=lambda x: x[1]))
    cs = time.time()
    new_pop = mating_pool(list(dicts.keys()), neurons, dicts, sample_weights)
    final_pop = mutate_pool(new_pop, neurons, number_of_mutate_node)
    ce = time.time()
    converge_time.append(ce - cs)
    dicts.clear()

    fit_result = multiprocess(pool, final_pop, x_train, y_train)
    prob_pop = probability(fit_result)
    dicts = {final_pop[i]: [fit_result[i], prob_pop[i]] for i in range(0, len(final_pop))}
    print('The mean of fitness is:', np.mean(fit_result))
    print('The best of fitness is:', np.max(fit_result))
    file.write('\nThe mean of fitness is:' + str(np.mean(fit_result)))
    file.write('\nThe best of fitness is:' + str(np.max(fit_result)))
    file.flush()
    gen_fit_file.write('\n' + str(fit_result))
    gen_fit_file.flush()
    gen_mean_file.write('\n' + str(np.mean(fit_result)))
    gen_mean_file.flush()
    gen_max_file.write('\n' + str(np.max(fit_result)))
    gen_max_file.flush()
Ejemplo n.º 59
0
class BacktestingEngine(object):
    """
    CTA回测引擎
    函数接口和策略引擎保持一样,
    从而实现同一套代码从回测到实盘。
    """

    TICK_MODE = 'tick'
    BAR_MODE = 'bar'

    #----------------------------------------------------------------------
    def __init__(self):
        """Constructor"""
        # 本地停止单编号计数
        self.stopOrderCount = 0
        # stopOrderID = STOPORDERPREFIX + str(stopOrderCount)

        # 本地停止单字典
        # key为stopOrderID,value为stopOrder对象
        self.stopOrderDict = {}  # 停止单撤销后不会从本字典中删除
        self.workingStopOrderDict = {}  # 停止单撤销后会从本字典中删除

        # 引擎类型为回测
        self.engineType = ENGINETYPE_BACKTESTING

        # 回测相关
        self.strategy = None  # 回测策略
        self.mode = self.BAR_MODE  # 回测模式,默认为K线

        self.startDate = ''
        self.initDays = 0
        self.endDate = ''

        self.slippage = 0  # 回测时假设的滑点
        self.rate = 0  # 回测时假设的佣金比例(适用于百分比佣金)
        self.size = 1  # 合约大小,默认为1
        self.priceTick = 0  # 价格最小变动

        self.dbClient = None  # 数据库客户端
        self.dbCursor = None  # 数据库指针

        #self.historyData = []       # 历史数据的列表,回测用
        self.initData = []  # 初始化用的数据
        #self.backtestingData = []   # 回测用的数据

        self.dbName = ''  # 回测数据库名
        self.symbol = ''  # 回测集合名

        self.dataStartDate = None  # 回测数据开始日期,datetime对象
        self.dataEndDate = None  # 回测数据结束日期,datetime对象
        self.strategyStartDate = None  # 策略启动日期(即前面的数据用于初始化),datetime对象

        self.limitOrderDict = OrderedDict()  # 限价单字典
        self.workingLimitOrderDict = OrderedDict()  # 活动限价单字典,用于进行撮合用
        self.limitOrderCount = 0  # 限价单编号

        self.tradeCount = 0  # 成交编号
        self.tradeDict = OrderedDict()  # 成交字典

        self.logList = []  # 日志记录

        # 当前最新数据,用于模拟成交用
        self.tick = None
        self.bar = None
        self.dt = None  # 最新的时间

    #----------------------------------------------------------------------
    def setStartDate(self, startDate='20100416', initDays=10):
        """设置回测的启动日期"""
        self.startDate = startDate
        self.initDays = initDays

        self.dataStartDate = datetime.strptime(startDate, '%Y%m%d')

        initTimeDelta = timedelta(initDays)
        self.strategyStartDate = self.dataStartDate + initTimeDelta

    #----------------------------------------------------------------------
    def setEndDate(self, endDate=''):
        """设置回测的结束日期"""
        self.endDate = endDate
        if endDate:
            self.dataEndDate = datetime.strptime(endDate, '%Y%m%d')
            # 若不修改时间则会导致不包含dataEndDate当天数据
            self.dataEndDate = self.dataEndDate.replace(hour=23, minute=59)

    #----------------------------------------------------------------------
    def setBacktestingMode(self, mode):
        """设置回测模式"""
        self.mode = mode

    #----------------------------------------------------------------------
    def setDatabase(self, dbName, symbol):
        """设置历史数据所用的数据库"""
        self.dbName = dbName
        self.symbol = symbol

    #----------------------------------------------------------------------
    def loadHistoryData(self):
        """载入历史数据"""
        host, port, logging = loadMongoSetting()

        self.dbClient = pymongo.MongoClient(host, port)
        collection = self.dbClient[self.dbName][self.symbol]

        self.output(u'开始载入数据')

        # 首先根据回测模式,确认要使用的数据类
        if self.mode == self.BAR_MODE:
            dataClass = CtaBarData
            func = self.newBar
        else:
            dataClass = CtaTickData
            func = self.newTick

        # 载入初始化需要用的数据
        flt = {
            'datetime': {
                '$gte': self.dataStartDate,
                '$lt': self.strategyStartDate
            }
        }
        initCursor = collection.find(flt)

        # 将数据从查询指针中读取出,并生成列表
        self.initData = []  # 清空initData列表
        for d in initCursor:
            data = dataClass()
            data.__dict__ = d
            self.initData.append(data)

        # 载入回测数据
        if not self.dataEndDate:
            flt = {'datetime': {'$gte': self.strategyStartDate}}  # 数据过滤条件
        else:
            flt = {
                'datetime': {
                    '$gte': self.strategyStartDate,
                    '$lte': self.dataEndDate
                }
            }
        self.dbCursor = collection.find(flt)

        self.output(u'载入完成,数据量:%s' %
                    (initCursor.count() + self.dbCursor.count()))

    #----------------------------------------------------------------------
    def runBacktesting(self):
        """运行回测"""
        # 载入历史数据
        self.loadHistoryData()

        # 首先根据回测模式,确认要使用的数据类
        if self.mode == self.BAR_MODE:
            dataClass = CtaBarData
            func = self.newBar
        else:
            dataClass = CtaTickData
            func = self.newTick

        self.output(u'开始回测')

        self.strategy.inited = True
        self.strategy.onInit()
        self.output(u'策略初始化完成')

        self.strategy.trading = True
        self.strategy.onStart()
        self.output(u'策略启动完成')

        self.output(u'开始回放数据')

        for d in self.dbCursor:
            data = dataClass()
            data.__dict__ = d
            func(data)

        self.output(u'数据回放结束')

    #----------------------------------------------------------------------
    def newBar(self, bar):
        """新的K线"""
        self.bar = bar
        self.dt = bar.datetime
        self.crossLimitOrder()  # 先撮合限价单
        self.crossStopOrder()  # 再撮合停止单
        self.strategy.onBar(bar)  # 推送K线到策略中

    #----------------------------------------------------------------------
    def newTick(self, tick):
        """新的Tick"""
        self.tick = tick
        self.dt = tick.datetime
        self.crossLimitOrder()
        self.crossStopOrder()
        self.strategy.onTick(tick)

    #----------------------------------------------------------------------
    def initStrategy(self, strategyClass, setting=None):
        """
        初始化策略
        setting是策略的参数设置,如果使用类中写好的默认设置则可以不传该参数
        """
        self.strategy = strategyClass(self, setting)
        self.strategy.name = self.strategy.className

    #----------------------------------------------------------------------
    def sendOrder(self, vtSymbol, orderType, price, volume, strategy):
        """发单"""
        self.limitOrderCount += 1
        orderID = str(self.limitOrderCount)

        order = VtOrderData()
        order.vtSymbol = vtSymbol
        order.price = self.roundToPriceTick(price)
        order.totalVolume = volume
        order.status = STATUS_NOTTRADED  # 刚提交尚未成交
        order.orderID = orderID
        order.vtOrderID = orderID
        order.orderTime = str(self.dt)

        # CTA委托类型映射
        if orderType == CTAORDER_BUY:
            order.direction = DIRECTION_LONG
            order.offset = OFFSET_OPEN
        elif orderType == CTAORDER_SELL:
            order.direction = DIRECTION_SHORT
            order.offset = OFFSET_CLOSE
        elif orderType == CTAORDER_SHORT:
            order.direction = DIRECTION_SHORT
            order.offset = OFFSET_OPEN
        elif orderType == CTAORDER_COVER:
            order.direction = DIRECTION_LONG
            order.offset = OFFSET_CLOSE

        # 保存到限价单字典中
        self.workingLimitOrderDict[orderID] = order
        self.limitOrderDict[orderID] = order

        return orderID

    #----------------------------------------------------------------------
    def cancelOrder(self, vtOrderID):
        """撤单"""
        if vtOrderID in self.workingLimitOrderDict:
            order = self.workingLimitOrderDict[vtOrderID]
            order.status = STATUS_CANCELLED
            order.cancelTime = str(self.dt)
            del self.workingLimitOrderDict[vtOrderID]

    #----------------------------------------------------------------------
    def sendStopOrder(self, vtSymbol, orderType, price, volume, strategy):
        """发停止单(本地实现)"""
        self.stopOrderCount += 1
        stopOrderID = STOPORDERPREFIX + str(self.stopOrderCount)

        so = StopOrder()
        so.vtSymbol = vtSymbol
        so.price = self.roundToPriceTick(price)
        so.volume = volume
        so.strategy = strategy
        so.stopOrderID = stopOrderID
        so.status = STOPORDER_WAITING

        if orderType == CTAORDER_BUY:
            so.direction = DIRECTION_LONG
            so.offset = OFFSET_OPEN
        elif orderType == CTAORDER_SELL:
            so.direction = DIRECTION_SHORT
            so.offset = OFFSET_CLOSE
        elif orderType == CTAORDER_SHORT:
            so.direction = DIRECTION_SHORT
            so.offset = OFFSET_OPEN
        elif orderType == CTAORDER_COVER:
            so.direction = DIRECTION_LONG
            so.offset = OFFSET_CLOSE

        # 保存stopOrder对象到字典中
        self.stopOrderDict[stopOrderID] = so
        self.workingStopOrderDict[stopOrderID] = so

        return stopOrderID

    #----------------------------------------------------------------------
    def cancelStopOrder(self, stopOrderID):
        """撤销停止单"""
        # 检查停止单是否存在
        if stopOrderID in self.workingStopOrderDict:
            so = self.workingStopOrderDict[stopOrderID]
            so.status = STOPORDER_CANCELLED
            del self.workingStopOrderDict[stopOrderID]

    #----------------------------------------------------------------------
    def crossLimitOrder(self):
        """基于最新数据撮合限价单"""
        # 先确定会撮合成交的价格
        if self.mode == self.BAR_MODE:
            buyCrossPrice = self.bar.low  # 若买入方向限价单价格高于该价格,则会成交
            sellCrossPrice = self.bar.high  # 若卖出方向限价单价格低于该价格,则会成交
            buyBestCrossPrice = self.bar.open  # 在当前时间点前发出的买入委托可能的最优成交价
            sellBestCrossPrice = self.bar.open  # 在当前时间点前发出的卖出委托可能的最优成交价
        else:
            buyCrossPrice = self.tick.askPrice1
            sellCrossPrice = self.tick.bidPrice1
            buyBestCrossPrice = self.tick.askPrice1
            sellBestCrossPrice = self.tick.bidPrice1

        # 遍历限价单字典中的所有限价单
        for orderID, order in self.workingLimitOrderDict.items():
            # 判断是否会成交
            buyCross = (order.direction == DIRECTION_LONG
                        and order.price >= buyCrossPrice and buyCrossPrice > 0
                        )  # 国内的tick行情在涨停时askPrice1为0,此时买无法成交

            sellCross = (order.direction == DIRECTION_SHORT
                         and order.price <= sellCrossPrice
                         and sellCrossPrice > 0
                         )  # 国内的tick行情在跌停时bidPrice1为0,此时卖无法成交

            # 如果发生了成交
            if buyCross or sellCross:
                # 推送成交数据
                self.tradeCount += 1  # 成交编号自增1
                tradeID = str(self.tradeCount)
                trade = VtTradeData()
                trade.vtSymbol = order.vtSymbol
                trade.tradeID = tradeID
                trade.vtTradeID = tradeID
                trade.orderID = order.orderID
                trade.vtOrderID = order.orderID
                trade.direction = order.direction
                trade.offset = order.offset

                # 以买入为例:
                # 1. 假设当根K线的OHLC分别为:100, 125, 90, 110
                # 2. 假设在上一根K线结束(也是当前K线开始)的时刻,策略发出的委托为限价105
                # 3. 则在实际中的成交价会是100而不是105,因为委托发出时市场的最优价格是100
                if buyCross:
                    trade.price = min(order.price, buyBestCrossPrice)
                    self.strategy.pos += order.totalVolume
                else:
                    trade.price = max(order.price, sellBestCrossPrice)
                    self.strategy.pos -= order.totalVolume

                trade.volume = order.totalVolume
                trade.tradeTime = str(self.dt)
                trade.dt = self.dt
                self.strategy.onTrade(trade)

                self.tradeDict[tradeID] = trade

                # 推送委托数据
                order.tradedVolume = order.totalVolume
                order.status = STATUS_ALLTRADED
                self.strategy.onOrder(order)

                # 从字典中删除该限价单
                del self.workingLimitOrderDict[orderID]

    #----------------------------------------------------------------------
    def crossStopOrder(self):
        """基于最新数据撮合停止单"""
        # 先确定会撮合成交的价格,这里和限价单规则相反
        if self.mode == self.BAR_MODE:
            buyCrossPrice = self.bar.high  # 若买入方向停止单价格低于该价格,则会成交
            sellCrossPrice = self.bar.low  # 若卖出方向限价单价格高于该价格,则会成交
            bestCrossPrice = self.bar.open  # 最优成交价,买入停止单不能低于,卖出停止单不能高于
        else:
            buyCrossPrice = self.tick.lastPrice
            sellCrossPrice = self.tick.lastPrice
            bestCrossPrice = self.tick.lastPrice

        # 遍历停止单字典中的所有停止单
        for stopOrderID, so in self.workingStopOrderDict.items():
            # 判断是否会成交
            buyCross = so.direction == DIRECTION_LONG and so.price <= buyCrossPrice
            sellCross = so.direction == DIRECTION_SHORT and so.price >= sellCrossPrice

            # 如果发生了成交
            if buyCross or sellCross:
                # 推送成交数据
                self.tradeCount += 1  # 成交编号自增1
                tradeID = str(self.tradeCount)
                trade = VtTradeData()
                trade.vtSymbol = so.vtSymbol
                trade.tradeID = tradeID
                trade.vtTradeID = tradeID

                if buyCross:
                    self.strategy.pos += so.volume
                    trade.price = max(bestCrossPrice, so.price)
                else:
                    self.strategy.pos -= so.volume
                    trade.price = min(bestCrossPrice, so.price)

                self.limitOrderCount += 1
                orderID = str(self.limitOrderCount)
                trade.orderID = orderID
                trade.vtOrderID = orderID

                trade.direction = so.direction
                trade.offset = so.offset
                trade.volume = so.volume
                trade.tradeTime = str(self.dt)
                trade.dt = self.dt
                self.strategy.onTrade(trade)

                self.tradeDict[tradeID] = trade

                # 推送委托数据
                so.status = STOPORDER_TRIGGERED

                order = VtOrderData()
                order.vtSymbol = so.vtSymbol
                order.symbol = so.vtSymbol
                order.orderID = orderID
                order.vtOrderID = orderID
                order.direction = so.direction
                order.offset = so.offset
                order.price = so.price
                order.totalVolume = so.volume
                order.tradedVolume = so.volume
                order.status = STATUS_ALLTRADED
                order.orderTime = trade.tradeTime
                self.strategy.onOrder(order)

                self.limitOrderDict[orderID] = order

                # 从字典中删除该限价单
                if stopOrderID in self.workingStopOrderDict:
                    del self.workingStopOrderDict[stopOrderID]

    #----------------------------------------------------------------------
    def insertData(self, dbName, collectionName, data):
        """考虑到回测中不允许向数据库插入数据,防止实盘交易中的一些代码出错"""
        pass

    #----------------------------------------------------------------------
    def loadBar(self, dbName, collectionName, startDate):
        """直接返回初始化数据列表中的Bar"""
        return self.initData

    #----------------------------------------------------------------------
    def loadTick(self, dbName, collectionName, startDate):
        """直接返回初始化数据列表中的Tick"""
        return self.initData

    #----------------------------------------------------------------------
    def writeCtaLog(self, content):
        """记录日志"""
        log = str(self.dt) + ' ' + content
        self.logList.append(log)

    #----------------------------------------------------------------------
    def output(self, content):
        """输出内容"""
        print(str(datetime.now()) + "\t" + content)

    #----------------------------------------------------------------------
    def calculateBacktestingResult(self):
        """
        计算回测结果
        """
        self.output(u'计算回测结果')

        # 首先基于回测后的成交记录,计算每笔交易的盈亏
        resultList = []  # 交易结果列表

        longTrade = []  # 未平仓的多头交易
        shortTrade = []  # 未平仓的空头交易

        tradeTimeList = []  # 每笔成交时间戳
        posList = [0]  # 每笔成交后的持仓情况

        for trade in self.tradeDict.values():
            # 多头交易
            if trade.direction == DIRECTION_LONG:
                # 如果尚无空头交易
                if not shortTrade:
                    longTrade.append(trade)
                # 当前多头交易为平空
                else:
                    while True:
                        entryTrade = shortTrade[0]
                        exitTrade = trade

                        # 清算开平仓交易
                        closedVolume = min(exitTrade.volume, entryTrade.volume)
                        result = TradingResult(entryTrade.price, entryTrade.dt,
                                               exitTrade.price, exitTrade.dt,
                                               -closedVolume, self.rate,
                                               self.slippage, self.size)
                        resultList.append(result)

                        posList.extend([-1, 0])
                        tradeTimeList.extend([result.entryDt, result.exitDt])

                        # 计算未清算部分
                        entryTrade.volume -= closedVolume
                        exitTrade.volume -= closedVolume

                        # 如果开仓交易已经全部清算,则从列表中移除
                        if not entryTrade.volume:
                            shortTrade.pop(0)

                        # 如果平仓交易已经全部清算,则退出循环
                        if not exitTrade.volume:
                            break

                        # 如果平仓交易未全部清算,
                        if exitTrade.volume:
                            # 且开仓交易已经全部清算完,则平仓交易剩余的部分
                            # 等于新的反向开仓交易,添加到队列中
                            if not shortTrade:
                                longTrade.append(exitTrade)
                                break
                            # 如果开仓交易还有剩余,则进入下一轮循环
                            else:
                                pass

            # 空头交易
            else:
                # 如果尚无多头交易
                if not longTrade:
                    shortTrade.append(trade)
                # 当前空头交易为平多
                else:
                    while True:
                        entryTrade = longTrade[0]
                        exitTrade = trade

                        # 清算开平仓交易
                        closedVolume = min(exitTrade.volume, entryTrade.volume)
                        result = TradingResult(entryTrade.price, entryTrade.dt,
                                               exitTrade.price, exitTrade.dt,
                                               closedVolume, self.rate,
                                               self.slippage, self.size)
                        resultList.append(result)

                        posList.extend([1, 0])
                        tradeTimeList.extend([result.entryDt, result.exitDt])

                        # 计算未清算部分
                        entryTrade.volume -= closedVolume
                        exitTrade.volume -= closedVolume

                        # 如果开仓交易已经全部清算,则从列表中移除
                        if not entryTrade.volume:
                            longTrade.pop(0)

                        # 如果平仓交易已经全部清算,则退出循环
                        if not exitTrade.volume:
                            break

                        # 如果平仓交易未全部清算,
                        if exitTrade.volume:
                            # 且开仓交易已经全部清算完,则平仓交易剩余的部分
                            # 等于新的反向开仓交易,添加到队列中
                            if not longTrade:
                                shortTrade.append(exitTrade)
                                break
                            # 如果开仓交易还有剩余,则进入下一轮循环
                            else:
                                pass

        # 检查是否有交易
        if not resultList:
            self.output(u'无交易结果')
            return {}

        # 然后基于每笔交易的结果,我们可以计算具体的盈亏曲线和最大回撤等
        capital = 0  # 资金
        maxCapital = 0  # 资金最高净值
        drawdown = 0  # 回撤

        totalResult = 0  # 总成交数量
        totalTurnover = 0  # 总成交金额(合约面值)
        totalCommission = 0  # 总手续费
        totalSlippage = 0  # 总滑点

        timeList = []  # 时间序列
        pnlList = []  # 每笔盈亏序列
        capitalList = []  # 盈亏汇总的时间序列
        drawdownList = []  # 回撤的时间序列

        winningResult = 0  # 盈利次数
        losingResult = 0  # 亏损次数
        totalWinning = 0  # 总盈利金额
        totalLosing = 0  # 总亏损金额

        for result in resultList:
            capital += result.pnl
            maxCapital = max(capital, maxCapital)
            drawdown = capital - maxCapital

            pnlList.append(result.pnl)
            timeList.append(result.exitDt)  # 交易的时间戳使用平仓时间
            capitalList.append(capital)
            drawdownList.append(drawdown)

            totalResult += 1
            totalTurnover += result.turnover
            totalCommission += result.commission
            totalSlippage += result.slippage

            if result.pnl >= 0:
                winningResult += 1
                totalWinning += result.pnl
            else:
                losingResult += 1
                totalLosing += result.pnl

        # 计算盈亏相关数据
        winningRate = winningResult / totalResult * 100  # 胜率

        averageWinning = 0  # 这里把数据都初始化为0
        averageLosing = 0
        profitLossRatio = 0

        if winningResult:
            averageWinning = totalWinning / winningResult  # 平均每笔盈利
        if losingResult:
            averageLosing = totalLosing / losingResult  # 平均每笔亏损
        if averageLosing:
            profitLossRatio = -averageWinning / averageLosing  # 盈亏比

        # 返回回测结果
        d = {}
        d['capital'] = capital
        d['maxCapital'] = maxCapital
        d['drawdown'] = drawdown
        d['totalResult'] = totalResult
        d['totalTurnover'] = totalTurnover
        d['totalCommission'] = totalCommission
        d['totalSlippage'] = totalSlippage
        d['timeList'] = timeList
        d['pnlList'] = pnlList
        d['capitalList'] = capitalList
        d['drawdownList'] = drawdownList
        d['winningRate'] = winningRate
        d['averageWinning'] = averageWinning
        d['averageLosing'] = averageLosing
        d['profitLossRatio'] = profitLossRatio
        d['posList'] = posList
        d['tradeTimeList'] = tradeTimeList

        return d

    #----------------------------------------------------------------------
    def showBacktestingResult(self):
        """显示回测结果"""
        d = self.calculateBacktestingResult()

        # 输出
        self.output('-' * 30)
        self.output(u'第一笔交易:\t%s' % d['timeList'][0])
        self.output(u'最后一笔交易:\t%s' % d['timeList'][-1])

        self.output(u'总交易次数:\t%s' % formatNumber(d['totalResult']))
        self.output(u'总盈亏:\t%s' % formatNumber(d['capital']))
        self.output(u'最大回撤: \t%s' % formatNumber(min(d['drawdownList'])))

        self.output(u'平均每笔盈利:\t%s' %
                    formatNumber(d['capital'] / d['totalResult']))
        self.output(u'平均每笔滑点:\t%s' %
                    formatNumber(d['totalSlippage'] / d['totalResult']))
        self.output(u'平均每笔佣金:\t%s' %
                    formatNumber(d['totalCommission'] / d['totalResult']))

        self.output(u'胜率\t\t%s%%' % formatNumber(d['winningRate']))
        self.output(u'盈利交易平均值\t%s' % formatNumber(d['averageWinning']))
        self.output(u'亏损交易平均值\t%s' % formatNumber(d['averageLosing']))
        self.output(u'盈亏比:\t%s' % formatNumber(d['profitLossRatio']))

        # 绘图
        import matplotlib.pyplot as plt
        import numpy as np

        try:
            import seaborn as sns  # 如果安装了seaborn则设置为白色风格
            sns.set_style('whitegrid')
        except ImportError:
            pass

        pCapital = plt.subplot(4, 1, 1)
        pCapital.set_ylabel("capital")
        pCapital.plot(d['capitalList'], color='r', lw=0.8)

        pDD = plt.subplot(4, 1, 2)
        pDD.set_ylabel("DD")
        pDD.bar(range(len(d['drawdownList'])), d['drawdownList'], color='g')

        pPnl = plt.subplot(4, 1, 3)
        pPnl.set_ylabel("pnl")
        pPnl.hist(d['pnlList'], bins=50, color='c')

        pPos = plt.subplot(4, 1, 4)
        pPos.set_ylabel("Position")
        if d['posList'][-1] == 0:
            del d['posList'][-1]
        tradeTimeIndex = [
            item.strftime("%m/%d %H:%M:%S") for item in d['tradeTimeList']
        ]
        xindex = np.arange(0, len(tradeTimeIndex),
                           np.int(len(tradeTimeIndex) / 10))
        tradeTimeIndex = map(lambda i: tradeTimeIndex[i], xindex)
        pPos.plot(d['posList'], color='k', drawstyle='steps-pre')
        pPos.set_ylim(-1.2, 1.2)
        plt.sca(pPos)
        plt.tight_layout()
        plt.xticks(xindex, tradeTimeIndex, rotation=30)  # 旋转15

        plt.show()

    #----------------------------------------------------------------------
    def putStrategyEvent(self, name):
        """发送策略更新事件,回测中忽略"""
        pass

    #----------------------------------------------------------------------
    def setSlippage(self, slippage):
        """设置滑点点数"""
        self.slippage = slippage

    #----------------------------------------------------------------------
    def setSize(self, size):
        """设置合约大小"""
        self.size = size

    #----------------------------------------------------------------------
    def setRate(self, rate):
        """设置佣金比例"""
        self.rate = rate

    #----------------------------------------------------------------------
    def setPriceTick(self, priceTick):
        """设置价格最小变动"""
        self.priceTick = priceTick

    #----------------------------------------------------------------------
    def runOptimization(self, strategyClass, optimizationSetting):
        """优化参数"""
        # 获取优化设置
        settingList = optimizationSetting.generateSetting()
        targetName = optimizationSetting.optimizeTarget

        # 检查参数设置问题
        if not settingList or not targetName:
            self.output(u'优化设置有问题,请检查')

        # 遍历优化
        resultList = []
        for setting in settingList:
            self.clearBacktestingResult()
            self.output('-' * 30)
            self.output('setting: %s' % str(setting))
            self.initStrategy(strategyClass, setting)
            self.runBacktesting()
            d = self.calculateBacktestingResult()
            try:
                targetValue = d[targetName]
            except KeyError:
                targetValue = 0
            resultList.append(([str(setting)], targetValue))

        # 显示结果
        resultList.sort(reverse=True, key=lambda result: result[1])
        self.output('-' * 30)
        self.output(u'优化结果:')
        for result in resultList:
            self.output(u'%s: %s' % (result[0], result[1]))
        return result

    #----------------------------------------------------------------------
    def clearBacktestingResult(self):
        """清空之前回测的结果"""
        # 清空限价单相关
        self.limitOrderCount = 0
        self.limitOrderDict.clear()
        self.workingLimitOrderDict.clear()

        # 清空停止单相关
        self.stopOrderCount = 0
        self.stopOrderDict.clear()
        self.workingStopOrderDict.clear()

        # 清空成交相关
        self.tradeCount = 0
        self.tradeDict.clear()

    #----------------------------------------------------------------------
    def runParallelOptimization(self, strategyClass, optimizationSetting):
        """并行优化参数"""
        # 获取优化设置
        settingList = optimizationSetting.generateSetting()
        targetName = optimizationSetting.optimizeTarget

        # 检查参数设置问题
        if not settingList or not targetName:
            self.output(u'优化设置有问题,请检查')

        # 多进程优化,启动一个对应CPU核心数量的进程池
        pool = multiprocessing.Pool(multiprocessing.cpu_count())
        l = []

        for setting in settingList:
            l.append(
                pool.apply_async(optimize,
                                 (strategyClass, setting, targetName,
                                  self.mode, self.startDate, self.initDays,
                                  self.endDate, self.slippage, self.rate,
                                  self.size, self.dbName, self.symbol)))
        pool.close()
        pool.join()

        # 显示结果
        resultList = [res.get() for res in l]
        resultList.sort(reverse=True, key=lambda result: result[1])
        self.output('-' * 30)
        self.output(u'优化结果:')
        for result in resultList:
            self.output(u'%s: %s' % (result[0], result[1]))

    #----------------------------------------------------------------------
    def roundToPriceTick(self, price):
        """取整价格到合约最小价格变动"""
        if not self.priceTick:
            return price

        newPrice = round(price / self.priceTick, 0) * self.priceTick
        return newPrice
Ejemplo n.º 60
0
class DownloadManager(ThreadManager):
    """"""
    def __init__(self):
        """"""
        ThreadManager.__init__(self)
        self.active_downloads = {}
        self.queue_downloads = OrderedDict()
        self.complete_downloads = {}
        self.stopped_downloads = {}
        self.global_slots = Slots()

    def reorder_queue(self, id_order_list):
        """"""
        ordered_items_dict = OrderedDict()
        for id_item in id_order_list:
            try:
                ordered_items_dict[id_item] = self.queue_downloads[id_item]
            except KeyError:
                pass
        if len(self.queue_downloads) == len(ordered_items_dict):
            self.queue_downloads.clear()
            self.queue_downloads.update(ordered_items_dict)
        else:
            logger.warning("reorder_queue failed")

    def start_all(self, id_order_list):
        """"""
        self.queue_downloads.update(self.stopped_downloads)
        self.stopped_downloads.clear()
        self.reorder_queue(id_order_list)
        self.next_download()

    def stop_all(self):
        """"""
        for id_item, download_item in self.active_downloads.iteritems():
            self.stop_thread(id_item)
        for id_item, download_item in self.queue_downloads.items():
            self.stopped_downloads[id_item] = download_item
            del self.queue_downloads[id_item]

    def start_download(self, id_item):
        """"""
        try:
            download_item = self.stopped_downloads.pop(id_item)
        except KeyError:
            return False
        else:
            self.queue_downloads[id_item] = download_item
            self.next_download()
            return True

    def stop_download(self, id_item):
        """"""
        try:
            download_item = self.active_downloads[id_item]
        except KeyError:
            try:
                download_item = self.queue_downloads.pop(id_item)
                self.stopped_downloads[id_item] = download_item
            except KeyError:
                return False
            else:
                return True
        else:  # active
            self.stop_thread(id_item)
            return True

    def delete_download(self, id_items_list, remove_file=False):
        """"""
        for id_item in id_items_list:
            th = None
            try:
                download_item = self.active_downloads.pop(id_item)
            except KeyError:
                try:
                    download_item = self.stopped_downloads.pop(id_item)
                except KeyError:
                    try:
                        download_item = self.queue_downloads.pop(id_item)
                    except KeyError:
                        try:
                            download_item = self.complete_downloads.pop(
                                id_item)
                        except KeyError:
                            raise
            else:  # active
                th = self.get_thread(id_item)
                self.stop_thread(id_item)
                self.delete_thread(id_item)
                self.global_slots.remove_slot()
                self.next_download()

            if remove_file:
                threading.Thread(group=None,
                                 target=self.remove_file,
                                 name=None,
                                 args=(download_item, th)).start()

    def remove_file(self, download_item, th):
        """"""
        if th is not None:
            th.join()
        try:
            os.remove(os.path.join(download_item.path, download_item.name))
        except Exception as err:
            logger.warning(err)

    def update_active_downloads(self):
        """
        This may change the active_downloads dict, you should get a dict copy before calling this method.
        """
        for id_item, download_item in self.active_downloads.items():
            th = self.thread_downloads[download_item.id]
            self.update_download_item(th, download_item)
            old_status = download_item.status
            if old_status in (cons.STATUS_STOPPED, cons.STATUS_FINISHED,
                              cons.STATUS_ERROR):

                if old_status == cons.STATUS_STOPPED:
                    self.stopped_downloads[id_item] = download_item
                elif old_status == cons.STATUS_FINISHED:
                    self.complete_downloads[id_item] = download_item
                else:  #status == cons.STATUS_ERROR
                    download_item.fail_count += 1
                    if download_item.fail_count > conf.get_retries_limit():
                        download_item.status = cons.STATUS_STOPPED
                        self.stopped_downloads[id_item] = download_item
                    else:
                        download_item.status = cons.STATUS_QUEUE
                        self.queue_downloads[id_item] = download_item

                del self.thread_downloads[id_item]
                del self.active_downloads[id_item]
                self.global_slots.remove_slot()
                self.next_download()

                if old_status == cons.STATUS_FINISHED:
                    events.download_complete.emit(download_item)
                if not self.active_downloads and old_status != cons.STATUS_STOPPED:
                    events.all_downloads_complete.emit()
                if th.limit_exceeded and self.active_downloads and old_status == cons.STATUS_ERROR:
                    events.limit_exceeded.emit()

    def update_download_item(self, th, download_item):
        #th = self.thread_downloads[download_item.id]

        if th.error_flag and th.stop_flag:  #fix on stopped failed download
            status = cons.STATUS_STOPPED
        else:
            status = th.status  #get before any other attr

        download_item.status = status
        download_item.chunks, download_item.size_complete = th.get_chunk_n_size(
        )
        download_item.name = th.file_name
        download_item.status_msg = th.status_msg
        download_item.size = th.size_file
        download_item.start_time = th.start_time
        download_item.size_resume = th.size_tmp
        download_item.can_resume = th.can_resume
        download_item.is_premium = th.is_premium
        download_item.video_quality = th.video_quality
        download_item.calc_stats()

    def add_to_downloader(self, download_item):
        self.queue_downloads[download_item.id] = download_item

    def next_download(self):
        for download_item in self.queue_downloads.values():
            self.download_starter(download_item)
            if not self.global_slots.available_slot():
                break

    def download_starter(self, download_item):
        """"""
        if not self.global_slots.available_slot():
            return
        elif not self.is_host_slot_available(
                download_item.host
        ):  # and host_accounts.get_account(download_item.host) is None:
            return
        else:
            download_item.fail_count = 0
            self.global_slots.add_slot()
            self.create_thread(download_item)  #threadmanager
            self.active_downloads[download_item.id] = download_item
            del self.queue_downloads[download_item.id]

    def is_host_slot_available(self, host):
        """"""
        count = 0
        host_slots = plugins_config.get_plugin_item(host).get_slots_limit()
        if host_slots > 0:  # -1 or 0 means unlimited slots
            for download_item in self.active_downloads.itervalues():
                if host == download_item.host:
                    count += 1
                if count >= host_slots:
                    return False
        return True

    def new_slot_limit(self, new_limit):
        """"""
        current_limit = self.global_slots.get_limit()
        self.global_slots.set_limit(new_limit)
        if new_limit > current_limit:
            self.next_download()