Example #1
0
 def _tagValuedProperties(self, content_type):
     """Document properties for property files having constructs like
      <ns:name>value</ns:name>
      @param content_type: contenttypes.CT_CORE_PROPS or contenttypes.CT_EXT_PROPS
      @return: mapping like {'property name': 'property value', ...}
     """
     rval = {}
     if not content_type in self.content_types.listMetaContentTypes:
         # We fail silently
         return rval
     for tree in self.content_types.getTreesFor(self, content_type):
         for elt in tree.getroot().getchildren():
             tag = elt.tag.split('}')[-1] # Removing namespace if any
             rval[toUnicode(tag)] = toUnicode(elt.text)
     return rval
Example #2
0
 def _tagValuedProperties(self, content_type):
     """Document properties for property files having constructs like
      <ns:name>value</ns:name>
      @param content_type: contenttypes.CT_CORE_PROPS or contenttypes.CT_EXT_PROPS
      @return: mapping like {'property name': 'property value', ...}
     """
     rval = {}
     if not content_type in self.content_types.listMetaContentTypes:
         # We fail silently
         return rval
     for tree in self.content_types.getTreesFor(self, content_type):
         for elt in tree.getroot().getchildren():
             tag = elt.tag.split('}')[-1]  # Removing namespace if any
             rval[toUnicode(tag)] = toUnicode(elt.text)
     return rval
Example #3
0
    def customProperties(self):
        """Document custom properties added by the document author.

        We canot convert the properties as indicated
        with the http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes
        namespace

        :return: mapping of metadata
        """
        rval = {}
        if len(
                self.content_types.getPathsForContentType(
                    contenttypes.CT_CUSTOM_PROPS)) == 0:
            # We may have no custom properties at all.
            return rval
        XPath = lxml.etree.XPath  # Class shortcut
        properties_xpath = XPath('custom-properties:property',
                                 namespaces=ns_map)
        propname_xpath = XPath('@name')
        propvalue_xpath = XPath('*/text()')
        for tree in self.content_types.getTreesFor(
                self, contenttypes.CT_CUSTOM_PROPS):
            for elt in properties_xpath(tree.getroot()):
                rval[toUnicode(propname_xpath(elt)[0])] = u" ".join(
                    propvalue_xpath(elt))
        return rval
Example #4
0
 def dbMemoizedFunction(self, url):
     stats.calls += 1
     cursor.execute("select result from mb where url=?", (toUnicode(url),))
     result = cursor.fetchone()
     
     if result:
         stats.hits += 1
         #log("Hit MusicBrainz DB cache (%s)." % stats)
         text = result[0].encode("UTF-8")
     else:
         #log("Missed MusicBrainz DB cache (%s)." % stats)
         sleep(1)
         result = fn(self, url)
         text = result.read()
         cursor.execute("insert into mb values (?, ?)", (toUnicode(url), toUnicode(text)))
     return StringIO.StringIO(text)   # fn must return a file-like object
Example #5
0
    def dbMemoizedFunction(self, url):
        stats.calls += 1
        cursor.execute("select result from mb where url=?", (toUnicode(url), ))
        result = cursor.fetchone()

        if result:
            stats.hits += 1
            #log("Hit MusicBrainz DB cache (%s)." % stats)
            text = result[0].encode("UTF-8")
        else:
            #log("Missed MusicBrainz DB cache (%s)." % stats)
            sleep(1)
            result = fn(self, url)
            text = result.read()
            cursor.execute("insert into mb values (?, ?)",
                           (toUnicode(url), toUnicode(text)))
        return StringIO.StringIO(text)  # fn must return a file-like object
Example #6
0
def to_uci_id(v):
    if v is None:
        return None
    try:
        v = unicode(long(v))
    except:
        v = toUnicode(v)
    return reUCIID.sub(u'', v)
Example #7
0
def to_int_str(v):
    if v is None:
        return None
    fix_spaces(v)
    try:
        return unicode(long(v))
    except:
        pass
    return toUnicode(v)
Example #8
0
    def dispatcher(self):
        if not self.message:
            pass

        # you can define your rule here, and return right handler
 
        if self.message.msgType == "text":
            content = self.message.content
            
            if content.find("help") >= 0 or content.find("Help") >= 0 \
                or content.find("?") >= 0 or content.find(toUnicode("?")) >= 0:
                logInfo("select HelpHandler to service request")
                return HelpHandler(self.message)
            
            if content.startswith("jj") or content.startswith("jijin") \
                or content.startswith("fund"):
                logInfo("select QueryFundHandler to service request")
                return QueryFundHandler(self.message)
            
            if content.startswith("gp") or content.startswith("gupiao") \
                or content.startswith("stock"):
                logInfo("select QueryStockHandler to service request")
                return QueryStockHandler(self.message)
            
            if content.startswith("days"):
                logInfo("select CalcDaysBeforeHandler to service request")
                return CalcDaysUntilNowHandler(self.message)
            
            if content.startswith("fm") or content.startswith("FM"):
                logInfo("select DoubanFMHandler to service request")
                return DoubanFMHandler(self.message)
            
            if content.startswith("sbgp") or content.startswith("SBGP"):
                logInfo("select SubscribeStockHandler to service request")
                return SubscribeStockHandler(self.message)
            
            if content == "ping" or content == "Ping":
                logInfo("select PingPongHandler to service request")
                return PingPongHandler(self.message)
            
            return HelpHandler(self.message)
        elif self.message.msgType == "event":
            event = self.message.event
            
            if event == "subscribe":
                logInfo("select SubscribeEventHandler to service request")
                return SubscribeEventHandler(self.message)
            elif event == "unsubscribe":
                logInfo("select UnsubscribeEventHandler to service request")
                return UnsubscribeEventHandler(self.message)
            else:
                return HelpHandler(self.message)
        else:
            return PingPongHandler(self.message)
Example #9
0
    def normalize(self, view, user):
        do_load = True

        for name, value in self.items():
            if isinstance(value, str):
                value = self[name] = utils.toUnicode(value)

            try:
                value = self._parseValue(name, value)
            except KeyError:
                if self.allow_extra_parameters:
                    continue

                raise InvalidParameterError(name)

            if not self._parameters.has_key(name) or self._parameters[name]["mandatory"] is not True:
                do_load = False

            if self._parameters[name]["save"] and self.has_key("_save"):
                user.setConfigValue(view, name, value)

            self[name] = value

        # Go through unset parameters.
        # - Error out on mandatory parameters,
        # - Load default value for optional parameters that got one.
        # - Load last user value for parameter.

        for name in self._parameters.keys():
            got_param = self.has_key(name)
            if not got_param:
                if self._parameters[name]["mandatory"]:
                    raise MissingParameterError(name)

                elif self._parameters[name]["default"] != None:
                    self[name] = self._parameters[name]["default"]

            if self._parameters[name]["save"]:
                try:
                    value = self._parseValue(name, user.getConfigValue(view, name))
                except KeyError:
                    continue

                self._default[name] = value
                if do_load and not got_param:
                    self[name] = value

        try:
            self.pop("_save")
        except:
            pass

        return do_load
Example #10
0
 def customProperties(self):
     """Document custom properties
     @return: mapping of metadata
     FIXME: This is ugly. We do not convert the properties as indicated
     with the http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes
     namespace
     """
     rval = {}
     if len(self.content_types.getPathsForContentType(contenttypes.CT_CUSTOM_PROPS)) == 0:
         # We may have no custom properties at all.
         return rval
     XPath = lxml.etree.XPath # Class shortcut
     properties_xpath = XPath('custom-properties:property', namespaces=ns_map)
     propname_xpath = XPath('@name')
     propvalue_xpath = XPath('*/text()')
     for tree in self.content_types.getTreesFor(self, contenttypes.CT_CUSTOM_PROPS):
         for elt in properties_xpath(tree.getroot()):
             rval[toUnicode(propname_xpath(elt)[0])] = u" ".join(propvalue_xpath(elt))
     return rval
Example #11
0
def init_ccn(fname=fnameDefault):

    #large_delete_all( LicenseHolder )
    #large_delete_all( Team )

    tstart = datetime.datetime.now()

    fix_bad_license_codes()

    discipline_id = dict(
        (discipline, Discipline.objects.get(name=discipline))
        for discipline in ['Road', 'Track', 'Cyclocross', 'MTB'])

    effective_date = datetime.date.today()

    # Process the records in large transactions for efficiency.
    @transaction.atomic
    def process_ur_records(ur_records):

        for i, ur in ur_records:
            try:
                date_of_birth = date_from_value(ur.get('DOB', ''))
            except Exception as e:
                safe_print(u'Row {}: Invalid birthdate "{}" ({}) {}'.format(
                    i, ur.get('DOB', ''), ur, e))
                continue

            if not ur.get('License Numbers', ''):
                safe_print(u'Row {}: Missing License Code '.format(i))
                continue

            attributes = {
                'license_code':
                to_int_str(ur.get('License Numbers', '')),
                'last_name':
                to_str(ur.get('Last Name', '')),
                'first_name':
                to_str(ur.get('First Name', '')),
                'gender':
                gender_from_str(to_str(ur.get('Sex', 'm'))),
                'date_of_birth':
                date_of_birth,
                'city':
                to_str(ur.get('Member City', '')),
                'state_prov':
                to_str(ur.get('Member Province', '')),
                'nationality':
                to_str(ur.get('Nationality', '')),
                'zip_postal':
                to_str(
                    get_key(ur, (
                        'ZipPostal',
                        'Zip'
                        'Postal',
                        'ZipCode',
                        'PostalCode',
                        'Zip Code',
                        'Postal Code',
                    ), None))
            }
            attributes = {a: v for a, v in attributes.items() if v is not None}

            if ur.get('Tag', '').strip():
                attributes['existing_tag'] = to_int_str(ur.get('Tag',
                                                               '')).strip()

            try:
                lh = LicenseHolder.objects.get(
                    license_code=attributes['license_code'])
                if set_attributes(lh, attributes):
                    lh.save()

            except LicenseHolder.DoesNotExist:
                lh = LicenseHolder(**attributes)
                lh.save()

            safe_print(
                u'{i:>6}: {license:>8} {dob:>10} {uci_code} {last}, {first}, {city}, {state_prov}'
                .format(i=i,
                        license=removeDiacritic(lh.license_code),
                        dob=lh.date_of_birth.strftime('%Y/%m/%d'),
                        uci_code=lh.uci_code,
                        last=removeDiacritic(lh.last_name),
                        first=removeDiacritic(lh.first_name),
                        city=removeDiacritic(lh.city),
                        state_prov=removeDiacritic(lh.state_prov)))
            TeamHint.objects.filter(license_holder=lh).delete()

            teams = [t.strip() for t in ur.get('Afiliates', '').split(',')]
            if teams:
                last_team = None
                for t in teams:
                    team = Team.objects.get_or_create(name=t)[0]
                    last_team = team
                pcd = ur.get('Primary Cycling Discipline', '').strip().lower()
                for id, d in six.iteritems(discipline_id):
                    if d.name.lower() in pcd:
                        TeamHint(discipline=d,
                                 license_holder=lh,
                                 effective_date=effective_date,
                                 team=last_team)
                        break

    suffix = 'CCN'
    ur_records = []
    wb = open_workbook(fname)
    import_utils.datemode = wb.datemode

    ws = None
    for sheet_name in wb.sheet_names():
        if sheet_name.endswith(suffix):
            safe_print('Reading sheet: {}'.format(sheet_name))
            ws = wb.sheet_by_name(sheet_name)
            break

    if not ws:
        safe_print(u'Cannot find sheet ending with "{}"'.format(suffix))
        return

    num_rows = ws.nrows
    num_cols = ws.ncols
    for r in six.moves.range(num_rows):
        row = ws.row(r)
        if r == 0:
            # Get the header fields from the first row.
            fields = [toUnicode(f.value).strip() for f in row]
            safe_print(u'\n'.join(fields))
            continue

        ur = dict((f, row[c].value) for c, f in enumerate(fields))
        ur_records.append((r + 1, ur))
        if len(ur_records) == 3000:
            process_ur_records(ur_records)
            ur_records = []

    process_ur_records(ur_records)

    safe_print('Initialization in: ', datetime.datetime.now() - tstart)
Example #12
0
def to_str(v):
    return toUnicode(v).strip()
Example #13
0
def to_int_str(v):
    try:
        return u'{}'.format(v)
    except:
        pass
    return toUnicode(v)
Example #14
0
def to_str(v):
    if v is None:
        return v
    return fix_spaces(toUnicode(v)).strip()
Example #15
0
def to_int_str( v ):
	try:
		return unicode(long(v))
	except:
		pass
	return toUnicode(v)
Example #16
0
def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs):
    csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs)
    for row in csv_reader:
        yield [toUnicode(cell) for cell in row]