def _tagValuedProperties(self, content_type): """Document properties for property files having constructs like <ns:name>value</ns:name> @param content_type: contenttypes.CT_CORE_PROPS or contenttypes.CT_EXT_PROPS @return: mapping like {'property name': 'property value', ...} """ rval = {} if not content_type in self.content_types.listMetaContentTypes: # We fail silently return rval for tree in self.content_types.getTreesFor(self, content_type): for elt in tree.getroot().getchildren(): tag = elt.tag.split('}')[-1] # Removing namespace if any rval[toUnicode(tag)] = toUnicode(elt.text) return rval
def customProperties(self): """Document custom properties added by the document author. We canot convert the properties as indicated with the http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes namespace :return: mapping of metadata """ rval = {} if len( self.content_types.getPathsForContentType( contenttypes.CT_CUSTOM_PROPS)) == 0: # We may have no custom properties at all. return rval XPath = lxml.etree.XPath # Class shortcut properties_xpath = XPath('custom-properties:property', namespaces=ns_map) propname_xpath = XPath('@name') propvalue_xpath = XPath('*/text()') for tree in self.content_types.getTreesFor( self, contenttypes.CT_CUSTOM_PROPS): for elt in properties_xpath(tree.getroot()): rval[toUnicode(propname_xpath(elt)[0])] = u" ".join( propvalue_xpath(elt)) return rval
def dbMemoizedFunction(self, url): stats.calls += 1 cursor.execute("select result from mb where url=?", (toUnicode(url),)) result = cursor.fetchone() if result: stats.hits += 1 #log("Hit MusicBrainz DB cache (%s)." % stats) text = result[0].encode("UTF-8") else: #log("Missed MusicBrainz DB cache (%s)." % stats) sleep(1) result = fn(self, url) text = result.read() cursor.execute("insert into mb values (?, ?)", (toUnicode(url), toUnicode(text))) return StringIO.StringIO(text) # fn must return a file-like object
def dbMemoizedFunction(self, url): stats.calls += 1 cursor.execute("select result from mb where url=?", (toUnicode(url), )) result = cursor.fetchone() if result: stats.hits += 1 #log("Hit MusicBrainz DB cache (%s)." % stats) text = result[0].encode("UTF-8") else: #log("Missed MusicBrainz DB cache (%s)." % stats) sleep(1) result = fn(self, url) text = result.read() cursor.execute("insert into mb values (?, ?)", (toUnicode(url), toUnicode(text))) return StringIO.StringIO(text) # fn must return a file-like object
def to_uci_id(v): if v is None: return None try: v = unicode(long(v)) except: v = toUnicode(v) return reUCIID.sub(u'', v)
def to_int_str(v): if v is None: return None fix_spaces(v) try: return unicode(long(v)) except: pass return toUnicode(v)
def dispatcher(self): if not self.message: pass # you can define your rule here, and return right handler if self.message.msgType == "text": content = self.message.content if content.find("help") >= 0 or content.find("Help") >= 0 \ or content.find("?") >= 0 or content.find(toUnicode("?")) >= 0: logInfo("select HelpHandler to service request") return HelpHandler(self.message) if content.startswith("jj") or content.startswith("jijin") \ or content.startswith("fund"): logInfo("select QueryFundHandler to service request") return QueryFundHandler(self.message) if content.startswith("gp") or content.startswith("gupiao") \ or content.startswith("stock"): logInfo("select QueryStockHandler to service request") return QueryStockHandler(self.message) if content.startswith("days"): logInfo("select CalcDaysBeforeHandler to service request") return CalcDaysUntilNowHandler(self.message) if content.startswith("fm") or content.startswith("FM"): logInfo("select DoubanFMHandler to service request") return DoubanFMHandler(self.message) if content.startswith("sbgp") or content.startswith("SBGP"): logInfo("select SubscribeStockHandler to service request") return SubscribeStockHandler(self.message) if content == "ping" or content == "Ping": logInfo("select PingPongHandler to service request") return PingPongHandler(self.message) return HelpHandler(self.message) elif self.message.msgType == "event": event = self.message.event if event == "subscribe": logInfo("select SubscribeEventHandler to service request") return SubscribeEventHandler(self.message) elif event == "unsubscribe": logInfo("select UnsubscribeEventHandler to service request") return UnsubscribeEventHandler(self.message) else: return HelpHandler(self.message) else: return PingPongHandler(self.message)
def normalize(self, view, user): do_load = True for name, value in self.items(): if isinstance(value, str): value = self[name] = utils.toUnicode(value) try: value = self._parseValue(name, value) except KeyError: if self.allow_extra_parameters: continue raise InvalidParameterError(name) if not self._parameters.has_key(name) or self._parameters[name]["mandatory"] is not True: do_load = False if self._parameters[name]["save"] and self.has_key("_save"): user.setConfigValue(view, name, value) self[name] = value # Go through unset parameters. # - Error out on mandatory parameters, # - Load default value for optional parameters that got one. # - Load last user value for parameter. for name in self._parameters.keys(): got_param = self.has_key(name) if not got_param: if self._parameters[name]["mandatory"]: raise MissingParameterError(name) elif self._parameters[name]["default"] != None: self[name] = self._parameters[name]["default"] if self._parameters[name]["save"]: try: value = self._parseValue(name, user.getConfigValue(view, name)) except KeyError: continue self._default[name] = value if do_load and not got_param: self[name] = value try: self.pop("_save") except: pass return do_load
def customProperties(self): """Document custom properties @return: mapping of metadata FIXME: This is ugly. We do not convert the properties as indicated with the http://schemas.openxmlformats.org/officeDocument/2006/docPropsVTypes namespace """ rval = {} if len(self.content_types.getPathsForContentType(contenttypes.CT_CUSTOM_PROPS)) == 0: # We may have no custom properties at all. return rval XPath = lxml.etree.XPath # Class shortcut properties_xpath = XPath('custom-properties:property', namespaces=ns_map) propname_xpath = XPath('@name') propvalue_xpath = XPath('*/text()') for tree in self.content_types.getTreesFor(self, contenttypes.CT_CUSTOM_PROPS): for elt in properties_xpath(tree.getroot()): rval[toUnicode(propname_xpath(elt)[0])] = u" ".join(propvalue_xpath(elt)) return rval
def init_ccn(fname=fnameDefault): #large_delete_all( LicenseHolder ) #large_delete_all( Team ) tstart = datetime.datetime.now() fix_bad_license_codes() discipline_id = dict( (discipline, Discipline.objects.get(name=discipline)) for discipline in ['Road', 'Track', 'Cyclocross', 'MTB']) effective_date = datetime.date.today() # Process the records in large transactions for efficiency. @transaction.atomic def process_ur_records(ur_records): for i, ur in ur_records: try: date_of_birth = date_from_value(ur.get('DOB', '')) except Exception as e: safe_print(u'Row {}: Invalid birthdate "{}" ({}) {}'.format( i, ur.get('DOB', ''), ur, e)) continue if not ur.get('License Numbers', ''): safe_print(u'Row {}: Missing License Code '.format(i)) continue attributes = { 'license_code': to_int_str(ur.get('License Numbers', '')), 'last_name': to_str(ur.get('Last Name', '')), 'first_name': to_str(ur.get('First Name', '')), 'gender': gender_from_str(to_str(ur.get('Sex', 'm'))), 'date_of_birth': date_of_birth, 'city': to_str(ur.get('Member City', '')), 'state_prov': to_str(ur.get('Member Province', '')), 'nationality': to_str(ur.get('Nationality', '')), 'zip_postal': to_str( get_key(ur, ( 'ZipPostal', 'Zip' 'Postal', 'ZipCode', 'PostalCode', 'Zip Code', 'Postal Code', ), None)) } attributes = {a: v for a, v in attributes.items() if v is not None} if ur.get('Tag', '').strip(): attributes['existing_tag'] = to_int_str(ur.get('Tag', '')).strip() try: lh = LicenseHolder.objects.get( license_code=attributes['license_code']) if set_attributes(lh, attributes): lh.save() except LicenseHolder.DoesNotExist: lh = LicenseHolder(**attributes) lh.save() safe_print( u'{i:>6}: {license:>8} {dob:>10} {uci_code} {last}, {first}, {city}, {state_prov}' .format(i=i, license=removeDiacritic(lh.license_code), dob=lh.date_of_birth.strftime('%Y/%m/%d'), uci_code=lh.uci_code, last=removeDiacritic(lh.last_name), first=removeDiacritic(lh.first_name), city=removeDiacritic(lh.city), state_prov=removeDiacritic(lh.state_prov))) TeamHint.objects.filter(license_holder=lh).delete() teams = [t.strip() for t in ur.get('Afiliates', '').split(',')] if teams: last_team = None for t in teams: team = Team.objects.get_or_create(name=t)[0] last_team = team pcd = ur.get('Primary Cycling Discipline', '').strip().lower() for id, d in six.iteritems(discipline_id): if d.name.lower() in pcd: TeamHint(discipline=d, license_holder=lh, effective_date=effective_date, team=last_team) break suffix = 'CCN' ur_records = [] wb = open_workbook(fname) import_utils.datemode = wb.datemode ws = None for sheet_name in wb.sheet_names(): if sheet_name.endswith(suffix): safe_print('Reading sheet: {}'.format(sheet_name)) ws = wb.sheet_by_name(sheet_name) break if not ws: safe_print(u'Cannot find sheet ending with "{}"'.format(suffix)) return num_rows = ws.nrows num_cols = ws.ncols for r in six.moves.range(num_rows): row = ws.row(r) if r == 0: # Get the header fields from the first row. fields = [toUnicode(f.value).strip() for f in row] safe_print(u'\n'.join(fields)) continue ur = dict((f, row[c].value) for c, f in enumerate(fields)) ur_records.append((r + 1, ur)) if len(ur_records) == 3000: process_ur_records(ur_records) ur_records = [] process_ur_records(ur_records) safe_print('Initialization in: ', datetime.datetime.now() - tstart)
def to_str(v): return toUnicode(v).strip()
def to_int_str(v): try: return u'{}'.format(v) except: pass return toUnicode(v)
def to_str(v): if v is None: return v return fix_spaces(toUnicode(v)).strip()
def to_int_str( v ): try: return unicode(long(v)) except: pass return toUnicode(v)
def unicode_csv_reader(utf8_data, dialect=csv.excel, **kwargs): csv_reader = csv.reader(utf8_data, dialect=dialect, **kwargs) for row in csv_reader: yield [toUnicode(cell) for cell in row]