def update_rss(self): handler = self.handler errors = [] errors_str = [] articles = [] feeds_summary = {} # backup the default timeout default_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(self.get_property('timeout')) # Download the feeds for uri, keywords, active in handler.get_rows(): if active is False: continue keywords = [x.strip().lower() for x in keywords.split(',')] # TODO Use itools.vfs instead of urllib2 try: req = urllib2.Request(uri) req.add_header('User-Agent', 'itools/%s' % itools_version) response = urllib2.urlopen(req) data = response.read() except (socket.error, socket.gaierror, Exception, urllib2.HTTPError), e: msg = '%s <br />-- Network error: "%s"' msg = msg % (XMLContent.encode(str(uri)), e) msg = msg.encode('utf-8') errors.append(XMLParser(msg)) errors_str.append(msg) summary = ('rssfeeds, Error downloading feed\n' 'uri: %s\n\n' % str(uri)) details = format_exc() log_warning(summary + details, domain='itws') continue # Parse try: feed = RSSFile(string=data) except Exception, e: msg = '%s <br />-- Error parsing: "%s"' msg = msg % (XMLContent.encode(str(uri)), e) msg = msg.encode('utf-8') errors.append(XMLParser(msg)) errors_str.append(msg) summary = ('rssfeeds, Error parsing feed\n' 'uri: %s\n\n' % str(uri)) details = format_exc() log_warning(summary + details, domain='itws') continue
def get_namespace(self, context): # Build namespace namespace = {'author': self.get_namespace_author(context), 'href': context.get_link(self), 'images': self.get_images(context)} for key in ['title', 'note', 'advantages', 'disadvantages']: namespace[key] = self.get_property(key) # Add informations about product product = self.parent.parent namespace['product'] = {'link': context.get_link(product), 'title': product.get_title()} # Context here = context.resource namespace['is_on_user_view'] = here.class_id == 'user' namespace['is_on_product_view'] = here.class_id == 'product' # Description description = self.get_property('description').encode('utf-8') description = XMLContent.encode(description) namespace['description'] = XMLParser(description.replace('\n', '<br/>')) # ctime ctime = self.get_property('ctime') accept = context.accept_language namespace['ctime'] = format_datetime(ctime, accept) # Recommendation recommended = self.get_property('recommended') or 0 namespace['recommendation'] = bool(int(recommended)) return namespace
def _update_data(self): limit = self.get_property('limit') uri = self._get_account_uri() data = None # errors errors = [] errors_str = [] # backup the default timeout default_timeout = socket.getdefaulttimeout() socket.setdefaulttimeout(3) # timeout in seconds # TODO Use itools.vfs instead of urllib2 try: req = urllib2.Request(uri) req.add_header('User-Agent', 'itools/%s' % itools_version) response = urllib2.urlopen(req) data = response.read() except (socket.error, socket.gaierror, Exception, urllib2.HTTPError, urllib2.URLError), e: msg = '%s -- Network error: "%s"' msg = msg % (XMLContent.encode(str(uri)), e) msg = msg.encode('utf-8') errors.append(XMLParser(msg)) errors_str.append(msg) summary = 'sidebar/twitter, Error downloading feed\n' details = format_exc() log_warning(summary + details, domain='itws')
def encode_element(name, value, encoding='UTF-8'): # Encode datatype = schema.get(name, String) if issubclass(datatype, Unicode): result = datatype.encode(value, encoding) else: result = datatype.encode(value) # To XML return XMLContent.encode(result)
def GET(self, resource, context): parent = resource.parent try: doctree = resource.get_doctree() except SystemMessage, e: # Critical context.message = ERR_SYNTAX_ERROR(error=e.message) content = XMLContent.encode(resource.handler.to_str()) return '<pre>' + content + '</pre>'
def to_str(self): # Attributes attributes = [] for attr_name in self.attributes: attr_value = self.attributes[attr_name] attr_value = XMLContent.encode(attr_value) if attr_name == 'lang': attr_name = 'xml:lang' attributes.append(' %s="%s"' % (attr_name, attr_value)) attributes = ''.join(attributes) # Ok return '<note%s>%s</note>\n' % (attributes, self.text)
def get_item_value(self, resource, context, item, column): if column == 'img_path': img_path = resource.handler.get_record_value(item, column) # NOTE img_path is unicode multiple -> multilingual image = resource.get_resource(str(img_path), soft=True) if not image: return None return get_resource_preview(image, 128, 64, 0, context) elif column == 'img_link': img_link = resource.handler.get_record_value(item, column) reference = get_reference(img_link) if reference.scheme: # Encode the reference '&' to avoid XMLError reference = XMLContent.encode(str(reference)) return XMLParser('<a href="%s">%s</a>' % (reference, reference)) # Split path/view reference_path = str(reference.path) view = None if reference_path.count(';'): reference_path, view = reference_path.split('/;' ,1) item_resource = resource.get_resource(reference_path, soft=True) if not item_resource: # Not found, just return the reference # Encode the reference '&' to avoid XMLError return XMLContent.encode(str(reference)) # Build the new reference with the right path reference2 = deepcopy(reference) reference2.path = context.get_link(item_resource) if view: # Append the view reference2.path = '%s/;%s' % (reference2.path, view) # Encode the reference '&' to avoid XMLError # Reference : the reference used for the a content reference = XMLContent.encode(str(reference)) # Reference2 : the reference used for href attribute reference2 = XMLContent.encode(str(reference2)) return XMLParser('<a href="%s">%s</a>' % (reference2, reference)) return Table_View.get_item_value(self, resource, context, item, column)
def get_item_value(self, resource, context, item, column): if column == 'img_path': img_path = resource.handler.get_record_value(item, column) image = resource.get_resource(str(img_path), soft=True) if not image: return None return get_resource_preview(image, 128, 64, 0, context) elif column == 'img_link': img_link = resource.handler.get_record_value(item, column) reference = get_reference(img_link) if isinstance(reference, EmptyReference): return None if reference.scheme: # Encode the reference '&' to avoid XMLError ref = XMLContent.encode(str(reference)) return XMLParser('<a href="{ref}">{ref}</a>'.format(ref=ref)) # Split path/view reference_path, view = get_path_and_view(reference.path) item_resource = resource.get_resource(reference_path, soft=True) if not item_resource: # Not found, just return the reference # Encode the reference '&' to avoid XMLError return XMLContent.encode(str(reference)) # Build the new reference with the right path reference2 = deepcopy(reference) reference2.path = context.get_link(item_resource) if view: # Append the view reference2.path = '%s/;%s' % (reference2.path, view) # Encode the reference '&' to avoid XMLError # Reference : the reference used for the a content reference = XMLContent.encode(str(reference)) # Reference2 : the reference used for href attribute reference2 = XMLContent.encode(str(reference2)) return XMLParser('<a href="%s">%s</a>' % (reference2, reference)) return OrderedTable_View.get_item_value(self, resource, context, item, column)
def get_namespace(self, resource, context): # Render txt version as webmail txt_data = resource.get_email_text(context) txt_data = XMLContent.encode(txt_data) txt_data = re.sub(r'(\A|\s)(http://(\w|\.|/|:|;|\?|=|%|&|-)+)', r'\1<a href="\2" target="_blank"> \2</a>', txt_data) txt_data = XMLParser(txt_data.replace('\n', '<br/>')) # Return namespace return {'title': resource.get_title(), 'email_subject': resource.get_email_subject(context), 'spool_size': context.server.get_spool_size(), 'nb_users': resource.parent.get_subscripters_nb(), 'is_sent': resource.get_property('is_sent'), 'number': resource.get_property('number'), 'txt_data': txt_data}
def to_str(self): s = [] attributes = ['xml:lang="%s"' % self.attributes['lang']] for attr_name in self.attributes: if attr_name != 'lang': attributes.append('%s="%s"' % (attr_name, self.attributes[attr_name])) s.append(' <tuv %s>\n' % ' '.join(attributes)) for note in self.notes: s.append(note.to_str()) s.append(' <seg>%s</seg>\n' % XMLContent.encode(self.text)) s.append(' </tuv>\n') return ''.join(s)
def get_linked_resources_message(resource, context, state='public'): # Customize message if webpage uses private/pending resources referenced_resources = list(get_linked_resources(resource)) if len(referenced_resources) == 0: return None message = MSG(u'This {title} uses pending/private resources ' u'please go to ' u'<a href="{path}/;backlinks">backlinks interface</a>.') path = context.get_link(resource) path = XMLContent.encode(path) class_title = resource.class_title.gettext() message = message.gettext(title=class_title, path=path).encode('utf8') message = XHTMLBody.decode(message) # Return custom message return message
def to_str(self): # format, version, schema format = self.format version = self.version # Opening lines = ['<?xml version="1.0" encoding="UTF-8"?>\n', '<metadata format="%s" version="%s">\n' % (format, version)] # Sort properties names = self.properties.keys() names.sort() # Properties for name in names: value = self.properties[name] datatype = get_datatype(format, name) default = datatype.get_default() is_multiple = getattr(datatype, 'multiple', False) # Multilingual properties if isinstance(value, dict): template = ' <%s xml:lang="%s">%s</%s>\n' for language, value in value.items(): if value != default: value = datatype.encode(value) value = XMLContent.encode(value) lines.append(template % (name, language, value, name)) # Multiple values elif is_multiple: if not isinstance(value, list): raise TypeError, 'multiple values must be lists' # Record if issubclass(datatype, Record): aux = datatype.schema for value in value: lines.append(' <%s>\n' % name) for key, value in value.items(): value = aux.get(key, String).encode(value) value = XMLContent.encode(value) lines.append(' <%s>%s</%s>\n' % (key, value, key)) lines.append(' </%s>\n' % name) continue # Regular field for value in value: value = datatype.encode(value) value = XMLContent.encode(value) lines.append(' <%s>%s</%s>\n' % (name, value, name)) continue # Simple properties elif value != default: value = datatype.encode(value) if type(value) is not str: message = 'in "%s", property "%s" not encoded' raise TypeError, message % (self.key, name) value = XMLContent.encode(value) lines.append(' <%s>%s</%s>\n' % (name, value, name)) lines.append('</metadata>\n') return ''.join(lines)
def test_decode(self): self.assertEqual(XMLContent.decode(self.result1), self.data) self.assertEqual(XMLAttribute.decode(self.result2), self.data)
def test_encode(self): self.assertEqual(XMLContent.encode(self.data), self.result1) self.assertEqual(XMLAttribute.encode(self.data), self.result2)
def build_warn_referenced_msg(resource, context, total): path = context.get_link(resource) path = XMLContent.encode(path) message = MSG_UNPUBLISHED_RESOURCES_LINKED(path=path, n=total) message = message.gettext().encode('utf8') return XHTMLBody.decode(message)
def _get_translatable_blocks(events): # Default value encoding = 'utf-8' # To identify the begin/end format id = 0 id_stack = [] context_stack = [None] stream = None message = Message() skip_level = 0 for event in events: type, value, line = event # Set the good encoding if type == XML_DECL: encoding = value[1] # And now, we catch only the good events elif type == START_ELEMENT: if skip_level > 0: skip_level += 1 if stream: stream.append(event) continue else: tag_uri, tag_name, attributes = value schema = get_element_schema(tag_uri, tag_name) # Context management if schema.context is not None: context_stack.append(schema.context) # Skip content ? if schema.skip_content: skip_level = 1 if id_stack: stream = [event] continue # Is inline ? elif schema.is_inline: id += 1 id_stack.append(id) start_format = _make_start_format(tag_uri, tag_name, attributes, encoding) message.append_start_format(start_format, id, line) continue elif id_stack: skip_level = 1 stream = [event] continue elif type == END_ELEMENT: if skip_level > 0: skip_level -= 1 if stream: stream.append(event) if skip_level == 0: id += 1 aux = stream_to_str(stream, encoding) aux = unicode(aux, encoding) aux = [(aux, False, context_stack[-1])] message.append_start_format(aux, id, line) message.append_end_format([], id, line) stream = None continue else: tag_uri, tag_name = value[:2] schema = get_element_schema(tag_uri, tag_name) # Context management if schema.context is not None: context_stack.pop() # Is inline ? if schema.is_inline: message.append_end_format( [(get_end_tag(value), False, None)], id_stack.pop(), line) continue elif type == TEXT: # Not empty ? if stream: stream.append(event) continue elif skip_level == 0 and (value.strip() != '' or message): value = XMLContent.encode(value) value = unicode(value, encoding) message.append_text(value, line, context_stack[-1]) continue elif type == COMMENT: if stream: stream.append(event) continue elif message: id += 1 if isinstance(value, str): value = unicode(value, encoding) value = u'<!--%s-->' % value message.append_start_format([(value, False, None)], id, line) message.append_end_format([], id, line) continue # Not a good event => break + send the event if message: yield MESSAGE, message, message.get_line() message = Message() yield event # Send the last message! if message: yield MESSAGE, message, message.get_line()
def _choice_widget(context, form, datatype, name, value, schema, fields, readonly, container, container_attributes, choice, choice_attributes, choice_checked, multiline, tabindex): html = [] # True -> '1' ; False -> '2' data = datatype.encode(value) if readonly: for option in datatype.get_namespace(data): attributes = merge_dicts(choice_attributes, value=option['name'], disabled=u"disabled") attributes.setdefault(u"class", []).append(u"disabled") if option['selected']: attributes[choice_checked] = choice_checked value = option['value'] if is_prototype(value, MSG): value = value.gettext() content = XMLContent.encode(value) input = make_element(choice, attributes, content) if multiline is True: # Une option par ligne html.append(make_element(u"div", content=input)) else: html.append(input) else: if type(data) == type("") or type(data) == type(u""): try: data_tmp = data.decode("utf-8").split('|') except: data_tmp = data else: data_tmp = data for option in datatype.get_namespace(data_tmp): js = [] opt_name = option['name'] attributes = merge_dicts(choice_attributes, value=opt_name) if option['selected'] or data == option['name'].strip().encode( 'utf-8'): attributes[choice_checked] = choice_checked if form.is_disabled_by_dependency(name, schema, fields): attributes[u"disabled"] = u"disabled" attributes.setdefault(u"class", []).append(u"disabled") # 0005970: Le Javascript pour (dés)activer les autres champs dep_names = form.get_dep_names(name, schema) for dep_name in dep_names: dependency = schema[dep_name].dependency if not dependency: continue # Add js action on simple boolean expressions. ie: "C11" if not Expression.is_simple(dependency): # Authorized operators are "==", "=" and "in" # ie:"C11=='others'", "C11='others'", "'others' in C11" if 'in' in dependency: dep_value = dependency.split('in', 1)[0].strip() elif '==' in dependency: exclusive, dep_value = dependency.rsplit('==', 1) elif '!=' in dependency: exclusive, dep_value = dependency.rsplit('!=', 1) else: msg = 'K col operator unknown in "%s"' % dependency log_warning(msg) continue if dep_value[:1] in ("u'", 'u"'): dep_value = dep_value[:2] dep_value = (dep_value.replace("'", '').replace('"', '')) #dep_value = checkid(dep_value) else: dep_value = 'false' js_code = ( '$("input[name=\\"%s\\"][value=\\"%s\\"]")' '.change(function(){switch_input($(this),"%s","%s");});' % (name, opt_name, dep_name, dep_value)) js.append(js_code) # 0005970 fin value = option['value'] if is_prototype(value, MSG): value = value.gettext() content = XMLContent.encode(value) input = make_element(choice, attributes, content) if multiline is True: # Une option par ligne html.append(make_element(u"div", content=input)) else: html.append(input) # Append JS code content = XMLContent.encode(u'\n'.join(js)) attributes = {u'type': "text/javascript"} html.append(make_element(u'script', attributes, content)) attributes = merge_dicts(container_attributes, id=u"field_{name}".format(name=name)) check_errors(context, form, datatype, name, data, schema, fields, readonly, attributes, tabindex=None) return make_element(container, attributes, u"".join(html))
def action(self, resource, context, form): filename, mimetype, body = form['file'] # Clean up f*ing Google export with no quote body = cleanup_gmail_csv(body) csv = CSVFile() csv.load_state_from_string(body) rows = csv.get_rows() # Decode header header = rows.next() root = context.root language = context.site_root.get_default_language() companies = resource.get_resource('companies') contacts = resource.get_resource('contacts') abspath = str(resource.get_canonical_path()) base_path_query = get_base_path_query(abspath) contacts_added = {} contacts_updated = [] companies_added = {} for row in rows: # Find company company_title = find_value_by_column(header, row, GMAIL_COMPANY) if company_title: company = None # FIXME the catalog should do this key = company_title.lower().translate(transmap) if key in companies_added: # Already added company = companies_added[key] else: # Search existing company query = AndQuery(base_path_query, PhraseQuery('format', 'company'), TextQuery('title', company_title)) results = root.search(query) for document in results.get_documents(): # Found company = root.get_resource(document.abspath) break else: # Creating company company = companies.add_company( title={language: company_title}) companies_added[key] = company else: company = thingy(name=None) # Find contact by name and company if available contact = None firstname = find_value_by_column(header, row, GMAIL_FIRST_NAME) lastname = find_value_by_column(header, row, GMAIL_LAST_NAME) email = find_value_by_column(header, row, GMAIL_EMAIL) key = (firstname, lastname, email, company_title) if key in contacts_added: # Already added contact = contacts_added[key] else: # Search existing contact query = AndQuery(base_path_query, PhraseQuery('format', 'contact')) if firstname: query.append(TextQuery('crm_p_firstname', firstname)) if lastname: query.append(TextQuery('crm_p_lastname', lastname)) if email: query.append(TextQuery('crm_p_email', email)) if company.name is not None: query.append(PhraseQuery('crm_p_company', company.name)) results = root.search(query) for document in results.get_documents(): # Found contact = root.get_resource(document.abspath) contacts_updated.append(contact) break else: # Creating contact contact = contacts.add_contact( crm_p_firstname=firstname, crm_p_lastname=lastname, crm_p_email=email, crm_p_company=company.name, crm_p_status='lead') contacts_added[key] = contact # Update contact for title, name in import_columns.iteritems(): if title in (GMAIL_FIRST_NAME, GMAIL_LAST_NAME, GMAIL_EMAIL, GMAIL_COMPANY): continue value = find_value_by_column(header, row, title) if value is not None: datatype = contact.get_property_datatype(name) if issubclass(datatype, String): value = value.encode('utf8') contact.set_property(name, value) message = [] pattern = u'<a href="{0}">{1}</a>' if contacts_added: n = len(contacts_added) contacts_added = u", ".join(pattern.format( context.get_link(contact), XMLContent.encode(contact.get_title())) for contact in contacts_added.itervalues()) message.append(MSG_CONTACTS_ADDED(n=n, added=contacts_added)) if contacts_updated: n = len(contacts_updated) contacts_updated = u", ".join(pattern.format( context.get_link(contact), XMLContent.encode(contact.get_title())) for contact in contacts_updated) message.append(MSG_CONTACTS_UPDATED(n=n, updated=contacts_updated)) if not message: message = ERR_NO_CONTACT_FOUND context.message = message
def to_str(self): # format, version, schema format = self.format version = self.version # Opening lines = [ '<?xml version="1.0" encoding="UTF-8"?>\n', '<metadata format="%s" version="%s">\n' % (format, version) ] # Sort properties names = self.properties.keys() names.sort() # Properties for name in names: value = self.properties[name] datatype = get_datatype(format, name) default = datatype.get_default() is_multiple = getattr(datatype, 'multiple', False) # Multilingual properties if isinstance(value, dict): template = ' <%s xml:lang="%s">%s</%s>\n' for language, value in value.items(): if value != default: value = datatype.encode(value) value = XMLContent.encode(value) lines.append(template % (name, language, value, name)) # Multiple values elif is_multiple: if not isinstance(value, list): raise TypeError, 'multiple values must be lists' # Record if issubclass(datatype, Record): aux = datatype.schema for value in value: lines.append(' <%s>\n' % name) for key, value in value.items(): value = aux.get(key, String).encode(value) value = XMLContent.encode(value) lines.append(' <%s>%s</%s>\n' % (key, value, key)) lines.append(' </%s>\n' % name) continue # Regular field for value in value: value = datatype.encode(value) value = XMLContent.encode(value) lines.append(' <%s>%s</%s>\n' % (name, value, name)) continue # Simple properties elif value != default: value = datatype.encode(value) if type(value) is not str: message = 'in "%s", property "%s" not encoded' raise TypeError, message % (self.key, name) value = XMLContent.encode(value) lines.append(' <%s>%s</%s>\n' % (name, value, name)) lines.append('</metadata>\n') return ''.join(lines)
def _get_translatable_blocks(events): # Default value encoding = 'utf-8' # To identify the begin/end format id = 0 id_stack = [] context_stack = [None] stream = None message = Message() skip_level = 0 for event in events: type, value, line = event # Set the good encoding if type == XML_DECL: encoding = value[1] # And now, we catch only the good events elif type == START_ELEMENT: if skip_level > 0: skip_level += 1 if stream: stream.append(event) continue else: tag_uri, tag_name, attributes = value schema = get_element_schema(tag_uri, tag_name) # Context management if schema.context is not None: context_stack.append(schema.context) # Skip content ? if schema.skip_content: skip_level = 1 if id_stack: stream = [event] continue # Is inline ? elif schema.is_inline: id += 1 id_stack.append(id) start_format = _make_start_format(tag_uri, tag_name, attributes, encoding) message.append_start_format(start_format, id, line) continue elif id_stack: skip_level = 1 stream = [event] continue elif type == END_ELEMENT: if skip_level > 0: skip_level -= 1 if stream: stream.append(event) if skip_level == 0: id += 1 aux = stream_to_str(stream, encoding) aux = unicode(aux, encoding) aux = [(aux, False, context_stack[-1])] message.append_start_format(aux, id, line) message.append_end_format([], id, line) stream = None continue else: tag_uri, tag_name = value[:2] schema = get_element_schema(tag_uri, tag_name) # Context management if schema.context is not None: context_stack.pop() # Is inline ? if schema.is_inline: message.append_end_format([(get_end_tag(value), False, None)], id_stack.pop(), line) continue elif type == TEXT: # Not empty ? if stream: stream.append(event) continue elif skip_level == 0 and (value.strip() != '' or message): value = XMLContent.encode(value) value = unicode(value, encoding) message.append_text(value, line, context_stack[-1]) continue elif type == COMMENT: if stream: stream.append(event) continue elif message: id += 1 if isinstance(value, str): value = unicode(value, encoding) value = u'<!--%s-->' % value message.append_start_format([(value, False, None)], id, line) message.append_end_format([], id, line) continue # Not a good event => break + send the event if message: yield MESSAGE, message, message.get_line() message = Message() yield event # Send the last message! if message: yield MESSAGE, message, message.get_line()
def file_widget(context, form, datatype, name, value, schema, fields, readonly, tabindex=None): if readonly: tagname = u"div" attributes = {u"class": [u"readonly"]} content = XMLContent.encode(value) else: tagname = u"input" attributes = { u"type": u"file", u"id": u"field_{name}".format(name=name), u"name": name, u"value": XMLAttribute.encode(value), u"size": str(datatype.size), u"maxlength": str(datatype.length) } content = u"" # Check for errors check_errors(context, form, datatype, name, value, schema, fields, readonly, attributes, tabindex=None) html = make_element(tagname, attributes, content) # Preview if not value: return html resource = form.parent.get_resource(value, soft=True) if resource is None: return html context = get_context() link = context.get_link(resource) href = u"{0}/;download".format(link) src = u"{0}/;thumb?width=128&height=128".format(link) preview = make_element(u"a", { u"href": href, u"target": u"_new" }, make_element(u"img", {u"src": src})) field_id = u"field_{name}_delete".format(name=name) if readonly: delete = u"" else: delete = make_element( u"input", { u"type": u"checkbox", u"id": field_id, u"name": u"{name}_delete".format(name=name), u"value": u"1" }, make_element(u"label", {u"for": field_id}, MSG(u"Delete"))) html = html + preview + delete return html