def test_map_compose_filter(self):
def filter_world(x):
return None if x == 'world' else x
proc = MapCompose(filter_world, str.upper)
self.assertEqual(proc(['hello', 'world', 'this', 'is', 'scrapy']),
['HELLO', 'THIS', 'IS', 'SCRAPY'])
def test_mapcompose(self):
filter_world = lambda x: None if x == 'world' else x
proc = MapCompose(filter_world, six.text_type.upper)
self.assertEqual(proc([u'hello', u'world', u'this', u'is', u'scrapy']),
[u'HELLO', u'THIS', u'IS', u'SCRAPY'])
class DefaultedItemLoader(NameItemLoader):
default_input_processor = MapCompose(lambda v: v[:-1])
class TstItemLoader(NameItemLoader):
name_in = MapCompose(lambda v: v.title())
class ChildItemLoader(TstItemLoader):
url_in = MapCompose(processor)
class ChildItemLoader(TstItemLoader):
url_in = MapCompose(processor_with_args, key=u'val')
class TstItemLoader(ItemLoader):
default_item_class = TstItem
name_out = MapCompose(float)
class ChildDefaultedItemLoader(DefaultedItemLoader):
name_in = MapCompose(DefaultedItemLoader.default_input_processor,
six.text_type.swapcase)
class ChildItemLoader(TstItemLoader):
name_in = MapCompose(TstItemLoader.name_in, six.text_type.swapcase)
class IdentityDefaultedItemLoader(DefaultedItemLoader):
name_in = MapCompose()
class ChildChildItemLoader(ChildItemLoader):
url_in = MapCompose(lambda v: v.upper())
summary_in = MapCompose(lambda v: v)
class ChildItemLoader(TstItemLoader):
url_in = MapCompose(lambda v: v.lower())
old_locale = localelib.getlocale(localelib.LC_TIME)
localelib.setlocale(localelib.LC_TIME, locale)
time_s = time.strptime(value, format)
dt = datetime.datetime(*time_s[0:5])
# 1900 is the default year from strptime, means no year parsed
if dt.year == 1900:
dt = dt.replace(year=datetime.datetime.utcnow().year)
if locale:
localelib.setlocale(localelib.LC_TIME, old_locale)
return dt
def to_date(value, format, locale=None):
return to_datetime(value, format, locale).date()
def to_time(value, format):
time_s = time.strptime(value, format)
return datetime.time(time_s[3], time_s[4])
# defaults
default_input_processor = MapCompose(replace_br, remove_tags, unquote_markup,
replace_escape, strip, clean_spaces)
default_output_processor = TakeFirst()