def test_unserializable(self): # Hide an unserializable object as deep as we can in supported collections value = ['a', ('b', {'c': {'d', object()}})] with pytest.raises(TypeError): serialization.serialize(value) with pytest.raises(TypeError): serialization.dumps(value)
def upgrade(ver, session): if ver == 0: table = table_schema('input_cache_entry', session) table_add_column(table, 'json', Unicode, session) # Make sure we get the new schema with the added column table = table_schema('input_cache_entry', session) for row in session.execute(select([table.c.id, table.c.entry])): try: p = pickle.loads(row['entry']) session.execute( table.update().where(table.c.id == row['id']).values( json=json.dumps(p, encode_datetime=True))) except KeyError as e: logger.error( 'Unable error upgrading input_cache pickle object due to {}', str(e)) ver = 1 if ver == 1: table = table_schema('input_cache_entry', session) for row in session.execute(select([table.c.id, table.c.json])): if not row['json']: # Seems there could be invalid data somehow. See #2590 continue data = json.loads(row['json'], decode_datetime=True) # If title looked like a date, make sure it's a string title = str(data.pop('title')) e = Entry(title=title, **data) session.execute(table.update().where( table.c.id == row['id']).values(json=serialization.dumps(e))) ver = 2 return ver
def upgrade(ver: int, session: DBSession) -> int: if ver == 0: table = table_schema('input_cache_entry', session) table_add_column(table, 'json', Unicode, session) # Make sure we get the new schema with the added column table = table_schema('input_cache_entry', session) for row in session.execute(select([table.c.id, table.c.entry])): try: p = pickle.loads(row['entry']) session.execute( table.update().where(table.c.id == row['id']).values( json=json.dumps(p, encode_datetime=True))) except KeyError as ex: logger.error( f'Unable error upgrading input_cache pickle object due to {ex}' ) ver = 1 if ver == 1: table = table_schema('input_cache_entry', session) for row in session.execute(select([table.c.id, table.c.json])): if not row['json']: # Seems there could be invalid data somehow. See #2590 continue data = json.loads(row['json'], decode_datetime=True) # If title looked like a date, make sure it's a string # Had a weird case of an entry without a title: https://github.com/Flexget/Flexget/issues/2636 title = data.pop('title', None) entry = partial(Entry, **data) e = entry(title=str(title)) if title else entry() session.execute(table.update().where( table.c.id == row['id']).values(json=serialization.dumps(e))) ver = 2 return ver
def test_entry_serialization(self): entry1 = entry.Entry( { 'title': 'blah', 'url': 'http://blah', 'listfield': ['a', 'b', 1, 2], 'dictfield': {'a': 1, 'b': 2}, 'intfield': 5, 'floatfield': 5.5, 'datefield': datetime.date(1999, 9, 9), 'datetimefield': datetime.datetime(1999, 9, 9, 9, 9), 'qualityfield': qualities.Quality('720p hdtv'), 'nestedlist': [qualities.Quality('1080p')], 'nesteddict': {'a': datetime.date(1999, 9, 9)}, } ) entry1.add_lazy_fields('lazy function', ['lazyfield']) assert entry1.is_lazy('lazyfield') serialized = serialization.dumps(entry1) print(serialized) entry2 = serialization.loads(serialized) # Use the underlying dict, so we compare all fields assert entry2.is_lazy('lazyfield') assert dict(entry1) == dict(entry2) assert entry2['lazyfield'] == 'value a'
def test_builtin_serialization(self): # Also test these things nest properly value = { 'a': 'aoeu', 'b': [1, 2, 3.5], 'c': (1, datetime.datetime(2019, 12, 12, 12, 12)), 'd': {'a', 1, datetime.date(2019, 11, 11)} } out = serialization.dumps(value) backin = serialization.loads(out) assert backin == value
def setter(self, entry): if isinstance(entry, dict): if entry.get('serializer') == 'Entry' and 'version' in entry and 'value' in entry: # This is already a serialized form of entry setattr(self, name, json.dumps(entry)) return entry = Entry(entry) if isinstance(entry, Entry): setattr(self, name, serialization.dumps(entry)) else: raise TypeError(f'{type(entry)!r} is not type Entry or dict.')
def upgrade(ver, session): if ver is None: # Make sure there is no data we can't load in the backlog table backlog_table = table_schema('backlog', session) try: for item in session.query('entry').select_from( backlog_table).all(): pickle.loads(item.entry) except (ImportError, TypeError): # If there were problems, we can drop the data. logger.info( 'Backlog table contains unloadable data, clearing old data.') session.execute(backlog_table.delete()) ver = 0 if ver == 0: backlog_table = table_schema('backlog', session) logger.info('Creating index on backlog table.') Index('ix_backlog_feed_expire', backlog_table.c.feed, backlog_table.c.expire).create(bind=session.bind) ver = 1 if ver == 1: table = table_schema('backlog', session) table_add_column(table, 'json', Unicode, session) # Make sure we get the new schema with the added column table = table_schema('backlog', session) for row in session.execute(select([table.c.id, table.c.entry])): try: p = pickle.loads(row['entry']) session.execute( table.update().where(table.c.id == row['id']).values( json=json.dumps(p, encode_datetime=True))) except KeyError as e: logger.error( 'Unable error upgrading backlog pickle object due to {}', str(e)) ver = 2 if ver == 2: table = table_schema('backlog', session) for row in session.execute(select([table.c.id, table.c.json])): if not row['json']: # Seems there could be invalid data somehow. See #2590 continue data = json.loads(row['json'], decode_datetime=True) # If title looked like a date, make sure it's a string title = str(data.pop('title')) e = Entry(title=title, **data) session.execute(table.update().where( table.c.id == row['id']).values(json=serialization.dumps(e))) ver = 3 return ver
def upgrade(ver, session): if ver == 0: table = table_schema('pending_entries', session) for row in session.execute(select([table.c.id, table.c.json])): if not row['json']: # Seems there could be invalid data somehow. See #2590 continue data = json.loads(row['json'], decode_datetime=True) # If title looked like a date, make sure it's a string title = str(data.pop('title')) e = Entry(title=title, **data) session.execute(table.update().where( table.c.id == row['id']).values(json=serialization.dumps(e))) ver = 1 return ver
def upgrade(ver, session): if ver is None: # Upgrade to version 0 was a failed attempt at cleaning bad entries from our table, better attempt in ver 1 ver = 1 if ver == 1: table = table_schema('delay', session) table_add_column(table, 'json', Unicode, session) # Make sure we get the new schema with the added column table = table_schema('delay', session) failures = 0 for row in session.execute(select([table.c.id, table.c.entry])): try: p = pickle.loads(row['entry']) session.execute( table.update() .where(table.c.id == row['id']) .values(json=json.dumps(p, encode_datetime=True)) ) except (KeyError, ImportError): failures += 1 if failures > 0: logger.error( 'Error upgrading {} pickle objects. Some delay information has been lost.', failures, ) ver = 2 if ver == 2: table = table_schema('delay', session) for row in session.execute(select([table.c.id, table.c.json])): if not row['json']: # Seems there could be invalid data somehow. See #2590 continue data = json.loads(row['json'], decode_datetime=True) # If title looked like a date, make sure it's a string title = str(data.pop('title')) e = Entry(title=title, **data) session.execute( table.update().where(table.c.id == row['id']).values(json=serialization.dumps(e)) ) ver = 3 return ver