def _encode_item(self, value):
# Document instance
if isinstance(value, Document):
if value.doc_id is None:
raise self.StructureError(msg="Trying to relate an unsaved "
"document; '%s'" %
type(value).__name__)
return value.doc_id
# CustomField instance
elif isinstance(value, CustomField):
return value.value
# datetime types
elif isinstance(value,
(datetime.datetime, datetime.date, datetime.time)):
if hasattr(value, 'tzinfo') and value.tzinfo is None:
# always timezone "aware" datetime and time
value = value.replace(tzinfo=tzutc())
pickler = jsonpickle.Pickler(unpicklable=False)
return pickler.flatten(value)
# list
elif isinstance(value, list):
return [self._encode_item(v) for v in value]
# dictionary, pass it to dict encoder
elif isinstance(value, dict):
return self._encode_dict(value)
# no need to encode
return value
def test_unpickleable(self):
"""
If 'unpickleable' is set on the Pickler, the date objects should be
simple, human-readable strings.
"""
obj = datetime.datetime.now()
pickler = jsonpickle.Pickler(unpicklable=False)
flattened = pickler.flatten(obj)
self.assertEqual(str(obj), flattened)
def flatten(data, **kwargs):
"""
Args:
data(Any):
**kwargs: see Pickler
Returns:
dict: state of data
"""
return jsonpickle.Pickler(**kwargs).flatten(data, reset=True)
def update(self):
"""
Insert or update the managed document
"""
p = jsonpickle.Pickler()
pickled_entry = p.flatten(self.entity)
if self.doc_id is None:
doc_id = self.table.insert(pickled_entry)
self.doc_id = doc_id
else:
self.table.update(pickled_entry, doc_ids=[self.doc_id])
def _get_flatten_value(self, type, value):
if isinstance(value, str):
return value
if type == 'due_date_time':
return value.strftime(constants.DATE_TIME_FORMAT)
if type in ['date', 'due_date']:
return value.strftime(constants.DATE_FORMAT)
if type == 'time':
if isinstance(value, time):
return time.strftime(value, constants.TIME_FORMAT)
return datetime.strftime(value, constants.TIME_FORMAT)
if type == 'file':
if not isinstance(value, list):
return
p = jsonpickle.Pickler(unpicklable=False)
return p.flatten(value)
def _index(self):
vectorizer = TfidfVectorizer(tokenizer=lambda text: extract_terms_list(
text, self._lemmatization))
doc_term_matrix = vectorizer.fit_transform([
read_all_file_text(self._input_dir / file['file'])
for file in self._files
])
print(f'Created {doc_term_matrix.shape} doc-term matrix')
# It is possible save only some vectorizer properties (like vocabulary, idf) instead of pickling the whole object
# but there is no way to do this with TfidfVectorizer except messing with private properties
# as shown here http://thiagomarzagao.com/2015/12/08/saving-TfidfVectorizer-without-pickles/
# which is probably not a good idea, can break in future sklearn versions, etc.
# The doc-term matrix is also serialized in not a very human-readable format unfortunately,
# it is possible to store it in a less efficient but simpler format via .toarray().tolist()
pickler = jsonpickle.Pickler()
return {
'files': self._files,
'tfidf_doc_term_matrix': pickler.flatten(doc_term_matrix),
'tfidf_vectorizer': pickler.flatten(vectorizer)
}
def _json_safe(self, mapping):
data = dict()
for key, value in mapping.iteritems():
# None values will be stripped out
if value is None:
continue
# KEY
# Document instance
if isinstance(key, Document):
# document instances are not hashable!
# should raise an error here
pass
# CustomField instance
elif isinstance(key, CustomField):
key = key.value
elif isinstance(key,
(datetime.datetime, datetime.date, datetime.time)):
pickler = jsonpickle.Pickler(unpicklable=False)
key = pickler.flatten(key)
# VALUE
# Document instance
if isinstance(value, Document):
if value.doc_id is None:
raise self.StructureError(
msg="Trying to relate an unsaved "
"document; '%s'" % type(value).__name__)
value = value.doc_id
# CustomField instance
elif isinstance(value, CustomField):
value = value.value
# a dictionary value, go recursive
elif isinstance(value, dict):
value = self._json_safe(value)
# good to json encode
data[key] = value
return data
if data['user_selected']:
playlists = data['user_selected']
else:
playlists = data['all_playlists']
for pl in playlists:
pl_dict = {'items': []}
pl_dict.update(vars(pl))
for k in list(pl_dict):
if k.startswith('_'):
del pl_dict[k]
items = plex.fetchItem(pl.ratingKey).items()
for item in items:
item_dict = object_cleaner(item)
pl_dict['items'].append(item_dict)
json_dump = jsonpickle.Pickler(unpicklable=False).flatten(
obj=pl_dict)
title = json_dump['title']
output_file = '{}-{}-Playlist.{}'.format(
user, title, opts.export)
if opts.export == 'json':
with open(output_file, 'w') as fp:
json.dump(json_dump, fp, indent=4, sort_keys=True)
elif opts.export == 'csv':
columns = []
data_list = []
for rows in json_dump['items']:
flat_data = flatten(rows)
columns += list(flat_data)
data_list.append(flat_data)
with open(output_file, 'w', encoding='UTF-8',
newline='') as data_file:
def to_json(obj, plain_json: bool = False):
return jsonpickle.Pickler(unpicklable=not plain_json,
keys=True).flatten(obj)
def toJson(self):
return jsonpickle.Pickler(unpicklable=True).flatten(self.__rawRepr__())
