class LocalFileBackend(StardateBackend): def __init__(self): self.name = u'localfile' self.parser = FileParser() self.social_auth = None def get_name(self): return self.name def set_social_auth(self, *args, **kwargs): return def serialize_posts(self, posts): """ Returns dictionary of individual Post """ posts_as_dicts = [] serialized = serialize( 'python', posts, fields=('title', 'slug', 'publish', 'stardate', 'body') ) for post in serialized: posts_as_dicts.append(post['fields']) return posts_as_dicts def _get_post_path(self, folder, post): """ Dynamically guess post file path from slug / blog folder """ filename = post['slug'] filename = '{0}.md'.format(filename) path = os.path.join(folder, filename) return path def _posts_from_file(self, file_path): """ Return list of post dictionaries from file """ if os.path.exists(file_path): with open(file_path, 'r') as f: content = f.read() posts = self.parser.unpack(content) else: posts = [] return posts def _posts_from_dir(self, folder, posts=[]): """ Get posts dicts from files in a directory """ files = os.listdir(folder) remote_posts = [] for filename in files: with open(filename, 'r') as f: remote_post = f.read() remote_post = self.parser.parse(remote_post) remote_posts.append(remote_post) return remote_posts def push(self, posts): """ Render posts to files posts: List of Post object instances """ # Grab the file or folder path associated # with a blog blog_path = posts[0].blog.backend_file # Separate blog path into directory and filename blog_dir, blog_file = os.path.split(blog_path) # pushing works differently depending on whether # We are using a single file or a directory of files if blog_file: responses = [self._push_blog_file(blog_path, posts)] else: responses = self._push_post_files(blog_dir, posts) return responses def _push_blog_file(self, file_path, posts): """ Update posts in a single blog file """ remote_posts = self._posts_from_file(file_path) # Use serialized version of posts to find # and update local_posts = self.serialize_posts(posts) # Update remote_posts with local versions ## FIXME: n^2 crawl, use stardate as keys ## in dicts instead of lists? for local_post in local_posts: exists = False for remote_post in remote_posts: if remote_post['stardate']: if local_post['stardate'] == remote_post['stardate']: exists = True remote_post.update(local_post) # Post may exist on backend with uuid, but # also exist in local from last pull where # uuid was assigned. Use 'title' field as # backup else: if local_post['title'] == remote_post['title']: exists = True remote_post.update(local_post) # Add new remote post if it does not exist yet if not exists: remote_posts.append(local_post) # Turn post list back into string content = self.parser.pack(remote_posts) with open(file_path, 'w') as f: f.write(content) return def _push_post_files(self, folder, posts): """ Update posts in multiple files """ local_posts = self.serialized_posts(posts) for local_post in local_posts: # Generate the post file path dynamically post_path = self._get_post_path(folder, local_post) # Get the existing remote post as a post dict if os.path.exists(post_path): with open(post_path, 'r') as f: remote_post = f.read() remote_post = self.parser.parse(remote_post) else: remote_post = {} # Update the contents of the remote post remote_post.update(local_post) content = self.parser.render(remote_post) with open(post_path, 'w') as f: f.write(content) return def _update_from_dict(self, blog, post_dict, post=None): """ Create or update a Post from a dictionary """ # If a post is not provided, try an fetch it if not post: if 'stardate' in post_dict: post = Post.objects.filter( blog=blog, stardate=post_dict['stardate'] ) if post: post = post[0] if not post: post = Post(blog=blog) # Update from dict values for att, value in post_dict.items(): setattr(post, att, value) post.save(push=False) return post def pull(self, blog): """ Update local posts from remote source blog: Blog instance """ blog_path = blog.backend_file blog_dir, blog_file = os.path.split(blog_path) # Extract remote posts from single file if blog_file: remote_posts = self._posts_from_file(blog_path) # Extract posts from multiple files else: remote_posts = self._posts_from_dir(blog_dir) updated_list = [] for remote_post in remote_posts: updated = self._update_from_dict(blog, remote_post) updated_list.append(updated) return updated_list
class FileParserTestCase(TestCase): def setUp(self): self.parser = FileParser() self.test_string = "publish: {0}\ntimezone: US/Eastern\ntitle: Tingling of the spine\n\n\nExtraordinary claims require extraordinary evidence!".format(TIMESTAMP) def tearDown(self): Blog.objects.all().delete() User.objects.all().delete() def test_pack(self): file_path = tempfile.mkstemp(suffix='.txt')[1] user = User.objects.create(username='******') blog = Blog.objects.create( backend_file=file_path, backend_class='stardate.backends.local_file.LocalFileBackend', name='test blog', user=user, ) Post.objects.create( blog=blog, title='My first post', publish=datetime.datetime(2015, 1, 1, 6, 0), body='This is the first post.' ) Post.objects.create( blog=blog, title='My second post', publish=datetime.datetime(2015, 1, 2, 6, 0), body='This is the second post.' ) post_list = [post.serialized() for post in blog.posts.all()] packed = self.parser.pack(post_list) self.assertIsInstance(post_list, list) self.assertEqual(len(post_list), 2) try: self.assertIsInstance(packed, basestring) except NameError: self.assertIsInstance(packed, str) self.assertTrue(u'title: {0}'.format(post_list[0]['title']) in packed) self.assertTrue(u'title: {0}'.format(post_list[1]['title']) in packed) self.assertTrue(u'stardate: {0}'.format(post_list[0]['stardate']) in packed) self.assertTrue(u'stardate: {0}'.format(post_list[1]['stardate']) in packed) self.assertTrue(u'\n\n\n{0}'.format(post_list[0]['body']) in packed) self.assertTrue(u'\n\n\n{0}'.format(post_list[1]['body']) in packed) self.assertTrue(u'publish: {0}'.format(post_list[0]['publish']) in packed) self.assertTrue(u'publish: {0}'.format(post_list[1]['publish']) in packed) def test_parse_publish(self): timestamp = '01-01-2015 06:00AM+0000' expected = datetime.datetime(2015, 1, 1, 6, 0, tzinfo=timezone.utc) self.assertEqual(self.parser.parse_publish(timestamp), expected) self.assertEqual(self.parser.parse_publish(expected), expected) self.assertEqual( self.parser.parse_publish('2016-01-01 00:00:00 -0500'), datetime.datetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern')) ) self.assertEqual( self.parser.parse_publish('2016-01-01'), datetime.datetime(2016, 1, 1, tzinfo=timezone.utc) ) self.assertEqual( self.parser.parse_publish('2016-01-01 00:00:00'), datetime.datetime(2016, 1, 1, tzinfo=timezone.utc) ) self.assertEqual( self.parser.parse_publish('2016-01-01 12AM', 'US/Eastern'), datetime.datetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern')) ) self.assertEqual( self.parser.parse_publish(datetime.date(2016, 1, 1)), datetime.datetime(2016, 1, 1, tzinfo=timezone.utc) ) self.assertEqual( self.parser.parse_publish('2016-01-01 12AM', 'EST'), datetime.datetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern')) ) self.assertEqual( self.parser.parse_publish('2016-07-01 12AM', 'US/Eastern'), datetime.datetime(2016, 7, 1, tzinfo=tz.gettz('US/Eastern')) ) self.assertEqual( self.parser.parse_publish(datetime.datetime(2016, 1, 1, 0, 0, tzinfo=timezone.utc), 'US/Eastern'), datetime.datetime(2016, 1, 1, tzinfo=tz.gettz('US/Eastern')) ) def test_parse(self): parsed = self.parser.parse(self.test_string) self.assertEqual(parsed['title'], 'Tingling of the spine') expected = datetime.datetime(2012, 1, 2, tzinfo=tz.gettz('US/Eastern')) self.assertEqual(parsed['publish'], expected) self.assertEqual(parsed['body'], 'Extraordinary claims require extraordinary evidence!') self.assertEqual(parsed['timezone'], 'US/Eastern') # Check that extra_field is parsed string = u"title: My title\nextra_field: Something arbitrary\n\n\nThe body.\n" parsed = self.parser.parse(string) self.assertTrue('title' in parsed.keys()) self.assertTrue('extra_field' in parsed.keys()) def test_render(self): file_path = tempfile.mkstemp(suffix='.txt')[1] user = User.objects.create(username='******') blog = Blog.objects.create( backend_file=file_path, backend_class='stardate.backends.local_file.LocalFileBackend', name='test blog', user=user, ) post = Post.objects.create( blog=blog, title='Test title', publish=datetime.datetime(2013, 6, 1), timezone='US/Eastern', body='The body.', ) packed = self.parser.pack([post.serialized()]) rendered = self.parser.render(post.serialized()) self.assertTrue('publish: 2013-06-01 12:00 AM -0400' in rendered) parsed = self.parser.parse(rendered) self.assertEqual(parsed.get('title'), post.title) self.assertEqual(parsed.get('timezone'), post.timezone) self.assertEqual(parsed.get('body'), post.body.raw) self.assertEqual(rendered, packed) def test_unpack(self): content = self.test_string post_list = self.parser.unpack(content) #The file has one post to unpack self.assertEqual(len(post_list), 1) post = post_list[0] self.assertEqual(post.get('title'), 'Tingling of the spine') self.assertEqual( post.get('publish'), datetime.datetime(2012, 1, 2, tzinfo=tz.gettz('US/Eastern')) ) self.assertEqual(post.get('body'), 'Extraordinary claims require extraordinary evidence!') @patch('stardate.parsers.logger') def test_bad_string(self, mock_logging): content = 'bad string\n\r' posts = self.parser.unpack(content) self.assertEqual(posts, []) mock_logging.warn.assert_called_once_with( 'Not enough information found to parse string.')