예제 #1
0
 def __init__(self, *args, **kwargs):
     self.start_idx = kwargs['start_idx'] if 'start_idx' in kwargs else 0
     self.end_idx = kwargs['end_idx'] if 'end_idx' in kwargs else 0
     self.env = kwargs['env'] if 'env' in kwargs else 'dev'
     self.conf = config[self.env]
     self.d = Debug(level=4, color=True)
     self.current_dir = os.path.dirname(os.path.abspath(__file__))
     self.project_dir = os.path.abspath(
         os.path.join(self.current_dir, os.pardir))
     self.admin_url = self.conf['url']['admin']
     self._default_image_id = 2
예제 #2
0
	def __init__(self, context, plugin_dir="plugins"):
		self.d = Debug()
		
		# apply plugin dir to PATH
		sys.path.append(plugin_dir)

		# context object
		self.context = context
		self.plugins = []
예제 #3
0
class PluginDispatcher(object):
	"""
	PluginDispatcher loads modules and handles event hooks in them

	:author: Anton Zering <*****@*****.**>
	:copyright: 2012, Anton Zering
	:license: Apache License, Version 2.0
	"""
	def __init__(self, context, plugin_dir="plugins"):
		self.d = Debug()
		
		# apply plugin dir to PATH
		sys.path.append(plugin_dir)

		# context object
		self.context = context
		self.plugins = []

	def load_plugins(self, path_list):
		self.d.info("Loading modules...")
		for path in path_list:
			cls = self.get_class_by_path(path)
			if cls:
				self.plugins.append(cls(self.context))

	def get_class_by_path(self, path):
		"""
		Returns a class object by path

		:param path: Path to the plugin in module notation: module.module.PluginClass
		:returns: Class object
		"""
		# separate class from modul name
		mod_name, cls_name = path.rsplit('.', 1)

		try:
			cls = __import__(mod_name).__dict__[cls_name]
		except Exception, e:
			# plugin could not be loaded due to lookup error or syntax errors.
			self.d.warn("!! FAILED loading module %s (%s)" % (path, e))
			return None

		return cls
예제 #4
0
파일: pybot.py 프로젝트: synthomat/pealbot
	def __init__(self, host, nick_name, real_name=None):
		""" 
		Constructor for the bot

		:param host: The server address as tuple of (HOST, PORT)
		:param nick_name: Nickname for the bot.
		:param real_name: Real name of the bot. It not provided, the nick is taken as real name.
		"""

		self.d = Debug()
		
		self.host = host
		self.nick_name = nick_name
		
		if real_name:
			self.real_name = real_name
		else:
			self.real_name = nick_name

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
예제 #5
0
파일: pybot.py 프로젝트: synthomat/pealbot
class BotBot(object):
	"""
	:author: Anton Zering <*****@*****.**>
	:author: Jules Held <*****@*****.**>
	:copyright: 2012, Anton Zering
	:license: Apache License, Version 2.0
	"""

	def __init__(self, host, nick_name, real_name=None):
		""" 
		Constructor for the bot

		:param host: The server address as tuple of (HOST, PORT)
		:param nick_name: Nickname for the bot.
		:param real_name: Real name of the bot. It not provided, the nick is taken as real name.
		"""

		self.d = Debug()
		
		self.host = host
		self.nick_name = nick_name
		
		if real_name:
			self.real_name = real_name
		else:
			self.real_name = nick_name

		self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)

	def connect(self, host=None):
		"""
		Connects to the server

		:param host: The server address as tuple of (HOST, PORT).
		"""

		if not host:
			host = self.host

		self.d.info("Connecting to " + str(host))

		# connect to server socket
		self.sock.connect(self.host)

	def disconnect(self):
		"""Disconnects from the server if already connected."""

		if not self.sock:
			return

		self.d.info("Disconnecting from " + str(self.host))
		self.sock.close()

	def send(self, msg):
		"""
		Sends raw messages to the server.

		:param msg: Message to be sent.

		:return: success of the send operation
		"""

		self.d.notice("<< " + msg)
		return self.sock.sendall(msg + "\r\n")

	def quit(self, quit_reason=""):
		"""
		Sends a QUIT command to the server. Then disconnects.

		:param quit_msg: Quit message.
		"""

		self.send('QUIT %s' % quit_reason)
		self.disconnect()

	def join(self, channels):
		"""
		Sends a JOIN message to the server to joins one or multiple channels

		:param channels: One or more channels separated by whitespaces.
		"""

		return self.send('JOIN %s' % channels)

	def part(self, channels, part_reason=None):
		"""
		Sends the PART message to the searver to leave one or multiple channels

		:param channels: One or more channels separated by whitespaces.
		:param part_msg: Part message to appear in the channel
		"""

		return self.send('PART %s :%s' % (channels, part_reason))

	def nick(self, new_nick):
		"""
		Sends the NICK message to the server to try a nickname change of the bot

		:param new_nick: The new nick
		"""

		return self.send("NICK %s" % new_nick)

	def notice(self, target, text):
		"""
		Sends a NOTICE message to a reciever

		:param target: Reciever of the message
		:param msg: Message to be sent
		"""

		return self.send("NOTICE %s :%s" % (target, text))

	def msg(self, target, text):
		"""
		Sends a PRIVMSG message to a channel or another client

		:param target: Reciever of the message. Channel or a nick name.
		:param msg: Message to be sent
		"""
		return self.send("PRIVMSG %s :%s" % (target, text))

	def on_ping(self, msg):
		"""
		Handles PING messages from the server

		"""

		self.send("PONG :%s" % msg.trail)

	def exit(self, exit_code=0):
		sys.exit(exit_code)

	def start(self):
		# connect to server
		self.connect()

		# set default nick name
		self.nick(self.nick_name)

		# set default user and real name
		self.send("USER %s 0 0 :%s" % (self.nick_name, self.real_name))
		
		# TODO: move to plugin!
		# Authenticate at Nickserv:
		# self.auth(self.config['password'])

		while True:
			try:
				recv = self.sock.recv(2048)
				# print recv
			except:
				self.exit(0)

			for line in recv.split("\r\n"):

				# ignore empty lines
				if not line: continue

				# handle irc commands
				msg = IRCMessage.parse(line)

				if msg:
					self.handle_irc(msg)

	def handle_irc(self, msg):
		method = lookup_hook(self, msg.cmd, codes)

		if method:
			return method(msg)
예제 #6
0
class WPImporter:
    def __init__(self, *args, **kwargs):
        self.start_idx = kwargs['start_idx'] if 'start_idx' in kwargs else 0
        self.end_idx = kwargs['end_idx'] if 'end_idx' in kwargs else 0
        self.env = kwargs['env'] if 'env' in kwargs else 'dev'
        self.conf = config[self.env]
        self.d = Debug(level=4, color=True)
        self.current_dir = os.path.dirname(os.path.abspath(__file__))
        self.project_dir = os.path.abspath(
            os.path.join(self.current_dir, os.pardir))
        self.admin_url = self.conf['url']['admin']
        self._default_image_id = 2

    def get_csrf_token_from_page(self, url):
        r = self.s.get(url, verify=False)
        soup = BeautifulSoup(r.text, "lxml")
        return soup.find('input', {'name': '_csrf'}).get('value')

    def login(self):
        self.s = requests.Session()
        #r = self.s.get('{}/auth/login'.format(self.admin_url), verify=False)
        #csrf_token = r.cookies['_csrf']
        csrf_token = self.get_csrf_token_from_page('{}/auth/login'.format(
            self.admin_url))
        p = {
            'email': self.conf['account']['email'],
            'password': self.conf['account']['password'],
            '_csrf': csrf_token
        }
        self.s.post('{}/auth/login'.format(self.admin_url), data=p)

    def import_categories(self):
        with open(self.project_dir + '/output/categories/categories.json',
                  'r') as f:
            ## import categories
            self.d.info('start importing categories ...')
            csrf_token = self.get_csrf_token_from_page(
                '{}/categories/new'.format(self.admin_url))
            categories = json.load(f)
            categories_count = len(categories)
            counter = 1
            cur_id = 0
            while len(categories) > 0:
                category = categories.pop(0)

                if category['id'] > cur_id:
                    cur_id = category['id']

                if category['parent'] > cur_id:
                    categories.append(category)
                    continue

                category['import_id'] = category['id']
                del category['id']

                category['parent_id'] = category['parent']
                del category['parent']

                category['_csrf'] = csrf_token
                z = self.s.post('{}/api/v1/categories/new'.format(
                    self.admin_url),
                                json=category,
                                headers=self._create_headers(csrf_token))
                result = json.loads(z.text)
                progress_info = '({}/{})'.format(counter, categories_count)
                if 'errors' in result['payload']:
                    for k in result['payload']['errors']:
                        self.d.error('ERROR', progress_info,
                                     'cat_id={}'.format(category['import_id']),
                                     k, result['payload']['errors'][k])
                else:
                    self.d.log('Done.', progress_info,
                               result['payload']['message'])

                counter += 1

    def import_tags(self):
        with open(self.project_dir + '/output/tags/tags.json', 'r') as f:
            ## import tags
            csrf_token = self.get_csrf_token_from_page('{}/tags/new'.format(
                self.admin_url))
            tags = json.load(f)
            for tag in tags:
                tag['import_id'] = tag['id']
                del tag['id']
                tag['_csrf'] = csrf_token
                z = s.post('{}/api/v1/tags/new'.format(self.admin_url),
                           json=tag,
                           headers=self._create_headers(csrf_token))
                result = json.loads(z.text)
                if 'errors' in result['payload']:
                    for k in result['payload']['errors']:
                        self.d.error('ERROR', progress_info,
                                     'tag_id={}'.format(category['import_id']),
                                     k, result['payload']['errors'][k])
                else:
                    self.d.log('Done.', result['payload']['message'])

    def import_operators(self):
        with open(self.project_dir + '/output/wp_users/wp_users.json',
                  'r') as f:
            ## import operators
            self.d.info('start importing operators ...')
            csrf_token = self.get_csrf_token_from_page(
                '{}/operators/new'.format(self.admin_url))
            operators = json.load(f)
            operators_count = len(operators)
            counter = 1
            for operator in operators:
                operator['import_id'] = operator['id']
                del operator['id']

                operator['_csrf'] = csrf_token
                z = self.s.post('{}/api/v1/operators/new'.format(
                    self.admin_url),
                                json=operator,
                                headers=self._create_headers(csrf_token))
                result = json.loads(z.text)
                progress_info = '({}/{})'.format(counter, operators_count)
                if 'errors' in result['payload']:
                    for k in result['payload']['errors']:
                        self.d.error(
                            'ERROR', progress_info,
                            'operator_id={}'.format(operator['import_id']), k,
                            result['payload']['errors'][k])
                else:
                    self.d.log('Done.', progress_info,
                               result['payload']['message'])

                counter += 1

    def import_articles_with_images(self):
        self.import_articles(with_image=True)

    def update_article_image(self):
        """
            COPY (SELECT ROW_TO_JSON(t) FROM (SELECT articles.id, image_files.filename FROM articles
                join images on images.id = articles.image_id join image_files on images.id = image_files.image_id) t) TO '/tmp/article_image_relations.json';
            mv /tmp/article_image_relations.json /var/lib/postgresql/data/
            mv $DOCKER_ALIAS/data/article_image_relations.json ~/wordpress-split-data/deps/
        """
        contents = []
        self.air = {}
        with open(self.project_dir + '/deps/article_image_relations.json',
                  'r') as f:
            contents = f.readlines()

        for line in contents:
            j = json.loads(line.strip())
            self.air[j['id']] = j

        self.import_articles(with_image=True,
                             update_only=True,
                             save_to_local=True)

    def import_articles(self,
                        with_image=False,
                        update_only=False,
                        save_to_local=False):
        h = HTMLParser()

        self.uii = {}
        with open(self.project_dir + '/deps/upload_images_id.json.c') as f:
            self.uii = json.load(f)

        features_alias = {}
        with open(self.project_dir + '/output/features.json', 'r') as f:
            series = json.load(f)
            categories = []
            with open(self.project_dir + '/output/categories/categories.json',
                      'r') as f:
                categories = json.load(f)

            for s in series:
                if s['term_group'] == 1:
                    for category in categories:
                        if category['slug'] == s['slug']:
                            features_alias[category['id']] = s
                            break

        for idx in xrange(self.start_idx, self.end_idx + 1):
            input_file = self.project_dir + '/output/wp_posts/wp_posts_{}.json'.format(
                idx)
            with open(input_file, 'r') as f:
                ## import articles
                self.d.info('start importing articles ...')
                self.d.info('read file ... {}'.format(input_file))
                csrf_token = self.get_csrf_token_from_page(
                    '{}/articles/new'.format(self.admin_url))
                articles = json.load(f)
                articles_count = len(articles)
                counter = 1
                operator_ids = self._get_operators_ids()
                for article in articles:
                    # replace cdn url into /media/image/:id
                    xds = re.findall(
                        '(https://stg.localhost/640/480/uploads/(.*))"',
                        article['html_content'])
                    article['html_content'] = self._replace_image_url_to_media(
                        xds, article['html_content'])

                    # replace s3 url into /media/image/:id
                    xds = re.findall(
                        '(https://cdn-prd.s3-ap-northeast-1.amazonaws.com/wp-content/uploads/(.*))"',
                        article['html_content'])
                    article['html_content'] = self._replace_image_url_to_media(
                        xds, article['html_content'])

                    if article['author_id'] not in operator_ids:
                        self.d.debug(
                            'operator id not exists. Set to author_id = 1.')
                        article['author_id'] = 1

                    article['import_id'] = article['id']

                    # set one space if subtitle is empty
                    if article[u'sub_title'] == '':
                        article[u'sub_title'] = ' '
                    else:
                        article[u'sub_title'] = article[u'sub_title'][:60]

                    if u'published_at' in article:
                        article['status'] = 2

                    if article['series_id'] != 0:
                        article['feature_ids'] = [article['series_id']]
                    else:
                        article['feature_ids'] = []

                    # limit title to 60 words
                    article[u'title'] = h.unescape(article[u'title'][:60])

                    # limit meta_description to 120 words
                    article[u'meta_description'] = article[
                        u'meta_description'][:100]

                    if article[u'meta_description'] == '':
                        article[u'meta_description'] = article[u'title']

                    # make content & html_content the same.
                    article[u'content'] = article['html_content']

                    # do not import tag_ids
                    article[u'tag_ids'] = []

                    article['_csrf'] = csrf_token

                    article['html_content'] = h.unescape(
                        article['html_content'])

                    # make category <-> features alias
                    if article['category_id'] != 0 and article[
                            'category_id'] in features_alias:
                        article['feature_ids'].append(
                            features_alias[article['category_id']]['term_id'])

                    article['image_id'] = self._default_image_id
                    if save_to_local:
                        if article['id'] in self.air:
                            article['image_id'] = self.air[
                                article['id']]['image_id']
                        else:
                            article['image_id'] = self._default_image_id

                    z = None
                    if update_only:
                        article = self._upload_media_image(article,
                                                           upload_flag=False)
                        #del article['image_id']
                        z = self.s.put(
                            '{}/api/v1/articles/{}'.format(
                                self.admin_url, article['import_id']),
                            json=article,
                            headers=self._create_headers(csrf_token))
                    else:
                        z = self.s.post(
                            '{}/api/v1/articles/new'.format(self.admin_url),
                            json=article,
                            headers=self._create_headers(csrf_token))

                    result = json.loads(z.text)
                    progress_info = '({}/{})'.format(counter, articles_count)
                    if 'errors' in result['payload']:
                        for k in result['payload']['errors']:
                            self.d.error('ERROR', progress_info,
                                         'id={}'.format(article['import_id']),
                                         k, result['payload']['errors'][k],
                                         article['category_id'])
                    else:
                        if result['status'] == 'Not Found':
                            self.d.error('Not Found.', progress_info,
                                         'id={}'.format(article['import_id']))
                        else:
                            self.d.log('Done.', progress_info,
                                       'id={}'.format(article['import_id']),
                                       result['payload']['message'])

                            if with_image:
                                if save_to_local:
                                    article = self._upload_media_image(
                                        article,
                                        upload_flag=False,
                                        save_to_local=True)
                                else:
                                    article = self._upload_media_image(article)

                                    # update image_id
                                    self._update_article_image_id(article)

                    counter += 1

    def import_widgets(self):
        link_short_code_dict = {}
        with open(self.project_dir + '/output/link_short_code.json', 'r') as f:
            link_short_code_dict = json.load(f)

        with open(self.project_dir + '/output/wp_posts/wp_links.json',
                  'r') as f:
            ## import wplink as widgets
            csrf_token = self.get_csrf_token_from_page('{}/widgets/new'.format(
                self.admin_url))
            wplinks = json.load(f)
            for key in wplinks:
                # insert widget
                wplink = wplinks[key]
                content = u'<div class="recommend-links"><h2 class="title">【編集部のオススメ記事】</h2><ul class="links">{}</ul></div>'.format(
                    ''.join([
                        u'<li><a href="{}"><i class="icons-right _color-darkgray"></i>{}</a></li>'
                        .format(wplink['data'][k][0], wplink['data'][k][1])
                        for k in wplink['data']
                    ]))

                widget_title = ",".join([k for k in wplink['data']])[:60]
                content = content.replace(u'https://prd.localhost/archives',
                                          u'/archives')
                content_soup = BeautifulSoup(content, "lxml")

                if content_soup.html is not None:
                    content_soup.html.unwrap()

                if content_soup.body is not None:
                    content_soup.body.unwrap()

                if wplink['id'] == 124:
                    widget_title += ' '

                widget_data = {
                    'import_id':
                    wplink['id'],
                    'name':
                    widget_title,
                    'content':
                    ' ',  #content_soup.prettify(),
                    'type_id':
                    10,  # おすすめ記事リンク
                    'link_ids':
                    [link_short_code_dict[k]['id'] for k in wplink['data']]
                }
                widget_data['_csrf'] = csrf_token
                z = self.s.post('{}/api/v1/widgets/new'.format(self.admin_url),
                                json=widget_data,
                                headers=self._create_headers(csrf_token))
                result = json.loads(z.text)
                if 'errors' in result['payload']:
                    for k in result['payload']['errors']:
                        self.d.error('ERROR', k,
                                     result['payload']['errors'][k])
                else:
                    self.d.log('Done.', result['payload']['message'])

                # insert panel
                panel_data = {
                    'import_id': wplink['id'],
                    'name': widget_title,
                    'type_id': 7,
                    'widget_ids': [wplink['id']],
                }
                z = self.s.post('{}/api/v1/panels/new'.format(self.admin_url),
                                json=panel_data,
                                headers=self._create_headers(csrf_token))
                result = json.loads(z.text)
                if 'errors' in result['payload']:
                    for k in result['payload']['errors']:
                        self.d.error('ERROR', k,
                                     result['payload']['errors'][k])
                else:
                    self.d.log('Done.', wplink['id'],
                               result['payload']['message'])

    def import_feature(self):
        with open(self.project_dir + '/output/features.json', 'r') as f:
            csrf_token = self.get_csrf_token_from_page(
                '{}/features/new'.format(self.admin_url))
            series = json.load(f)
            for s in series:
                # insert widget
                feature_data = {
                    'import_id': s['term_id'],
                    'author_id': 1001,
                    'title': s['name'],
                    'slug': s['slug'],
                    'image_id': 50002,  # self._default_image_id,
                    'description': ' ',
                    '_csrf': csrf_token,
                }
                z = self.s.post('{}/api/v1/features/new'.format(
                    self.admin_url),
                                json=feature_data,
                                headers=self._create_headers(csrf_token))
                result = json.loads(z.text)
                if 'errors' in result['payload']:
                    for k in result['payload']['errors']:
                        self.d.error('ERROR', k,
                                     result['payload']['errors'][k],
                                     s['term_id'], s['name'])
                else:
                    self.d.log('Done.', result['payload']['message'])

    def import_link_short_code(self):
        with open(self.project_dir + '/output/link_short_code.json', 'r') as f:
            # TODO: change to correct url
            csrf_token = self.get_csrf_token_from_page(
                '{}/features/new'.format(self.admin_url))
            link_short_codes = json.load(f)
            for key in link_short_codes:
                lsc_data = {
                    'import_id': link_short_codes[key]['id'],
                    'name': key,
                    'display_name': link_short_codes[key]['title'],
                    'href': link_short_codes[key]['link'],
                    '_csrf': csrf_token,
                }
                z = self.s.post('{}/api/v1/link_short_codes/new'.format(
                    self.admin_url),
                                json=lsc_data,
                                headers=self._create_headers(csrf_token))
                result = json.loads(z.text)
                if 'errors' in result['payload']:
                    for k in result['payload']['errors']:
                        self.d.error('ERROR', k,
                                     result['payload']['errors'][k])
                else:
                    self.d.log('Done.', result['payload']['message'])

    def download_images(self):
        input_file = self.project_dir + '/output/s3_images/s3_images.json'
        with open(input_file, 'r') as f:
            ## import articles
            self.d.info('read file ... {}'.format(input_file))
            s3_images = json.load(f)
            print len(s3_images.keys())

            for d in s3_images:
                for origin_url in s3_images[d]:
                    # create dirs
                    image_path = origin_url.split('https://prd.localhost/')[1]
                    image_dir = '/'.join(image_path.split('/')[:-1])
                    helpers.create_dirs(self.project_dir + '/output',
                                        [image_dir])

                    s3_url = s3_images[d][origin_url]
                    urllib.urlretrieve(
                        origin_url, 'output/' + image_dir + '/' +
                        origin_url.split('/')[-1])

    def _get_operators_ids(self):
        with open(self.project_dir + '/output/wp_users/wp_users.json',
                  'r') as f:
            operators = json.load(f)
            return [operator['id'] for operator in operators]

    def _upload_media_image(self,
                            article,
                            upload_flag=True,
                            save_to_local=False):
        thumb_url = article['thumb_url']

        if thumb_url == "":
            if len(article['image_urls']) > 0:
                thumb_url = article['image_urls'][0]
            else:
                article['image_id'] = self._default_image_id
                return article

        fn = thumb_url.split('/')[-1]

        open_fn = fn
        file_exist_flag = os.path.isfile(self.project_dir +
                                         '/output/wp-content-dl/uploads/' + fn)
        if not file_exist_flag:
            for ch in '12345':
                tmp = os.path.splitext(fn)
                open_fn = '{}{}{}'.format(tmp[0], ch, tmp[1])

                if os.path.isfile(self.project_dir +
                                  '/output/wp-content-dl/uploads/' + open_fn):
                    file_exist_flag = True
                    break

        if not file_exist_flag:
            # fetch image
            image_url = 'https://prd.localhost/wp-content/uploads/{}'.format(
                fn)
            self.d.log('no image file found. start fetch files ...', image_url)
            image_dir = 'wp-content-dl'
            helpers.create_dirs(self.project_dir + '/output', [image_dir])
            urllib.urlretrieve(
                image_url,
                'output/' + image_dir + '/uploads/' + image_url.split('/')[-1])
            open_fn = fn
            file_exist_flag = True

        image_path = self._get_image_full_path(open_fn)
        is_image = False
        try:
            with Image.open(image_path) as f:
                is_image = True
        except:
            pass

        if file_exist_flag and is_image:
            ## use default image
            if fn == 'logo_thumbnail1.jpg':
                article['image_id'] = self._default_image_id
                return article

            ## resize image before upload
            self._resize_eyecatch_image(image_path, fn)

            if upload_flag:
                self.d.debug('start upload image... {}'.format(fn))
                result = self._create_image_info()
                image_file_id, image_id, upload_url, content_type = self._generate_image_file_id(
                    fn, result['payload']['Id'])
                #self.d.debug('upload url = {}'.format(upload_url))

                with open('/tmp/{}'.format(fn), 'r') as f:
                    if upload_flag:
                        image_data = f.read()
                        headers = {
                            'content-type': content_type,
                            'content-length': str(len(image_data)),
                        }
                        r = self.s.put(upload_url,
                                       data=image_data,
                                       headers=headers)

                    if (upload_flag
                            and r.status_code == 200) or not upload_flag:
                        if upload_flag:
                            # doing image validation. ( /api/batch/media/image/validate/:id )
                            self.s.get(
                                '{}/api/batch/media/image/validate/{}'.format(
                                    self.admin_url, image_file_id))
                            article['image_id'] = image_id
                            self.d.debug(
                                'image uploaded. post_id={}, image_id={}'.
                                format(article['id'], image_file_id))

                        # remove img tag in html_content
                        #article = self._remove_eyecatch_image_in_content(article)
                    else:
                        article['image_id'] = self._default_image_id
                        self.d.error('error uploading image. ID={}'.format(
                            article['id']))

                # remove tmp image after upload finished
                os.remove(self.conf['path']['tmp_image'] + '/' + fn)

            if save_to_local:
                dst_dir = '/tmp/u/' + self.air[
                    article['id']]['filename'].split('/')[0]
                if not os.path.exists(dst_dir):
                    os.makedirs(dst_dir)

                dst_file = '/tmp/u/' + self.air[article['id']]['filename']
                src_file = self.conf['path']['tmp_image'] + '/' + fn
                if os.path.exists(dst_file):
                    os.remove(dst_file)

                try:
                    shutil.move(src_file, dst_file)
                except:
                    pass

        else:
            article['image_id'] = self._default_image_id
            self.d.error('thumb image not found. [post_id = {}][{}]'.format(
                article['id'], fn))

        return article

    def _remove_eyecatch_image_in_content(self, article):
        content_soup = BeautifulSoup(article['html_content'], 'lxml')
        for img_tag in content_soup.findAll('img'):
            # remove "?asd=1" parameter
            img_tag['src'] = img_tag['src'].split('?')[0]

            # remove "(1)" something like medium_210421689(1).jpg
            img_tag['src'] = re.sub(r'\(\d\)', '', img_tag['src'])

            img_fn = img_tag['src'].split('/')[-1]
            split_by_dot = img_fn.split('.')
            split_by_hypen = split_by_dot[0].split('-')

            # handle some rare case which the url doesn't have file ext
            if len(split_by_dot) == 1:
                split_by_dot.append('')

            img_fn2 = '-'.join(split_by_hypen[0:len(split_by_hypen) -
                                              1]) + '.' + split_by_dot[1]
            match_rate = 0.95
            o = os.path.splitext(img_fn)
            m = re.match(r'{}\d?\.{}'.format(o[0], o[1][1:]), img_fn)
            if Levenshtein.ratio(
                    unicode(img_fn),
                    unicode(fn)) > match_rate or Levenshtein.ratio(
                        unicode(img_fn2),
                        unicode(fn)) > match_rate or m is not None:
                img_tag.extract()
                content_soup.html.unwrap()
                content_soup.body.unwrap()

                article['html_content'] = content_soup.prettify(indent_width=2)
                # only for first one
                break
            else:
                self.d.error("img_tag's not matched. {} / {}".format(
                    img_fn, fn))

    def _create_image_info(self):
        # https://admin.localhost/api/v1/media/image/new
        # alt= , caption= , use_type=4
        csrf_token = self.get_csrf_token_from_page('{}/media'.format(
            self.admin_url))
        create_image_params = {
            'alt': '',
            'caption': '',
            'use_type': 4,
        }
        result = self.s.post('{}/api/v1/media/image/new'.format(
            self.admin_url),
                             json=create_image_params,
                             headers=self._create_headers(csrf_token))
        return json.loads(result.text)

    def _generate_image_file_id(self, fn, fid):
        # /api/v1/media/image/3/upload/new
        # POST
        # request_filename: "randString.png"
        csrf_token = self.get_csrf_token_from_page('{}/media'.format(
            self.admin_url))
        create_upload_url_params = {
            'request_filename': fn,
        }
        r = self.s.post('{}/api/v1/media/image/{}/upload/new'.format(
            self.admin_url, fid),
                        json=create_upload_url_params,
                        headers=self._create_headers(csrf_token))
        result = json.loads(r.text)
        return (result['payload']['ImageFileId'],
                result['payload']['ImageFileId2'],
                result['payload']['UploadURL'],
                result['payload']['ContentType'])

    def _get_image_full_path(self, fn):
        return self.project_dir + '/output/wp-content-dl/uploads/' + fn

    def _get_admin_url(self):
        return self.conf['url']['admin']

    def _create_headers(self, csrf_token):
        return {
            'x-csrf-token': csrf_token,
            'content-type': 'application/json;charset=UTF-8',
        }

    def _resize_eyecatch_image(self, image_path, fn):
        basewidth = 600
        baseheight = 400
        img = Image.open(image_path)
        if img.size[0] < basewidth:
            wpercent = (basewidth / float(img.size[0]))
            hsize = int((float(img.size[1]) * float(wpercent)))
            img = img.resize((basewidth, hsize), Image.ANTIALIAS)

        if img.size[1] < baseheight:
            wpercent = (baseheight / float(img.size[1]))
            wsize = int((float(img.size[0]) * float(wpercent)))
            img = img.resize((wsize, baseheight), Image.ANTIALIAS)

        ratio = float(img.size[0]) / float(img.size[1])

        y1 = (img.size[1] - baseheight) / 2
        y2 = y1 + baseheight
        x1 = (img.size[0] - basewidth) / 2
        x2 = x1 + basewidth

        if ratio > 1.5:
            w = int(float(img.size[1]) * 1.5)
            x1 = (img.size[0] - w) / 2
            x2 = x1 + w
            img = img.crop((x1, 0, x2, img.size[1]))
        else:
            h = int(float(img.size[0]) / 1.5)
            y1 = (img.size[1] - h) / 2
            y2 = y1 + h
            img = img.crop((0, y1, img.size[0], y2))

        if fn.find('.') == -1:
            fn = 'rand.' + fn

        img.save(self.conf['path']['tmp_image'] + '/' + fn)

    def _update_article_image_id(self, article):
        csrf_token = self.get_csrf_token_from_page(
            '{}/articles/edit/{}'.format(self.admin_url, article['id']))
        update_params = article
        r = self.s.put('{}/api/v1/articles/{}'.format(self.admin_url,
                                                      article['id']),
                       json=update_params,
                       headers=self._create_headers(csrf_token))
        result = json.loads(r.text)
        if r.status_code == 200:
            self.d.log('image_id update Done.', article['id'],
                       result['payload']['message'])
        else:
            if 'errors' in result['payload']:
                for k in result['payload']['errors']:
                    self.d.error('ERROR', k, result['payload']['errors'][k])

    def _replace_image_url_to_media(self, xds, article_html_content):
        for xd in xds:
            xt = xd[0].split(u'"')[0]
            xr = xd[1].split(u'"')[0]
            try:
                article_html_content = article_html_content.replace(
                    xt, 'https://admin.localhost/media/image/{}'.format(
                        self.uii[xr]))
            except:
                self.d.error('failed replace image url to media ... key =', xr)

        return article_html_content
예제 #7
0
from tools import Debug

orig_prettify = BeautifulSoup.prettify
r = re.compile(r'^(\s*)', re.MULTILINE)


def prettify(self, encoding=None, formatter="minimal", indent_width=4):
    return r.sub(r'\1' * indent_width, orig_prettify(self, encoding,
                                                     formatter))


BeautifulSoup.prettify = prettify

if __name__ == '__main__':
    ## init logger
    d = Debug(level=4, color=True)

    db_conn = None
    try:
        d.info('connect to database ...')
        db_conn = DBConn(config['dev']['db'])
        d.info('[OK]\n')
    except Exception as e:
        d.error(e)
        exit()

    cursor = db_conn.get_cursor()

    ## init directories
    current_dir = os.path.dirname(os.path.abspath(__file__))
    output_dir = os.path.abspath(os.path.join(current_dir, os.pardir,