async def save(data: str,
filepath: Path,
mode: str = 'wt',
encoding: str = 'utf-8'):
"""
Save **data** to **filepath**
:param data: Contains data to be saved
:type data: str
:param filepath: Path to the file where **data** will be saved
:type filepath: Path
:param mode: Mode to upon which the file opened
:type mode: str
:param encoding: Encoding to encode write data to file
:return: None
"""
create_dirs(filepath)
with filepath.open(mode=mode, encoding=encoding) as f:
f.write(data)
def download_images(self):
input_file = self.project_dir + '/output/s3_images/s3_images.json'
with open(input_file, 'r') as f:
## import articles
self.d.info('read file ... {}'.format(input_file))
s3_images = json.load(f)
print len(s3_images.keys())
for d in s3_images:
for origin_url in s3_images[d]:
# create dirs
image_path = origin_url.split('https://prd.localhost/')[1]
image_dir = '/'.join(image_path.split('/')[:-1])
helpers.create_dirs(self.project_dir + '/output',
[image_dir])
s3_url = s3_images[d][origin_url]
urllib.urlretrieve(
origin_url, 'output/' + image_dir + '/' +
origin_url.split('/')[-1])
def load(self) -> None:
"""
Download thumbnails if THUMBS_DIR is empty
or JSON_FILE don't exist.
Otherwise fill stacked widget with existing thumbnails
"""
create_dirs(THUMBS_DIR, CURRENT_DIR, SAVED_DIR)
if JSON_FILE.exists():
if not is_dir_contains_images(THUMBS_DIR):
logger.debug(
f"{short_path(THUMBS_DIR)} is empty. Downloading new thumbnails"
)
self.progressbar.show()
self.download_thumbs()
else:
logger.debug('Filling stacked widget')
self.sw.fill()
self.change_info()
else:
logger.debug(f"{short_path(JSON_FILE)} doesn't exist. Updating")
self.update_()
def _upload_media_image(self,
article,
upload_flag=True,
save_to_local=False):
thumb_url = article['thumb_url']
if thumb_url == "":
if len(article['image_urls']) > 0:
thumb_url = article['image_urls'][0]
else:
article['image_id'] = self._default_image_id
return article
fn = thumb_url.split('/')[-1]
open_fn = fn
file_exist_flag = os.path.isfile(self.project_dir +
'/output/wp-content-dl/uploads/' + fn)
if not file_exist_flag:
for ch in '12345':
tmp = os.path.splitext(fn)
open_fn = '{}{}{}'.format(tmp[0], ch, tmp[1])
if os.path.isfile(self.project_dir +
'/output/wp-content-dl/uploads/' + open_fn):
file_exist_flag = True
break
if not file_exist_flag:
# fetch image
image_url = 'https://prd.localhost/wp-content/uploads/{}'.format(
fn)
self.d.log('no image file found. start fetch files ...', image_url)
image_dir = 'wp-content-dl'
helpers.create_dirs(self.project_dir + '/output', [image_dir])
urllib.urlretrieve(
image_url,
'output/' + image_dir + '/uploads/' + image_url.split('/')[-1])
open_fn = fn
file_exist_flag = True
image_path = self._get_image_full_path(open_fn)
is_image = False
try:
with Image.open(image_path) as f:
is_image = True
except:
pass
if file_exist_flag and is_image:
## use default image
if fn == 'logo_thumbnail1.jpg':
article['image_id'] = self._default_image_id
return article
## resize image before upload
self._resize_eyecatch_image(image_path, fn)
if upload_flag:
self.d.debug('start upload image... {}'.format(fn))
result = self._create_image_info()
image_file_id, image_id, upload_url, content_type = self._generate_image_file_id(
fn, result['payload']['Id'])
#self.d.debug('upload url = {}'.format(upload_url))
with open('/tmp/{}'.format(fn), 'r') as f:
if upload_flag:
image_data = f.read()
headers = {
'content-type': content_type,
'content-length': str(len(image_data)),
}
r = self.s.put(upload_url,
data=image_data,
headers=headers)
if (upload_flag
and r.status_code == 200) or not upload_flag:
if upload_flag:
# doing image validation. ( /api/batch/media/image/validate/:id )
self.s.get(
'{}/api/batch/media/image/validate/{}'.format(
self.admin_url, image_file_id))
article['image_id'] = image_id
self.d.debug(
'image uploaded. post_id={}, image_id={}'.
format(article['id'], image_file_id))
# remove img tag in html_content
#article = self._remove_eyecatch_image_in_content(article)
else:
article['image_id'] = self._default_image_id
self.d.error('error uploading image. ID={}'.format(
article['id']))
# remove tmp image after upload finished
os.remove(self.conf['path']['tmp_image'] + '/' + fn)
if save_to_local:
dst_dir = '/tmp/u/' + self.air[
article['id']]['filename'].split('/')[0]
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
dst_file = '/tmp/u/' + self.air[article['id']]['filename']
src_file = self.conf['path']['tmp_image'] + '/' + fn
if os.path.exists(dst_file):
os.remove(dst_file)
try:
shutil.move(src_file, dst_file)
except:
pass
else:
article['image_id'] = self._default_image_id
self.d.error('thumb image not found. [post_id = {}][{}]'.format(
article['id'], fn))
return article
try:
d.info('connect to database ...')
db_conn = DBConn(config['dev']['db'])
d.info('[OK]\n')
except Exception as e:
d.error(e)
exit()
cursor = db_conn.get_cursor()
## init directories
current_dir = os.path.dirname(os.path.abspath(__file__))
output_dir = os.path.abspath(os.path.join(current_dir, os.pardir,
'output'))
output_dirs = ['wp_users', 'wp_posts', 'categories', 'tags', 's3_images']
helpers.create_dirs(output_dir, output_dirs)
## get shorten url mappings
shorten_url_dict = get_shorten_urls(cursor)
shorten_url_keys = shorten_url_dict.keys()
shorten_url_id = 1
for k in shorten_url_dict:
shorten_url_dict[k]['id'] = shorten_url_id
shorten_url_id += 1
shorten_url_dict[k]['link'] = shorten_url_dict[k]['link'].replace(
'https://prd.localhost//', '/')
shorten_url_dict[k]['link'] = shorten_url_dict[k]['link'].replace(
'https://prd.localhost/', '/')