def processing(ljname: str, skip=None or int, maxcount=None or int, forced=False):
lj.title('processing raw -- to --> data')
filename = '{0}/calendar/_{0}_.txt'.format(ljname)
if lj.checkfileexist(filename):
posts = lj.loadfromfile(filename, lines=True)
for post in posts[skip:maxcount]:
print(post)
url = post.split(' ', 1)[0]
parseljpost(url, forced)
print('-'*40)
print(len(posts))
def makeupdate(ljname: str, count: int):
if not count:
return
lj.title('make web update')
dir = lambda ljname, dir: '{ljname}/{dir}'.format(ljname=ljname, dir=dir)
udir = lambda ljname, dir: '{ljname}/_update_/{dir}'.format(ljname=ljname, dir=dir if ljname not in dir else dir.repl(ljname, ljname+'/_update_', 1))
# lj.checkpath(udir(ljname,'data/'))
# lj.checkpath(udir(ljname,'images/userpics/'))
# lj.checkpath(udir(ljname,'images/comments/'))
posts = lj.loadfromfile('{0}/calendar/_{0}_.txt'.format(ljname), True)[:count+1]
files = list()
upics = list()
ids = list()
for post in posts:
postid = lj.extractpostid(post)
ids.append(postid)
jsondata = json.loads(lj.loadfromfile('{dir}/{postid}.htm.json'.format(dir=dir(ljname, 'data'), postid=postid)))
for comment in jsondata['comments']:
filename = comment.get('userpic')
if filename:
filename = filename.split('/')[-1]
if filename not in upics:
upics.append(filename)
files += [postid + '.htm', postid + '.htm.json']
print(len(upics))
imgext = ['.jpg', '.png', '.gif', '.svg']
images = [file for file in os.listdir(dir(ljname, 'images/')) if file.endswith(tuple(imgext))]
images = [file for file in images if file.split('-', 1)[0] in ids]
cimages = [file for file in os.listdir(dir(ljname, 'images/comments/')) if file.endswith(tuple(imgext))]
cimages = [file for file in cimages if file.split('-', 1)[0] in ids]
# copyallfiles(files, dir(ljname, 'data/'), udir(ljname, 'data/'), True)
# copyallfiles(images, dir(ljname, 'images/'), udir(ljname, 'images/'), False)
# copyallfiles(upics, dir(ljname, 'images/userpics/'), udir(ljname, 'images/userpics/'), False)
# print('creating update is complete!')
lj.title('uploading data to ftp...')
ftptransfer(files, dir(ljname, 'data/'), 'data/', True)
lj.title('uploading images to ftp...')
ftptransfer(images, dir(ljname, 'images/'), 'images/', False)
lj.title('uploading userpics to ftp...')
ftptransfer(cimages, dir(ljname, 'images/comments/'), 'images/comments/', False)
lj.title('uploading comment images to ftp...')
ftptransfer(upics, dir(ljname, 'images/userpics/'), 'images/userpics/', False)
ftptransfer(['_{}_.txt'.format(ljname)], dir(ljname, 'calendar/'), 'calendar/', True)
print('uploading is complete!')
def makehtml(ljname: str,
skip=None,
maxcount=None,
mkposts=True,
mkindex=True):
# генерирует html контент на основе обработанных данных
lj.title('generate posts .htmls')
filename = '{0}/calendar/_{0}_.txt'.format(ljname)
lj.savetofile('', '{0}/calendar/_collapsed_.txt'.format(ljname),
'w') # clear comment errors file
if lj.checkfileexist(filename):
posts = lj.loadfromfile(filename, lines=True)
if mkposts:
for post in posts[skip:maxcount]:
print(post)
url = post.split(' ', 1)[0]
makepost(url)
if mkindex:
makeindex(ljname, posts)
uniqfilelines('{0}/calendar/_collapsed_.txt'.format(ljname), sort=True)
def makeindex(ljname, posts):
lj.title('generate html index')
csstime = int(time.time())
template = lj.loadfromfile('_index_templ.html').split('~')
templateblocks = {
'start': template[0],
'ok': template[1],
'none': template[2],
'end': template[3]
}
fields = {'ljname': ljname, 'time': csstime}
html = templateblocks['start'].format(**fields)
for item in posts:
post = item.split(' ', 1)
url = post[0]
title = post[1]
postid = lj.extractpostid(url)
block = 'ok' if lj.checkfileexist('{}/post/{}.html'.format(
ljname, postid)) else 'none'
html += templateblocks[block].format(postid=postid, title=title)
html += templateblocks['end']
lj.savetofile(html, '{}/index.html'.format(ljname))
def testdata(ljname: str):
lj.title('testing')
dir = '{}/raw/'.format(ljname)
posts = lj.loadfromfile('{0}/calendar/_{0}_.txt'.format(ljname), True)
postsids = list(map(lj.extractpostid, posts))
for file in os.listdir(dir):
if file.endswith('.json'):
postid = int(file.split('.',1)[0])
filename = dir + file
jsondata = json.loads(lj.loadfromfile(filename))
nav = jsondata['entry']['nav']
if len(nav['prev']) < 5 or len(nav['next']) < 5:
print('{} * {} * {}'.format(nav['prev'], postid, nav['next']))
if str(postid) in postsids:
nav = lj.getljnavigation(posts, postsids.index(str(postid)))
if len(nav['next']) > 5 or len(nav['prev']) > 5:
jsondata['entry']['nav'] = nav
lj.savetofile(jsondata, filename, 'w')
print('Навигация успешно исправлена: {} {} {}'.format(postid, nav['prev'], nav['next']))
else:
print('Не удалось исправить навигацию: {}'.format(postid))
else:
print('Пост {} отсутствует в списке постов. Обновите список постов!'.format(postid))