def clearfinal(): """ if a directory doesn't exist on start, it must not be also in destination """ dir1 = sgconf.cfgget("dirstart") dir2 = sgconf.cfgget("dirfinal") # first, I clear directories that don't exist anymore for root, subf, files in os.walk(dir2, topdown=False): for fn in subf: chkdir = os.path.join(root, fn) exdir = chkdir.replace(dir2, dir1) if not os.path.exists(exdir): try: showmsg("Clearing directory " + fn, 0) shutil.rmtree(chkdir) except: pass # then, I delete files: two ways. html & php are composed so others are only copied for root, subf, files in os.walk(dir2): for fn in files: chkfile = os.path.join(root, fn) chkext = os.path.splitext(chkfile)[1] if chkext == ".html" or chkext == ".php": oldfile = str( os.path.splitext(chkfile)[0].replace( dir2, dir1)) + sgconf.cfgget("processingext") if not os.path.exists(oldfile): os.remove(chkfile) else: oldfile = chkfile.replace(dir2, dir1) if not os.path.exists(oldfile): os.remove(chkfile)
def conformitynamecheck(): """ :return: none """ spaths = [] # two folders are checked, main and newposts, then i advise for empty values if sgconf.cfgget("dirstart") == "": showmsg("You've to set the start directory before check filenames", 99) return else: spaths.append(sgconf.cfgget("dirstart")) if sgconf.cfgget("dirnewposts") != "": spaths.append(sgconf.cfgget("dirnewposts")) for sdir in spaths: for root, subFolders, files in os.walk(sdir): for fn in files: newfile = fn.lower() newfile = newfile.replace("'", "_") newfile = newfile.replace(" ", "-") newfile = newfile.replace("..", ".") if fn != newfile: try: showmsg(" renaming " + os.path.join(root, fn), 0) os.rename(os.path.join(root, fn), os.path.join(root, newfile)) except: showmsg(" error renaming " + os.path.join(root, fn), 0)
def extgraphicinfo(immagine, cosa): """ extracts the information for image :param immagine: name of image file :param cosa: what to extract linksize=800x600 arrdim=[800, 600] :return: """ if cosa == "linksize": cmdl = sgconf.cfgget("appidentify") + " -format \'%w %h\' '" + immagine + "'" v = extgetcmd(cmdl) arr = v.split(" ") v = "width='" + arr[0] + "' height='" + arr[1] + "'" return v elif cosa == "exif": cmdl = sgconf.cfgget("appidentify") + " -format %[exif:*] '" + immagine + "'" v = extgetcmd(cmdl) return v elif cosa == "arrdim": cmdl = sgconf.cfgget("appidentify") + " -format \'%w %h\' '" + immagine + "'" v = extgetcmd(cmdl) arr = v.split(" ") # if an image is corrupted, it will return strings, not numbers if not arr[0].isnumeric(): arr[0] = 1 if not arr[1].isnumeric(): arr[1] = 1 return arr
def extgraphicresize(immagine, larghezza, thumb): """imagemagick command that resizes images :param immagine: image file :param larghezza: width :param thumb: is a thumbnail? :return: none """ compressione = sgconf.cfgget("imagescompression") densita = sgconf.cfgget("imagesresolution") if thumb == "": extbackup(immagine) cmdline = sgconf.cfgget("appmogrify") + ' ' + sgconf.cfgget("imagestriptags") + ' -density ' + str(densita) + ' -quality ' + str(compressione) + ' -scale ' + str(larghezza) + ' "' + immagine + '"' extruncmd(cmdline, True) else: cmdline = sgconf.cfgget("appconvert") + ' ' + sgconf.cfgget("imagestriptags") + ' -quality ' + str(compressione) + ' -scale ' + str(larghezza) + ' "' + immagine + '" "' + thumb + '"' extruncmd(cmdline, True) # this will resize thumbnails if width < height arr = extgraphicinfo(thumb, "arrdim") w = int(arr[0]) h = int(arr[1]) if w < h: cmdline = sgconf.cfgget("appmogrify") + ' ' + sgconf.cfgget("imagestriptags") + ' -quality ' + str(compressione) + ' -crop ' + str(larghezza) + 'x' + str(larghezza) + '+0-' + str(round((h - w) / 2)) + ' "' + thumb + '"' extruncmd(cmdline, True) #
def file_search_replace(txt): """ replace some user defined variables :param txt: :return: """ global listarepl if not listarepl: replfile = os.path.join(sgconf.cfgget("dirstart"), "site", "replace.conf") if os.path.exists(replfile): showmsg(" replacing contents in " + replfile, 0) with open(replfile, newline='') as csvfile: sr = csv.reader(csvfile, delimiter='|') for row in sr: a = row[0] b = row[1] listarepl[a] = b csvfile.close() else: listarepl["zzz"] = "zzz" for key, value in listarepl.items(): txt = txt.replace(key, value) return txt
def rsssetauthor(proposedname, proposedmail): """ if author is not setted, than get defaults function should return an email address :param proposedname: default author mode :param proposedmail: default author mail :return: author name for rss feed """ if proposedname == "": res = sgconf.cfgget("rssauthor") else: if proposedmail.find("@") < 2 or proposedname == "": res = sgconf.cfgget("rssauthor") else: res = proposedmail + " (" + proposedname + ")" return res
def process(cosa, nomi, elenco): """ new proc for templating chunks :param cosa: what is going to be templated, tipically a variable set from chunks. See [1] :param nomi: list of variable names in the substitution, can be empty an empty list if elenco is a dict :param elenco: a list if nomi is set, or a dictionary :return: """ mydiz = {} if cosa.startswith("chunk"): text = sgconf.cfgget(cosa) else: text = cosa # simpliest substitution: one or more parameters, without # any kind of repeating if isinstance(nomi, str) and isinstance(elenco, dict): if nomi == "": mydiz = elenco text = Template(text).safe_substitute(mydiz) return text # repeat a single value for more than 1 block: it can be repeated as a list # (name repeated is named as "nomi", while if it is a dict in the blocks are # templated key and value if isinstance(nomi, str) and isinstance(elenco, list): res = "" kindoflist = "" regstart = "<for " + nomi + ">" regend = "<next " + nomi + ">" regex = regstart + ".*?" + regend re.compile(regex) if all(isinstance(i, str) for i in elenco): kindoflist = "lst" if all(isinstance(i, dict) for i in elenco): kindoflist = "dct" matches = re.findall(regex, text, re.DOTALL) if matches: for m in matches: newdata = m[len(regstart):] newdata = newdata[:-len(regend)] if kindoflist == "lst": for e in elenco: mydiz[nomi] = e res += Template(newdata).safe_substitute(mydiz) if kindoflist == "dct": for mydiz in elenco: res += Template(newdata).safe_substitute(mydiz) text = text.replace(m, res) return text
def getlinkreplace(linea): """replace a fast link with the correct link""" flink = linea fdir = os.path.dirname(sgconf.cfgget("z_currentfile")) fname = os.path.splitext(flink)[0] fext = os.path.splitext(flink)[1] appl = "" #res = "" listimages = [".bmp", ".jpg", ".png"] listaudio = ["dsd", "dsf", ".flac", ".mp3", ".ogg", ".wav", "wv"] listvideo = [".flv", ".mkv", ".mp4", ".ogm", ".webm"] # image files ------------------------------------------------------------------------------- if fext in listimages: res = "<img src='" + flink + "' " + sgexternal.extgraphicinfo( os.path.join(fdir, linea), "linksize") + " alt='" + fname + "' class='sgfastimage'>" # audio files ------------------------------------------------------------------------------- elif fext in listaudio: res = "<audio controls class='sgfastaudio'>\n" if fext == ".ogg": appl = "audio/ogg" elif fext == ".mp3": appl = "audio/mpeg" elif fext == ".wav": appl = "audio/wave" res += " <source src='" + flink + "' type='" + appl + "'>\n" res += " <a href='" + flink + "'>" + fname + "</a>\<br>n" res += " Your browser does not support the audio element.\n</audio>\n" # video files -------------------------------------------------------------------------------- elif fext in listvideo: res = Template( sgconf.cfgget("chunk_video_link")).safe_substitute(filelink=fname) elif fext == ".youtube": res = Template( sgconf.cfgget("chunk_youtube_link")).safe_substitute(p1=fname) # all others, including html and php ---------------------------------------------------------- else: res = "<a href='" + seturlencode(flink) + "'>" + fname + "</a>" return res
def showmsg(message, modo): """ :param message: message to be showed :param modo: the way message is showed 0=show, 9=log :return: link corrected """ if sgconf.cfgget("hideprogressmessages") == "no" and modo != 1: return if message == "": return if modo == 0: # normal print(message) elif modo == 1: # error print(message) elif modo == 9: # log file file_write(sgconf.cfgget("logfile"), message + "\n", "a") elif modo == 99: # debug print("..................\n.. " + message + "\n..................")
def cuttext(text, mode): """ cut the text passed to function :param text: text to be cutted :param mode: the way :return: """ if mode == "summary": maxdim = sgconf.cfgget("postsummarylength") return textwrap.shorten(sterilizetemplates(text), width=maxdim, placeholder="")
def getuniqueid(nf, modo): """ get an unique random string. For file id, it's simply an unique id. For 'salt', if you think to secure relatively the process you can set in action file the string, or put in it the value 'random' or simply nothing, to make prefix always changing. A consideration is that you need to upload always the hashed password files. This is not generally a problem, due they are often small files, but think to transfer mode due security, to passwords length, to simple words, etc etc :param nf: file name if useful, :param modo: the kind of id :return: """ if modo == "fileid": res = ''.join(random.choice(string.ascii_lowercase) for i in range(12)) res += "-" + os.path.basename(os.path.splitext(nf)[0]) elif modo == "salt": if sgconf.cfgget("saltprefix") == "": res = ''.join( random.choice(string.ascii_lowercase) for i in range(2)) else: res = sgconf.cfgget("saltprefix") else: res = "" return res
def markup(elenco): """try to convert some rows as utility and some as an implementation of a markup""" testo = '\n'.join(elenco) # nobracket = r'[^\]\[]*' # brk = (r'\[(' + (nobracket + r'(\[' + nobracket) * 6 + (nobracket + r'\])*' + nobracket) * 6 + nobracket + r')\]') local_re = r'<:([^>]*)>' # <:link> #mark_re = r'<m:([^>]*)>' # <m: * > notes_re = r'<n:([^>]*)>' # <n: * > # quote_re = r'<q:([^>]*)>' # <q: * > # filelist_re = r'\$\{?listgeneric\(.*\)\}?' # listgeneric(kindoffiles) # history_re = r'\$\{?history\([0-9][0-9]\)\}?' # listgeneric(kindoffiles) scripts_re = r'\$\{?script\(.*\)\}?' # script(scriptname) lang_re = r'\$\{?lang\([a-z][a-z]\)\}?' # listgeneric(kindoffiles) # for internal use square_re = r'<sqr:([^>]*)>' # <sqr: * > for line in re.findall(lang_re, testo): testo = testo.replace(line, sgproc.replacelanguage(line)) for line in re.findall(local_re, testo): testo = testo.replace("<:" + line + ">", getlinkreplace(line)) for line in re.findall(filelist_re, testo): testo = testo.replace(line, sgproc.replacefilelist(line)) for line in re.findall(scripts_re, testo): testo = testo.replace(line, sgproc.pagescriptresult(line)) # for line in re.findall(mark_re, testo): # testo = testo.replace("<m:" + line + ">", "<mark class='sgmark'>" + line + "</mark>") for line in re.findall(square_re, testo): testo = testo.replace("<sqr:" + line + ">", getindexsquare(line)) # for line in re.findall(quote_re, testo): # testo = testo.replace("<q:" + line + ">", sgchunks.process(sgconf.cfgget("chunk_note"), "", {"quote": line})) myconds = sgconf.cfgget("replacetagfile") if myconds: for r in myconds: if testo.find(r[0]) < testo.rfind(r[0]): m_re = r[0] + '(.*?)' + r[0] for line in re.findall(m_re, testo): testo = testo.replace(r[0] + line + r[0], r[1] + line + r[2]) testo = sgmistune.markdown(testo, False) # these lines about correcting the results from sgmistune if testo[:3] == "<p>": if testo.rstrip()[-4:] == "</p>": testo = testo.rstrip()[3:-4] testo = testo.replace("${}", "</div>") return testo
def dir_clean(nomedir): """clean directory tree recursively removing empty dirs :param nomedir: the name of directory :return: none """ if nomedir.find(sgconf.cfgget("dirstart")) < 0: return for dirpath, dirnames, filenames in os.walk(nomedir, topdown=False): try: os.rmdir(dirpath) except: pass
def extbackup(fn): # noinspection PyPep8 """backup an image should copy a backup of the modified images in a directory, and adding a number at the end if it is already present a file with that name :param fn: filename :return: none """ if sgconf.cfgget("dirbackup") != "": today = time.strftime("%Y-%m-%d") db = os.path.join(sgconf.cfgget("dirbackup"), today) if not os.path.exists(db): os.makedirs(db) if os.path.exists(db): newfile = os.path.join(db, os.path.basename(fn)) if not os.path.exists(newfile): shutil.copy(fn, newfile) else: for numero in range(1, 99): newfile = os.path.join(db, os.path.basename(fn) + "." + str(numero)) if not os.path.exists(newfile): shutil.copy(fn, newfile) return
def file_trash(filecont): """ remove all files containings this in name :param filecont: files that contains this will be deleted :return: none """ sdir = sgconf.cfgget("dirstart") for root, subdirs, files in os.walk(sdir): for fl in files: if fl.find(filecont) >= 0: showmsg("Removing " + os.path.join(root, fl), 0) try: os.remove(os.path.join(root, fl)) except: pass
def rsspreview(text, alternative, linkf): """ get a bit of the text to be used in rss summary. Different from the other summary in index files, it doesn't consider actually the fact that languages for article can be more of one. :param text: text of article :param alternative: if text is none, generally title is :param linkf: :return: smaller text """ misura = int(sgconf.cfgget("rsssummarylength")) thumbpath = os.path.join( os.path.dirname(linkf), "thumbnails", os.path.splitext(os.path.basename(linkf))[0] + ".jpg") if text != "": text = sgutils.removehtml(text) text = textwrap.shorten(text, width=misura, placeholder="") if text.rfind(".") > misura - 10: text = text[:text.rfind(".")] elif text.rfind(" ") > misura - 20: text = text[:text.rfind(" ")] else: text = alternative res = text + " [...]" res = sgutils.removehtml(res) res = res.replace("<", "") if os.path.exists(thumbpath): res = "<img src='http://" + sgconf.cfgget( "sitename") + "/" + thumbpath.replace( sgconf.cfgget("dirstart"), "") + "' " + sgexternal.extgraphicinfo( thumbpath, "linksize") + "><p>\n" + res return res
def clearstart(): """ clear some kinds of files if they don't have to exists anymore """ listimagedirs = sgconf.cfgget("dirimages").split("|") for e in listimagedirs: for root, subf, files in os.walk( os.path.join(sgconf.cfgget("dirstart"), e)): for fn in files: basefile = os.path.splitext(fn)[0] # base name extfile = os.path.splitext(fn)[1] # extension chkfile = os.path.join(root, basefile) # all path except extension if extfile == sgconf.cfgget("processingext"): if not os.path.exists(chkfile + sgconf.cfgget("processingext")): showmsg( " removing " + chkfile + sgconf.cfgget("processingext"), 0) try: os.remove(chkfile + sgconf.cfgget("processingext")) except: pass
def getsummary(fn): # return a summary of the article conto = 0 # to be sure that variable is set maxdim = sgconf.cfgget("postsummarylength") txt = sgutils.file_read(fn) pos = txt.find("\n:> lang:") if pos >= 0: multilang = True else: multilang = False lista = txt.split("\n") dictl = {} for i in range(0, len(lista)): if lista[i][:3] != ":> " or lista[1][:8] == ":> lang:": conto = i - 1 break lista = lista[-(len(lista) - conto):] if multilang: condition = "all" dictl[condition] = "" for i in range(0, len(lista)): if lista[i][:8] == ":> lang:": try: condition = lista[i][-2:] if not condition in dictl: dictl[condition] = "" except: pass else: if len(dictl[condition]) < maxdim: if len(lista[i]) > 0: dictl[condition] = dictl[condition] + "\n" + lista[i] else: dictl[condition] = dictl[condition][:maxdim] txt = "" if not multilang: for n in lista: if len(txt) < maxdim and n[:3] != ":> ": txt = txt + "\n" + n if len(txt) >= maxdim: txt = sgutils.cuttext(txt, "summary") break txt = sgutils.removehtml(txt) txt = "<span class='sgsummary'>" + txt + "</span>" else: txt = "" for n in dictl: if n != "all": txt += "<span class='sgsummary sglang " + n + "' style='display:none;'>" + dictl[ n] + "...\n</span>" if txt.find("{") > 0: txt = txt.replace("{", "{") txt = txt.replace("}", "}") return txt
def getindexsquare(linea): """ function that returns the content of a block in the list files :param linea: content :return: the block """ dic = {} fname = os.path.join(sgconf.cfgget("dirstart"), linea) sdir = os.path.join(sgconf.cfgget("dirstart"), sgconf.cfgget("dirposts")) if not os.path.exists(fname): return linea nomefile = sgproc.fwe(fname) immagine = nomefile + ".jpg" thumbnail = os.path.join(os.path.dirname(immagine), "thumbnails", os.path.basename(immagine)) dati = sgproc.Pagina() sgproc.textget(fname, dati) titles = dati.title.split("|") mylink = sgproc.createpostslink(fname, sdir, "link") if len(titles) == 1: dic["title"] = "<a href='" + mylink + "'>" + sgproc.replacetitle( dati.title, 0) + "</a>" else: tmp = "" for j in range(0, len(titles)): tmp = tmp + "<a href='" + mylink + "' class='" + titles[ j][:2] + "' style='display:none;'>" + sgproc.replacetitle( titles[j], 0) + "</a>" dic["title"] = tmp dic["linktofullarticle"] = mylink dic["linkposts"] = sgproc.createpostslink(nomefile, sdir, "data") dic["tags"] = sgproc.replacetagfiles( nomefile.replace(sgconf.cfgget("dirstart"), ""), dati.tags, True, True) if os.path.exists(os.path.join(sdir, immagine)): if not os.path.exists(os.path.join(sdir, thumbnail)): sgproc.graphicresize(immagine, sgconf.cfgget("postsimagewidth"), sgconf.cfgget("poststhumbwidth")) if os.path.exists(os.path.join(sdir, thumbnail)): tmp = thumbnail.replace( os.path.join(sgconf.cfgget("dirstart"), sgconf.cfgget("dirposts")), "")[1:] desc = sgproc.replacetitle(sgproc.fwe(os.path.basename(tmp)), 0) tmp = seturlencode(tmp) dic["thumbdesc"] = desc dic["thumblink"] = sgutils.lnkc(tmp) dic["thumbsize"] = sgexternal.extgraphicinfo(thumbnail, "linksize") else: dic["thumbdesc"] = "" dic["thumblink"] = "" dic["thumbsize"] = "" #sort of summary if sgconf.cfgget("chunk_article_square").find("summary") > 0: nomefile = os.path.join(sgconf.cfgget("dirstart"), fname) dic["summary"] = getsummary(nomefile) else: dic["summary"] = "" return Template(sgconf.cfgget("chunk_article_square")).safe_substitute(dic)
def rssaddtolist(nomefile, mydate): global listarss listarefuse = [] maxnum = sgconf.cfgget("rsslistlength") if os.path.splitext(nomefile)[1] == sgconf.cfgget("processingext"): cln = nomefile[len(sgconf.cfgget("dirstart")):] # nome = mydate + " " + cln if listarss: if (mydate + " " + nomefile) in listarss: return if len(listarss) > maxnum * 2: listarss = sorted(listarss, reverse=True)[:maxnum] # no upper files if os.path.dirname(cln) == "": return # no files in site directory if fnmatch.fnmatch(cln, os.path.join("site", "*.*")): return # no image galleries if fnmatch.fnmatch(cln, os.path.join(sgconf.cfgget("dirimages"), "*")): return if os.path.dirname(cln) == sgconf.cfgget("dirposts"): return if os.path.dirname(cln) == sgconf.cfgget("dirperma"): return if fnmatch.fnmatch( cln, os.path.join(sgconf.cfgget("dirposts"), "*", sgconf.cfgget("indexfile") + ".*")): return # managing to no add some directories or files to rss.xml if sgconf.cfgget("hiddenpaths") != "": listarefuse = sgconf.cfgget("hiddenpaths").split("|") if sgconf.cfgget("rssnoadd") != "": listarefuse += sgconf.cfgget("rssnoadd").split("|") for lritem in listarefuse: if nomefile.find(lritem) > 0: return listarss.append(mydate + " " + nomefile)
def rssgo(): """ main function """ global listarss filerss = os.path.join(sgconf.cfgget("dirstart"), "site", "rss", "rss.xml") res = [] mp = sgproc.Pagina() lista = sorted(listarss, reverse=True)[:sgconf.cfgget("rsslistlength")] if len(lista) == 0: return # will avoid creation of a new rss file if there are no file added if os.path.exists(filerss): fileh = open(filerss) lines = fileh.readlines() fileh.close() try: oldfirst = os.path.splitext( os.path.basename(lines[14].replace("</link>", "")))[0] newfirst = os.path.splitext( os.path.basename(sgconf.cfgget("lastpost")))[0] if oldfirst == newfirst: return except: pass dt = strftime("%a, %d %b %Y %H:%M:%S %z", localtime()) res.append("<?xml version=\"1.0\" encoding=\"UTF-8\" ?>") res.append( "<rss version=\"2.0\" xmlns:atom=\"http://www.w3.org/2005/Atom\">") res.append("<channel>") # res.append("<atom:link href=\"http://" + cfgget("sitename") + "/" + cfgget("dirposts") + "/" + "rss.xml" + "\" rel=\"self\" type=\"application/rss+xml\">") res.append("<title>" + sgconf.cfgget("sitename") + "</title>") res.append("<link>http://" + sgconf.cfgget("sitename") + "/</link>") res.append("<description><![CDATA[" + sgconf.cfgget("rssdescription") + "]]></description>") res.append("<language>" + sgconf.cfgget("defaultlang") + "</language>") res.append("<pubDate>" + dt + "</pubDate>") res.append("<lastBuildDate>" + dt + "</lastBuildDate>") res.append("<docs>http://blogs.law.harvard.edu/tech/rss</docs>") res.append("<generator>" + sggen.getversion(True) + "</generator>") for ele in lista: myfile = ele[9:] fn = myfile.replace(sgconf.cfgget("dirstart"), "") fn = os.path.splitext(fn)[0] + ".html" dt = ele[:8] f = time.mktime((rssint(dt[0:4]), rssint(dt[4:6]), rssint(dt[6:8]), 0, 0, 1, 0, 0, 0)) dt = time.strftime("%a, %d %b %Y %H:%M:%S %z", time.localtime(f)) sgproc.textget(myfile, mp) descr = rsspreview(mp.text, mp.title, mp.filepath) # xtext, xtitle, xtags, xfilesize, xfilelastm, xauthor,xfilename sitelink = os.path.join(sgconf.cfgget("sitename"), fn) res.append(" <item>") res.append(" <title><![CDATA[" + mp.title + "]]></title>") res.append(" <author><![CDATA[" + rsssetauthor(mp.author, mp.authormail) + "]]></author>") res.append(" <link>http://" + sitelink + "</link>") res.append(" <description><![CDATA[" + descr + "]]></description>") res.append(" <pubDate>" + dt + "</pubDate>") res.append(" <comments>http://" + sitelink + "</comments>") res.append(" <guid isPermaLink=\"false\">" + rsssetguid(ele) + "</guid>") res.append(" </item>") # res.append("</atom:link>") res.append("</channel>") res.append("</rss>") if os.path.exists(filerss): os.remove(filerss) filecontent = "\n".join(res) sgutils.file_write(filerss, filecontent, "w")