def main():
logging.basicConfig(
#filename=LOGDIR + "/rssSocial.log",
stream=sys.stdout,
level=logging.INFO,
format="%(asctime)s [%(filename).12s] %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
msgLog = "Launched at %s" % time.asctime()
logMsg(msgLog, 1, 2)
rules = moduleRules()
srcs, dsts, ruls, impRuls = rules.checkRules()
rules.printList(srcs, "Sources")
rules.printList(dsts, "Destinations")
print("Rules")
for i,rul in enumerate(sorted(ruls)):
print(f"{i}) {rul}")
for j, action in enumerate(ruls[rul]):
print(f" └---->{j}) {action}")
return
def readConfigSrc(self, indent, src, more):
msgLog = f"Src: Src {src}"
logMsg(msgLog, 2, 0)
msgLog = f"More: Src {more}"
logMsg(msgLog, 1, 0)
if src[0] == 'cache':
apiSrc = getApi(src[0], src[1:])
apiSrc.fileName = apiSrc.fileNameBase(src[1:])
apiSrc.postaction = 'delete'
else:
apiSrc = getApi(src[0], src[2])
for option in more:
if option == 'posts':
nameMethod = f"setPostsType"
else:
nameMethod = f"set{option.capitalize()}"
if nameMethod in apiSrc.__dir__():
# setCache ¿?
# url, time, max, posts,
cmd = getattr(apiSrc, nameMethod)
if inspect.ismethod(cmd):
cmd(more[option])
else:
for name in apiSrc.__dir__():
if name.lower() == nameMethod.lower():
cmd = getattr(apiSrc, name)
if inspect.ismethod(cmd):
cmd(more[option])
break
if not apiSrc.getPostsType():
apiSrc.setPostsType('posts')
# apiSrc.setPosts()
# print(f"Postsss: {apiSrc.getPosts()}")
return apiSrc
def hasPublishMethod(self, service):
clsService = getModule(service)
listMethods = clsService.__dir__()
methods = []
target = None
for method in listMethods:
if method.find("publish") >= 0:
action = "publish"
target = ""
moduleService = clsService.publishPost.__module__
if method.find("Api") >= 0:
target = method[len("publishApi"):].lower()
msgLog = (f"Target api {target}")
logMsg(msgLog, 2, 0)
elif moduleService == f"module{service.capitalize()}":
target = method[len("publish"):].lower()
msgLog = (f"Target mod {target}")
logMsg(msgLog, 2, 0)
if target and (target!='image'):
toAppend = (action, target)
if not (toAppend in methods):
methods.append(toAppend)
return methods
def readConfigDst(self, indent, action, more, apiSrc):
profile = action[2]
nick = action[3]
socialNetwork = (profile, nick)
msgLog = (f"{indent}socialNetwork: {socialNetwork}")
logMsg(msgLog, 2, 0)
msgLog = (f"{indent}Action: {action}")
logMsg(msgLog, 1, 0)
msgLog = (f"{indent}More: Dst {more}")
logMsg(msgLog, 1, 0)
if action[0] == "cache":
print(f"Dst: {action}")
apiDst = getApi("cache", ((more['service'], action[1]),
f"{action[2]}@{action[3]}", 'posts'))
apiDst.socialNetwork = action[2]
apiDst.nick = action[3]
apiDst.fileName = apiDst.fileNameBase(apiSrc)
else:
apiDst = getApi(profile, nick)
apiDst.setUser(nick)
apiDst.setPostsType('posts')
msgLog = (f"{indent}Api dst: {apiDst}")
logMsg(msgLog, 2, 0)
if 'max' in more:
mmax = more['max']
elif 'buffermax' in more:
mmax = more['buffermax']
else:
mmax = 0
apiDst.setMax(mmax)
if 'time' in more:
apiDst.setTime(more['time'])
return apiDst
def checkRules(self):
msgLog = "Checking rules"
logMsg(msgLog, 1, 2)
config = configparser.ConfigParser()
config.read(CONFIGDIR + "/.rssBlogs")
services = self.getServices()
services['regular'].append('cache')
# cache objects can be considered as regular for publishing elements
# stored there
indent = 3*" "+4*" "
srcs = []
srcsA = []
more = []
dsts = []
ruls = {}
mor = {}
impRuls = []
for section in config.sections():
url = config.get(section, "url")
msgLog = f"Section: {section} Url: {url}"
logMsg(msgLog, 1, 1)
# Sources
moreS = dict(config.items(section))
moreSS = None
if "rss" in config.options(section):
rss = config.get(section, "rss")
msgLog = (f"Service: rss -> {rss}")
logMsg(msgLog, 2, 0)
toAppend = ("rss", "set",
urllib.parse.urljoin(url, rss), "posts")
srcs.append(toAppend)
more.append(moreS)
else:
msgLog = (f"url {url}")
logMsg(msgLog, 2, 0)
for service in services["regular"]:
if (
("service" in config[section])
and (service == config[section]["service"])
) or (url.find(service) >= 0):
methods = self.hasSetMethods(service)
logging.debug(f"Service: {service} has set {methods}")
for method in methods:
msgLog = (f"Method: {method}")
logMsg(msgLog, 2, 0)
msgLog = (f"moreS: {moreS}")
logMsg(msgLog, 2, 0)
# If it has a method for setting, we can set
# directly using this
if service in config[section]:
nick = config[section][service]
else:
nick = url
if ((nick.find("slack") < 0)
and (nick.find("gitter") < 0)):
nick = nick.split("/")[-1]
if (service == 'imdb') or (service == 'imgur'):
nick = url
elif ('twitter' in url):
nick = url.split("/")[-1]
if 'posts' in moreS:
if moreS['posts'] == method[1]:
toAppend = (service, "set", nick, method[1])
else:
toAppend = (service, "set", nick, method[1])
msgLog = (f"toAppend: {toAppend}")
logMsg(msgLog, 2, 0)
if not (toAppend in srcs):
if (('posts' in moreS)
and (moreS['posts'] == method[1])):
srcs.append(toAppend)
more.append(moreS)
else:
# Available, but with no rules
srcsA.append(toAppend)
fromSrv = toAppend
msgLog = (f"fromSrv toAppend: {toAppend}")
logMsg(msgLog, 2, 0)
msgLog = (f"fromSrv moreS: {moreS}")
logMsg(msgLog, 2, 0)
if "time" in config.options(section):
timeW = config.get(section, "time")
else:
timeW = 0
if "buffermax" in config.options(section):
bufferMax = config.get(section, "buffermax")
else:
bufferMax = 0
if "max" in config.options(section):
bufferMax = config.get(section, "max")
# Destinations
hasSpecial = False
if "posts" in config[section]:
postsType = config[section]["posts"]
else:
postsType = "posts"
if fromSrv:
fromSrv = ( fromSrv[0], fromSrv[1], fromSrv[2], postsType,)
for service in services["special"]:
toAppend = ""
msgLog = (f"Service: {service}")
logMsg(msgLog, 2, 0)
if service in config.options(section):
valueE = config.get(section, service).split("\n")
for val in valueE:
nick = config.get(section, val)
msgLog = (f"Service special: {service} "
f"({val}, {nick})")
logMsg(msgLog, 2, 0)
if service == "direct":
url = "posts"
toAppend = (service, url, val, nick) #, timeW, bufferMax)
msgLog = (f"Service special toAppend: {toAppend} ")
logMsg(msgLog, 2, 0)
msgLog = (f"Service special from: {fromSrv} ")
logMsg(msgLog, 2, 0)
if toAppend not in dsts:
dsts.append(toAppend)
if toAppend:
if fromSrv not in mor:
mor[fromSrv] = moreS
if fromSrv in ruls:
if not toAppend in ruls[fromSrv]:
ruls[fromSrv].append(toAppend)
msgLog = (f"1 added: {toAppend} "
f"in {fromSrv} ")
logMsg(msgLog, 2, 0)
else:
ruls[fromSrv] = []
ruls[fromSrv].append(toAppend)
msgLog = (f"1.1 added: {toAppend} "
f"in {fromSrv} ")
logMsg(msgLog, 2, 0)
hasSpecial = True
for service in services["regular"]:
if (service == 'cache'):
continue
toAppend = ""
if service in config.options(section):
methods = self.hasPublishMethod(service)
msgLog = (f"Service: {service} ({section}) has "
f"{methods}")
logMsg(msgLog, 2, 0)
for method in methods:
msgLog = (f"Method: {method}")
logMsg(msgLog, 2, 0)
# If it has a method for publishing, we can
# publish directly using this
if not method[1]:
mmethod = 'post'
else:
mmethod = method[1]
toAppend = (
"direct",
mmethod,
service,
config.get(section, service) #,
# timeW,
# bufferMax,
)
if not (toAppend in dsts):
dsts.append(toAppend)
if toAppend:
if hasSpecial:
msgLog = (f"hasSpecial: {fromSrv}---")
logMsg(msgLog, 2, 0)
msgLog = (f"hasSpecial: {toAppend}---")
logMsg(msgLog, 2, 0)
nickSn = f"{toAppend[2]}@{toAppend[3]}"
fromSrvSp = (
"cache",
(fromSrv[0], fromSrv[2]),
nickSn,
"posts",
)
impRuls.append((fromSrvSp, toAppend))
if fromSrvSp not in mor:
mor[fromSrvSp] = moreS
if fromSrvSp in ruls:
if not toAppend in ruls[fromSrvSp]:
ruls[fromSrvSp].append(toAppend)
msgLog = (f"2 added: {toAppend} "
f"in {fromSrvSp} ")
logMsg(msgLog, 1, 0)
else:
ruls[fromSrvSp] = []
ruls[fromSrvSp].append(toAppend)
if url:
msgLog = (f"2.1 added: {toAppend} "
f"in {fromSrvSp} "
f"with {url}")
else:
msgLog = (f"2.1 added: {toAppend} "
f"in {fromSrvSp} "
f"with no url")
logMsg(msgLog, 1, 0)
else:
msgLog = (f"From {fromSrv}")
logMsg(msgLog, 2, 0)
msgLog = (f"direct: {dsts}---")
logMsg(msgLog, 2, 0)
if fromSrv not in mor:
msgLog = (f"Adding {moreS}")
logMsg(msgLog, 2, 0)
mor[fromSrv] = moreS
if fromSrv in ruls:
if not toAppend in ruls[fromSrv]:
ruls[fromSrv].append(toAppend)
msgLog = (f"3 added: {toAppend} in "
f"{fromSrv} ")
logMsg(msgLog, 2, 0)
else:
ruls[fromSrv] = []
ruls[fromSrv].append(toAppend)
msgLog = (f"3.1 added: {toAppend} in "
f"{fromSrv} ")
logMsg(msgLog, 2, 0)
# Now we can add the sources not added.
for src in srcsA:
if not src in srcs:
msgLog = (f"Adding implicit {src}")
logMsg(msgLog, 2, 0)
srcs.append(src)
more.append({})
# Now we can see which destinations can be also sources
for dst in dsts:
if dst[0] == "direct":
service = dst[2]
methods = self.hasSetMethods(service)
for method in methods:
msgLog = (f"cache dst {dst}")
logMsg(msgLog, 2, 0)
toAppend = (service, "set", dst[3], method[1])#, dst[4])
msgLog = (f"toAppend src {toAppend}")
logMsg(msgLog, 2, 0)
if not (toAppend[:4] in srcs):
srcs.append(toAppend[:4])
more.append({})
elif dst[0] == "cache":
if len(dst)>4 :
toAppend = (dst[0], "set", (dst[1], (dst[2], dst[3])),
"posts", dst[4], 1)
else:
toAppend = (dst[0], "set", (dst[1], (dst[2], dst[3])),
"posts", 0, 1)
if not (toAppend[:4] in srcs):
srcs.append(toAppend[:4])
more.append({})
available = {}
myKeys = {}
myIniKeys = []
# for i, src in enumerate(srcs):
for i, src in enumerate(ruls.keys()):
if not src:
continue
iniK, nameK = self.getIniKey(src[0], myKeys, myIniKeys)
if not (iniK in available):
available[iniK] = {"name": src[0], "data": [], "social": []}
available[iniK]["data"] = [{'src': src[1:], 'more': more[i]}]
else:
available[iniK]["data"].append({'src': src[1:],
'more': more[i]})
# srcC = (src[0], "set", src[1], src[2])
# if srcC not in ruls:
# ruls[srcC] =
myList = []
for elem in available:
component = (
f"{elem}) "
f"{available[elem]['name']}: "
f"{len(available[elem]['data'])}"
)
myList.append(component)
if myList:
availableList = myList
else:
availableList = []
self.available = available
self.availableList = availableList
msgLog = (f"Avail: {self.available}")
logMsg(msgLog, 2, 0)
msgLog = (f"Ruls: {ruls}")
logMsg(msgLog, 2, 0)
self.rules = ruls
self.more = mor
return (srcs, dsts, ruls, impRuls)
def executeRules(self, args):
msgLog = "Execute rules"
logMsg(msgLog, 1, 2)
# print(args)
select = args.checkBlog
simmulate = args.simmulate
import concurrent.futures
delayedPosts = []
# import pprint
# pprint.pprint(self.rules)
# textEnd = ""
threads = 0
with concurrent.futures.ThreadPoolExecutor(max_workers=75) as pool:
i = 0
previous = ""
for src in sorted(self.rules.keys()):
if src[0] != previous:
i = 0
previous = src[0]
name = f"{src[0]}{i}>"
if src in self.more:
# f" More: {self.more[src]}")
more = self.more[src]
else:
# f" More: empty")
more = None
if src[0] in ['cache', 'gitter', 'gmail']:
srcName = more['url']
# FIXME
if 'slack' in srcName:
srcName = f"Slack({srcName.split('/')[2].split('.')[0]})"
elif 'gitter' in srcName:
srcName = f"Gitter({srcName.split('/')[-2]})"
elif 'imgur' in srcName:
srcName = f"Imgur({srcName.split('/')[-1]})"
elif '.com' in srcName:
srcName = f"Gmail({srcName.split('.')[0]})"
text = (f"Source: {srcName} ({src[3]})")
else:
text = (f"Source: {src[2]} ({src[3]})")
textEnd = (f"Source: {name} {src[2]} {src[3]}")
actions = self.rules[src]
for k, action in enumerate(actions):
if (select and (select.lower() != f"{src[0].lower()}{i}")):
actionMsg = f"Skip."
else:
actionMsg = (f"Scheduling...")
nameA = f"{name} {actionMsg} "
if action[1].startswith('http'):
# FIXME
theAction = 'posts'
else:
theAction = action[1]
msgLog = (f"{nameA} {text} Action {k}:"
f" {action[3]}@{action[2]} ({theAction})")
textEnd = f"{textEnd}\n{msgLog}"
logMsg(msgLog, 1, 1)
nameA = f"{name} [({theAction})"
# The '[' is closed in executeAction TODO
if actionMsg == "Skip.":
continue
timeSlots = args.timeSlots
noWait = args.noWait
# Is this the correct place?
if ((action[0] in 'cache') or
((action[0] == 'direct') and (action[2] == 'pocket'))
):
# We will always load new items in the cache
timeSlots = 0
noWait=True
threads = threads + 1
delayedPosts.append(pool.submit(self.executeAction,
src, more, action,
noWait,
timeSlots,
args.simmulate,
nameA))
i = i + 1
messages = []
for future in concurrent.futures.as_completed(delayedPosts):
try:
res = future.result()
msgLog = (f"End Delay: {res}")
logMsg(msgLog, 1, 1)
if res:
messages.append(
f" Published in: {future}\n{res} "
)
except Exception as exc:
#else:
msgLog = (f"{future} generated an exception: {exc} "
f"Src: {src}. Action: {action}")
logMsg(msgLog, 1, 1)
msgLog = (f"{sys.exc_info()}")
logMsg(msgLog, 1, 1)
import traceback
msgLog = (f"{traceback.print_exc()}")
logMsg(msgLog, 1, 1)
msgLog = (f"End Execute rules with {threads} threads.")
logMsg(msgLog, 1, 2)
return
def executeAction(self, src, more, action,
noWait, timeSlots, simmulate, name="",
nextPost = True, pos = -1, delete=False):
sys.path.append(path)
from configMod import logMsg
indent = f" {name}->({action[3]}@{action[2]})] -> "+" "
# The ']' is opened in executeRules FIXME
msgLog = (f"{indent}Sleeping to launch all processes")
logMsg(msgLog, 1, 0)
# 'Cometic' waiting to allow all the processes to be launched.
time.sleep(1)
msgAction = (f"{action[0]} {action[3]}@{action[2]} "
f"({action[1]})")
# Destination
apiSrc = self.readConfigSrc(indent, src, more)
if apiSrc.getName():
msgLog = (f"{indent}Source: {apiSrc.getName()} ({src[3]}) -> "
f"Action: {msgAction})")
else:
msgLog = (f"{indent}Source: {src[2]} ({src[3]}) -> "
f"Action: {msgAction})")
logMsg(msgLog, 1, 0)
textEnd = (f"{msgLog}")
# print(f"Srcccc: {src}")
# print(f"Srcccc: {apiSrc.getNick()}")
# return
if (apiSrc.getHold() == 'yes'):
time.sleep(1)
msgHold = f"{indent} In hold"
logging.info(msgHold)
return msgHold
if not apiSrc.getClient():
msgLog = (f"{indent}Error. No client for {src[2]} ({src[3]})")
logMsg(msgLog, 1, 1)
return f"End: {msgLog}"
apiSrc.setPosts()
# print(f"apiSrc: {apiSrc}")
# print(f"action: {action}")
# print(f"more {more}")
apiDst = self.readConfigDst(indent, action, more, apiSrc)
if not apiDst.getClient():
msgLog = (f"{indent}Error. No client for {action[2]}")
logMsg(msgLog, 1, 1)
return f"End: {msgLog}"
apiDst.setUrl(apiSrc.getUrl())
# print(f"{indent}action: {action}")
# print(f"-->{indent}apiDst: {apiDst.getPostsType()} {action[1]}")
# print(f"-->{indent}apiDst: {apiDst.getPostsType()[:-1]} {action[1]}")
# print(f"-->{indent}apiDst: {apiDst.getPostsType()} {action[1]}")
if ((apiDst.getPostsType() != action[1])
and (apiDst.getPostsType()[:-1] != action[1])
and (action[0] != 'cache')):
# FIXME: Can we do better?
return f"{indent}{action}"
apiDst.setPosts()
# print(f"Posttttts: {apiDst.getPosts()}")
#FIXME: Is it always needed? Only in caches?
indent = f"{indent} "
apiSrc.setLastLink(apiDst)
lastLink = apiSrc.getLastLinkPublished()
lastTime = apiSrc.getLastTimePublished()
time.sleep(1)
if lastLink:
msgLog = (f"Last link: {lastLink} ")
logMsg(f"{indent}{msgLog}", 1, 1)
msgLog = ''
if lastTime:
myTime = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(lastTime))
msgLog = (f"{msgLog}Last time: {myTime}")
if nextPost:
num = apiDst.getMax()
else:
num = 1
msgLog = (f"{indent}{msgLog} Num: {num}")
logMsg(msgLog, 1, 1)
listPosts = []
link = ''
if (num > 0):
tNow = time.time()
hours = float(apiDst.getTime())*60*60
if lastTime:
diffTime = tNow - lastTime
else:
# If there is no lasTime, we will publish
diffTime = hours + 1
msgLog = (f"{indent}Src time: {apiSrc.getTime()} "
f"Dst time: {apiDst.getTime()}")
logMsg(msgLog, 2, 0)
numAvailable = 0
if (noWait or (diffTime>hours)):
tSleep = random.random()*float(timeSlots)*60
if nextPost:
post = apiSrc.getNextPost()
else:
post = apiSrc.getPost(pos)
if post:
apiSrc.setNextTime(tNow, tSleep, apiDst)
else:
apiSrc.setNextAvailableTime(tNow, tSleep, apiDst)
if (tSleep>0.0):
msgLog= f"{indent}Waiting {tSleep/60:2.2f} minutes"
else:
tSleep = 2.0
msgLog= f"{indent}No Waiting"
msgLog = f"{msgLog} for action: {msgAction}"
logMsg(msgLog, 1, 1)
for i in range(num):
time.sleep(tSleep)
if nextPost:
self.executePublishAction(indent, msgAction, apiSrc,
apiDst, simmulate)
else:
self.executePublishAction(indent, msgAction, apiSrc,
apiDst, simmulate,
nextPost, pos)
elif (diffTime<=hours):
msgLog = (f"Not enough time passed. "
f"We will wait at least "
f"{(hours-diffTime)/(60*60):2.2f} hours.")
# logMsg(msgLog, 1, 1)
textEnd = f"{textEnd} {msgLog}"
else:
if (num<=0):
msgLog = (f"{indent}No posts available")
logMsg(msgLog, 1, 1)
return textEnd
def executePublishAction(self, indent, msgAction, apiSrc, apiDst,
simmulate, nextPost=True, pos=-1):
res = ''
msgLog = (f"{indent}Go! Action: {msgAction}")
logMsg(msgLog, 1, 1)
# The source of data can have changes while we were waiting
apiSrc.setPosts()
if nextPost:
# FIXME is this needed?
post = apiSrc.getNextPost()
else:
post = apiSrc.getPost(pos)
if post:
msgLog = f"{indent}Would schedule in {msgAction} ..."
logMsg(msgLog, 1, 1)
indent = f"{indent} "
logMsg(msgLog, 2, 0)
logMsg(f"{indent}- {apiSrc.getPostTitle(post)}", 1, 1)
else:
msgLog = f"{indent}No post to schedule in {msgAction}"
logMsg(msgLog, 1, 1)
indent = f"{indent[:-1]}"
resMsg = ''
postaction = ''
if not simmulate:
if nextPost:
res = apiDst.publishNextPost(apiSrc)
else:
res = apiDst.publishPosPost(apiSrc, pos)
resMsg = f"Publish: {res}. "
# print(f"{indent}res: {res}")
if (nextPost and
((not res) or ('SAVELINK' in res) or not ('Fail!' in res))):
resUpdate = apiSrc.updateLastLink(apiDst, '')
resMsg += f"Update: {resUpdate}"
if ((not res) or ('SAVELINK' in res) or not ('Fail!' in res)):
postaction = apiSrc.getPostAction()
if postaction:
msgLog = (f"{indent}Post Action {postaction}")
logMsg(msgLog, 1, 1)
if nextPost:
cmdPost = getattr(apiSrc, f"{postaction}NextPost")
resPost = cmdPost()
else:
cmdPost = getattr(apiSrc, f"{postaction}")
resPost = cmdPost(pos)
# FIXME inconsistent
msgLog = (f"{indent}Post Action command {cmdPost}")
logMsg(msgLog, 1, 1)
msgLog = (f"{indent}End {postaction}, reply: {resPost} ")
logMsg(msgLog, 1, 1)
resMsg += f"Post Action: {resPost}"
else:
msgLog = (f"{indent}No Post Action")
logMsg(msgLog, 1, 1)
msgLog = (f"{indent}End publish, reply: {resMsg}")
logMsg(msgLog, 1, 1)
else:
msgLog = (f"{indent}This is a simmulation")
logMsg(msgLog, 1, 1)
resMsg = f"Simulate: {msgLog}\n"
post = apiSrc.getNextPost()
if post:
link = apiSrc.getPostLink(post)
if link:
msgLog = (f"{indent}I'd record link: {link}")
logMsg(msgLog, 1, 1)
resMsg += f"{msgLog}\n"
# fN = fileNamePath(apiDst.getUrl(), socialNetwork)
# msgLog = (f"{indent}in file ", f"{fN}.last")
# logMsg(msgLog, 1, 1)
msgLog = (f"{indent}in file "
f"{apiSrc.fileNameBase(apiDst)}.last")
logMsg(msgLog, 1, 1)
resMsg += f"{msgLog}\n"
if postaction == 'delete':
msgLog = (f"{indent}Available {len(apiSrc.getPosts())-1}")
else:
msgLog = (f"{indent}Available {len(apiSrc.getPosts())}")
logMsg(msgLog, 1, 1)
return resMsg
