def handleRegisterCopy(self, msg):
debug("handleRegisterCopy: " + str(msg))
assert (msg.type == Message.REGISTER_COPY)
retval = False
fileinfo = msg.content[0:3]
fileserver = msg.content[3]
copyToken = self.copycount.startCopyFromServer(fileserver,
self.clientName,
fileinfo[0])
debug("copyToken: %d" % copyToken)
if copyToken == 0:
if not self.conn.sendMessage(
Message(Message.WAIT, [str(self.config.CLIENT_WAIT)])):
debug("client died")
else:
retval = True
else:
if not self.conn.sendMessage(Message(Message.FILE_OK)):
debug("client died")
self.copycount.endCopy(fileserver, self.clientName, copyToken)
else:
msg = self.conn.receiveMessage()
debug("recv: " + str(msg))
self.copycount.endCopy(fileserver, self.clientName, copyToken)
if msg == None:
debug("client died")
elif msg.type == Message.COPY_OK:
newloc = Location(fileinfo[0], fileinfo[1], fileinfo[2],
self.clientName)
self.db.addLocation(msg.content[0], newloc)
else:
debug("copy failed")
debug("retval: " + str(retval))
return retval
def handleIsActive(self, msg):
debug("handleIsActive: " + str(msg))
assert (msg.type == Message.IS_ACTIVE)
dest = msg.content[0]
if self.copycount.isActiveTransfer(self.clientName, dest):
reply = Message(Message.WAIT, [str(self.config.CLIENT_WAIT)])
else:
reply = Message(Message.FILE_OK)
if not self.conn.sendMessage(reply):
debug("client died")
def _copySingleFile(self, source, destination, register):
try:
shutil.copy2(source, destination)
log("copied %s to %s" % (source, destination))
if register:
reply = Message(Message.COPY_OK, [ destination ])
else:
debug("don't send location")
reply = Message(Message.COPY_FAILED)
retval = 0
except Exception, e:
error("cannot copy %s to %s: %s" % (source, destination, str(e)))
reply = Message(Message.COPY_FAILED)
retval = 1
def send_pm(recipient, subject, body, bypass_opt_out=False):
opt_in = True
# If there is not a bypass to opt in, check the status
if not bypass_opt_out:
try:
acct = Account.select(
Account.opt_in).where(Account.username == recipient).get()
opt_in = acct.opt_in
except Account.DoesNotExist:
pass
# if the user has opted in, or if there is an override to send the PM even if they have not
if opt_in or not bypass_opt_out:
msg = Message(username=recipient, subject=subject, message=body)
msg.save()
def copyFromOrigin(self, requestedFile):
debug("copy from origin -> %s:%s" %
(self.clientName, requestedFile[4]))
cnt = 0
fileserver = requestedFile[3]
if fileserver == "": fileserver = "unknown"
copyToken = self.copycount.startCopyFromServer(fileserver,
self.clientName,
requestedFile[4])
if copyToken == 0:
self.stat.inc("wait")
return (True, True)
debug("start copy")
self.conn.sendMessage(Message(Message.COPY_FROM_SERVER))
msg = self.conn.receiveMessage()
while msg and msg.type == Message.PING:
msg = self.conn.receiveMessage()
debug("recv ping: " + str(msg))
debug("end copy: " + str(msg))
if msg == None:
self.stat.inc("aborted")
r = (True, False)
elif msg.type == Message.COPY_OK:
newloc = Location(msg.content[0], requestedFile[1], requestedFile[2], \
self.clientName)
self.db.addLocation(requestedFile[0], newloc)
self.stat.inc("copyFromServer")
r = (True, False)
else:
r = (False, False)
# release lock on local copy _after_ adding the new location to the DB
# such that parallel running clients on the same node use the existing
# copy from now on
self.copycount.endCopy(fileserver, self.clientName, copyToken)
return r
class CmClient:
""" CacheManager Client"""
def __init__(self, config, single=True):
"""initialize client.
set single=False if you need to fetch several files.
"""
self.config = config
self.single = single
self.connection = self._connectToServer(config)
def __del__(self):
if not self.single:
try:
self.connection.sendMessage(Message(Message.EXIT, []))
except Exception:
pass
def _connectToServer(self, config):
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server = settings.clientEnvironment().server(config)
if not server[0] in range(10):
server = socket.gethostbyname(server)
s.connect((server, config.MASTER_PORT))
s.settimeout(config.SOCKET_TIMEOUT)
except Exception, e:
error(str(e))
return None
conn = Connection(s)
if not self.single:
r = conn.sendMessage(Message(Message.KEEP_ALIVE, []))
return conn
def run(self):
while not self.finished.isSet():
try:
self.finished.wait(self.interval)
if not self.finished.isSet():
self.conn.sendMessage(Message(Message.PING, []))
except Exception, e:
warning("exception in PingThread: %s" % str(e))
break
def handleGetLocations(self, msg):
debug("handleGetLocations: " + str(msg))
assert (msg.type == Message.GET_LOCATIONS)
requestedFile = msg.content
if not self.db.hasFile(requestedFile[0]):
debug("file not found in db: %s" % requestedFile[0])
self.conn.sendMessage(Message(Message.EXIT, []))
else:
locations = self.db.getAllLocations(requestedFile[0])
debug("%d locations found" % len(locations))
for loc in locations:
if loc.host == self.clientName:
found = self.checkLocal(loc, requestedFile)
else:
found, abort = self.checkRemote(loc, requestedFile)
if abort:
debug("client died")
return
self.conn.sendMessage(Message(Message.EXIT, []))
def handleFileRequest(self, msg):
debug("handleFileRequest: " + str(msg))
assert (msg.type == Message.REQUEST_FILE)
requestedFile = msg.content
localDestination = requestedFile[4]
found = False
wait = False
forceWait = False
while True:
loc = self.findLocation(requestedFile)
debug("loc: " + str(loc))
if self.copycount.isActiveTransfer(self.clientName,
localDestination):
debug("currently active transfer. send wait")
forceWait = wait = True
elif loc != None:
forceWait = False
if loc.host == self.clientName:
found = self.checkLocal(loc, requestedFile)
else:
found, abort = self.checkRemote(loc, requestedFile)
debug("checkRemote -> found=%s, abort=%s" % (found, abort))
if not abort and found:
found, wait = self.copyFromRemote(loc, requestedFile)
else:
forceWait = False
break
debug("found: " + str(found))
debug("wait: " + str(wait))
if found or forceWait: break
if (not found) or wait:
# if file was not found on a node or if we would have to wait for it,
# check if we can get it without waiting from the server
if not forceWait:
found, wait = self.copyFromOrigin(requestedFile)
if not found:
log("copyFromOrigin failed: " + requestedFile[0])
self.conn.sendMessage(Message(Message.FALLBACK))
if wait:
debug("send wait")
self.conn.sendMessage(
Message(Message.WAIT, [str(self.config.CLIENT_WAIT)]))
return wait
def handle_comment(message):
response = send_from_comment(message)
response_text = text.make_response_text(message, response)
# check if subreddit is untracked or silent. If so, PM the users.
if response["subreddit_status"] in ["silent", "untracked", "hostile"]:
message_recipient = str(message.author)
if response["status"] < 100:
subject = text.SUBJECTS["success"]
else:
subject = text.SUBJECTS["failure"]
message_text = response_text + text.COMMENT_FOOTER
msg = Message(username=message_recipient,
subject=subject,
message=message_text)
msg.save()
else:
message.reply(response_text + text.COMMENT_FOOTER)
def isActive(self, destination):
""" return: waiting time """
debug("isActive: %s" % destination)
r = self.conn.sendMessage(Message(Message.IS_ACTIVE, [destination]))
msg = self.conn.receiveMessage()
if not msg:
error("connection reset")
return 0
elif msg.type == Message.WAIT:
return int(msg.content[0])
else:
return 0
def _copyFile(self, source, destination, register, isBundle):
debug("copyFile: %s, %s" % (source, destination))
if not os.path.isfile(source):
error("file not found '%s'" % source)
return 1
debug("isBundle: %d" % isBundle)
if not isBundle:
if os.path.isdir(destination):
destination = os.path.normpath(destination + "/" + os.path.basename(source))
debug("destination: " + destination)
if os.path.isfile(destination):
log("warning: will overwrite %s" % destination)
else:
if not os.path.isdir(destination):
error("destination is not a directory")
return 1
fs = FileSystem(self.config)
fileinfo = fs.getFileInfo(source)
debug("fileinfo: " + str(fileinfo))
fileserver = fs.getFileServer(destination)
debug("fileserver: " + fileserver)
request = Message(Message.REGISTER_COPY, fileinfo + [fileserver])
if not self.connection.sendMessage(request):
error("cannot send message")
return 1
retval = 0
while True:
msg = self.connection.receiveMessage()
if msg == None:
error("cannot receive message")
retval = 1
break
elif msg.type == Message.WAIT:
time.sleep(int(msg.content[0]))
if not self.connection.sendMessage(request):
error("cannot send message")
retval = 1
break
elif msg.type == Message.FILE_OK:
if not isBundle:
retval, reply = self._copySingleFile(source, destination, register)
else:
retval, reply = self._copyBundleFile(source, destination)
if not self.connection.sendMessage(reply):
error("cannot send message")
retval = 1
break
else:
error("unexpected message: %s" % str(msg))
retval = 1
break
return retval
def checkLocal(self, loc, requestedFile):
debug("checkLocal")
self.conn.sendMessage(Message(Message.CHECK_LOCAL, [loc.path]))
debug("send")
msg = self.conn.receiveMessage()
debug("recv: " + str(msg))
if msg == None:
return True # client died, don't care
if msg.type != Message.FILE_OK:
debug("local file invalid. remove " + loc.path)
self.db.removeLocation(requestedFile[0], loc)
return False
return True
def checkRemote(self, loc, requestedFile):
""" return (found, abort) """
debug("check remote")
self.conn.sendMessage(
Message(Message.CHECK_REMOTE, [loc.host, loc.path]))
debug("send")
msg = self.conn.receiveMessage()
debug("recv: " + str(msg))
if msg == None:
return (True, True)
if msg.type != Message.FILE_OK:
debug("remote file invalid. remove " + loc.path)
self.db.removeLocation(requestedFile[0], loc)
return (False, False)
else:
return (True, False)
def copyFromRemote(self, loc, requestedFile):
""" return (copyOk, wait) """
debug("copy from remote -> %s:%s" %
(self.clientName, requestedFile[4]))
cnt = 0
copyToken = self.copycount.startCopyFromNode(loc.host, self.clientName,
requestedFile[4])
if copyToken == 0:
self.stat.inc("wait")
return (True, True)
debug("start copy")
self.conn.sendMessage(
Message(Message.COPY_FROM_NODE, [loc.host, loc.path]))
msg = self.conn.receiveMessage()
debug("recv: " + str(msg))
while msg and msg.type == Message.PING:
msg = self.conn.receiveMessage()
debug("recv ping: " + str(msg))
debug("end copy: " + str(msg))
if msg == None:
self.stat.inc("aborted")
r = (True, False)
elif msg.type == Message.COPY_OK:
debug("COPY OK")
newloc = copy.copy(loc)
newloc.host = self.clientName
newloc.path = msg.content[0]
self.db.addLocation(requestedFile[0], newloc)
self.stat.inc("copyFromNode")
r = (True, False)
else:
debug("copy failed")
#TODO: don't remove location if there is just not enogh disk space!
#DONE: disk space is checked before sending the request
self.db.removeLocation(requestedFile[0], loc)
r = (False, False)
self.copycount.endCopy(loc.host, self.clientName, copyToken)
return r
def sendFileLocation(self, fileinfo, destination):
debug("sendFileLocation: %s, %s" % (str(fileinfo), destination))
r = self.conn.sendMessage(Message(Message.HAVE_FILE, fileinfo + [destination]))
debug(" => " + str(r))
retval = 0
cnt = 0
fail = 0
for line in file(source):
fileItem = line.strip()
if fileItem == "":
continue
fileItem = os.path.realpath(fileItem)
destFile = os.path.normpath(destination + "/" + os.path.basename(fileItem))
debug("destFile: " + destFile)
if os.path.isfile(destFile):
warning("will overwrite %s" % destFile)
try:
shutil.copy2(fileItem, destFile)
log("copied %s to %s" % (fileItem, destFile))
cnt += 1
except Exception, e:
error("cannot copy %s to %s: %s" % (fileItem, destFile, str(e)))
fail += 1
retval = 1
log("copied %d files, %d errors" % (cnt, fail))
# force a copy failed such that the server does not register
# the bundle file location as cache copy.
warning("local copies not registered in database")
return (retval, Message(Message.COPY_FAILED))
def _isBundleFile(self, filename):
return filename.endswith(".bundle") and not self.config.IGNORE_BUNDLE
def handleMessage(self, fileinfo, destination, msg):
""" return (retFile, retval, terminate) """
debug("handleMessage %s" % str(msg))
debug("destination = %s" % destination)
retval = False
retFile = None
reply = None
terminate = False
if msg == None:
error("no connection to master")
retFile = fileinfo[0]
retval = True
elif msg.type == Message.CHECK_LOCAL:
if self.checkLocal(fileinfo, msg.content[0]):
reply = Message(Message.FILE_OK)
log("using existing copy %s" % msg.content[0])
retFile = msg.content[0]
retval = True
else:
reply = Message(Message.FILE_NOT_OK)
elif msg.type == Message.CHECK_REMOTE:
if self.checkRemote(fileinfo, msg.content[0], msg.content[1]):
self.brandFile(msg.content[0], msg.content[1])
reply = Message(Message.FILE_OK)
retFile = (msg.content[0], msg.content[1])
else:
reply = Message(Message.FILE_NOT_OK)
elif msg.type == Message.COPY_FROM_NODE:
if self.copyFromNode(fileinfo, msg.content[0], msg.content[1], destination):
reply = Message(Message.COPY_OK, [destination])
debug("OK")
retFile = destination
retval = True
else:
reply = Message(Message.COPY_FAILED)
elif msg.type == Message.COPY_FROM_SERVER:
if self.copyFromServer(fileinfo, destination):
reply = Message(Message.COPY_OK, [destination])
debug("OK")
retFile = destination
retval = True
else:
reply = Message(Message.COPY_FAILED, [])
elif msg.type == Message.FALLBACK:
log("no local cache available")
retFile = fileinfo[0]
retval = True
elif msg.type == Message.WAIT:
log("no copy slot available. waiting.")
time.sleep(int(msg.content[0]))
self.requestFile(fileinfo, destination)
elif msg.type == Message.EXIT:
debug("exit received")
terminate = True
retval = True
else:
error("unknown message received: %d" % msg.type)
retFile = fileinfo[0]
retval = True
debug("reply: " + str(reply))
debug("retval: " + str(retval))
debug("retFile: " + str(retFile))
if reply != None:
self.conn.sendMessage(reply)
return (retFile, retval, terminate)
def sendKeepAlive(self):
debug("sendKeepAlive")
r = self.conn.sendMessage(Message(Message.KEEP_ALIVE, []))
debug(" => " + str(r))
def requestFile(self, fileinfo, destination):
debug("requestFile: " + str(fileinfo))
fileserver = self.fileSystem.getFileServer(fileinfo[0])
debug("file server: " + fileserver)
return self.conn.sendMessage(Message(Message.REQUEST_FILE, fileinfo + [fileserver, destination]))
def requestFileLocations(self, fileinfo):
debug("requestFileLocations: " + str(fileinfo))
return self.conn.sendMessage(Message(Message.GET_LOCATIONS, fileinfo))
def __del__(self): if not self.single: try: self.connection.sendMessage(Message(Message.EXIT, [])) except Exception: pass
def sendFileRemoved(self, fileinfo, destination):
debug("sendFileRemoved: %s, %s" % (str(fileinfo), destination))
r = self.conn.sendMessage(Message(Message.DELETED_COPY, fileinfo + [destination]))
debug(" => " + str(r))
from time import sleep
from shared import REDDIT, LOGGER, Message
LOGGER.info("Starting messenger")
while True:
results = Message.select()
for result in results:
LOGGER.info(
"%s %s %s" %
(result.username, result.subject, repr(result.message)[:50]))
try:
REDDIT.redditor(str(result.username)).message(
str(result.subject), str(result.message))
except:
pass
Message.delete().where(Message.id == result.id).execute()
sleep(6)
def sendExit(self):
debug("sendExit")
r = self.conn.sendMessage(Message(Message.EXIT, []))
debug(" => " + str(r))