def run(self):
"""
call this module with argument "<fmfid>"
"""
# get files hash from DB
fmfid=int(self.jobdata['mfid'])
result = upqdb.UpqDB().query("SELECT f.md5, mf.fid, m.url_prefix, m.url_daemon, m.url_daemon_password, mf.path \
FROM mirror_file as mf \
LEFT JOIN file as f on f.fid = mf.fid \
LEFT JOIN mirror as m on mf.mid = m.mid \
WHERE mf.mfid = %d" % fmfid)
res = result.first()
if not res:
self.msg("File with jobdata='%s' not found in DB."%(self.jobdata))
return False
script_url = res['url_prefix']+"/"+res['url_daemon']
params = urllib.urlencode({'p': res['path'], 'pw': res['url_daemon_password']})
hash_url = "%s?%s" % (script_url, params)
file_path = res['path']
md5 = res['md5']
# retrieve md5 hash from deamon.php on remote mirror server
self.logger.debug("retrieving '%s'", hash_url)
try:
hash_file = urllib.urlopen(hash_url)
hash = hash_file.read()
hash_file.close()
except Exception as e:
self.logger.error(str(e))
return False
self.msg("received md5 hash for filename=%s on fmfid=%s is %s" %(file_path, fmfid, hash))
self.result = hash
if md5 == hash:
#TODO add notify job here
upqdb.UpqDB().query("UPDATE mirror_file SET lastcheck=NOW(), status=1 WHERE mfid=%d" %(fmfid))
self.msg('Remote hash matches hash in DB.')
return True
else:
query="DELETE FROM mirror_file WHERE mfid=%s " %(fmfid)
upqdb.UpqDB().query(query)
self.msg('Remote hash does NOT match hash in DB for mfid %s = %s != %s.' %(fmfid, hash, md5))
return False
def program_cleanup(sig_num, frame):
logger = log.getLogger("upq")
logger.info("Shutting down socket server...")
server.shutdown()
logger.info("Disconnecting from DB...")
upqdb.UpqDB().cleanup()
log.getLogger("upq").info("Good bye.")
sys.exit(0)
def revive_jobs(self):
"""
Fetches all jobs from DB that are in state "new" or "running", and
recreates the objects from its pickled representation.
returns : list of alive, unqueued jobs
"""
results = upqdb.UpqDB().query(
"SELECT * FROM upqueue WHERE status = 1 OR status = 2")
jobs = []
for res in results:
job = res['jobname']
modclass = module_loader.load_module(job)
obj = modclass(job, json.loads(res['jobdata']))
obj.jobid = res['jobid']
obj.thread = "Thread-revived-UpqJob"
jobs.append(obj)
logger.debug("revived jobs='%s'", jobs)
return jobs
from upqdb import UpqDB
import upqconfig
import sys
from xmlrpc.client import ServerProxy
import json
import os
import upqjob
import upqdb
class Sf_sync(upqjob.UpqJob):
def versionfetch(self):
from jobs import versionfetch
j = versionfetch.Versionfetch("versionfetch", {})
j.run()
return j.run()
def run(self):
self.versionfetch()
upqconfig.UpqConfig()
upqconfig.UpqConfig().readConfig()
db = upqdb.UpqDB()
db.connect(upqconfig.UpqConfig().db['url'], upqconfig.UpqConfig().db['debug'])
s = Sf_sync("sf_sync", dict())
s.run()
def main(argv=None):
if argv is None:
argv = sys.argv
server = None
# SIGINT signal handler
def program_cleanup(sig_num, frame):
logger = log.getLogger("upq")
logger.info("Shutting down socket server...")
server.shutdown()
logger.info("Disconnecting from DB...")
upqdb.UpqDB().cleanup()
log.getLogger("upq").info("Good bye.")
sys.exit(0)
usage = "usage: %prog -c CONFIGFILE [options]"
parser = OptionParser(usage)
parser.add_option("-c",
"--config",
dest="configfile",
default="",
help="path to config file CONFIGFILE")
#TODO: use this to en/disable daemonization
# parser.add_option("-d", "--daemonize",
# help="detach from terminal etc")
parser.add_option("-l",
"--logfile",
dest="logfile",
default="",
help="path to logfile LOGFILE")
(options, argv) = parser.parse_args()
try:
# read ini file
UpqConfig(options.configfile, options.logfile)
UpqConfig().readConfig()
#FIXME: remove following line + how does this $%$!" work?
del UpqConfig().daemon['pidfile']
# if UpqConfig().daemon.has_key('pidfile'):
# lockfile=UpqConfig().daemon['pidfile']
# UpqConfig().daemon['pidfile']=pidlockfile.TimeoutPIDLockFile(lockfile, acquire_timeout=1)
context = daemon.DaemonContext(**UpqConfig().daemon)
# daemonize
context.stdout = sys.stderr
context.stderr = sys.stderr
upq = Upq()
with context:
# initialize logging
logger = log.init_logging(UpqConfig().logging)
logger.info("Starting logging...")
logger.debug(UpqConfig().config_log)
# setup and test DB
logger.info("Connecting to DB...")
db = upqdb.UpqDB()
db.connect(UpqConfig().db['url'], UpqConfig().db['debug'])
db.version()
# start server
logger.info("Starting socket server...")
server = upq.start_server()
# except SIGINT and SIGTERM
signal.signal(signal.SIGINT, program_cleanup)
signal.signal(signal.SIGTERM, program_cleanup)
log.getLogger("upq").info(
"Server running until receiving SIGTERM or SIGINT / Ctrl+C.")
signal.pause()
except Exception:
traceback.print_exc(file=sys.stderr)
try:
db.cleanup()
except:
pass
sys.exit(1)