def callback(ch, method, properties, body):
global callback_flag
global file_logger
global logger
global scanner
global trans
global vclient
global mqclient
global id
stats = "NOT OK"
trx_id = misc.generate_id(size=15)
procid = id + " " + trx_id
file_logger.info(procid + " Start Migration: " + repr(body))
meta = json.loads(body)
key = meta["path"]
info = FileInfo(meta["base"], key)
file_logger.info("FileInfo " + repr(info))
fullpath = info.get_absolute_path()
key_exist = False
try:
key_exist = trans.key_exist(key=key)
except mogilefs.MogileFSError:
pass
if key_exist:
file_logger.warning(procid + " Key exist: " + key)
elif not os.path.isfile(fullpath):
file_logger.warning(procid + " File does not exist: " + fullpath)
else:
file_logger.info("Scanning file: " + fullpath)
scan_result = scanner.scan_file(fullpath)
file_logger.info(procid + " Scanned file {0}: {1}".format(fullpath,scan_result))
if scan_result:
trans_result = trans.send_file(source=fullpath, key=key, clas=migconfig.clas)
message = procid + " MogileFS key {0}: {1}".format(key, trans_result)
file_logger.info(message)
if trans_result == True:
coll = info.to_collection()
logger.file_saved(coll)
file_logger.info(procid + " Saved metadata: " + repr(coll))
vclient.send(coll["_id"])
file_logger.info(procid + " Validation task: " + coll["_id"])
stats = "OK"
elif trans_result == None:
file_logger.warning(procid + " Not saved because key exist: " + key)
else:
message = procid + " Error sent file to MogileFS: " + info.to_string()
file_logger.error(message)
mqclient.send(body)
else:
coll = info.to_collection()
coll['status'] = 'infected'
logger.file_saved(coll)
file_logger.error(procid + " Infected file: " + repr(coll))
ch.basic_ack(delivery_tag = method.delivery_tag)
file_logger.info(procid + " End migration %s: %s " %(repr(body), stats))
def callback(ch, method, properties, body):
stats = "OK"
global id
procid = id + "-" + misc.generate_id(size=15)
file_logger.info(procid + " Start validation: " + repr(body))
dbmeta = logger.file_get(body)
file_logger.info(procid + " Database meta data: " + repr(dbmeta))
if dbmeta:
base = "/tmp"
name = os.path.basename(dbmeta["path"])
fullpath = os.path.join(base, name)
try:
if os.path.isfile(fullpath):
os.remove(fullpath)
except OSError:
suffix = misc.generate_id()
fullpath = fullpath + "-" + suffix
trans.download_file(key=dbmeta["path"], name=fullpath)
mogmeta = FileInfo("/tmp", name)
file_logger.info(procid + " MogileFS meta data: " + repr(mogmeta.to_collection()))
if mogmeta.equal_meta(dbmeta):
logger.file_validated(dbmeta)
else:
logger.file_corrupted(dbmeta)
migration_job = {"path": dbmeta["path"], "base": dbmeta["base"]}
mqclient.send(migration_job)
msg = procid + " File corrupted. Sent new job to migration queue: " + json.dumps(dbmeta)
logger.warning(msg)
stats = "CORRUPTED"
os.remove(fullpath)
else:
if not trans.key_exist(body):
file_logger.error("Missing key: " + body)
ch.basic_ack(delivery_tag = method.delivery_tag)
file_logger.info("%s End validation %s: %s" %(procid, repr(body), stats))
