def callback(ch, method, properties, body): global callback_flag global file_logger global logger global scanner global trans global vclient global mqclient global id stats = "NOT OK" trx_id = misc.generate_id(size=15) procid = id + " " + trx_id file_logger.info(procid + " Start Migration: " + repr(body)) meta = json.loads(body) key = meta["path"] info = FileInfo(meta["base"], key) file_logger.info("FileInfo " + repr(info)) fullpath = info.get_absolute_path() key_exist = False try: key_exist = trans.key_exist(key=key) except mogilefs.MogileFSError: pass if key_exist: file_logger.warning(procid + " Key exist: " + key) elif not os.path.isfile(fullpath): file_logger.warning(procid + " File does not exist: " + fullpath) else: file_logger.info("Scanning file: " + fullpath) scan_result = scanner.scan_file(fullpath) file_logger.info(procid + " Scanned file {0}: {1}".format(fullpath,scan_result)) if scan_result: trans_result = trans.send_file(source=fullpath, key=key, clas=migconfig.clas) message = procid + " MogileFS key {0}: {1}".format(key, trans_result) file_logger.info(message) if trans_result == True: coll = info.to_collection() logger.file_saved(coll) file_logger.info(procid + " Saved metadata: " + repr(coll)) vclient.send(coll["_id"]) file_logger.info(procid + " Validation task: " + coll["_id"]) stats = "OK" elif trans_result == None: file_logger.warning(procid + " Not saved because key exist: " + key) else: message = procid + " Error sent file to MogileFS: " + info.to_string() file_logger.error(message) mqclient.send(body) else: coll = info.to_collection() coll['status'] = 'infected' logger.file_saved(coll) file_logger.error(procid + " Infected file: " + repr(coll)) ch.basic_ack(delivery_tag = method.delivery_tag) file_logger.info(procid + " End migration %s: %s " %(repr(body), stats))
def callback(ch, method, properties, body): stats = "OK" global id procid = id + "-" + misc.generate_id(size=15) file_logger.info(procid + " Start validation: " + repr(body)) dbmeta = logger.file_get(body) file_logger.info(procid + " Database meta data: " + repr(dbmeta)) if dbmeta: base = "/tmp" name = os.path.basename(dbmeta["path"]) fullpath = os.path.join(base, name) try: if os.path.isfile(fullpath): os.remove(fullpath) except OSError: suffix = misc.generate_id() fullpath = fullpath + "-" + suffix trans.download_file(key=dbmeta["path"], name=fullpath) mogmeta = FileInfo("/tmp", name) file_logger.info(procid + " MogileFS meta data: " + repr(mogmeta.to_collection())) if mogmeta.equal_meta(dbmeta): logger.file_validated(dbmeta) else: logger.file_corrupted(dbmeta) migration_job = {"path": dbmeta["path"], "base": dbmeta["base"]} mqclient.send(migration_job) msg = procid + " File corrupted. Sent new job to migration queue: " + json.dumps(dbmeta) logger.warning(msg) stats = "CORRUPTED" os.remove(fullpath) else: if not trans.key_exist(body): file_logger.error("Missing key: " + body) ch.basic_ack(delivery_tag = method.delivery_tag) file_logger.info("%s End validation %s: %s" %(procid, repr(body), stats))