def DogQueueConsumer(): logger.info('DogQueue consumer begin') q = DogQueue() try: rs = q.getAll() for r in rs: if not q.isConsuming(r): UploadFile.apply_async(args=[r]) logger.info('DogQueue consuming {filepath}'.format(filepath=r)) except: logger.exception('DogQueueConsumer') else: logger.info('DogQueue consumer end')
def UploadFile(filepath): # import shutil from tools.ftptools import ( FTPInfo, StandardFTPFactory ) q = DogQueue() q.doing(filepath) pendingFile(filepath) f = FileRoute(filepath) try: logger.info('Send file to route') transtype, ftpname, dest_path, filename = f.routeInfo() logger.info('Copy file {filepath} Begin'.format(filepath=filepath)) info = FTPInfo().get_info(ftpname) host = info.get('host', None) port = info.get('port', 22) username = info.get('username', None) password = info.get('password', None) key = info.get('key', None) f = StandardFTPFactory.get_factory(transtype) c = f.get_client(host, port, username, password, key) local = filepath if dest_path is None: dest_path = '' remote = os.path.join(dest_path, filename) temp = os.path.join(dest_path, filename + '.transfering') c.put(local, temp) c.rename(temp, remote) c.close() os.remove(filepath) # shutil.copyfile(filepath, os.path.join(dest_path, filename)) q.done(filepath) except TypeError: q.done(filepath) logger.info('{filepath} is not our file'.format(filepath=filepath)) return False except: logger.exception('UploadFile') return False else: logger.info('Copy file {filepath} Success'.format(filepath=filepath)) return True