def loadReceipeModel(): ingrs_vocab = pickle.load( open(os.path.join(data_dir, 'ingr_vocab.pkl'), 'rb')) vocab = pickle.load(open(os.path.join(data_dir, 'instr_vocab.pkl'), 'rb')) title = '' ingr_vocab_size = len(ingrs_vocab) instrs_vocab_size = len(vocab) output_dim = instrs_vocab_size t = time.time() import sys sys.argv = [''] del sys args = get_parser() args.maxseqlen = 15 args.ingrs_only = False model = get_model(args, ingr_vocab_size, instrs_vocab_size) # Load the trained model parameters model_path = os.path.join(data_dir, 'modelbest.ckpt') model.load_state_dict(torch.load(model_path, map_location=map_loc)) model.to(device) model.eval() model.ingrs_only = False model.recipe_only = False print('loaded model') print("Elapsed time:", time.time() - t) return model
def main(): """ Utility to perform actions on HTML Required arguments: -a | --action <action_name> : Action to perform. 2 valid options exist so far: 'retrieve-content' or 'strip-tags' -u | --url <url> : URL for the HTML content """ exit_code = 0 logs.init(logfile='html') ap = args_util.get_parser() ap.add_argument('-a', '--action', type=str, help='Action to perform. 2 valid options exist so far: ' '\'retrieve-content\' or \'strip-tags\'') ap.add_argument('-u', '--url', type=str, help='URL for the HTML content') ap.add_argument('-t', '--tag', type=str, default=None, help='Optional HTML tag to strip and return the content') ap.add_argument('--text', type=str, default=True, help='Optional text within HTML tags to strip') args = ap.parse_args() assert args.action, ('--action is required. The valid options are: ' '\'retrieve-content\' or \'strip-tags\'') assert args.url, '--url is required' try: html = retrieve_html(args.url) if html is None: print 'HTML was not retrieved' exit_code = 1 exit(exit_code) if args.action == 'retrieve-content': contents = extract_content_and_language(html, url=args.url) if contents is None: print 'Content could not be retrieved. Check log.' else: for content in contents: print content, ' = ', contents[content] elif args.action == 'strip-tags': print strip_tags(html, node_text=args.text, tag=args.tag) else: log.error('Invalid action provided: \'%s\'' % args.action) print 'Invalid action provided: \'%s\'' % args.action print 'Must be either \'retrieve-content\' or \'strip-tags\'' exit_code = 2 except Exception as e: log.exception('Exception encountered: %s' % e) exit_code = 1 exit(exit_code)
def main(): import warnings warnings.filterwarnings("ignore") #set_seeds() args = get_parser() os.environ['CUDA_VISIBLE_DEVICES'] = args.cuda_index print('CUDA', args.cuda_index) datasets = ('ENZYMES', 'DD', 'REDDIT-MULTI-12K', 'COLLAB', 'PROTEINS_full', 'REDDIT-BINARY') benchmark = datasets[0] args.dataset = benchmark now = datetime.now().strftime('%Y-%m-%d_%H:%M:%S') args.logdir = f'logs/{benchmark}/{now}' if not os.path.exists(f'{args.logdir}/checkpoints'): os.makedirs(f'{args.logdir}/checkpoints') ds = Dataset(name=benchmark, max_nodes=1000, num_folds=10) args.input_dim = ds.feat_dim args.output_dim = args.input_dim args.num_classes = ds.num_class args.num_centroids = [int(x) for x in args.num_centroids.split(',') if x.strip().isdigit()] val_accs = train(ds, args) args.mean_validation_accuracy = np.mean(val_accs) args.std_validation_accuracy = np.std(val_accs) args.best_fold = int(np.argmax(val_accs)) args.best_validation_accuracy = np.max(val_accs) args.validation_accuracies = val_accs with open(f'{args.logdir}/summary.json', 'w') as f: json.dump(args.__dict__, f, indent=2)
def load_model(): global model args = get_parser() args.maxseqlen = 15 args.ingrs_only = False model = get_model(args, ingr_vocab_size, instrs_vocab_size) model.load_state_dict(torch.load(MODEL_PATH, map_location=map_loc)) model.to(device) model.eval() model.ingrs_only = False model.recipe_only = False logger.debug("Loaded model")
def main(): """ Utility to perform actions on HTML Required arguments: -a | --action <action_name> : Action to perform. 2 valid options exist so far: 'retrieve-content' or 'strip-tags' -u | --url <url> : URL for the HTML content """ exit_code = 0 logs.init(logfile='html') ap = args_util.get_parser() ap.add_argument('-a', '--action', type=str, help='Action to perform. 2 valid options exist so far: ' '\'retrieve-content\' or \'strip-tags\'') ap.add_argument('-u', '--url', type=str, help='URL for the HTML content') ap.add_argument('-t', '--tag', type=str, default=None, help='Optional HTML tag to strip and return the content') ap.add_argument('--text', type=str, default=True, help='Optional text within HTML tags to strip') args = ap.parse_args() assert args.action, ('--action is required. The valid options are: ' '\'retrieve-content\' or \'strip-tags\'') assert args.url, '--url is required' try: html = retrieve_html(args.url) if html is None: print 'HTML was not retrieved' exit_code = 1 exit(exit_code) if args.action == 'retrieve-content': contents = extract_content_and_language(html, url=args.url) if contents is None: print 'Content could not be retrieved. Check log.' else: for content in contents: print content, ' = ', contents[content] elif args.action == 'strip-tags': print strip_tags(html, node_text=args.text, tag=args.tag) else: log.error('Invalid action provided: \'%s\'' % args.action) print 'Invalid action provided: \'%s\'' % args.action print 'Must be either \'retrieve-content\' or \'strip-tags\'' exit_code = 2 except Exception as e: log.exception('Exception encountered: %s' % e) exit_code = 1 exit(exit_code)
def test(): import args p = args.get_parser() a = p.parse_args() d = init_domain(args=a) print d.name for i in d.select("select count(*) from %s" % (d.name,)): print i d = init_domain(domain='gsr_ingest', args=a) print d.name for i in d.select("select count(*) from %s" % (d.name,)): print i
def imread_indexed(filename): """ Load image given filename.""" # Dataset configuration initialization parser = get_parser() args = parser.parse_args() if args.dataset == 'kittimots': im = Image.open(filename).convert('P') else: im = Image.open(filename) annotation = np.atleast_3d(im)[...,0] reshape = np.array(im.getpalette()).reshape(-1, 3) return annotation, reshape
def __init__(self, ): transf_list = [] transf_list.append(transforms.Resize(256)) transf_list.append(transforms.CenterCrop(224)) self.transform = transforms.Compose(transf_list) transf_list_batch = [] transf_list_batch.append(transforms.ToTensor()) transf_list_batch.append( transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))) self.to_input_transf = transforms.Compose(transf_list_batch) data_dir = "data" # code will run in gpu if available and if the flag is set to True, else it will run on cpu use_gpu = False self.device = torch.device( "cuda" if torch.cuda.is_available() and use_gpu else "cpu") map_loc = None if torch.cuda.is_available() and use_gpu else "cpu" self.ingrs_vocab = pickle.load( open(os.path.join(data_dir, "ingr_vocab.pkl"), "rb")) self.vocab = pickle.load( open(os.path.join(data_dir, "instr_vocab.pkl"), "rb")) ingr_vocab_size = len(self.ingrs_vocab) instrs_vocab_size = len(self.vocab) # output_dim = instrs_vocab_size args = get_parser() args.maxseqlen = 15 args.ingrs_only = False model = get_model(args, ingr_vocab_size, instrs_vocab_size) # Load the trained model parameters model_path = os.path.join(data_dir, "modelbest.ckpt") model.load_state_dict(torch.load(model_path, map_location=map_loc)) model.to(self.device) model.eval() model.ingrs_only = False model.recipe_only = False self.model = model print("loaded model") self.ingr_co2 = pickle.load( open(os.path.join(data_dir, "ingr_co2.pkl"), "rb")) self.ingr_alternatives = pickle.load( open(os.path.join(data_dir, "ingr_alt.pkl"), "rb"))
def g_model(): data_dir = 'M:/Final Project/code/inversecooking-master/data' ingrs_vocab = pickle.load(open(os.path.join(data_dir, 'ingr_vocab.pkl'), 'rb')) vocab = pickle.load(open(os.path.join(data_dir, 'instr_vocab.pkl'), 'rb')) ingr_vocab_size = len(ingrs_vocab) instrs_vocab_size = len(vocab) output_dim = instrs_vocab_size import sys; sys.argv=['']; del sys args = get_parser() args.maxseqlen = 15 args.ingrs_only=False model = get_model(args, ingr_vocab_size, instrs_vocab_size) # Load the trained model parameters model_path = os.path.join(data_dir, 'modelbest.ckpt') model.load_state_dict(torch.load(model_path, map_location='cpu')) model.to('cpu') model.eval() model.ingrs_only = False model.recipe_only = False return model
import torch from sklearn.manifold import TSNE from torch.autograd import Variable import PIL.ImageOps import torch.nn as nn from torch import optim import torch.nn.functional as F #import self-built function from LoadData import FaceData from Net import MLP, VAE #import network from Train import ModelTrain, AverageMeter #import trainer from Helper import plotfunc, TraverseDataset #import some helper function from args import get_parser #import args #================================================== mp = get_parser() opts = mp.parse_args() #================================================== def test(test_loader, MLPModel, AEModel): losses = AverageMeter() top1 = AverageMeter() print("=> loading checkpoint '{}'".format(opts.model_path)) checkpoint = torch.load(opts.model_path) opts.start_epoch = checkpoint['epoch'] MLPModel.load_state_dict(checkpoint['MLPstate_dict']) AEModel.load_state_dict(checkpoint['AEstate_dict']) print("=> loaded checkpoint '{}' (epoch {})".format( opts.model_path, checkpoint['epoch']))
def main(): import args import logs # preliminary stab at some useful command line functionality (only cluster is currently needed) ap = args.get_parser() ap.add_argument('-c', '--cluster', action="store_true", help='Switch to output names of cluster nodes.') ap.add_argument( '--get', metavar='REQUEST', nargs='?', type=str, help= 'Information to get from embers.conf [service|data|cluster|host|input|output]' ) ap.add_argument('--s3only', action="store_true", help="Write debug messages.") ap.add_argument('--s3onlyprod', action="store_true", help="Write debug messages.") ap.add_argument('--prefixes', action="store_true", help="Write debug messages.") ap.add_argument('--prefixpairs', action="store_true", help="Write debug messages.") group = ap.add_mutually_exclusive_group() group.add_argument('--host', metavar='HOST', nargs='?', type=str, help='Name of the host to get information about') group.add_argument( '--data', metavar='DATA', nargs='?', type=str, help='Name of the data (queue, etc.) to get information about') arg = ap.parse_args() logs.init(arg) init(arg) if arg.get: assert arg.get in ('service', 'data', 'cluster', 'host', 'inputs', 'outputs', 'services'), 'Improper get request' try: if arg.s3only or arg.s3onlyprod: if arg.s3only: prodOnly = False else: prodOnly = True plst = get_all_s3only(prodOnly) if plst: print string.join(plst) return if arg.prefixpairs: plst, qlst = get_all_prefixpairs() for i in range(len(plst)): print "%s %s" % (qlst[i], plst[i]) return if arg.prefixes: plst = get_all_prefixes() for p in plst: print p return if arg.host: assert arg.host in conf.get( 'cluster'), 'Host is not listed in embers.conf' if arg.get == 'services': print conf.get('cluster')[arg.host]['services'] if arg.get == 'data': print[ data for service in conf['cluster'][arg.host]['services'] for data in conf['services'][service]['outputs'] ] if arg.data: assert arg.data in conf.get( 'data'), 'Data is not listed in embers.conf' (host, service) = get_host_and_service(arg.data) if arg.get == 'host': print host if arg.get == 'service': print service if arg.service: assert arg.service in conf.get( 'services'), 'Service is not listed in embers.conf' if arg.get == 'host': print[ host for host in conf['cluster'] if arg.service in conf['cluster'][host]['services'] ] if arg.get == 'inputs': print conf['services'][arg.service]['inputs'] if arg.get == 'outputs': print conf['services'][arg.service]['outputs'] if arg.cluster or arg.get == 'cluster': print ' '.join(conf.get('cluster', '')) if arg.get == 'data' and not (arg.service or arg.data or arg.host): print ' '.join(conf.get('data', '')) if arg.get == 'host' and not (arg.service or arg.data or arg.host): print ' '.join(conf.get('cluster', '')) if arg.get == 'service' and not (arg.service or arg.data or arg.host): print ' '.join(conf.get('services', '')) except Exception, e: log.error("Requested information not in embers.conf, error %s", e)
import numpy as np import torch import torch.nn as nn import torch.nn.parallel import torch.optim import torch.utils.data import torchvision.transforms as transforms import torchvision.datasets as datasets import torchvision.models as models import torch.backends.cudnn as cudnn from data_loader import ImagerLoader from trijoint import im2recipe # ============================================================================= import args parser = args.get_parser() opts = parser.parse_args() data_path, snapshots, logdir = args.show_train_opts(opts) # create suffix-added path # ============================================================================= from torch.utils.tensorboard import SummaryWriter dt_now = snapshots.split('/')[-1] train_writer = SummaryWriter(os.path.join(logdir, "train_" + dt_now)) val_writer = SummaryWriter(os.path.join(logdir, "val_" + dt_now)) # ============================================================================= if not(torch.cuda.device_count()): device = torch.device(*('cpu',0)) else: torch.cuda.manual_seed(opts.seed) device = torch.device(*('cuda',0))
import numpy as np import os import time import torch from torch import nn, optim from tensorboard_logger import configure, log_value from ntm import NTM from ntm.datasets import CopyDataset, RepeatCopyDataset, AssociativeDataset, NGram, PrioritySort from args import get_parser from marnn import * from dnc import DNC from dnc.sam import SAM args = get_parser().parse_args() print("args:\n",args) configure("runs/") print('name:',args.name) # ---------------------------------------------------------------------------- # -- initialize datasets, model, criterion and optimizer # ---------------------------------------------------------------------------- ''' ''' if args.task=='copy': args.task_json = 'ntm/tasks/copy.json'
with open("./run_and_start.sh", "r") as f: command = re.sub( r'python.*?\n', '', f.read() ) # TODO is this really the best way to be compliant with the fork's parent? def roscore(): proc = subprocess.Popen(command, shell=True, executable='/bin/bash', universal_newlines=True) proc.wait() print("\n\n!!!\nROSCORE CRASHED. EXITING NOW!\n!!!\n\n", file=sys.stderr) exit(-128) thread = threading.Thread(target=roscore) thread.start() parsed = get_parser().parse_args() print(parsed) if get_parser().parse_args().test: os.environ["SOLUTION_TEST"] = "1" print("\n\n\n\n\n\n\n\nTESTING\n\n\n\n\n\n\n\n") import test else: print("\n\n\n\n\n\n\n\nTRAINING\n\n\n\n\n\n\n\n") os.environ["SOLUTION_TEST"] = "0" import train
def davis_toolbox_evaluation(output_dir, eval_split, skip_F=1, title=None): """ Use default DAVIS toolbox to evaluate the performance (J, F) Print the result on the table Save the result into a yaml file Args: output_dir: the folder includes all final annotations """ from dmm.dataloader import db_eval, Segmentation from dmm.dataloader.youtubeVOS import YoutubeVOSLoader from dmm.misc.config_youtubeVOS import phase, cfg from args import get_parser phase = phase.VAL parser = get_parser() args = parser.parse_args() db = YoutubeVOSLoader(args, split=eval_split) print('Loading video segmentations from: {}'.format(output_dir)) # Load segmentation segmentations = [ Segmentation('trainval', osp.join(output_dir, s), False) for s in db.iternames() ] # Evaluate results if skip_F: evaluation = db_eval(db, segmentations, ['J']) JF = ['J'] else: evaluation = db_eval(db, segmentations, ['J', 'F']) JF = ['J', 'F'] # Print results table = PrettyTable(['Method'] + [ p[0] + '_' + p[1] for p in itertools.product(JF, ['mean', 'recall', 'decay']) ]) table.add_row([osp.basename(output_dir)] + [ "%.3f" % np.round(evaluation['dataset'][metric][statistic], 3) for metric, statistic in itertools.product(JF, ['mean', 'recall', 'decay']) ]) print(str(table) + "\n") # Save results into yaml file with open(osp.join(output_dir, 'davis_eval_results.yaml'), 'w') as f: yaml.dump(evaluation, f) tim = '{}'.format(time.strftime('%m-%d-%H')) print(tim) headers = ['Method'] + [ p[0] + '_' + p[1] for p in itertools.product(JF, ['mean', 'recall', 'decay']) ] if title is None: title = output_dir tables = ["{}".format(title)] + [ "%.3f" % np.round(evaluation['dataset'][metric][statistic], 3) for metric, statistic in itertools.product(JF, ['mean', 'recall', 'decay']) ] print(tables) info = '\n{}\n'.format(tabulate([tables], headers, tablefmt="github")) print(info) f = open("EXPERIMENT.md", "a") f.write(info) f.close()
def main(): parser = get_parser("generation", MODEL_CLASSES, ALL_MODELS) args = parser.parse_args() if args.local_rank == -1: args.device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() else: torch.cuda.set_device(args.local_rank) args.device = torch.device("cuda", args.local_rank) torch.distributed.init_process_group(backend='nccl') args.n_gpu = 1 logging.basicConfig( format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN) logger.warning( "Process rank: %s, device: %s, n_gpu: %s, distributed training: %s", args.local_rank, args.device, args.n_gpu, bool(args.local_rank != -1)) set_seed(args) if args.local_rank not in [-1, 0]: torch.distributed.barrier() args.model_type = args.model_type.lower() model_class, tokenizer_class = MODEL_CLASSES[args.model_type] gpt2_model, tokenizer = init_gpt2_model( checkpoint_dir=args.model_name_or_path, args=args, model_class=model_class, tokenizer_class=tokenizer_class) gpt2_model.eval() if args.local_rank == 0: torch.distributed.barrier() config = gpt2_model.gpt2.config if args.length < 0 and config.max_position_embeddings > 0: args.length = config.max_position_embeddings elif 0 < config.max_position_embeddings < args.length: args.length = config.max_position_embeddings # No generation bigger than model size elif args.length < 0: args.length = MAX_LENGTH # avoid infinite loop logger.info(args) if args.local_rank not in [-1, 0]: torch.distributed.barrier( ) # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache eval_dataset = load_and_cache_examples(args, tokenizer) if args.local_rank == 0: torch.distributed.barrier() eval_sampler = SequentialSampler( eval_dataset) if args.local_rank == -1 else DistributedSampler( eval_dataset, shuffle=False) eval_dataloader = DataLoader(eval_dataset, sampler=eval_sampler, batch_size=args.per_gpu_eval_batch_size) output_log = { "true_text": [], "generated_text": [], "context": [], "recall_score": [], "context_suffix_styles": [], "original_styles": [], "metadata": [] } for batch in tqdm(eval_dataloader, desc="Evaluating", disable=args.local_rank not in [-1, 0]): sentences = batch["sentence"].to(args.device) segments = batch["segment"].to(args.device) global_dense_vectors = batch["global_dense_vectors"].to(args.device) suffix_styles = batch["suffix_style"] original_styles = batch["original_style"] metadata = batch["metadata"] # Assume init_context_size is same for all examples in minibatch init_context_size = batch["init_context_size"][0].item() out, dense_length, scores = gpt2_model.generate( gpt2_sentences=sentences, segments=segments, eos_token_id=tokenizer.eos_token_id, global_dense_vectors=global_dense_vectors, init_context_size=init_context_size) for sent_num in range(sentences.shape[0]): output_sequence = out[sent_num][init_context_size:].tolist() if tokenizer.eos_token_id in output_sequence: output_sequence = output_sequence[:output_sequence.index( tokenizer.eos_token_id)] true_text = tokenizer.decode( sentences[sent_num, init_context_size:].tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True) generated_text = tokenizer.decode( output_sequence, clean_up_tokenization_spaces=True, skip_special_tokens=True) context = tokenizer.decode( sentences[sent_num, :init_context_size].tolist(), clean_up_tokenization_spaces=True, skip_special_tokens=True) recall_score = recall(true_text, context) output_log["true_text"].append(true_text) output_log["generated_text"].append(generated_text) output_log["context"].append(context) output_log["recall_score"].append("%.4f" % recall_score) output_log["metadata"].append(metadata[sent_num]) if hasattr(eval_dataset, "reverse_label_dict"): output_log["context_suffix_styles"].append( class_number_to_str(eval_dataset, suffix_styles[sent_num])) output_log["original_styles"].append( class_number_to_str(eval_dataset, original_styles[sent_num])) else: output_log["context_suffix_styles"].append("<none>") output_log["original_styles"].append("<none>") with open( os.path.join(args.generation_output_dir, "reference_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["true_text"]) + "\n") with open( os.path.join(args.generation_output_dir, "generated_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["generated_text"]) + "\n") with open( os.path.join(args.generation_output_dir, "context_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["context"]) + "\n") with open( os.path.join(args.generation_output_dir, "recall_score_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["recall_score"]) + "\n") with open( os.path.join( args.generation_output_dir, "context_suffix_styles_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["context_suffix_styles"]) + "\n") with open( os.path.join(args.generation_output_dir, "original_styles_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["original_styles"]) + "\n") with open( os.path.join(args.generation_output_dir, "metadata_%d.txt" % max(args.local_rank, 0)), "w") as f: f.write("\n".join(output_log["metadata"]) + "\n")
from torchvision import transforms from torch.autograd import Variable import numpy as np from PIL import Image import torchvision.transforms.functional as TF import random import transformers import h5py from refer.refer import REFER from args import get_parser # Dataset configuration initialization parser = get_parser() args = parser.parse_args() class ReferDataset(data.Dataset): def __init__(self, args, input_size, image_transforms=None, target_transforms=None, split='train', eval_mode=False): self.classes = [] self.input_size = input_size self.image_transforms = image_transforms
status = proc.wait() if status != 0: print(proc.stdout.read().decode("ascii").strip().split("\n")[-1]) def login(username): password = getpass.getpass() url = urls.auth() args = {"response_type": "token", "client_id": clientid(), "redirect_uri": "http://localhost", "scope": "user_read"} response = requests.get(url, params=args) print(response.status_code) print(response.headers) print(response.text) parser = args.get_parser() args = parser.parse_args() print(args) if args.top: printtopgames(args.top) elif args.game: printgamestreams(args.game) elif args.stream: openlivestreamer(args.stream) elif args.follows: print(args.follows) elif args.user: login(args.user) else: parser.print_usage()
#-*- coding:UTF-8 -*- import torch import torch.nn as nn import torch.optim as optim import os import shutil import time import numpy as np from torch.autograd import Variable from model import imvstxt from args import get_parser #===================================================== parse = get_parser() opts = parse.parse_args() #===================================================== def adjust_learning_rate(optimizer, epoch, opts): """Switching between modalities""" # parameters corresponding to the rest of the network optimizer.param_groups[0]['lr'] = opts.lr * opts.freeRecipe # parameters corresponding to visionMLP optimizer.param_groups[1]['lr'] = opts.lr * opts.freeVision print('Initial base params lr: %f' % optimizer.param_groups[0]['lr']) print('Initial vision lr: %f' % optimizer.param_groups[1]['lr']) # after first modality change we set patience to 3
def main(dir_file, image_folder, demo_path, lights): use_gpu = True device = torch.device( 'cuda' if torch.cuda.is_available() and use_gpu else 'cpu') map_loc = None if torch.cuda.is_available() and use_gpu else 'cpu' ingrs_vocab = pickle.load( open(os.path.join(dir_file, 'recipe1m_vocab_unit.pkl'), 'rb')) ingr_vocab_size = len(ingrs_vocab) t = time.time() import sys sys.argv = [''] del sys args = get_parser() args.maxseqlen = 15 args.ingrs_only = False model = get_model(args, ingr_vocab_size) # Load the trained model parameters model_dir = '/home/r8v10/git/InvCo/dataset/new_model/inversecooking/model/checkpoints' # model_dir = F'{ROOT_DIR}/dataset/model/inversecooking/model/checkpoints' model_path = os.path.join(model_dir, 'modelbest.ckpt') model.load_state_dict(torch.load(model_path, map_location=map_loc)) model.to(device) model.eval() model.ingrs_only = False model.recipe_only = False print('loaded model') print("Elapsed time:", time.time() - t) transf_list_batch = [] transf_list_batch.append(transforms.ToTensor()) transf_list_batch.append( transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))) to_input_transf = transforms.Compose(transf_list_batch) # set to true to load images from demo_urls instead of those in test_imgs folder use_urls = False #if True, it will show the recipe even if it's not valid show_anyways = True if use_urls: response = requests.get(demo_path) image = Image.open(BytesIO(response.content)) else: image_path = os.path.join(image_folder, demo_path) image = Image.open(image_path).convert('RGB') # print('Data path:', image_path) transf_list = [] transf_list.append(transforms.Resize(256)) transf_list.append(transforms.CenterCrop(224)) transform = transforms.Compose(transf_list) image_transf = transform(image) image_tensor = to_input_transf(image_transf).unsqueeze(0).to(device) num_valid = 1 temperature = 1.0 # greedy = [True, False, False, False] # beam = [-1, -1, -1, -1] while True: with torch.no_grad(): outputs = model.sample(image_tensor, greedy=False, temperature=temperature, beam=-1, true_ingrs=None) recipe_ids = outputs['recipe_ids'].cpu().numpy() outs, valid = prepare_output(recipe_ids[0], ingrs_vocab) num_valid += 1 if valid['is_valid'] or show_anyways: if valid['reason'] == 'All ok.': # print ('RECIPE', num_valid) # BOLD = '\033[1m' # END = '\033[0m' # print (BOLD + '\nTitle:' + END,outs['title']) # print (BOLD + '\nInstructions:'+END) # print ('-'+'\n-'.join(outs['recipe'])) # print ('='*20) #print ("Reason: ", valid['reason']) break recommend_id, recommend_lights = search(dir_file, outs['recipe'], lights) recommend_title, recommend_url = get_recipe(dir_file, recommend_id) # print('Recommendation of recipe:', recommend_id) # print('Title:', recommend_title) # print('Lights:', recommend_lights) # print('url:', recommend_url) return outs['title'], outs[ 'recipe'], recommend_lights, recommend_title, recommend_url
#-*- coding:UTF-8 -*- import time import torch import shutil import numpy as np import torch.nn as nn import torchvision import torch.utils.data import torch.optim as optim from torch.autograd import Variable import torch.nn.functional as F from args import get_parser from Deidentification import Scrambling # from Loss import CenterLoss,RankingLoss #================================================ myparser=get_parser() opts=myparser.parse_args() #================================================ class AverageMeter(object): """Compute and store the average and current value""" def __init__(self): self.reset() def reset(self): self.val=0 self.avg=0 self.sum=0 self.count=0 def update(self,val,n=1):
def main(): """ A little utility to handle reading and writing streams to and from a queue. --pub <queue> : publish what's read from stdin to <queue> --sub <queue> : read from <queue> and write the messages to stdout --cat : when used with --pub, write all published messages to stdout --clean : check in incoming and outgoing messages. Verify the message is correct JSON and add an embersId if needed. --log_file : Path to write the log file to --log_level : Logging level Other standard EMBERS options (e.g. --verbose). """ import args import message global log ap = args.get_parser() ap.add_argument( '--clean', action="store_true", help='Verify message format and add standard fields such as embersId.') ap.add_argument('--addfeed', action="store_true", help='Add feed and feedPath fields to published message.') ap.add_argument('--cat', action="store_true", help='Write all published messages to stdout.') ap.add_argument('--rm', nargs="+", help="delete queue") arg = ap.parse_args() log = logs.getLogger(log_name=arg.log_file) logs.init(arg, l=arg.log_level, logfile=arg.log_file) init(arg) if arg.rm and not arg.sub: for queue in arg.rm: print "Deleting", queue, queue = ikqueue.Queue(queue) queue.maybe_bind(connect()) queue.delete() print "." return try: # need to use the raw/utf handler unless we are doing clean marshal = UnicodeMarshal() if arg.clean or arg.addfeed: marshal = JsonMarshal() if arg.sub is None and os.environ.get('UPSTART_JOB') is None: arg.sub = '-' # stdin subq = open( arg.sub, 'r') #, marshal=marshal, ssh_key=arg.ssh_key, ssh_conn=arg.tunnel) if arg.pub is None and os.environ.get('UPSTART_JOB') is None: arg.pub = '-' # stdout pubq = open(arg.pub, 'w', capture=arg.cat, marshal=marshal) except Exception as e: log.exception("Exception opening queues: %s" % e) # "Human-readable" queue name can be retrieved as # # sname = subq.get_name() # pname = pubq.get_name() rc = 0 try: it = subq.__iter__() while True: m = '' try: m = it.next() if arg.clean: m = message.clean(m) if m: if arg.addfeed: m = message.add_embers_ids(m, feed=pubq.get_name(), feedPath=pubq.get_name()) pubq.write(m) except StopIteration: break except KeyboardInterrupt: break except Exception as e: rc += 1 if m: log.exception('Could not process message %s: %s' % (m, e)) else: log.exception('Unknown processing error %s' % e) except KeyboardInterrupt: pass except Exception as e: rc = 1 log.exception('Top level exception %s' % e) return rc
def main(): mp = get_parser() #argv[0]:dataset. argv[1]:scramble type opts = mp.parse_args() #pltfunc=plotfunc()#plot image ###################################### 0.Image Scrambling and preprocess images ########################### if opts.dataset_name == 'Yale': dataParams = YaleParams Scrambleimglist, class_number = TraverseDataset( dataParams.dataset_dir, opts.n).preprocessYaleDataset(opts.scramble_method) # Yale elif opts.dataset_name == 'ORL': dataParams = ORLParams Scrambleimglist, class_number = TraverseDataset( dataParams.dataset_dir, opts.n).preprocessORLDataset(opts.scramble_method) #ORL elif opts.dataset_name == 'CMUPIE': dataParams = CMUPIEParams Scrambleimglist, class_number = TraverseDataset( dataParams.dataset_dir, opts.n).preprocessCMUPIEDataset(opts.scramble_method) #CMUPIE else: dataParams = PUBFIGParams Scrambleimglist, class_number = TraverseDataset( dataParams.dataset_dir, opts.n).preprocessPUBFIG83Dataset(opts.scramble_method) # PUBFIG opts.num_classes = class_number print('Data Image Scrambling transforms done. Class number is :{}'.format( class_number)) ###################################### 1.Data Loader################################################# VAE_traindata = FaceData( Scrambleimglist, class_number, dataParams, mode='VAEtrain', transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]), #normalization ]), should_invert=False) print('VAE train data loaded done') MLP_traindata = FaceData(Scrambleimglist, class_number, dataParams, mode='mlptrain', transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]), ]), should_invert=False) print('MLP train data loaded done') MLP_valdata = FaceData(Scrambleimglist, class_number, dataParams, mode='mlpvalidate', transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]), ]), should_invert=False) print('MLP validate data loaded done') MLP_testdata = FaceData(Scrambleimglist, class_number, dataParams, mode='mlptest', transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize([.5, .5, .5], [.5, .5, .5]), ]), should_invert=False) print('MLP test data loaded done') VAE_train_loader = DataLoader(VAE_traindata, shuffle=True, num_workers=opts.num_workers, batch_size=opts.VAE_training_batch_size) MLP_train_loader = DataLoader(MLP_traindata, shuffle=True, num_workers=opts.num_workers, batch_size=opts.MLP_training_batch_size) MLP_val_loader = DataLoader(MLP_valdata, shuffle=True, num_workers=opts.num_workers, batch_size=opts.MLP_validate_batch_size) MLP_test_loader = DataLoader(MLP_testdata, shuffle=True, num_workers=opts.num_workers, batch_size=opts.MLP_test_batch_size) ###################################### 2. Model ############################################################ opts.cuda = torch.cuda.is_available() inputsize = Scrambleimglist[0][0].size[0] #image size MLPModel = MLP( inputsize * int(inputsize / 16), dataParams.num_classes ) #Yale is 15,ORL face class number is 40,CMUPIE class number is 68,PUBFIG is 83 VAEModel = VAE(inputsize) print(VAEModel) torch.manual_seed(opts.seed) if opts.cuda: MLPModel = MLPModel.cuda() VAEModel = VAEModel.cuda() torch.cuda.manual_seed(opts.seed) MLPModel.apply(weights_init) #weight initial VAEModel.apply(weights_init) #weight initial print('model done.') ####################################### 3.Optimizer ################################################################ MLPOptimizer = optim.Adam(MLPModel.parameters(), lr=opts.lr) # optimzer4mlp VAEOptimizer = optim.Adam(VAEModel.parameters(), lr=opts.lr) # optimzer4vae best_val = 0 print('optimizer done.') #################################################### 4.training##################################################### modelTrain = ModelTrain(opts, VAEModel, MLPModel, VAE_train_loader, MLP_train_loader, MLP_val_loader, MLP_test_loader, VAEOptimizer, MLPOptimizer) VAEcount = modelTrain.VAEtrain() mlpcount = modelTrain.mlptrain(best_val)
def main(): """ A little utility to handle reading and writing streams to and from a queue. --pub <queue> : publish what's read from stdin to <queue> --sub <queue> : read from <queue> and write the messages to stdout --cat : when used with --pub, write all published messages to stdout --clean : check in incoming and outgoing messages. Verify the message is correct JSON and add an embersId if needed. --log_file : Path to write the log file to --log_level : Logging level Other standard EMBERS options (e.g. --verbose). """ import args import message global log ap = args.get_parser() ap.add_argument('--clean', action="store_true", help='Verify message format and add standard fields such as embersId.') ap.add_argument('--addfeed', action="store_true", help='Add feed and feedPath fields to published message.') ap.add_argument('--cat', action="store_true", help='Write all published messages to stdout.') ap.add_argument('--rm', nargs="+", help="delete queue") arg = ap.parse_args() log = logs.getLogger(log_name=arg.log_file) logs.init(arg, l=arg.log_level, logfile=arg.log_file) init(arg) if arg.rm and not arg.sub: for queue in arg.rm: print "Deleting", queue, queue = ikqueue.Queue(queue) queue.maybe_bind(connect()) queue.delete() print "." return try: # need to use the raw/utf handler unless we are doing clean marshal = UnicodeMarshal() if arg.clean or arg.addfeed: marshal = JsonMarshal() if arg.sub is None and os.environ.get('UPSTART_JOB') is None: arg.sub = '-' # stdin subq = open(arg.sub, 'r') #, marshal=marshal, ssh_key=arg.ssh_key, ssh_conn=arg.tunnel) if arg.pub is None and os.environ.get('UPSTART_JOB') is None: arg.pub = '-' # stdout pubq = open(arg.pub, 'w', capture=arg.cat, marshal=marshal) except Exception as e: log.exception("Exception opening queues: %s" % e) # "Human-readable" queue name can be retrieved as # # sname = subq.get_name() # pname = pubq.get_name() rc = 0 try: it = subq.__iter__() while True: m = '' try: m = it.next() if arg.clean: m = message.clean(m) if m: if arg.addfeed: m = message.add_embers_ids(m, feed=pubq.get_name(), feedPath=pubq.get_name()) pubq.write(m) except StopIteration: break except KeyboardInterrupt: break except Exception as e: rc += 1 if m: log.exception('Could not process message %s: %s' % (m, e)) else: log.exception('Unknown processing error %s' % e) except KeyboardInterrupt: pass except Exception as e: rc = 1 log.exception('Top level exception %s' % e) return rc
def main(): """A little utility to handle reading and writing streams to and from a queue. --pub <queue> : publish what's read from stdin to <queue> --sub <queue> : read from <queue> and write the messages to stdout --cat : when used with --pub, write all published messages to stdout --clean : check in incoming and outgoing messages. Verify the message is correct JSON and add an embersId if needed. Other standard EMBERS options (e.g. --verbose). """ import args import logs import message ap = args.get_parser() ap.add_argument('--clean', action="store_true", help='Verify message format and add standard fields such as embersId.') ap.add_argument('--cat', action="store_true", help='Write all published messages to stdout.') arg = ap.parse_args() logs.init(arg) init(arg) assert arg.sub or arg.pub, "Need to subscribe or publish to something." # need to use the raw/utf handler unless we are doing clean marshal = UnicodeMarshal() if arg.clean: marshal = JsonMarshal() subq = None if arg.sub: # read a queue subq = open(arg.sub, 'sub', marshal=marshal, ssh_key=arg.ssh_key, ssh_conn=arg.tunnel) else: # read stdin subq = StreamQueue(sys.stdin, marshal=marshal) pubq = None if arg.pub: # send to queue pubq = open(arg.pub, 'pub', capture=arg.cat, marshal=marshal) else: # send to stdout pubq = StreamQueue(sys.stdout, mode='w', marshal=marshal) rc = 0 try: it = subq.__iter__() while True: m = '' try: m = it.next() if arg.clean: m = message.clean(m) if m: pubq.write(m) except StopIteration: break except KeyboardInterrupt: break except: rc += 1 if m: log.exception('Could not process message %s' % (m,)) else: log.exception('Unknown processing error') except KeyboardInterrupt: pass except: rc = 1 log.exception('Top level exception') return rc
def main(): parser = get_parser("finetuning") args = parser.parse_args() if (os.path.exists(args.output_dir) and os.listdir(args.output_dir) and args.do_train and not args.overwrite_output_dir): raise ValueError("Output directory ({}) already exists and is not empty. Use --overwrite_output_dir to overcome.".format(args.output_dir)) # Setup CUDA, GPU & distributed training if args.local_rank == -1 or args.no_cuda: device = torch.device("cuda" if torch.cuda.is_available() and not args.no_cuda else "cpu") args.n_gpu = torch.cuda.device_count() else: # Initializes the distributed backend which will take care of sychronizing nodes/GPUs torch.cuda.set_device(args.local_rank) device = torch.device("cuda", args.local_rank) torch.distributed.init_process_group(backend='nccl') args.n_gpu = 1 args.device = device # Setup logging logging.basicConfig(format='%(asctime)s - %(levelname)s - %(name)s - %(message)s', datefmt='%m/%d/%Y %H:%M:%S', level=logging.INFO if args.local_rank in [-1, 0] else logging.WARN) logger.warning("Process rank: %s, device: %s, n_gpu: %s, distributed training: %s, 16-bits training: %s", args.local_rank, device, args.n_gpu, bool(args.local_rank != -1), args.fp16) # Set seed set_seed(args) # Load pretrained model and tokenizer if args.local_rank not in [-1, 0]: # Barrier to make sure only the first process in distributed training download model & vocab torch.distributed.barrier() config_class, model_class, tokenizer_class = MODEL_CLASSES[args.model_type] config = config_class.from_pretrained(args.config_name if args.config_name else args.model_name_or_path, cache_dir=args.cache_dir if args.cache_dir else None) # Adding an extra embedding dimension for style/content vectors config.extra_embedding_dim = args.extra_embedding_dim tokenizer = tokenizer_class.from_pretrained(args.tokenizer_name if args.tokenizer_name else args.model_name_or_path, do_lower_case=args.do_lower_case, cache_dir=args.cache_dir if args.cache_dir else None) model = model_class.from_pretrained(args.model_name_or_path, from_tf=bool('.ckpt' in args.model_name_or_path), config=config, cache_dir=args.cache_dir if args.cache_dir else None) tokenizer.add_special_tokens(SPECIAL_TOKENS) model.resize_token_embeddings(len(tokenizer)) model.to(args.device) gpt2_model = GPT2ParentModule(args=args, gpt2=model) if args.local_rank == 0: torch.distributed.barrier() # End of barrier to make sure only the first process in distributed training download model & vocab logger.info("Training/evaluation parameters %s", args) # Training if args.do_train: if args.local_rank not in [-1, 0]: torch.distributed.barrier() # Barrier to make sure only the first process in distributed training process the dataset, and the others will use the cache train_dataset = load_and_cache_examples(args, tokenizer, evaluate=False) if args.local_rank == 0: torch.distributed.barrier() global_step, tr_loss = train(args, gpt2_model, train_dataset, tokenizer) logger.info(" global_step = %s, average loss = %s", global_step, tr_loss) if args.do_train and (args.local_rank == -1 or torch.distributed.get_rank() == 0): output_dir = os.path.join(args.output_dir, 'checkpoint-{}'.format(global_step)) if not os.path.exists(output_dir) and args.local_rank in [-1, 0]: os.makedirs(output_dir) save_model(gpt2_model, output_dir, args, global_step, tokenizer) gpt2_model, tokenizer = init_gpt2_model(checkpoint_dir=args.output_dir, args=args, model_class=model_class, tokenizer_class=tokenizer_class) # Evaluation if args.do_eval and args.local_rank in [-1, 0]: eval_done = False all_results = {} top_checkpoint = None patience = 0 while not eval_done: checkpoints = [] if not args.evaluate_specific: checkpoints = list(os.path.dirname(c) for c in sorted(glob.glob(args.output_dir + '/checkpoint-*/' + WEIGHTS_NAME, recursive=True))) logging.getLogger("transformers.modeling_utils").setLevel(logging.WARN) # Reduce logging # Sort checkpoints according to the step number if len(checkpoints) > 0: checkpoints.sort(key=lambda x: int(x.split("-")[-1])) else: checkpoints.append(args.evaluate_specific) checkpoints = [x for x in checkpoints if x not in all_results] # Count the number of while loop iterations no new checkpoints were found if len(checkpoints) == 0: patience += 1 else: patience = 0 logger.info("Evaluate the following checkpoints: %s", checkpoints) for checkpoint in checkpoints: prefix = checkpoint.split('/')[-1] if checkpoint.find('checkpoint') != -1 else "" gpt2_model, _ = init_gpt2_model(checkpoint_dir=checkpoint, args=args, model_class=model_class) result = evaluate(args, gpt2_model, tokenizer, prefix=prefix) all_results[checkpoint] = result["perplexity"] sorted_results = [(k, v) for k, v in all_results.items()] sorted_results.sort(key=lambda x: x[1].item()) if not args.evaluate_specific and args.do_delete_old and len(sorted_results) > args.save_total_limit: logger.info("Deleting worse checkpoints...") # delete all but the top save_total_limit checkpoints for res in sorted_results[args.save_total_limit:]: if os.path.exists(res[0]): logger.info("Deleting {}...".format(res[0])) shutil.rmtree(res[0]) # move top checkpoint to root directory if not args.evaluate_specific and len(sorted_results) > 0 and sorted_results[0][0] != top_checkpoint: command = "cp {}/* {}".format(sorted_results[0][0], args.output_dir) logger.info("executing {}...".format(command)) subprocess.check_output(command, shell=True) top_checkpoint = sorted_results[0][0] sorted_results_summary = "\n".join(["{} = {:.4f}".format(x[0], x[1]) for x in sorted_results]) logger.info("Top checkpoints:\n{}".format(sorted_results_summary)) if args.eval_frequency_min == 0 or args.evaluate_specific or patience > args.eval_patience: eval_done = True else: logger.info("Sleeping for {:d} minutes...zzzz...".format(args.eval_frequency_min)) time.sleep(args.eval_frequency_min * 60) return all_results
def main(): import args import logs # preliminary stab at some useful command line functionality (only cluster is currently needed) ap = args.get_parser() ap.add_argument('-c', '--cluster', action="store_true", help='Switch to output names of cluster nodes.') ap.add_argument('--get', metavar='REQUEST', nargs='?', type=str, help='Information to get from embers.conf [service|data|cluster|host|input|output]') ap.add_argument('--s3only', action="store_true", help="Write debug messages.") ap.add_argument('--s3onlyprod', action="store_true", help="Write debug messages.") ap.add_argument('--prefixes', action="store_true", help="Write debug messages.") ap.add_argument('--prefixpairs', action="store_true", help="Write debug messages.") group = ap.add_mutually_exclusive_group() group.add_argument('--host', metavar='HOST', nargs='?', type=str, help='Name of the host to get information about') group.add_argument('--data', metavar='DATA', nargs='?', type=str, help='Name of the data (queue, etc.) to get information about') arg = ap.parse_args() logs.init(arg) init(arg) if arg.get: assert arg.get in ('service','data','cluster','host','inputs','outputs','services'), 'Improper get request' try: if arg.s3only or arg.s3onlyprod: if arg.s3only: prodOnly = False else: prodOnly = True plst = get_all_s3only(prodOnly) if plst: print string.join(plst) return if arg.prefixpairs: plst, qlst = get_all_prefixpairs() for i in range(len(plst)): print "%s %s" % (qlst[i], plst[i]) return if arg.prefixes: plst = get_all_prefixes() for p in plst: print p return if arg.host: assert arg.host in conf.get('cluster'), 'Host is not listed in embers.conf' if arg.get == 'services': print conf.get('cluster')[arg.host]['services'] if arg.get == 'data': print [data for service in conf['cluster'][arg.host]['services'] for data in conf['services'][service]['outputs']] if arg.data: assert arg.data in conf.get('data'), 'Data is not listed in embers.conf' (host, service) = get_host_and_service(arg.data) if arg.get == 'host': print host if arg.get == 'service': print service if arg.service: assert arg.service in conf.get('services'), 'Service is not listed in embers.conf' if arg.get == 'host': print [host for host in conf['cluster'] if arg.service in conf['cluster'][host]['services']] if arg.get == 'inputs': print conf['services'][arg.service]['inputs'] if arg.get == 'outputs': print conf['services'][arg.service]['outputs'] if arg.cluster or arg.get == 'cluster': print ' '.join(conf.get('cluster','')) if arg.get == 'data' and not (arg.service or arg.data or arg.host): print ' '.join(conf.get('data','')) if arg.get == 'host' and not (arg.service or arg.data or arg.host): print ' '.join(conf.get('cluster','')) if arg.get == 'service' and not (arg.service or arg.data or arg.host): print ' '.join(conf.get('services','')) except Exception, e: log.error("Requested information not in embers.conf, error %s", e)
if best_at_checkpoint_metric: curr_pat = 0 else: curr_pat += 1 args.current_epoch = epoch + 1 # Save the epoch at which the model needs to start save_checkpoint(model, optimizer, args, es_best, epoch_best, 0, curr_pat, checkpoint_filename) stdout_logger.info('Saved checkpoint for epoch {}.'.format(epoch)) if curr_pat > args.patience: break # Mark job as finished f = open(HALT_filename, 'w') for metric in es_best.keys(): f.write('{}:{}\n'.format(metric, es_best[metric])) f.close() if args.tensorboard: logger.close() if __name__ == '__main__': args = get_parser() torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) random.seed(args.seed) np.random.seed(args.seed) main(args)
def main(): """A little utility to handle reading and writing streams to and from a queue. --pub <queue> : publish what's read from stdin to <queue> --sub <queue> : read from <queue> and write the messages to stdout --cat : when used with --pub, write all published messages to stdout --clean : check in incoming and outgoing messages. Verify the message is correct JSON and add an embersId if needed. Other standard EMBERS options (e.g. --verbose). """ import args import logs import message ap = args.get_parser() ap.add_argument( '--clean', action="store_true", help='Verify message format and add standard fields such as embersId.') ap.add_argument('--cat', action="store_true", help='Write all published messages to stdout.') arg = ap.parse_args() logs.init(arg) init(arg) assert arg.sub or arg.pub, "Need to subscribe or publish to something." # need to use the raw/utf handler unless we are doing clean marshal = UnicodeMarshal() if arg.clean: marshal = JsonMarshal() subq = None if arg.sub: # read a queue subq = open(arg.sub, 'sub', marshal=marshal, ssh_key=arg.ssh_key, ssh_conn=arg.tunnel) else: # read stdin subq = StreamQueue(sys.stdin, marshal=marshal) pubq = None if arg.pub: # send to queue pubq = open(arg.pub, 'pub', capture=arg.cat, marshal=marshal) else: # send to stdout pubq = StreamQueue(sys.stdout, mode='w', marshal=marshal) rc = 0 try: it = subq.__iter__() while True: m = '' try: m = it.next() if arg.clean: m = message.clean(m) if m: pubq.write(m) except StopIteration: break except KeyboardInterrupt: break except: rc += 1 if m: log.exception('Could not process message %s' % (m, )) else: log.exception('Unknown processing error') except KeyboardInterrupt: pass except: rc = 1 log.exception('Top level exception') return rc