Esempio n. 1
0
def parseArgs(args):
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(args)
    args = arg_data[0]
    args.script_args = arg_data[1]

    args.disk_cache = False if args.disk_cache == 'no' else True
    args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
    args.load_images = True if args.load_images == 'yes' else False
    args.load_plugins = False if args.load_plugins == 'no' else True

    if args.proxy:
        item = args.proxy.split(':')
        if len(item) < 2 or not len(item[1]):
            p.print_help()
            sys.exit(1)
        args.proxy = item

    do_action('ParseArgs', Bunch(locals()))

    if not args.script:
        p.print_help()
        sys.exit(1)

    try:
        with codecs.open(args.script, encoding='utf-8') as script:
            args.script_name = script.name
            args.script = script.read()
    except IOError as (errno, stderr):
        sys.exit('%s: \'%s\'' % (stderr, args.script))
Esempio n. 2
0
def parseArgs(args):
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(args)
    args = arg_data[0]
    args.script_args = arg_data[1]

    args.disk_cache = False if args.disk_cache == 'no' else True
    args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
    args.load_images = True if args.load_images == 'yes' else False
    args.load_plugins = False if args.load_plugins == 'no' else True
    args.local_access_remote = False if args.local_access_remote == 'no' else True

    if args.proxy:
        item = args.proxy.split(':')
        if len(item) < 2 or not len(item[1]):
            p.print_help()
            sys.exit(1)
        args.proxy = item

    if args.cookies and not os.path.exists(args.cookies):
        sys.exit("No such file or directory: '%s'" % args.cookies)

    do_action('ParseArgs')

    if not args.script:
        p.print_help()
        sys.exit(1)

    if not os.path.exists(args.script):
        sys.exit("No such file or directory: '%s'" % args.script)

    return args
Esempio n. 3
0
def main():

    args = argParser()

    cifarLoader = CifarLoader(args)
    if not os.path.exists(args.logdir):
        os.makedirs(args.logdir)

    device = torch.device("cuda" if args.cuda else "cpu")
    net = args.model(args.logdir, device).to(device)
    print('The log is recorded in ')
    print(net.logFile.name)

    criterion = net.criterion().to(device)
    optimizer = net.optimizer()

    for epoch in trange(args.epochs):  # loop over the dataset multiple times
        net.adjust_learning_rate(optimizer, epoch, args)
        train(net, cifarLoader, optimizer, criterion, epoch, device)
        if epoch % 10 == 0:  # Comment out this part if you want a faster training
            test(net, cifarLoader, device, 'Train')
            test(net, cifarLoader, device, 'Test')

    print('The log is recorded in ')
    print(net.logFile.name)
Esempio n. 4
0
def main():

    args = argParser()

    cifarLoader = CifarLoader(args)
    net = args.model()
    print('The log is recorded in ')
    print(net.logFile.name)

    if os.path.isfile('model.pth'):
        net.load_state_dict(torch.load('model.pth'))

    criterion = net.criterion()
    optimizer = net.optimizer()

    for epoch in range(args.epochs):  # loop over the dataset multiple times
        net.adjust_learning_rate(optimizer, epoch, args)
        train(net, cifarLoader, optimizer, criterion, epoch)
        if epoch % 10 == 0:  # Comment out this part if you want a faster training
            test(net, cifarLoader, 'Train')
            test(net, cifarLoader, 'Test')

    torch.save(net.state_dict(), 'model.pth')
    print('The log is recorded in ')
    print(net.logFile.name)
Esempio n. 5
0
def parseArgs(args):
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(args)
    args = arg_data[0]
    args.script_args = arg_data[1]

    args.disk_cache = False if args.disk_cache == 'no' else True
    args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
    args.load_images = True if args.load_images == 'yes' else False
    args.load_plugins = False if args.load_plugins == 'no' else True
    args.local_access_remote = False if args.local_access_remote == 'no' else True

    if args.proxy:
        item = args.proxy.split(':')
        if len(item) < 2 or not len(item[1]):
            p.print_help()
            sys.exit(1)
        args.proxy = item

    do_action('ParseArgs')

    if not args.script:
        p.print_help()
        sys.exit(1)

    if not os.path.exists(args.script):
        sys.exit('No such file or directory: \'%s\'' % args.script)

    return args
Esempio n. 6
0
def parseArgs(app, args):
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(args)
    args = arg_data[0]
    args.script_args = arg_data[1]

    args.disk_cache = False if args.disk_cache == 'no' else True
    args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
    args.load_images = True if args.load_images == 'yes' else False
    args.load_plugins = False if args.load_plugins == 'no' else True
    args.local_access_remote = False if args.local_access_remote == 'no' else True

    # register an alternative Message Handler
    messageHandler = MessageHandler(args.verbose)
    qInstallMsgHandler(messageHandler.process)

    file_check = (args.cookies, args.config)
    for file_ in file_check:
        if file_ is not None and not os.path.exists(file_):
            sys.exit("No such file or directory: '%s'" % file_)

    if args.config:
        config = Config(app, args.config)
        # apply settings
        for setting in config.settings:
            setattr(args, config.settings[setting]['mapping'], config.property(setting))

            # special case for verbose arg, which will need to be re-applied
            if setting == 'verbose':
                messageHandler.verbose = args.verbose

    split_check = (
        (args.proxy, 'proxy'),
        (args.auth, 'auth')
    )
    for arg, name in split_check:
        if arg:
            item = arg.split(':')
            if len(item) < 2 or not len(item[1]):
                p.print_help()
                sys.exit(1)
            setattr(args, name, item)

    do_action('ParseArgs')

    if args.script is None:
        p.print_help()
        sys.exit(1)

    if not os.path.exists(args.script):
        sys.exit("No such file or directory: '%s'" % args.script)

    return args
Esempio n. 7
0
def parseArgs(app, args):
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(args)
    args = arg_data[0]
    args.script_args = arg_data[1]

    # register an alternative Message Handler
    messageHandler = MessageHandler(args.verbose)
    qInstallMsgHandler(messageHandler.process)

    file_check = (args.cookies_file, args.config)
    for file_ in file_check:
        if file_ is not None and not os.path.exists(file_):
            sys.exit("No such file or directory: '%s'" % file_)

    if args.config:
        config = Config(app, args.config)
        # apply settings
        for setting in config.settings:
            setattr(args, config.settings[setting]['mapping'], config.property(setting))

    split_check = (
        (args.proxy, 'proxy'),
    )
    for arg, name in split_check:
        if arg:
            item = arg.split(':')
            if len(item) < 2 or not len(item[1]):
                p.print_help()
                sys.exit(1)
            setattr(args, name, item)

    do_action('ParseArgs', args)

    if args.debug:
        debug(args.debug)

    # verbose flag got changed on us, so we reload the flag
    if messageHandler.verbose != args.verbose:
        messageHandler.verbose = args.verbose

    if args.script is None:
        p.print_help()
        sys.exit(1)

    if not os.path.exists(args.script):
        sys.exit("No such file or directory: '%s'" % args.script)

    return args
Esempio n. 8
0
def main():

    args = argParser()

    cifarLoader = YohoLoader(args)
    net = args.model()

    criterion = net.criterion()
    optimizer = net.optimizer()

    for epoch in range(args.epochs):  # loop over the dataset multiple times
        net.adjust_learning_rate(optimizer, epoch, args)
        train(net, cifarLoader, optimizer, criterion, epoch)
        if epoch % 4 == 0:  # Comment out this part if you want a faster training
            test(net, cifarLoader, 'Train')
            test(net, cifarLoader, 'Test')
        torch.save(net, "weights/epoch" + str(epoch) + ".yoho")
Esempio n. 9
0
def main():

    args = argParser()

    loader = FaceLoader(args)
    net = args.model()
    print('The log is recorded in ')
    print(net.logFile.name)

    criterion = net.criterion()
    optimizer = net.optimizer()

    for epoch in range(args.epochs):  # loop over the dataset multiple times
        net.adjust_learning_rate(optimizer, epoch, args)
        train(net, loader, optimizer, criterion, epoch)
        if epoch % 1 == 0:  # Comment out this part if you want a faster training
            test(net, loader, 'Train')
            test(net, loader, 'Test')

    print('The log is recorded in ')
    print(net.logFile.name)
Esempio n. 10
0
def main():

    args = argParser()

    cifarLoader = CifarLoader(args)
    net = args.model()
    print('The log is recorded in ')
    print(net.logFile.name)

    criterion = net.criterion()
    optimizer = net.optimizer()

    for epoch in range(args.epochs):  # loop over the dataset multiple times
        net.adjust_learning_rate(optimizer, epoch, args)
        train(net, cifarLoader, optimizer, criterion, epoch)
        if epoch % 1 == 0: # Comment out this part if you want a faster training
            test(net, cifarLoader, 'Train')
            test(net, cifarLoader, 'Test')


    print('The log is recorded in ')
    print(net.logFile.name)
Esempio n. 11
0
def main():
    # cuda = torch.device('cuda')
    # torch.set_default_tensor_type('torch.cuda.FloatTensor')
    args = argParser()

    cifarLoader = CifarLoader(args)
    net = args.model()
    # net.cuda()
    print('The log is recorded in ')
    print(net.logFile.name)

    criterion = net.criterion()
    optimizer = net.optimizer()

    for epoch in range(args.epochs):  # loop over the dataset multiple times
        net.adjust_learning_rate(optimizer, epoch, args)
        train(net, cifarLoader, optimizer, criterion, epoch)
        if epoch % 1 == 0:  # Comment out this part if you want a faster training
            test(net, cifarLoader, 'Train')
            test(net, cifarLoader, 'Test')

    print('The log is recorded in ')
    print(net.logFile.name)
Esempio n. 12
0
def main():

    args = argParser()

    cifarLoader = CifarLoader(args)
    net = args.model()
    print('The log is recorded in ')
    print(net.logFile.name)

    criterion = net.criterion()
    optimizer = net.optimizer()

    device = "cuda" if torch.cuda.is_available() else "cpu"
    net.to(device)

    for epoch in range(args.epochs):  # loop over the dataset multiple times
        net.adjust_learning_rate(optimizer, epoch, args)
        train(net, cifarLoader, optimizer, criterion, epoch, device)
        if epoch % 1 == 0:  # Comment out this part if you want a faster training
            test(net, cifarLoader, device, 'Train')
            test(net, cifarLoader, device, 'Test')

    print('The log is recorded in ')
    print(net.logFile.name)
Esempio n. 13
0
def parseArgs(args):
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(args)
    args = arg_data[0]
    args.script_args = arg_data[1]

    if args.upload_file:
        # process the tags
        item_buffer = {}
        for i in range(len(args.upload_file)):
            item = args.upload_file[i].split('=')
            if len(item) < 2 or not len(item[1]):
                # if buffer is empty, or tag has no
                # value 'tag=', print help and exit
                if not len(item_buffer) or \
                item[1:] and not item[1:][0]:
                    p.print_help()
                    sys.exit(1)

                # this is a bug workaround for argparse.
                # if you call parse_known_args, and you
                # have an --option script arg, the args
                # get jumbled up, and it's inconsistent
                #
                # we're just going to check for -- and
                # swap it all back to the right order
                if args.script_args:
                    for i in range(len(args.upload_file)):
                        if not args.upload_file[i].count('='):
                            # insert the arg after --option (make sure it's not None)
                            if args.script:
                                args.script_args.insert(1, args.script)
                            # insert value args before --option
                            if args.upload_file[i+1:]:
                                arg_buffer = args.upload_file[i+1:]
                                arg_buffer.reverse()
                                for val in arg_buffer:
                                    args.script_args.insert(0, val)
                            args.script = args.upload_file[i]
                            break
                else:
                    args.script = args.upload_file[i]
                    args.script_args = args.upload_file[i+1:]
                break

            # duplicate tag checking
            if item[0] in item_buffer:
                sys.exit('Multiple tags named \'%s\' were found' % item[0])

            item_buffer[item[0]] = item[1]

        # make sure files exist
        for tag in item_buffer:
            if not os.path.exists(item_buffer[tag]):
                sys.exit('No such file or directory: \'%s\'' % item_buffer[tag])
        args.upload_file = item_buffer

    if args.proxy:
        item = args.proxy.split(':')
        if len(item) < 2 or not len(item[1]):
            p.print_help()
            sys.exit(1)
        args.proxy = item

    do_action('ParseArgs', Bunch(locals()))

    if not args.script:
        p.print_help()
        sys.exit(1)

    try:
        args.script = codecs.open(args.script, encoding='utf-8')
    except IOError as (errno, stderr):
        sys.exit('%s: \'%s\'' % (stderr, args.script))
Esempio n. 14
0
def parseArgs(args):
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(args)
    args = arg_data[0]
    args.script_args = arg_data[1]

    if args.upload_file:
        # process the tags
        item_buffer = {}
        for i in range(len(args.upload_file)):
            item = args.upload_file[i].split('=')
            if len(item) < 2 or not len(item[1]):
                # if buffer is empty, or tag has no
                # value 'tag=', print help and exit
                if not len(item_buffer) or \
                item[1:] and not item[1:][0]:
                    p.print_help()
                    sys.exit(1)

                # this is a bug workaround for argparse.
                # if you call parse_known_args, and you
                # have an --option script arg, the args
                # get jumbled up, and it's inconsistent
                #
                # we're just going to check for -- and
                # swap it all back to the right order
                if args.script_args:
                    for i in range(len(args.upload_file)):
                        if not args.upload_file[i].count('='):
                            # insert the arg after --option (make sure it's not None)
                            if args.script:
                                args.script_args.insert(1, args.script)
                            # insert value args before --option
                            if args.upload_file[i + 1:]:
                                arg_buffer = args.upload_file[i + 1:]
                                arg_buffer.reverse()
                                for val in arg_buffer:
                                    args.script_args.insert(0, val)
                            args.script = args.upload_file[i]
                            break
                else:
                    args.script = args.upload_file[i]
                    args.script_args = args.upload_file[i + 1:]
                break

            # duplicate tag checking
            if item[0] in item_buffer:
                sys.exit('Multiple tags named \'%s\' were found' % item[0])

            item_buffer[item[0]] = item[1]

        # make sure files exist
        for tag in item_buffer:
            if not os.path.exists(item_buffer[tag]):
                sys.exit('No such file or directory: \'%s\'' %
                         item_buffer[tag])
        args.upload_file = item_buffer

    if args.proxy:
        item = args.proxy.split(':')
        if len(item) < 2 or not len(item[1]):
            p.print_help()
            sys.exit(1)
        args.proxy = item

    do_action('ParseArgs', Bunch(locals()))

    if not args.script:
        p.print_help()
        sys.exit(1)

    try:
        args.script = codecs.open(args.script, encoding='utf-8')
    except IOError as (errno, stderr):
        sys.exit('%s: \'%s\'' % (stderr, args.script))
Esempio n. 15
0
import os, sys, resources

from phantom import Phantom
from utils import argParser, MessageHandler, version

from PyQt4.QtCore import QString, qInstallMsgHandler, qFatal
from PyQt4.QtGui import *

# make keyboard interrupt quit program
import signal

signal.signal(signal.SIGINT, signal.SIG_DFL)

if __name__ == "__main__":
    # Handle all command-line options
    p = argParser()
    args = p.parse_args()

    # register an alternative Message Handler
    messageHandler = MessageHandler(args.verbose)
    qInstallMsgHandler(messageHandler.process)

    if args.upload_file:
        item_buffer = {}
        for i in range(len(args.upload_file)):
            item = args.upload_file[i].split("=")
            if len(item) < 2 or not len(item[1]):
                if len(item_buffer) == 0:
                    p.print_help()
                    sys.exit(1)
                args.script = args.upload_file[i:]
Esempio n. 16
0
            if index > 1: continue
            print item_dir
            for item in os.listdir(item_dir):
                emulFile = item[:item.rfind(
                    '_')] + '_Emulator_' + item[:-5].split('_')[-1] + '.root'
                if not os.path.exists(list_emulDir[index] + '/' + emulFile):
                    print list_emulDir[
                        index] + '/' + emulFile + "does not exist"
                list_data.append(item)
                list_emul.append(emulFile)

            if args.compare:
                pool = mp.Pool(processes=50)
                with tqdm(total=len(list_data)) as pbar:
                    for _ in pool.imap_unordered(cp.compareDataEmul, [
                            utils.argParser(item_dir, list_emulDir[index],
                                            item, outDir, True)
                            for item in list_data
                    ]):
                        pbar.update()
                pool.close()
                pool.join()
            # NEED TO FIX
            if args.efficiency:
                pool = mp.Pool(processes=50)
                with tqdm(total=len(list_emul)) as pbar:
                    for _ in pool.imap_unordered(ef.calculateEfficiency, [
                            utils.argParser(list_emulDir[index], item, outDir,
                                            True) for item in list_emul
                    ]):
                        pbar.update()
                pool.close()
Esempio n. 17
0
import os, sys, resources

from phantom import Phantom
from utils import argParser, MessageHandler, version

from PyQt4.QtCore import QString, qInstallMsgHandler, qFatal
from PyQt4.QtGui import QIcon, QApplication

# make keyboard interrupt quit program
import signal
signal.signal(signal.SIGINT, signal.SIG_DFL)

if __name__ == '__main__':
    # Handle all command-line options
    p = argParser()
    arg_data = p.parse_known_args(sys.argv[1:])
    args = arg_data[0]
    args.script_args = arg_data[1]

    # register an alternative Message Handler
    messageHandler = MessageHandler(args.verbose)
    qInstallMsgHandler(messageHandler.process)

    if args.upload_file:
        item_buffer = {}
        for i in range(len(args.upload_file)):
            item = args.upload_file[i].split('=')
            if len(item) < 2 or not len(item[1]):
                if len(item_buffer) == 0:
                    p.print_help()
Esempio n. 18
0
def main():

    args = argParser()
    #torch.cuda.set_device(1)
    device = torch.device("cuda:0")
    #device = torch.device('cpu')
    print(device)
    transform = transforms.Compose([
        # TODO: Use these data augmentations later
        transforms.RandomHorizontalFlip(),
        transforms.RandomResizedCrop(224),
        transforms.ToTensor(),
        #transforms.ColorJitter(),
        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
    ])
    #print(os.path.isdir('newtrain'))
    #trainset = torchvision.datasets.ImageFolder('newtrain', transform=transform)

    #cifarLoader = torch.utils.data.DataLoader(trainset, batch_size=4, shuffle=True, num_workers=2)
    cifarLoader = BirdLoader(args)
    #print('BirdLoader initialized')
    outputFile = open(args.outputfile, 'w+')
    logFile = open(args.logfile, 'w+')
    net = args.model()
    net = net.to(device)
    #print('The log is recorded in ')
    #print(net.logFile.name)

    if not args.getoutput:
        criterion = net.criterion().to(device)
        params = list(net.parameters())

        for p in params:
            print(p.requires_grad)

        #optimizer = optim.Adam(params)
        optimizer = net.optimizer()
        convergeCount = 5
        currentLoss = float("inf")
        startTime = datetime.datetime.now()
        log(logFile, 'Training began at ' + str(startTime))
        for epoch in range(
                args.epochs):  # loop over the dataset multiple times
            log(logFile, 'Epoch ' + str(epoch + 1))
            net.adjust_learning_rate(optimizer, epoch, args)
            loss = train(net, cifarLoader.trainloader, optimizer, criterion,
                         epoch, device, logFile)
            if epoch % 1 == 0:  # Comment out this part if you want a faster training
                test(net, cifarLoader, device, logFile, 'Train')
            if abs(currentLoss - loss) < 0.001:
                convergeCount -= 1
            else:
                convergeCount = 5
            if loss < currentLoss:
                currentLoss = loss
                torch.save(net.state_dict(), args.modelfile)
            if convergeCount == 0:
                break
        endTime = datetime.datetime.now()
        log(
            logFile, 'Training ended at ' + str(endTime) +
            '. It trained for ' + str(endTime - startTime))
    else:
        net.load_state_dict(torch.load(args.modelfile))

    output(
        net, outputFile,
        transforms.Compose([
            transforms.Resize(256),
            transforms.CenterCrop(224),
            transforms.ToTensor(),
            transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
        ]), device)
    #test(net)

    #net.save_state_dict('mytraining.pt')

    log(logFile, 'The log is recorded in ')
    log(logFile, args.logfile)
    log(logFile, 'The output is recorded in ')
    log(logFile, args.outputfile)
    log(logFile, 'The model is recorded in ')
    log(logFile, args.modelfile)
    logFile.close()
    outputFile.close()