Esempio n. 1
1
def main():
    size, smooth, src_dir, dest_dir, num_procs = handle_commandline()
    Qtrac.report("starting...")
    summary = scale(size, smooth, src_dir, dest_dir, num_procs)
    summarize(summary, num_procs)
Esempio n. 2
0
def scale(size, smooth, source, target, concurrency):
    
    canceled = False
    
    jobs = multiprocessing.JoinableQueue()
    
    results = multiprocessing.Queue()
    
    create_processes(size, smooth, jobs, results, concurrency)
    
    todo = add_jobs(source, target, jobs)
    
    try:
        jobs.join()
    except KeyboardInterrupt: # May not work on Windows
        Qtrac.report("canceling...")
        canceled = True
    
    copied = scaled = 0
    
    while not results.empty(): # Safe because all jobs have finished
        result = results.get_nowait()
        copied += result.copied
        scaled += result.scaled
    
    return Summary(todo, copied, scaled, canceled)
def results():
    while True:
        result = (yield)
        results.todo += result.todo
        results.copied += result.copied
        results.scaled += result.scaled
        Qtrac.report("{} {}".format("copied" if result.copied else "scaled",
                os.path.basename(result.name)))
Esempio n. 4
0
def main():
    limit, concurrency = handle_commandline()
    Qtrac.report("starting...")
    filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
    jobs = multiprocessing.JoinableQueue()
    results = multiprocessing.Queue()
    create_processes(limit, jobs, results, concurrency)
    todo = add_jobs(filename, jobs)
    process(todo, jobs, results, concurrency)
Esempio n. 5
0
def summarize(summary, concurrency):
    message = "copied {} scaled {} ".format(summary.copied, summary.scaled)
    difference = summary.todo - (summary.copied + summary.scaled)
    if difference:
        message += "skipped {} ".format(difference)
    message += "using {} processes".format(concurrency)
    if summary.canceled:
        message += " [canceled]"
    Qtrac.report(message)
    print()
Esempio n. 6
0
def main():
    limit, how_many_threads = handle_commandline()
    Qtrac.report("starting...")
    filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")

    jobs = queue.Queue()
    results = queue.Queue()
    create_threads(limit, jobs, results, how_many_threads)
    todo = add_jobs(filename, jobs)
    process(todo, jobs, results, how_many_threads)
def summarize(concurrency, canceled):
    message = "copied {} scaled {} ".format(results.copied, results.scaled)
    difference = results.todo - (results.copied + results.scaled)
    if difference:
        message += "skipped {} ".format(difference)
    message += "using {} coroutines".format(concurrency)
    if canceled:
        message += " [canceled]"
    Qtrac.report(message)
    print()
Esempio n. 8
0
def summarize(summary):
    message = "copied {} scaled {} ".format(summary.copied, summary.scaled)
    difference = summary.todo - (summary.copied + summary.scaled)
    if difference:
        message += "skipped {} ".format(difference)
    message += "single-threaded"
    if summary.canceled:
        message += " [canceled]"
    Qtrac.report(message)
    print()
def main():
    size, smooth, source, target, concurrency = handle_commandline()
    Qtrac.report("starting...")
    canceled = False
    try:
        scale(size, smooth, source, target, concurrency)
    except KeyboardInterrupt:
        Qtrac.report("canceling...")
        canceled = True
    summarize(concurrency, canceled)
def scaler(receiver, sink, size, smooth, me):
    while True:
        sourceImage, targetImage, who = (yield)
        if who == me:
            try:
                result = scale_one(size, smooth, sourceImage, targetImage)
                sink.send(result)
            except Image.Error as err:
                Qtrac.report(str(err), True)
        elif receiver is not None:
            receiver.send((sourceImage, targetImage, who))
Esempio n. 11
0
def worker(limit, jobs, results):
    while True:
        try:
            feed = jobs.get()
            ok, result = Feed.read(feed, limit)
            if not ok:
                Qtrac.report(result, True)
            elif result is not None:
                Qtrac.report("read {}".format(result[0][4:-6]))
                results.put(result)
        finally:
            jobs.task_done()
def worker(size, smooth, jobs, results):
    while True:
        try:
            sourceImage, targetImage = jobs.get()
            try:
                result = scale_one(size, smooth, sourceImage, targetImage)
                Qtrac.report("{} {}".format("copied" if result.copied else
                        "scaled", os.path.basename(result.name)))
                results.put(result)
            except Image.Error as err:
                Qtrac.report(str(err), True)
        finally:
            jobs.task_done()
def reader(receiver, sink, limit, me):
    while True:
        feed, who = (yield)
        if who == me:
            ok, result = Feed.read(feed, limit)
            if not ok:
                Qtrac.report(result, True)
                result = None
            else:
                Qtrac.report("read {} at {}".format(feed.title, feed.url))
            sink.send(result)
        elif receiver is not None:
            receiver.send((feed, who))
Esempio n. 14
0
def main():
    
    t1 = time.time()
    
    print(multiprocessing.cpu_count())
    
    size, smooth, source, target, concurrency = (100,1,".",".",4) #handle_commandline()
    
    Qtrac.report("starting...")
    
    summary = scale(size, smooth, source, target, concurrency)
    
    v = time.time() - t1
    print("=="*20)
    print "time:%s" %(v)
Esempio n. 15
0
def process(todo, jobs, results, concurrency):
    canceled = False
    try:
        jobs.join() # Wait for all the work to be done
    except KeyboardInterrupt: # May not work on Windows
        Qtrac.report("canceling...")
        canceled = True
    if canceled:
        done = results.qsize()
    else:
        done, filename = output(results)
    Qtrac.report("read {}/{} feeds using {} threads{}".format(done, todo,
            concurrency, " [canceled]" if canceled else ""))
    print()
    if not canceled:
        webbrowser.open(filename)
def main():
    limit = handle_commandline()
    filename = os.path.join(tempfile.gettempdir(), "whatsnew.html") 
    canceled = False
    todo = done = 0
    with open(filename, "wt", encoding="utf-8") as file:
        file.write("<!doctype html>\n")
        file.write("<html><head><title>What's New</title></head>\n")
        file.write("<body><h1>What's New</h1>\n")
        todo, done, canceled = write_body(file, limit)
        file.write("</body></html>\n")
    Qtrac.report("read {}/{} feeds{}".format(done, todo, " [canceled]" if
            canceled else ""))
    print()
    if not canceled:
        webbrowser.open(filename)
Esempio n. 17
0
def scale(size, smooth, src_dir, dest_dir, num_procs):
    canceled = False
    jobs = multiprocessing.JoinableQueue()
    results = multiprocessing.Queue()
    create_processes(size, smooth, jobs, results, num_procs)
    todo = add_jobs(src_dir, dest_dir, jobs)
    try:
        jobs.join()
    except KeyboardInterrupt: # catch Ctrl-C (may not work on Windows)
        Qtrac.report("canceling...")
        canceled = True
    copied = scaled = 0
    while not results.empty():  # Safe because all jobs have finished
        result = results.get_nowait()
        copied += result.copied
        scaled += result.scaled
    return Summary(todo, copied, scaled, canceled)
Esempio n. 18
0
def main():
    regression = False
    size = int(1e6)
    if len(sys.argv) > 1 and sys.argv[1] == "-P":
        regression = True
        size = 20
    Qtrac.remove_if_exists(os.path.join(tempfile.gettempdir(), "point.db"))
    start = time.clock()
    points = []
    for i in range(size):
        points.append(Point(i, i ** 2, i // 2))
    end = time.clock() - start
    assert points[size - 1].x == size - 1
    print(len(points))
    if not regression:  # wait until we can see how much memory is used
        print("took {} secs to create {:,} points".format(end, size))
        input("press Enter to finish")
def main():
    limit, concurrency = handle_commandline()
    Qtrac.report("starting...")
    filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
    futures = set()
    with concurrent.futures.ThreadPoolExecutor(
            max_workers=concurrency) as executor:
        for feed in Feed.iter(filename):
            future = executor.submit(Feed.read, feed, limit)
            futures.add(future)
        done, filename, canceled = process(futures)
        if canceled:
            executor.shutdown()
    Qtrac.report("read {}/{} feeds using {} threads{}".format(done,
            len(futures), concurrency, " [canceled]" if canceled else ""))
    print()
    if not canceled:
        webbrowser.open(filename)
def main():
    limit, concurrency = handle_commandline()
    Qtrac.report("starting...")
    datafile = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
    filename = os.path.join(tempfile.gettempdir(), "whatsnew.html") 
    canceled = False
    with open(filename, "wt", encoding="utf-8") as file:
        write_header(file)
        pipeline = create_pipeline(limit, concurrency, file)
        try:
            for i, feed in enumerate(Feed.iter(datafile)):
                pipeline.send((feed, i % concurrency))
        except KeyboardInterrupt:
            Qtrac.report("canceling...")
            canceled = True
        write_footer(file, results.ok, results.todo, canceled,
                concurrency)
    if not canceled:
        webbrowser.open(filename)
Esempio n. 21
0
def scale_one(size, smooth, sourceImage, targetImage):
    try:
        oldImage = Image.from_file(sourceImage)
        if oldImage.width <= size and oldImage.height <= size:
            oldImage.save(targetImage)
            return Result(1, 0, targetImage)
        else:
            if smooth:
                scale = min(size / oldImage.width, size / oldImage.height)
                newImage = oldImage.scale(scale)
            else:
                stride = int(math.ceil(max(oldImage.width / size,
                                           oldImage.height / size)))
                newImage = oldImage.subsample(stride)
            newImage.save(targetImage)
            return Result(0, 1, targetImage)
    except Exception as e:
        Qtrac.report("Problem scaling " +
                     os.path.basename(sourceImage), error=True)
        Qtrac.report(str(e), error=True)
        return Result(0, 0, sourceImage)
Esempio n. 22
0
def scale(size, smooth, source, target):
    canceled = False
    todo = copied = scaled = 0
    for sourceImage, targetImage in get_jobs(source, target):
        try:
            todo += 1
            result = scale_one(size, smooth, sourceImage, targetImage)
            copied += result.copied
            scaled += result.scaled
            Qtrac.report("{} {}".format("copied" if result.copied else "scaled", os.path.basename(targetImage)))
        except Image.Error as err:
            Qtrac.report(str(err), True)
        except KeyboardInterrupt:
            Qtrac.report("canceling...")
            canceled = True
            break
    return Summary(todo, copied, scaled, canceled)
def write_body(file, limit):
    canceled = False
    todo = done = 0
    filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
    for feed in Feed.iter(filename):
        todo += 1
        try:
            ok, result = Feed.read(feed, limit)
            if not ok:
                Qtrac.report(result, True)
            elif result is not None:
                Qtrac.report("read {} at {}".format(feed.title, feed.url))
                for item in result:
                    file.write(item)
                done += 1
        except KeyboardInterrupt:
            Qtrac.report("canceling...")
            canceled = True
            break
    return todo, done, canceled
def wait_for(futures):
    canceled = False
    results = []
    try:
        for future in concurrent.futures.as_completed(futures):
            err = future.exception()
            if err is None:
                ok, result = future.result()
                if not ok:
                    Qtrac.report(result, True)
                elif result is not None:
                    Qtrac.report("read {}".format(result[0][4:-6]))
                results.append((ok, result))
            else:
                raise err # Unanticipated
    except KeyboardInterrupt:
        Qtrac.report("canceling...")
        canceled = True
        for future in futures:
            future.cancel()
    return canceled, results
Esempio n. 25
0
def wait_for(futures):
    canceled = False
    copied = scaled = 0
    try:
        for future in concurrent.futures.as_completed(futures):
            err = future.exception()
            if err is None:
                result = future.result()
                copied += result.copied
                scaled += result.scaled
                Qtrac.report("{} {}".format("copied" if result.copied else
                        "scaled", os.path.basename(result.name)))
            elif isinstance(err, Image.Error):
                Qtrac.report(str(err), True)
            else:
                raise err # Unanticipated
    except KeyboardInterrupt:
        Qtrac.report("canceling...")
        canceled = True
        for future in futures:
            future.cancel()
    return Summary(len(futures), copied, scaled, canceled)
Esempio n. 26
0
def main():
    size, smooth, source, target, concurrency = handle_commandline()
    Qtrac.report("starting...")
    summary = scale(size, smooth, source, target, concurrency)
    summarize(summary, concurrency)
def write_footer(file, ok, todo, canceled, concurrency):
    file.write("</body></html>\n")
    Qtrac.report("read {}/{} feeds using {} coroutines{}".format(ok, todo,
            concurrency, " [canceled]" if canceled else ""))
    print()