def upload_photos(set, photos, key, secret, checkpoint):
try:
auth = Auth(key, secret)
auth.authenticate()
except urllib2.HTTPError as e:
print e.read()
raise
set_controller = Photosets(auth)
# Work queue to print upload status
ui_wq = WorkQueue(print_status, num_workers=1)
upload_and_add(photos[0], set, auth, set_controller, ui_wq, checkpoint)
wq = WorkQueue(upload_and_add,
num_workers=16,
max_queue_size=50,
set=set,
auth=auth,
set_controller=set_controller,
ui_wq=ui_wq,
checkpoint=checkpoint)
for photo in photos[1:]:
wq.add(photo)
wq.done()
ui_wq.done()
def __init__(self, *args, **kws):
port = kws.pop('port', 9123)
master_name = kws.pop('name', 'bs.sge.wq')
catalog = kws.pop('catalog', True)
exclusive = kws.pop('exclusive', False)
wq_alg = kws.pop('wq_alg', WORK_QUEUE_SCHEDULE_FCFS)
wq_debug = kws.pop('ccl_debug', 'all')
set_debug_flag(wq_debug)
backend.Backend.__init__(self, *args, **kws)
self.workqueue = WorkQueue(port=port,
name=master_name,
catalog=catalog,
exclusive=exclusive)
self.workqueue.specify_algorithm(wq_alg)
# self.workqueue.specify_master_mode(WORK_QUEUE_MASTER_MODE_STANDALONE)
self.workqueue.specify_worker_mode(WORK_QUEUE_WORKER_MODE_SHARED)
def work():
with tempfile.TemporaryDirectory() as td:
with ThreadPoolExecutor(2, thread_name_prefix="test-executor-") as ex:
yield WorkQueue(__file__, ex, Path(td))
