def main():
csv = CsvFeed(settings.FEED_CSV_PATH)
feed.subscribe(csv)
csv.writeline(feed.header())
credentials = service_account.Credentials\
.from_service_account_file(settings.GCLOUD_CREDENTIALS_FILE_PATH)
subscriber = pubsub.SubscriberClient(credentials=credentials)
print("subscriber created!")
sub_path = subscriber.subscription_path(settings.PUB_SUB.get('project_id'),
settings.PUB_SUB.get('sub_name'))
subs_future = subscriber.subscribe(sub_path, callback)
print("subscriber subcribed to {}!".format(
settings.PUB_SUB.get('sub_name')))
# ftp = ChimolaFTP('ftp.c0310255.ferozo.com', 'c0310255', 'biDIri20fo')
ftp = ChimolaFTP(settings.FTP.get('hostname'),
settings.FTP.get('username'),
settings.FTP.get('password'))
upload_future = ThreadPoolExecutor(max_workers=1).submit(upload_feed, ftp)
print("executor created and submited!")
try:
print("wating for messages...")
subs_future.result()
print("wating for upload feed...")
upload_future.result()
print("messages to ack: {}".format(len(ack_ids)))
subscriber.acknowldege(sub_path, ack_ids)
print("acked message Ids: {}".format(ack_ids))
ack_ids.clear()
except (KeyboardInterrupt, Exception) as ex:
subs_future.cancel()
upload_future.cancel()
print(ex)
raise
def _test_thread_impl():
from concurrent.futures import ThreadPoolExecutor
from matplotlib import pyplot as plt, rcParams
rcParams.update({
"webagg.open_in_browser": False,
"webagg.port_retries": 1,
})
# Test artist creation and drawing does not crash from thread
# No other guarantees!
fig, ax = plt.subplots()
# plt.pause needed vs plt.show(block=False) at least on toolbar2-tkagg
plt.pause(0.5)
future = ThreadPoolExecutor().submit(ax.plot, [1, 3, 6])
future.result() # Joins the thread; rethrows any exception.
fig.canvas.mpl_connect("close_event", print)
future = ThreadPoolExecutor().submit(fig.canvas.draw)
plt.pause(0.5) # flush_events fails here on at least Tkagg (bpo-41176)
future.result() # Joins the thread; rethrows any exception.
plt.close()
fig.canvas.flush_events() # pause doesn't process events after close
def _test_thread_impl():
from concurrent.futures import ThreadPoolExecutor
from matplotlib import pyplot as plt, rcParams
rcParams.update({
"webagg.open_in_browser": False,
"webagg.port_retries": 1,
})
# Test artist creation and drawing does not crash from thread
# No other guarantees!
fig, ax = plt.subplots()
# plt.pause needed vs plt.show(block=False) at least on toolbar2-tkagg
plt.pause(0.5)
future = ThreadPoolExecutor().submit(ax.plot, [1, 3, 6])
future.result() # Joins the thread; rethrows any exception.
fig.canvas.mpl_connect("close_event", print)
future = ThreadPoolExecutor().submit(fig.canvas.draw)
plt.pause(0.5) # flush_events fails here on at least Tkagg (bpo-41176)
future.result() # Joins the thread; rethrows any exception.
plt.close() # backend is responsible for flushing any events here
if plt.rcParams["backend"].startswith("WX"):
# TODO: debug why WX needs this only on py3.8
fig.canvas.flush_events()
def main():
# print(sys.version)
# スレッドベースの非同期実行
future = ThreadPoolExecutor().submit(f, 'wef')
is_feature = isinstance(future, Future)
print(is_feature)
result = future.result()
print(result)
is_running = future.running()
print(is_running)
is_done = future.done()
print(is_done)
is_cancelled = future.cancelled()
print(is_cancelled)
# print(download(urls[0]))
get_sequentials()
get_multi_thread()
counter = Counter()
threads = 2
with ThreadPoolExecutor() as e:
# 2つのスレッドを用意し、それぞれcount_upを呼び出す
futures = [e.submit(count_up, counter) for _ in range(threads)]
done, not_done = wait(futures)
# print(done, not_done)
print(f'{counter.count} != 2000000')
counter2 = ThreadSafeCounter()
threads = 2
with ThreadPoolExecutor() as e:
# 2つのスレッドを用意し、それぞれcount_upを呼び出す
futures = [e.submit(count_up, counter2) for _ in range(threads)]
done, not_done = wait(futures)
print(done, not_done)
print(f'{counter2.count} == 2000000')
with ThreadPoolExecutor(max_workers=3) as e:
d = Downloader()
futures = [e.submit(d.get(url)) for url in urls]
for future in as_completed(futures):
# r = future.result()
print(future)
c = MyClass()
c.execute()
def _wrapper(self, *args, **kwargs):
def callback(future):
return future.result()
try:
f = lambda future: tornado.ioloop.IOLoop.current(
).add_callback_from_signal(functools.partial(callback, future))
future = ThreadPoolExecutor(max_workers=4).submit(
functools.partial(func, self, *args, **kwargs))
future.add_done_callback(f)
return future.result()
except Exception, e:
print e
from concurrent.futures import (ThreadPoolExecutor, Future)
def func():
return 1
if __name__ == '__main__':
# 非同期に行いたい処理をsubmitに渡すと、
# その処理の実行がスケジューリングされて
# Futureクラスのインスタンスが返される。
# ThreadPoolExecutorはスレッドベースの非同期実行
future = ThreadPoolExecutor().submit(func)
print(isinstance(future, Future))
# futureはスケジューリングされた呼び出し可能オブジェクトを
# カプセル化したもので、実行が先送りされている事を表現している。
# スケジューリングはconcurrent.futuresモジュールが行う。
# 非同期で実行した処理の戻り値を取得
print(future.result())
# 現在の状態を確認する
print(future.done())
print(future.running())
print(future.cancelled())
@author: TakahiroKurokawa
"""
#ThreadPoolExecutorクラスはExecutorクラスのサブクラス
from concurrent.futures import ThreadPoolExecutor, Future
#非同期に行いたい処理
def func():
return 1
#非同期に行いたい処理をThreadPoolExecutorクラスのメソッドsubmit()に渡す
future = ThreadPoolExecutor().submit(func)
print(1, isinstance(future, Future))
result = future.result()
print(2, f"result={result}")
print(3, f"done={future.done()}")
print(4, f"running={future.running()}")
print(5, f"cancelled={future.cancelled()}")
#対象ページのurl一覧
urls = ["https://twitter.com", "https://facebook.com", "https://instagram.com"]
from hashlib import md5
from pathlib import Path
from urllib import request
def download(url):
req = request.Request(url)
def _build(self, build_dir, cancel_event, lock: lockfile.LockFile = None):
"""Perform the build. This assumes there actually is a build to perform.
:param Path build_dir: The directory in which to perform the build.
:param threading.Event cancel_event: Event to signal that the build
should stop.
:param lock: The lockfile object. This will need to be refreshed to
keep it from expiring.
:returns: True or False, depending on whether the build appears to have
been successful.
"""
try:
future = ThreadPoolExecutor().submit(self._setup_build_dir,
build_dir)
while future.running():
time.sleep(lock.SLEEP_PERIOD)
lock.renew()
# Raise any errors raised by the thread.
future.result()
except TestBuilderError as err:
self.tracker.error(
note=("Error setting up build directory '{}': {}".format(
build_dir, err)))
return False
try:
# Do the build, and wait for it to complete.
with self.tmp_log_path.open('w') as build_log:
# Build scripts take the test id as a first argument.
cmd = [self._script_path.as_posix(), str(self.test.id)]
proc = subprocess.Popen(cmd,
cwd=build_dir.as_posix(),
stdout=build_log,
stderr=build_log)
result = None
timeout = self._timeout
while result is None:
try:
result = proc.wait(timeout=1)
except subprocess.TimeoutExpired:
lock.renew()
if self._timeout_file.exists():
timeout_file = self._timeout_file
else:
timeout_file = self.tmp_log_path
try:
timeout = max(
timeout,
timeout_file.stat().st_mtime + self._timeout)
except OSError:
pass
# Has the output file changed recently?
if time.time() > timeout:
# Give up on the build, and call it a failure.
proc.kill()
cancel_event.set()
self.tracker.fail(
state=STATES.BUILD_TIMEOUT,
note="Build timed out after {} seconds.".
format(self._timeout))
return False
if cancel_event is not None and cancel_event.is_set():
proc.kill()
self.tracker.update(
state=STATES.ABORTED,
note="Build canceled due to other builds "
"failing.")
return False
except subprocess.CalledProcessError as err:
if cancel_event is not None:
cancel_event.set()
self.tracker.error(
note="Error running build process: {}".format(err))
return False
except (IOError, OSError) as err:
if cancel_event is not None:
cancel_event.set()
self.tracker.error(
note="Error that's probably related to writing the "
"build output: {}".format(err))
return False
finally:
try:
self.tmp_log_path.rename(build_dir / self.LOG_NAME)
except OSError as err:
self.tracker.warn(
"Could not move build log from '{}' to final location "
"'{}': {}".format(self.tmp_log_path, build_dir, err))
try:
self._fix_build_permissions(build_dir)
except OSError as err:
self.tracker.warn("Error fixing build permissions: %s".format(err))
if result != 0:
if cancel_event is not None:
cancel_event.set()
self.tracker.fail(note="Build returned a non-zero result.")
return False
else:
self.tracker.update(state=STATES.BUILD_DONE,
note="Build completed successfully.")
return True
def run(self, coro):
if self.threaded:
future = ThreadPoolExecutor().submit(asyncio.run, coro)
return future.result()
else:
return asyncio.run(coro)
