def _connect_rover_fired(self):
"""Handle callback for HTTP rover connections.
"""
if not self.device_uid:
msg = "\nDevice ID not found!\n\nConnection requires a valid Piksi device ID."
self._prompt_setting_error(msg)
return
if not self.http:
self._prompt_networking_error("\nNetworking disabled!")
return
try:
_base_pragma = self.base_pragma
if not self.http.connect_write(self.link, self.whitelist, pragma=_base_pragma):
msg = ("\nUnable to connect to Skylark!\n\n"
"Please check that you have a network connection.")
self._prompt_networking_error(msg)
self.http.close()
self.connected_rover = False
return
self.connected_rover = True
print "Connected as a base station!"
executor = ThreadPoolExecutor(max_workers=2)
executor.submit(self._retry_read)
except:
self.connected_rover = False
import traceback
print traceback.format_exc()
def search(rootTitle, destTitle, session, maxDepth=-1):
'''
Given the title of a wikipedia page, find how many 'hops' it takes
to get to the given destination page.
Uses the breadth first search algorithm.
'''
visited = set()
consumerQueue = queue.Queue(MAX_CONSUMER_QUEUE_SIZE)
consumerQueue.put(page.Page(rootTitle, session, 0))
producerQueue = queue.Queue()
executor = ThreadPoolExecutor(2)
# Start thread to consume pages
consumerArgs = [destTitle, consumerQueue, producerQueue, visited]
consumerFuture = executor.submit(consumer, *consumerArgs)
# Start thread to produce child pages and add them to consumer queue
producerArgs = [consumerQueue, producerQueue, session]
executor.submit(producer, *producerArgs)
return consumerFuture.result()
def start(self):
left_lng = 103.9213455517
top_lat = 30.7828453209
right_lng = 104.2178123382
bottom_lat = 30.4781772402
offset = 0.002
if os.path.isfile(self.db_name):
os.remove(self.db_name)
try:
with sqlite3.connect(self.db_name) as c:
c.execute('''CREATE TABLE mobike
(Time DATETIME, bikeIds VARCHAR(12), bikeType TINYINT,distId INTEGER,distNum TINYINT, type TINYINT, x DOUBLE, y DOUBLE)''')
except Exception as ex:
pass
executor = ThreadPoolExecutor(max_workers=250)
print("Start")
self.total = 0
lat_range = np.arange(top_lat, bottom_lat, -offset)
for lat in lat_range:
lng_range = np.arange(left_lng, right_lng, offset)
for lon in lng_range:
self.total += 1
executor.submit(self.get_nearby_bikes, (lat, lon))
executor.shutdown()
self.group_data()
class DriveSink:
def __init__(self, parser_args):
self.cloud_drive = CloudDrive(parser_args)
self.sync_progress = SyncProgress()
self.executor = ThreadPoolExecutor(max_workers=12)
self.root_node = CloudNode(
self.cloud_drive,
parser_args.skipmd5,
frozenset(parser_args.exclude_dirs.lower().split(
",")) if parser_args.exclude_dirs else None,
frozenset(parser_args.extensions.lower().split(",")),
parser_args.source,
self.cloud_drive.get_pictures_root_node_id(),
self.sync_progress)
def upload(self):
futures = deque([self.executor.submit(self.root_node.upload)])
while futures:
current = futures.popleft().result()
for child in current.children():
futures.append(self.executor.submit(child.upload))
print("")
self.sync_progress.report_and_wait_for_uploads()
def download(self):
# parallel download NYI
pass
class RemoteHelloConsumer(object):
def __init__(self):
self._helloservice = None
self._name = "Python"
self._msg = "Hello Java"
self._executor = ThreadPoolExecutor()
@Validate
def _validate(self, bundle_context):
# call it!
resp = self._helloservice.sayHello(self._name + "Sync", self._msg)
print(
"{0} IHello service consumer received sync response: {1}".format(
self._name, resp
)
)
# call sayHelloAsync which returns Future and we add lambda to print
# the result when done
self._executor.submit(
self._helloservice.sayHelloAsync, self._name + "Async", self._msg
).add_done_callback(
lambda f: print("async response: {0}".format(f.result()))
)
print("done with sayHelloAsync method")
# call sayHelloAsync which returns Future and we add lambda to print
# the result when done
self._executor.submit(
self._helloservice.sayHelloPromise,
self._name + "Promise",
self._msg,
).add_done_callback(
lambda f: print("promise response: {0}".format(f.result()))
)
print("done with sayHelloPromise method")
def handle(self, *args, **options):
#return
start_date = datetime.date(2013, 11, 24)
#start_date = datetime.date(2015, 4, 29)
now = datetime.date.today()+datetime.timedelta(days=1)
#now = start_date + datetime.timedelta(days=3)
day = start_date
threads = []
executor = ThreadPoolExecutor(max_workers=32)
while day < now:
print (day)
try:
pass
#gain(day.strftime('%Y-%m-%d'))
#t = Thread(target=gain, args=(day.strftime('%Y-%m-%d'),))
#threads.append( t )
#t.start()
#time.sleep(1)
executor.submit (gain, day.strftime('%Y-%m-%d'))
except Exception as e:
print(e)
pass
day += datetime.timedelta(days=1)
#for x in threads:
# print (x)
# x.join()
class ActiveVoice:
def __init__(self):
# See https://msdn.microsoft.com/en-us/library/ms723602%28v=vs.85%29.aspx for doc
self._voice = win32com.client.Dispatch("SAPI.SpVoice")
self._executor = ThreadPoolExecutor(max_workers=1)
def shutdown(self):
self._executor.shutdown(False)
def speak(self, text, volume=DEFAULT_VOLUME, rate=DEFAULT_RATE):
volume = max(MIN_VOLUME, min(MAX_VOLUME, volume))
rate = max(MIN_RATE, min(MAX_RATE, rate))
def do_speak():
self._voice.Volume = volume
self._voice.Rate = rate
self._voice.Speak(text)
self._executor.submit(do_speak)
def wait(self, time_in_sec):
self._executor.submit(time.sleep, time_in_sec)
def list_avatars(self): # pointless, only one on Windows 10!
avatars = []
for index, token in enumerate(self._voice.GetVoices("", "")):
avatars.append( '%d: %s' % (index+1, token.GetDescription()) )
return avatars
def main():
formatter = logging.Formatter(
'[%(asctime)s] %(name)s<%(levelname)s> %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
args = get_parser().parse_args()
if args.subworker and args.worker != 'process':
logger.error('using subworkers requires worker set to be "process"')
return
if args.worker == 'thread':
from concurrent.futures import ThreadPoolExecutor
e = ThreadPoolExecutor(args.concurrency)
for _ in range(args.concurrency):
e.submit(run_worker, args.odq, args.queue, args.worker)
elif args.worker == 'process':
from concurrent.futures import ProcessPoolExecutor
e = ProcessPoolExecutor(args.concurrency)
for _ in range(args.concurrency):
e.submit(run_worker, args.odq, args.queue, args.worker,
args.subworker, args.subconcurrency)
elif args.worker == 'gevent':
from gevent import monkey
monkey.patch_all()
from gevent.pool import Pool
pool = Pool(args.concurrency)
for _ in range(args.concurrency):
pool.spawn(run_worker, args.odq, args.queue, args.worker)
pool.join()
class ElasticDataSink:
def __init__(self, name, conn, model_identifier, workers=5, bound=10000):
self.name = name
self.conn = conn
self.model_identifier = model_identifier
self.queue = Queue(maxsize=bound)
self.pool = ThreadPoolExecutor(max_workers=workers)
def start(self):
self.model_identifier.model_class.init(using=self.conn)
def __sink_item(self):
try:
item = self.queue.get_nowait()
save_status = item.save(using=self.conn)
if not save_status:
logger.error("Error saving the item to the sink")
else:
logger.info("item saved to the sink")
except Empty as e:
logger.warn("sink queue is empty")
logger.warn(e)
def sink_item(self, item):
assert isinstance(item, self.model_identifier.model_class), \
" item must be instance of " + self.model_identifier.model_class
try:
self.queue.put(item, timeout=10)
self.pool.submit(self.__sink_item)
except Full as e:
logger.error("sink queue is full")
logger.error(e)
class AsyncClient(object):
"""Client which uses the base to be more performant.
This client uses Futures with a ThreadPoolExecutor. This allows requests to
be executed asynchronously. Asynchronous execution with multiple Clients
enables requests to be processed in parallel and with pipeline execution at
the server, which can drastically improve achievable interoperability rate
as observed at the client.
Note that methods return Future objects. Users should handle the response
and errors appropriately. If serial request execution is desired, ensure the
Future response or error is received prior to making another request.
"""
def __init__(self, url, username, password, timeout=1):
"""Create a new AsyncClient and login.
Args:
url: Base URL of interoperability server
(e.g., http://localhost:8000)
username: Interoperability username
password: Interoperability password
timeout: Individual session request timeout (seconds)
"""
self.client = Client(url, username, password, timeout)
self.server_info_executor = ThreadPoolExecutor(max_workers=1)
self.uas_telemetry_executor = ThreadPoolExecutor(max_workers=1)
self.obstacles_executor = ThreadPoolExecutor(max_workers=1)
def get_server_info(self):
"""GET server information, to be displayed to judges.
Returns:
Future object which contains the return value or error from the
underlying Client.
"""
return self.server_info_executor.submit(self.client.get_server_info)
def post_telemetry(self, telem):
"""POST new telemetry.
Args:
telem: Telemetry object containing telemetry state.
Returns:
Future object which contains the return value or error from the
underlying Client.
"""
return self.uas_telemetry_executor.submit(self.client.post_telemetry,
telem)
def get_obstacles(self):
"""GET obstacles.
Returns:
Future object which contains the return value or error from the
underlying Client.
"""
return self.obstacles_executor.submit(self.client.get_obstacles)
class BaseDataLayer(Layer):
def setup(self, bottom, top):
param = eval(self.param_str_)
self.batch_size_ = param['batch_size']
self.data_setup(bottom, top)
top[0].reshape(*self.data_.shape)
self.executor_ = ThreadPoolExecutor(max_workers=1)
self.thread_ = self.executor_.submit(self.internal_thread_entry)
def reshape(self, bottom, top):
pass
def forward(self, bottom, top):
self.thread_.result()
top[0].reshape(*self.data_.shape)
top[0].data[...] = self.data_
self.thread_ = self.executor_.submit(self.internal_thread_entry)
def data_setup(self, bottom, top):
raise NotImplementedError()
def internal_thread_entry(self):
raise NotImplementedError()
def __del__(self):
self.thread_.result()
self.executor_.shutdown()
super(self.__class__, self).__del__()
def main():
"""Main method executed when run"""
if len(argv) != 2:
print("Usage: flymirror.py [rules_file]")
return
if not file_exists(argv[1]):
print("Error: rules file", argv[1], "does not exist.")
return
config = read_config(argv[1])
URLS.put(config.start)
# Start the loops in another thread
# include 2 extra threads for the loopers
pool = ThreadPoolExecutor(int(config.workers) + 2)
perfprint("[START]")
pool.submit(download_loop, pool)
pool.submit(handle_response_loop, pool, config)
# Join on both the queues at once (Yeah, this is hacky -- may break in later versions)
while URLS.unfinished_tasks or RESPONSES.unfinished_tasks:
sleep(0.3)
# Shut everything down (may take 1 second)
DONE.put(True)
pool.shutdown()
perfprint("[END]")
def main():
# Update configuration from the local file
from .configuration import Setting
Setting.read_cfg_from_file()
# Print instance information
print("Node name: {0}\nNode address: {1}".format(Setting.get_node_name(), Setting.get_node_addr()))
# Reset data in the database
from .meta_storage import MetaStorage
meta_storage = MetaStorage()
meta_storage.drop_database()
meta_storage.close_connection()
print("Clear data in the database complete.")
# Reset data in the local storage
from general.services import Services
if not Services.is_folder_exist(Setting.get_local_storage()):
Services.t_print(Setting.get_local_storage() + " does not exist! (Local Storage).")
# Get file from the folder
import glob
import os
files = glob.glob(Setting.get_local_storage() + "*")
for file in files:
os.remove(file)
print("Clear {0} files in the local storage complete.".format(len(files)))
# Create a thread for running REST service
from concurrent.futures import ThreadPoolExecutor
pool = ThreadPoolExecutor()
pool.submit(run_rest_service)
def main(cmd, args):
logging.basicConfig(
level=logging.INFO,
format=("%(relativeCreated)04d %(process)05d %(threadName)-10s "
"%(levelname)-5s %(msg)s"))
pool = Pool(max_workers=1)
arg1 = tmp_fname
arg2 = 'arg2'
stage_file_name = cmd
if (glob_stage == 1):
f = pool.submit(check_output, ["ec-perl", "-w", stage_file_name, args, arg2], shell=True)
print "stage 1 started. Initiate call to" + stage_file_name
print "Stage is " + str(glob_stage)
else:
f = pool.submit(check_output, ["ec-perl", stage_file_name, " ", args, arg2], shell=True)
print "stage 2 started. Initiate call to" + stage_file_name + " " + args + " " + arg2 + "'"
print "Stage is " + str(glob_stage)
f.add_done_callback(callback)
pool.shutdown(wait=False)
class CreateQueue:
''' Queue of creation requests '''
def __init__(self):
self.create_executor = ThreadPoolExecutor(max_workers=32)
self.tasks = []
self.lock = threading.Lock()
self.init()
def init(self):
for creation in models.QueuedCreation.objects.all():
req = VmCreationRequest(creation.owner.name, creation.stats, creation.resource.id)
self.tasks.append(
Task(created=util.datetime_to_unix(creation.created),
request=req,
params=creation.params))
def queue(self, request, params):
if len(self.tasks) > settings.QUEUE_LIMIT:
raise errors.QueueFull()
models.QueuedCreation(owner=models.Account.objects.get(name=request.owner),
stats=request.stats,
resource=models.DerivativeResource.objects.get(id=request.res_id),
params=params).save()
self.tasks.append(Task(time.time(), request, params))
def maybe_unqueue(self, spawner):
# TODO: this may be too expensive if queue is really big
with self.lock:
tasks = list(self.tasks)
tasks.sort(key=lambda k: -k.current_priority)
for task in tasks:
if spawner.check_estimates(task.request):
logging.info('submitting queued task %s', task.request)
self._submit(spawner, task)
self.tasks.remove(task)
break
def start(self):
pass
def _requeue(self, task):
self.tasks.append(task)
def _submit(self, spawner, task):
def execute():
try:
response = spawner.create_vm_if_possible(task.request)
logging.info('sandbox created (response=%s)', response)
if not response:
self._requeue(task)
else:
task.finish(response)
except:
logging.exception('create vm from queue')
raise
self.create_executor.submit(execute).add_done_callback(lambda val: None)
def echo_server(addr):
pool = ThreadPoolExecutor(128)
sock = socket(AF_INET, SOCK_STREAM)
sock.bind(addr)
sock.listen(5)
while True:
client_sock, client_addr = sock.accept()
pool.submit(echo_client, client_sock, client_addr)
class ThreadPool(object):
def __init__(self):
self.pool = ThreadPoolExecutor(128)
def execute(self, func, *args, **kwargs):
LOG.info(_('execute %s in a thread pool') % (func.__name__))
self.pool.submit(func, *args, **kwargs)
def compress_dir(in_dir, out_dir):
if not out_dir.exists():
out_dir.mkdir()
executor = ThreadPoolExecutor(4)
for file in (f for f in in_dir.iterdir() if f.suffix == '.bmp'):
out_file = (out_dir / file.name).with_suffix('.rle')
executor.submit(compress_image, str(file), str(out_file))
def pool_run(self):
"""
通过线程池跑
:return:
"""
pool = ThreadPoolExecutor(max_workers=10)
for i in range(10):
pool.submit(self.run, i)
def __iter__(self): pool = ThreadPoolExecutor(1) it = iter(self.it) future = pool.submit(self._next, it) while True: result = future.result() future = pool.submit(self._next, it) yield result
class Core(object):
def __init__(self, driver, mapping, store, default_state):
self.driver = driver
self.mapping = mapping
self.store = store
self.default_state = default_state
self.logger = logging.getLogger(__name__)
self.executor = ThreadPoolExecutor(max_workers=1)
def set_pdu_outlet_command(self, pdu, outlet, command):
self.logger.info("PDU '{}', outlet '{}' has new command: '{}'".format(
pdu, outlet, command)
)
self.store[(pdu, outlet)] = FINAL_STATES[command]
try:
device = self._get_device(pdu, outlet)
self.logger.debug(
"Found server '{}' on PDU '{}' outlet '{}'".format(
device, pdu, outlet)
)
self.executor.submit(self._switch_power, command, device)
except KeyError:
pass
def _switch_power(self, command, device):
if command == POWER_ON:
self.driver.power_on(device)
elif command == POWER_OFF:
self.driver.power_off(device)
elif command == REBOOT:
self.driver.power_off(device)
self.driver.power_on(device)
else:
self.logger.error("Unknown power command: {}".format(command))
return
def get_pdu_outlet_state(self, pdu, outlet):
try:
return self.store[(pdu, outlet)]
except KeyError:
pass
try:
device = self._get_device(pdu, outlet)
power_state = self.driver.get_power_state(device)
except KeyError:
power_state = self.default_state
self.store[(pdu, outlet)] = power_state
return power_state
def _get_device(self, pdu, outlet):
return self.mapping[(pdu, outlet)]
def echo_server(addr):
print('Serwer Echo działa pod adresem', addr)
pool = ThreadPoolExecutor(128)
sock = socket(AF_INET, SOCK_STREAM)
sock.bind(addr)
sock.listen(5)
while True:
client_sock, client_addr = sock.accept()
pool.submit(echo_client, client_sock, client_addr)
def main():
# Update configuration from the local file
from .configuration import Setting
Setting.read_configuration_from_file()
# Create a thread for running REST service
from concurrent.futures import ThreadPoolExecutor
pool = ThreadPoolExecutor()
pool.submit(run_rest_service)
def __init__(self, ordererStub, entity, directory, nodeAdminTuple, timeout = 600):
pool = ThreadPoolExecutor(1)
StreamHelper.__init__(self)
self.nodeAdminTuple = nodeAdminTuple
self.directory = directory
self.entity = entity
# Set the UpdateMessage and start the stream
sendGenerator = self.createSendGenerator(timeout)
self.replyGenerator = ordererStub.Deliver(sendGenerator, timeout + 1)
pool.submit(self._start_receive)
def search(self, query):
n_days = (self.until - self.since).days
tp = ThreadPoolExecutor(max_workers=self.n_threads)
for i in range(0, n_days):
since_query = self.since + datetime.timedelta(days=i)
until_query = self.since + datetime.timedelta(days=(i + 1))
day_query = "%s since:%s until:%s" % (query, since_query.strftime("%Y-%m-%d"),
until_query.strftime("%Y-%m-%d"))
tp.submit(self.perform_search, day_query)
tp.shutdown(wait=True)
class Listener:
BUF_SIZE = 4096
PORT = 3439
def __init__(self):
self.pool = ThreadPoolExecutor(max_workers=1)
self.queue = queue.Queue()
self.socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.socket.bind(('', Listener.PORT))
handler = threading.Thread(target=self.handle)
handler.daemon = True
handler.start()
self.listen()
def listen(self):
while True:
try:
(buf, address) = self.socket.recvfrom(Listener.BUF_SIZE)
self.queue.put((buf, address))
self.pool.submit(print, (buf, address))
except KeyboardInterrupt:
traceback.print_exc()
self.queue.join()
self.socket.close()
sys.exit(0)
def handle(self):
instances = {}
counts = {}
while True:
while not self.queue.empty():
try:
buf, _ = self.queue.get()
items = buf.split(','.encode('utf8'))
instances[items[0]] = (items[1], items[2], items[3])
counts[items[0]] = 5
finally:
self.queue.task_done()
for key in instances.keys():
counts[key] -= 1
if counts[key] == 0:
del instances[key]
del counts[key]
# print('Instances: {0}'.format(instances))
# print('Counts: {0}'.format(counts))
time.sleep(5)
def console_main():
""" This function handles all the console action. """
setproctitle('image-scraper')
scraper = ImageScraper()
scraper.get_arguments()
print("\nImageScraper\n============\nRequesting page....\n")
try:
scraper.get_html()
except PageLoadError as err:
if err.status_code is None:
print("ImageScraper is unable to acces the internet.")
else:
print("Page failed to load. Status code: {0}".format(err.status_code))
sys.exit()
scraper.get_img_list()
if len(scraper.images) == 0:
sys.exit("Sorry, no images found.")
if scraper.no_to_download is None:
scraper.no_to_download = len(scraper.images)
print("Found {0} images: ".format(len(scraper.images)))
try:
scraper.process_download_path()
except DirectoryAccessError:
print("Sorry, the directory can't be accessed.")
sys.exit()
except DirectoryCreateError:
print("Sorry, the directory can't be created.")
sys.exit()
if scraper.dump_urls:
for img_url in scraper.images:
print(img_url)
status_flags = {'count': 0, 'percent': 0.0, 'failed': 0, 'under_min_or_over_max_filesize': 0}
widgets = ['Progress: ', Percentage(), ' ', Bar(marker=RotatingMarker()),
' ', ETA(), ' ', FileTransferSpeed()]
pbar = ProgressBar(widgets=widgets, maxval=100).start()
pool = ThreadPoolExecutor(max_workers=scraper.nthreads)
status_lock = threading.Lock()
for img_url in scraper.images:
if status_flags['count'] == scraper.no_to_download:
break
pool.submit(download_worker_fn, scraper, img_url, pbar, status_flags, status_lock)
status_flags['count'] += 1
pool.shutdown(wait=True)
pbar.finish()
print("\nDone!\nDownloaded {0} images\nFailed: {1}\n".format(
status_flags['count']-status_flags['failed']-status_flags['under_min_or_over_max_filesize'],
status_flags['failed']))
return
def test_watcher(self):
executor = ThreadPoolExecutor(max_workers=1)
f = executor.submit(lambda: 42)
w = FutureWatcher(f)
def spies(w):
return SimpleNamespace(
done=QSignalSpy(w.done),
finished=QSignalSpy(w.finished),
result=QSignalSpy(w.resultReady),
error=QSignalSpy(w.exceptionReady),
cancelled=QSignalSpy(w.cancelled)
)
spy = spies(w)
self.assertTrue(spy.done.wait())
self.assertEqual(list(spy.done), [[f]])
self.assertEqual(list(spy.finished), [[f]])
self.assertEqual(list(spy.result), [[42]])
self.assertEqual(list(spy.error), [])
self.assertEqual(list(spy.cancelled), [])
f = executor.submit(lambda: 1/0)
w = FutureWatcher(f)
spy = spies(w)
self.assertTrue(spy.done.wait())
self.assertEqual(list(spy.done), [[f]])
self.assertEqual(list(spy.finished), [[f]])
self.assertEqual(len(spy.error), 1)
self.assertIsInstance(spy.error[0][0], ZeroDivisionError)
self.assertEqual(list(spy.result), [])
self.assertEqual(list(spy.cancelled), [])
ev = threading.Event()
# block the executor to test cancellation
executor.submit(lambda: ev.wait())
f = executor.submit(lambda: 0)
w = FutureWatcher(f)
self.assertTrue(f.cancel())
ev.set()
spy = spies(w)
self.assertTrue(spy.done.wait())
self.assertEqual(list(spy.done), [[f]])
self.assertEqual(list(spy.finished), [])
self.assertEqual(list(spy.error), [])
self.assertEqual(list(spy.result), [])
self.assertEqual(list(spy.cancelled), [[f]])
class Processor(object):
PROCESS_TIME_LIMIT = 30
EXCEPTION_LIMIT = 3
executor = ThreadPoolExecutor(max_workers=5)
def __init__(self, newtask_queue, resultdb, projectdb, enable_stdout_capture=True):
self.newtask_queue = newtask_queue
self.resultdb = resultdb
self.enable_stdout_capture = enable_stdout_capture
self.project_manager = ProjectManager(projectdb, dict(
enable_stdout_capture=self.enable_stdout_capture,
))
self.executor = ThreadPoolExecutor(max_workers=5)
def handle_result(self, task, result):
'''Deal one response result'''
start_time = time.time()
response = rebuild_response(result)
follows = []
try:
assert 'taskid' in task, 'need taskid in task'
project_name = task['project_name']
project_id = task['project_id']
project_data = self.project_manager.get(project_name, project_id)
assert project_data, "No such project!"
if project_data.get('exception'):
logger.error(project_data.get('exception_log'))
else:
logger.info('Sccceed in getting project data.')
results, follows, db_name, coll_name = project_data['instance'].run_task(
project_data['module'],
task,
response
)
return (results, follows, db_name, coll_name)
except Exception as e:
logger.exception(e)
process_time = time.time() - start_time
logger.info('Process time cost: %s' % str(process_time))
return (None, None, None, None)
def put_tasks(self, tasks):
self.executor.submit(self.newtask_queue.put, *tasks)
def put_results(self, results, db_name, coll_name, task, query=None):
self.executor.submit(self.save2db, results, db_name, coll_name, task, query=query)
def save2db(self, results, db_name, coll_name, task, query=None):
self.resultdb.update(db_name, coll_name, results, task, query=query)
def on_message(self, parameters):
input_p = json.loads(parameters) # Decode JSON dictionary into a python dictionary to get key information.
pool = ThreadPoolExecutor(max_workers=settingmain.MAX_WORKERS)
if input_p['task'] == 'uniprotparser':
# Input key to look for result.
result = yield pool.submit(uptasks.get_results, input_p['key'])
pool.shutdown()
self.write_message(result)
if input_p['task'] == 'uniprotparser2':
# This statement have no function as of yet.
result = yield pool.submit(uniparser.uniprot_parser, input_p)
pool.shutdown()
self.write_message(json.encode(result))
class Datafeed(object):
def __init__(self):
self.markets = []
self.observers = []
self.depths = {}
self.init_markets(config.markets)
self.init_observers(config.observers)
self.threadpool = ThreadPoolExecutor(max_workers=10)
def init_markets(self, _markets):
logging.debug("init_markets:%s" % _markets)
self.market_names = _markets
markets = create_markets(_markets)
for market_name, market in markets.items():
if self.get_market(market_name):
continue
self.markets.append(market)
def init_observers(self, _observers):
logging.debug("init_observers:%s" % _observers)
self.observer_names = _observers
for observer_name in _observers:
try:
exec('import observers.' + observer_name.lower())
observer = eval('observers.' + observer_name.lower() + '.' +
observer_name + '()')
self.observers.append(observer)
except (ImportError, AttributeError) as e:
print("%s observer name is invalid: Ignored (you should check your config file)" % (observer_name))
print(e)
def register_observer(self, _observer):
logging.debug("register_observer:%s" % _observer)
self.observers.append(_observer)
def get_market(self, market_name):
for market in self.markets:
if market.name == market_name:
return market
return None
def observer_tick(self):
for observer in self.observers:
observer.tick(self.depths)
def tick(self):
self.print_tickers()
self.observer_tick()
def __get_market_depth(self, market, depths):
depth = market.get_depth()
if depth:
depths[market.name] = depth
def update_depths(self):
depths = {}
futures = []
for market in self.markets:
futures.append(self.threadpool.submit(self.__get_market_depth,
market, depths))
wait(futures, timeout=20)
return depths
def print_tickers(self):
for market in self.markets:
logging.debug("ticker: " + market.name + " - " + str(market.get_ticker()))
def replay_history(self, directory):
import os
import json
import pprint
files = os.listdir(directory)
files.sort()
for f in files:
depths = json.load(open(directory + '/' + f, 'r'))
self.depths = {}
for market in self.market_names:
if market in depths:
self.depths[market] = depths[market]
self.tick()
def update_balance(self):
pass
def terminate(self):
for observer in self.observers:
observer.terminate()
for market in self.markets:
market.terminate()
def _run_loop(self, is_feed = True):
if not is_feed and len(self.observers) == 0:
print('empty observers')
return
if is_feed and len(self.markets) == 0:
print('empty markets')
return
#
signal.signal(signal.SIGINT, sigint_handler)
#以下那句在windows python2.4不通过,但在freebsd下通过
signal.signal(signal.SIGHUP, sigint_handler)
signal.signal(signal.SIGTERM, sigint_handler)
kafka_topic = config.kafka_topic
bootstrap_servers = config.bootstrap_servers
try:
if is_feed:
producer = KafkaProducer(bootstrap_servers=bootstrap_servers,
# key_serializer=str.encode,
value_serializer=lambda v: json.dumps(v).encode('utf-8'))
else:
consumer = KafkaConsumer(kafka_topic,
value_deserializer=lambda m: json.loads(
m.decode('utf-8')),
bootstrap_servers=bootstrap_servers)
except Exception as ex:
logging.warn("exception depths:%s" % ex)
traceback.print_exc()
return
if is_feed:
while True:
self.depths = self.update_depths()
future = producer.send(kafka_topic, value=self.depths)
# Block for 'synchronous' sends
try:
record_metadata = future.get(timeout=20)
logging.info(record_metadata)
except Exception as ex:
logging.warn("exception in producer:%s" % ex)
traceback.print_exc()
continue
if is_sigint_up:
# 中断时需要处理的代码
logging.info("exit in producer")
self.terminate()
break
sys.stdout.write(".")
sys.stdout.flush()
time.sleep(config.refresh_rate)
else:
try:
for message in consumer:
if is_sigint_up:
# 中断时需要处理的代码
logging.info("exit in consumer begin")
self.terminate()
break
# print (message)
# message value and key are raw bytes -- decode if necessary!
# e.g., for unicode: `message.value.decode('utf-8')`
logging.info ("datafeed: %s:%d:%d: key=%s" % (message.topic, message.partition,
message.offset, message.key))
# print(message)
self.update_balance()
self.depths = message.value
self.tick()
if is_sigint_up:
# 中断时需要处理的代码
logging.info("exit in consumer end")
self.terminate()
break
logging.info('consumer done...')
# self.tick()
except Exception as ex:
logging.warn("exception in consumer:%s" % ex)
traceback.print_exc()
self.terminate()
return
logging.info('app exist.')
def run_loop(self):
is_feed = False
self._run_loop(is_feed)
"""
使用ThreadPoolExecutor向线程池进行任务提交者更
方便的从被调用函数中获取返回值
"""
from concurrent.futures import ThreadPoolExecutor
import urllib.request
def fetch_url(url):
u = urllib.request.urlopen(url)
data = u.read()
return data
pool = ThreadPoolExecutor(10)
# 提交到线程池
a = pool.submit(fetch_url, 'http://www.python.org')
b = pool.submit(fetch_url, 'http://www.pypy.org')
# 获得的handle对象会处理所有的阻塞与协作，然后从工作线程中返回数据
x = a.result()
y = b.result()
print(x)
print(y)
class UpdateTask(TaskQueue.Task):
def __init__(self, ui, data_hub, data_center, force: bool):
super(UpdateTask, self).__init__('UpdateTask')
self.__ui = ui
self.__force = force
self.__data_hub = data_hub
self.__data_center = data_center
self.__quit = False
# Thread pool
self.__patch_count = 0
self.__apply_count = 0
self.__future = None
self.__pool = ThreadPoolExecutor(max_workers=1)
# Parameters
self.uri = ''
self.identities = []
self.clock = Clock(False)
self.progress = ProgressRate()
def in_work_package(self, uri: str) -> bool:
return self.uri == uri
def set_work_package(self, uri: str, identities: list or str or None):
if isinstance(identities, str):
identities = [identities]
self.uri = uri
self.identities = identities
# ----------------------------------------------------------------------------------------
# def __build_click_table(self):
# self.__clock_table = {}
# for uri, identities in self.__update_pack:
# self.__clock_table[uri] = Clock()
#
# def __build_progress_table(self):
# self.__progress_table = {}
# for uri, identities in self.__update_pack:
# progress_rate = ProgressRate()
# self.__progress_table[uri] = progress_rate
# if identities is not None:
# progress_rate.set_progress(uri, 0, len(identities))
# for identity in identities:
# progress_rate.set_progress([uri, identity], 0, 1)
# else:
# progress_rate.set_progress(uri, 0, 1)
def run(self):
print('Update task start.')
self.__patch_count = 0
self.__apply_count = 0
try:
# Catch "pymongo.errors.ServerSelectionTimeoutError: No servers found yet" exception and continue.
self.__execute_update()
except Exception as e:
print('Update got Exception: ')
print(e)
print('Continue...')
finally:
if self.__future is not None:
self.__future.cancel()
print('Update task finished.')
def quit(self):
self.__quit = True
def identity(self) -> str:
return self.uri
# ------------------------------------- Task -------------------------------------
def __execute_update(self):
self.clock.reset()
self.progress.reset()
self.progress.set_progress(self.uri, 0, len(self.identities) if self.identities is not None else 1)
identities = self.identities if self.identities is not None else [None]
for identity in identities:
while (self.__patch_count - self.__apply_count > 20) and not self.__quit:
time.sleep(0.5)
continue
if self.__quit:
break
print('------------------------------------------------------------------------------------')
if identity is not None:
# Optimise: Update not earlier than listing date.
listing_date = self.__data_hub.get_data_utility().get_securities_listing_date(identity, default_since())
if self.__force:
since, until = listing_date, now()
else:
since, until = self.__data_center.calc_update_range(self.uri, identity)
since = max(listing_date, since)
time_serial = (since, until)
else:
time_serial = None
patch = self.__data_center.build_local_data_patch(self.uri, identity, time_serial, force=self.__force)
self.__patch_count += 1
print('Patch count: ' + str(self.__patch_count))
self.__future = self.__pool.submit(self.__execute_persistence, self.uri, identity, patch)
if self.__future is not None:
print('Waiting for persistence task finish...')
self.__future.result()
self.clock.freeze()
# self.__ui.task_finish_signal[UpdateTask].emit(self)
def __execute_persistence(self, uri: str, identity: str, patch: tuple) -> bool:
try:
if patch is not None:
self.__data_center.apply_local_data_patch(patch)
if identity is not None:
self.progress.set_progress([uri, identity], 1, 1)
self.progress.increase_progress(uri)
except Exception as e:
print('e')
return False
finally:
self.__apply_count += 1
print('Persistence count: ' + str(self.__apply_count))
return True
try:
os.mkdir(config['certs_folder'])
except FileExistsError as e:
print('mkdir failed:\n\t', e)
exit(0)
init_trust_certs(config['trusted_certs_folder'])
count = 0
if os.path.isfile(config['extract_from_folder']):
count += 1
extract_file(config['extract_from_folder'], count)
elif os.path.isdir(config['extract_from_folder']):
p = ThreadPoolExecutor() #线程池
for root, _, files in os.walk(config['extract_from_folder']):
for f in files:
ext = os.path.splitext(f)[1]
f = os.path.join(root, f)
if ext == '.pcap':
count += 1
p.submit(extract_file, f, count)
# extract_file(f)
p.shutdown()
else:
print(f"{config['extract_from_folder']} not found.")
print("total count:", count)
print("time consumed:", time.time() - start)
# print(sum)
# extract_file('pcaps/1.pcap')
def get_page(url):
print('<%s> is getting [%s]' % (os.getpid(), url))
response = requests.get(url)
time.sleep(random.randint(1, 7))
return {'url': url, 'text': response.text}
# 解析网页
def parse_page(res):
res = res.result()
print("res", res)
print("<%s> parse [%s]" % (os.getpid(), res['url']))
with open('content.txt', 'a') as f:
parse_res = "[url]: %s [size]: %s\n" % (res['url'], len(res['text']))
f.write(parse_res)
if __name__ == '__main__':
urls = [
'https://www.baidu.com', 'https://www.python.org',
'https://www.openstack.org', 'https://help.github.com/',
'http://www.sina.com.cn/'
]
start = time.time()
# pool = ThreadPoolExecutor()
pool = ThreadPoolExecutor()
for url in urls:
pool.submit(get_page, url).add_done_callback(parse_page)
pool.shutdown()
print("主进程PID：%s,运行时间:%s" % (os.getpid(), time.time() - start))
class HttpClient(object):
"""A http based client for submitting Spark-based jobs to a Livy backend.
Parameters
----------
url_str : string
Livy server url to create a new session or the url of an existing
session
load_defaults : boolean, optional
This parameter decides if the default config needs to be loaded
Default is True
conf_dict : dict, optional
The key-value pairs in the conf_dict will be loaded to the config
Default is None
Examples
--------
Imports needed to create an instance of HttpClient
>>> from livy.client import HttpClient
1) Creates a client that is loaded with default config
as 'load_defaults' is True by default
>>> client = HttpClient("http://example:8998/")
2) Creates a client that does not load default config, but loads
config that are passed in 'config_dict'
>>> config_dict = {'spark.app.name', 'Test App'}
>>> client = HttpClient("http://example:8998/", load_defaults=False,
>>> config_dict=config_dict)
"""
_CONFIG_SECTION = 'env'
_LIVY_CLIENT_CONF_DIR = "LIVY_CLIENT_CONF_DIR"
def __init__(self, url, load_defaults=True, conf_dict=None):
uri = urlparse(url)
self._config = ConfigParser()
self._load_config(load_defaults, conf_dict)
match = re.match(r'(.*)/sessions/([0-9]+)', uri.path)
if match:
base = ParseResult(scheme=uri.scheme,
netloc=uri.netloc,
path=match.group(1),
params=uri.params,
query=uri.query,
fragment=uri.fragment)
self._set_uri(base)
self._conn = _LivyConnection(base, self._config)
self._session_id = int(match.group(2))
self._reconnect_to_existing_session()
else:
self._set_uri(uri)
session_conf_dict = dict(self._config.items(self._CONFIG_SECTION))
self._conn = _LivyConnection(uri, self._config)
self._session_id = self._create_new_session(
session_conf_dict).json()['id']
self._executor = ThreadPoolExecutor(max_workers=1)
self._stopped = False
self.lock = threading.Lock()
def submit(self, job):
"""
Submits a job for execution to the spark cluster.
Parameters
----------
job : function
The function must accept a single parameter, which is an instance
of JobContext.
Returns
-------
job_handle : an instance of the class JobHandle
A handle that can be used to monitor the job
Examples
-------
>>> def simple_spark_job(context):
>>> elements = [10, 20, 30, 40, 50]
>>> return context.sc.parallelize(elements, 2).count()
>>> client.submit(simple_spark_job)
"""
return self._send_job('submit-job', job)
def run(self, job):
"""
Asks the remote context to run a job immediately.
Normally, the remote context will queue jobs and execute them based on
how many worker threads have been configured. This method will run
the submitted job in the same thread processing the RPC message,
so that queueing does not apply.
It's recommended that this method only be used to run code that
finishes quickly. This avoids interfering with the normal operation
of the context.
Parameters
----------
job : function
The function must accept a single parameter, which is an instance
of JobContext. Spark jobs can be created with the help of
JobContext, which exposes the Spark libraries.
Returns
-------
future : concurrent.futures.Future
A future to monitor the status of the job
Examples
-------
>>> def simple_job(context):
>>> return "hello"
>>> client.run(simple_job)
"""
return self._send_job("run-job", job)
def add_file(self, file_uri):
"""
Adds a file to the running remote context.
Note that the URL should be reachable by the Spark driver process. If
running the driver in cluster mode, it may reside on a different
host, meaning "file:" URLs have to exist on that node (and not on
the client machine).
Parameters
----------
file_uri : string
String representation of the uri that points to the location
of the file
Returns
-------
future : concurrent.futures.Future
A future to monitor the status of the job
Examples
-------
>>> client.add_file("file:/test_add.txt")
>>> # Example job using the file added using add_file function
>>> def add_file_job(context):
>>> from pyspark import SparkFiles
>>> def func(iterator):
>>> with open(SparkFiles.get("test_add.txt")) as testFile:
>>> fileVal = int(testFile.readline())
>>> return [x * fileVal for x in iterator]
>>> return context.sc.parallelize([1, 2, 3, 4])
>>> .mapPartitions(func).collect()
>>> client.submit(add_file_job)
"""
return self._add_file_or_pyfile_job("add-file", file_uri)
def add_jar(self, file_uri):
"""
Adds a jar file to the running remote context.
Note that the URL should be reachable by the Spark driver process. If
running the driver in cluster mode, it may reside on a different host,
meaning "file:" URLs have to exist on that node (and not on the
client machine).
Parameters
----------
file_uri : string
String representation of the uri that points to the location
of the file
Returns
-------
future : concurrent.futures.Future
A future to monitor the status of the job
Examples
-------
>>> client.add_jar("file:/test_package.jar")
"""
return self._add_file_or_pyfile_job("add-jar", file_uri)
def add_pyfile(self, file_uri):
"""
Adds a .py or .zip to the running remote context.
Note that the URL should be reachable by the Spark driver process. If
running the driver in cluster mode, it may reside on a different host,
meaning "file:" URLs have to exist on that node (and not on the
client machine).
Parameters
----------
file_uri : string
String representation of the uri that points to the location
of the file
Returns
-------
future : concurrent.futures.Future
A future to monitor the status of the job
Examples
-------
>>> client.add_pyfile("file:/test_package.egg")
>>> # Example job using the file added using add_pyfile function
>>> def add_pyfile_job(context):
>>> # Importing module from test_package.egg
>>> from test.pyfile_test import TestClass
>>> test_class = TestClass()
>>> return test_class.say_hello()
>>> client.submit(add_pyfile_job)
"""
return self._add_file_or_pyfile_job("add-pyfile", file_uri)
def upload_file(self, file_path):
"""
Upload a file to be passed to the Spark application.
Parameters
----------
file_path : string
File path of the local file to be uploaded.
Returns
-------
future : concurrent.futures.Future
A future to monitor the status of the job
Examples
-------
>>> client.upload_file("/test_upload.txt")
>>> # Example job using the file uploaded using upload_file function
>>> def upload_file_job(context):
>>> from pyspark import SparkFiles
>>> def func(iterator):
>>> with open(SparkFiles.get("test_upload.txt")) as testFile:
>>> fileVal = int(testFile.readline())
>>> return [x * fileVal for x in iterator]
>>> return context.sc.parallelize([1, 2, 3, 4])
>>> .mapPartitions(func).collect()
>>> client.submit(add_file_job)
"""
return self._upload_file_or_pyfile("upload-file",
open(file_path, 'rb'))
def upload_pyfile(self, file_path):
"""
Upload a .py or .zip dependency to be passed to the Spark application.
Parameters
----------
file_path : string
File path of the local file to be uploaded.
Returns
-------
future : concurrent.futures.Future
A future to monitor the status of the job
Examples
-------
>>> client.upload_pyfile("/test_package.egg")
>>> # Example job using the file uploaded using upload_pyfile function
>>> def upload_pyfile_job(context):
>>> # Importing module from test_package.egg
>>> from test.pyfile_test import TestClass
>>> test_class = TestClass()
>>> return test_class.say_hello()
>>> client.submit(upload_pyfile_job)
"""
return self._upload_file_or_pyfile("upload-pyfile",
open(file_path, 'rb'))
def stop(self, shutdown_context):
"""
Stops the remote context.
The function will return immediately and will not wait for the pending
jobs to get completed
Parameters
----------
shutdown_context : Boolean
Whether to shutdown the underlying Spark context. If false, the
context will keep running and it's still possible to send commands
to it, if the backend being used supports it.
"""
with self.lock:
if not self._stopped:
self._executor.shutdown(wait=False)
try:
if shutdown_context:
session_uri = "/" + str(self._session_id)
headers = {'X-Requested-By': 'livy'}
self._conn.send_request("DELETE",
session_uri,
headers=headers)
except:
raise Exception(traceback.format_exc())
self._stopped = True
def _set_uri(self, uri):
if uri is not None and uri.scheme in ('http', 'https'):
self._config.set(self._CONFIG_SECTION, 'livy.uri', uri.geturl())
else:
url_exception = uri.geturl if uri is not None else None
raise ValueError('Cannot create client - Uri not supported - ',
url_exception)
def _set_conf(self, key, value):
if value is not None:
self._config.set(self._CONFIG_SECTION, key, value)
else:
self._delete_conf(key)
def _delete_conf(self, key):
self._config.remove_option(self._CONFIG_SECTION, key)
def _set_multiple_conf(self, conf_dict):
for key, value in conf_dict.items():
self._set_conf(key, value)
def _load_config(self, load_defaults, conf_dict):
self._config.add_section(self._CONFIG_SECTION)
if load_defaults:
self._load_default_config()
if conf_dict is not None and len(conf_dict) > 0:
self._set_multiple_conf(conf_dict)
def _load_default_config(self):
config_dir = os.environ.get(self._LIVY_CLIENT_CONF_DIR)
if config_dir is not None:
config_files = os.listdir(config_dir)
default_conf_files = ['spark-defaults.conf', 'livy-client.conf']
for default_conf_file in default_conf_files:
if default_conf_file in config_files:
self._load_config_from_file(config_dir, default_conf_file)
def _load_config_from_file(self, config_dir, config_file):
path = os.path.join(config_dir, config_file)
data = "[" + self._CONFIG_SECTION + "]\n" + \
open(path, encoding='utf-8').read()
self._config.readfp(StringIO(data))
def _create_new_session(self, session_conf_dict):
data = {'kind': 'pyspark', 'conf': session_conf_dict}
response = self._conn.send_request('POST',
"/",
headers=self._conn._JSON_HEADERS,
data=data)
return response
def _reconnect_to_existing_session(self):
reconnect_uri = "/" + str(self._session_id) + "/connect"
self._conn.send_request('POST',
reconnect_uri,
headers=self._conn._JSON_HEADERS)
def _send_job(self, command, job):
pickled_job = cloudpickle.dumps(job)
base64_pickled_job = base64.b64encode(pickled_job).decode('utf-8')
base64_pickled_job_data = {'job': base64_pickled_job}
handle = JobHandle(self._conn, self._session_id, self._executor)
handle._start(command, base64_pickled_job_data)
return handle
def _add_file_or_pyfile_job(self, command, file_uri):
data = {'uri': file_uri}
suffix_url = "/" + str(self._session_id) + "/" + command
return self._executor.submit(self._add_or_upload_resource,
suffix_url,
data=data,
headers=self._conn._JSON_HEADERS)
def _upload_file_or_pyfile(self, command, open_file):
files = {'file': open_file}
suffix_url = "/" + str(self._session_id) + "/" + command
return self._executor.submit(self._add_or_upload_resource,
suffix_url,
files=files)
def _add_or_upload_resource(self,
suffix_url,
files=None,
data=None,
headers=None):
return self._conn.send_request('POST',
suffix_url,
files=files,
data=data,
headers=headers).content
class JobExecutor:
# The behaviour with blocking_submit == True is that the submit will block after queuing a number of jobs.
# In the case of a storage write for example this ensures that we don't enqueue to many blocks at once and
# so use up all available memory.
# In the other case there is no limit on the number of submitted jobs. But the number of simultaneous
# outstanding results is limited.
# In the case of a storage read for example this ensures that we don't have to many outstanding read blocks
# at once and so use up all available memory.
def __init__(self, *, workers: int, blocking_submit: bool, name: str) -> None:
self._name = name
self._executor = ThreadPoolExecutor(max_workers=workers, thread_name_prefix=name)
self._futures: List[Future] = []
self._blocking_submit = blocking_submit
# Set the queue limit to two times the number of workers plus one to ensure that there are always
# enough jobs available even when all futures finish at the same time.
self._semaphore = BoundedSemaphore(2 * workers + 1)
def submit(self, function: Callable) -> None:
if self._blocking_submit:
self._semaphore.acquire()
def execute_with_release():
try:
return function()
finally:
self._semaphore.release()
self._futures.append(self._executor.submit(execute_with_release))
else:
def execute_with_acquire():
self._semaphore.acquire()
return function()
self._futures.append(self._executor.submit(execute_with_acquire))
# This is tricky to implement as we need to make sure that we don't hold a reference to the completed Future anymore.
# Indeed it's so tricky that older Python versions had the same problem. See https://bugs.python.org/issue27144.
def get_completed(self, timeout: int = None) -> Iterator[Any]:
for future in concurrent.futures.as_completed(self._futures, timeout=timeout):
self._futures.remove(future)
if not self._blocking_submit and not future.cancelled():
self._semaphore.release()
try:
result = future.result()
except Exception as exception:
result = exception
del future
yield result
def shutdown(self) -> None:
if len(self._futures) > 0:
logger.warning('Job executor "{}" is being shutdown with {} outstanding jobs, cancelling them.'.format(
self._name, len(self._futures)))
for future in self._futures:
future.cancel()
logger.debug('Job executor "{}" cancelled all outstanding jobs.'.format(self._name))
if not self._blocking_submit:
# Get all jobs so that the semaphore gets released and still waiting jobs can complete
for _ in self.get_completed():
pass
logger.debug('Job executor "{}" read results for all outstanding jobs.'.format(self._name))
self._executor.shutdown()
def wait_for_all(self) -> None:
concurrent.futures.wait(self._futures)
def main() -> None:
"""Main application entry point."""
parser = argparse.ArgumentParser()
parser.add_argument('--ticket-path',
default='tickets/eng/dev/aws',
type=str,
help="Vanadium ticket used to access Grail resources.")
parser.add_argument('--install', action='store_true')
parser.add_argument('--install-adhoc-2', action='store_true')
parser.add_argument(
'--adhoc-name',
default='',
type=str,
help="Name of the adhoc instance (may be a glob pattern)")
parser.add_argument('--release',
default='',
type=str,
help="""Copy gqljupyter and
jupyter.py to S3. The arg is the local path of the gqljupyter binary""")
parser.add_argument('--port', default=8888, type=int)
args = parser.parse_args()
if args.install_adhoc_2:
install_jupyter_kernel()
sys.exit(0)
config = Config(adhoc_addr=get_adhoc_addr(args.adhoc_name),
ticket_path=args.ticket_path)
if args.install:
install_jupyter(config)
sys.exit(0)
if args.release:
if os.environ.get('USER') != 'ysaito':
raise Exception('--release should be done only by ysaito')
sha256_got = check_output(['sha256sum', args.release])
with tempfile.NamedTemporaryFile(mode='w') as fd:
fd.write(sha256_got)
fd.flush()
s3_cp(config, fd.name, 's3://grail-ysaito/gql/gqljupyter.sha256')
s3_cp(config, args.release, 's3://grail-ysaito/gql/gqljupyter')
s3_cp(config, __file__, 's3://grail-ysaito/gql/jupyter.py')
sys.exit(0)
if os.environ.get('USER') == 'ubuntu':
logging.error('This script must run on your desktop/laptop, not adhoc')
sys.exit(1)
if not validate_gql_binary(config):
install_jupyter(config)
if not validate_gql_binary(config):
raise Exception("Failed to install gql")
subprocess.call([
'ssh', 'ubuntu@' + config.adhoc_addr,
"ps awx | grep jupyter | awk '{print $1}' | xargs kill -9"
])
pool = ThreadPoolExecutor(max_workers=128)
r = pool.submit(start_browser, args.port)
logging.info('Starting jupyter notebook at localhost:%s', args.port)
subprocess.check_call([
'ssh', '-L',
'%d:127.0.0.1:%d' % (args.port, args.port),
'ubuntu@' + config.adhoc_addr,
'export V23_CREDENTIALS=/home/ubuntu/.v23; ',
'/home/ubuntu/.local/bin/jupyter', 'notebook', '--ip=127.0.0.1',
'--port=%d' % (args.port, ), '--no-browser', '--NotebookApp.token=',
'--NotebookApp.password='
])
class Downloader:
from ..TuiTools.Bar import DataTransformBar
def __init__(self, url: str, num: int, name: str = '', proxy: str = '',
referer: str = '', output_error: bool = False, failed2exit: bool = False, exit_if_exist: bool = False,
disableStatus: bool = False, disableParallel: bool = False):
"""
qs普通文件下载引擎
Qs general file download engine
:param url: 文件url
:param num: 线程数量
"""
signal.signal(signal.SIGINT, self._kill_self)
info_flag = True
self.url, self.num, self.output_error, self.proxies = url, num, output_error, {}
self.exit_if_exist = exit_if_exist
self.enabled = True
if not disableStatus:
with qs_default_console.status('Getting file info..' if user_lang != 'zh' else '获取文件信息中..'):
self.url, self.name, r = get_fileinfo(url, proxy, referer)
else:
self.url, self.name, r = get_fileinfo(url, proxy, referer)
if not (self.url and self.name and r):
info_flag = False
qs_default_console.print(qs_error_string if failed2exit else qs_warning_string,
'Get File information failed, please check network!'
if user_lang != 'zh' else '获取文件信息失败，请检查网络!')
if failed2exit:
self.enabled = False
return
if self.name and '.' not in self.name:
self.name = os.path.basename(url)
if name:
self.name = name
if os.path.exists(self.name) and self.exit_if_exist:
self.enabled = False
return
if proxy:
self.proxies = {
'http': 'http://'+proxy,
'https': 'https://'+proxy
}
self.headers = headers
if referer:
self.headers['Referer'] = referer
if not self.url:
qs_default_console.print(qs_error_string, self.name)
raise Exception('Connection Error!' if user_lang != 'zh' else '连接失败!')
try:
if not info_flag:
raise KeyError
self.size = int(r.headers['content-length'])
self.main_progress = Downloader.DataTransformBar()
self.dl_id = self.main_progress.add_task('Download', filename=self.name, start=False)
self.main_progress.update(self.dl_id, total=self.size)
if self.size < 5e6 or disableParallel:
qs_default_console.print(qs_info_string, 'FILE SIZE' if user_lang != 'zh' else '文件大小'
, size_format(self.size))
self.size = -self.size
else:
self.fileBlock = GetBlockSize(self.size)
except KeyError:
self.size = -1
self.main_progress = Downloader.DataTransformBar(False)
self.dl_id = self.main_progress.add_task('Download', filename=self.name, start=False)
if self.size > 0:
self.pool = ThreadPoolExecutor(max_workers=self.num)
self.futures, self.fileLock = [], Lock()
self.job_queue = queue.Queue()
if os.path.exists(self.name + '.qs_dl'):
self.ctn_file = open(self.name + '.qs_dl', 'r+')
self.ctn = [int(i) for i in self.ctn_file.read().strip().split()]
else:
self.ctn_file = open(self.name + '.qs_dl', 'w')
self.ctn = []
self.writers = FileWriters(self.name, max(2, int(core_num / 2)), "rb+" if self.ctn else "wb")
qs_default_console.print(qs_info_string, 'FILE SIZE' if user_lang != 'zh' else '文件大小'
, size_format(self.size, align=True))
qs_default_console.print(qs_info_string, 'THRAED NUM' if user_lang != 'zh' else '线程数量'
, '%7d' % num)
def _kill_self(self, signum, frame):
"""
终止下载任务，保存断点
Terminate the download and save the breakpoint
:param signum: 信号
:param frame: frame
:return: None
"""
if self.size > 0:
self.main_progress.stop_task(self.dl_id)
self.main_progress.stop()
qs_default_console.print(qs_info_string,
'Get Ctrl-C, exiting...' if user_lang != 'zh' else '捕获Ctrl-C, 正在退出...')
self.ctn_file.close()
self.pool.shutdown(wait=False)
self.writers.wait()
qs_default_console.print(qs_info_string, 'Deal Done!' if user_lang != 'zh' else '处理完成!')
os._exit(0)
def _dl(self, start):
"""
执行文件块下载任务
Perform the file block download task
:param start: 文件块起始偏移量
:return: None
"""
try:
_sz = min(start + self.fileBlock, self.size - 1)
_headers = self.headers.copy()
_headers['Range'] = 'bytes={}-{}'.format(start, _sz)
_content = b''
for chunk in get(self.url, headers=_headers, timeout=50, proxies=self.proxies).iter_content(65536):
_content += chunk
self.main_progress.advance(self.dl_id, sys.getsizeof(chunk))
self.writers.new_job(_content, start)
except Exception as e:
if self.output_error:
qs_default_console.print(qs_error_string, repr(e))
self.job_queue.put(start)
else:
self.fileLock.acquire()
self.ctn.append(start)
self.ctn_file.write('%d\n' % start)
self.fileLock.release()
def _single_dl(self):
"""
无法并行下载或无需并行下载时采用串行下载
Parallel downloads are not possible or serial downloads are not required
:return: None
"""
r = get(self.url, stream=True, proxies=self.proxies, headers=self.headers)
flag = self.size != -1
if flag:
self.main_progress.start_task(self.dl_id)
else:
self.main_progress.update(self.dl_id, total=-1)
with open(self.name, 'wb') as f:
for chunk in r.iter_content(32768):
f.write(chunk)
self.main_progress.advance(self.dl_id, sys.getsizeof(chunk))
def run(self):
"""
规划下载任务并开始下载
qs可以有效应对网络问题对下载任务造成的影响，如：
1.下载过程中切换代理
2.链路异常
并确保下载任务顺利完成
:return: None
"""
if not self.enabled:
return self.name if self.exit_if_exist else ''
self.main_progress.start()
if self.size > 0:
self.main_progress.start_task(self.dl_id)
if not self.ctn:
with open(self.name, "wb") as fp:
fp.truncate(self.size)
self.main_progress.advance(self.dl_id, self.fileBlock*len(self.ctn))
for i in range(0, self.size, self.fileBlock):
if self.ctn and i in self.ctn:
continue
else:
self.job_queue.put(i)
retry_cnt = 0
while not self.job_queue.empty():
self.futures.clear()
while not self.job_queue.empty():
cur = self.job_queue.get()
if cur not in self.ctn:
self.futures.append(self.pool.submit(self._dl, cur))
wait(self.futures)
if not self.job_queue.empty() and retry_cnt > 2:
qs_default_console.print(qs_warning_string, 'Exists File Block Lost, Retrying after 0.5 sec'
if user_lang != 'zh' else '存在文件块丢失，0.5秒后重试')
time.sleep(0.5)
retry_cnt += 1
self.writers.wait()
self.ctn_file.close()
os.remove(self.name + '.qs_dl')
else:
self._single_dl()
self.main_progress.stop()
qs_default_console.print(qs_info_string, self.name, 'download done!' if user_lang != 'zh' else '下载完成!')
return self.name
# concurrent的案例
from concurrent.futures import ThreadPoolExecutor
import time
def return_future(msg):
time.sleep(3)
return msg
# 创建一个线程池,设定最多线程池个数
pool = ThreadPoolExecutor(max_workers=2)
# 往线程池加入2个task
f1 = pool.submit(return_future, 'hello')
f2 = pool.submit(return_future, 'world')
# 等待执行完毕
print(f1.done())
time.sleep(3)
print(f2.done())
# 结果
print(f1.result())
print(f2.result())
class WavDown(WavDownBasic):
def __init__(self, seqid, list_wav_url, callback_url, consumer_logger):
# 创建线程池
self.executor = ThreadPoolExecutor(WavDownBasic.PoolNum)
self.consumer_logger = consumer_logger
# 批次号，在消息队列里获取
self.seqid = seqid
self.list_wav_url = list_wav_url
self.callback_url = callback_url
# todo 文件夹加时间戳
# 创建md5对象
m = hashlib.md5()
m.update((str(self.seqid) + str(time.time())).encode())
self.filename = m.hexdigest()
# 创建一个下载文件夹暂存文件
self.download_temppath = os.path.join(WavDownBasic.share_path,
str(self.filename))
self.consumer_logger.info("下载文件夹暂存文件 {}".format(
self.download_temppath))
self.consumer_logger.info("创建打包文件名 {}".format(self.filename))
os.makedirs(self.download_temppath, exist_ok=True)
def run(self):
"""
主执行函数，执行批量下载 并打包
:param list_Wav_url: 下载url列表
:return res_json: 返回打包zip文件全路径
"""
print("@@@@@@@@@@@@@@@@@@@@@@run ", self.list_wav_url)
future_list = []
for urlnode in self.list_wav_url:
url = urlnode.strip()
name = urlnode.split("/")[-1]
res_flag = re.match('http', url)
if res_flag is not None:
future = self.executor.submit(self.do_down_wav, url, name)
else:
future = self.executor.submit(self.do_copy_wav, url, name)
future_list.append(future)
all_n = len(future_list)
n = 0
while n != all_n:
n = 0
for future in future_list:
if future.done():
n = n + 1
time.sleep(3)
schedule = int(n * 100 / all_n)
self.consumer_logger.info("schedule {}".format(schedule))
# 返回结果放在一个字典中
res_json_schedule = {
"task_id": str(self.seqid),
"download": "",
"schedule": str(schedule)
}
# todo 结果检验，异常处理
if n != all_n:
self.notify_schedule(send_json_schedule=res_json_schedule)
self.executor.shutdown()
self.consumer_logger.info('''begin make_archive {}'''.format(
self.download_temppath))
t1 = time.time()
# 第一个参数是归档文件名称，第二个参数是指定的格式，不仅是支持zip，第三个参数是要压缩文件/文件夹的路径
# shutil.make_archive(os.path.join(WavDownBasic.share_path, self.filename), 'zip', self.download_temppath)
self.consumer_logger.info(
'''zip -r -q -o {zippath} {download_temppath}'''.format(
zippath=os.path.join(WavDownBasic.share_path,
self.filename + ".zip "),
download_temppath=self.download_temppath))
subprocess.run(
'''zip -r -q -o {zippath} {download_temppath}'''.format(
zippath=os.path.join(WavDownBasic.share_path,
self.filename + ".zip"),
download_temppath=self.download_temppath),
shell=True,
cwd=WavDownBasic.share_path)
t2 = time.time()
self.consumer_logger.debug(
'''end make_archive {download_temppath} ,cost time {costtime}'''.
format(download_temppath=self.download_temppath,
costtime=(t2 - t1) / 60))
t1 = time.time()
# 删除暂存文件夹
time.sleep(1)
self.consumer_logger.debug('''begin del {}'''.format(
self.download_temppath))
# shutil.rmtree(self.download_temppath, ignore_errors=True)
subprocess.run('''rm -rf {download_temppath}'''.format(
download_temppath=self.download_temppath),
shell=True,
cwd=WavDownBasic.share_path)
t2 = time.time()
self.consumer_logger.debug(
'''end del {download_temppath} ,cost time {costtime}'''.format(
download_temppath=self.download_temppath,
costtime=(t2 - t1) / 60))
download_url = "http://{domain_name}:{LocalPort}/{filename}".format(
domain_name=WavDownBasic.domain_name,
LocalPort=WavDownBasic.LocalPort,
filename=self.filename + ".zip")
self.consumer_logger.info(
"task_id: {task_id} download_url: {download_url}".format(
task_id=self.seqid, download_url=download_url))
# 返回结果放在一个字典中
res_json = {
"task_id": str(self.seqid),
"download": str(download_url),
"schedule": "100"
}
return res_json
def do_down_wav(self, url, name):
"""
下载单个网络上录音文件
:param url:
:param name:
:return:
"""
# print("下载地址为：",url,"文件名为：",name)
try:
self.consumer_logger.debug(
"begin download wav {url} {name}".format(url=url, name=name))
res = requests.get(url=url)
if res.status_code == 200:
self.consumer_logger.debug(
"begin write wav {name} to localfile".format(url=url,
name=name))
with open(os.path.join(self.download_temppath, name),
mode='bw') as f:
f.write(res.content)
self.consumer_logger.debug(
"end write wav {name} to localfile".format(url=url,
name=name))
else:
self.consumer_logger.error(
"[ERROR],wav文件下载异常，可能找不到 wav文件 {}".format(url))
except Exception as e:
self.consumer_logger.error(
"[ERROR],wav文件下载异常，可能无法访问 远程服务器{e} {url}".format(e=str(e),
url=str(url)))
def do_copy_wav(self, filepath, name):
"""
下载单个网络上录音文件
:param url:
:param name:
:return:
"""
des_path = os.path.join(self.download_temppath, name)
sou_path = filepath
if os.path.exists(sou_path):
print("sou_path为：", sou_path, "des_path 为：", des_path)
try:
self.consumer_logger.debug(
"begin copy wav {filepath} {name}".format(
filepath=filepath, name=name))
shutil.copy(sou_path, des_path)
except Exception as e:
self.consumer_logger.error(
"[ERROR], wav文件copy异常，可能无法访问 远程服务器{}".format(str(e)))
else:
self.consumer_logger.error("[ERROR], wav文件copy异常，文件不存在")
def notify_schedule(self, send_json_schedule):
"""
发生进度情况给 回调通知地址
:param send_json_schedule: 需要发生的消息体 json Post请求
:return:
"""
try:
res = requests.post(url=self.callback_url, json=send_json_schedule)
res_content_json = json.loads(res.text)
self.consumer_logger.debug("schedule notify code = {}".format(
res_content_json['code']))
if int(res_content_json['code']) != 1:
self.consumer_logger.error("schedule notify error! {}".format(
res_content_json['msg']))
else:
self.consumer_logger.debug("schedule notify OK! {}".format(
res_content_json['msg']))
except Exception as e:
self.consumer_logger.error(
"schedule notify error! Network anomaly {}".format(str(e)))
from concurrent.futures import ThreadPoolExecutor
def crawlPageDate(url):
print(url, kwargs)
if __name__ == "__main__":
pool = ThreadPoolExecutor(max_workers=8)
for page in range(1, 57):
# 往线程池中提交任务
url = 'https: // www.meishij.net / chufang / diy / jiangchangcaipu /? & page =' + str(
page)
pool.submit(crawlPageDate, url)
delta = price - underlying_prices[symbol]
pnl = Pnl(delta, day, u_delta, u_gamma, u_theta)
data = pd.DataFrame({
"coin": [symbol],
"price": [price],
"day": [day],
"pnl": [pnl]
})
return data
################################################
# pull initial data on app start
get_new_data()
# for the data refresh
executor = ThreadPoolExecutor(max_workers=1)
executor.submit(get_new_data_every)
data = pd.read_csv("input.csv")
data = data.to_dict()
address = data.get("address")[0]
symbol = data.get("coin")[0]
price = data.get("price")[0]
day = data.get("day")[0]
# run the calculator
X = trigger_calculator(address, symbol, price, day)
name = input('enter your name: ')
pygame.init()
pygame.display.set_caption("Pandemic")
screen = pygame.display.set_mode((1000, 800))
channel = ServerChannel(ip, 1066)
chat_channel = ChatChannel(ip, 1099)
server_state = channel.read_new_state()
scene = Lobby(screen, channel, chat_channel, server_state, name)
game_started = False
state_fetch_executor = ThreadPoolExecutor(max_workers=1)
state_fetch_future = state_fetch_executor.submit(channel.read_new_state)
running = True
while running:
if state_fetch_future.done():
server_state = state_fetch_future.result()
state_fetch_future = state_fetch_executor.submit(
channel.read_new_state)
if server_state.is_game_mode and not game_started:
scene = Game(screen, channel, server_state)
game_started = True
events = pygame.event.get()
scene.update(server_state, events)
scene.draw()
class Executor(object):
def __init__(self, env, pool, config, io, parallel=None):
self._env = env
self._io = io
self._dry_run = False
self._enabled = True
self._verbose = False
self._authenticator = Authenticator(config, self._io)
self._chef = Chef(config, self._env)
self._chooser = Chooser(pool, self._env)
if parallel is None:
parallel = config.get("installer.parallel", True)
if parallel and not (PY2 and WINDOWS):
# This should be directly handled by ThreadPoolExecutor
# however, on some systems the number of CPUs cannot be determined
# (it raises a NotImplementedError), so, in this case, we assume
# that the system only has one CPU.
try:
self._max_workers = cpu_count() + 4
except NotImplementedError:
self._max_workers = 5
else:
self._max_workers = 1
self._executor = ThreadPoolExecutor(max_workers=self._max_workers)
self._total_operations = 0
self._executed_operations = 0
self._executed = {"install": 0, "update": 0, "uninstall": 0}
self._skipped = {"install": 0, "update": 0, "uninstall": 0}
self._sections = OrderedDict()
self._lock = threading.Lock()
self._shutdown = False
@property
def installations_count(self): # type: () -> int
return self._executed["install"]
@property
def updates_count(self): # type: () -> int
return self._executed["update"]
@property
def removals_count(self): # type: () -> int
return self._executed["uninstall"]
def supports_fancy_output(self): # type: () -> bool
return self._io.supports_ansi() and not self._dry_run
def disable(self):
self._enabled = False
return self
def dry_run(self, dry_run=True):
self._dry_run = dry_run
return self
def verbose(self, verbose=True):
self._verbose = verbose
return self
def execute(self, operations): # type: (Operation) -> int
self._total_operations = len(operations)
for job_type in self._executed:
self._executed[job_type] = 0
self._skipped[job_type] = 0
if operations and (self._enabled or self._dry_run):
self._display_summary(operations)
# We group operations by priority
groups = itertools.groupby(operations, key=lambda o: -o.priority)
self._sections = OrderedDict()
for _, group in groups:
tasks = []
serial_operations = []
for operation in group:
if self._shutdown:
break
# Some operations are unsafe, we mus execute them serially in a group
# https://github.com/python-poetry/poetry/issues/3086
# https://github.com/python-poetry/poetry/issues/2658
#
# We need to explicitly check source type here, see:
# https://github.com/python-poetry/poetry-core/pull/98
is_parallel_unsafe = operation.job_type == "uninstall" or (
operation.package.develop
and operation.package.source_type in {"directory", "git"})
if not operation.skipped and is_parallel_unsafe:
serial_operations.append(operation)
continue
tasks.append(
self._executor.submit(self._execute_operation, operation))
try:
wait(tasks)
for operation in serial_operations:
wait([
self._executor.submit(self._execute_operation,
operation)
])
except KeyboardInterrupt:
self._shutdown = True
if self._shutdown:
# Cancelling further tasks from being executed
[task.cancel() for task in tasks]
self._executor.shutdown(wait=True)
break
return 1 if self._shutdown else 0
def _write(self, operation, line):
if not self.supports_fancy_output(
) or not self._should_write_operation(operation):
return
if self._io.is_debug():
with self._lock:
section = self._sections[id(operation)]
section.write_line(line)
return
with self._lock:
section = self._sections[id(operation)]
section.output.clear()
section.write(line)
def _execute_operation(self, operation):
try:
if self.supports_fancy_output():
if id(operation) not in self._sections:
if self._should_write_operation(operation):
with self._lock:
self._sections[id(operation)] = self._io.section()
self._sections[id(operation)].write_line(
" <fg=blue;options=bold>•</> {message}: <fg=blue>Pending...</>"
.format(message=self.get_operation_message(
operation), ), )
else:
if self._should_write_operation(operation):
if not operation.skipped:
self._io.write_line(
" <fg=blue;options=bold>•</> {message}".format(
message=self.get_operation_message(
operation), ), )
else:
self._io.write_line(
" <fg=default;options=bold,dark>•</> {message}: "
"<fg=default;options=bold,dark>Skipped</> "
"<fg=default;options=dark>for the following reason:</> "
"<fg=default;options=bold,dark>{reason}</>".format(
message=self.get_operation_message(operation),
reason=operation.skip_reason,
))
try:
result = self._do_execute_operation(operation)
except EnvCommandError as e:
if e.e.returncode == -2:
result = -2
else:
raise
# If we have a result of -2 it means a KeyboardInterrupt
# in the any python subprocess, so we raise a KeyboardInterrupt
# error to be picked up by the error handler.
if result == -2:
raise KeyboardInterrupt
except Exception as e:
try:
from clikit.ui.components.exception_trace import ExceptionTrace
if not self.supports_fancy_output():
io = self._io
else:
message = " <error>•</error> {message}: <error>Failed</error>".format(
message=self.get_operation_message(operation,
error=True), )
self._write(operation, message)
io = self._sections.get(id(operation), self._io)
with self._lock:
trace = ExceptionTrace(e)
trace.render(io)
io.write_line("")
finally:
with self._lock:
self._shutdown = True
except KeyboardInterrupt:
try:
message = " <warning>•</warning> {message}: <warning>Cancelled</warning>".format(
message=self.get_operation_message(operation,
warning=True), )
if not self.supports_fancy_output():
self._io.write_line(message)
else:
self._write(operation, message)
finally:
with self._lock:
self._shutdown = True
def _do_execute_operation(self, operation):
method = operation.job_type
operation_message = self.get_operation_message(operation)
if operation.skipped:
if self.supports_fancy_output():
self._write(
operation,
" <fg=default;options=bold,dark>•</> {message}: "
"<fg=default;options=bold,dark>Skipped</> "
"<fg=default;options=dark>for the following reason:</> "
"<fg=default;options=bold,dark>{reason}</>".format(
message=operation_message,
reason=operation.skip_reason,
),
)
self._skipped[operation.job_type] += 1
return 0
if not self._enabled or self._dry_run:
self._io.write_line(
" <fg=blue;options=bold>•</> {message}".format(
message=operation_message, ))
return 0
result = getattr(self, "_execute_{}".format(method))(operation)
if result != 0:
return result
message = " <fg=green;options=bold>•</> {message}".format(
message=self.get_operation_message(operation, done=True), )
self._write(operation, message)
self._increment_operations_count(operation, True)
return result
def _increment_operations_count(self, operation, executed):
with self._lock:
if executed:
self._executed_operations += 1
self._executed[operation.job_type] += 1
else:
self._skipped[operation.job_type] += 1
def run_pip(self, *args, **kwargs): # type: (...) -> int
try:
self._env.run_pip(*args, **kwargs)
except EnvCommandError as e:
output = decode(e.e.output)
if ("KeyboardInterrupt" in output
or "ERROR: Operation cancelled by user" in output):
return -2
raise
return 0
def get_operation_message(self,
operation,
done=False,
error=False,
warning=False):
base_tag = "fg=default"
operation_color = "c2"
source_operation_color = "c2"
package_color = "c1"
if error:
operation_color = "error"
elif warning:
operation_color = "warning"
elif done:
operation_color = "success"
if operation.skipped:
base_tag = "fg=default;options=dark"
operation_color += "_dark"
source_operation_color += "_dark"
package_color += "_dark"
if operation.job_type == "install":
return "<{}>Installing <{}>{}</{}> (<{}>{}</>)</>".format(
base_tag,
package_color,
operation.package.name,
package_color,
operation_color,
operation.package.full_pretty_version,
)
if operation.job_type == "uninstall":
return "<{}>Removing <{}>{}</{}> (<{}>{}</>)</>".format(
base_tag,
package_color,
operation.package.name,
package_color,
operation_color,
operation.package.full_pretty_version,
)
if operation.job_type == "update":
return "<{}>Updating <{}>{}</{}> (<{}>{}</{}> -> <{}>{}</>)</>".format(
base_tag,
package_color,
operation.initial_package.name,
package_color,
source_operation_color,
operation.initial_package.full_pretty_version,
source_operation_color,
operation_color,
operation.target_package.full_pretty_version,
)
return ""
def _display_summary(self, operations):
installs = 0
updates = 0
uninstalls = 0
skipped = 0
for op in operations:
if op.skipped:
skipped += 1
continue
if op.job_type == "install":
installs += 1
elif op.job_type == "update":
updates += 1
elif op.job_type == "uninstall":
uninstalls += 1
if not installs and not updates and not uninstalls and not self._verbose:
self._io.write_line("")
self._io.write_line("No dependencies to install or update")
return
self._io.write_line("")
self._io.write_line("<b>Package operations</b>: "
"<info>{}</> install{}, "
"<info>{}</> update{}, "
"<info>{}</> removal{}"
"{}".format(
installs,
"" if installs == 1 else "s",
updates,
"" if updates == 1 else "s",
uninstalls,
"" if uninstalls == 1 else "s",
", <info>{}</> skipped".format(skipped)
if skipped and self._verbose else "",
))
self._io.write_line("")
def _execute_install(self, operation): # type: (Install) -> None
return self._install(operation)
def _execute_update(self, operation): # type: (Update) -> None
return self._update(operation)
def _execute_uninstall(self, operation): # type: (Uninstall) -> None
message = " <fg=blue;options=bold>•</> {message}: <info>Removing...</info>".format(
message=self.get_operation_message(operation), )
self._write(operation, message)
return self._remove(operation)
def _install(self, operation):
package = operation.package
if package.source_type == "directory":
return self._install_directory(operation)
if package.source_type == "git":
return self._install_git(operation)
if package.source_type == "file":
archive = self._prepare_file(operation)
elif package.source_type == "url":
archive = self._download_link(operation, Link(package.source_url))
else:
archive = self._download(operation)
operation_message = self.get_operation_message(operation)
message = " <fg=blue;options=bold>•</> {message}: <info>Installing...</info>".format(
message=operation_message, )
self._write(operation, message)
args = ["install", "--no-deps", str(archive)]
if operation.job_type == "update":
args.insert(2, "-U")
return self.run_pip(*args)
def _update(self, operation):
return self._install(operation)
def _remove(self, operation):
package = operation.package
# If we have a VCS package, remove its source directory
if package.source_type == "git":
src_dir = self._env.path / "src" / package.name
if src_dir.exists():
safe_rmtree(str(src_dir))
try:
return self.run_pip("uninstall", package.name, "-y")
except CalledProcessError as e:
if "not installed" in str(e):
return 0
raise
def _prepare_file(self, operation):
package = operation.package
message = " <fg=blue;options=bold>•</> {message}: <info>Preparing...</info>".format(
message=self.get_operation_message(operation), )
self._write(operation, message)
archive = Path(package.source_url)
if not Path(package.source_url).is_absolute() and package.root_dir:
archive = package.root_dir / archive
archive = self._chef.prepare(archive)
return archive
def _install_directory(self, operation):
from poetry.factory import Factory
package = operation.package
operation_message = self.get_operation_message(operation)
message = " <fg=blue;options=bold>•</> {message}: <info>Building...</info>".format(
message=operation_message, )
self._write(operation, message)
if package.root_dir:
req = os.path.join(str(package.root_dir), package.source_url)
else:
req = os.path.realpath(package.source_url)
args = ["install", "--no-deps", "-U"]
pyproject = PyProjectTOML(os.path.join(req, "pyproject.toml"))
if pyproject.is_poetry_project():
# Even if there is a build system specified
# some versions of pip (< 19.0.0) don't understand it
# so we need to check the version of pip to know
# if we can rely on the build system
legacy_pip = self._env.pip_version < self._env.pip_version.__class__(
19, 0, 0)
package_poetry = Factory().create_poetry(
pyproject.file.path.parent)
if package.develop and not package_poetry.package.build_script:
from poetry.masonry.builders.editable import EditableBuilder
# This is a Poetry package in editable mode
# we can use the EditableBuilder without going through pip
# to install it, unless it has a build script.
builder = EditableBuilder(package_poetry, self._env, NullIO())
builder.build()
return 0
elif legacy_pip or package_poetry.package.build_script:
from poetry.core.masonry.builders.sdist import SdistBuilder
# We need to rely on creating a temporary setup.py
# file since the version of pip does not support
# build-systems
# We also need it for non-PEP-517 packages
builder = SdistBuilder(package_poetry)
with builder.setup_py():
if package.develop:
args.append("-e")
args.append(req)
return self.run_pip(*args)
if package.develop:
args.append("-e")
args.append(req)
return self.run_pip(*args)
def _install_git(self, operation):
from poetry.core.vcs import Git
package = operation.package
operation_message = self.get_operation_message(operation)
message = " <fg=blue;options=bold>•</> {message}: <info>Cloning...</info>".format(
message=operation_message, )
self._write(operation, message)
src_dir = self._env.path / "src" / package.name
if src_dir.exists():
safe_rmtree(str(src_dir))
src_dir.parent.mkdir(exist_ok=True)
git = Git()
git.clone(package.source_url, src_dir)
reference = package.source_resolved_reference
if not reference:
reference = package.source_reference
git.checkout(reference, src_dir)
# Now we just need to install from the source directory
package._source_url = str(src_dir)
return self._install_directory(operation)
def _download(self, operation): # type: (Operation) -> Path
link = self._chooser.choose_for(operation.package)
return self._download_link(operation, link)
def _download_link(self, operation, link):
package = operation.package
archive = self._chef.get_cached_archive_for_link(link)
if archive is link:
# No cached distributions was found, so we download and prepare it
try:
archive = self._download_archive(operation, link)
except BaseException:
cache_directory = self._chef.get_cache_directory_for_link(link)
cached_file = cache_directory.joinpath(link.filename)
# We can't use unlink(missing_ok=True) because it's not available
# in pathlib2 for Python 2.7
if cached_file.exists():
cached_file.unlink()
raise
# TODO: Check readability of the created archive
if not link.is_wheel:
archive = self._chef.prepare(archive)
if package.files:
hashes = {f["hash"] for f in package.files}
hash_types = {h.split(":")[0] for h in hashes}
archive_hashes = set()
for hash_type in hash_types:
archive_hashes.add("{}:{}".format(
hash_type,
FileDependency(
package.name,
Path(archive.path)
if isinstance(archive, Link) else archive,
).hash(hash_type),
))
if archive_hashes.isdisjoint(hashes):
raise RuntimeError(
"Invalid hashes ({}) for {} using archive {}. Expected one of {}."
.format(
", ".join(sorted(archive_hashes)),
package,
archive.name,
", ".join(sorted(hashes)),
))
return archive
def _download_archive(self, operation,
link): # type: (Operation, Link) -> Path
response = self._authenticator.request("get",
link.url,
stream=True,
io=self._sections.get(
id(operation), self._io))
wheel_size = response.headers.get("content-length")
operation_message = self.get_operation_message(operation)
message = " <fg=blue;options=bold>•</> {message}: <info>Downloading...</>".format(
message=operation_message, )
progress = None
if self.supports_fancy_output():
if wheel_size is None:
self._write(operation, message)
else:
from clikit.ui.components.progress_bar import ProgressBar
progress = ProgressBar(self._sections[id(operation)].output,
max=int(wheel_size))
progress.set_format(message + " <b>%percent%%</b>")
if progress:
with self._lock:
progress.start()
done = 0
archive = self._chef.get_cache_directory_for_link(link) / link.filename
archive.parent.mkdir(parents=True, exist_ok=True)
with archive.open("wb") as f:
for chunk in response.iter_content(chunk_size=4096):
if not chunk:
break
done += len(chunk)
if progress:
with self._lock:
progress.set_progress(done)
f.write(chunk)
if progress:
with self._lock:
progress.finish()
return archive
def _should_write_operation(self, operation): # type: (Operation) -> bool
if not operation.skipped:
return True
return self._dry_run or self._verbose
class WebInterface:
"""
The WebInterface is a executor like class for the submission of runs to the VerifierCloud
"""
def __init__(
self,
web_interface_url,
user_pwd,
revision="trunk:HEAD",
thread_count=1,
result_poll_interval=2,
user_agent=None,
version=None,
):
"""
Creates a new WebInterface object.
The given svn revision is resolved (e.g. 'HEAD' -> 17495).
@param web_interface_url: the base URL of the VerifierCloud's web interface
@param user_pwd: user name (and password) in the format '<user_name>[:<password>]' or none if no authentification is required
@param revision: the svn revision string, defaults to 'trunk:HEAD'
@param thread_count: the number of threads for fetching results in parallel
@param result_poll_interval: the number of seconds to wait between polling results
"""
# this attribute is used to communicate shutdown to the futures
# of the run submission in case the user cancels while the run submission is still ongoing:
self.active = True
if not (1 <= thread_count <= MAX_SUBMISSION_THREADS):
sys.exit(
"Invalid number {} of client threads, needs to be between 1 and {}.".format(
thread_count, MAX_SUBMISSION_THREADS
)
)
if not 1 <= result_poll_interval:
sys.exit(
"Poll interval {} is too small, needs to be at least 1s.".format(
result_poll_interval
)
)
if not web_interface_url[-1] == "/":
web_interface_url += "/"
default_headers = {"Connection": "Keep-Alive"}
if user_agent:
default_headers["User-Agent"] = "{}/{} (Python/{} {}/{})".format(
user_agent,
version,
platform.python_version(),
platform.system(),
platform.release(),
)
urllib.parse.urlparse(web_interface_url) # sanity check
self._web_interface_url = web_interface_url
logging.info("Using VerifierCloud at %s", web_interface_url)
self._connection = requests.Session()
# increase the pool size a bit to get rid of warnings on aborting with SIGINT:
self._connection.mount(
"https://", requests.adapters.HTTPAdapter(pool_maxsize=100)
)
self._connection.headers.update(default_headers)
try:
cert_paths = ssl.get_default_verify_paths()
cert_path = cert_paths.cafile or cert_paths.capath # both might be None
except AttributeError: # not available on old Python
cert_path = None
# make sure that certificate verification is enabled
self._connection.verify = cert_path or True
if user_pwd:
self._connection.auth = self.getUserAndPassword(user_pwd)
self._unfinished_runs = {}
self._unfinished_runs_lock = threading.Lock()
self._downloading_result_futures = {}
self._download_attempts = {}
self.thread_count = thread_count
self._executor = ThreadPoolExecutor(thread_count)
self._thread_local = threading.local()
self._hash_code_cache = {}
self._group_id = str(random.randint(0, 1000000)) # noqa: S311
self._read_hash_code_cache()
self._revision = self._request_tool_revision(revision)
self._tool_name = self._request_tool_name()
if re.match("^.*:[0-9]*$", revision) and revision != self._revision:
logging.warning(
"Using %s version %s, which is different than the requested version %s!",
self._tool_name,
self._revision,
revision,
)
else:
logging.info("Using %s version %s.", self._tool_name, self._revision)
if HAS_SSECLIENT:
self._result_downloader = SseResultDownloader(self, result_poll_interval)
else:
self._result_downloader = PollingResultDownloader(
self, result_poll_interval
)
def getUserAndPassword(self, user_pwd):
# split only once, password might contain special char ':'
tokens = user_pwd.split(":", maxsplit=1)
if len(tokens) == 2 and all(tokens):
user, password = tokens
else:
user = user_pwd
password = getpass("Please enter password for user '" + user + "': ")
return user, password
def _read_hash_code_cache(self):
if not os.path.isfile(HASH_CODE_CACHE_PATH):
return
with open(HASH_CODE_CACHE_PATH, mode="r") as hashCodeCacheFile:
for line in hashCodeCacheFile:
tokens = line.strip().split("\t")
if len(tokens) == 3:
self._hash_code_cache[(tokens[0], tokens[1])] = tokens[2]
def _write_hash_code_cache(self):
# make snapshot of hash code cache to avoid concurrent modification.
# We reset self._hash_code_cache to {} to save memory and because
# it will not be needed any more (this method is only called upon shutdown).
hash_code_cache = self._hash_code_cache
self._hash_code_cache = {}
directory = os.path.dirname(HASH_CODE_CACHE_PATH)
try:
os.makedirs(directory, exist_ok=True)
with tempfile.NamedTemporaryFile(dir=directory, delete=False) as tmpFile:
for (path, mTime), hashValue in hash_code_cache.items():
line = path + "\t" + mTime + "\t" + hashValue + "\n"
tmpFile.write(line.encode())
os.renames(tmpFile.name, HASH_CODE_CACHE_PATH)
except OSError as e:
logging.warning(
"Could not write hash-code cache file to %s: %s",
HASH_CODE_CACHE_PATH,
e.strerror,
)
def _request_tool_revision(self, revision):
path = "tool/version_string?revision=" + revision
(resolved_svn_revision, _) = self._request("GET", path)
return resolved_svn_revision.decode("UTF-8")
def _request_tool_name(self):
path = "tool/name"
(tool_name, _) = self._request("GET", path)
return tool_name.decode("UTF-8")
def tool_revision(self):
return self._revision
def tool_name(self):
return self._tool_name
def _get_sha256_hash(self, path):
path = os.path.abspath(path)
mTime = str(os.path.getmtime(path))
if (path, mTime) in self._hash_code_cache:
return self._hash_code_cache[(path, mTime)]
else:
with open(path, "rb") as opened_file:
hashValue = hashlib.sha256(opened_file.read()).hexdigest()
self._hash_code_cache[(path, mTime)] = hashValue
return hashValue
def _create_and_add_run_future(self, run_id):
result = RunResultFuture(self, run_id)
with self._unfinished_runs_lock:
self._unfinished_runs[run_id] = result
return result
def submit_witness_validation(
self, witness_path, program_path, configuration=None, user_pwd=None
):
"""
Submits a single witness validation run to the VerifierCloud.
@note: flush() should be called after the submission of the last run.
@param witness_path: path to the file containing the witness
@param program_path: path to the file containing the program
@param configuration: name of configuration (optional)
@param user_pwd: overrides the user name and password given in the constructor (optional)
"""
# collect parameters
params = {}
with open(witness_path, "rb") as witness_file:
params["witnessText"] = witness_file.read()
with open(program_path, "rb") as program_file:
params["programText"] = program_file.read()
if configuration:
params["configuration"] = configuration
# prepare request
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Content-Encoding": "deflate",
"Accept": "text/plain",
}
paramsCompressed = zlib.compress(
urllib.parse.urlencode(params, doseq=True).encode("utf-8")
)
path = "runs/witness_validation/"
(response, _) = self._request(
"POST", path, paramsCompressed, headers, user_pwd=user_pwd
)
try:
run_id = response.decode("UTF-8")
except UnicodeDecodeError as e:
raise WebClientError(
"Malformed response from server while submitting witness-validation run:\n{}".format(
response
)
) from e
logging.debug("Submitted witness validation run with id %s", run_id)
return self._create_and_add_run_future(run_id)
def submit(
self,
run,
limits,
cpu_model,
result_files_pattern=None,
meta_information=None,
priority="IDLE",
user_pwd=None,
revision=None,
result_files_patterns=(),
required_files=(),
):
"""
Submits a single run to the VerifierCloud.
@note: flush() should be called after the submission of the last run.
@param run: The input for the run: command line options (run.options),
source files (run.sourcefiles),
property file (run.propertyfile),
identifier for error messages (run.identifier)
@param limits: dict of limitations for the run (memlimit, timelimit, corelimit, softtimelimit)
@param cpu_model: substring of CPU model to use or 'None' for no restriction
@param result_files_pattern: the result is filtered with the given glob pattern, '**' is no restriction and None or the empty string do not match any file.
@param meta_information: meta information about the submitted run as JSON string
@param priority: the priority of the submitted run, defaults to 'IDLE'
@param user_pwd: overrides the user name and password given in the constructor (optional)
@param revision: overrides the revision given in the constructor (optional)
@param result_files_patterns: list of result_files_pattern (optional)
@param required_files: list of additional file required to execute the run (optional)
@raise WebClientError: if the HTTP request could not be created
@raise UserAbortError: if the user already requested shutdown on this instance
@raise HTTPError: if the HTTP request was not successful
"""
if not self.active:
raise UserAbortError("User interrupt detected while submitting runs.")
if result_files_pattern:
if result_files_patterns:
raise ValueError(
"Cannot specify result_files_pattern and result_files_patterns "
"at the same time."
)
result_files_patterns = [result_files_pattern]
return self._submit(
run,
limits,
cpu_model,
required_files,
result_files_patterns,
meta_information,
priority,
user_pwd,
revision,
)
def _submit(
self,
run,
limits,
cpu_model,
required_files,
result_files_patterns,
meta_information,
priority,
user_pwd,
revision,
counter=0,
):
params = []
opened_files = [] # open file handles are passed to the request library
for programPath in run.sourcefiles:
norm_path = self._normalize_path_for_cloud(programPath)
params.append(
("programTextHash", (norm_path, self._get_sha256_hash(programPath)))
)
for required_file in get_files(required_files):
norm_path = self._normalize_path_for_cloud(required_file)
params.append(
("requiredFileHash", (norm_path, self._get_sha256_hash(required_file)))
)
params.append(("revision", revision or self._revision))
if run.propertyfile:
property_file = self._add_file_to_params(
params, "propertyText", run.propertyfile
)
opened_files.append(property_file)
if MEMLIMIT in limits:
params.append(("memoryLimitation", str(limits[MEMLIMIT])))
if TIMELIMIT in limits:
params.append(("timeLimitation", str(limits[TIMELIMIT])))
if SOFTTIMELIMIT in limits:
params.append(("softTimeLimitation", str(limits[SOFTTIMELIMIT])))
if CORELIMIT in limits:
params.append(("coreLimitation", str(limits[CORELIMIT])))
if cpu_model:
params.append(("cpuModel", cpu_model))
if result_files_patterns:
for pattern in result_files_patterns:
params.append(("resultFilesPattern", pattern))
else:
params.append(("resultFilesPattern", ""))
if priority:
params.append(("priority", priority))
(invalidOption, files) = self._handle_options(run, params, limits)
opened_files.extend(files)
if invalidOption:
raise WebClientError(
'Command {0} contains option "{1}" that is not usable with the webclient. '.format(
run.options, invalidOption
)
)
params.append(("groupId", str(self._group_id)))
if meta_information:
params.append(("metaInformation", meta_information))
# prepare request
headers = {"Accept": "text/plain"}
path = "runs/"
(response, statusCode) = self._request(
"POST",
path,
files=params,
headers=headers,
expectedStatusCodes=[200, 412],
user_pwd=user_pwd,
)
for opened_file in opened_files:
opened_file.close()
# program files or required files given as hash value are not known by the cloud system
if statusCode == 412:
if counter >= 1:
raise WebClientError(
"Files still missing on server for run {0} even after uploading them:\n{1}".format(
run.identifier, response
)
)
headers = {
"Content-Type": "application/octet-stream",
"Content-Encoding": "deflate",
}
filePath = "files/"
# upload all used program files
for programPath in run.sourcefiles:
with open(programPath, "rb") as programFile:
compressedProgramText = zlib.compress(programFile.read(), 9)
self._request(
"POST",
filePath,
data=compressedProgramText,
headers=headers,
expectedStatusCodes=[200, 204],
user_pwd=user_pwd,
)
# upload all required files
for required_file_path in required_files:
with open(required_file_path, "rb") as required_file:
compressed_required_file = zlib.compress(required_file.read(), 9)
self._request(
"POST",
filePath,
data=compressed_required_file,
headers=headers,
expectedStatusCodes=[200, 204],
user_pwd=user_pwd,
)
# retry submission of run
return self._submit(
run,
limits,
cpu_model,
required_files,
result_files_patterns,
meta_information,
priority,
user_pwd,
revision,
counter + 1,
)
else:
try:
run_id = response.decode("UTF-8")
except UnicodeDecodeError as e:
raise WebClientError(
"Malformed response from server while submitting run {0}:\n{1}".format(
run.identifier, response
)
) from e
if not VALID_RUN_ID.match(run_id):
raise WebClientError(
"Malformed response from server while submitting run {0}:\n{1}".format(
run.identifier, run_id
)
)
logging.debug("Submitted run with id %s", run_id)
return self._create_and_add_run_future(run_id)
def _handle_options(self, run, params, rlimits):
opened_files = []
config = None
# TODO use code from CPAchecker module, it add -stats and sets -timelimit,
# instead of doing it here manually, too
if self._tool_name == "CPAchecker":
params.append(("option", "statistics.print=true"))
if "softtimelimit" in rlimits:
params.append(
("option", "limits.time.cpu=" + str(rlimits["softtimelimit"]) + "s")
)
task_options = getattr(run, "task_options", None)
if isinstance(task_options, dict) and task_options.get("language") == "C":
data_model = task_options.get("data_model")
if data_model:
data_model_option = {"ILP32": "Linux32", "LP64": "Linux64"}.get(
data_model
)
if data_model_option:
params.append(
("option", "analysis.machineModel=" + data_model_option)
)
else:
raise WebClientError(
"Unsupported data_model '{}' defined for task '{}'".format(
data_model, run.identifier
)
)
if run.options:
i = iter(run.options)
disableAssertions = False
while True:
try:
option = next(i)
if len(option) == 0:
continue
if option == "-heap":
params.append(("heap", next(i)))
elif option == "-stack":
params.append(("stack", next(i)))
elif option == "-noout":
params.append(("option", "output.disable=true"))
elif option == "-outputpath":
params.append(("option", "output.path=" + next(i)))
elif option == "-logfile":
params.append(("option", "log.file=" + next(i)))
elif option == "-nolog":
params.append(("option", "log.level=OFF"))
params.append(("option", "log.consoleLevel=OFF"))
elif option == "-stats":
# ignore, is always set by this script
pass
elif option == "-disable-java-assertions":
disableAssertions = True
elif option == "-java":
params.append(("option", "language=JAVA"))
elif option == "-32":
params.append(("option", "analysis.machineModel=Linux32"))
elif option == "-64":
params.append(("option", "analysis.machineModel=Linux64"))
elif option == "-entryfunction":
params.append(("option", "analysis.entryFunction=" + next(i)))
elif option == "-timelimit":
params.append(("option", "limits.time.cpu=" + next(i)))
elif option == "-skipRecursion":
params.append(("option", "cpa.callstack.skipRecursion=true"))
params.append(("option", "analysis.summaryEdges=true"))
elif option == "-cbmc":
params.append(("option", "analysis.checkCounterexamples=true"))
params.append(("option", "counterexample.checker=CBMC"))
elif option == "-clang":
params.append(("option", "parser.useClang=true"))
elif option == "-preprocess":
params.append(("option", "parser.usePreprocessor=true"))
elif option == "-generateReport":
params.append(("generateReport", "true"))
elif option == "-sourcepath":
params.append(("option", "java.sourcepath=" + next(i)))
elif option in ["-cp", "-classpath"]:
params.append(("option", "java.classpath=" + next(i)))
elif option == "-spec":
spec_path = next(i)
spec_file = self._add_file_to_params(
params, "specificationText", spec_path
)
opened_files.append(spec_file)
elif option == "-config":
configPath = next(i)
tokens = configPath.split("/")
if tokens[0] == "config" and len(tokens) == 2:
config = tokens[1].split(".")[0]
params.append(("configuration", config))
else:
params.append(
("option", "configuration.file=" + configPath)
)
elif option == "-setprop":
params.append(("option", next(i)))
elif option == "-benchmark":
params.append(("option", "coverage.enabled=true"))
params.append(("option", "output.disable=true"))
params.append(("option", "statistics.memory=false"))
disableAssertions = True
elif option[0] == "-":
if config:
raise WebClientError(
"More than one configuration: '{}' and '{}'".format(
config, option[1:]
)
)
else:
params.append(("configuration", option[1:]))
config = option[1:]
else:
return (option, opened_files)
except StopIteration:
break
if disableAssertions:
params.append(("disableJavaAssertions", "true"))
return (None, opened_files)
def _add_file_to_params(self, params, name, path):
norm_path = self._normalize_path_for_cloud(path)
opened_file = open(path, "rb")
params.append((name, (norm_path, opened_file)))
return opened_file
def _normalize_path_for_cloud(self, path):
norm_path = os.path.normpath(path)
if ".." in norm_path or os.path.isabs(norm_path):
norm_path = os.path.basename(norm_path)
return norm_path.replace("\\", "/")
def flush_runs(self):
"""
Starts the execution of all previous submitted runs in the VerifierCloud.
The web interface groups runs and submits them to the VerifierCloud only from time to time.
This method forces the web interface to do this immediately and starts downloading of results.
@return: the ids of the RunCollections created since the last flush request
"""
headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Connection": "Keep-Alive",
}
params = {"groupId": self._group_id}
path = "runs/flush"
(response, _) = self._request(
"POST", path, data=params, headers=headers, expectedStatusCodes=[200, 204]
)
run_collections = response.decode("utf-8").split("\n")
if len(run_collections) == 0:
logging.warning(
"No runs were submitted to the VerifierCloud before or a rate limit is hit."
)
else:
logging.info(
"Submitted %s run collection: %s",
len(run_collections),
",".join(run_collections),
)
self._result_downloader.start()
return run_collections
def _is_finished(self, run_id):
headers = {"Accept": "text/plain"}
path = "runs/" + run_id + "/state"
try:
(state, _) = self._request("GET", path, headers=headers)
state = state.decode("utf-8")
if state == "FINISHED":
logging.debug("Run %s finished.", run_id)
if state == "UNKNOWN":
logging.debug(
"Run %s is not known by the webclient, trying to get the result.",
run_id,
)
return state
except requests.HTTPError as e:
logging.warning(
"Could not get state for run %s: %s\n%s",
run_id,
getattr(e, "reason", ""),
e.response.content or "",
)
return False
def _download_result(self, run_id):
# download result as zip file
headers = {"Accept": "application/zip"}
path = "runs/" + run_id + "/result"
(zip_content, _) = self._request("GET", path, headers=headers)
return zip_content
def _download_result_async(self, run_id):
def callback(downloaded_result):
run_id = self._downloading_result_futures.pop(downloaded_result)
exception = downloaded_result.exception()
if not exception:
with self._unfinished_runs_lock:
result_future = self._unfinished_runs.pop(run_id, None)
if result_future:
result_future.set_result(downloaded_result.result())
else:
attempts = self._download_attempts.pop(run_id, 1)
logging.info(
"Could not get result of run %s on attempt %d: %s",
run_id,
attempts,
exception,
)
if attempts < 10:
self._download_attempts[run_id] = attempts + 1
self._download_result_async(run_id)
else:
self._run_failed(run_id)
if run_id not in self._downloading_result_futures.values():
# result is not downloaded
future = self._executor.submit(self._download_result, run_id)
self._downloading_result_futures[future] = run_id
future.add_done_callback(callback)
def _run_failed(self, run_id):
run_result_future = self._unfinished_runs.pop(run_id, None)
def getErrorForRun(run_id):
headers = {"Accept": "text/plain"}
path = "runs/" + run_id + "/failurecause"
(error_msg, _) = self._request("GET", path, headers=headers)
return error_msg
if run_result_future:
error_msg = getErrorForRun(run_id)
logging.warning("Execution of run %s failed. Error(%s)", run_id, error_msg)
run_result_future.set_exception(
WebClientError("Execution failed. Error({})".format(error_msg))
)
def shutdown(self):
"""
Cancels all unfinished runs and stops all internal threads.
"""
self.active = False
self._result_downloader.shutdown()
if len(self._unfinished_runs) > 0:
logging.info("Stopping tasks on server...")
stop_executor = ThreadPoolExecutor(max_workers=5 * self.thread_count)
stop_tasks = set()
with self._unfinished_runs_lock:
for runId in self._unfinished_runs.keys():
stop_tasks.add(stop_executor.submit(self._stop_run, runId))
self._unfinished_runs[runId].set_exception(
UserAbortError(
"Run was canceled because user requested shutdown."
)
)
self._unfinished_runs.clear()
for task in stop_tasks:
task.result()
stop_executor.shutdown(wait=True)
logging.info("Stopped all tasks.")
self._write_hash_code_cache()
self._executor.shutdown(wait=True)
self._connection.close()
def _stop_run(self, run_id):
with self._unfinished_runs_lock:
self._unfinished_runs.pop(run_id, None)
path = "runs/" + run_id
try:
self._request("DELETE", path, expectedStatusCodes=[200, 204, 404])
except HTTPError as e:
reason = e.reason if hasattr(e, "reason") else "<unknown>"
content = (
e.response.content
if hasattr(e, "response") and hasattr(e.response, "content")
else "<unknown>"
)
logging.info(
"Stopping of run %s failed: %s\n%s\n%s",
run_id,
e,
reason,
content or "",
)
def _request(
self,
method,
path,
data=None,
headers=None,
files=None,
expectedStatusCodes=(200,),
user_pwd=None,
):
url = self._web_interface_url + path
if user_pwd:
auth = (user_pwd.split(":")[0], user_pwd.split(":")[1])
else:
auth = None
counter = 0
while counter < 5:
counter += 1
# send request
try:
response = self._connection.request(
method, url, data=data, files=files, headers=headers, auth=auth
)
except Exception as e:
if counter < 5:
logging.debug(
"Exception during %s request to %s: %s", method, path, e
)
sleep(1)
continue
else:
raise
if response.status_code in expectedStatusCodes:
return (response.content, response.status_code)
else:
if response.status_code == 401:
message = "Permission denied. Please check the URL given to --cloudMaster and specify credentials if necessary."
elif response.status_code == 404:
message = "Not found. Please check the URL given to --cloudMaster."
elif response.status_code == 503:
message = "Service Unavailable."
if counter < 5:
logging.debug(message)
sleep(60)
continue
else:
message = response.text
# HTTPError.request is automatically filled with response.request so no need to pass it.
# Also HTTPError extends IOError, so there is a constructor IOError(errno, strerror, filename)
raise requests.HTTPError(
response.status_code, message, path, response=response
)
class FtpServer:
limitasyon = 1000000
def __init__(self, port):
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.bind(('localhost', port))
self.clients = {}
self.client_cwd = {}
self.client_data = {}
self.executor = ThreadPoolExecutor(10)
self.init()
def init(self):
with open('password.txt') as file:
content = file.readlines()
lines = [line.strip() for line in content]
for line in lines:
parts = line.split(':', 2)
cwd = os.getcwd().replace('\\', '/')
self.clients[parts[0]] = parts[1]
self.client_cwd[parts[0]] = cwd
self.client_data[parts[0]] = []
def send(self, data, address):
self.sock.sendto(data, address)
def listen(self):
while True:
data, address = self.sock.recvfrom(self.limitasyon)
received_data = data.decode()
self.executor.submit(self.handle, received_data, address)
def handle(self, received_data, address):
ftp_message = json.loads(received_data)
method = ftp_message['method']
credentials = ftp_message['credentials']
data = ftp_message['data']
result = ftp_message['result']
if credentials['username'] not in self.clients:
response_data = create_response(ftp_message['session_id'], method,
404)
self.send(response_data, address)
return
password = self.clients[credentials['username']]
if password != credentials['password']:
response_data = create_response(ftp_message['session_id'], method,
403)
self.send(response_data, address)
return
client_cwd = self.client_cwd[credentials['username']]
if method == 'List':
file_data = ""
prefix = ""
files = [f for f in os.listdir(client_cwd)]
for f in files:
file_data += prefix
file_data += f
prefix = ";"
response_data = create_response(ftp_message['session_id'],
method,
200,
content=file_data)
self.send(response_data, address)
elif method == 'cd':
folder = data['header']
if folder == '.':
result_code = 200
elif folder == '..':
dirs = client_cwd.replace('\\', '/').split('/')
if len(dirs) > 2:
dirs.pop()
client_cwd = '/'.join(dirs)
self.client_cwd[credentials['username']] = client_cwd
result_code = 200
else:
result_code = 403
else:
dirs = client_cwd.replace('\\', '/').split('/')
dirs.append(folder)
cwd = '/'.join(dirs)
if os.path.isdir(cwd):
client_cwd = cwd
self.client_cwd[credentials['username']] = client_cwd
result_code = 200
else:
result_code = 404
response_data = create_response(ftp_message['session_id'],
method,
result_code,
header=folder)
self.send(response_data, address)
# Handle 'get' command : get the file from the server
elif method == 'Get':
file_name = data['header']
file_path = client_cwd + '/' + file_name
if os.path.isfile(file_path):
did = 1
with open(file_path, mode='rb') as file:
while True:
file_content = file.read(1024)
if not file_content:
break
b64_content = base64.b64encode(file_content).decode()
response_data = create_response(
ftp_message['session_id'],
method,
200,
data_id=did,
is_last=False,
header=file_name,
content=b64_content)
self.send(response_data, address)
did += 1
sleep(0.05)
response_data = create_response(ftp_message['session_id'],
method,
200,
data_id=did,
is_last=True,
header=file_name,
content="")
self.send(response_data, address)
else:
response_data = create_response(ftp_message['session_id'],
method,
404,
header=file_name)
self.send(response_data, address)
elif method == 'Put':
file_name = data['header']
did = data['data_id']
if did == 1:
self.client_data[credentials['username']] = []
if data['is_last']:
content = bytearray()
file_name = data['header']
file_path = client_cwd + '/' + file_name
for c in self.client_data[credentials['username']]:
content += c
write_file(file_path, content)
else:
self.client_data[credentials['username']].append(
base64.b64decode(data['content']))
response_data = create_response(ftp_message['session_id'],
method,
200,
data_id=data['data_id'],
header=file_name)
self.send(response_data, address)
# Not implemented command
else:
response_data = create_response(ftp_message['session_id'], method,
501)
self.send(response_data, address)
class SseResultDownloader:
def __init__(self, web_interface, result_poll_interval):
logging.debug("Server-Send Events are used to get state of runs.")
self._web_interface = web_interface
self._run_finished_url = web_interface._web_interface_url + "runs/finished"
self._result_poll_interval = result_poll_interval
self._sse_client = None
self._shutdown = False
self._new_runs = False
self._state_receive_executor = ThreadPoolExecutor(max_workers=1)
def _log_future_exception_and_fallback(self, result):
e = result.exception()
if e is not None:
if (
self._shutdown
and isinstance(e, AttributeError)
and str(e) == "'NoneType' object has no attribute 'read'"
):
# This is harmless, it occurs because SSEClient reads on closed connection.
logging.debug("Error during result processing:", exc_info=True)
else:
logging.warning("Error during result processing:", exc_info=True)
if not self._shutdown:
self._fall_back()
def _should_reconnect(self, error):
if self._new_runs:
return False
elif (
type(error) == HTTPError
and error.response is not None
and error.response.status >= 400
and error.response.status < 500
):
logging.debug("Exception in SSE connection: %s", error)
return False
else:
return True
def _start_sse_connection(self):
while self._new_runs:
run_ids = set(self._web_interface._unfinished_runs.keys())
self._new_runs = False
# nothing to do
if len(run_ids) == 0:
return
params = []
for run_id in run_ids:
params.append(("run", run_id))
headers = {"Accept-Encoding": "UTF-8"}
logging.debug("Creating Server-Send Event connection.")
try:
self._sse_client = ShouldReconnectSeeClient(
self._run_finished_url,
self._should_reconnect,
session=self._web_interface._connection,
headers=headers,
data=params,
)
except Exception as e:
logging.warning("Creating SSE connection failed: %s", e)
self._fall_back()
return
for message in self._sse_client:
if self._shutdown:
self._sse_client.resp.close()
break
data = message.data
tokens = data.split(" ")
if len(tokens) == 2:
run_id = tokens[0]
state = tokens[1]
if state == "FINISHED":
if run_id in run_ids:
logging.debug("Run %s finished.", run_id)
self._web_interface._download_result_async(run_id)
elif state == "UNKNOWN":
logging.debug(
"Run %s is not known by the webclient, trying to get the result.",
run_id,
)
self._web_interface._download_async(run_id)
elif state == "ERROR":
self._web_interface._run_failed(run_id)
else:
logging.warning(
"Received unknown run state %s for run %s.",
state,
run_id,
)
run_ids.discard(run_id)
if self._shutdown or self._new_runs or len(run_ids) == 0:
break
else:
logging.warning("Received invalid message %s", data)
self._sse_client = None
# fall back to polling if Server-Send Event based approach failed
if len(run_ids) != 0 and not self._shutdown:
self._fall_back()
def _fall_back(self):
logging.info("Fall back to polling.")
self._web_interface._result_downloader = PollingResultDownloader(
self._web_interface, self._result_poll_interval
)
self._web_interface._result_downloader.start()
self.shutdown(wait=False)
def start(self):
self._new_runs = True
if self._sse_client:
self._sse_client.resp.close()
else:
future = self._state_receive_executor.submit(self._start_sse_connection)
future.add_done_callback(self._log_future_exception_and_fallback)
def shutdown(self, wait=True):
self._shutdown = True
self._state_receive_executor.shutdown(wait=wait)
class PipelineModel:
"""Controls IO (camera, video file, recording, saving frames). Methods run
in worker threads."""
def __init__(self,
update_view,
camera_config_file=None,
rgbd_video=None,
device=None):
"""Initialize.
Args:
update_view (callback): Callback to update display elements for a
frame.
camera_config_file (str): Camera configuration json file.
rgbd_video (str): RS bag file containing the RGBD video. If this is
provided, connected cameras are ignored.
device (str): Compute device (e.g.: 'cpu:0' or 'cuda:0').
"""
self.update_view = update_view
if device:
self.device = device.lower()
else:
self.device = 'cuda:0' if o3d.core.cuda.is_available() else 'cpu:0'
self.o3d_device = o3d.core.Device(self.device)
self.video = None
self.camera = None
self.flag_capture = False
self.cv_capture = threading.Condition() # condition variable
self.recording = False # Are we currently recording
self.flag_record = False # Request to start/stop recording
if rgbd_video: # Video file
self.video = o3d.t.io.RGBDVideoReader.create(rgbd_video)
self.rgbd_metadata = self.video.metadata
self.status_message = f"Video {rgbd_video} opened."
else: # RGBD camera
now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
filename = f"{now}.bag"
self.camera = o3d.t.io.RealSenseSensor()
if camera_config_file:
with open(camera_config_file) as ccf:
self.camera.init_sensor(o3d.t.io.RealSenseSensorConfig(
json.load(ccf)),
filename=filename)
else:
self.camera.init_sensor(filename=filename)
self.camera.start_capture(start_record=False)
self.rgbd_metadata = self.camera.get_metadata()
self.status_message = f"Camera {self.rgbd_metadata.serial_number} opened."
log.info(self.rgbd_metadata)
# RGBD -> PCD
self.extrinsics = o3d.core.Tensor.eye(4,
dtype=o3d.core.Dtype.Float32,
device=self.o3d_device)
self.intrinsic_matrix = o3d.core.Tensor(
self.rgbd_metadata.intrinsics.intrinsic_matrix,
dtype=o3d.core.Dtype.Float32,
device=self.o3d_device)
self.depth_max = 3.0 # m
self.pcd_stride = 2 # downsample point cloud, may increase frame rate
self.flag_normals = False
self.flag_save_rgbd = False
self.flag_save_pcd = False
self.pcd_frame = None
self.rgbd_frame = None
self.executor = ThreadPoolExecutor(max_workers=3,
thread_name_prefix='Capture-Save')
self.flag_exit = False
@property
def max_points(self):
"""Max points in one frame for the camera or RGBD video resolution."""
return self.rgbd_metadata.width * self.rgbd_metadata.height
@property
def vfov(self):
"""Camera or RGBD video vertical field of view."""
return np.rad2deg(2 * np.arctan(self.intrinsic_matrix[1, 2].item() /
self.intrinsic_matrix[1, 1].item()))
def run(self):
"""Run pipeline."""
n_pts = 0
frame_id = 0
t1 = time.perf_counter()
if self.video:
self.rgbd_frame = self.video.next_frame()
else:
self.rgbd_frame = self.camera.capture_frame(
wait=True, align_depth_to_color=True)
pcd_errors = 0
while (not self.flag_exit
and (self.video is None or # Camera
(self.video and not self.video.is_eof()))): # Video
if self.video:
future_rgbd_frame = self.executor.submit(self.video.next_frame)
else:
future_rgbd_frame = self.executor.submit(
self.camera.capture_frame,
wait=True,
align_depth_to_color=True)
if self.flag_save_pcd:
self.save_pcd()
self.flag_save_pcd = False
try:
self.rgbd_frame = self.rgbd_frame.to(self.o3d_device)
self.pcd_frame = o3d.t.geometry.PointCloud.create_from_rgbd_image(
self.rgbd_frame, self.intrinsic_matrix, self.extrinsics,
self.rgbd_metadata.depth_scale, self.depth_max,
self.pcd_stride, self.flag_normals)
depth_in_color = self.rgbd_frame.depth.colorize_depth(
self.rgbd_metadata.depth_scale, 0, self.depth_max)
except RuntimeError:
pcd_errors += 1
if self.pcd_frame.is_empty():
log.warning(f"No valid depth data in frame {frame_id})")
continue
n_pts += self.pcd_frame.point['positions'].shape[0]
if frame_id % 60 == 0 and frame_id > 0:
t0, t1 = t1, time.perf_counter()
log.debug(
f"\nframe_id = {frame_id}, \t {(t1-t0)*1000./60:0.2f}"
f"ms/frame \t {(t1-t0)*1e9/n_pts} ms/Mp\t")
n_pts = 0
frame_elements = {
'color': self.rgbd_frame.color.cpu(),
'depth': depth_in_color.cpu(),
'pcd': self.pcd_frame.cpu(),
'status_message': self.status_message
}
self.update_view(frame_elements)
if self.flag_save_rgbd:
self.save_rgbd()
self.flag_save_rgbd = False
self.rgbd_frame = future_rgbd_frame.result()
with self.cv_capture: # Wait for capture to be enabled
self.cv_capture.wait_for(
predicate=lambda: self.flag_capture or self.flag_exit)
self.toggle_record()
frame_id += 1
if self.camera:
self.camera.stop_capture()
else:
self.video.close()
self.executor.shutdown()
log.debug(f"create_from_depth_image() errors = {pcd_errors}")
def toggle_record(self):
if self.camera is not None:
if self.flag_record and not self.recording:
self.camera.resume_record()
self.recording = True
elif not self.flag_record and self.recording:
self.camera.pause_record()
self.recording = False
def save_pcd(self):
"""Save current point cloud."""
now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
filename = f"{self.rgbd_metadata.serial_number}_pcd_{now}.ply"
# Convert colors to uint8 for compatibility
self.pcd_frame.point['colors'] = (self.pcd_frame.point['colors'] *
255).to(o3d.core.Dtype.UInt8)
self.executor.submit(o3d.t.io.write_point_cloud,
filename,
self.pcd_frame,
write_ascii=False,
compressed=True,
print_progress=False)
self.status_message = f"Saving point cloud to {filename}."
def save_rgbd(self):
"""Save current RGBD image pair."""
now = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
filename = f"{self.rgbd_metadata.serial_number}_color_{now}.jpg"
self.executor.submit(o3d.t.io.write_image, filename,
self.rgbd_frame.color)
filename = f"{self.rgbd_metadata.serial_number}_depth_{now}.png"
self.executor.submit(o3d.t.io.write_image, filename,
self.rgbd_frame.depth)
self.status_message = (
f"Saving RGBD images to {filename[:-3]}.{{jpg,png}}.")
class ma10CrossMa20Pair(BasePair):
def __init__(self, coin_type, time_type,binance,stop_loss):
self.coin_type = coin_type
self.time_type = time_type
self.threadpool = ThreadPoolExecutor(max_workers=1)
self.strategy_type = config.MaCrossAverage
self.stop_loss = stop_loss
self.binance = binance
#市场竞价-操作
def __market_broker(self,buy_flag, amount, coin_type,rate_close,rate_average,average_line):
if(buy_flag):
data=self.binance.get_order_book(symbol=coin_type,limit=5)
bidPrice=data["bids"][0][0]
two_str1=data["bids"][0][0][8:]
two_str2=data["bids"][1][0][8:]
bool_flag=re.match("00", two_str1) and re.match("00", two_str2)
bidPrice=float(bidPrice)*(1+0.005)
if(bool_flag != None):
bidPrice=round(bidPrice,6)
else:
bidPrice=round(bidPrice,8)
self.buy(bidPrice,coin_type,rate_close,rate_average,average_line,amount=amount)
else:
data=self.binance.get_order_book(symbol=coin_type,limit=5)
askPrice=data["asks"][0][0]
two_str1=data["asks"][0][0][8:]
two_str2=data["asks"][1][0][8:]
bool_flag=re.match("00", two_str1) and re.match("00", two_str2)
askPrice=float(askPrice)*(1-0.005)
if(bool_flag != None):
askPrice=round(askPrice,6)
else:
askPrice=round(askPrice,8)
self.sell(askPrice,average_line,amount=amount,coin_type=coin_type)
def get_opportunity_ma10CrossMa20(self,data):
#不顾一切
ma10,ma20,ma60=self.get_maLiner(data)
jincha=CROSS(ma10,ma20)
sicha=CROSS(ma20,ma10)
jincha_flag=jincha.series[-1] and ma60.series[-1]>ma60.series[-2]
sicha_flag=sicha.series[-1]
if(jincha_flag
):
futures = []
futures.append(self.threadpool.submit(self.__market_broker,jincha_flag,
config.min_tx_volume, self.coin_type,rate_close=0,rate_average=0,average_line=0))
elif(sicha_flag):
futures = []
futures.append(self.threadpool.submit(self.__market_broker,jincha_flag,
config.min_tx_volume, self.coin_type,rate_close=0,rate_average=0,average_line=0))
def sell(self, askPirce,average_line,coin_type, amount=config.min_tx_volume):
#对数据库进行更新操作
# 创建session对象:
session = DBSession()
#查询有没有买单的
#时间
update_date=time.strftime('%Y-%m-%d %H:%M:%S')
order_result = session.query(Order).filter(Order.type == 0).filter(Order.coin_type == coin_type). \
filter(Order.end_date < update_date).filter(Order.time_type == self.time_type). \
filter(Order.strategy_type == self.strategy_type)
count=order_result.count()
if(count>0):
#说明有成交的买单，那么就可以进行卖出操作了
#更新数据库
order=order_result.one()
order.sell_price = askPirce
order.buy_price=float(order.buy_price)
order.profit_rate = askPirce/order.buy_price -1
order.sell_fee = (askPirce * amount)*0.001
order.profit = (float(order.sell_price)-float(order.buy_price))*(float(order.amount))- \
(float(order.buy_fee) + float(order.sell_fee))
order.type = 1
order.ask_order_id = "test_ask_004"
# 添加到session:
message="%s:%s执行下单操作成功，卖价为%.8f,数量为%.2f"%(self.coin_type,self.time_type,askPirce,amount)
logger.warn(message)
if(order.profit_rate>self.stop_loss and order.profit_rate<0):
#亏损在0-1%中间就不要动
# 关闭session:
session.close()
return
#亏损超过1%，卖出
# 提交即保存到数据库:
if(config.market_game_start == 1):
for pair_map in config.binance_game_coin_pairs:
if(pair_map["symbol"] == self.coin_type):
#实体竞技
try:
balance_data_list=self.binance.get_account()["balances"]
asset=self.coin_type.replace("BTC","")
for value_map in balance_data_list:
if(value_map["asset"]==asset):
free=value_map["free"]
free=float(free)
amount=min(free,amount)
break
result=self.binance.order_limit_sell(symbol=self.coin_type, quantity=amount, price=askPirce )
order.ask_order_id = result["orderId"]
message="下单 %s"%(order.ask_order_id)
logger.warn(message)
except Exception as e:
traceback.print_exc()
logger.fatal(e)
session.commit()
# 关闭session:
session.close()
def double_cross(self, average_line,average_data,bool_data):
bool_data = NumericSeries(bool_data)
ref_bool_data = REF(bool_data,1)
select_bool = bool_data >ref_bool_data
select_bool=select_bool.series[99-20:]
ret=np.where(select_bool==True)
size=ret[0].size
if(size<2):
return False
average_line=average_line[100-20:]
first_one_position = ret[0][-1]
first_one = average_line[first_one_position]
second_one_position = ret[0][-2]
second_one = average_line[second_one_position]
if(first_one>second_one and select_bool[-1]==True):
return True
return False
def get_maLiner(self, data):
X = np.array(data)
my_close_list=np.array(X[:,4],dtype=np.float)
my_close_list = pd.Series(my_close_list)
close_data=np.array(my_close_list)
close_data = NumericSeries(close_data)
ma10=MA(close_data,10)
ma20=MA(close_data,20)
ma60=MA(close_data,60)
return ma10,ma20,ma60
class NewDataHolder(object):
def __init__(self):
self.__data = {}
self.__ignore = []
self.__buyed = []
self.__selled = []
self.__setting = Config()
self.__tpe = ThreadPoolExecutor(5)
self.__engine = create_engine(self.__setting.get_DBurl())
def get_buyed(self):
return self.__buyed
def add_buyed(self, code):
self.__buyed.append(code)
self.__tpe.submit(self.saveData, self.__data[code].get_data())
def get_selled(self):
return self.__selled
def add_selled(self, code, save=False):
self.__selled.append(code)
if save:
self.__tpe.submit(self.saveData, self.__data[code].get_data())
def get_ignore(self):
return self.__ignore
def add_ignore(self, code, save=False):
self.__ignore.append(code)
if save:
self.__tpe.submit(self.saveData, self.__data[code].get_data())
def get_data(self):
return self.__data
def addDataHandler(self, row):
code = row['code']
if code in self.get_buyed() or code in self.get_ignore():
return
if code in self.__data and self.__data[code].len() > 0:
self.__data[code].add_Line(row)
else:
self.__data[code] = NewStock(code, row)
if float(row['pre_close']) != 0 and (float(row['price']) - float(
row['pre_close'])) / float(row['pre_close']) * 100 >= 9.9:
MyLog.info('[%s] reach 10' % code)
save = False
if float(row['price']) != float(row['open']):
save = True
self.add_ignore(code, save=save)
def addSellDataHandler(self, row):
code = row['code']
if code in self.get_selled():
return
if code in self.__data and self.__data[code].len() > 0:
self.__data[code].add_Line(row)
else:
self.__data[code] = NewStock(code, row)
def addData(self, df):
df.apply(self.addDataHandler, axis=1)
def addSellData(self, df):
df.apply(self.addSellDataHandler, axis=1)
def saveData(self, data):
try:
line = data[0]
code = line['code']
df = pd.DataFrame(data)
df.to_sql('live_' + code,
con=self.__engine,
if_exists='replace',
index=False)
MyLog.info('[%s] save data' % code)
except Exception as e:
MyLog.error('[%s %s] save [%s] data error \n' %
(line['date'], line['time'], code))
MyLog.error(str(e) + '\n')
"//div[@class='user-tab active tab get-list']//i/text()")))
user_info['liked_num'] = int(''.join(
share_douyin_content.xpath(
"//div[@class='like-tab tab get-list']//i/text()")))
print(user_info)
insert_mysql(user_info)
# 处理抖音用户信息的主方法
def handle_douyin_web_share(user_id):
share_douyin_url = 'https://www.douyin.com/share/user/{0}'.format(user_id)
share_web_header = {'User-Agent': UserAgent().chrome}
share_douyin_response = requests.get(url=share_douyin_url,
headers=share_web_header)
share_douyin_content = handle_font_decode(share_douyin_response.text)
item_insert(share_douyin_content)
# 创建队列，利用多线程请求数据库的抖音ID
queue_list = Queue()
pool = ThreadPoolExecutor(max_workers=20)
for i in range(100):
for j in range(100):
share_id = get_random_id()['share_id']
queue_list.put(share_id)
while queue_list.qsize() > 0:
try:
pool.submit(handle_douyin_web_share, queue_list.get())
except Exception as e:
print(e.args)
class QA_Tdx_Executor():
def __init__(self, thread_num=2, *args, **kwargs):
self.thread_num = thread_num
self._queue = queue.Queue(maxsize=200)
self.api_no_connection = TdxHq_API()
self._api_worker = Thread(target=self.api_worker,
args=(),
name='API Worker')
self._api_worker.start()
self.executor = ThreadPoolExecutor(self.thread_num)
def __getattr__(self, item):
try:
api = self.get_available()
func = api.__getattribute__(item)
def wrapper(*args, **kwargs):
res = self.executor.submit(func, *args, **kwargs)
self._queue.put(api)
return res
return wrapper
except:
return self.__getattr__(item)
def _queue_clean(self):
self._queue = queue.Queue(maxsize=200)
def _test_speed(self, ip, port=7709):
api = TdxHq_API(raise_exception=True, auto_retry=False)
#api.need_setup = False
_time = datetime.datetime.now()
try:
with api.connect(ip, port, time_out=0.05):
if len(api.get_security_list(0, 1)) > 800:
return (datetime.datetime.now() - _time).total_seconds()
else:
return datetime.timedelta(9, 9, 0).total_seconds()
except Exception as e:
#print('BAD IP {}, DEL for Reason{}'.format(ip,e))
return datetime.timedelta(9, 9, 0).total_seconds()
def get_market(self, code):
code = str(code)
if code[0] in ['5', '6', '9'] or code[:3] in [
"009", "126", "110", "201", "202", "203", "204"
]:
return 1
return 0
def get_level(self, level):
if level in ['day', 'd', 'D', 'DAY', 'Day']:
level = 9
elif level in ['w', 'W', 'Week', 'week']:
level = 5
elif level in ['month', 'M', 'm', 'Month']:
level = 6
elif level in ['Q', 'Quarter', 'q']:
level = 10
elif level in ['y', 'Y', 'year', 'Year']:
level = 11
elif str(level) in ['5', '5m', '5min', 'five']:
level = 0
elif str(level) in ['1', '1m', '1min', 'one']:
level = 8
elif str(level) in ['15', '15m', '15min', 'fifteen']:
level = 1
elif str(level) in ['30', '30m', '30min', 'half']:
level = 2
elif str(level) in ['60', '60m', '60min', '1h']:
level = 3
return level
@property
def ipsize(self):
return len(self._queue.qsize())
@property
def api(self):
return self.get_available()
def get_available(self):
if self._queue.empty() is False:
return self._queue.get_nowait()
else:
Timer(0, self.api_worker).start()
return self._queue.get()
def api_worker(self):
data = []
if self._queue.qsize() < 80:
for item in info_ip_list:
_sec = self._test_speed(item)
if _sec < 0.1:
self._queue.put(
TdxHq_API(heartbeat=False).connect(ip=item,
time_out=0.05))
else:
self._queue_clean()
Timer(0, self.api_worker).start()
Timer(300, self.api_worker).start()
def _singal_job(self, context, id_, time_out=0.5):
try:
_api = self.get_available()
__data = context.append(
self.api_no_connection.to_df(
_api.get_security_quotes([
(self._select_market_code(x), x)
for x in code[80 * id_:80 * (id_ + 1)]
])))
__data['datetime'] = datetime.datetime.now()
self._queue.put(_api) # 加入注销
return __data
except:
return self.singal_job(context, id_)
def get_realtime(self, code):
context = pd.DataFrame()
code = [code] if type(code) is str else code
try:
for id_ in range(int(len(code) / 80) + 1):
context = self._singal_job(context, id_)
data = context[[
'datetime', 'code', 'open', 'high', 'low', 'price', 'ask1',
'ask_vol1', 'ask2', 'ask_vol2', 'ask3', 'ask_vol3', 'ask4',
'ask_vol4', 'ask5', 'ask_vol5'
]]
return data.set_index('code', drop=False, inplace=False)
except:
return None
def get_realtime_concurrent(self, code):
code = [code] if type(code) is str else code
try:
# for id_ in range(int(len(code) / 80) + 1):
data = {
self.get_security_quotes([
(self.get_market(x), x)
for x in code[80 * pos:80 * (pos + 1)]
])
for pos in range(int(len(code) / 80) + 1)
}
return (pd.concat([
self.api_no_connection.to_df(i.result()) for i in data
]), datetime.datetime.now())
except:
pass
def get_security_bar_concurrent(self, code, _type, lens):
#code = [code] if type(code) is str else code
try:
#[api.get_security_bars(level, __select_market_code(str(code)), str(code), (25 - i) * 800, 800) for i in range(26)]
data = {[
self.get_security_bars(self.get_level(_type),
self.get_market(str(code)), str(code),
(25 - i) * 800, 800)
for i in range(int(lens / 800) + 1)
]}
print([i.result() for i in data])
except:
raise Exception
def _get_security_bars(self, context, code, _type, lens):
try:
_api = self.get_available()
for i in range(1, int(lens / 800) + 2):
context.extend(
_api.get_security_bars(self.get_level(_type),
self.get_market(str(code)),
str(code), (i - 1) * 800, 800))
#print(context)
self._queue.put(_api)
return context
except Exception as e:
#print(e)
return self._get_security_bars(context, code, _type, lens)
def get_security_bars(self, code, _type, lens):
code = [code] if type(code) is str else code
context = []
try:
for item in code:
context = self._get_security_bars(context, item, _type, lens)
return context
except Exception as e:
raise e
def save_mongo(self):
pass
fakeua = {}
fakeua['user-agent'] = random.choice(USER_AGENTS)
try:
sp = bs(
rq.get('https://e-hentai.org/favorites.php',
cookies=cookies,
headers=fakeua).text, 'html.parser')
except AttributeError:
print("请检查cookie是否配置正确、ip是否被ban")
# 以下逻辑仅为获取收藏数，如果发现获取失败，但确认cookies配置正确，可以手动删掉以下逻辑，配置 pagenum=[你的收藏数]
favornum = int((sp.find(attrs={
'name': 'favform'
}).p.string.split(' ')[1]).replace(',', '').replace(' ', ''))
pagenum = favornum // 50 + 1
urlList = ['https://e-hentai.org/favorites.php']
for i in range(1, pagenum):
urlList.append('https://e-hentai.org/favorites.php?page=' + str(i))
# 此处三行表示使用大小为8的线程池并行导出
# 如果发现没有反应（即无法创建线程），可以使用串行版本如下：
# for url in urlList:
# getfavor(url)
pool = ThreadPoolExecutor(8)
for url in urlList:
pool.submit(getfavor, url)
input('task running...\n')
from concurrent.futures import ThreadPoolExecutor
import time
from log import log
log(globals())
def wait_on_b():
log('b function')
log('b-->', globals())
time.sleep(5)
log(b.result())
return 5
def wait_on_a():
log('a function')
log('a-->', globals())
time.sleep(5)
log(a.result())
return 6
executor = ThreadPoolExecutor(max_workers=2)
a = executor.submit(wait_on_b)
b = executor.submit(wait_on_a)
class ZTO(object):
def __init__(self):
self.dh_list = [] # 保存单号的列表
self.task = None # 线程
self.data_info = [] # 保存获取到的信息，方便导出
self.data_erro = [] # 保存未成功获取的单号
self.pool = ThreadPoolExecutor(10)
self.init_gui()
def init_gui(self):
self.root = tkinter.Tk()
self.root_width = 500 # 窗口款多
self.root_height = 480 # 窗口高度
self.init_root_wind() # 初始化主窗口信息
header_frame = tkinter.Frame(self.root)
header_frame.pack(fill=tkinter.X, pady=20, padx=20)
header_frame_2 = tkinter.Frame(self.root)
header_frame_2.pack(fill=tkinter.X, pady=12, padx=20)
header_frame_3 = tkinter.Frame(self.root)
header_frame_3.pack(fill=tkinter.X, pady=12, padx=50)
# zto_session
self.zto_session = tkinter.StringVar()
tkinter.Label(header_frame, text='zto_session:').pack(side=tkinter.LEFT)
self.zto_session_input = tkinter.Entry(header_frame, textvariable=self.zto_session, width=40)
self.zto_session_input.pack(padx=12)
# session
tkinter.Label(header_frame_2, text=' session:').pack(side=tkinter.LEFT)
self.session = tkinter.StringVar()
self.session_input = tkinter.Entry(header_frame_2, textvariable=self.session, width=40)
self.session_input.pack(padx=12)
# 导入单号按钮， 短信接口获取， 查询接口获取
# 导入单号按钮
self.import_excel_btn = tkinter.Button(header_frame_3, text='导入单号', command=self.import_excel)
self.import_excel_btn.pack(side=tkinter.LEFT, padx=12)
self.select_api = tkinter.StringVar()
tkinter.Radiobutton(header_frame_3, variable=self.select_api, text='短信接口获取', value=0).pack(side=tkinter.LEFT)
tkinter.Radiobutton(header_frame_3, variable=self.select_api, text='查询接口获取', value=1).pack(side=tkinter.LEFT)
self.select_api.set(0)
# 开始检测按钮
self.start_btn = tkinter.Button(header_frame_3, text='开始获取', command=self.start_get)
self.start_btn.pack(side=tkinter.LEFT, padx=12)
# 展示信息的表格
# 商品表格的frame
self.goods_Frame = tkinter.Frame(self.root)
self.goods_Frame.pack(fill=tkinter.X)
# 定义中心列表区域
self.info = ttk.Treeview(self.goods_Frame, show="headings", columns=('单号', '联系方式'))
self.vbar = ttk.Scrollbar(self.goods_Frame, orient=tkinter.VERTICAL, command=self.info.yview)
self.info.configure(yscrollcommand=self.vbar.set)
self.info.column("单号", anchor="center", width=280)
self.info.column("联系方式", anchor="center", width=200)
self.info.heading("单号", text="单号")
self.info.heading("联系方式", text="联系方式")
self.info.grid(row=0, column=0, sticky=tkinter.NSEW)
self.vbar.grid(row=0, column=1, sticky=tkinter.NS)
# 导出信息按钮
self.export_excel_btn = tkinter.Button(self.root, text='导出信息', width=40, command=self.export_info)
self.export_excel_btn.pack(pady=10)
self.root.mainloop()
# 初始化主窗口
def init_root_wind(self):
self.root.title('中天系统v2.2')
self.center_window() # 设置窗口剧中
self.root.resizable(width=False, height=False) # 禁止窗口拉伸
# 设置窗口剧中
def center_window(self):
"""
设置窗口剧中
:param width: 窗口的宽度
:param height: 窗口的高度
:return: 自动设置窗口剧中
"""
screenwidth = self.root.winfo_screenwidth()
screenheight = self.root.winfo_screenheight()
size = '%dx%d+%d+%d' % (
self.root_width, self.root_height, (screenwidth - self.root_width) / 2,
(screenheight - self.root_height) / 2)
self.root.geometry(size)
# 导入excel表格
def import_excel(self):
excel_file = askopenfilename()
if excel_file:
if not os.path.isfile(excel_file):
messagebox.showinfo('导入失败', '导入失败当前文件不存在！')
return
if not excel_file.endswith('xlsx'):
messagebox.showinfo('导入失败', '导入文件应该是"xlsx"文件!')
return
self.dh_list = []
workbook = openpyxl.load_workbook(excel_file)
worksheet = workbook.active
for row in worksheet.rows:
value = row[0].value
if value:
self.dh_list.append(value)
# temp_list = [str(row[0].value).strip() for row in worksheet.rows if str(row[0].value)]
if self.dh_list:
messagebox.showinfo('导入成功', '导入成功，当前导入单号数量: %s' % len(self.dh_list))
print(self.dh_list)
return
else:
messagebox.showinfo('导入失败', '当前文件没有数据！')
return
# 开始获取每个单号联系方式的按钮事件
def start_get(self):
# 判断单号是否导入
if not self.dh_list:
messagebox.showerror('Erro', '请先导入单号！')
return
# 判断session是否输入
if not self.zto_session.get() or not self.session.get():
tkinter.messagebox.showerror('ERRO', 'zto_session 或 session未输入!')
return
# 判断是否有任务在执行
if self.task:
if self.task.isAlive():
tkinter.messagebox.showerror('ERRO', '当前有任务正在进行，请等待任务结束')
return
if messagebox.askyesno('开始？', '确定开始获取？'):
self.task = Thread(target=self.inner_task)
self.task.start()
# 线程内的函数
def inner_task(self):
try:
for _ in map(self.info.delete, self.info.get_children("")):
pass
# export_dh_list = [self.dh_list[i:i + 500] for i in range(0, len(self.dh_list), 500)]
export_dh_list = self.dh_list
self.data_info = []
self.data_erro = []
zto_session = self.zto_session.get().strip()
session = self.session.get().strip()
func = self.get_info_msg
if int(self.select_api.get()) == 1:
func = self.get_info
result = [self.pool.submit(func, zto_session, session, dh) for dh in export_dh_list]
wait(result)
messagebox.showinfo('成功', '全部单号获取完毕，请导出查看')
except Exception as e:
messagebox.showerror('出现BUG', 'BUG信息:%s' % e)
self.data_info = []
self.data_erro = []
# 导出信息按钮事件
def export_info(self):
try:
t = time.strftime('%Y-%m-%d-%H%M%S', time.localtime())
diretory = askdirectory()
if diretory:
success_file = os.path.join(diretory, '%s-success.xlsx' % t)
workbook = openpyxl.Workbook()
worksheet = workbook.active
for dd in self.data_info:
worksheet.append(dd)
workbook.save(success_file)
erro_file = os.path.join(diretory, '%s-erro.xlsx' % t)
erro_workbook = openpyxl.Workbook()
erro_sheet = erro_workbook.active
for dd in self.data_erro:
erro_sheet.append(dd)
erro_workbook.save(erro_file)
messagebox.showinfo('导出成功', '成功：%s\n失败：%s' % (success_file, erro_file))
except Exception as e:
messagebox.showerror('BUG', '导出文件出现BUG:%s' % e)
# 程序核心, 获取指定单号的联系人
def get_info(self, zto_session, session, dh):
headers = {
'X-Canvas-Fingerprint': 'a5830a699863251ddd8fb1c4af8e336v',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'
}
cookies = {
'com.zto.sessionid': zto_session,
'SESSION': session
}
try:
response = requests.get(
url='https://sso.zto.com/security-services/billtrack/billinfo-query-preauth?bill_id=%s&type=A014' % dh,
headers=headers, cookies=cookies, timeout=10)
time.sleep(0.1)
print(response.text)
# {"error":"no_perm_query:not_scan_node","error_description":"运单不属于贵网点","message":"该运单未在贵网点进行到件、发件、中转扫描或不是发放给贵网点使用的，不能进行「订单信息」查询操作。\n如您是客服中心或服务网点用户，请向您的主管确认您所在的网点具有此运单流经路由网点的查询权限。"}
# {"ticket":"4He3WNFTEeiX3QBQVoRltw"}
# {"error":"you_need_login_first","error_description":"需要登录","message":"您需要登录才可以继续此操作。"}
data = json.loads(response.text)
if data.get('error', None):
info = data.get('error_description')
self.data_erro.append([dh, info])
self.info.insert('', 0, values=(dh, info))
return dh, info
headers_2 = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36',
'_type': 'A014',
'_token': data.get('ticket', '')
}
try:
response = requests.get(url='https://newbill.zt-express.com/order-query/get?billCode=%s' % dh,
headers=headers_2, cookies=cookies, timeout=10)
time.sleep(0.1)
print(dh, response.text)
except TooManyRedirects as e:
self.data_erro.append([dh, 'SESSION失效'])
self.info.insert('', 0, values=(dh, 'SESSION失效'))
return dh, 'SESSION失效'
data2 = json.loads(response.text)
if data2['status']:
if data2.get('result', None):
ii = data2['result'][0]['receiveInfo']
if ii == "R11:没有获取到收件人信息":
self.data_erro.append([dh, ii])
self.info.insert('', 0, values=(dh, ii))
else:
self.info.insert('', 0, values=(dh, ii))
self.data_info.append([dh, ii])
return dh, ii
else:
self.data_erro.append([dh, '暂无数据'])
self.info.insert('', 0, values=(dh, '暂无数据'))
return dh, '暂无数据'
else:
self.data_erro.append([dh, response.text])
self.info.insert('', 0, values=(dh, response.text))
return dh, data2
except Exception as e:
info = '查询接口获取单号:%s出错:%s\n' % e
self.data_erro.append([dh, info])
return dh, str(e)
# 短信接口获取联系方式
def get_info_msg(self, zto_session, session, dh):
try:
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'
}
data = {"billCode": dh}
cookies = {
'com.zto.sessionid': zto_session,
'SESSION': session
}
try:
time.sleep(0.1)
response = requests.post(url='https://sms.zto.com/mobile/billcode', json=data, cookies=cookies, timeout=10, headers=headers)
except TooManyRedirects as e:
self.data_erro.append([dh, 'SESSION失效'])
self.info.insert('', 0, values=(dh, 'SESSION失效'))
return dh, 'SESSION失效'
info = json.loads(response.text)
print(dh, info)
# {"resultData":null,"requestStatus":false,"StatusMessage":"无法正确获取手机号"}
# {"resultData":{"RecMobile":"13916437644","RecName":""},"requestStatus":true,"StatusMessage":"成功获取"}
if info.get('requestStatus'):
ii = info['resultData']['RecMobile']
self.data_info.append([dh, ii])
self.info.insert('', 0, values=(dh, ii))
return dh, ii
else:
self.data_erro.append([dh, '无法正确获取手机号'])
self.info.insert('', 0, values=(dh, '无法正确获取手机号'))
return dh, '无法正确获取手机号'
except Exception as e:
info = '短信接口获取单号:%s时候出错%s\n' % (dh, e)
self.data_erro.append([dh, info])
return dh, str(e)
def main():
parser = argparse.ArgumentParser(
description='Produce a single message taken from input')
parser.add_argument('--server',
type=str,
metavar='HOST',
default='localhost',
help='Kafka bootstrap-server address')
parser.add_argument('--port',
type=int,
metavar='PORT',
default=9092,
help='Kafka bootstrap-server port')
parser.add_argument('--client',
type=str,
default='ch-kafka-python',
help='custom client id for this producer')
parser.add_argument('--topic',
type=str,
required=True,
help='name of Kafka topic to store in')
parser.add_argument('--retries',
type=int,
default=0,
help='number of retries to send on failure')
parser.add_argument('--multiply',
type=int,
default=1,
help='multiplies incoming string many times')
parser.add_argument('--repeat',
type=int,
default=1,
help='send same (multiplied) message many times')
mode_group = parser.add_mutually_exclusive_group()
mode_group.add_argument('--jobs',
type=int,
default=multiprocessing.cpu_count(),
help='number of concurrent jobs')
mode_group.add_argument('--delay',
type=int,
metavar='SECONDS',
default=0,
help='delay before sending next message')
args = parser.parse_args()
config = {
'bootstrap_servers': f'{args.server}:{args.port}',
'client_id': args.client,
'retries': args.retries,
}
client = kafka.KafkaProducer(**config)
message = sys.stdin.buffer.read() * args.multiply
def send(num):
if args.delay > 0:
time.sleep(args.delay)
client.send(topic=args.topic, value=message)
print(
f'iteration {num}: sent a message multiplied {args.multiply} times'
)
if args.delay > 0:
args.jobs = 1
pool = ThreadPoolExecutor(max_workers=args.jobs)
for num in range(args.repeat):
pool.submit(send, num)
pool.shutdown()
client.flush()
client.close()
class EvaluationRunner(object):
"""
Class for efficiently evaluating an algorithm across multiple threads for every
mixture in a dataset. If the algorithm requires something that must happen in a
single thread, then the execution of the algorithm can happen in two steps - once
on a blocking thread, and then continue on a parallel thread by specifying
use_blocking_executor as True:
.. code-block:: python
# run on a blocking executor (e.g. on one GPU), in self.blocking_func
features = algorithm.extract_features()
# run the rest on the main executor, using the result self.blocking_func
# this is in self.run_func
algorithm.run(features=features)
algorithm.make_audio_signals()
This class is built assuming that you are using NUSSL style algorithms for source
separation. If you're evaluating on other tasks (e.g. sound classification or
music transcription), then you can use new testers or edit this class to your
liking. If you're editing the class, your new code likely belongs in either
main_func, run_func, or blocking_func.
Args:
testers ([type]): [description]
algorithm_config ([type]): [description]
dataset ([type]): [description]
output_path ([type]): [description]
max_workers (int, optional): [description]. Defaults to 1.
use_blocking_executor (bool, optional): [description]. Defaults to False.
"""
def __init__(self,
testers,
algorithm_config,
dataset,
output_path,
max_workers=1,
use_blocking_executor=False):
self.evaluation_executor = ThreadPoolExecutor(max_workers=max_workers)
self.main_executor = ThreadPoolExecutor(max_workers=max_workers)
self.dataset = dataset
self.testers = testers
self.algorithm_config = algorithm_config
self.use_blocking_executor = use_blocking_executor
self.output_path = os.path.join(output_path, 'results')
os.makedirs(self.output_path, exist_ok=True)
self.AlgorithmClass = getattr(algorithms, algorithm_config['class'])
dummy_mixture = self.dataset.load_audio_files(self.dataset.files[0])[0]
if self.use_blocking_executor:
blocking_algorithm_config = copy.deepcopy(
self.algorithm_config['args'])
if 'use_cuda' in inspect.getargspec(self.AlgorithmClass).args:
blocking_algorithm_config['use_cuda'] = True
self.algorithm_config['args']['use_cuda'] = False
args = inspect.getfullargspec(self.AlgorithmClass)[0]
if 'extra_modules' in args:
blocking_algorithm_config['extra_modules'] = model.extras
self.blocking_algorithm = self.AlgorithmClass(
dummy_mixture, **blocking_algorithm_config)
self.blocking_executor = ThreadPoolExecutor(max_workers=1)
def blocking_func(self, file_path):
"""
[summary]
[extended_summary]
Args:
file_path ([type]): [description]
Returns:
[type]: [description]
"""
mixture, _, _ = self.dataset.load_audio_files(file_path)
self.blocking_algorithm.set_audio_signal(mixture)
self.blocking_algorithm._compute_spectrograms()
features = self.blocking_algorithm.extract_features()
return {'features': features}
def log_scores(self, scores):
"""
[summary]
[extended_summary]
Args:
scores ([type]): [description]
"""
for key in scores:
if key != 'permutation':
logging_str = f"{key}: "
for metric in scores[key]:
logging_str += f"{metric} => {np.mean(scores[key][metric])}, "
logging.info(logging_str)
def run_func(self, file_path, data=None):
"""
[summary]
[extended_summary]
Args:
file_path ([type]): [description]
data ([type], optional): [description]. Defaults to None.
"""
mixture, sources, labels = self.dataset.load_audio_files(file_path)
classes = self.dataset.options['source_labels']
labels = [classes[np.argmax(l)] for l in labels]
args = inspect.getfullargspec(self.AlgorithmClass)[0]
algorithm_args = copy.deepcopy(self.algorithm_config['args'])
if 'extra_modules' in args:
algorithm_args['extra_modules'] = model.extras
algorithm = self.AlgorithmClass(mixture, **algorithm_args)
if data:
for key in data:
if key not in inspect.getargspec(algorithm.run).args:
data.pop(key)
algorithm.run(**data)
else:
algorithm.run()
estimates = algorithm.make_audio_signals()
tester_args = {
'true_sources_list': sources,
'estimated_sources_list': estimates,
'source_labels': labels
}
all_scores = []
try:
for tester in self.testers:
TestClass = tester[0]
kwargs = tester[1]
args = {}
for k in tester_args:
if k in inspect.getargspec(TestClass).args:
args[k] = tester_args[k]
args.update(kwargs)
evaluator = TestClass(**args)
scores = evaluator.evaluate()
self.log_scores(scores)
all_scores.append(scores)
path_to_yml = os.path.join(
self.output_path,
os.path.splitext(os.path.basename(file_path))[0] + '.yml')
logging.info(path_to_yml)
with open(path_to_yml, 'w') as f:
yaml.dump(all_scores, f)
except:
logging.exception()
def main_func(self, file_path):
"""
[summary]
[extended_summary]
Args:
file_path ([type]): [description]
"""
data = None
if self.use_blocking_executor:
task = self.blocking_executor.submit(self.blocking_func, file_path)
data = task.result()
self.evaluation_executor.submit(self.run_func, file_path, data)
def run(self):
"""
[summary]
[extended_summary]
"""
for file_path in self.dataset.files:
self.main_executor.submit(self.main_func, file_path)
self.main_executor.shutdown(wait=True)
self.evaluation_executor.shutdown(wait=True)
