def main():
""" Starts the AdminServer. """
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
parser = argparse.ArgumentParser(
prog='appscale-admin', description='Manages AppScale-related processes')
subparsers = parser.add_subparsers(dest='command')
subparsers.required = True
serve_parser = subparsers.add_parser(
'serve', description='Starts the server that manages AppScale processes')
serve_parser.add_argument(
'-p', '--port', type=int, default=constants.DEFAULT_PORT,
help='The port to listen on')
serve_parser.add_argument(
'-v', '--verbose', action='store_true', help='Output debug-level logging')
subparsers.add_parser(
'summary', description='Lists AppScale processes running on this machine')
restart_parser = subparsers.add_parser(
'restart',
description='Restart AppScale processes running on this machine')
restart_parser.add_argument('service', nargs='+',
help='The process or service ID to restart')
args = parser.parse_args()
if args.command == 'summary':
table = sorted(list(get_combined_services().items()))
print(tabulate(table, headers=['Service', 'State']))
sys.exit(0)
if args.command == 'restart':
socket_path = urlquote(ServiceManagerHandler.SOCKET_PATH, safe='')
session = requests_unixsocket.Session()
response = session.post(
'http+unix://{}/'.format(socket_path),
data={'command': 'restart', 'arg': [args.service]})
response.raise_for_status()
return
if args.verbose:
logging.getLogger('appscale').setLevel(logging.DEBUG)
options.define('secret', appscale_info.get_secret())
options.define('login_ip', appscale_info.get_login_ip())
options.define('private_ip', appscale_info.get_private_ip())
options.define('zk_locations', appscale_info.get_zk_node_ips())
options.define('load_balancers', appscale_info.get_load_balancer_ips())
acc = appscale_info.get_appcontroller_client()
ua_client = UAClient(appscale_info.get_db_master_ip(), options.secret)
zk_client = KazooClient(
hosts=','.join(options.zk_locations),
connection_retry=ZK_PERSISTENT_RECONNECTS)
zk_client.start()
version_update_lock = zk_client.Lock(constants.VERSION_UPDATE_LOCK_NODE)
thread_pool = ThreadPoolExecutor(4)
monit_operator = MonitOperator()
all_resources = {
'acc': acc,
'ua_client': ua_client,
'zk_client': zk_client,
'version_update_lock': version_update_lock,
'thread_pool': thread_pool
}
if options.private_ip in appscale_info.get_taskqueue_nodes():
logger.info('Starting push worker manager')
GlobalPushWorkerManager(zk_client, monit_operator)
if options.private_ip in appscale_info.get_load_balancer_ips():
logger.info('Starting RoutingManager')
routing_manager = RoutingManager(zk_client)
routing_manager.start()
service_manager = ServiceManager(zk_client)
service_manager.start()
controller_state = ControllerState(zk_client)
app = web.Application([
('/oauth/token', OAuthHandler, {'ua_client': ua_client}),
('/v1/apps/([^/]*)/services/([^/]*)/versions', VersionsHandler,
{'acc': acc, 'ua_client': ua_client, 'zk_client': zk_client,
'version_update_lock': version_update_lock, 'thread_pool': thread_pool,
'controller_state': controller_state}),
('/v1/projects', ProjectsHandler, all_resources),
('/v1/projects/([a-z0-9-]+)', ProjectHandler, all_resources),
('/v1/apps/([^/]*)/services', ServicesHandler,
{'ua_client': ua_client, 'zk_client': zk_client}),
('/v1/apps/([^/]*)/services/([^/]*)', ServiceHandler,
all_resources),
('/v1/apps/([^/]*)/services/([^/]*)/versions/([^/]*)',
VersionHandler,
{'acc': acc, 'ua_client': ua_client, 'zk_client': zk_client,
'version_update_lock': version_update_lock, 'thread_pool': thread_pool,
'controller_state': controller_state}),
('/v1/apps/([^/]*)/operations/([a-z0-9-]+)', OperationsHandler,
{'ua_client': ua_client}),
('/api/cron/update', UpdateCronHandler,
{'acc': acc, 'zk_client': zk_client, 'ua_client': ua_client}),
('/api/datastore/index/add', UpdateIndexesHandler,
{'zk_client': zk_client, 'ua_client': ua_client}),
('/api/queue/update', UpdateQueuesHandler,
{'zk_client': zk_client, 'ua_client': ua_client})
])
logger.info('Starting AdminServer')
app.listen(args.port)
management_app = web.Application([
('/', ServiceManagerHandler, {'service_manager': service_manager})])
management_server = HTTPServer(management_app)
management_socket = bind_unix_socket(ServiceManagerHandler.SOCKET_PATH)
management_server.add_socket(management_socket)
io_loop = IOLoop.current()
io_loop.start()
def download_many(suffixs):
workers = min(MAX_WORKERS, len(suffixs))
with ThreadPoolExecutor(workers) as executor:
res = executor.map(download_one, suffixs)
return len(list(res))
def __init__(self, client, bg_host=None, bg_port=None, ssl_enabled=None, ca_cert=None,
client_cert=None, system=None, name=None, description=None, version=None,
icon_name=None, instance_name=None, logger=None, parser=None, multithreaded=None,
metadata=None, max_concurrent=None, bg_url_prefix=None, **kwargs):
# If a logger is specified or the logging module already has additional handlers
# then we assume that logging has already been configured
if logger or len(logging.getLogger(__name__).root.handlers) > 0:
self.logger = logger or logging.getLogger(__name__)
self._custom_logger = True
else:
logging.config.dictConfig(DEFAULT_LOGGING_CONFIG)
self.logger = logging.getLogger(__name__)
self._custom_logger = False
connection_parameters = brewtils.get_connection_info(
bg_host=bg_host,
bg_port=bg_port,
ssl_enabled=ssl_enabled,
ca_cert=ca_cert,
client_cert=client_cert,
url_prefix=bg_url_prefix,
ca_verify=kwargs.get('ca_verify', None),
username=kwargs.get('username', None),
password=kwargs.get('password', None),
client_timeout=kwargs.get('client_timeout', None),
)
self.bg_host = connection_parameters['bg_host']
self.bg_port = connection_parameters['bg_port']
self.ssl_enabled = connection_parameters['ssl_enabled']
self.ca_cert = connection_parameters['ca_cert']
self.client_cert = connection_parameters['client_cert']
self.bg_url_prefix = connection_parameters['url_prefix']
self.ca_verify = connection_parameters['ca_verify']
self.max_attempts = kwargs.get('max_attempts', -1)
self.max_timeout = kwargs.get('max_timeout', 30)
self.starting_timeout = kwargs.get('starting_timeout', 5)
self.max_concurrent = self._setup_max_concurrent(multithreaded, max_concurrent)
self.instance_name = instance_name or os.environ.get('BG_INSTANCE_NAME', 'default')
self.metadata = metadata or {}
self.instance = None
self.queue_connection_params = None
self.admin_consumer = None
self.request_consumer = None
self.connection_poll_thread = None
self.client = client
self.shutdown_event = threading.Event()
self.parser = parser or SchemaParser()
self.system = self._setup_system(
client, self.instance_name, system, name, description, version,
icon_name, self.metadata, kwargs.get("display_name", None),
kwargs.get('max_instances', None),
)
self.unique_name = (
'%s[%s]-%s' %
(self.system.name, self.instance_name, self.system.version)
)
# Tightly manage when we're in an 'error' state, aka Brew-view is down
self.brew_view_error_condition = threading.Condition()
self.brew_view_down = False
self.pool = ThreadPoolExecutor(max_workers=self.max_concurrent)
self.admin_pool = ThreadPoolExecutor(max_workers=1)
self.bm_client = EasyClient(
logger=self.logger, parser=self.parser, **connection_parameters)
from concurrent.futures import ThreadPoolExecutor, as_completed
import time
import random
from functools import partial
def get_html(sleep_time, num):
time.sleep(sleep_time)
# print("get page {} success".format(sleep_time))
return num
executor = ThreadPoolExecutor(max_workers=2)
# 通过sumbit提交到线程池中
tasks = list()
for i in range(10):
sleep_time = random.randint(2, 5)
#把右边函数看成一个整体
tasks.append(executor.submit(partial(get_html, sleep_time), (i)))
#阻塞 等待完成的函数
for i in as_completed(tasks):
data = i.result()
print('num {} success'.format(data))
def __init__(self, fxn, max_workers=120):
self._fxn = fxn
self._executor = ThreadPoolExecutor(max_workers)
self._future_to_key = {} # map from future to a key for later access
from concurrent.futures import ThreadPoolExecutor, as_completed
from sqlalchemy.orm import scoped_session
from db import queries, session_factory, engine
def execute_query(query):
session = scoped_session(session_factory)()
result = session.execute(query)
session.close()
return result
with ThreadPoolExecutor(max_workers=engine.pool.size()) as executor:
futures = [executor.submit(execute_query, query) for query in queries]
for future in as_completed(futures):
result = future.result()
print(list(result)[0])
print(result.rowcount)
def task(exec_host_li, cmd):
with ThreadPoolExecutor(max_workers=10) as executor:
futures = [executor.submit(ssh_command, host_dic, cmd) for host_dic in exec_host_li]
for future in as_completed(futures):
print(future.result())
def executor(self):
cls = self.__class__
if cls._executor is None:
cls._executor = ThreadPoolExecutor(1)
return cls._executor
def run(self, use_updated=False):
try:
usr_choice = easygui.ynbox('是否执行v2ray云彩姬一键安装脚本？', install_title)
if usr_choice:
# 首次安装
if use_updated is False:
INIT_airport_docTree()
INIT_process_docTree()
for x in range(3):
self.open_dir = easygui.diropenbox('请选择安装路径', install_title, default=SYS_LOCAL_fPATH)
# 退出-放弃更新
if self.open_dir is None:
return False
# 选择限制
if os.listdir(self.open_dir):
easygui.msgbox('当前目录下存在其他文件，请选择独立的文件夹！', TITLE)
else:
# 记录用户选择的下载目录，便于软件更新时的项目文件拉取
with open(YAML_PATH, 'w', encoding='utf-8') as f:
proj = YAML_PROJECT
proj['path'] = self.open_dir
yaml.dump(proj, f)
break
# 给头铁的孩子一点教育
else:
easygui.msgbox('操作有误，请重试！', TITLE)
return False
# 软件更新
else:
try:
VersionControlSystem.kill_main()
# fixme:将updated模块移植到系统路径，通过外部操作控制软件更新；
# TODO: 将self.open_dir赋值为软件所在路径
with open(YAML_PATH, 'r', encoding='utf-8') as f:
data = yaml.load(f, Loader=yaml.FullLoader)
print(data['path'])
# self.open_dir = easygui.diropenbox()
self.open_dir = data['path']
except Exception as e:
print(e)
# 下载线程
os.startfile(self.open_dir)
print(f"install path >> {self.open_dir}")
with ThreadPoolExecutor(max_workers=1) as t:
t.submit(self.download)
# t.submit(easygui.msgbox, '正在拉取项目文件，请等待下载', install_title)
easygui.msgbox('下载完成', install_title)
# 解压线程
with ThreadPoolExecutor(max_workers=2) as t:
t.submit(UnZipManager, self.open_fp)
t.submit(easygui.msgbox, '正在解压核心组件，请等待解压', title=install_title)
easygui.msgbox('解压完成', install_title)
# 自启动
target_file = self.open_fp.replace('.zip', '') + f'_v{VersionControlSystem.get_server_version()[0]}'
try:
os.startfile(os.path.join(
target_file,
InstallGuider.v2raycs_name))
except OSError:
pass
finally:
for filename in os.listdir(self.open_dir):
if '.zip' in filename:
try:
os.remove(os.path.join(self.open_dir, filename))
except OSError:
pass
elif os.path.basename(target_file).split('_')[-1] != filename.split('_')[-1]:
if os.path.basename(target_file).split('_')[0] in filename:
try:
shutil.rmtree(os.path.join(self.open_dir, filename))
os.rmdir(os.path.join(self.open_dir, filename))
except OSError:
pass
except Exception as e:
easygui.exceptionbox(f'{e}')
# over
finally:
easygui.msgbox('感谢使用', install_title)
def send_command_to_devices(devices, command):
with ThreadPoolExecutor(max_workers=30) as executor:
result = list(executor.map(connect_ssh, devices, repeat(command)))
return result
import requests
from bs4 import BeautifulSoup
from concurrent.futures import ThreadPoolExecutor, as_completed
file = open('snps.txt', 'r')
lines = file.readlines()
base_url = 'https://selfdecode.com/snp/'
urls = [base_url + snp.rstrip() for snp in lines]
def getData(url):
print(url)
html = requests.get(url)
return html.status_code
processes = []
with ThreadPoolExecutor(max_workers=100) as executor:
for url in urls:
processes.append(executor.submit(getData, url))
for task in as_completed(processes):
print(task.result())
self.settings['redis'].delete(sess)
self.close(code=1000)
return
routes = [
(r"/", MainHandler),
(r"/signup", WebSocketHandler),
(r"/sms", SmsHandler),
]
settings = {
"static_path": "assets",
"static_url_prefix": "/assets/",
"template_path": "html",
"compress_response": True,
"cookie_secret": uuid.uuid4().hex,
"redis": redis.StrictRedis(),
"debug": True,
"db": motor.MotorClient().readdressdb,
"tpool": ThreadPoolExecutor(10),
}
def make_app():
return tornado.web.Application(routes,**settings)
if __name__ == "__main__":
app = make_app()
app.listen(8888)
tornado.log.logging.basicConfig(stream=sys.stdout,level=tornado.log.logging.DEBUG)
tornado.ioloop.IOLoop.current().start()
tr=res,
t_srs=dst_srs,
t_projwin=site_extent,
threads=8)
dz_cmd_list.append(cmd)
if make_stacks:
from concurrent.futures import ThreadPoolExecutor
#threads = 8
threads = iolib.cpu_count()
iolib.setstripe(outdir, threads)
delay = 3.0
outf = open(os.devnull, 'w')
if mos_cmd_list:
with ThreadPoolExecutor(max_workers=threads) as executor:
for cmd in mos_cmd_list:
#print(cmd)
#executor.submit(subprocess.call, cmd, stdout=outf, stderr=subprocess.STDOUT)
executor.submit(subprocess.call, cmd)
time.sleep(delay)
if std_cmd_list:
with ThreadPoolExecutor(max_workers=threads) as executor:
for cmd in std_cmd_list:
#print(cmd)
#executor.submit(subprocess.call, cmd, stdout=outf, stderr=subprocess.STDOUT)
executor.submit(subprocess.call, cmd)
time.sleep(delay)
if dz_cmd_list:
def generate_acoustic_conds(utterances_paths):
with ThreadPoolExecutor(max_workers=50) as executor:
for utterance_path in utterances_paths:
executor.submit(ahoproc_tools.io.wav2aco, utterance_path)
def yify_movie_scraper(signals, movie_title=None, print_output=False):
signals.log_data.emit("Application", "")
signals.log_data.emit("yify(movies)", f"Scraping using yify movies...")
final_result_dict = {}
count = 0
if movie_title is None:
signals.log_data.emit("yify(movies)",
f"Query Error! Enter a valid movie title")
return final_result_dict, count, False
else:
start_time = time.time()
corected_movie_title = movie_title.lower().replace(' ', '%20').replace(
'-', '%20')
url = f"https://yts.mx/browse-movies/{corected_movie_title}/all/all/0/latest/0/all"
signals.log_data.emit(
"yify(movies)",
f"Scrape url set to : {url} for search title : {movie_title}")
def fetch_data(url):
signals.log_data.emit("yify(movies)",
f"[worker - {url}] = working on {url}")
result_dict = {}
signals.log_data.emit(
"yify(movies)", f"[worker - {url}] = staring requests session...")
session = HTMLSession()
r = session.get(url)
signals.log_data.emit(
"yify(movies)",
f"[worker - {url}] = done, now getting torrent matches...")
matches = r.html.find("div.modal-torrent", first=False)
count = 0
for match in matches:
torrent_name = match.find("a.download-torrent",
first=True).attrs['title'].replace(
"Download", "").strip()
magnet_link = match.find("a.magnet", first=True).attrs['href']
torrent_link = match.find("a.button-green-download2-big",
first=True).attrs['href']
size = match.find("p.quality-size", first=False)[-1].text
movie_mode = match.find("p.quality-size", first=True).text
seeders = 999
season = ""
episode = ""
searched_title = ""
torrent_name = f"{torrent_name} - {movie_mode}"
result_dict[count] = [
searched_title, torrent_name, torrent_link, magnet_link, size,
seeders, season, episode
]
count += 1
signals.log_data.emit(
"yify(movies)",
f"[worker - {url}] = found torrent : {torrent_name}, currently found {count} torrents"
)
return result_dict
## Get the list of movie results
signals.log_data.emit("yify(movies)",
f"getting the list of pagination urls...")
session = HTMLSession()
r = session.get(url)
if print_output:
print("Getting the list of movie results...")
movie_result_list = [
movie.attrs['href'] for movie in r.html.find(
"div.row div.browse-movie-wrap a.browse-movie-link", first=False)
]
signals.log_data.emit(
"yify(movies)",
f"done getting pagination url list, list = {movie_result_list}")
if print_output:
print("List of movies found : ", movie_result_list)
## Now visit each movie url and get torrent data
signals.log_data.emit("yify(movies)",
f"Now visit each movie url and get torrent data.")
if print_output:
print("Now visit each movie url and get torrent data")
signals.log_data.emit("yify(movies)",
f"Starting thread-pool with 25 workers...")
with ThreadPoolExecutor(max_workers=25) as executor:
results = executor.map(fetch_data, movie_result_list)
signals.log_data.emit(
"yify(movies)",
f"Done getting individual torrents, now merging into one result dictionary..."
)
try:
for result in results:
for val in result.values():
val[0] = movie_title.title()
final_result_dict[count] = val
count += 1
except Exception as e:
signals.log_data.emit("yify(movies)[error]", f"exception : {e}")
if print_output: print("Exception, error = ", e)
else:
pass
if print_output:
print(json.dumps(final_result_dict, indent=2))
print(f"Finished all ops in {time.time() - start_time} seconds")
print(f"Found {count} items")
print("Done")
signals.log_data.emit(
"yify(movies)",
f"Finished all operations in {time.time() - start_time} seconds, found {len(final_result_dict.keys())} torrents."
)
return final_result_dict, count, True
import functools
from concurrent.futures import ThreadPoolExecutor
from itertools import repeat
# 提前初始化好
thread_pool = ThreadPoolExecutor(max_workers=30)
def threads(concurrent_size, timeout=0):
def decorate(func):
@functools.wraps(func)
def wrapper(*args, **kw):
futures = []
for i in range(concurrent_size):
# 向线程池提交任务。
futures.append(thread_pool.submit(func, *args, **kw))
for _ in repeat(None):
for i in futures:
if i.done():
result = i.result()
return result
return None
return wrapper
return decorate
from concurrent.futures import ThreadPoolExecutor
import time
def boil_udon():
print(' うどんを茹でます。')
time.sleep(3)
print(' うどんが茹であがりました。')
tpe = ThreadPoolExecutor(max_workers=3)
print('うどんを100個茹でます。')
for _ in range(10):
tpe.submit(boil_udon)
tpe.shutdown()
print('うどんが100個茹で上がりました。')
def run_script_with_mutiple_threads():
with ThreadPoolExecutor(max_workers=16) as Executor:
# jobs = [Executor.submit(get_test_news, u) for u in range(66466, 44882, -1)]
jobs = [Executor.submit(get_news, u) for u in range(66485, 44882, -1)]
def _setUpExecutor(self):
self._executor = ThreadPoolExecutor(max_workers=1)
def generate(self, writer, parameters, context, feedback):
self.feedback = feedback
feedback.setProgress(1)
extent = self.parameterAsExtent(parameters, self.EXTENT, context)
self.min_zoom = self.parameterAsInt(parameters, self.ZOOM_MIN, context)
self.max_zoom = self.parameterAsInt(parameters, self.ZOOM_MAX, context)
dpi = self.parameterAsInt(parameters, self.DPI, context)
self.color = self.parameterAsColor(parameters, self.BACKGROUND_COLOR, context)
self.tile_format = self.formats[self.parameterAsEnum(parameters, self.TILE_FORMAT, context)]
self.quality = self.parameterAsInt(parameters, self.QUALITY, context)
self.metatilesize = self.parameterAsInt(parameters, self.METATILESIZE, context)
self.maxThreads = int(ProcessingConfig.getSetting(ProcessingConfig.MAX_THREADS))
try:
self.tile_width = self.parameterAsInt(parameters, self.TILE_WIDTH, context)
self.tile_height = self.parameterAsInt(parameters, self.TILE_HEIGHT, context)
except AttributeError:
self.tile_width = 256
self.tile_height = 256
wgs_crs = QgsCoordinateReferenceSystem('EPSG:4326')
dest_crs = QgsCoordinateReferenceSystem('EPSG:3857')
project = context.project()
self.src_to_wgs = QgsCoordinateTransform(project.crs(), wgs_crs, context.transformContext())
self.wgs_to_dest = QgsCoordinateTransform(wgs_crs, dest_crs, context.transformContext())
# without re-writing, we need a different settings for each thread to stop async errors
# naming doesn't always line up, but the last number does
self.settingsDictionary = {str(i): QgsMapSettings() for i in range(self.maxThreads)}
for thread in self.settingsDictionary:
self.settingsDictionary[thread].setOutputImageFormat(QImage.Format_ARGB32_Premultiplied)
self.settingsDictionary[thread].setDestinationCrs(dest_crs)
self.settingsDictionary[thread].setLayers(self.layers)
self.settingsDictionary[thread].setOutputDpi(dpi)
if self.tile_format == 'PNG':
self.settingsDictionary[thread].setBackgroundColor(self.color)
## disable partial labels (they would be cut at the edge of tiles)
labeling_engine_settings = self.settingsDictionary[thread].labelingEngineSettings()
labeling_engine_settings.setFlag(QgsLabelingEngineSettings.UsePartialCandidates, False)
self.settingsDictionary[thread].setLabelingEngineSettings(labeling_engine_settings)
# Transfer context scopes to MapSettings
self.settingsDictionary[thread].setExpressionContext(context.expressionContext())
self.wgs_extent = self.src_to_wgs.transformBoundingBox(extent)
self.wgs_extent = [self.wgs_extent.xMinimum(), self.wgs_extent.yMinimum(), self.wgs_extent.xMaximum(),
self.wgs_extent.yMaximum()]
metatiles_by_zoom = {}
self.totalMetatiles = 0
allMetatiles = []
for zoom in range(self.min_zoom, self.max_zoom + 1):
metatiles = get_metatiles(self.wgs_extent, zoom, self.metatilesize)
metatiles_by_zoom[zoom] = metatiles
allMetatiles += metatiles
self.totalMetatiles += len(metatiles)
lab_buffer_px = 100
self.progress = 0
tile_params = {
'format': self.tile_format,
'quality': self.quality,
'width': self.tile_width,
'height': self.tile_height,
'min_zoom': self.min_zoom,
'max_zoom': self.max_zoom,
'extent': self.wgs_extent,
}
writer.set_parameters(tile_params)
self.writer = writer
self.progressThreadLock = threading.Lock()
if self.maxThreads > 1:
feedback.pushConsoleInfo(self.tr('Using {max_threads} CPU Threads:').format(max_threads=self.maxThreads))
for zoom in range(self.min_zoom, self.max_zoom + 1):
feedback.pushConsoleInfo(self.tr('Generating tiles for zoom level: {zoom}').format(zoom=zoom))
with ThreadPoolExecutor(max_workers=self.maxThreads) as threadPool:
threadPool.map(self.renderSingleMetatile, metatiles_by_zoom[zoom])
else:
feedback.pushConsoleInfo(self.tr('Using 1 CPU Thread:'))
for zoom in range(self.min_zoom, self.max_zoom + 1):
feedback.pushConsoleInfo(self.tr('Generating tiles for zoom level: {zoom}').format(zoom=zoom))
for i, metatile in enumerate(metatiles_by_zoom[zoom]):
self.renderSingleMetatile(metatile)
writer.close()
def setUp(self):
global DeferredException
self.sys_path_save = sys.path
# Setting up the config module
from xosconfig import Config
config = os.path.join(test_path, "../test_config.yaml")
Config.clear()
Config.init(config, "synchronizer-config-schema.yaml")
# END Setting up the config module
from xossynchronizer.mock_modelaccessor_build import mock_modelaccessor_config
mock_modelaccessor_config(test_path, [("olt-service", "volt.xproto"),
("rcord", "rcord.xproto")])
import xossynchronizer.modelaccessor
reload(xossynchronizer.modelaccessor
) # in case nose2 loaded it in a previous test
from xossynchronizer.modelaccessor import model_accessor
self.model_accessor = model_accessor
from pull_onus import ONUDevicePullStep
# import all class names to globals
for (k, v) in model_accessor.all_model_classes.items():
globals()[k] = v
self.sync_step = ONUDevicePullStep
# mock volt service
self.volt_service = Mock()
self.volt_service.id = "volt_service_id"
self.volt_service.voltha_url = MOCK_VOLTHA_SERVER_ADDRESS
self.volt_service.voltha_port = MOCK_VOLTHA_SERVER_PORT
# mock OLTDevice
self.olt = Mock()
self.olt.id = 1
# second mock OLTDevice
self.olt2 = Mock()
self.olt2.id = 2
# mock pon port
self.pon_port = Mock()
self.pon_port.id = 1
# mock pon port
self.pon_port2 = Mock()
self.pon_port2.id = 2
# Mock gRPC server
clear_voltha_client_cache()
self.server = grpc.server(ThreadPoolExecutor(max_workers=5))
self.voltha_mock, _, _ = VolthaServerMock.start_voltha_server(
self.server)
# Server is actually started on single tests because different tests build different mocks
# mock voltha responses
self.devices = {
"items": [{
"id": "0001130158f01b2d",
"type": "broadcom_onu",
"vendor": "Broadcom",
"serial_number": "BRCM22222222",
"vendor_id": "BRCM",
"adapter": "broadcom_onu",
"vlan": 0,
"admin_state": "ENABLED",
"oper_status": "ACTIVE",
"connect_status": "REACHABLE",
"reason": "starting-omci",
"parent_id": "00010fc93996afea",
"parent_port_no": 1
}]
}
self.ports_dict = {"0001130158f01b2d": {"items": []}}
self.two_devices = {
"items": [{
"id": "0001130158f01b2d",
"type": "broadcom_onu",
"vendor": "Broadcom",
"serial_number": "BRCM22222222",
"vendor_id": "BRCM",
"adapter": "broadcom_onu",
"vlan": 0,
"admin_state": "ENABLED",
"oper_status": "ACTIVE",
"connect_status": "REACHABLE",
"reason": "starting-omci",
"parent_id": "00010fc93996afea",
"parent_port_no": 1
}, {
"id": "0001130158f01b2e",
"type": "broadcom_onu",
"vendor": "Broadcom",
"serial_number": "BRCM22222223",
"vendor_id": "BRCM",
"adapter": "broadcom_onu",
"vlan": 0,
"admin_state": "ENABLED",
"oper_status": "ACTIVE",
"connect_status": "REACHABLE",
"reason": "omci-admin-lock",
"parent_id": "00010fc93996afeb",
"parent_port_no": 1
}],
}
# TODO add ports
self.two_ports_dict = {
"0001130158f01b2d": {
"items": []
},
"0001130158f01b2e": {
"items": []
}
}
def kick_ass_scraper_anime(signals, movie_title=None, print_output=False):
"""Use thereadpool to open the next pages"""
signals.log_data.emit("Application", "")
signals.log_data.emit("kickass(anime)", "Scraping using kickass(anime)...")
def fetch(page_url):
## Get torrent data from all pages
return_dict = {}
dict_count = 0
# print(f"working on {page_url}")
signals.log_data.emit("kickass(anime)",
f"[worker- {page_url}] = working on {page_url}")
scrape_url = page_url
searched_title = movie_title
## Create the requests session
signals.log_data.emit(
"kickass(anime)",
f"[worker- {page_url}] = Starting the requests session...")
main_session = HTMLSession()
r_main = main_session.get(scrape_url, timeout=40)
## Start scraping for data
signals.log_data.emit(
"kickass(anime)",
f"[worker- {page_url}] = session successfully established, now getting torrent matches..."
)
matches = r_main.html.find("tr.even,tr.odd", first=False)
for match in matches:
# get the magnet link
torrent_page_url = match.find("a.cellMainLink",
first=True).attrs['href']
torrent_page_url = "https://kickasstorrents.to" + torrent_page_url
# print(f"getting magnet link from {torrent_page_url}")
session2 = HTMLSession()
r2 = session2.get(torrent_page_url, timeout=40)
# Magnet link
try:
magnet_link = r2.html.find("a.kaGiantButton",
first=True).attrs['href']
except Exception as e:
if print_output: print(e)
signals.log_data.emit(
"kickass(anime)[ERROR]",
f"[worker- {page_url}] = Error getting magnet link: {e}")
magnet_link = ''
# Torrent Title
try:
torrent_title = match.find("a.cellMainLink", first=True).text
except Exception as e:
if print_output: print(e)
signals.log_data.emit(
"kickass(anime)[ERROR]",
f"[worker- {page_url}] = Error getting magnet link: {e}")
torrent_title = ''
# Size
try:
size = match.find("td.nobr", first=True).text
except Exception as e:
if print_output: print(e)
signals.log_data.emit(
"kickass(anime)[ERROR]",
f"[worker- {page_url}] = Error getting magnet link: {e}")
size = ''
# seeders
try:
seeders = match.find("td.green", first=True).text
except Exception as e:
if print_output: print(e)
signals.log_data.emit(
"kickass(anime)[ERROR]",
f"[worker- {page_url}] = Error getting magnet link: {e}")
seeders = ''
# Season And Episode
try:
query_string = list(
re.compile(r'([SE][0-9]+|[0-9]+x[0-9]+)').finditer(
torrent_title))[0].group()
if query_string.startswith("S"):
se_ep_ptring = list(
re.compile(r'[SE][0-9]+').finditer(torrent_title))
season = se_ep_ptring[0].group().replace("S", "")
episode = se_ep_ptring[1].group().replace("E", "")
elif len(query_string.split("x")) == 2:
se_ep_ptring = list(
re.compile(r'[0-9]+x[0-9]+').finditer(torrent_title))
season = "{:02}".format(
int(se_ep_ptring[0].group().split("x")[0]))
episode = se_ep_ptring[0].group().split("x")[1]
else:
season = None
episode = None
except Exception as e:
season = ''
episode = ''
## add item to dict
return_dict[dict_count] = [
movie_title, torrent_title, torrent_page_url, magnet_link,
size, seeders, season, episode
]
dict_count += 1
signals.log_data.emit(
"kickass(anime)",
f"[worker- {page_url}] = found torrent : {torrent_title}")
return return_dict, dict_count, True
final_result_dict = {}
count = 0
## generagte list of pages to visit
signals.log_data.emit(
"kickass(anime)",
f"getting a list of all torrent results pages to scrape...")
start_time = time.time()
if print_output: print("Started process")
signals.log_data.emit("kickass(anime)", f"Started process...")
main_url = "https://kickasstorrents.to/search/{}/category/anime/1/".format(
movie_title)
signals.log_data.emit(
"kickass(anime)",
f"scrape url set to : {main_url} for searched title : {movie_title}")
if print_output: print("getting pages")
signals.log_data.emit("kickass(anime)", f"getting pages...")
main_session = HTMLSession()
r_main = main_session.get(main_url)
count_str = r_main.html.find("tr td a.plain", first=True).text
if count_str == 'omikrosgavri':
signals.log_data.emit("kickass(anime)",
f"omikrosgavri error, found 0 torrens")
return final_result_dict, count, False
page_count = math.ceil(
int(count_str.split(" ")[-1]) /
int(count_str.split(" ")[-3].split('-')[1]))
pages_list = [
main_url.replace(main_url.split('/')[-2], str(i))
for i in range(1, page_count + 1)
]
if print_output: print(f"Scrapping {page_count} pages")
signals.log_data.emit("kickass(anime)", f"Scrapping {page_count} pages.")
if print_output: print("Starting threadpool...")
signals.log_data.emit("kickass(anime)",
f"starting thread-pool with 25 workers")
with ThreadPoolExecutor(max_workers=25) as executor:
results = executor.map(fetch, pages_list)
try:
for result in results:
for val in result[0].values():
final_result_dict[count] = val
count += 1
except Exception as e:
signals.log_data.emit("kickass(anime)[ERROR]",
f"Error, exception : {e}")
if print_output: print("Exception, error = ", e)
else:
pass
if print_output:
print(json.dumps(final_result_dict, indent=2))
print(f"Finished all ops in {time.time() - start_time} seconds")
print("Done")
signals.log_data.emit(
"kickass(anime)",
f"Finished all ops in {time.time() - start_time} seconds, found {len(final_result_dict.keys())} torrents."
)
return final_result_dict, count, True
def thread_pool_executor(cls, n_workers):
return ThreadPoolExecutor(n_workers)
def kick_ass_scraper_all(torrent_name=None, print_output=False):
def fetch(page_url):
## Get torrent data from all pages
return_dict = {}
dict_count = 0
print(f"working on {page_url}\n")
scrape_url = page_url
searched_title = torrent_name
## Create the requests session
main_session = HTMLSession()
r_main = main_session.get(scrape_url, timeout=40)
## Start scraping for data
matches = r_main.html.find("tr.even,tr.odd", first=False)
for match in matches:
# get the magnet link
torrent_page_url = match.find("a.cellMainLink",
first=True).attrs['href']
torrent_page_url = "https://kat.am/usearch/" + torrent_page_url
print(f"getting magnet link from {torrent_page_url}")
session2 = HTMLSession()
r2 = session2.get(torrent_page_url, timeout=40)
# Magnet link
try:
magnet_link = r2.html.find("a.kaGiantButton",
first=True).attrs['href']
except Exception as e:
print(e)
magnet_link = ''
# Torrent Title
try:
torrent_title = match.find("a.cellMainLink", first=True).text
except Exception as e:
print(e)
torrent_title = ''
# Size
try:
size = match.find("td.nobr", first=True).text
except Exception as e:
print(e)
size = ''
# seeders
try:
seeders = match.find("td.green", first=True).text
except Exception as e:
print(e)
seeders = ''
# Season And Episode
try:
query_string = list(
re.compile(r'([SE][0-9]+|[0-9]+x[0-9]+)').finditer(
torrent_title))[0].group()
if query_string.startswith("S"):
se_ep_ptring = list(
re.compile(r'[SE][0-9]+').finditer(torrent_title))
season = se_ep_ptring[0].group().replace("S", "")
episode = se_ep_ptring[1].group().replace("E", "")
elif len(query_string.split("x")) == 2:
se_ep_ptring = list(
re.compile(r'[0-9]+x[0-9]+').finditer(torrent_title))
season = "{:02}".format(
int(se_ep_ptring[0].group().split("x")[0]))
episode = se_ep_ptring[0].group().split("x")[1]
else:
season = None
episode = None
except Exception as e:
season = ''
episode = ''
## add item to dict
return_dict[dict_count] = [
torrent_name, torrent_title, torrent_page_url, magnet_link,
size, seeders, season, episode
]
dict_count += 1
return return_dict, dict_count, True
final_result_dict = {}
count = 0
## generagte list of pages to visit
start_time = time.time()
if print_output: print("Started process")
main_url = "https://kat.am/search/{}/".format(torrent_name)
if print_output: print("getting pages")
main_session = HTMLSession()
r_main = main_session.get(main_url)
count_str = r_main.html.find("tr td a.plain", first=True).text
if count_str == 'omikrosgavri':
return final_result_dict, count, False
page_count = math.ceil(
int(count_str.split(" ")[-1]) /
int(count_str.split(" ")[-3].split('-')[1]))
pages_list = [
main_url.replace(main_url.split('/')[-2], str(i))
for i in range(1, page_count + 1)
][:50]
if print_output: print(f"Scrapping {page_count} pages")
if print_output: print("Starting threadpool...")
with ThreadPoolExecutor(max_workers=25) as executor:
results = executor.map(fetch, pages_list)
try:
for result in results:
for val in result[0].values():
final_result_dict[count] = val
count += 1
except Exception as e:
if print_output: print("Exception, error = ", e)
else:
pass
if print_output:
print(json.dumps(final_result_dict, indent=2))
print(f"Finished all ops in {time.time() - start_time} seconds")
print("Done")
return final_result_dict, count, True
self.log.info("Slack engine connected")
while True:
data = self.client.rtm_read()
if len(data) > 0:
for d in data:
if d['type'] == 'message':
if 'subtype' not in d or d['subtype'] != 'bot_message':
try:
self.process_message(d)
self.log.info(
'Incoming message: {0}'.format(data))
except:
self.log.exception(
"Error while processing message")
time.sleep(1)
if __name__ == '__main__':
wd = WatchdogBot(SLACK_CHANNEL)
with ThreadPoolExecutor(max_workers=4) as e:
try:
e.submit(wd.run_bot)
e.submit(wd.listen_apps)
except KeyboardInterrupt:
e.shutdown(wait=False)
e._threads.clear()
thread._threads_queues.clear()
raise
def nyaa_scraper(signals, movie_title=None, print_output=False):
"""
:keywords : movie_title=None
A string containing the movie title or movie eztv ID, If None, exits.
:returns : > Dict containing torrent links.
:keys = torrent result count (int)
:values = [searched_title, ez_title, torrent_link, magnet_link, size, seeders, season, episode] (list)
> The number of torrents found
> Weather the scrape was successful
"""
signals.log_data.emit("Application", "")
signals.log_data.emit("nyaa", "Scraping using nyaa...")
final_result_dict = {}
count = 0
if movie_title is None:
# raise Exception("Querry Error! Enter a valid movie title")
signals.log_data.emit("nyaa[Error]",
"Query Error! Enter a valid movie title.")
return final_result_dict, count, False
if movie_title is not None:
scrape_url = f"https://nyaa.si/?f=0&c=0_0&q={movie_title}"
searched_title = movie_title
signals.log_data.emit(
"nyaa",
f"scrape url set to : {scrape_url} for searched title : {searched_title}"
)
# print(scrape_url)
else:
return final_result_dict, count, False
## Get the number of pages to visit
signals.log_data.emit("nyaa", f"getting the number of pages to visit...")
pagnation_session = HTMLSession()
pagnation_request = pagnation_session.get(scrape_url)
t = pagnation_request.html.find("div.pagination-page-info",
first=True).text
results_count = int(
list(re.finditer(r'of [0-9]+ results', t))[0].group().split(" ")[1])
per_page_count = int(
list(re.finditer(r'results [0-9]+-[0-9]+ out',
t))[0].group().split(" ")[1].split("-")[1])
try:
total_pages = math.ceil(results_count / per_page_count)
except:
total_pages = 1
if print_output: print(total_pages)
signals.log_data.emit(
"nyaa", f"Done, Number of pages to be visited : {total_pages}.")
## Generate a list of result page urls
signals.log_data.emit(
"nyaa", f"generating a list of urls (from the parent url) to visit...")
pages_urls = [
f"https://nyaa.si/?f=0&c=0_0&q={movie_title}&p={page}"
for page in range(1, total_pages + 1)
]
signals.log_data.emit("nyaa", f"done, pages list = {pages_urls}")
def get_page_data(page_url):
signals.log_data.emit(
"nyaa", f"[worker - {page_url}] = working on {page_url}")
return_dict = {}
dict_count = 0
signals.log_data.emit(
"nyaa", f"[worker - {page_url}] = starting requests session...")
session = HTMLSession()
r = session.get(page_url)
signals.log_data.emit(
"nyaa",
f"[worker - {page_url}] = done, now getting all torrent matches")
matches = r.html.find("div.table-responsive tbody tr", first=False)
for match in matches:
## Get the title
try:
torrent_title = match.find(
"td:not(.text-center) a:not(.comments)",
first=False)[-1].attrs['title']
except:
torrent_title = " "
## Get The Magnet Link
try:
magnet_link = match.find(
"td.text-center",
first=False)[0].find('a')[1].attrs['href']
except:
magnet_link = " "
## Get The .torrent link
try:
torrent_link = " "
except:
torrent_link = " "
## Get size
try:
size = match.find("td.text-center", first=False)[1].text
except:
size = " "
## Get the seeders
try:
seeders = match.find("td.text-center", first=False)[3].text
except:
seeders = " "
## Get season:
try:
season = " "
except:
season = " "
## Get Episode:
try:
episode = " "
except:
episode = " "
return_dict[dict_count] = [
movie_title, torrent_title, torrent_link, magnet_link, size,
seeders, season, episode
]
dict_count += 1
signals.log_data.emit(
"nyaa",
f"[worker - {page_url}] = Found torrent : {torrent_title}, currently the {dict_count}'th torrent."
)
return return_dict, dict_count, True
## Start the threadpool executor
signals.log_data.emit("nyaa", f"starting the thread-pool...")
start_time = time.time()
with ThreadPoolExecutor(max_workers=5) as executor:
results = executor.map(get_page_data, pages_urls)
try:
for result in results:
for val in result[0].values():
final_result_dict[count] = val
count += 1
except Exception as e:
signals.log_data.emit("nyaa[error]", f"Exception : {e}")
if print_output: print("Exception, error = ", e)
else:
pass
if print_output:
print(json.dumps(final_result_dict, indent=2))
print(f"Finished all ops in {time.time() - start_time} seconds")
print(f"Found {count} items")
print("Done")
signals.log_data.emit(
"nyaa",
f"Done with all operations in {time.time() - start_time} seconds, found {len(final_result_dict.keys())} torrents."
)
return final_result_dict, count, True
def __init__(self,
web_interface_url,
user_pwd,
svn_branch='trunk',
svn_revision='HEAD',
thread_count=1,
result_poll_interval=2,
user_agent=None,
version=None):
"""
Creates a new WebInterface object.
The given svn revision is resolved (e.g. 'HEAD' -> 17495).
@param web_interface_url: the base URL of the VerifierCloud's web interface
@param user_pwd: user name and password in the format '<user_name>:<password>' or none if no authentification is required
@param svn_branch: the svn branch name or 'trunk', defaults to 'trunk'
@param svn_revision: the svn revision number or 'HEAD', defaults to 'HEAD'
@param thread_count: the number of threads for fetching results in parallel
@param result_poll_interval: the number of seconds to wait between polling results
"""
if not (1 <= thread_count <= MAX_SUBMISSION_THREADS):
sys.exit(
"Invalid number {} of client threads, needs to be between 1 and {}."
.format(thread_count, MAX_SUBMISSION_THREADS))
if not 1 <= result_poll_interval:
sys.exit("Poll interval {} is too small, needs to be at least 1s.".
format(result_poll_interval))
if not web_interface_url[-1] == '/':
web_interface_url += '/'
default_headers = {'Connection': 'Keep-Alive'}
if user_agent:
default_headers['User-Agent'] = \
'{}/{} (Python/{} {}/{})'.format(user_agent, version, platform.python_version(), platform.system(), platform.release())
urllib.parse.urlparse(web_interface_url) # sanity check
self._web_interface_url = web_interface_url
logging.info('Using VerifierCloud at %s', web_interface_url)
self._connection = requests.Session()
self._connection.headers.update(default_headers)
self._connection.verify = '/etc/ssl/certs'
if user_pwd:
self._connection.auth = (user_pwd.split(":")[0],
user_pwd.split(":")[1])
self._base64_user_pwd = base64.b64encode(
user_pwd.encode("utf-8")).decode("utf-8")
else:
self._base64_user_pwd = None
self._unfinished_runs = {}
self._unfinished_runs_lock = threading.Lock()
self._downloading_result_futures = {}
self._download_attempts = {}
self.thread_count = thread_count
self._executor = ThreadPoolExecutor(thread_count)
self._thread_local = threading.local()
self._hash_code_cache = {}
self._group_id = str(random.randint(0, 1000000))
self._read_hash_code_cache()
self._resolved_tool_revision(svn_branch, svn_revision)
self._tool_name = self._request_tool_name()
try:
self._result_downloader = SseResultDownloader(
self, result_poll_interval)
except:
self._result_downloader = PollingResultDownloader(
self, result_poll_interval)
def anime_tosho_scraper(signals, movie_title=None, print_output=False):
signals.log_data.emit("Application", "")
signals.log_data.emit("anime-tosho", f"Scraping using anime tosho...")
final_result_dict = {}
count = 0
start_time = time.time()
if movie_title is None:
# raise Exception("Querry Error! Enter a valid movie title")
signals.log_data.emit("anime-tosho",
f"Query Error! Enter a valid movie title")
return final_result_dict, count, False
else:
scrape_url = f"https://animetosho.org/search?q={movie_title.replace(' ', '%20')}"
searched_title = movie_title
signals.log_data.emit(
"anime-tosho",
f"scrape url set to : {scrape_url} for the search title : {searched_title}"
)
## Get The List Of pages available using pagination
signals.log_data.emit(
"anime-tosho",
f"Getting the list of pages available for pagination...")
if print_output:
print("Getting The Pagination Pages")
paginarion_session = HTMLSession()
r = paginarion_session.get(scrape_url)
pages_list = [html.url for html in r.html] # A list Of Result pages
signals.log_data.emit("anime-tosho",
f"Done, pagination pages url list = {pages_list}")
# pages_list = ["https://animetosho.org/search?q=Fairy%20Tail", "https://animetosho.org/search?q=Fairy%20Tail&page=2", "https://animetosho.org/search?q=Fairy%20Tail&page=3", "https://animetosho.org/search?q=Fairy%20Tail&page=4"]
## Construct The Info Getting Function
def get_torrent_data(url):
return_dict = {}
dict_count = 0
signals.log_data.emit("anime-tosho",
f"[worker - {url}] = Working on url : {url}.")
signals.log_data.emit(
"anime-tosho", f"[worker - {url}] = starting requests session...")
session = HTMLSession()
r = session.get(url)
signals.log_data.emit(
"anime-tosho",
f"[worker - {url}] = Done, now getting the torrent matches...")
matches = r.html.find("div div.home_list_entry", first=False)
for match in matches:
try:
size = match.find("div.size")[0].text
except:
size = ""
try:
seeds = match.find("div.links span[title]"
)[0].attrs['title'].split('/')[0].replace(
':',
'').strip(string.ascii_letters).strip()
except:
seeds = "-"
try:
magnet_link = [
i for i in match.find("a[href]")
if 'magnet' in i.attrs['href']
][0].attrs['href']
except:
magnet_link = ""
try:
torrent_link = [
i for i in match.find("a[href]")
if '.torrent' in i.attrs['href']
][0].attrs['href']
except:
torrent_link = ""
try:
torrent_title = match.find("div.link a")[0].text
except:
torrent_title = "NONE"
season = ""
episode = ""
## add item to dict
return_dict[dict_count] = [
movie_title, torrent_title, torrent_link, magnet_link, size,
seeds, season, episode
]
dict_count += 1
signals.log_data.emit(
"anime-tosho",
f"[worker - {url}] = found torrent : {torrent_title}")
return return_dict
## Get torrent info from each paginatied page
signals.log_data.emit(
"anime-tosho", f"getting torrent info from each pagination page...")
if print_output:
print("Now getting the torrent data from each paginated page")
signals.log_data.emit("anime-tosho",
f"starting thread-pool using 25 workers")
with ThreadPoolExecutor(max_workers=25) as executor:
results = executor.map(get_torrent_data, pages_list)
## Merge the individual page results to one final dictionary
signals.log_data.emit("anime-tosho",
f"merging individual worker results into one...")
if print_output:
print("Now Merging pages results to final dictionary copy")
try:
for result in results:
for val in result.values():
final_result_dict[count] = val
count += 1
except Exception as e:
signals.log_data.emit("anime-tosho[error]", f"exception : {e}")
if print_output: print("Exception, error = ", e)
else:
pass
if print_output:
print(json.dumps(final_result_dict, indent=2))
print(f"Finished all ops in {time.time() - start_time} seconds")
print(f"Found {count} items")
print("Done")
signals.log_data.emit(
"anime-tosho",
f"Finished all operations in {time.time() - start_time} seconds, and found {len(final_result_dict.keys())} torrents"
)
return final_result_dict, count, True
'zookeeper address and port are not written!')
if cls not in instances:
logging.info('no zk client, init one')
instances[cls] = cls(*args, **kw)
if instances[cls].zk is None:
logging.info('zk client disconnect, init another one')
instances[cls] = cls(*args, **kw)
return instances[cls]
return _zk_singleton
default_executor = ThreadPoolExecutor(20)
def run_on_executor(executor=default_executor):
def run_on_executor_decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
task = Task(executor.submit, func, *args, **kwargs)
return task
return wrapper
return run_on_executor_decorator
def run_callback(func):
from flask import jsonify
from flask_adminlte import AdminLTE
from models import AdminUser, SimulationTask, ResultsPerDay, Base
from concurrent.futures import ThreadPoolExecutor
import sys
import json
from models import db_session, engine, Base
from tasks import run_long_task
from tasks import print_exception
from sqlalchemy import func
import logging
import logging.config
from config import g_logger
#Only One Task per time
executor = ThreadPoolExecutor(1)
task_future = None
#Main function contains multiple routes
def create_rido_application(configfile=None):
app = Flask(__name__)
AdminLTE(app)
current_user = AdminUser()
@app.teardown_appcontext
def shutdown_session(exception=None):
db_session.remove()
@app.before_first_request
