def instance_owners(request):
if request.user.is_superuser or request.user.has_perm('ganeti.view_instances'):
p = Pool(20)
instancesall = []
bad_clusters = []
def _get_instances(cluster):
try:
instancesall.extend(cluster.get_user_instances(request.user))
except (GanetiApiError, Exception):
bad_clusters.append(cluster)
if not request.user.is_anonymous():
p.imap(_get_instances, Cluster.objects.all())
p.join()
instances = [i for i in instancesall if i.users]
def cmp_users(x, y):
return cmp(",".join([ u.username for u in x.users]),
",".join([ u.username for u in y.users]))
instances.sort(cmp=cmp_users)
return render_to_response("instance_owners.html",
{"instances": instances},
context_instance=RequestContext(request))
else:
return HttpResponseRedirect(reverse('user-instances'))
def prepare_clusternodes(cluster=None):
if not cluster:
clusters = Cluster.objects.all()
else:
clusters = Cluster.objects.filter(slug=cluster)
p = Pool(15)
nodes = []
bad_clusters = []
bad_nodes = []
def _get_nodes(cluster):
try:
for node in cluster.get_cluster_nodes():
nodes.append(node)
if node['offline'] is True:
bad_nodes.append(node['name'])
except (GanetiApiError, Exception):
cluster._client = None
bad_clusters.append(cluster)
finally:
close_connection()
p.imap(_get_nodes, clusters)
p.join()
return nodes, bad_clusters, bad_nodes
def instance_owners(request):
if request.user.is_superuser or request.user.has_perm('ganeti.view_instances'):
p = Pool(20)
instancesall = []
bad_clusters = []
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instancesall.extend(cluster.get_user_instances(request.user))
except (GanetiApiError, Timeout):
bad_clusters.append(cluster)
finally:
t.cancel()
if not request.user.is_anonymous():
p.imap(_get_instances, Cluster.objects.all())
p.join()
instances = [i for i in instancesall if i.users]
def cmp_users(x, y):
return cmp(",".join([ u.username for u in x.users]),
",".join([ u.username for u in y.users]))
instances.sort(cmp=cmp_users)
return render_to_response("instance_owners.html",
{"instances": instances},
context_instance=RequestContext(request))
else:
return HttpResponseRedirect(reverse('user-instances'))
def match_reports():
report_set = json.loads(request.data)
coordinates = []
coord_indices = {}
matched_reports = []
matching_result = []
lookupPool = Pool(50)
for found, matched, report in lookupPool.imap(lookup_and_match,
((r, report_set['repository'], report_set['readsetId'])
for r in report_set['reports'])):
if report['clinicalSignificance'] in ('Uncertain significance', 'not provided', 'conflicting data from submitters', 'other'):
continue
if not found:
new_coord = Coordinate(report['chrom'], report['seqStart'], report['seqEnd'])
coordinates, coord_index = push_coordinates(coordinates, new_coord)
coord_indices.setdefault(coord_index, []).append(report)
elif matched:
matched_reports.append(report)
read_search = partial(make_read_search,
report_set['repository'],
report_set['readsetId'])
def search_and_match(coord_index):
coord = coordinates[coord_index]
result = read_search(coord, coord_index)
reads = ijson.items(HTTPStream(result), 'reads.item')
reports = coord_indices[coord_index]
reports_visited = []
for read in reads:
read_coord = Coordinate(read['referenceSequenceName'],
read['position'],
read['position']+get_ref_length(read['cigar']))
covered_reports = [report for report in reports
if (read_coord.chrom == report['chrom'] and
read_coord.start <= report['seqStart'] and
read_coord.end > report['seqEnd'])]
if covered_reports:
new_matched_reports = [report for report in covered_reports
if (report['reportId'] not in reports_visited and
match(report, read))]
matched_reports.extend(new_matched_reports)
reports_visited.extend([report['reportId'] for report in covered_reports
if report['reportId'] not in reports_visited])
#push read into cache:
read['repository'] = report_set['repository']
read['readsetId'] = report_set['readsetId']
read['start'] = read['position']
read['end'] = read['position'] + get_ref_length(read['cigar'])
CachedReads.save(read)
if len(reports_visited) >= len(reports):
break
searchPool = Pool(50)
(_ for _ in searchPool.imap(search_and_match, coord_indices.keys()))
return Response(json.dumps(matched_reports), content_type='application/json; charset=UTF-8')
def all(self):
users, orgs, groups, instanceapps, networks = preload_instance_data()
p = Pool(20)
instances = []
def _get_instances(cluster):
try:
instances.extend(cluster.get_instances())
except (GanetiApiError, Exception):
pass
finally:
close_connection()
clusters = Cluster.objects.all()
p.imap(_get_instances, clusters)
p.join()
return instances
def test():
ids = image_search("moon")
# let's be nice to archive.org and limit concurrent requests to 2
gpool = Pool(2)
for urls in gpool.imap(get_image_urls, ids):
print urls
def _path_to_xb(self, paths):
d = self.embedding_service.dim()
xb = np.ndarray(shape=(len(paths), d), dtype=np.float32)
p = Pool(12)
for i, emb in enumerate(p.imap(path_to_embedding, paths)):
xb[i] = emb
return xb
def call_main():
rawsql = None
try:
import sys
import getopt
options, args = getopt.getopt(sys.argv[1:], "hr:", ['help', "raw="])
for name, value in options:
if name in ('-h', '--help'):
print("error")
elif name in ('-r', '--raw'):
rawsql = value
except getopt.GetoptError:
print("error")
ora_rawsql = "select * from test where rownum < 10"
ora_conn = OraPlumberConn(user=ora_con_dict['user'],
passwd=ora_con_dict['passwd'],
schema=ora_con_dict['schema'])
ora_conn.execute(ora_rawsql)
pg_rawsql = "select * from test limit 10"
pg_conn = PgPlumberConn(database=pg_con_dict['database'],
user=pg_con_dict['user'],
password=pg_con_dict['password'],
host=pg_con_dict['host'],
port=pg_con_dict['port'])
pg_conn.execute(pg_rawsql)
insert_sql = 'insert into test values {}'
process = ProcessInsert(pg_conn, insert_sql)
from gevent.pool import Pool
p = Pool(4)
for _d in p.imap(process.handle, ora_conn.find_batch()):
pass
def job(): # gevent.joinall([gevent.spawn(dosomething1,2),gevent.spawn(dosomething2,2)]) import time import gevent from gevent.pool import Pool from gevent import threadpool # todo：注意map的 Pool和threadPool的区别。 pool 处理 非IO 耗时计算时没有 显式gevent.sleep() 就会变成同步程序。。。换成threadPool pool = Pool(3) start = time.time() # for _ in xrange(2): # pool.add(gevent.spawn(time.sleep, 1)) pool.imap(time.sleep,xrange(3)) gevent.wait() delay = time.time() - start print 'Running "time.sleep(1)" 4 times with 3 threads. Should take about 2 seconds: %.3fs' %delay
def main():
top_story_ids = requests.get(
"{}/topstories.json".format(API_PREFIX)).json()
pool = Pool(50)
rust_stories = list(
filter(lambda story: "Rust" in story.get("title", ""),
pool.imap(fetch_story, top_story_ids)))[:MAX_COUNT]
stories_length = len(rust_stories)
if stories_length < MAX_COUNT:
existed_story_ids = set(map(int, redis.lrange(REDIS_KEY, 0, -1)))
existed_story_ids -= set(item["id"] for item in rust_stories)
rust_stories.extend(
pool.imap(fetch_story,
list(existed_story_ids)[:MAX_COUNT - stories_length]))
redis.lpush(REDIS_KEY, *[item["id"] for item in rust_stories])
redis.ltrim(REDIS_KEY, 0, MAX_COUNT - 1)
render(rust_stories)
def main():
top_story_ids = requests.get(
"{}/topstories.json".format(API_PREFIX)).json()
pool = Pool(50)
rust_stories = list(
filter(lambda story: "Rust" in story.get("title", ""),
pool.imap(fetch_story, top_story_ids)))[:MAX_COUNT]
stories_length = len(rust_stories)
if stories_length < MAX_COUNT:
existed_story_ids = set(map(int, redis.lrange(REDIS_KEY, 0, -1)))
existed_story_ids -= set(item["id"] for item in rust_stories)
rust_stories.extend(
pool.imap(fetch_story,
list(existed_story_ids)[:MAX_COUNT - stories_length]))
redis.lpush(REDIS_KEY, *[item["id"] for item in rust_stories])
redis.ltrim(REDIS_KEY, 0, MAX_COUNT - 1)
render(rust_stories)
def calc_pi(tries, n):
# ------------------------------------------------------------------------------------------
from gevent.pool import Pool
pool = Pool()
result = pool.imap(test, [tries]*n) # gevent
# ------------------------------------------------------------------------------------------
pi = 4.0 * sum(result)/(tries * n)
return pi
def calc_pi(tries, n):
# ------------------------------------------------------------------------------------------
from gevent.pool import Pool
pool = Pool()
result = pool.imap(test, [tries] * n) # gevent
# ------------------------------------------------------------------------------------------
pi = 4.0 * sum(result) / (tries * n)
return pi
def _load_records(self, options):
pool_size = self.settings['__workers__']
connection_pool = Pool(size=pool_size)
for device_info in connection_pool.imap(self.prepare_device,
self.get_all_devices(),
maxsize=pool_size):
yield device_info
def all(self):
users, orgs, groups, instanceapps, networks = preload_instance_data()
p = Pool(20)
instances = []
bad_clusters = []
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instances.extend(cluster.get_instances())
except (GanetiApiError, Timeout):
pass
finally:
t.cancel()
clusters = Cluster.objects.all()
p.imap(_get_instances, clusters)
p.join()
return instances
def run_with_imap(cls):
__greenlet_number = 5
pool = Pool(size=__greenlet_number)
# result = pool.imap(cls.main_run_fun, ("index_1", "index_2", "index_3", "index_4", "index_5", "index_6"))
# result = pool.imap(cls.main_run_fun, (("index_1", "index_2"), ("index_3", "index_4"), ("index_5", "index_6")))
result = pool.imap(cls.main_run_fun_with_args,
(("index_1", "index_2"), ("index_3", "index_4"),
("index_5", "index_6")))
result_list = [r for r in result]
print("result: ", result_list)
def _load_records(self, options):
pool_size = self.settings['__workers__']
connection_pool = Pool(size=pool_size)
for device_info in connection_pool.imap(self.fetch_asset_details, self.fetch_asset_ids(), maxsize=pool_size):
if device_info:
yield device_info
else:
raise StopIteration
def all(self):
users, orgs, groups, instanceapps, networks = preload_instance_data()
p = Pool(20)
instances = []
bad_clusters = []
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instances.extend(cluster.get_instances())
except (GanetiApiError, Timeout):
pass
finally:
t.cancel()
clusters = Cluster.objects.all()
p.imap(_get_instances, clusters)
p.join()
return instances
def cmd_assistant_discovery_queue(args):
with init_client(args) as s:
web = s.get_web_session()
if not web:
LOG.error("Failed to get web session")
return 1 # error
sessionid = web.cookies.get('sessionid',
domain='store.steampowered.com')
LOG.info("Generating new discovery queue...")
try:
data = web.post(
'https://store.steampowered.com/explore/generatenewdiscoveryqueue',
{
'sessionid': sessionid,
'queuetype': 0
}).json()
except Exception as exp:
LOG.debug("Exception: %s", str(exp))
data = None
if not isinstance(data, dict) or not data.get('queue', None):
LOG.error("Invalid/empty discovery response")
return 1 # error
def explore_app(appid):
for delay in (1, 3, 5, 8, 14):
resp = web.post('https://store.steampowered.com/app/10', {
'appid_to_clear_from_queue': appid,
'sessionid': sessionid
})
if resp.status_code == 200:
return True
LOG.warning(
'Failed to explore app %s, retrying in %s second(s)',
appid, delay)
s.sleep(delay)
return False
pool = Pool(6)
result = pool.imap(explore_app, data['queue'])
if all(result):
LOG.info("Discovery queue explored successfully")
else:
LOG.error("Failed to explore some apps, try again")
return 1 #error
def periodic_parsing(self):
major_name_list = []
pool = Pool(70)
for major in self.all_major:
if major.on_off and major.should_parse():
major_name_list.append(major.major_name)
for _ in pool.imap(self.parsing_major_name, major_name_list):
pass
def fetch_package_infos(self):
self.load()
get_url = urllib2.urlopen
pool = Pool(self.concurrency)
pkg_idx = self.package_index
if not pkg_idx:
pkg_idx = self._fetch_package_index()
pkg_info_map = self.pkg_info_map
if not pkg_info_map:
pkg_info_map = PackageInfoMap(pkg_idx=pkg_idx)
pkg_info_map.path = self.package_info_path
pb = ProgressBar(widgets=[Percentage(),
' ', Bar(),
' ', SimpleProgress()],
maxval=len(pkg_idx) + 1)
pb.start()
pb.update(len(pkg_info_map))
to_fetch = sorted(set(pkg_idx.package_rel_urls) -
set(pkg_info_map.pkg_infos.viewkeys()))
def _get_package_info(package_rel_url):
pkg_url = self.default_pypi_url + 'pypi/%s/json' % package_rel_url
try:
resp = get_url(pkg_url)
except Exception as e:
ret = {'error': repr(e)}
else:
ret = json.loads(resp.read())
ret['rel_url'] = package_rel_url
return ret
pkg_info_iter = pool.imap(_get_package_info, to_fetch)
err_count = 0
for pkg_info in pkg_info_iter:
try:
pkg_info_map.add_dict(pkg_info)
except KeyError:
err_count += 1
pkg_info_map.add_dict(pkg_info)
pb.update(len(pkg_info_map))
if len(pkg_info_map) % self.concurrency == 0:
pkg_info_map.save()
pool.join(timeout=0.3, raise_error=True)
print 'Done fetching. Saving', len(pkg_info_map), 'package infos.'
try:
pkg_info_map.save()
except Exception:
print ExceptionInfo.from_current().get_formatted()
import pdb;pdb.post_mortem()
import pdb;pdb.set_trace()
return
def load_devices_api_v1(self, *a, **kw):
for partition in self.fetch_all_partitions():
if self.settings['partitions'] == "All" or partition[
'name'] in self.settings['partitions']:
pool_size = self.settings.get('__workers__', 2)
connection_pool = Pool(size=pool_size)
for device in connection_pool.imap(
self.load_hardware_and_serial_for_windows_devices,
self.fetch_all_devices_for_partition(partition['id']),
maxsize=pool_size):
yield device
else:
LOG.debug("Skipping partition %r", partition)
def _load_records(self, options):
pool_size = self.settings['__workers__']
connection_pool = Pool(size=pool_size)
# fetch all the assets ordered by unique sys_id value
all_asset_url = self.settings['url'] + "/api/now/table/alm_asset?" \
"sysparm_display_value=all&" \
"sysparm_query=^ORDERBYsys_id"
for asset in connection_pool.imap(self.prepare_asset_payload,
self.paginator(all_asset_url),
maxsize=pool_size):
yield asset
def messages_p2p(config: Config) -> None:
login_pool = Pool(size=config.number_of_users)
number_of_senders = config.number_of_users // 2
number_of_receivers = config.number_of_users // 2
# login senders and receivers concurrently
senders_users_results = login_pool.imap(
new_user,
repeat(config.sender_matrix_server_url, times=number_of_senders))
receivers_users_results = login_pool.imap(
new_user,
repeat(config.receiver_matrix_server_url, times=number_of_receivers))
senders = list(senders_users_results)
receivers = list(receivers_users_results)
invite_pool = Pool(size=config.number_of_concurrent_chat_rooms)
rooms = list(login_pool.imap(init_clients_and_rooms, senders, receivers))
invite_pool.join()
message_pool = Pool(size=config.number_of_parallel_messages)
for room in rooms:
message_pool.spawn(send, room)
message_pool.join()
def generate_arrays_from_file():
'''
读取图片
:return:
'''
files = glob("./data/image/*.png")
files = files
x = []
pool = Pool(size=100)
res = pool.imap(lambda filename: img_to_array(load_img(filename)) / 255,
files)
for img_area in res:
x.append(img_area)
x = np.array(x)
return x
def prepare_clusternodes(cluster=None):
if not cluster:
clusters = Cluster.objects.all()
else:
clusters = Cluster.objects.filter(slug=cluster)
p = Pool(15)
nodes = []
bad_clusters = []
bad_nodes = []
def _get_nodes(cluster):
try:
for node in cluster.get_cluster_nodes():
nodes.append(node)
if node['offline'] is True:
bad_nodes.append(node['name'])
except (GanetiApiError, Exception):
cluster._client = None
bad_clusters.append(cluster)
finally:
close_connection()
p.imap(_get_nodes, clusters)
p.join()
return nodes, bad_clusters, bad_nodes
def parsing_all(self):
#-----조회할 데이터 옵션 선택-----#
pool = Pool(70)
# self.course_info_list = list()
params_list = []
for i in range(len(self.gubun_list)):
if i == 0:
for j in range(len(self.major_code_list)):
params = {
'tab_lang': 'K',
'type': '',
'ag_ledg_year': self.default_year, # 년도
'ag_ledgr_sessn':
self.default_session, # 1=1학기, 2=여름계절, 3=2학기, 4=겨울계절
'ag_org_sect':
'A', # A=학부, B=대학원, D=통번역대학원, E=교육대학원, G=정치행정언론대학원, H=국제지역대학원, I=경영대학원(주간), J=경영대학원(야간), L=법학전문대학원, M=TESOL대학원, T=TESOL전문교육원
'campus_sect': 'H1', # H1=서울, H2=글로벌
'gubun': self.gubun_list[i], # 1=전공/부전공, 2=실용외국어/교양과목
'ag_crs_strct_cd': self.major_code_list[j], # 전공 목록
'ag_compt_fld_cd': '' # 교양 목록
}
params_list.append(params)
# self.major_data = list(self.parsing(params))
else:
for k in range(len(self.liberal_code_list)):
params = {
'tab_lang': 'K',
'type': '',
'ag_ledg_year': self.default_year, # 년도
'ag_ledgr_sessn':
self.default_session, # 1=1학기, 2=여름계절, 3=2학기, 4=겨울계절
'ag_org_sect':
'A', # A=학부, B=대학원, D=통번역대학원, E=교육대학원, G=정치행정언론대학원, H=국제지역대학원, I=경영대학원(주간), J=경영대학원(야간), L=법학전문대학원, M=TESOL대학원, T=TESOL전문교육원
'campus_sect': 'H1', # H1=서울, H2=글로벌
'gubun': self.gubun_list[i], # 1=전공/부전공, 2=실용외국어/교양과목
'ag_crs_strct_cd': '', # 전공 목록
'ag_compt_fld_cd': self.liberal_code_list[k] # 교양 목록
}
params_list.append(params)
for _ in pool.imap(self.parsing, params_list):
pass
def device_page_generator(self, options):
"""
Page info generator
:param options:
:return:
"""
pool_size = self.settings['__workers__']
connection_pool = Pool(size=pool_size)
for page in connection_pool.imap(
self.get_device_page_info,
self.device_page_url_generator(options),
maxsize=pool_size):
if not page:
raise StopIteration
yield page
def collect_friends(self, uid=None, level=1):
slot = []
slot_dict = {}
uid = uid or self.renren.uid
uid = str(uid)
def get_fs_by_level(lv):
return filter(lambda s: s.level == lv, slot)
@gtimeout(360, mute=True)
def _collect(fo):
friends = set(self.renren.get_friends(fo.uid))
if uid in friends:
friends.remove(uid)
fo.friends = friends
return fo
fs = FriendsStore(uid, 0)
slot.append(fs)
def find_fs(uid, lv, parent):
if uid not in slot_dict:
slot_dict[uid] = FriendsStore(uid, lv, parent)
return slot_dict[uid]
pool = Pool(30)
for l in range(level):
pool_jobs = pool.imap(_collect, get_fs_by_level(l))
for fo in pool_jobs:
if not fo:
continue
slot.extend(
[find_fs(u, l+1, fo.uid) for u in fo.friends]
)
print 'collect done'
slot.pop(0)
return slot
def main():
net_devices = get_devices()
start_time = datetime.now()
results = []
pool = Pool(4)
for a_result in pool.imap(run_task, (a_device for a_device in net_devices.values())):
results.append(a_result)
elapsed_time = datetime.now() - start_time
print('\n--- Host task times ---')
for a_result in results:
if 'exception' in a_result:
print(f"{a_result['host']} - {a_result['exception']}")
#print(a_result['traceback'])
else:
print(f"{a_result['host']} - {a_result['ssh_runtime']}")
print(f"\nTotal Elapsed time: {format(elapsed_time)}")
def main():
p = Pool(20)
r = s.get("https://api.epandda.org/annotations?limit=1")
r.raise_for_status()
total = r.json()["counts"]["totalCount"]
print(total)
fetched = 0
result_iter = p.imap(get_url, urls(total))
with jsonlines.open("/home/godfoder/Downloads/annotations.jsonl",
mode="w") as writer:
for res in result_iter:
fetched += len(res)
writer.write_all(res)
if fetched % 10000 == 0:
print(fetched)
print(fetched)
def call_main():
rawsql = None
try:
import sys
import getopt
options, args = getopt.getopt(sys.argv[1:], "hr:", ['help', "raw="])
for name, value in options:
if name in ('-h', '--help'):
print("error")
elif name in ('-r', '--raw'):
rawsql = value
except getopt.GetoptError:
print("error")
ora_rawsql = "select * from test where rownum < 10"
ora_conn = OraPlumberConn(
user=ora_con_dict['user'],
passwd=ora_con_dict['passwd'],
schema=ora_con_dict['schema'])
ora_conn.execute(ora_rawsql)
pg_rawsql = "select * from test limit 10"
pg_conn = PgPlumberConn(
database=pg_con_dict['database'],
user=pg_con_dict['user'],
password=pg_con_dict['password'],
host=pg_con_dict['host'],
port=pg_con_dict['port'])
pg_conn.execute(pg_rawsql)
insert_sql = 'insert into test values {}'
process = ProcessInsert(pg_conn, insert_sql)
from gevent.pool import Pool
p = Pool(4)
for _d in p.imap(process.handle, ora_conn.find_batch()):
pass
def _load_records(self, options):
if self.settings.get('dep_uuid'):
# if the dep_uuid is given, we have to retrieve the different subset of devices from the separate API
# it is not clear from the docs if the API supports pagination, looks like not
# also this API is supported only by the AirWatch starting from 9.2(?)
dep_api_url = '%s/api/mdm/dep/groups/%s/devices' % (
self.settings['url'], self.settings['dep_uuid'])
self.dep_devices = {
_['deviceSerialNumber']: _
for _ in self.get(dep_api_url).json()
}
pool_size = self.settings['__workers__']
connection_pool = Pool(size=pool_size)
for device_info in connection_pool.imap(
self.retrieve_device_info,
self.device_page_generator(options),
maxsize=pool_size):
if not device_info:
raise StopIteration
yield device_info
data = str(random.randint(10, 10000))
w.write('%s, from client %d\n' % (data, cid))
w.flush()
print 'client', cid, 'send:', data
def recv():
while True:
line = r.readline()
print 'client', cid, 'recive:', line,
if not line:
break
send_job = gevent.spawn_later(1, send)
recv_job = gevent.spawn(recv)
def clear(*args):
gevent.killall([send_job, recv_job])
s.close()
send_job.link(clear)
recv_job.link(clear)
gevent.joinall([send_job, recv_job])
print 'client', cid, 'finish'
clients = pool.imap(client, xrange(1000))
gevent.spawn_later(60, lambda: clients.kill()).start()
clients.start()
gevent.run()
def get_user_group_list(request):
if request.user.is_superuser or request.user.has_perm('ganeti.view_instances'):
q_params = None
try:
q_params = request.GET['q']
except:
pass
users = User.objects.all()
groups = Group.objects.all()
instances = []
clusters = Cluster.objects.all()
p = Pool(20)
bad_clusters = []
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instances.extend(cluster.get_user_instances(request.user))
except (GanetiApiError, Timeout):
bad_clusters.append(cluster)
finally:
t.cancel()
if not request.user.is_anonymous():
p.imap(_get_instances, Cluster.objects.all())
p.join()
if q_params:
users = users.filter(username__icontains=q_params)
groups = groups.filter(name__icontains=q_params)
instances = Instance.objects.filter(name__icontains=q_params)
clusters = clusters.filter(slug__icontains=q_params)
ret_list = []
for user in users:
userd = {}
userd['text']=user.username
userd['email']=user.email
userd['id']="u_%s"%user.pk
userd['type']="user"
ret_list.append(userd)
for group in groups:
groupd = {}
groupd['text']=group.name
groupd['id']="g_%s"%group.pk
groupd['type']="group"
ret_list.append(groupd)
for instance in instances:
instd = {}
instd['text']=instance.name
instd['id']="i_%s"%instance.name
instd['type']="vm"
ret_list.append(instd)
for cluster in clusters:
cld = {}
cld['text']=cluster.slug
cld['id']="c_%s"%cluster.pk
cld['type']="cluster"
ret_list.append(cld)
action = ret_list
return HttpResponse(json.dumps(action), mimetype='application/json')
else:
action = {'error':"Permissions' violation. This action has been logged and our admins will be notified about it"}
return HttpResponse(json.dumps(action), mimetype='application/json')
class OssUtils(object):
def __init__(self, public_net=True, max_conn_pool=10):
# default connection pool size
# the pool size mast bg or eq than put&get thread number.
"""
init function
:param public_net: use public or aliyun private net.
:param max_conn_pool: max num of threads
"""
oss2.defaults.connection_pool_size = max_conn_pool
self.is_public_net = public_net
self.access_key_id = 'LTAIipB2qbeW3V99'
self.access_key_secret = 'Dtyx9XKvFdQvmniSlns18j7lXpmA03'
self.bucket_name = 'mojimeteo'
if public_net:
# public net like bj29 use this.
# DO NOT USE THIS ADDRESS TO DOWNLOAD.
self.endpoint = 'oss-cn-beijing.aliyuncs.com'
else:
# ali private net use this.
self.endpoint = 'vpc100-oss-cn-beijing.aliyuncs.com'
self.bucket = oss2.Bucket(
oss2.Auth(self.access_key_id, self.access_key_secret),
self.endpoint, self.bucket_name)
self.greenlet_pool = Pool(max_conn_pool)
def get_obj(self,
key,
dst,
use_resume=True,
part_size=(20 * 1024 * 1024),
num_threads=4):
"""
get files from oss,
:param key: oss key
:param dst: The path to save obj.
:return obj save path at last
"""
try:
if self.is_public_net:
raise Exception('Do not download from public')
if use_resume:
oss2.resumable_download(
self.bucket,
key,
dst,
store=oss2.ResumableDownloadStore(root='/tmp'),
multiget_threshold=20 * 1024 * 1024,
part_size=part_size,
num_threads=num_threads)
else:
self.bucket.get_object_to_file(key, dst)
return dst
except Exception as ex:
print ex.message
return None
def put_obj(self,
key,
src,
use_resume=True,
part_size=(20 * 1024 * 1024),
num_threads=4):
"""
put file to oss
:param key:
:param src:
"""
# use resume
try:
if use_resume:
oss2.resumable_upload(self.bucket,
key,
src,
store=oss2.ResumableStore(root='/tmp'),
multipart_threshold=100 * 1024,
part_size=part_size,
num_threads=num_threads)
else:
self.bucket.put_object_from_file(key, src)
except Exception as ex:
print ex.message
def exists(self, key):
"""
judge if the key exists
:param key:
:return:
"""
return self.bucket.object_exists(key)
def copy_obj(self, src_key, dst_key):
"""
copy obj from src_key to dst_key
:param key:
:return:
"""
return self.bucket.copy_object(self.bucket_name, src_key, dst_key)
def list_obj(self, prefix='', delimiter='/', max_count=1000):
"""
return a iterator in bucket
:param prefix:
:param max_count:
:return:
"""
return islice(
oss2.ObjectIterator(self.bucket,
delimiter=delimiter,
prefix=prefix), max_count)
def get_objs(self, src_root, dst_root):
"""
get objs in oss
"""
monkey.patch_socket()
task_list = []
if src_root.startswith('/'):
src_root = src_root[1:]
if not src_root.endswith('/'):
src_root += '/'
# make local dir if not exist
def _touch_dir(path):
result = False
try:
path = path.strip().rstrip("\\")
if not os.path.exists(path):
os.makedirs(path)
result = True
else:
result = True
except:
result = False
return result
_touch_dir(dst_root)
def _get_objs(key):
for obj in self.list_obj(key):
# remove prefix of src_root
_rel_path = obj.key[obj.key.index(src_root) + len(src_root):]
local_obj = os.path.join(dst_root, _rel_path)
if obj.is_prefix(): # directory
_touch_dir(local_obj)
_get_objs(obj.key)
else: # file
task_list.append((obj.key, local_obj, False))
self.get_obj(obj.key, local_obj, False)
print 'file: %s->%s' % (obj.key, local_obj)
_get_objs(src_root)
tasks = self.greenlet_pool.imap(self.get_obj, task_list)
tasks.join()
def put_objs(self, src_root, dst_root):
monkey.patch_all()
task_list = []
if dst_root.startswith('/'):
dst_root = dst_root[1:]
def _put_objs(_rel_path):
local_path = os.path.join(src_root, _rel_path)
filelist = os.listdir(local_path)
for filename in filelist:
oss_path = os.path.join(dst_root, _rel_path)
key = os.path.join(oss_path, filename)
local_obj = os.path.join(local_path, filename)
_rel_path_obj = os.path.join(_rel_path, filename)
if os.path.isdir(local_obj):
_put_objs(_rel_path_obj)
else:
self.greenlet_pool.add(
gevent.spawn(self.put_obj, key, local_obj, False))
_put_objs('')
self.greenlet_pool.join()
zfile = '%s.zip' % site_id
with zipfile.ZipFile(zfile) as z:
z.extractall('tmp')
file_workers = [
pool.spawn(upload_files, i, worker_id, file_queue) for i in xrange(concurrency)
]
for dirname, dirnames, filenames in os.walk('tmp/%s' % site_id):
# print path to all subdirectories first.
files = []
for filename in filenames:
files.append(os.path.join(dirname, filename))
for f in files:
file_queue.put(f, block=False)
print "START_DIRS"
dirs = []
for subdirname in dirnames:
dirs.append(os.path.join(dirname, subdirname))
if dirs:
print "POOLING:", dirs
dir_pool.imap(mkdirs, dirs)
print "END"
#joinall(dir_jobs)
#joinall([
# spawn([s_dir] + dirs) for s_dir, dirs in skel_dirs.iteritems()
#])
file_queue.join()
'support', 'tags', 'wins')
h_stats = {k+"_per_hour": float(data[k])/float(data['hours']) for k in per_stat_cols if float(data['hours']) > 0}
g_stats = {k+"_per_game": float(data[k])/float(data['games']) for k in per_stat_cols if float(data['games']) > 0}
return dict(data.items()+h_stats.items()+g_stats.items())
player.set(last_updated=datetime.now(), **extract_data(all))
if player.monthly is not None:
player.monthly.set(**extract_data(monthly))
else:
Monthly(player=player, **extract_data(monthly))
if player.weekly is not None:
player.weekly.set(**extract_data(weekly))
else:
Weekly(player=player, **extract_data(weekly))
if player.daily is not None:
player.daily.set(**extract_data(daily))
else:
Daily(player=player, **extract_data(daily))
c+=1
if not c%100:
print "{}/{} done.".format(c, total)
with db_session:
players = select(x for x in Players)
p = Pool(10)
total = len(players)
res = p.imap(update_player, players)
gevent.wait([res])
print "Committing..."
commit()
print "Done."
client = paramiko.SSHClient()
client.get_host_keys()
client.set_missing_host_key_policy(paramiko.WarningPolicy())
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(host, port=22)
stdin, stdout, stderr = client.exec_command('uptime')
result = stdout.read()
client.close()
return host, result
if __name__ == '__main__':
if len(sys.argv) < 2:
sys.exit('Usage: %s filename' % sys.argv[0])
file_name = sys.argv[1]
if not os.path.exists(file_name):
sys.exit('ERROR: data file %s was not found!' % sys.argv[1])
pool = Pool(25)
hosts = []
with open(file_name) as fh:
for line in fh:
hosts.append(line.strip())
for host, result in pool.imap(uptime, hosts):
print "Host: %s" % host, "Response: ", result
fmt = struct.Struct('>i%ds' % data_len)
data = fmt.pack(data_len, data)
s.sendall(data)
print 'client', cid, 'send:', nums
def recv():
while True:
data = s.recv(4)
data = head_fmt.unpack(data)
length = data[0]
data = s.recv(length)
print 'client', cid, 'recive:', data
send_job = gevent.spawn_later(1, send)
recv_job = gevent.spawn(recv)
def clear(*args):
gevent.killall([send_job, recv_job])
s.close()
send_job.link(clear)
recv_job.link(clear)
gevent.joinall([send_job, recv_job])
print 'client', cid, 'finish'
clients = pool.imap(client, xrange(1000))
gevent.spawn_later(60, lambda: clients.kill()).start()
clients.start()
gevent.run()
def get_user_group_list(request):
if request.user.is_superuser or request.user.has_perm(
'ganeti.view_instances'):
q_params = None
try:
q_params = request.GET['q']
except:
pass
users = User.objects.all()
groups = Group.objects.all()
instances = []
clusters = Cluster.objects.all()
p = Pool(20)
bad_clusters = []
def _get_instances(cluster):
t = Timeout(RAPI_TIMEOUT)
t.start()
try:
instances.extend(cluster.get_user_instances(request.user))
except (GanetiApiError, Timeout):
bad_clusters.append(cluster)
finally:
t.cancel()
if not request.user.is_anonymous():
p.imap(_get_instances, Cluster.objects.all())
p.join()
if q_params:
users = users.filter(username__icontains=q_params)
groups = groups.filter(name__icontains=q_params)
instances = Instance.objects.filter(name__icontains=q_params)
clusters = clusters.filter(slug__icontains=q_params)
ret_list = []
for user in users:
userd = {}
userd['text'] = user.username
userd['email'] = user.email
userd['id'] = "u_%s" % user.pk
userd['type'] = "user"
ret_list.append(userd)
for group in groups:
groupd = {}
groupd['text'] = group.name
groupd['id'] = "g_%s" % group.pk
groupd['type'] = "group"
ret_list.append(groupd)
for instance in instances:
instd = {}
instd['text'] = instance.name
instd['id'] = "i_%s" % instance.name
instd['type'] = "vm"
ret_list.append(instd)
for cluster in clusters:
cld = {}
cld['text'] = cluster.slug
cld['id'] = "c_%s" % cluster.pk
cld['type'] = "cluster"
ret_list.append(cld)
action = ret_list
return HttpResponse(json.dumps(action), mimetype='application/json')
else:
action = {
'error':
"Permissions' violation. This action has been logged and our admins will be notified about it"
}
return HttpResponse(json.dumps(action), mimetype='application/json')
break
#print "User: {user}; playtime: {playtime}; WaitTime: {wait}; Action:{action}".format(user=uid, playtime=playing_time,
# wait=sleep_time, action=action)
gevent.sleep(sleep_time)
if action == "stop":
inst.stop_instance()
elif action == "noinput":
inst.notify_instance('20')
elif action == "crash":
inst.notify_instance('11')
else:
pass
print getcurrent()
pool = Pool(parallen)
pool.imap(cloud_play, range(1, 400))
weight['overtime'] = 0
while now_time < end_time - 30000:
time.sleep(2)
free_num = pool.free_count()
print "==========", free_num
if free_num > 0:
pool.imap(cloud_play, range(end_num, end_num + free_num))
end_num += free_num
now_time = int(time.time() * 1000)
def multi(self, F, iterable): pool = Pool(size=self.pool_size) res = pool.imap(F, iterable) return res
links.append({'source':source,
'target':target,
'value':value,
'count':rec['count'],
'count2':rec['count2'],
'percentile': percentile
})
return links
if __name__=="__main__":
members = get_all_members()
members = ( member for member in members if member['is_current'] )
reduced_member_info=[]
full_member_info = pool.imap(get_single_member,members)
for info in full_member_info:
reduced_member_info.append( {
'id': info['id'],
'party':int(info['party_url'].split('/')[-2]),
'name':"%(name)s" % info,
'fullname':"%(name)s - %(party_name)s" % info
} )
reduced_member_info = dict((x['id'],x) for x in reduced_member_info)
simple_bill_info = get_simple_bill_info()
full_bill_info = pool.imap(get_full_bill,itertools.chain.from_iterable(simple_bill_info))
full_bill_info = (bill for bill in full_bill_info if int(bill['stage_date'].split('-')[0])>=2013)
proposer_list = ([ int(x.split('/')[-2]) for x in bill['proposers'] ] for bill in full_bill_info)
links = process_links(proposer_list,reduced_member_info)
