def _fetch_lines(path: str, seq_num: int, ip1: str, ip2: str = None):
with rpc.ServerProxy("http://%s:8001/" % ip1) as proxy:
multicall = rpc.MultiCall(proxy)
multicall.fetching(path, seq_num)
file_string = tuple(multicall())[0]
print(file_string)
if file_string:
with open(path, 'a') as f:
for elem in file_string:
f.write(elem + '\n')
f.close()
return
if ip2 is not None:
with rpc.ServerProxy("http://%s:8001/" % ip2) as proxy:
multicall = rpc.MultiCall(proxy)
multicall.fetching(path, seq_num)
file_string = tuple(multicall())[0]
print(file_string)
if file_string:
with open(path, 'a') as f:
for elem in file_string:
f.write(elem + '\n')
f.close()
return
def get_release_info(packages, json=False):
"""
"""
if json:
for package in packages:
data = get_jsonparsed_data(PYPI_JSON % package)
for release in data['releases']:
urls = data['releases'][release]
yield urls, data['info']
return
mcall = xmlrpc.MultiCall(PYPI_XML)
i = 0
for package, releases in get_releases(packages):
for version in releases:
mcall.release_urls(package, version)
mcall.release_data(package, version)
i += 1
if i % 50 == 49:
result = mcall()
mcall = xmlrpc.MultiCall(PYPI_XML)
for urls, data in by_two(result):
yield urls, data
result = mcall()
for urls, data in by_two(result):
yield urls, data
def get_releases(packages):
"""
"""
mcall = xmlrpc.MultiCall(PYPI_XML)
called_packages = deque()
for package in packages:
mcall.package_releases(package, True)
called_packages.append(package)
if len(called_packages) == 100:
result = mcall()
mcall = xmlrpc.MultiCall(PYPI_XML)
for releases in result:
yield called_packages.popleft(), releases
result = mcall()
for releases in result:
yield called_packages.popleft(), releases
def game_is_updated(path, ip):
"""
I get pinged that an update is made, therefore I request the update of the machine that pinged me.
Parameters
----------
path: The machines share the same location, but the server does not know where the file is, so it must be
passed as an argument
ip: This is the IP that pinged and from which I must request now.
Returns
-------
"""
print('File got updated, I request now')
print(ip)
with rpc.ServerProxy("http://%s:8001/" % ip) as proxy:
multicall = rpc.MultiCall(proxy)
multicall.game_request(path)
multicall()
updated_game = tuple(multicall())[0]
print('test ', updated_game)
with open(path, 'a') as f:
f.write(updated_game + '\n')
f.close()
def call(self):
"""Execute added multicall calls.
@return: the results (post-processed), in the order they were added
@rtype: tuple
"""
m = xmlrpc_client.MultiCall(self.rt_obj._get_conn())
for call in self.calls:
method, args = call
rpc_call = getattr(method, 'rpc_call')
getattr(m, rpc_call)(*args)
results = m()
results = tuple(results)
results_processed = []
for r, c in list(zip(results, self.calls)):
method = c[0] # Method instance
result = process_result(method, r)
results_processed.append(result)
# assign result to class_obj
exists = hasattr(self.class_obj, method.varname)
if not exists or not inspect.ismethod(
getattr(self.class_obj, method.varname)):
setattr(self.class_obj, method.varname, result)
return (tuple(results_processed))
def update(self, info_hash, fields):
multi_call = xmlrpc_client.MultiCall(self._server)
for key, val in fields.items():
method_name = 'd.%s.set' % key
getattr(multi_call, method_name)(info_hash, val)
return multi_call()[0]
def request_new_game_file(path: str, ip: str):
with rpc.ServerProxy("http://%s:8001/" % ip) as proxy:
multicall = rpc.MultiCall(proxy)
multicall.game_request(path)
file_string = tuple(multicall())[0]
with open(path, 'a') as f:
f.write(file_string + '\n')
f.close()
def ping_the_updates(path, ip1, ip2, my_ip):
print('i am pinging', ip1, ip2, my_ip)
try:
with rpc.ServerProxy("http://%s:8001/" % ip1) as proxy:
multicall = rpc.MultiCall(proxy)
multicall.game_is_updated(path, my_ip)
multicall()
except:
print('No server connection with %s' % ip1)
if ip2 is not None:
try:
with rpc.ServerProxy("http://%s:8001/" % ip2) as proxy:
multicall = rpc.MultiCall(proxy)
multicall.game_is_updated(path, my_ip)
multicall()
except:
print('No server connection with %s' % ip2)
time.sleep(3)
def get_release_info(packages, json=False):
"""Retrieve statistics for package passed in by calling other methods.
Based on JSON toggle, retrieve package data as JSON or via XMLRPC requests
and return the data.
@param packages: List of package names
@type packages: list
@param json: JSON Toggle
@type json: bool
@r_param urls: Details of a particular release
@r_type urls: dict
@r_param data[info]/data: General information about package
@r_type data[info]/data: dict
"""
if json:
for package in packages:
data = get_jsonparsed_data(PYPI_JSON % package)
for release in data['releases']:
urls = data['releases'][release]
yield urls, data['info']
return
mcall = xmlrpc.MultiCall(PYPI_XML)
i = 0
for package, releases in get_releases(packages):
for version in releases:
mcall.release_urls(package, version)
mcall.release_data(package, version)
i += 1
if i % 50 == 49:
result = mcall()
mcall = xmlrpc.MultiCall(PYPI_XML)
for urls, data in by_two(result):
yield urls, data
result = mcall()
for urls, data in by_two(result):
yield urls, data
def test_multicall(self):
try:
p = xmlrpclib.ServerProxy(URL)
multicall = xmlrpclib.MultiCall(p)
multicall.add(2, 3)
multicall.pow(6, 8)
multicall.div(127, 42)
add_result, pow_result, div_result = multicall()
self.assertEqual(add_result, 2 + 3)
self.assertEqual(pow_result, 6**8)
self.assertEqual(div_result, 127 // 42)
except (xmlrpclib.ProtocolError, OSError) as e:
if not is_unavailable_exception(e):
self.fail('%s\n%s' % (e, getattr(e, 'headers', '')))
def test_non_existing_multicall(self):
try:
p = xmlrpclib.ServerProxy(URL)
multicall = xmlrpclib.MultiCall(p)
multicall.this_is_not_exists()
result = multicall()
self.assertEqual(result.results[0]['faultCode'], 1)
self.assertEqual(
result.results[0]['faultString'],
'<class \'Exception\'>:method "this_is_not_exists" is not supported'
)
except (xmlrpclib.ProtocolError, OSError) as e:
if not is_unavailable_exception(e):
self.fail('%s\n%s' % (e, getattr(e, 'headers', '')))
def get_releases(packages):
"""Retrieve and return package data via XMLRPC requests.
@param packages: List of packages
@type packages: list
@r_param "called_packages.popleft()": Called package ame
@r_type "called_packages.popleft()": str
@r_param releases: Items from Multicall Generator Object
@r_type releases: XMLRPC Proxy Object
"""
mcall = xmlrpc.MultiCall(PYPI_XML)
called_packages = deque()
for package in packages:
mcall.package_releases(package, True)
called_packages.append(package)
if len(called_packages) == 100:
result = mcall()
mcall = xmlrpc.MultiCall(PYPI_XML)
for releases in result:
yield called_packages.popleft(), releases
result = mcall()
for releases in result:
yield called_packages.popleft(), releases
def torrent(self, info_hash, fields=None):
"""Get the details of a torrent"""
if not fields:
fields = list(self.default_fields)
fields = self._clean_fields(fields)
multi_call = xmlrpc_client.MultiCall(self._server)
for field in fields:
method_name = 'd.%s' % field
getattr(multi_call, method_name)(info_hash)
resp = multi_call()
# TODO: Maybe we should return a named tuple or a Torrent class?
return dict(list(zip(self._clean_fields(fields, reverse=True), [val for val in resp])))
def test_multicall(self):
try:
p = xmlrpclib.ServerProxy(URL)
multicall = xmlrpclib.MultiCall(p)
multicall.add(2, 3)
multicall.pow(6, 8)
multicall.div(127, 42)
add_result, pow_result, div_result = multicall()
self.assertEqual(add_result, 2 + 3)
self.assertEqual(pow_result, 6**8)
self.assertEqual(div_result, 127 // 42)
except (xmlrpclib.ProtocolError, OSError) as e:
# ignore failures due to non-blocking socket 'unavailable' errors
if not is_unavailable_exception(e):
# protocol error; provide additional information in test output
self.fail("%s\n%s" % (e, getattr(e, "headers", "")))
def test_non_existing_multicall(self):
try:
p = xmlrpclib.ServerProxy(URL)
multicall = xmlrpclib.MultiCall(p)
multicall.this_is_not_exists()
result = multicall()
# result.results contains;
# [{'faultCode': 1, 'faultString': '<class \'exceptions.Exception\'>:'
# 'method "this_is_not_exists" is not supported'>}]
self.assertEqual(result.results[0]['faultCode'], 1)
self.assertEqual(result.results[0]['faultString'],
'<class \'Exception\'>:method "this_is_not_exists" '
'is not supported')
except (xmlrpclib.ProtocolError, socket.error) as e:
# ignore failures due to non-blocking socket 'unavailable' errors
if not is_unavailable_exception(e):
# protocol error; provide additional information in test output
self.fail("%s\n%s" % (e, getattr(e, "headers", "")))
def test_add_modules_used(self):
server = self.server
today = datetime.datetime.today()
today = today.strftime("%Y-%m-%d")
with self.assertRaises(xmlrpclib.Fault) as cm:
server.add_modules_used("tango", "aqws12", "jobid", [], today)
self.assertEqual(cm.exception.faultCode, 81)
self.assertEqual(cm.exception.faultString,
'Username and/or password is incorrect')
result = server.add_modules_used(
"tango", "aq12ws", "942337.tango-m.vpac.org", [
"modules", "pgi/12.10", "gmp/5.0.5", "mpfr/3.1.1", "mpc/1.0",
"gcc/4.7.2", "intel/12.1.3", "openmpi-pgi/1.6.3",
"vpac/config", "lapack/3.4.2", "octave/3.6.3"
], today)
self.assertEqual(result, True)
multicall = xmlrpclib.MultiCall(server)
multicall.add_modules_used(
"tango", "aq12ws", "942337.tango-m.vpac.org", [
"modules", "pgi/12.10", "gmp/5.0.5", "mpfr/3.1.1", "mpc/1.0",
"gcc/4.7.2", "intel/12.1.3", "openmpi-pgi/1.6.3",
"vpac/config", "lapack/3.4.2", "octave/3.6.3"
], today)
multicall.add_modules_used(
"tango", "aq12ws", "942338.tango-m.vpac.org", [
"modules", "pgi/12.10", "gmp/5.0.5", "mpfr/3.1.1", "mpc/1.0",
"gcc/4.7.2", "intel/12.1.3", "openmpi-pgi/1.6.3",
"vpac/config", "lapack/3.4.2", "octave/3.6.3"
], today)
result = tuple(multicall())
self.assertEqual(result, (True, True))
def call(self):
"""Execute added multicall calls
@return: the results (post-processed), in the order they were added
@rtype: tuple
"""
m = xmlrpclib.MultiCall(self.rt_obj._get_xmlrpc_conn())
for call in self.calls:
method, args = call
rpc_call = getattr(method, "rpc_call")
getattr(m, rpc_call)(*args)
results = m()
results = tuple(results)
results_processed = []
for r, c in zip(results, self.calls):
method = c[0] # Method instance
result = process_result(method, r)
results_processed.append(result)
# assign result to class_obj
setattr(self.class_obj, method.varname, result)
return (tuple(results_processed))
def cleanup(self, reason):
"""
Cleans up registrations with master and releases topic and service resources
@param reason: human-reasonable debug string
@type reason: str
"""
self.logger.debug("registration cleanup starting")
try:
self.cond.acquire()
self.cond.notifyAll()
finally:
self.cond.release()
# we never successfully initialized master_uri
if not self.master_uri:
return
master = xmlrpcapi(self.master_uri)
# we never successfully initialized master
if master is None:
return
caller_id = get_caller_id()
# clear the registration listeners as we are going to do a quick unregister here
rl = get_registration_listeners()
if rl is not None:
rl.clear()
tm = get_topic_manager()
sm = get_service_manager()
try:
multi = xmlrpcclient.MultiCall(master)
if tm is not None:
for resolved_name, _ in tm.get_subscriptions():
self.logger.debug("unregisterSubscriber [%s]"%resolved_name)
multi.unregisterSubscriber(caller_id, resolved_name, self.uri)
for resolved_name, _ in tm.get_publications():
self.logger.debug("unregisterPublisher [%s]"%resolved_name)
multi.unregisterPublisher(caller_id, resolved_name, self.uri)
if sm is not None:
for resolved_name, service_uri in sm.get_services():
self.logger.debug("unregisterService [%s]"%resolved_name)
multi.unregisterService(caller_id, resolved_name, service_uri)
multi()
except socket.error as se:
(errno, msg) = se.args
if errno == 111 or errno == 61: #can't talk to master, nothing we can do about it
self.logger.warn("cannot unregister with master due to network issues")
else:
self.logger.warn("unclean shutdown\n%s"%traceback.format_exc())
except:
self.logger.warn("unclean shutdown\n%s"%traceback.format_exc())
self.logger.debug("registration cleanup: master calls complete")
#TODO: cleanup() should actually be orchestrated by a separate
#cleanup routine that calls the reg manager/sm/tm
if tm is not None:
tm.close_all()
if sm is not None:
sm.unregister_all()
def migrate_tickets(self, ticket_range=None):
print("Loading information from Trac…", file=sys.stderr)
get_all_tickets = xmlrpclib.MultiCall(self.trac)
ticket_list = self.trac.ticket.query("max=0&order=id")
# Optional argument ticket_range (2-sequence) can restrict
# range of tickets to be processed
if ticket_range is None or ticket_range[0] is None or ticket_range[
1] is None:
first_ticket, last_ticket = min(ticket_list), max(ticket_list)
else:
first_ticket, last_ticket = ticket_range
for ticket in ticket_list:
# Only get tickets within requested range
if first_ticket <= ticket <= last_ticket:
get_all_tickets.ticket.get(ticket)
print(
f"Creating GitHub tickets {first_ticket} to {last_ticket} …",
file=sys.stderr,
)
for trac_id, time_created, time_changed, attributes in sorted(
get_all_tickets(), key=lambda t: int(t[0])):
# need to keep trac # in title to have unique titles
title = attributes["summary"]
r = self.get_github_username(attributes["reporter"])
if r == GithubObject.NotSet:
rep = attributes["reporter"]
else:
try:
rep = "@" + r.login
except:
rep = attributes["reporter"]
body = self.fix_wiki_syntax(attributes["description"])
body += "\n\n"
newCC = []
for u in attributes["cc"].strip().split(", "):
if u:
newU = self.get_github_username(u)
if newU is GithubObject.NotSet:
newCC.append(u)
else:
newCC.append("@" + newU.login)
if newCC:
body += "_cc: %s_\n" % " ".join(newCC)
body += "\n\n"
body += "_Reported by " + rep + ", migrated from %s\n" % urljoin(
self.trac_public_url, "ticket/%d" % trac_id + "_")
#text_attributes = {
# k: convert_value_for_json(v) for k, v in list(attributes.items())
#}
#body += "```json\n" + json.dumps(text_attributes, indent=4) + "\n```\n"
milestone = self.get_gh_milestone(attributes["milestone"])
assignee = self.get_github_username(attributes["owner"])
labels = []
# User does not exist in GitHub -> Add username as label
if assignee is GithubObject.NotSet and (
attributes["owner"] and attributes["owner"].strip()):
labels = self.get_mapped_labels("owner", attributes["owner"])
for attr in ("type", "component", "resolution", "priority",
"keywords"):
labels += self.get_mapped_labels(attr, attributes.get(attr))
if title in self.gh_issues and not self.is_dry_run:
# Set should_reassign_existing_issues=False when the script
# needs to run multiple times without assigning
# tickets (which is slow and error prone)
gh_issue = self.gh_issues[title]
print("\tIssue exists: %s" % str(gh_issue), file=sys.stderr)
if self.should_reassign_existing_issues:
if assignee is not GithubObject.NotSet and (
not gh_issue.assignee or
(gh_issue.assignee.login != assignee.login)):
print("\t\tChanging assignee: %s" % (assignee),
file=sys.stderr)
gh_issue.edit(assignee=assignee)
continue
else:
comments = self.get_trac_comments(trac_id)
if self.should_import_attachments:
# This will overwrite any comment that has the
# same timestamp as an attachment, but those
# should all be "changed attachment" edits so that
# is OK I think
comments.update(
self.get_trac_attachments_as_comments(trac_id))
if self.is_dry_run:
print(f"\tDry run for issue: {title}", file=sys.stderr)
print(f"\t\tAssignee: {assignee}", file=sys.stderr)
print(f"\t\tBody: {body}", file=sys.stderr)
print(f"\t\tMilestone: {milestone}", file=sys.stderr)
print(f"\t\tLabels: {labels}", file=sys.stderr)
print(f"\t\tComments: {comments}", file=sys.stderr)
else:
gh_issue = self.import_issue(title, assignee, body,
milestone, labels, attributes,
comments)
print(
"\tInitiated issue: %s (%s)" % (title, gh_issue),
file=sys.stderr,
)
self.gh_issues[title] = gh_issue
if not self.is_dry_run:
print("\tChecking completion…", file=sys.stderr)
failure = True
sleep = 0
while failure:
try:
gh_issue = self.github_repo.get_issue(trac_id)
print(
"\t%s (%s)" % (gh_issue.title, gh_issue.html_url),
file=sys.stderr,
)
failure = False
except (UnknownObjectException, ssl.SSLError) as e:
sleep += 1
print("\t\tnot completed waiting",
e,
sleep,
file=sys.stderr)
time.sleep(sleep)
try:
import xmlrpc.client as xmlrpclib
except ImportError:
import xmlrpclib
s = xmlrpclib.Server("http://127.0.0.1:5000/skitai")
print s.indians(10)
m = xmlrpclib.MultiCall(s)
m.indians(100)
m.indians(200)
m.indians(100)
r = m()
print tuple(r)
def connect():
wiki = xmlrpclib.ServerProxy(WIKI_URL + "?action=xmlrpc2", allow_none=True)
auth_token = wiki.getAuthToken(BOT_USERNAME, read_password())
multi_call = xmlrpclib.MultiCall(wiki)
multi_call.applyAuthToken(auth_token)
return multi_call
from xmlrpc import client as xmlrpclib
proxy = xmlrpclib.ServerProxy("http://localhost:8000/")
multicall = xmlrpclib.MultiCall(proxy)
multicall.add(7, 3)
multicall.subtract(7, 3)
multicall.multiply(7, 3)
multicall.divide(7, 3)
result = multicall()
print("7+3=%d, 7-3=%d, 7*3=%d, 7/3=%d" % tuple(result))
