def sqla_to_hash(arg):
if isinstance(arg, list):
l = arg
else:
l = [arg]
r = []
for instance in l:
hash = {}
klass = instance.__class__
for key in klass.__mapper__.columns.keys():
t = type(klass.__mapper__.columns[key].type)
# print "{} is a {}".format(key, t)
if t == sqlalchemy.types.LargeBinary:
if sys.version[0] == '3':
hash[key] = xmlrpclib.Binary(getattr(instance, key))
else:
hash[key] = xmlrpclib.Binary(str(getattr(instance, key)))
elif t == DeclEnumType:
hash[key] = getattr(instance, key).value
else:
hash[key] = getattr(instance, key)
hash['__klass__'] = klass.__name__
hash['__pk__'] = klass.__mapper__.primary_key_from_instance(instance)
r.append(hash)
if isinstance(arg, list):
return r
else:
return r[0]
def populate_db(self, proxy):
# create student in db
proxy.create_student(
'AB123456',
"*****@*****.**",
"foo",
"bar",
"passport.png",
xmlrpcclient.Binary(b"FakedPNGFile"),
{
"1": xmlrpcclient.Binary(b"FP1Fake"),
},
)
def store_fingerprint(url, student_id, finger_num, data_file_path):
"""Store a fingerprint on a Kofa server.
Returns `None` on success or some message otherwise.
`url` is the Kofa server to connect to. It must contain a scheme
(``http://`` or ``https://``), a username, password, and of course
the hostname and port. Something like
``http://myname:secret@localhost:8080``, for instance.
`student_id` must be a student identifier existing on the server.
`finger_num` is the number of finger that was scanned.
`data_file_path` is the path to a file that contains the
fingerprint minutiae, i.e. the fingerprint data as produced by
libfprint.
"""
server_proxy = xmlrpcclient.ServerProxy(url)
data_to_store = xmlrpcclient.Binary(open(data_file_path, 'rb').read())
fingerprint = {str(finger_num): data_to_store}
result = None
try:
result = server_proxy.put_student_fingerprints(student_id, fingerprint)
except xmlrpcclient.Fault as err:
result = "Error %s: %s" % (err.faultCode, err.faultString)
except xmlrpcclient.ProtocolError as err:
result = "Error: %s %s" % (err.errcode, err.errmsg)
except Exception as err:
result = "Error: %s" % err
return result
def test_load_standard_types(self):
check = self.check_loads
check('string', 'string')
check('<string>string</string>', 'string')
check('<string>𝔘𝔫𝔦𝔠𝔬𝔡𝔢 string</string>', '𝔘𝔫𝔦𝔠𝔬𝔡𝔢 string')
check('<int>2056183947</int>', 2056183947)
check('<int>-2056183947</int>', -2056183947)
check('<i4>2056183947</i4>', 2056183947)
check('<double>46093.78125</double>', 46093.78125)
check('<boolean>0</boolean>', False)
check('<base64>AGJ5dGUgc3RyaW5n/w==</base64>',
xmlrpclib.Binary(b'\x00byte string\xff'))
check('<base64>AGJ5dGUgc3RyaW5n/w==</base64>',
b'\x00byte string\xff',
use_builtin_types=True)
check('<dateTime.iso8601>20050210T11:41:23</dateTime.iso8601>',
xmlrpclib.DateTime('20050210T11:41:23'))
check('<dateTime.iso8601>20050210T11:41:23</dateTime.iso8601>',
datetime.datetime(2005, 2, 10, 11, 41, 23),
use_builtin_types=True)
check(
'<array><data><value><int>1</int></value><value><int>2</int></value></data></array>',
[1, 2])
check(
'<struct><member><name>b</name><value><int>2</int></value></member><member><name>a</name><value><int>1</int></value></member></struct>',
{
'a': 1,
'b': 2
})
def start_down(file_type, data, author):
if file_type == 'link' or file_type == 'meta':
gid = server.aria2.addUri(key, [data])
name = server.aria2.tellStatus(key,
gid)['files'][0]['path'].split("/")[-1]
confirmed_text = f"Your download has started with the file name {name}"
client.send(Message(text=confirmed_text),
thread_id=author,
thread_type=ThreadType.USER)
down_started = f"download started from {who_sent(author)} with the file name {name}"
with open(log_file, 'a') as log:
log.write(down_started)
print(down_started)
elif file_type == 'torrent':
url = urllib.parse.unquote(data).split("php?u=")[1]
fil = requests.get(url).content
#print(fil)
gid = server.aria2.addTorrent(key, rpc.Binary(fil))
name = server.aria2.tellStatus(key,
gid)['files'][0]['path'].split("/")[-1]
confirmed_text = f"You download has started with the file name {name}"
client.send(Message(text=confirmed_text),
thread_id=author,
thread_type=ThreadType.USER)
down_started = f"download started from {who_sent(author)} with the file name {name}"
with open(log_file, 'a') as log:
log.write(down_started)
print(down_started)
else:
print("not a valid data type")
def main(parameters, arguments):
"""Main entry point.
:param parameters: Command line parameters.
:param arguments: Command line arguments.
"""
address = "%s://%s:%s@%s:%s%s" % (
PROTOCOL,
parameters.username,
parameters.password,
parameters.server,
parameters.port,
ENDPOINT)
print ("Connecting to: %s" % hide_password(address))
server = xmlrpclib.ServerProxy(address, verbose=VERBOSE)
try:
plugin_id, version_id = server.plugin.upload(
xmlrpclib.Binary(open(arguments[0]).read()))
print ("Plugin ID: %s" % plugin_id)
print ("Version ID: %s" % version_id)
except xmlrpclib.ProtocolError as err:
print ("A protocol error occurred")
print ("URL: %s" % hide_password(err.url, 0))
print ("HTTP/HTTPS headers: %s" % err.headers)
print ("Error code: %d" % err.errcode)
print ("Error message: %s" % err.errmsg)
except xmlrpclib.Fault as err:
print ("A fault occurred")
print ("Fault code: %d" % err.faultCode)
print ("Fault string: %s" % err.faultString)
def upload_diff(self, plagiatext, title, diff_id):
if self.server is None:
self._init_server()
pywikibot.output("\tUpload text to server...")
SUBMIT_TO_GENERATE_REPORT = 1
SUBMIT_TO_STORE_IN_REPOSITORY = 2
SUBMIT_TO_STORE_IN_REPOSITORY_AND_GENERATE_REPORT = 3
submit_response = self.server.document.add({
'sid':
self.sid,
'submit_to':
SUBMIT_TO_GENERATE_REPORT,
'folder':
self.folder['id'],
'uploads': [{
'title':
'{}{}'.format(title,
diff_id), #'%s - %i' % (title, uuid.uuid4()),
'author_first': 'Random',
'author_last': 'Author',
'filename': diff_id,
'upload': xmlrpclib.Binary(plagiatext)
}]
})
if submit_response['status'] != 200:
pywikibot.output(submit_response)
raise Exception("Invalid status from server")
try:
upload = submit_response['uploaded'][0]
return upload['id']
except:
print(submit_response)
raise
def load_torrent(self, torrent, start=False, verbose=False, verify_load=True, verify_retries=3):
"""
Loads torrent into rTorrent (with various enhancements)
@param torrent: can be a url, a path to a local file, or the raw data
of a torrent file
@type torrent: str
@param start: start torrent when loaded
@type start: bool
@param verbose: print error messages to rTorrent log
@type verbose: bool
@param verify_load: verify that torrent was added to rTorrent successfully
@type verify_load: bool
@return: Depends on verify_load:
- if verify_load is True, (and the torrent was
loaded successfully), it'll return a L{Torrent} instance
- if verify_load is False, it'll return None
@rtype: L{Torrent} instance or None
@raise AssertionError: If the torrent wasn't successfully added to rTorrent
- Check L{TorrentParser} for the AssertionError's
it raises
@note: Because this function includes url verification (if a url was input)
as well as verification as to whether the torrent was successfully added,
this function doesn't execute instantaneously. If that's what you're
looking for, use load_torrent_simple() instead.
"""
p = self._get_conn()
tp = TorrentParser(torrent)
torrent = xmlrpclib.Binary(tp._raw_torrent)
info_hash = tp.info_hash
func_name = self._get_load_function("raw", start, verbose)
# load torrent
getattr(p, func_name)(torrent)
if verify_load:
i = 0
while i < verify_retries:
self.get_torrents()
if info_hash in [t.info_hash for t in self.torrents]:
break
# was still getting AssertionErrors, delay should help
time.sleep(1)
i += 1
assert info_hash in [t.info_hash for t in self.torrents],\
"Adding torrent was unsuccessful."
return(find_torrent(info_hash, self.torrents))
def test_decode(self):
d = b'\x01\x02\x03abc123\xff\xfe'
de = base64.encodebytes(d)
t1 = xmlrpclib.Binary()
t1.decode(de)
self.assertEqual(str(t1), str(d, 'latin-1'))
t2 = xmlrpclib._binary(de)
self.assertEqual(str(t2), str(d, 'latin-1'))
def put_stats_on_storage(self, host_id, data):
"""
Puts data on the shared storage according to the parameters.
Data should be passed in as a string.
"""
self._log.debug("Storing blocks on storage")
# broker expects blocks in hex format
self._proxy.put_stats(host_id, xmlrpc_client.Binary(data.encode()))
def leer(self, nombreArchivo):
archivo = verifica.join(self.root, nombreArchivo)
if not (verifica.exists(archivo)):
return "El archivo que quieres leer no existe"
elif verifica.isdir(archivo):
return "El path proporcionado no es un archivo"
with open(archivo, "rb") as file:
return datos.Binary(file.read())
def parse(self, message, custom_values=None, save_original=None):
# pass message as bytes because we don't know its encoding until we parse its headers
# and hence can't convert it to utf-8 for transport
return self.rpc('mail.thread',
'message_process',
self.model,
xmlrpclib.Binary(message),
custom_values or {},
save_original or False)
def get_stats(self):
result = {}
with self._status_broker_instance_access_lock:
state = self._listener.status_broker_instance.get_stats()
# As raw binary data can not be transferred via xmlrpc link,
# we need to wrap state's contents into Binary wrapper.
for k, v in six.iteritems(state):
result[k] = xmlrpc_client.Binary(v)
return result
def add(self, torrent, download_dir=None):
payload = xmlrpclib.Binary(torrent.torrent_data)
info_hash = Torrent.from_str(torrent.torrent_data).info_hash.upper()
# Make sure the xml-rpc size limit doesnt overflow
self._server.set_xmlrpc_size_limit(len(torrent.torrent_data) * 2)
self._server.load_raw(payload)
if download_dir:
self._server.d.set_directory_base(info_hash, download_dir)
self._server.d.start(info_hash)
return True
def get_var(var_str):
d = self._dict
if var_str == '':
cstr = zlib.compress(pickle.dumps(d, PICKLE_PROTOCOL), 9)
else:
exec('q = d.' + var_str, locals())
var = q
if hasattr(var, 'value'): var = var.value
cstr = zlib.compress(pickle.dumps(var, PICKLE_PROTOCOL), 9)
return xmlrpclib.Binary(cstr)
def load_torrent_simple(self, torrent, file_type, **kwargs):
"""Load torrent into rTorrent.
@param torrent: can be a url, a path to a local file, or the raw data
of a torrent file
@type torrent: str
@param file_type: valid options: 'url', 'file', or 'raw'
@type file_type: str
@param start: start torrent when loaded
@type start: bool
@param verbose: print error messages to rTorrent log
@type verbose: bool
@return: None
@raise AssertionError: if incorrect file_type is specified
@note: This function was written for speed, it includes no enhancements.
If you input a url, it won't check if it's valid. You also can't get
verification that the torrent was successfully added to rTorrent.
Use load_torrent() if you would like these features.
"""
start = kwargs.pop('start', False)
verbose = kwargs.pop('verbose', False)
params = kwargs.pop('params', [])
target = kwargs.pop('target', '')
p = self._get_conn()
if file_type not in ['raw', 'file', 'url']:
print("Invalid file_type, options are: 'url', 'file', 'raw'.")
return False
func_name = self._get_load_function(file_type, start, verbose)
if file_type == 'file':
# since we have to assume we're connected to a remote rTorrent
# client, we have to read the file and send it to rT as raw
if not os.path.isfile(torrent):
print('Invalid path: {0}'.format(torrent))
return False
torrent = open(torrent, 'rb').read()
if file_type in ['raw', 'file']:
finput = xmlrpc_client.Binary(torrent)
elif file_type == 'url':
finput = torrent
getattr(p, func_name)(target, finput, *params)
return True
def get_all_stats(self):
"""
Reads all files in storage_dir for the given service_type, returning a
space-delimited string of "<host_id>=<hex data>" for each host.
"""
d = self.get_raw_stats()
result = {}
for host_id in sorted(d.keys()):
result[str(host_id)] = xmlrpc_client.Binary(d.get(host_id))
return result
def test_internal_put_student_fingerprints(self, waeup_proxy):
# make sure the faked method is faked properly
waeup_proxy.create_student('AB123456')
# invalid student id
with pytest.raises(xmlrpcclient.Fault):
waeup_proxy.put_student_fingerprints('invalid-id')
# fingerprint dict not a dict
with pytest.raises(xmlrpcclient.Fault):
waeup_proxy.put_student_fingerprints('AB123456', 'not-a-dict')
# invalid fingerprint file type
with pytest.raises(xmlrpcclient.Fault):
waeup_proxy.put_student_fingerprints('AB123456', {'1': 12})
# invalid file format
with pytest.raises(xmlrpcclient.Fault):
waeup_proxy.put_student_fingerprints(
'AB123456', {'1': xmlrpcclient.Binary(b'not-an-fpm-file')})
# valid fingerprint dict
assert waeup_proxy.put_student_fingerprints(
'AB123456',
{'1': xmlrpcclient.Binary(b'FP1-faked-fpm-file')}) is True
# empty fingerprint dict
assert waeup_proxy.put_student_fingerprints('AB123456', {}) is False
def __init__(self, source, instance_only=False):
self.instance_only = instance_only
with TemporaryDirectory() as tempdir:
self.source = source
self.tempdir = tempdir
if self.is_http:
self.filename = Package._download(self.source, self.tempdir)
else:
self.filename = os.path.basename(self.source)
copyfile(os.path.expanduser(self.source), os.path.join(self.tempdir, self.filename))
self.path = os.path.join(self.tempdir, self.filename)
with open(self.path, 'rb') as f:
self.body = xmlrpclib.Binary(f.read())
self._extract_files()
self._parse_metadata()
def load_torrent_simple(self,
torrent,
file_type,
start=False,
verbose=False):
"""Loads torrent into rTorrent
@param torrent: can be a url, a path to a local file, or the raw data
of a torrent file
@type torrent: str
@param file_type: valid options: "url", "file", or "raw"
@type file_type: str
@param start: start torrent when loaded
@type start: bool
@param verbose: print error messages to rTorrent log
@type verbose: bool
@return: None
@raise AssertionError: if incorrect file_type is specified
@note: This function was written for speed, it includes no enhancements.
If you input a url, it won't check if it's valid. You also can't get
verification that the torrent was successfully added to rTorrent.
Use load_torrent() if you would like these features.
"""
p = self._get_conn()
assert file_type in ["raw", "file", "url"], \
"Invalid file_type, options are: 'url', 'file', 'raw'."
func_name = self._get_load_function(file_type, start, verbose)
if file_type == "file":
# since we have to assume we're connected to a remote rTorrent
# client, we have to read the file and send it to rT as raw
assert os.path.isfile(torrent), \
"Invalid path: \"{0}\"".format(torrent)
torrent = open(torrent, "rb").read()
if file_type in ["raw", "file"]:
finput = xmlrpclib.Binary(torrent)
elif file_type == "url":
finput = torrent
getattr(p, func_name)(finput)
def test_dump_bytes(self):
sample = b'my dog has fleas'
self.assertEqual(sample, xmlrpclib.Binary(sample))
for type_ in (bytes, bytearray, xmlrpclib.Binary):
value = type_(sample)
s = xmlrpclib.dumps((value, ))
result, m = xmlrpclib.loads(s, use_builtin_types=True)
newvalue, = result
self.assertEqual(newvalue, sample)
self.assertIs(type(newvalue), bytes)
self.assertIsNone(m)
result, m = xmlrpclib.loads(s, use_builtin_types=False)
newvalue, = result
self.assertEqual(newvalue, sample)
self.assertIs(type(newvalue), xmlrpclib.Binary)
self.assertIsNone(m)
def attachmentTupFromFilepath(filepath):
"""
Creates attachment struct and binary data object
for use with addAttachment method via the confluence2 xmlrpc API.
Used in e.g. server, experiment, page and limspage.
"""
filename = os.path.basename(filepath)
mimetype = getmimetype(filepath)
# Note: comment not required, I believe.
attInfo = dict(fileName=filename, contentType=mimetype)
with open(filepath, 'rb') as fd:
# Remember to open file in binary mode, otherwise you'll get padding errors (and wrong data).
# xmlrpclib.Binary also does base64.encode, but adds xml tag before and after.
attData = xmlrpclib.Binary(fd.read())
logger.debug("Read data for attachment '%s' with byte-length %s.", attInfo,
len(str(attData)))
return attInfo, attData
def extract_upload_images(post,
server,
title,
user,
password):
"""Extract the images from a Jupyter notebook and upload them to
the defined wordpress server.
Params:
=======
post : str
The converted information from the notebook to HTML.
server: obj
A `xmlrpclib.ServerProxy` instance.
title : str
Title for the post
Returns:
========
A string with the converted HTML once the images has been extracted
and replaced with urls to the wordpress site.
"""
# Let's extract the images and upload to wp
pat = re.compile('src="data:image/(.*?);base64,(.*?)"', re.DOTALL)
count = 1
postnew = post
for (ext, data) in pat.findall(post):
datab = a2b_base64(data)
datab = xmlrpclib.Binary(datab)
imgtitle = title.replace(' ','_').replace('.','-')
out = {'name': imgtitle + str(count) + '.' + ext,
'type': 'image/' + ext,
'bits': datab,
'overwrite': 'true'}
count += 1
image_id = server.wp.uploadFile("",
user,
password,
out)
urlimg = image_id['url']
postnew = postnew.replace('data:image/' + ext + ';base64,' + data,
urlimg)
return postnew
def marshall(obj):
typ = type(obj)
# take care of compound types
if isinstance(obj, list):
return list(map(marshall, obj))
if isinstance(obj, tuple):
return tuple(map(marshall, obj))
if isinstance(obj, dict):
return dict(zip(obj.keys(), map(marshall, obj.values())))
# check if object is a large binary object
if isinstance(obj, Blob):
return xmlrpclib.Binary(obj.buf)
if typ not in base_types:
obj = undefined
return obj
def test_load(self, mocked_proxy):
mocked_proxy = mocked_proxy()
mocked_proxy.execute.throw.return_value = 0
mocked_proxy.load.raw_start.return_value = 0
client = RTorrent('http://localhost/RPC2')
resp = client.load(
torrent_raw,
fields={
'priority': 3,
'directory': '/data/downloads',
'custom1': 'testing'
},
start=True,
mkdir=True,
)
assert resp == 0
# Ensure mkdir was called
mocked_proxy.execute.throw.assert_called_with('', 'mkdir', '-p',
'/data/downloads')
# Ensure load was called
assert mocked_proxy.load.raw_start.called
match_binary = Matcher(compare_binary,
xmlrpc_client.Binary(torrent_raw))
called_args = mocked_proxy.load.raw_start.call_args_list[0][0]
assert len(called_args) == 5
assert '' == called_args[0]
assert match_binary in called_args
fields = [p for p in called_args[2:]]
assert len(fields) == 3
# TODO: check the note in clients/rtorrent.py about this escaping.
# The client should be fixed to work consistenly on all python versions
# Calling re.escape here is a workaround so test works on python 3.7 and older versions
assert ('d.directory.set=' + re.escape('/data/downloads')) in fields
assert 'd.custom1.set=testing' in fields
assert 'd.priority.set=3' in fields
def post_to_confluence(space, parent_title, child_title, html_string, images):
# parent_title: there should be existing parent page
# child_title: child page is created with this title
# images : list of images to attach to confluence
# usage
# post_to_confluence("PCDS", "sample delivery system testing", "test child")
try:
server = xmlrpclib.ServerProxy(
"https://confluence.slac.stanford.edu/rpc/xmlrpc", allow_none=True)
username = input("username:"******"password:"******"Created confluence page")
# add attachment to child page
for image in images:
child_page = server.confluence2.getPage(token, space, child_title)
attachment = {}
attachment["fileName"] = image
attachment["contentType"] = 'image/png'
with open(image, 'rb') as f:
data = f.read()
server.confluence2.addAttachment(token,
child_page['id'], attachment,
xmlrpclib.Binary(data))
print(
"visit the link to view the child page \n https://confluence.slac.stanford.edu/display/PCDS/sample+delivery+system+automated+test"
)
except xmlrpclib.Fault as err:
print("Error accessing Confluence:",
sys.exc_info()[0], err.faultString)
except Exception as err:
print("Unexpected error:", err)
def test_upload(self):
image_bits = open(self.png_file_path(), 'rb').read()
filename = 'a / b % 2.png' # Test escaping weird chars.
response = self.fetch_rpc(
'metaWeblog.newMediaObject',
(
1, # Blog id, always 1.
tornado_options.user,
tornado_options.password,
{
'name': filename,
'bits': xmlrpclib.Binary(image_bits),
'type': 'image/png'}))
# Make sure we can now fetch the image.
self.http_client.fetch(response['url'], self.stop)
response = self.wait()
self.assertEqual(200, response.code)
self.assertEqual('image/png', response.headers['Content-Type'])
self.assertEqual(image_bits, response.body)
def send_to_ithenticate(article, file):
# 1. Init Server
server = build_server(article.journal)
# 2. Try to get this journal's folder
folder_id = find_folder(server['server'], server['sid'], article.journal)
# 2.1 If the journal doesn't have a folder, make one
if not folder_id:
folder_id = make_folder(server['server'], server['sid'],
article.journal)
# 3. Prepare the submission
first_author = article.correspondence_author
open_file = codecs.open(file.self_article_path(), 'rb')
data = xmlrpclib.Binary(bytes(open_file.read()))
article_dict = {
'title': article.title,
'author_first': first_author.first_name,
'author_last': first_author.last_name,
'filename': os.path.basename(file.original_filename),
'upload': data,
}
submission_dict = {
'sid': server['sid'],
'folder': folder_id,
'uploads': [article_dict],
'submit_to': 1,
}
submission = server['server'].document.add(submission_dict)
if submission['status'] == 200:
return submission['uploaded'][0].get('id')
else:
return None
def load(self, raw_torrent, fields=None, start=False, mkdir=True):
if fields is None:
fields = {}
# First param is empty 'target'
params = ['', xmlrpc_client.Binary(raw_torrent)]
# Additional fields to set
for key, val in fields.items():
# Values must be escaped if within params
# TODO: What are the escaping requirements? re.escape works differently on python 3.7+
params.append('d.%s.set=%s' % (key, re.escape(str(val))))
if mkdir and 'directory' in fields:
result = self._server.execute.throw('', 'mkdir', '-p',
fields['directory'])
if result != 0:
raise xmlrpc_client.Error('Failed creating directory %s' %
fields['directory'])
# by default rtorrent won't allow calls over 512kb in size.
xmlrpc_size = (len(xmlrpc_client.dumps(tuple(params), 'raw_start')) +
71680) # Add 70kb for buffer
if xmlrpc_size > 524288:
prev_size = self._server.network.xmlrpc.size_limit()
self._server.network.xmlrpc.size_limit.set('', xmlrpc_size)
# Call load method and return the response
if start:
result = self._server.load.raw_start(*params)
else:
result = self._server.load.raw(*params)
if xmlrpc_size > 524288:
self._server.network.xmlrpc.size_limit.set('', prev_size)
return result
def test_string(self):
d = b'\x01\x02\x03abc123\xff\xfe'
t = xmlrpclib.Binary(d)
self.assertEqual(str(t), str(d, "latin-1"))
