def open_binary(ctx, server_relative_url):
url = r"{0}web/getfilebyserverrelativeurl('{1}')/\$value".format(
ctx.serviceRootUrl, server_relative_url)
request = RequestOptions(url)
request.method = HttpMethod.Get
response = ctx.execute_request_direct(request)
return response
def execute(self, chunk_uploaded=None):
ctx = self._target_folder.context
file_name = os.path.basename(self._source_path)
# 1. create an empty file
target_item = self._target_folder.upload(file_name, None)
ctx.execute_query()
# 2. create upload session
item = DriveItemUploadableProperties()
item.name = file_name
session_result = target_item.create_upload_session(item)
ctx.execute_query()
# 3. start upload
fh = open(self._source_path, 'rb')
st = os.stat(self._source_path)
f_pos = 0
for piece in read_in_chunks(fh, chunk_size=self._chunk_size):
req = RequestOptions(session_result.value.uploadUrl)
req.method = HttpMethod.Put
req.set_header('Content-Length', str(len(piece)))
req.set_header(
'Content-Range',
'bytes {0}-{1}/{2}'.format(f_pos, (f_pos + len(piece) - 1),
st.st_size))
req.set_header('Accept', '*/*')
req.data = piece
resp = ctx.execute_request_direct(req)
f_pos += len(piece)
if chunk_uploaded is not None:
chunk_uploaded(f_pos)
return target_item
def test7_upload_file_session(self):
file_name = "big_buck_bunny.mp4"
path = "{0}/data/{1}".format(os.path.dirname(__file__), file_name)
# 1. create a file
target_item = self.client.me.drive.root.upload(file_name, None)
self.client.execute_query()
self.assertIsNotNone(target_item.properties['id'])
# 2. create upload session
item = DriveItemUploadableProperties()
item.name = file_name
session_result = target_item.create_upload_session(item)
self.client.execute_query()
self.assertIsNotNone(session_result.value)
# 3. start upload
f = open(path, 'rb')
st = os.stat(path)
f_pos = 0
for piece in read_in_chunks(f, chunk_size=1000000):
req = RequestOptions(session_result.value.uploadUrl)
req.method = HttpMethod.Put
req.set_header('Content-Length', str(len(piece)))
req.set_header('Content-Range', 'bytes {0}-{1}/{2}'.format(f_pos, (f_pos + len(piece) - 1), st.st_size))
req.set_header('Accept', '*/*')
req.data = piece
resp = self.client.execute_request_direct(req)
self.assertTrue(resp.ok)
f_pos += len(piece)
def open_binary(ctx, server_relative_url):
server_relative_url = urllib.quote(server_relative_url)
url = "{0}web/getfilebyserverrelativeurl('{1}')/\$value".format(
ctx.service_root_url, server_relative_url)
request = RequestOptions(url)
request.method = HttpMethod.Get
response = ctx.execute_query_direct(request)
return response
def checkout_file(target_url, ctx, l):
url = "{0}web/getfilebyserverrelativeurl('{1}')/" \
"CheckOut()".format(ctx.service_root_url, target_url)
try:
request = RequestOptions(url)
request.method = HttpMethod.Post
ctx.execute_request_direct(request)
except Exception as e:
l.error(e)
def checkin_file(target_url, ctx, checkin_type, l):
url = "{0}web/getfilebyserverrelativeurl('{1}')/" \
"CheckIn(comment='',checkintype={2})".format(ctx.service_root_url, target_url, checkin_type)
try:
request = RequestOptions(url)
request.method = HttpMethod.Post
ctx.execute_request_direct(request)
except Exception as e:
l.error(e)
def upload(self, fileobj, remote_path, buffer_size=None, log=False, progress_obj=False):
iid = uuid.uuid1()
if progress_obj:
log = True
if log:
logger = logging.getLogger('cloud.backup')
try:
size = os.path.getsize(fileobj.name)
except:
size = None
offset = -1
if not buffer_size:
buffer_size = 10* 1024 * 1024
x = ""
webUri = '%s%s' % (self.path, remote_path)
while True:
if offset == -1:
request_url = "%s/_api/web/GetFolderByServerRelativeUrl('%s')/Files/add(url='%s',overwrite=true)" % (self.baseurl, self.path, remote_path)
offset = 0
elif not offset:
if len(x) == buffer_size:
request_url="%s/_api/web/getfilebyserverrelativeurl('%s')/startupload(uploadId=guid'%s')" % (self.baseurl, webUri, iid)
else:
request_url = "%s/_api/web/GetFolderByServerRelativeUrl('%s')/Files/add(url='%s',overwrite=true)" % (self.baseurl, self.path, remote_path)
elif len(x) == buffer_size:
request_url = "%s/_api/web/getfilebyserverrelativeurl('%s')/continueupload(uploadId=guid'%s',fileOffset=%s)" % (self.baseurl, webUri, iid, offset)
else:
request_url = "%s/_api/web/getfilebyserverrelativeurl('%s')/finishupload(uploadId=guid'%s',fileOffset=%s)" % (self.baseurl, webUri, iid, offset)
offset += len(x)
options = RequestOptions(request_url)
options.method = HttpMethod.Post
self.request.context.authenticate_request(options)
self.request.context.ensure_form_digest(options)
result = requests.post(url=request_url, data=x, headers=options.headers, auth=options.auth)
if result.status_code not in (200, 201):
raise Exception(result.content)
if log and offset and offset % buffer_size*10 == 0:
percent_txt = ''
if size:
percent = round(offset*100/size)
percent_txt = '%d%%' % percent
if progress_obj:
progress_obj.write({'name': percent})
logger.info('OneDrive: %d bytes sent on %s bytes %s' % (offset, size or 'unknown', percent_txt))
x = fileobj.read(buffer_size)
if not x:
break
return True
def request_form_digest(self):
"""Request Form Digest"""
request = RequestOptions(self.serviceRootUrl + "contextinfo")
request.method = HttpMethod.Post
request.set_headers(self.json_format.build_http_headers())
response = self.execute_request_direct(request)
payload = response.json()
if self.json_format.metadata == ODataMetadataLevel.Verbose:
payload = payload['d']['GetContextWebInformation']
self.contextWebInformation = ContextWebInformation()
self.contextWebInformation.from_json(payload)
def read_list_items(context, list_title, url):
"""Read list items example"""
request = ClientRequest(context)
options = RequestOptions("{0}/web/lists/getbyTitle('{1}')/items".format(url, list_title))
options.set_header('Accept', 'application/json; odata=nometadata')
print("Retrieving list items from List {0}".format(list_title))
response = request.execute_request_direct(options)
data = json.loads(response.content)
for item in data['value']:
print("Item title: {0}".format(item["Title"]))
def open_binary(ctx, server_relative_url):
try:
from urllib import quote # Python 2.X
except ImportError:
from urllib.parse import quote # Python 3+
server_relative_url = quote(server_relative_url)
url = "{0}web/getfilebyserverrelativeurl('{1}')/\$value".format(ctx.service_root_url, server_relative_url)
request = RequestOptions(url)
request.method = HttpMethod.Get
response = ctx.execute_query_direct(request)
return response
def delete(self, remote_path):
webUri = '%s%s' % (self.path, remote_path)
request_url = "%s/_api/web/getfilebyserverrelativeurl('%s')" % (self.baseurl, webUri)
options = RequestOptions(request_url)
options.method = HttpMethod.Delete
options.set_header("X-HTTP-Method", "DELETE")
self.request.context.authenticate_request(options)
self.request.context.ensure_form_digest(options)
result = requests.post(url=request_url, data="", headers=options.headers, auth=options.auth)
if result.status_code not in (200, 201):
raise Exception(result.content)
return True
def request_form_digest(self):
"""Request Form Digest"""
request = RequestOptions(self.service_root_url + "contextinfo")
self.authenticate_request(request)
request.set_headers(self.json_format.build_http_headers())
response = requests.post(url=request.url,
headers=request.headers,
auth=request.auth)
payload = response.json()
if self.json_format.metadata == ODataMetadataLevel.Verbose:
payload = payload['d']['GetContextWebInformation']
self.contextWebInformation = ContextWebInformation()
self.contextWebInformation.from_json(payload)
def delete_list_item(context, list_title, item_id, url):
"""Delete list item example"""
request = ClientRequest(context)
options = RequestOptions(
"{0}/web/lists/getbyTitle('{1}')/items({2})".format(url, list_title, item_id))
options.set_header('Accept', 'application/json; odata=nometadata') # JSON Light nometadata mode!
options.set_header('IF-MATCH', '*')
options.set_header('X-HTTP-Method', 'DELETE')
options.data = {'Title': 'New Task (updated)'}
options.method = HttpMethod.Post
print("Deleting list item...")
request.execute_request_direct(options)
print("Task has been successfully [deleted]")
def post_raw(self, path, headers={}, data={}):
options = RequestOptions('{}{}'.format(self.site_url, path))
options.method = HttpMethod.Post
options.set_headers(headers)
if isinstance(data, abc.Mapping):
options.data.update(data)
data = self.request.execute_request_direct(options)
if data.status_code == 403:
import traceback
traceback.print_tb()
print(options.url)
print(data.content)
print(options.headers)
return data
def list(self, remote_path):
#webUri = '%s%s' % (self.path, remote_path)
#request_url = "%s_api/web/getfilebyserverrelativeurl('%s')/files" % (self.baseurl, webUri)
request_url = "%s_api/web/getfolderbyserverrelativeurl('%s')/files" % (
self.baseurl, remote_path)
print request_url
options = RequestOptions(request_url)
options.method = HttpMethod.Get
options.set_header("X-HTTP-Method", "GET")
options.set_header('accept', 'application/json;odata=verbose')
self.request.context.authenticate_request(options)
self.request.context.ensure_form_digest(options)
result = requests.get(url=request_url,
headers=options.headers,
auth=options.auth)
#result = requests.post(url=request_url, data="", headers=options.headers, auth=options.auth)
result = result.json()
'''if result.status_code not in (200, 201):
print 'Error code: '
print result.status_code
raise Exception(result.content)
'''
#return True
files = []
for i in range(len(result['d']['results'])):
item = result['d']['results'][i]
files.append(item)
print files[0]['Name']
return files
def download(self, remote_path, filename):
request_url = "%s_api/web/getfilebyserverrelativeurl('%s')/$value" % (self.baseurl, remote_path)
options = RequestOptions(request_url)
options.method = HttpMethod.Get
options.set_header("X-HTTP-Method", "GET")
options.set_header('accept', 'application/json;odata=verbose')
self.request.context.authenticate_request(options)
self.request.context.ensure_form_digest(options)
retry = 5
while retry:
with requests.get(url=request_url, headers=options.headers, auth=options.auth, stream=True, timeout=120) as r:
if r.status_code not in (200, 201):
error = self.parse_error(r)
if 'timed out' in error or '2130575252' in error:
time.sleep(2)
self.login()
retry -= 1
continue
raise Exception(error)
with open(filename, 'wb') as file:
for chunk in r.iter_content(chunk_size=8192):
if chunk:
file.write(chunk)
retry = 0
return filename
def download_file(uri):
"""Give the URI (returned from retrieve file query)
and the path you want to write to."""
uriR = uri + '/$value'
if ctx.acquire_token_for_user(username=USERNAME, password=PASSWORD):
options = RequestOptions(BASE_URL)
context = ClientContext(BASE_URL, ctx)
context.request_form_digest()
options.method = 'GET'
context.authenticate_request(options)
byte_doc = requests.get(url=uriR,
headers=options.headers,
auth=options.auth)
hack_file_name = uri.split("/")
hack_file = hack_file_name[len(hack_file_name) - 1]
file_name = hack_file[0:len(hack_file) - 2]
return file_name, byte_doc.content
else:
print('Incorrect login credentials')
def GetAllData(auth_creds, request, uri):
MSP_ROOT_URL = auth_creds[0]
MSP_USERNAME = auth_creds[1]
MSP_PASS = auth_creds[2]
dList = []
skip = 0
options = RequestOptions(MSP_ROOT_URL + uri + "?$skip=" + str(skip))
data = request.execute_request_direct(options)
dList.append(data)
dataLen = len(feedparser.parse(data.text)["entries"])
while dataLen != 0:
skip += dataLen
options = RequestOptions(MSP_ROOT_URL + uri + "?$skip=" + str(skip))
data = request.execute_request_direct(options)
dList.append(data)
dataLen = len(feedparser.parse(data.text)["entries"])
return dList
def update_list_item(context, list_title, item_id, values_to_send):
"""
Updates item with given id in given list with given properties
:param context: auth context
:param list_title: name of list
:param item_id:
:param values_to_send: dict with key-value pairs
:return: requests/result object
"""
request = ClientRequest(context)
options = RequestOptions(
"{2}/_api/web/lists/getbyTitle('{0}')/items({1})".format(
list_title, item_id, URL))
options.set_header('Accept', 'application/json; odata=nometadata')
options.set_header('IF-MATCH', '*')
options.set_header('X-HTTP-Method', 'MERGE')
options.data = values_to_send
options.method = HttpMethod.Post
result = request.execute_request_direct(options)
return result
def delete_list_item(context, list_title, item_id):
"""
Deletes item with given id in given list
:param context: auth context
:param list_title: name of list
:param item_id:
:return: requests/response object
"""
req = ClientRequest(context)
options = RequestOptions(
f"{URL}/_api/web/lists/getbyTitle('{list_title}')/items({item_id})")
options.set_header('Accept', 'application/json; odata=nometadata')
options.set_header('IF-MATCH', '*')
options.set_header('X-HTTP-Method', 'DELETE')
options.method = HttpMethod.Post
res = req.execute_request_direct(options)
return res
def retrieve_file_list(url):
url = url + '/Files'
if ctx.acquire_token_for_user(username=USERNAME, password=PASSWORD):
options = RequestOptions(BASE_URL)
context = ClientContext(BASE_URL, ctx)
context.request_form_digest()
options.set_header('Accept', 'application/json; odata=verbose')
options.method = 'GET'
context.authenticate_request(options)
data = requests.get(url=url,
headers=options.headers,
auth=options.auth)
if data.status_code == 200:
file_list = []
datam = json.loads(data.text)
for f in range(len(datam['d']['results'])):
uri = datam['d']['results'][f]['__metadata']['uri']
file_list.append(uri)
return file_list
else:
print(data.content)
def __iter__(self):
for _object in self._data:
yield _object
while self.__next_query_url:
# create a request with the __next_query_url
request = RequestOptions(self.__next_query_url)
request.set_headers(self.context.json_format.build_http_headers())
response = self.context.execute_request_direct(request)
# process the response
payload = self.context.pending_request.process_response(response)
self.__next_query_url = payload["next"]
child_client_objects = []
# add the new objects to the collection before yielding the results
for properties in payload["collection"]:
child_client_object = self.create_typed_object(
properties, self._item_type)
self.add_child(child_client_object)
child_client_objects.append(child_client_object)
for child_client_object in child_client_objects:
yield child_client_object
def checkin_file(file_path, base_url, folder_url, ctx_auth):
"""Attempt to check in a file in SharePoint"""
file_name = basename(file_path)
files_url = "{0}/_api/web/GetFileByServerRelativeUrl('{1}/{2}')/CheckIn(comment='Comment',checkintype=0)"
full_url = files_url.format(base_url, folder_url, file_name)
options = RequestOptions(base_url)
context = ClientContext(base_url, ctx_auth)
context.request_form_digest()
options.set_header('X-RequestDigest',
context.contextWebInformation.form_digest_value)
options.method = 'POST'
context.authenticate_request(options)
data = requests.post(url=full_url,
headers=options.headers,
auth=options.auth)
if data.status_code != 200:
print("checkin_file error code: " + str(data.status_code))
def save_binary(ctx, server_relative_url, content):
server_relative_url = urllib.quote(server_relative_url)
url = "{0}web/getfilebyserverrelativeurl('{1}')/\$value".format(
ctx.service_root_url, server_relative_url)
request = RequestOptions(url)
request.method = HttpMethod.Post
request.set_header('X-HTTP-Method', 'PUT')
request.data = content
ctx.execute_query_direct(request)
def build_request(self, query):
request = RequestOptions(query.url)
# set json format headers
request.set_headers(self.context.json_format.build_http_headers())
# set method
request.method = query.method
# set request payload
if query.payload is not None:
request.data = ODataEncoder(self.context.json_format).default(query.payload)
return request
def get(self, path):
# request = ClientRequest(self.ctx_auth)
options = RequestOptions('{}_api/web/{}'.format(self.site_url, path))
options.set_header('Accept', 'application/json')
options.set_header('Content-Type', 'application/json')
data = self.request.execute_request_direct(options)
if data.status_code == 404:
raise ValueError('Site does not exist')
s = json.loads(data.content)
return s
def save_binary(ctx, server_relative_url, content):
"""Uploads a file"""
url = r"{0}web/getfilebyserverrelativeurl('{1}')/\$value".format(
ctx.serviceRootUrl, server_relative_url)
request = RequestOptions(url)
request.method = HttpMethod.Post
request.set_header('X-HTTP-Method', 'PUT')
request.data = content
response = ctx.execute_request_direct(request)
return response
def FetchMSPAvailableFields(auth_creds):
MSP_ROOT_URL = auth_creds[0]
MSP_USERNAME = auth_creds[1]
MSP_PASS = auth_creds[2]
ctx_auth = AuthenticationContext(MSP_ROOT_URL)
if ctx_auth.acquire_token_for_user(MSP_USERNAME, MSP_PASS):
request = ClientRequest(ctx_auth)
options = RequestOptions(MSP_ROOT_URL + "/sites/pwa/_api/projectdata")
data = request.execute_request_direct(options)
root = ET.fromstring(data.content)
l = set()
for elem in root.iter():
if elem.text:
if not (elem.text == "None" or elem.text == "Default"):
l.add(elem.text)
return l
def _build_request(self):
for qry in self._queries:
request = RequestOptions(qry.entity_type.resourceUrl)
# set json format headers
request.set_headers(self.context.json_format.build_http_headers())
# set method
request.method = HttpMethod.Get
if isinstance(qry, CreateEntityQuery) \
or isinstance(qry, UpdateEntityQuery) \
or isinstance(qry, DeleteEntityQuery):
request.method = HttpMethod.Post
elif isinstance(qry, ServiceOperationQuery):
request.url = '/'.join(
[qry.entity_type.resourceUrl, qry.method_path.segment])
request.method = HttpMethod.Post
# set request payload
if qry.parameters is not None:
request.data = self._normalize_payload(qry.parameters)
if 'before' in self._events:
self._events['before'](request, qry)
self.__current_query = qry
yield request
def save_binary(ctx, server_relative_url, content):
try:
from urllib import quote # Python 2.X
except ImportError:
from urllib.parse import quote # Python 3+
server_relative_url = quote(server_relative_url)
url = "{0}web/getfilebyserverrelativeurl('{1}')/\$value".format(ctx.service_root_url, server_relative_url)
request = RequestOptions(url)
request.method = HttpMethod.Post
request.set_header('X-HTTP-Method', 'PUT')
request.data = content
ctx.execute_query_direct(request)
