def _upload(self, package):
data = package.metadata_dictionary()
data.update({
# action
":action": "file_upload",
"protcol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
print("Uploading {0}".format(package.basefilename))
with open(package.filename, "rb") as fp:
data_to_send.append((
"content",
(package.basefilename, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = ProgressBar(expected_size=encoder.len, filled_char='=')
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.show(monitor.bytes_read))
resp = self.session.post(
self.url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type},
)
bar.done()
return resp
def _upload(self, package):
data = package.metadata_dictionary()
data.update({
# action
":action": "file_upload",
"protcol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
print("Uploading {0}".format(package.basefilename))
with open(package.filename, "rb") as fp:
data_to_send.append((
"content",
(package.basefilename, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = ProgressBar(expected_size=encoder.len, filled_char='=')
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.show(monitor.bytes_read)
)
resp = self.session.post(
self.url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type},
)
bar.done()
return resp
def download_artifacts(session):
response = session.get("{0}/lastBuild/api/json/".format(JENKINS_URL),
headers={"Accept": "application/json"})
response.raise_for_status()
json_response = response.json()
assert not json_response["building"]
assert json_response["result"] == "SUCCESS"
paths = []
for artifact in json_response["artifacts"]:
response = session.get("{0}artifact/{1}".format(
json_response["url"], artifact["relativePath"]),
stream=True)
assert response.headers["content-length"]
print("Downloading {0}".format(artifact["fileName"]))
bar = ProgressBar(expected_size=int(
response.headers["content-length"]),
filled_char="=")
content = io.BytesIO()
for data in response.iter_content(chunk_size=8192):
content.write(data)
bar.show(content.tell())
assert bar.expected_size == content.tell()
bar.done()
out_path = os.path.join(
os.path.dirname(__file__),
"dist",
artifact["fileName"],
)
with open(out_path, "wb") as f:
f.write(content.getvalue())
paths.append(out_path)
return paths
def download(url, dest):
m = re.match(r'^https://send.firefox.com/download/(\w+)/#([\w_-]+)$', url)
if not m:
raise ValueError("URL format appears to be incorrect")
fid = m.group(1)
key = base64.urlsafe_b64decode(m.group(2) + '==')
print("Downloading %s..." % url)
url = "https://send.firefox.com/api/download/" + fid
resp = requests.get(url, stream=True)
resp.raise_for_status()
flen = int(resp.headers.get('Content-Length'))
metadata = json.loads(resp.headers.get('X-File-Metadata'))
filename = metadata['filename']
if os.path.isdir(dest):
filename = os.path.join(dest, filename)
else:
filename = dest
iv = binascii.unhexlify(metadata['id'])
cipher = AES.new(key, AES.MODE_GCM, iv)
ho = sha256()
print("Downloading to %s..." % filename)
try:
with open(filename + '.tmp', 'wb') as outf:
bar = ProgressBar(expected_size=flen, filled_char='=')
dl = 0
tag = b''
taglen = 16
for data in resp.iter_content(chunk_size=8192):
dl += len(data)
bar.show(dl)
if dl > flen - taglen:
dend = max(len(data) - (dl - (flen - taglen)), 0)
tag += data[dend:]
data = data[:dend]
chunk = cipher.decrypt(data)
ho.update(chunk)
outf.write(chunk)
if len(tag) == taglen:
break
print()
cipher.verify(tag)
except Exception as e:
print("File download failed:", e)
os.unlink(filename + '.tmp')
else:
os.rename(filename + '.tmp', filename)
print("Done, file verified!")
def download(fid, secret, dest, password=None, url=None):
metadata, nonce = get_metadata(fid, secret, password, url)
encryptKey = deriveFileKey(secret)
authKey = deriveAuthKey(secret, password, url)
sig = hmac.new(authKey, nonce, sha256).digest()
url = "https://send.firefox.com/api/download/" + fid
resp = requests.get(url,
headers={'Authorization': 'send-v1 ' + b64encode(sig)},
stream=True)
resp.raise_for_status()
flen = int(resp.headers.get('Content-Length'))
filename = metadata['metadata']['name']
if os.path.isdir(dest):
filename = os.path.join(dest, filename)
else:
filename = dest
iv = b64decode(metadata['metadata']['iv'])
cipher = AES.new(encryptKey, AES.MODE_GCM, iv, mac_len=16)
ho = sha256()
print("Downloading to %s..." % filename)
try:
with open(filename + '.tmp', 'wb') as outf:
bar = ProgressBar(expected_size=flen, filled_char='=')
dl = 0
tag = b''
taglen = 16
for data in resp.iter_content(chunk_size=8192):
dl += len(data)
bar.show(dl)
if dl > flen - taglen:
dend = max(len(data) - (dl - (flen - taglen)), 0)
tag += data[dend:]
data = data[:dend]
chunk = cipher.decrypt(data)
ho.update(chunk)
outf.write(chunk)
if len(tag) == taglen:
break
print()
cipher.verify(tag)
except Exception as e:
print("File download failed:", e)
os.unlink(filename + '.tmp')
else:
os.rename(filename + '.tmp', filename)
print("Done, file verified!")
class CallBack:
def __init__(self):
self.bar = None
def __call__(self, size_rd, total):
if self.bar is None:
self.bar = ProgressBar(expected_size=total,
filled_char='=')
self.bar.show(size_rd)
def download_artifacts(session):
response = session.get(
"{0}/lastBuild/api/json/".format(JENKINS_URL),
headers={
"Accept": "application/json"
}
)
response.raise_for_status()
assert not response.json()["building"]
assert response.json()["result"] == "SUCCESS"
paths = []
last_build_number = response.json()["number"]
for run in response.json()["runs"]:
if run["number"] != last_build_number:
print(
"Skipping {0} as it is not from the latest build ({1})".format(
run["url"], last_build_number
)
)
continue
response = session.get(
run["url"] + "api/json/",
headers={
"Accept": "application/json",
}
)
response.raise_for_status()
for artifact in response.json()["artifacts"]:
response = session.get(
"{0}artifact/{1}".format(run["url"], artifact["relativePath"]),
stream=True
)
assert response.headers["content-length"]
print("Downloading {0}".format(artifact["fileName"]))
bar = ProgressBar(
expected_size=int(response.headers["content-length"]),
filled_char="="
)
content = io.BytesIO()
for data in response.iter_content(chunk_size=8192):
content.write(data)
bar.show(content.tell())
assert bar.expected_size == content.tell()
bar.done()
out_path = os.path.join(
os.path.dirname(__file__),
"dist",
artifact["fileName"],
)
with open(out_path, "wb") as f:
f.write(content.getvalue())
paths.append(out_path)
return paths
class ProgressBar():
def __init__(self, length):
self.bar = Bar(expected_size=length, filled_char='=')
self.status = 0
def increment(self, value):
self.status += value
def progress(self, value):
self.bar.show(self.status + value)
def mkprogress(encoder):
"""
Returns a function that can be used as a callback for
:class:`requests_toolbelt.MultipartEncoderMonitor`.
:param encoder: instance of :class:`requests_toolbelt.MultipartEncoder`
"""
expected_size = encoder.len
bar = Bar(expected_size=expected_size, filled_char='=')
waiting = [False]
def callback(monitor):
"""
prints progess bar
:param monitor: instance of
:class:`requests_toolbelt.MultipartEncoderMonitor`
"""
if monitor.bytes_read < expected_size:
bar.show(monitor.bytes_read)
elif not waiting[0]:
waiting[0] = True
bar.done()
print('\nwaiting for server-side processing ...')
return callback
def create_progress_callback(encoder):
encoder_len = encoder.len
progress_bar = Bar(expected_size=encoder_len, filled_char='=')
def callback(monitor):
progress_bar.show(monitor.bytes_read)
return callback, progress_bar
def do_artifacts_artifact_add(opts):
logging.debug('add artifact %r', opts)
url = artifacts_url(opts.service)
image = {
'name': opts.name,
'description': opts.description,
}
# build contents of multipart/form-data, image meta must come first, hence
# we use an OrderedDict to preserve the order
files = OrderedDict()
for k, v in image.items():
files[k] = (None, io.StringIO(v))
# followed by firmware data
# but first, try to find out the size of firmware data
files['size'] = str(os.stat(opts.infile).st_size)
files['artifact'] = (opts.infile, open(opts.infile, 'rb'),
"application/octet-stream", {})
encoder = MultipartEncoder(files)
if sys.stderr.isatty():
try:
from requests_toolbelt import MultipartEncoderMonitor
from clint.textui.progress import Bar as ProgressBar
pb = ProgressBar(expected_size=encoder.len,
filled_char='=',
every=1024 * 1024)
monitor = MultipartEncoderMonitor(
encoder, lambda mon: pb.show(mon.bytes_read))
encoder = monitor
except ImportError:
pass
with api_from_opts(opts) as api:
rsp = api.post(url,
data=encoder,
headers={'Content-Type': encoder.content_type})
if rsp.status_code == 201:
# created
location = rsp.headers.get('Location', '')
print("created with URL: {}".format(location))
print('artifact ID: ', location.rsplit('/')[-1])
else:
errorprinter(rsp)
class ProgressDownloadStream(object):
"""
Progress bar buffer class that can monitor a file download by displaying
progress to the user while also writing to the desired file.
Basic usage:
download_stream = ProgressDownloadStream(stream=api_response_obj,
expected_size=total_file_bytes)
download_stream.stream_to_file(desired_file_path)
"""
def __init__(self, stream=None, expected_size=None, chunk_size=512):
"""
:param stream:
Streaming API response object returned by the 'requests' library
:type: requests.Response
:param int expected_size:
Total size of file being downloaded (in bytes)
:param int chunk_size:
Chunk size to use when iterating over streamed response content
"""
self.stream = stream
self.prog_bar = ProgressBar(expected_size=expected_size,
filled_char='=')
self.progress = 0
self.chunk_size = chunk_size
def stream_to_file(self, path):
"""
Download file to given path and display progress to user
:param str path: Desired file path
"""
# Using default state of files being overwritten for now
if os.path.exists(path):
os.remove(path)
# Stream downloaded contents to file and show progress
with open(path, 'wb') as f:
for chunk in self.stream.iter_content(chunk_size=self.chunk_size):
f.write(chunk)
self.progress += int(len(chunk))
self.prog_bar.show(self.progress)
def _download_with_progressbar(response, content_length):
"""
:type response: requests.Response
:type content_length: int
"""
with Bar(expected_size=content_length) as bar:
for i, block in enumerate(response.iter_content(1024), 1):
yield block
bar.show(min(i * 1024, content_length))
class FileChunks(object):
def __init__(self,
filename,
chunk_size=2 * 1024 * 1024,
progress_hook=None):
self.chunk_size = chunk_size
self.amount_seen = 0
self.filename = filename
self.file_obj = open(filename, 'rb')
self.file_size = os.fstat(self.file_obj.fileno()).st_size
self.progress_hook = progress_hook
if self.progress_hook is None and ProgressBar is not None:
self.progress_bar = ProgressBar(label=filename,
expected_size=self.file_size,
filled_char='=')
#self.progress_bar.show(self.amount_seen)
else:
self.progress_bar = None
def get(self):
try:
data = self.file_obj.read(self.chunk_size)
while len(data) > 0:
self.amount_seen += len(data)
if self.progress_hook is not None:
self.progress_hook(self.filename, self.amount_seen,
self.file_size)
if self.progress_bar is not None:
self.progress_bar.show(self.amount_seen)
if self.progress_bar is None and self.progress_hook is None:
print('Uploading %s: %s / %s\r' %
(self.filename, self.amount_seen, self.file_size))
yield data
data = self.file_obj.read(self.chunk_size)
finally:
if self.progress_bar is not None:
self.progress_bar.done()
if self.progress_bar is None and self.progress_hook is None:
print('Done Uploading %s' % (self.filename, ))
self.file_obj.close()
def close(self):
if not self.file_obj.closed:
self.file_obj.close()
def resume_upload(self,
file_path,
file_endpoint,
chunk_size=None,
headers=None,
auth=None,
offset=None):
chunk_size = chunk_size or self.chunk_size
try:
offset = self._get_offset(file_endpoint,
headers=headers,
auth=auth)
except FloydException as e:
floyd_logger.error(
"Failed to fetch offset data from upload server! %s",
e.message)
return False
except requests.exceptions.ConnectionError as e:
floyd_logger.error(
"Cannot connect to the Floyd data upload server for offset. "
"Check your internet connection.")
return False
total_sent = 0
file_size = os.path.getsize(file_path)
with open(file_path, 'rb') as f:
pb = ProgressBar(filled_char="=", expected_size=file_size)
while offset < file_size:
pb.show(offset)
f.seek(offset)
data = f.read(chunk_size)
try:
offset = self._upload_chunk(data,
offset,
file_endpoint,
headers=headers,
auth=auth)
total_sent += len(data)
floyd_logger.debug("%s bytes sent", total_sent)
except FloydException as e:
floyd_logger.error(
"Failed to fetch offset data from upload server! %s",
e.message)
return False
except requests.exceptions.ConnectionError as e:
floyd_logger.error(
"Cannot connect to the Floyd data upload server. "
"Check your internet connection.")
return False
# Complete the progress bar with one more call to show()
pb.show(offset)
pb.done()
return True
def __init__(self, filename, chunk_size=2*1024*1024,progress_hook=None):
self.chunk_size = chunk_size
self.amount_seen = 0
self.filename = filename
self.file_obj = open(filename,'rb')
self.file_size = os.fstat(self.file_obj.fileno()).st_size
self.progress_hook = progress_hook
if self.progress_hook is None and ProgressBar is not None:
self.progress_bar = ProgressBar(label=filename,expected_size=self.file_size, filled_char='=')
#self.progress_bar.show(self.amount_seen)
else:
self.progress_bar = None
def course_exports(course_list):
'''Do course exports for each course in the list'''
#pprint.pprint(course_list)
bar = Bar(label='Exporting Courses ',expected_size=len(course_list))
export_list = []
for idx,course in enumerate(course_list):
export_list.append(course_export(course))
bar.show(idx)
blen = len(export_list)
bar.label = "Checking Export Statuses: "
bar.expected_length = blen
bar.show(1)
while export_list:
if len(export_list) < 10:
time.sleep(2)
bar.show(blen-len(export_list))
#print export_list
for idx,cm in enumerate(export_list):
course,res = cm
url = res.get('progress_url',None)
if not url:
del(export_list[idx])
print 'not able to generate export for course',course
else:
download_progress = requests.get(url,headers=config['headers']).json()
bar.label = "Checking Export Status: {} {}% ".format(course[course_id_field],download_progress['completion'])
if download_progress['workflow_state'] not in ['queued','running']:
if download_progress['workflow_state'] == 'completed':
url = "https://{domain}/api/v1/courses/{}/content_exports/{}".format(course[course_id_field],res['id'],**config)
export_info = requests.get(url,headers=config['headers']).json()
#print 'export_info',export_info
export_url = export_info['attachment']['url']
course['export_url'] = export_url
yield course
else:
course['export_url'] = None
del(export_list[idx])
def create_console_callback(size):
"""
Create progress callback suitable for displaying the progress of a
requests_toolbelt.MultipartEncoderMonitor to a text console.
@param size int representing the number of bytes to be read be the
MultipartEncoderMonitor
"""
bar = Bar(expected_size=size, filled_char='\u25A0', every=1024)
def callback(monitor):
bar.show(monitor.bytes_read)
return callback
def __init__(self, stream=None, expected_size=None, chunk_size=512):
"""
:param stream:
Streaming API response object returned by the 'requests' library
:type: requests.Response
:param int expected_size:
Total size of file being downloaded (in bytes)
:param int chunk_size:
Chunk size to use when iterating over streamed response content
"""
self.stream = stream
self.prog_bar = ProgressBar(expected_size=expected_size,
filled_char='=')
self.progress = 0
self.chunk_size = chunk_size
def runMigrations(copies):
pool = Pool(processes=num_processes)
#copies.reverse()
bar = Bar()
#res = pool.map(massDoCopies,((bar,x) for x in copies))
#for x in copies:
# massDoCopies((bar,x))
res = pool.map_async(massDoCopies, ((bar, x) for x in copies))
stats = []
try:
stats.append(res.get(0xFFFF))
except KeyboardInterrupt:
print 'kill processes'
#pool.terminate()
exit()
except TypeError, err:
print 'err', err
pass
def wget(url,target=None,printhook=None,proxies=None,connect_timeout=10,download_timeout=None,verify_cert=False,referer=None,user_agent=None,cert=None,resume=False,md5=None,sha1=None,sha256=None,cache_dir=None):
r"""Copy the contents of a file from a given URL to a local file.
Args:
url (str): URL to document
target (str) : full file path of downloaded file. If None, put in a temporary dir with supplied url filename (final part of url)
proxies (dict) : proxies to use. eg {'http':'http://wpad:3128','https':'http://wpad:3128'}
timeout (int) : seconds to wait for answer before giving up
auth (list) : (user,password) to authenticate wirh basic auth
verify_cert (bool or str) : either False, True (verify with embedded CA list), or path to a directory or PEM encoded CA bundle file
to check https certificate signature against.
cert (list) : pair of (x509certfilename,pemkeyfilename) for authenticating the client
referer (str):
user_agent:
resume (bool):
md5 (str) :
sha1 (str) :
sha256 (str) :
cache_dir (str) : if file exists here, and md5 matches, copy from here instead of downloading. If not, put a copy of the file here after downloading.
Returns:
str : path to downloaded file
>>> respath = wget('http://wapt.tranquil.it/wapt/tis-firefox_28.0.0-1_all.wapt','c:\\tmp\\test.wapt',proxies={'http':'http://proxy:3128'})
???
>>> os.stat(respath).st_size>10000
True
>>> respath = wget('http://localhost:8088/runstatus','c:\\tmp\\test.json')
???
"""
start_time = time.time()
last_time_display = 0.0
last_downloaded = 0
def reporthook(received,total):
total = float(total)
if total>1 and received>1:
# print only every second or at end
if (time.time()-start_time>1) and ((time.time()-last_time_display>=1) or (received>=total)):
speed = received /(1024.0 * (time.time()-start_time))
if printhook:
printhook(received,total,speed,url)
elif sys.stdout is not None:
try:
if received == 0:
print(u"Downloading %s (%.1f Mb)" % (url,int(total)/1024/1024))
elif received>=total:
print(u" -> download finished (%.0f Kb/s)" % (total /(1024.0*(time.time()+.001-start_time))))
else:
print(u'%i / %i (%.0f%%) (%.0f KB/s)\r' % (received,total,100.0*received/total,speed))
except:
return False
return True
else:
return False
if target is None:
target = tempfile.gettempdir()
if os.path.isdir(target):
target = os.path.join(target,'')
(dir,filename) = os.path.split(target)
if not filename:
url_parts = urlparse.urlparse(url)
filename = url_parts.path.split('/')[-1]
if not dir:
dir = os.getcwd()
if not os.path.isdir(dir):
os.makedirs(dir)
if verify_cert == False:
requests.packages.urllib3.disable_warnings() # pylint: disable=no-member
header=default_http_headers()
if referer != None:
header.update({'referer': '%s' % referer})
if user_agent != None:
header.update({'user-agent': '%s' % user_agent})
target_fn = os.path.join(dir,filename)
# return cached file if md5 matches.
if (md5 is not None or sha1 is not None or sha256 is not None) and cache_dir is not None and os.path.isdir(cache_dir):
cached_filename = os.path.join(cache_dir,filename)
if os.path.isfile(cached_filename):
if _check_hash_for_file(cached_filename,md5=md5,sha1=sha1,sha256=sha256):
resume = False
if cached_filename != target_fn:
shutil.copy2(cached_filename,target_fn)
return target_fn
else:
cached_filename = None
if os.path.isfile(target_fn) and resume:
try:
actual_size = os.stat(target_fn).st_size
size_req = requests.head(url,
proxies=proxies,
timeout=connect_timeout,
verify=verify_cert,
headers=header,
cert = cert,
allow_redirects=True)
target_size = int(size_req.headers['content-length'])
file_date = size_req.headers.get('last-modified',None)
if target_size > actual_size:
header.update({'Range':'bytes=%s-' % (actual_size,)})
write_mode = 'ab'
elif target_size < actual_size:
target_size = None
write_mode = 'wb'
except Exception as e:
target_size = None
write_mode = 'wb'
else:
file_date = None
actual_size = 0
target_size = None
write_mode = 'wb'
# check hashes if size equal
if resume and (md5 is not None or sha1 is not None or sha256 is not None) and target_size is not None and (target_size == actual_size):
if not _check_hash_for_file(target_fn,md5=md5,sha1=sha1,sha256=sha256):
# restart download...
target_size = None
write_mode = 'wb'
if not resume or target_size is None or (target_size - actual_size) > 0:
httpreq = requests.get(url,stream=True,
proxies=proxies,
timeout=connect_timeout,
verify=verify_cert,
headers=header,
cert = cert,
allow_redirects=True)
httpreq.raise_for_status()
total_bytes = int(httpreq.headers['content-length'])
target_free_bytes = get_disk_free_space(os.path.dirname(os.path.abspath(target)))
if total_bytes > target_free_bytes:
raise Exception('wget : not enough free space on target drive to get %s MB. Total size: %s MB. Free space: %s MB' % (url,total_bytes // (1024*1024),target_free_bytes // (1024*1024)))
# 1Mb max, 1kb min
chunk_size = min([1024*1024,max([total_bytes/100,2048])])
cnt = 0
if printhook is None and ProgressBar is not None:
progress_bar = ProgressBar(label=filename,expected_size=target_size or total_bytes, filled_char='=')
progress_bar.show(actual_size)
with open(target_fn,write_mode) as output_file:
last_time_display = time.time()
last_downloaded = 0
if httpreq.ok:
for chunk in httpreq.iter_content(chunk_size=chunk_size):
output_file.write(chunk)
output_file.flush()
if download_timeout is not None and (time.time()-start_time>download_timeout):
raise requests.Timeout(r'Download of %s takes more than the requested %ss'%(url,download_timeout))
if printhook is None and ProgressBar is not None:
if (time.time()-start_time>0.2) and (time.time()-last_time_display>=0.2):
progress_bar.show(actual_size + cnt*len(chunk))
last_time_display = time.time()
else:
if reporthook(cnt*len(chunk),total_bytes):
last_time_display = time.time()
last_downloaded += len(chunk)
cnt +=1
if printhook is None and ProgressBar is not None:
progress_bar.show(total_bytes)
progress_bar.done()
last_time_display = time.time()
elif reporthook(last_downloaded,total_bytes):
last_time_display = time.time()
# check hashes
if sha256 is not None:
file_hash = _hash_file(target_fn,hash_func=hashlib.sha256)
if file_hash != sha256:
raise Exception(u'Downloaded file %s sha256 %s does not match expected %s' % (url,file_hash,sha256))
elif sha1 is not None:
file_hash = _hash_file(target_fn,hash_func=hashlib.sha1)
if file_hash != sha1:
raise Exception(u'Downloaded file %s sha1 %s does not match expected %s' % (url,file_hash,sha1))
elif md5 is not None:
file_hash = _hash_file(target_fn,hash_func=hashlib.md5)
if file_hash != md5:
raise Exception(u'Downloaded file %s md5 %s does not match expected %s' % (url,file_hash,md5))
file_date = httpreq.headers.get('last-modified',None)
if file_date:
file_datetime = httpdatetime2time(file_date)
os.utime(target_fn,(file_datetime,file_datetime))
# cache result
if cache_dir:
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
cached_filename = os.path.join(cache_dir,filename)
if target_fn != cached_filename:
shutil.copy2(target_fn,cached_filename)
return target_fn
progr.show(monitor.bytes_read)
fileobj = jfs.up(p, testfile, upload_callback=UP)
_end = time.time()
puts(colored.magenta("Network upload speed %s/sec \n" % ( humanizeFileSize( (filesize / (_end-_start)) ) )))
# DOWNLOAD TEST 1
puts(colored.green('Test 2 | requests | Read | File size: %s' % humanizeFileSize(filesize)))
_start = time.time()
x = fileobj.read()
_end = time.time()
puts(colored.magenta("Network download speed %s/sec \n" % ( humanizeFileSize( (filesize / (_end-_start)) ) )))
del x
# DOWNLOAD TEST 2
puts(colored.green('Test 2 | requests | Stream | File size: %s' % humanizeFileSize(filesize)))
progr2 = ProgressBar(expected_size=filesize)
_start = time.time()
_bytesread = 0
for chunk in fileobj.stream():
_bytesread = _bytesread + len(chunk)
progr2.show(_bytesread)
_end = time.time()
puts(colored.magenta("Network download speed %s/sec \n" % ( humanizeFileSize( (filesize / (_end-_start)) ) )))
# TODO: PRINT STATS IN A TABLE FOR COMPARISON / BOOKKEEPING
# Versions: jottalib, urllib3, requests, jottaAPI
# Server version from jottacloud.com
# TEST WITHOUR REQUESTS, ONLY urllib3
# UPLOAD TEST
#
def runMigrations(copies): pool = Pool(processes=num_processes) #copies.reverse() bar = Bar() res = pool.map(massDoCopies,((bar,x) for x in copies))
puts(
colored.green('Test 2 | requests | Read | File size: %s' %
humanizeFileSize(filesize)))
_start = time.time()
x = fileobj.read()
_end = time.time()
puts(
colored.magenta("Network download speed %s/sec \n" % (humanizeFileSize(
(filesize / (_end - _start))))))
del x
# DOWNLOAD TEST 2
puts(
colored.green('Test 2 | requests | Stream | File size: %s' %
humanizeFileSize(filesize)))
progr2 = ProgressBar(expected_size=filesize)
_start = time.time()
_bytesread = 0
for chunk in fileobj.stream():
_bytesread = _bytesread + len(chunk)
progr2.show(_bytesread)
_end = time.time()
puts(
colored.magenta("Network download speed %s/sec \n" % (humanizeFileSize(
(filesize / (_end - _start))))))
# TODO: PRINT STATS IN A TABLE FOR COMPARISON / BOOKKEEPING
# Versions: jottalib, urllib3, requests, jottaAPI
# Server version from jottacloud.com
# TEST WITHOUR REQUESTS, ONLY urllib3
def create_tarfile(self):
"""
Create a tar file with the contents of the current directory
"""
floyd_logger.info("Compressing data...")
# Show progress bar (file_compressed/file_to_compress)
self.__compression_bar = ProgressBar(
expected_size=self.__files_to_compress, filled_char='=')
# Auxiliary functions
def dfilter_file_counter(tarinfo):
"""
Dummy filter function used to track the progression at file levels.
"""
self.__compression_bar.show(self.__files_compressed)
self.__files_compressed += 1
return tarinfo
def warn_purge_exit(info_msg, filename, progress_bar, exit_msg):
"""
Warn the user that's something went wrong,
remove the tarball and provide an exit message.
"""
progress_bar.done()
floyd_logger.info(info_msg)
rmtree(os.path.dirname(filename))
sys.exit(exit_msg)
try:
# Define the default signal handler for catching: Ctrl-C
signal.signal(signal.SIGINT, signal.default_int_handler)
with tarfile.open(self.filename, "w:gz") as tar:
tar.add(self.source_dir,
arcname=os.path.basename(self.source_dir),
filter=dfilter_file_counter)
self.__compression_bar.done()
except (OSError, IOError) as e:
# OSError: [Errno 13] Permission denied
if e.errno == errno.EACCES:
self.source_dir = os.getcwd(
) if self.source_dir == '.' else self.source_dir # Expand cwd
warn_purge_exit(
info_msg="Permission denied. Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg=(
"Permission denied. Make sure to have read permission "
"for all the files and directories in the path: %s") %
(self.source_dir))
# OSError: [Errno 28] No Space Left on Device (IOError on python2.7)
elif e.errno == errno.ENOSPC:
dir_path = os.path.dirname(self.filename)
warn_purge_exit(
info_msg="No space left. Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg=
("No space left when compressing your data in: %s.\n"
"Make sure to have enough space before uploading your data."
) % (os.path.abspath(dir_path)))
except KeyboardInterrupt: # Purge tarball on Ctrl-C
warn_purge_exit(
info_msg="Ctrl-C signal detected: Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg="Stopped the data upload gracefully.")
class DataCompressor(object):
"""
Local Data Compression with progress bar.
"""
def __init__(self, source_dir, filename):
# Data directory to compress
self.source_dir = source_dir
# Archive (Tar file) name
# e.g. "/tmp/contents.tar.gz"
self.filename = filename
# Prgress Bar for tracking data compression
self.__compression_bar = None
# Number of files to compress
self.__files_to_compress = 0
self.__get_nfiles_to_compress()
# Number of files already compressed
self.__files_compressed = 0
def __get_nfiles_to_compress(self):
"""
Return the number of files to compress
Note: it should take about 0.1s for counting 100k files on a dual core machine
"""
floyd_logger.info(
"Get number of files to compress... (this could take a few seconds)"
)
paths = [self.source_dir]
try:
# Traverse each subdirs of source_dir and count files/dirs
while paths:
path = paths.pop()
for item in scandir(path):
if item.is_dir():
paths.append(item.path)
self.__files_to_compress += 1
elif item.is_file():
self.__files_to_compress += 1
except OSError as e:
# OSError: [Errno 13] Permission denied
if e.errno == errno.EACCES:
self.source_dir = os.getcwd(
) if self.source_dir == '.' else self.source_dir # Expand cwd
sys.exit(
("Permission denied. Make sure to have read permission "
"for all the files and directories in the path: %s") %
(self.source_dir))
floyd_logger.info("Compressing %d files", self.__files_to_compress)
def create_tarfile(self):
"""
Create a tar file with the contents of the current directory
"""
floyd_logger.info("Compressing data...")
# Show progress bar (file_compressed/file_to_compress)
self.__compression_bar = ProgressBar(
expected_size=self.__files_to_compress, filled_char='=')
# Auxiliary functions
def dfilter_file_counter(tarinfo):
"""
Dummy filter function used to track the progression at file levels.
"""
self.__compression_bar.show(self.__files_compressed)
self.__files_compressed += 1
return tarinfo
def warn_purge_exit(info_msg, filename, progress_bar, exit_msg):
"""
Warn the user that's something went wrong,
remove the tarball and provide an exit message.
"""
progress_bar.done()
floyd_logger.info(info_msg)
rmtree(os.path.dirname(filename))
sys.exit(exit_msg)
try:
# Define the default signal handler for catching: Ctrl-C
signal.signal(signal.SIGINT, signal.default_int_handler)
with tarfile.open(self.filename, "w:gz") as tar:
tar.add(self.source_dir,
arcname=os.path.basename(self.source_dir),
filter=dfilter_file_counter)
self.__compression_bar.done()
except (OSError, IOError) as e:
# OSError: [Errno 13] Permission denied
if e.errno == errno.EACCES:
self.source_dir = os.getcwd(
) if self.source_dir == '.' else self.source_dir # Expand cwd
warn_purge_exit(
info_msg="Permission denied. Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg=(
"Permission denied. Make sure to have read permission "
"for all the files and directories in the path: %s") %
(self.source_dir))
# OSError: [Errno 28] No Space Left on Device (IOError on python2.7)
elif e.errno == errno.ENOSPC:
dir_path = os.path.dirname(self.filename)
warn_purge_exit(
info_msg="No space left. Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg=
("No space left when compressing your data in: %s.\n"
"Make sure to have enough space before uploading your data."
) % (os.path.abspath(dir_path)))
except KeyboardInterrupt: # Purge tarball on Ctrl-C
warn_purge_exit(
info_msg="Ctrl-C signal detected: Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg="Stopped the data upload gracefully.")
def __call__(self, size_rd, total):
if self.bar is None:
self.bar = ProgressBar(expected_size=total,
filled_char='=')
self.bar.show(size_rd)
def course_exports(course_list):
'''Do course exports for each course in the list'''
#pprint.pprint(course_list)
bar = Bar(label='Exporting Courses ', expected_size=len(course_list))
export_list = []
for idx, course in enumerate(course_list):
export_list.append(course_export(course))
bar.show(idx)
blen = len(export_list)
bar.label = "Checking Export Statuses: "
bar.expected_length = blen
bar.show(1)
while export_list:
if len(export_list) < 10:
time.sleep(2)
bar.show(blen - len(export_list))
#print export_list
for idx, cm in enumerate(export_list):
course, res = cm
url = res.get('progress_url', None)
if not url:
del (export_list[idx])
print 'not able to generate export for course', course
else:
download_progress = requests.get(
url, headers=config['headers']).json()
bar.label = "Checking Export Status: {} {}% ".format(
course[course_id_field], download_progress['completion'])
if download_progress['workflow_state'] not in [
'queued', 'running'
]:
if download_progress['workflow_state'] == 'completed':
url = "https://{domain}/api/v1/courses/{}/content_exports/{}".format(
course[course_id_field], res['id'], **config)
export_info = requests.get(
url, headers=config['headers']).json()
#print 'export_info',export_info
export_url = export_info['attachment']['url']
course['export_url'] = export_url
yield course
else:
course['export_url'] = None
del (export_list[idx])
def __init__(self, length):
self.bar = Bar(expected_size=length, filled_char='=')
self.status = 0
