def _upload(self, package):
data = package.metadata_dictionary()
data.update({
# action
":action": "file_upload",
"protcol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
print("Uploading {0}".format(package.basefilename))
with open(package.filename, "rb") as fp:
data_to_send.append((
"content",
(package.basefilename, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = ProgressBar(expected_size=encoder.len, filled_char='=')
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.show(monitor.bytes_read))
resp = self.session.post(
self.url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type},
)
bar.done()
return resp
def download_artifacts(session):
response = session.get("{0}/lastBuild/api/json/".format(JENKINS_URL),
headers={"Accept": "application/json"})
response.raise_for_status()
json_response = response.json()
assert not json_response["building"]
assert json_response["result"] == "SUCCESS"
paths = []
for artifact in json_response["artifacts"]:
response = session.get("{0}artifact/{1}".format(
json_response["url"], artifact["relativePath"]),
stream=True)
assert response.headers["content-length"]
print("Downloading {0}".format(artifact["fileName"]))
bar = ProgressBar(expected_size=int(
response.headers["content-length"]),
filled_char="=")
content = io.BytesIO()
for data in response.iter_content(chunk_size=8192):
content.write(data)
bar.show(content.tell())
assert bar.expected_size == content.tell()
bar.done()
out_path = os.path.join(
os.path.dirname(__file__),
"dist",
artifact["fileName"],
)
with open(out_path, "wb") as f:
f.write(content.getvalue())
paths.append(out_path)
return paths
def _upload(self, package):
data = package.metadata_dictionary()
data.update({
# action
":action": "file_upload",
"protcol_version": "1",
})
data_to_send = self._convert_data_to_list_of_tuples(data)
print("Uploading {0}".format(package.basefilename))
with open(package.filename, "rb") as fp:
data_to_send.append((
"content",
(package.basefilename, fp, "application/octet-stream"),
))
encoder = MultipartEncoder(data_to_send)
bar = ProgressBar(expected_size=encoder.len, filled_char='=')
monitor = MultipartEncoderMonitor(
encoder, lambda monitor: bar.show(monitor.bytes_read)
)
resp = self.session.post(
self.url,
data=monitor,
allow_redirects=False,
headers={'Content-Type': monitor.content_type},
)
bar.done()
return resp
def resume_upload(self,
file_path,
file_endpoint,
chunk_size=None,
headers=None,
auth=None,
offset=None):
chunk_size = chunk_size or self.chunk_size
try:
offset = self._get_offset(file_endpoint,
headers=headers,
auth=auth)
except FloydException as e:
floyd_logger.error(
"Failed to fetch offset data from upload server! %s",
e.message)
return False
except requests.exceptions.ConnectionError as e:
floyd_logger.error(
"Cannot connect to the Floyd data upload server for offset. "
"Check your internet connection.")
return False
total_sent = 0
file_size = os.path.getsize(file_path)
with open(file_path, 'rb') as f:
pb = ProgressBar(filled_char="=", expected_size=file_size)
while offset < file_size:
pb.show(offset)
f.seek(offset)
data = f.read(chunk_size)
try:
offset = self._upload_chunk(data,
offset,
file_endpoint,
headers=headers,
auth=auth)
total_sent += len(data)
floyd_logger.debug("%s bytes sent", total_sent)
except FloydException as e:
floyd_logger.error(
"Failed to fetch offset data from upload server! %s",
e.message)
return False
except requests.exceptions.ConnectionError as e:
floyd_logger.error(
"Cannot connect to the Floyd data upload server. "
"Check your internet connection.")
return False
# Complete the progress bar with one more call to show()
pb.show(offset)
pb.done()
return True
def download_artifacts(session):
response = session.get(
"{0}/lastBuild/api/json/".format(JENKINS_URL),
headers={
"Accept": "application/json"
}
)
response.raise_for_status()
assert not response.json()["building"]
assert response.json()["result"] == "SUCCESS"
paths = []
last_build_number = response.json()["number"]
for run in response.json()["runs"]:
if run["number"] != last_build_number:
print(
"Skipping {0} as it is not from the latest build ({1})".format(
run["url"], last_build_number
)
)
continue
response = session.get(
run["url"] + "api/json/",
headers={
"Accept": "application/json",
}
)
response.raise_for_status()
for artifact in response.json()["artifacts"]:
response = session.get(
"{0}artifact/{1}".format(run["url"], artifact["relativePath"]),
stream=True
)
assert response.headers["content-length"]
print("Downloading {0}".format(artifact["fileName"]))
bar = ProgressBar(
expected_size=int(response.headers["content-length"]),
filled_char="="
)
content = io.BytesIO()
for data in response.iter_content(chunk_size=8192):
content.write(data)
bar.show(content.tell())
assert bar.expected_size == content.tell()
bar.done()
out_path = os.path.join(
os.path.dirname(__file__),
"dist",
artifact["fileName"],
)
with open(out_path, "wb") as f:
f.write(content.getvalue())
paths.append(out_path)
return paths
class FileChunks(object):
def __init__(self,
filename,
chunk_size=2 * 1024 * 1024,
progress_hook=None):
self.chunk_size = chunk_size
self.amount_seen = 0
self.filename = filename
self.file_obj = open(filename, 'rb')
self.file_size = os.fstat(self.file_obj.fileno()).st_size
self.progress_hook = progress_hook
if self.progress_hook is None and ProgressBar is not None:
self.progress_bar = ProgressBar(label=filename,
expected_size=self.file_size,
filled_char='=')
#self.progress_bar.show(self.amount_seen)
else:
self.progress_bar = None
def get(self):
try:
data = self.file_obj.read(self.chunk_size)
while len(data) > 0:
self.amount_seen += len(data)
if self.progress_hook is not None:
self.progress_hook(self.filename, self.amount_seen,
self.file_size)
if self.progress_bar is not None:
self.progress_bar.show(self.amount_seen)
if self.progress_bar is None and self.progress_hook is None:
print('Uploading %s: %s / %s\r' %
(self.filename, self.amount_seen, self.file_size))
yield data
data = self.file_obj.read(self.chunk_size)
finally:
if self.progress_bar is not None:
self.progress_bar.done()
if self.progress_bar is None and self.progress_hook is None:
print('Done Uploading %s' % (self.filename, ))
self.file_obj.close()
def close(self):
if not self.file_obj.closed:
self.file_obj.close()
def wget(url,target=None,printhook=None,proxies=None,connect_timeout=10,download_timeout=None,verify_cert=False,referer=None,user_agent=None,cert=None,resume=False,md5=None,sha1=None,sha256=None,cache_dir=None):
r"""Copy the contents of a file from a given URL to a local file.
Args:
url (str): URL to document
target (str) : full file path of downloaded file. If None, put in a temporary dir with supplied url filename (final part of url)
proxies (dict) : proxies to use. eg {'http':'http://wpad:3128','https':'http://wpad:3128'}
timeout (int) : seconds to wait for answer before giving up
auth (list) : (user,password) to authenticate wirh basic auth
verify_cert (bool or str) : either False, True (verify with embedded CA list), or path to a directory or PEM encoded CA bundle file
to check https certificate signature against.
cert (list) : pair of (x509certfilename,pemkeyfilename) for authenticating the client
referer (str):
user_agent:
resume (bool):
md5 (str) :
sha1 (str) :
sha256 (str) :
cache_dir (str) : if file exists here, and md5 matches, copy from here instead of downloading. If not, put a copy of the file here after downloading.
Returns:
str : path to downloaded file
>>> respath = wget('http://wapt.tranquil.it/wapt/tis-firefox_28.0.0-1_all.wapt','c:\\tmp\\test.wapt',proxies={'http':'http://proxy:3128'})
???
>>> os.stat(respath).st_size>10000
True
>>> respath = wget('http://localhost:8088/runstatus','c:\\tmp\\test.json')
???
"""
start_time = time.time()
last_time_display = 0.0
last_downloaded = 0
def reporthook(received,total):
total = float(total)
if total>1 and received>1:
# print only every second or at end
if (time.time()-start_time>1) and ((time.time()-last_time_display>=1) or (received>=total)):
speed = received /(1024.0 * (time.time()-start_time))
if printhook:
printhook(received,total,speed,url)
elif sys.stdout is not None:
try:
if received == 0:
print(u"Downloading %s (%.1f Mb)" % (url,int(total)/1024/1024))
elif received>=total:
print(u" -> download finished (%.0f Kb/s)" % (total /(1024.0*(time.time()+.001-start_time))))
else:
print(u'%i / %i (%.0f%%) (%.0f KB/s)\r' % (received,total,100.0*received/total,speed))
except:
return False
return True
else:
return False
if target is None:
target = tempfile.gettempdir()
if os.path.isdir(target):
target = os.path.join(target,'')
(dir,filename) = os.path.split(target)
if not filename:
url_parts = urlparse.urlparse(url)
filename = url_parts.path.split('/')[-1]
if not dir:
dir = os.getcwd()
if not os.path.isdir(dir):
os.makedirs(dir)
if verify_cert == False:
requests.packages.urllib3.disable_warnings() # pylint: disable=no-member
header=default_http_headers()
if referer != None:
header.update({'referer': '%s' % referer})
if user_agent != None:
header.update({'user-agent': '%s' % user_agent})
target_fn = os.path.join(dir,filename)
# return cached file if md5 matches.
if (md5 is not None or sha1 is not None or sha256 is not None) and cache_dir is not None and os.path.isdir(cache_dir):
cached_filename = os.path.join(cache_dir,filename)
if os.path.isfile(cached_filename):
if _check_hash_for_file(cached_filename,md5=md5,sha1=sha1,sha256=sha256):
resume = False
if cached_filename != target_fn:
shutil.copy2(cached_filename,target_fn)
return target_fn
else:
cached_filename = None
if os.path.isfile(target_fn) and resume:
try:
actual_size = os.stat(target_fn).st_size
size_req = requests.head(url,
proxies=proxies,
timeout=connect_timeout,
verify=verify_cert,
headers=header,
cert = cert,
allow_redirects=True)
target_size = int(size_req.headers['content-length'])
file_date = size_req.headers.get('last-modified',None)
if target_size > actual_size:
header.update({'Range':'bytes=%s-' % (actual_size,)})
write_mode = 'ab'
elif target_size < actual_size:
target_size = None
write_mode = 'wb'
except Exception as e:
target_size = None
write_mode = 'wb'
else:
file_date = None
actual_size = 0
target_size = None
write_mode = 'wb'
# check hashes if size equal
if resume and (md5 is not None or sha1 is not None or sha256 is not None) and target_size is not None and (target_size == actual_size):
if not _check_hash_for_file(target_fn,md5=md5,sha1=sha1,sha256=sha256):
# restart download...
target_size = None
write_mode = 'wb'
if not resume or target_size is None or (target_size - actual_size) > 0:
httpreq = requests.get(url,stream=True,
proxies=proxies,
timeout=connect_timeout,
verify=verify_cert,
headers=header,
cert = cert,
allow_redirects=True)
httpreq.raise_for_status()
total_bytes = int(httpreq.headers['content-length'])
target_free_bytes = get_disk_free_space(os.path.dirname(os.path.abspath(target)))
if total_bytes > target_free_bytes:
raise Exception('wget : not enough free space on target drive to get %s MB. Total size: %s MB. Free space: %s MB' % (url,total_bytes // (1024*1024),target_free_bytes // (1024*1024)))
# 1Mb max, 1kb min
chunk_size = min([1024*1024,max([total_bytes/100,2048])])
cnt = 0
if printhook is None and ProgressBar is not None:
progress_bar = ProgressBar(label=filename,expected_size=target_size or total_bytes, filled_char='=')
progress_bar.show(actual_size)
with open(target_fn,write_mode) as output_file:
last_time_display = time.time()
last_downloaded = 0
if httpreq.ok:
for chunk in httpreq.iter_content(chunk_size=chunk_size):
output_file.write(chunk)
output_file.flush()
if download_timeout is not None and (time.time()-start_time>download_timeout):
raise requests.Timeout(r'Download of %s takes more than the requested %ss'%(url,download_timeout))
if printhook is None and ProgressBar is not None:
if (time.time()-start_time>0.2) and (time.time()-last_time_display>=0.2):
progress_bar.show(actual_size + cnt*len(chunk))
last_time_display = time.time()
else:
if reporthook(cnt*len(chunk),total_bytes):
last_time_display = time.time()
last_downloaded += len(chunk)
cnt +=1
if printhook is None and ProgressBar is not None:
progress_bar.show(total_bytes)
progress_bar.done()
last_time_display = time.time()
elif reporthook(last_downloaded,total_bytes):
last_time_display = time.time()
# check hashes
if sha256 is not None:
file_hash = _hash_file(target_fn,hash_func=hashlib.sha256)
if file_hash != sha256:
raise Exception(u'Downloaded file %s sha256 %s does not match expected %s' % (url,file_hash,sha256))
elif sha1 is not None:
file_hash = _hash_file(target_fn,hash_func=hashlib.sha1)
if file_hash != sha1:
raise Exception(u'Downloaded file %s sha1 %s does not match expected %s' % (url,file_hash,sha1))
elif md5 is not None:
file_hash = _hash_file(target_fn,hash_func=hashlib.md5)
if file_hash != md5:
raise Exception(u'Downloaded file %s md5 %s does not match expected %s' % (url,file_hash,md5))
file_date = httpreq.headers.get('last-modified',None)
if file_date:
file_datetime = httpdatetime2time(file_date)
os.utime(target_fn,(file_datetime,file_datetime))
# cache result
if cache_dir:
if not os.path.isdir(cache_dir):
os.makedirs(cache_dir)
cached_filename = os.path.join(cache_dir,filename)
if target_fn != cached_filename:
shutil.copy2(target_fn,cached_filename)
return target_fn
class DataCompressor(object):
"""
Local Data Compression with progress bar.
"""
def __init__(self, source_dir, filename):
# Data directory to compress
self.source_dir = source_dir
# Archive (Tar file) name
# e.g. "/tmp/contents.tar.gz"
self.filename = filename
# Prgress Bar for tracking data compression
self.__compression_bar = None
# Number of files to compress
self.__files_to_compress = 0
self.__get_nfiles_to_compress()
# Number of files already compressed
self.__files_compressed = 0
def __get_nfiles_to_compress(self):
"""
Return the number of files to compress
Note: it should take about 0.1s for counting 100k files on a dual core machine
"""
floyd_logger.info(
"Get number of files to compress... (this could take a few seconds)"
)
paths = [self.source_dir]
try:
# Traverse each subdirs of source_dir and count files/dirs
while paths:
path = paths.pop()
for item in scandir(path):
if item.is_dir():
paths.append(item.path)
self.__files_to_compress += 1
elif item.is_file():
self.__files_to_compress += 1
except OSError as e:
# OSError: [Errno 13] Permission denied
if e.errno == errno.EACCES:
self.source_dir = os.getcwd(
) if self.source_dir == '.' else self.source_dir # Expand cwd
sys.exit(
("Permission denied. Make sure to have read permission "
"for all the files and directories in the path: %s") %
(self.source_dir))
floyd_logger.info("Compressing %d files", self.__files_to_compress)
def create_tarfile(self):
"""
Create a tar file with the contents of the current directory
"""
floyd_logger.info("Compressing data...")
# Show progress bar (file_compressed/file_to_compress)
self.__compression_bar = ProgressBar(
expected_size=self.__files_to_compress, filled_char='=')
# Auxiliary functions
def dfilter_file_counter(tarinfo):
"""
Dummy filter function used to track the progression at file levels.
"""
self.__compression_bar.show(self.__files_compressed)
self.__files_compressed += 1
return tarinfo
def warn_purge_exit(info_msg, filename, progress_bar, exit_msg):
"""
Warn the user that's something went wrong,
remove the tarball and provide an exit message.
"""
progress_bar.done()
floyd_logger.info(info_msg)
rmtree(os.path.dirname(filename))
sys.exit(exit_msg)
try:
# Define the default signal handler for catching: Ctrl-C
signal.signal(signal.SIGINT, signal.default_int_handler)
with tarfile.open(self.filename, "w:gz") as tar:
tar.add(self.source_dir,
arcname=os.path.basename(self.source_dir),
filter=dfilter_file_counter)
self.__compression_bar.done()
except (OSError, IOError) as e:
# OSError: [Errno 13] Permission denied
if e.errno == errno.EACCES:
self.source_dir = os.getcwd(
) if self.source_dir == '.' else self.source_dir # Expand cwd
warn_purge_exit(
info_msg="Permission denied. Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg=(
"Permission denied. Make sure to have read permission "
"for all the files and directories in the path: %s") %
(self.source_dir))
# OSError: [Errno 28] No Space Left on Device (IOError on python2.7)
elif e.errno == errno.ENOSPC:
dir_path = os.path.dirname(self.filename)
warn_purge_exit(
info_msg="No space left. Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg=
("No space left when compressing your data in: %s.\n"
"Make sure to have enough space before uploading your data."
) % (os.path.abspath(dir_path)))
except KeyboardInterrupt: # Purge tarball on Ctrl-C
warn_purge_exit(
info_msg="Ctrl-C signal detected: Removing compressed data...",
filename=self.filename,
progress_bar=self.__compression_bar,
exit_msg="Stopped the data upload gracefully.")
