def test_task_copy_files(self, d):
os.chdir(d.path)
file_path1 = os.path.join(d.path, 'test-copy-1.txt')
file_path2 = os.path.join(d.path, 'test-copy-2.txt')
to_path = os.path.join(d.path, 'files')
d.write(file_path1, 'sample data'.encode('utf-8'))
d.write(file_path2, 'sample data'.encode('utf-8'))
FileUtil.create_dir(to_path)
task = Task(task_type=Task.TYPE_COPY_FILES,
task_name='Sample copy files task',
task_params={
'from': os.path.join(d.path, '*.txt'),
'to': to_path
})
process_data = ProcessData()
template_data = {}
task.parse(process_data)
task.run(process_data=process_data,
template_data=template_data,
working_dir=d.path)
self.assertTrue(
os.path.exists(os.path.join(to_path, 'test-copy-1.txt')))
self.assertTrue(
os.path.exists(os.path.join(to_path, 'test-copy-2.txt')))
def download_from_github(self):
# download
Logger.i('Downloading repository: {0}...'.format(self.get_name()))
download_url = self.get_download_url()
download_filename = self.get_download_filename()
download_dest_dir = Constants.TEMP_DIR
download_dest_path = os.path.join(Constants.TEMP_DIR,
download_filename)
unpacked_dir = self.get_temp_dir()
unpack_dir = Constants.TEMP_DIR
force_download = False
_, git_data_type, git_data_version = self.get_git_data()
if git_data_type == Repository.GIT_TYPE_BRANCH:
force_download = True
# skip if exists
if not force_download and os.path.isfile(download_dest_path):
Logger.i('Repository already downloaded: {0}'.format(
self.get_name()))
else:
FileUtil.remove_file(download_dest_path)
DownloadUtil.download_file(download_url, download_dest_dir,
download_filename)
# check if file was downloaded
if os.path.isfile(download_dest_path):
Logger.i('Repository downloaded: {0}'.format(self.get_name()))
else:
Logger.f('Problems when download repository: {0}'.format(
self.get_name()))
# unpack
Logger.i('Unpacking repository: {0}...'.format(self.get_name()))
if not force_download and os.path.isdir(unpacked_dir):
Logger.i('Repository already unpacked: {0}...'.format(
self.get_name()))
else:
FileUtil.remove_dir(unpacked_dir)
# untar file
FileUtil.create_dir(unpack_dir)
tar = tarfile.open(download_dest_path)
tar.extractall(path=unpack_dir)
tar.close()
if os.path.isdir(unpacked_dir):
Logger.i('Repository unpacked: {0}'.format(self.get_name()))
else:
Logger.f('Problems when unpack repository: {0}'.format(
self.get_name()))
def test_create_dir(self, d):
os.chdir(d.path)
dir_name_1 = 'new-dir-1'
dir_name_2 = 'new-dir-2'
FileUtil.create_dir(dir_name_1)
FileUtil.create_dir(os.path.join(dir_name_1, dir_name_2))
self.assertTrue(os.path.isdir(dir_name_1))
self.assertTrue(os.path.isdir(os.path.join(dir_name_1, dir_name_2)))
def download_from_zip(self):
# download
Logger.i('Downloading repository: {0}...'.format(self.get_name()))
download_url = self.get_download_url()
download_filename = self.get_download_filename()
download_dest_dir = Constants.TEMP_DIR
download_dest_path = os.path.join(download_dest_dir, download_filename)
unpacked_dir = self.get_temp_dir()
unpack_dir = download_dest_dir
# skip if exists
if os.path.isfile(download_dest_path):
Logger.i('Repository already downloaded: {0}'.format(
self.get_name()))
else:
FileUtil.remove_file(download_dest_path)
DownloadUtil.download_file(download_url, download_dest_dir,
download_filename)
# check if file was downloaded
if os.path.isfile(download_dest_path):
Logger.i('Repository downloaded: {0}'.format(self.get_name()))
else:
Logger.f('Problems when download repository: {0}'.format(
self.get_name()))
# unpack
Logger.i('Unpacking repository: {0}...'.format(self.get_name()))
if os.path.isdir(unpacked_dir):
Logger.i('Repository already unpacked: {0}...'.format(
self.get_name()))
else:
FileUtil.remove_dir(unpacked_dir)
# unpack file
FileUtil.create_dir(unpack_dir)
zipref = zipfile.ZipFile(download_dest_path, 'r')
zipref.extractall(path=unpack_dir)
zipref.close()
if os.path.isdir(unpacked_dir):
Logger.i('Repository unpacked: {0}'.format(self.get_name()))
else:
Logger.f('Problems when unpack repository: {0}'.format(
self.get_name()))
def download_file(url, dest=None, filename=None):
"""
Download and save a file specified by url to dest directory.
"""
Logger.d('New download request: {0}'.format(url))
Logger.d('Destination: {0}'.format(dest))
Logger.d('Filename: {0}'.format(filename))
req = urllib2.Request(url, headers={'User-Agent': 'Mozilla/5.0'})
u = urllib2.urlopen(req)
scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
if not filename:
filename = DownloadUtil.get_filename_from_url(path)
if dest:
FileUtil.create_dir(dest)
filename = os.path.join(dest, filename)
Logger.d('Getting file metadata...')
with open(filename, 'wb') as f:
meta = u.info()
meta_func = meta.getheaders if hasattr(
meta, 'getheaders') else meta.get_all
meta_length = meta_func('Content-Length')
file_size = None
pbar = None
if meta_length:
file_size = int(meta_length[0])
if file_size:
Logger.d('File size in bytes: {0}'.format(file_size))
Logger.clean('')
pbar = tqdm(total=file_size)
file_size_dl = 0
block_sz = 8192
if not pbar:
Logger.d('Downloading, please wait...')
while True:
dbuffer = u.read(block_sz)
if not dbuffer:
break
dbuffer_len = len(dbuffer)
file_size_dl += dbuffer_len
f.write(dbuffer)
if pbar:
pbar.update(dbuffer_len)
if pbar:
pbar.close()
Logger.clean('')
return filename