def test_task_copy_files(self, d): os.chdir(d.path) file_path1 = os.path.join(d.path, 'test-copy-1.txt') file_path2 = os.path.join(d.path, 'test-copy-2.txt') to_path = os.path.join(d.path, 'files') d.write(file_path1, 'sample data'.encode('utf-8')) d.write(file_path2, 'sample data'.encode('utf-8')) FileUtil.create_dir(to_path) task = Task(task_type=Task.TYPE_COPY_FILES, task_name='Sample copy files task', task_params={ 'from': os.path.join(d.path, '*.txt'), 'to': to_path }) process_data = ProcessData() template_data = {} task.parse(process_data) task.run(process_data=process_data, template_data=template_data, working_dir=d.path) self.assertTrue( os.path.exists(os.path.join(to_path, 'test-copy-1.txt'))) self.assertTrue( os.path.exists(os.path.join(to_path, 'test-copy-2.txt')))
def download_from_github(self): # download Logger.i('Downloading repository: {0}...'.format(self.get_name())) download_url = self.get_download_url() download_filename = self.get_download_filename() download_dest_dir = Constants.TEMP_DIR download_dest_path = os.path.join(Constants.TEMP_DIR, download_filename) unpacked_dir = self.get_temp_dir() unpack_dir = Constants.TEMP_DIR force_download = False _, git_data_type, git_data_version = self.get_git_data() if git_data_type == Repository.GIT_TYPE_BRANCH: force_download = True # skip if exists if not force_download and os.path.isfile(download_dest_path): Logger.i('Repository already downloaded: {0}'.format( self.get_name())) else: FileUtil.remove_file(download_dest_path) DownloadUtil.download_file(download_url, download_dest_dir, download_filename) # check if file was downloaded if os.path.isfile(download_dest_path): Logger.i('Repository downloaded: {0}'.format(self.get_name())) else: Logger.f('Problems when download repository: {0}'.format( self.get_name())) # unpack Logger.i('Unpacking repository: {0}...'.format(self.get_name())) if not force_download and os.path.isdir(unpacked_dir): Logger.i('Repository already unpacked: {0}...'.format( self.get_name())) else: FileUtil.remove_dir(unpacked_dir) # untar file FileUtil.create_dir(unpack_dir) tar = tarfile.open(download_dest_path) tar.extractall(path=unpack_dir) tar.close() if os.path.isdir(unpacked_dir): Logger.i('Repository unpacked: {0}'.format(self.get_name())) else: Logger.f('Problems when unpack repository: {0}'.format( self.get_name()))
def test_create_dir(self, d): os.chdir(d.path) dir_name_1 = 'new-dir-1' dir_name_2 = 'new-dir-2' FileUtil.create_dir(dir_name_1) FileUtil.create_dir(os.path.join(dir_name_1, dir_name_2)) self.assertTrue(os.path.isdir(dir_name_1)) self.assertTrue(os.path.isdir(os.path.join(dir_name_1, dir_name_2)))
def download_from_zip(self): # download Logger.i('Downloading repository: {0}...'.format(self.get_name())) download_url = self.get_download_url() download_filename = self.get_download_filename() download_dest_dir = Constants.TEMP_DIR download_dest_path = os.path.join(download_dest_dir, download_filename) unpacked_dir = self.get_temp_dir() unpack_dir = download_dest_dir # skip if exists if os.path.isfile(download_dest_path): Logger.i('Repository already downloaded: {0}'.format( self.get_name())) else: FileUtil.remove_file(download_dest_path) DownloadUtil.download_file(download_url, download_dest_dir, download_filename) # check if file was downloaded if os.path.isfile(download_dest_path): Logger.i('Repository downloaded: {0}'.format(self.get_name())) else: Logger.f('Problems when download repository: {0}'.format( self.get_name())) # unpack Logger.i('Unpacking repository: {0}...'.format(self.get_name())) if os.path.isdir(unpacked_dir): Logger.i('Repository already unpacked: {0}...'.format( self.get_name())) else: FileUtil.remove_dir(unpacked_dir) # unpack file FileUtil.create_dir(unpack_dir) zipref = zipfile.ZipFile(download_dest_path, 'r') zipref.extractall(path=unpack_dir) zipref.close() if os.path.isdir(unpacked_dir): Logger.i('Repository unpacked: {0}'.format(self.get_name())) else: Logger.f('Problems when unpack repository: {0}'.format( self.get_name()))
def download_file(url, dest=None, filename=None): """ Download and save a file specified by url to dest directory. """ Logger.d('New download request: {0}'.format(url)) Logger.d('Destination: {0}'.format(dest)) Logger.d('Filename: {0}'.format(filename)) req = urllib2.Request(url, headers={'User-Agent': 'Mozilla/5.0'}) u = urllib2.urlopen(req) scheme, netloc, path, query, fragment = urlparse.urlsplit(url) if not filename: filename = DownloadUtil.get_filename_from_url(path) if dest: FileUtil.create_dir(dest) filename = os.path.join(dest, filename) Logger.d('Getting file metadata...') with open(filename, 'wb') as f: meta = u.info() meta_func = meta.getheaders if hasattr( meta, 'getheaders') else meta.get_all meta_length = meta_func('Content-Length') file_size = None pbar = None if meta_length: file_size = int(meta_length[0]) if file_size: Logger.d('File size in bytes: {0}'.format(file_size)) Logger.clean('') pbar = tqdm(total=file_size) file_size_dl = 0 block_sz = 8192 if not pbar: Logger.d('Downloading, please wait...') while True: dbuffer = u.read(block_sz) if not dbuffer: break dbuffer_len = len(dbuffer) file_size_dl += dbuffer_len f.write(dbuffer) if pbar: pbar.update(dbuffer_len) if pbar: pbar.close() Logger.clean('') return filename