def get_resources(self) -> List[str]: """ Returns list of resources' names """ return Query.folder( self.resources_location).select(lambda z: z.name).to_list()
def test_files(self): folder = Path(__file__).parent / "temp" os.makedirs(folder, exist_ok=True) if os.path.isdir(folder): # pragma: no cover shutil.rmtree(folder) plan = self.get_default_plan() self.kraken_run(kraken_simple_method, plan, cache_to_folder=folder, special_iterations=[3]) file = Query.folder(folder).single() self.assertEqual('3.kraken.pkl', file.name) results = self.kraken_run( kraken_simple_method, plan, cache_to_folder=folder) # type: List[kraken.IterationResult] for index, result in enumerate(results): self.assertEqual( kraken.IterationStatus.Skipped if index == 3 else kraken.IterationStatus.Success, result.status) self.assertIsNone(result.condition) self.assertIsNone(result.result) loaded_results = Query.en(kraken.Kraken.load( folder, None)).order_by(lambda z: z.result).to_list() self.assertResult([11, 12, 21, 22], loaded_results) shutil.rmtree(folder)
def _load_data(self, data: Union[str, Path, pd.DataFrame]): if isinstance(data, pd.DataFrame): return data if isinstance(data, str) or isinstance(data, Path): data = str(data) if os.path.isfile(data): return pd.read_parquet(data) elif os.path.isdir(data): dfs = Query.folder(data).select(pd.read_parquet).to_list() return pd.concat(dfs, sort=False) else: raise ValueError(f'Data was `{data}`, but there is neither file nor folder on this location') else: raise ValueError(f"Data was `{data}`, but the format is not supported")
def upload_folder(bucket_name: str, s3_path: str, folder: Path): aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', None) aws_secret_access = os.environ.get('AWS_SECRET_ACCESS_KEY', None) if aws_access_key_id is not None and aws_secret_access is not None: kwargs = dict(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access) else: kwargs = {} s3 = boto3.resource('s3', **kwargs) bucket = s3.Bucket(bucket_name) bucket.objects.filter(Prefix=s3_path).delete() client = boto3.client('s3', **kwargs) for file_path in Query.folder(folder): file_path_str = file_path.__str__() joint_path = os.path.join(s3_path, file_path.name) client.upload_file(file_path_str, bucket_name, joint_path)
def send(self): aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', None) aws_secret_access = os.environ.get('AWS_SECRET_ACCESS_KEY', None) if aws_access_key_id is not None and aws_secret_access is not None: kwargs = dict(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access) else: kwargs = {} s3 = boto3.resource('s3', **kwargs) bucket = s3.Bucket(self.s3_bucket) for name in self.names: s3_full_path = os.path.join(self.s3_path, name) bucket.objects.filter(Prefix=s3_full_path).delete() client = boto3.client('s3', **kwargs) for name in self.names: for file_path in Query.folder(self.location.joinpath(name)): client.upload_file( file_path.__str__(), self.s3_bucket, os.path.join(self.s3_path, name, file_path.name))
def upload_folder(self, local_path, remote_path): for file in Query.folder(local_path,'**/*').where(lambda z: z.is_file()): inner = str(file)[len(str(local_path)):] if inner.startswith('/'): inner = inner[1:] self.upload_file(os.path.join(local_path, inner), os.path.join(remote_path, inner))
def make_package( task: PackagingTask, dst_location: Optional[Union[Path, str]] = None) -> PackageInfo: """ Creates the package out of the :class:``PackagingTask``, and returns :class:``PackagingInfo``` describing this package """ if dst_location is None: dst_location = Loc.temp_path.joinpath('release/package') elif isinstance(dst_location, str): dst_location = Path(dst_location) elif not isinstance(dst_location, Path): raise ValueError( f'dst_location was {dst_location}, while str or Path is expected') if not os.path.isdir(dst_location): os.makedirs(dst_location, exist_ok=True) root = Loc.tg_path # type:Path release = Loc.temp_path.joinpath('release/package_tmp') # type:Path try: shutil.rmtree(release.__str__()) except: pass os.makedirs(release.__str__()) full_module_name = _full_module_name(task.name, task.version) lib = release.joinpath(full_module_name) shutil.copytree(root.__str__(), lib.joinpath(Loc.tg_name).__str__()) resources = lib.joinpath('resources') # type: Path os.makedirs(resources.__str__()) props = dict( module_name=task.name, version=task.version, full_module_name=full_module_name, dependencies=','.join(f"'{z}'" for dep_list in task.dependencies for z in dep_list.dependencies), tg_name=Loc.tg_name, full_tg_name=full_module_name + '.' + Loc.tg_name, ) for key, value in task.payload.items(): FileIO.write_pickle(value, resources.joinpath(key)) FileIO.write_text(_MANIFEST_TEMPLATE.format(**props), release.joinpath('MANIFEST.in')) FileIO.write_text(_SETUP_TEMPLATE.format(**props), release.joinpath('setup.py')) FileIO.write_json(props, release.joinpath('properties.json')) FileIO.write_text(_INIT_TEMPLATE.format(**props), lib.joinpath('__init__.py')) pwd = os.getcwd() os.chdir(release.__str__()) subprocess.call([sys.executable, 'setup.py', 'sdist']) os.chdir(pwd) file = Query.folder(release.joinpath('dist')).single() dst_location = dst_location.joinpath( f'{full_module_name}-{task.version}.tar.gz') shutil.copy(file.__str__(), dst_location.__str__()) shutil.rmtree(release.__str__()) return PackageInfo(task, full_module_name, dst_location)