def get_resources(self) -> List[str]:
"""
Returns list of resources' names
"""
return Query.folder(
self.resources_location).select(lambda z: z.name).to_list()
def test_files(self):
folder = Path(__file__).parent / "temp"
os.makedirs(folder, exist_ok=True)
if os.path.isdir(folder): # pragma: no cover
shutil.rmtree(folder)
plan = self.get_default_plan()
self.kraken_run(kraken_simple_method,
plan,
cache_to_folder=folder,
special_iterations=[3])
file = Query.folder(folder).single()
self.assertEqual('3.kraken.pkl', file.name)
results = self.kraken_run(
kraken_simple_method, plan,
cache_to_folder=folder) # type: List[kraken.IterationResult]
for index, result in enumerate(results):
self.assertEqual(
kraken.IterationStatus.Skipped if index == 3 else
kraken.IterationStatus.Success, result.status)
self.assertIsNone(result.condition)
self.assertIsNone(result.result)
loaded_results = Query.en(kraken.Kraken.load(
folder, None)).order_by(lambda z: z.result).to_list()
self.assertResult([11, 12, 21, 22], loaded_results)
shutil.rmtree(folder)
def _load_data(self, data: Union[str, Path, pd.DataFrame]):
if isinstance(data, pd.DataFrame):
return data
if isinstance(data, str) or isinstance(data, Path):
data = str(data)
if os.path.isfile(data):
return pd.read_parquet(data)
elif os.path.isdir(data):
dfs = Query.folder(data).select(pd.read_parquet).to_list()
return pd.concat(dfs, sort=False)
else:
raise ValueError(f'Data was `{data}`, but there is neither file nor folder on this location')
else:
raise ValueError(f"Data was `{data}`, but the format is not supported")
def upload_folder(bucket_name: str, s3_path: str, folder: Path):
aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', None)
aws_secret_access = os.environ.get('AWS_SECRET_ACCESS_KEY', None)
if aws_access_key_id is not None and aws_secret_access is not None:
kwargs = dict(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access)
else:
kwargs = {}
s3 = boto3.resource('s3', **kwargs)
bucket = s3.Bucket(bucket_name)
bucket.objects.filter(Prefix=s3_path).delete()
client = boto3.client('s3', **kwargs)
for file_path in Query.folder(folder):
file_path_str = file_path.__str__()
joint_path = os.path.join(s3_path, file_path.name)
client.upload_file(file_path_str, bucket_name, joint_path)
def send(self):
aws_access_key_id = os.environ.get('AWS_ACCESS_KEY_ID', None)
aws_secret_access = os.environ.get('AWS_SECRET_ACCESS_KEY', None)
if aws_access_key_id is not None and aws_secret_access is not None:
kwargs = dict(aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access)
else:
kwargs = {}
s3 = boto3.resource('s3', **kwargs)
bucket = s3.Bucket(self.s3_bucket)
for name in self.names:
s3_full_path = os.path.join(self.s3_path, name)
bucket.objects.filter(Prefix=s3_full_path).delete()
client = boto3.client('s3', **kwargs)
for name in self.names:
for file_path in Query.folder(self.location.joinpath(name)):
client.upload_file(
file_path.__str__(), self.s3_bucket,
os.path.join(self.s3_path, name, file_path.name))
def upload_folder(self, local_path, remote_path):
for file in Query.folder(local_path,'**/*').where(lambda z: z.is_file()):
inner = str(file)[len(str(local_path)):]
if inner.startswith('/'):
inner = inner[1:]
self.upload_file(os.path.join(local_path, inner), os.path.join(remote_path, inner))
def make_package(
task: PackagingTask,
dst_location: Optional[Union[Path, str]] = None) -> PackageInfo:
"""
Creates the package out of the :class:``PackagingTask``, and returns :class:``PackagingInfo``` describing this package
"""
if dst_location is None:
dst_location = Loc.temp_path.joinpath('release/package')
elif isinstance(dst_location, str):
dst_location = Path(dst_location)
elif not isinstance(dst_location, Path):
raise ValueError(
f'dst_location was {dst_location}, while str or Path is expected')
if not os.path.isdir(dst_location):
os.makedirs(dst_location, exist_ok=True)
root = Loc.tg_path # type:Path
release = Loc.temp_path.joinpath('release/package_tmp') # type:Path
try:
shutil.rmtree(release.__str__())
except:
pass
os.makedirs(release.__str__())
full_module_name = _full_module_name(task.name, task.version)
lib = release.joinpath(full_module_name)
shutil.copytree(root.__str__(), lib.joinpath(Loc.tg_name).__str__())
resources = lib.joinpath('resources') # type: Path
os.makedirs(resources.__str__())
props = dict(
module_name=task.name,
version=task.version,
full_module_name=full_module_name,
dependencies=','.join(f"'{z}'" for dep_list in task.dependencies
for z in dep_list.dependencies),
tg_name=Loc.tg_name,
full_tg_name=full_module_name + '.' + Loc.tg_name,
)
for key, value in task.payload.items():
FileIO.write_pickle(value, resources.joinpath(key))
FileIO.write_text(_MANIFEST_TEMPLATE.format(**props),
release.joinpath('MANIFEST.in'))
FileIO.write_text(_SETUP_TEMPLATE.format(**props),
release.joinpath('setup.py'))
FileIO.write_json(props, release.joinpath('properties.json'))
FileIO.write_text(_INIT_TEMPLATE.format(**props),
lib.joinpath('__init__.py'))
pwd = os.getcwd()
os.chdir(release.__str__())
subprocess.call([sys.executable, 'setup.py', 'sdist'])
os.chdir(pwd)
file = Query.folder(release.joinpath('dist')).single()
dst_location = dst_location.joinpath(
f'{full_module_name}-{task.version}.tar.gz')
shutil.copy(file.__str__(), dst_location.__str__())
shutil.rmtree(release.__str__())
return PackageInfo(task, full_module_name, dst_location)