def tc_start(self):
port = self.get_exposed_port(KafkaContainer.KAFKA_PORT)
data = f"""#!/bin/bash
echo 'clientPort=2181' > zookeeper.properties
echo 'dataDir=/var/lib/zookeeper/data' >> zookeeper.properties
echo 'dataLogDir=/var/lib/zookeeper/log' >> zookeeper.properties
zookeeper-server-start zookeeper.properties &
export KAFKA_ZOOKEEPER_CONNECT='localhost:2181'
export KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:{port},BROKER://$(hostname -i):9092
. /etc/confluent/docker/bash-config
/etc/confluent/docker/configure
/etc/confluent/docker/launch""".encode('utf-8')
tar_stream = BytesIO()
tar = tarfile.TarFile(fileobj=tar_stream, mode='w')
tarinfo = tarfile.TarInfo(name=KafkaContainer.TC_START_SCRIPT)
tarinfo.size = len(data)
tarinfo.mtime = time.time()
tar.addfile(tarinfo, BytesIO(data))
tar.close()
tar_stream.seek(0)
self.get_wrapped_container().put_archive('/', tar_stream)
def put_ssh_tar(containers_id):
pw_tarstream = BytesIO()
tarinfo = tarfile.TarInfo(name='gq.tar')
tarinfo.size = os.path.getsize('/root/data/gq.tar')
pw_tar = tarfile.TarFile(fileobj=pw_tarstream, mode='w')
pw_tar.addfile(tarinfo, open('/root/data/gq.tar'))
pw_tar.close()
pw_tarstream.seek(0)
# 根据container id上传tar包
# import pdb ;pdb.set_trace()
for c_id in containers_id:
container = docker_hd.containers.get(c_id)
container.put_archive(path='/root/gq/', data=pw_tarstream)
# container.exec_run('mkdir /root/.ssh')
pw_tarstream.seek(0)
res = container.exec_run('tar -xvf /root/gq/gq.tar -C /root/gq')
# 做第一次登陆
# c_ip = container.get_ip
container.exec_run('ssh -o stricthostkeychecking=no C_ip')
print res
def _generate_examples(self, split_name, file_paths):
base_dir = file_paths["data"]
filename = self.PYTHON_FILE_MAPPING[split_name]
mark_file = os.path.join(base_dir, ".mark")
if not os.path.exists(mark_file):
import gzip
import tarfile
gzip_filename = os.path.join(base_dir, "data.tar.gz")
with gzip.open(gzip_filename, "rb") as gzip_file:
t = tarfile.TarFile(fileobj=gzip_file)
t.extractall(path=base_dir)
with open(mark_file, "w") as f:
f.write("finished")
idx = 0
for entry in self.py_tokenize(base_dir=base_dir, file_name=filename):
path, out_tokens = entry
path = path[len("data/"):]
yield idx, dict(id=idx, path=path, code=out_tokens)
idx += 1
def string_to_container(container, s, dest_path):
pw_tarstream = BytesIO()
pw_tar = tarfile.TarFile(fileobj=pw_tarstream, mode='w')
file_data = s.encode('utf-8')
path = str(uuid.uuid1()).replace('-', '').lower() + '.py'
tarinfo = tarfile.TarInfo(name=path)
tarinfo.size = len(s)
tarinfo.mtime = time.time()
pw_tar.addfile(tarinfo, BytesIO(file_data))
pw_tar.close()
pw_tarstream.seek(0)
success = container.put_archive(dest_path, pw_tarstream)
if not success:
print('uhoh')
else:
print('seemed to be ok')
return dest_path + path
def save_tmpfs_dump(site):
# type: (SiteContext) -> None
"""Dump tmpfs content for later restore after remount
Creates a tar archive from the current tmpfs contents that is restored to the
tmpfs later after mounting it again.
Please note that this only preserves specific files, not the whole tmpfs.
"""
save_paths = [
Path(site.tmp_dir) / "check_mk" / "piggyback",
Path(site.tmp_dir) / "check_mk" / "piggyback_sources",
]
dump_path = _tmpfs_dump_path(site)
dump_path.parent.mkdir(parents=True, exist_ok=True)
with tarfile.TarFile(dump_path, mode="w") as f:
for save_path in save_paths:
if save_path.exists():
f.add(str(save_path),
arcname=str(save_path.relative_to(site.tmp_dir)))
assert dump_path.exists()
def _setup_advertised_host(cls, project, service):
"""
There are services like kafka that announce an advertised address
to clients, who should reconnect to this address. This method
sends the proper address to use to the container by adding a
environment file with the SERVICE_HOST variable set to this value.
"""
host = cls.compose_host(service=service)
content = "SERVICE_HOST=%s" % host
info = tarfile.TarInfo(name="/run/compose_env")
info.mode = 0100644
info.size = len(content)
data = StringIO.StringIO()
tar = tarfile.TarFile(fileobj=data, mode='w')
tar.addfile(info, StringIO.StringIO(content))
tar.close()
containers = project.containers(service_names=[service])
for container in containers:
container.client.put_archive(container=container.id, path="/", data=data.getvalue())
def variables_from_tar(filename: str, variables: _PyTree, i: int):
"""
Loads the variables of a variational state from the i-th element of a `.tar`
archive.
Args:
filename: the tar archive name. Assumes a .tar
extension and adds it if missing and no file exists.
variables: An object variables with the same structure and shape
of the object to be deserialized.
i: the index of the variables to load
"""
if not _path.isfile(filename):
if filename[-4:] != ".tar":
filename = filename + ".tar"
with _tarfile.TarFile(filename, "r") as file:
inner_files = file.getnames()
info = file.getmember(str(i) + ".mpack")
with file.extractfile(info) as f:
return _serialization.from_bytes(variables, f.read())
def zip2tar(zip_bytes):
with TemporaryDirectory() as td:
tarname = os.path.join(td, "data.tar")
timeshift = int(
(datetime.datetime.now() - datetime.datetime.utcnow()).total_seconds()
)
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zipf, tarfile.TarFile(
tarname, "w"
) as tarf:
for zipinfo in zipf.infolist():
if zipinfo.filename[-1] == "/": # is_dir() is py3.6+
continue
tarinfo = tarfile.TarInfo(name=zipinfo.filename)
tarinfo.size = zipinfo.file_size
tarinfo.mtime = calendar.timegm(zipinfo.date_time) - timeshift
infile = zipf.open(zipinfo.filename)
tarf.addfile(tarinfo, infile)
with open(tarname, "rb") as f:
tar_data = f.read()
return tar_data
def get_client_info(cls, flow_name, metadata):
if cls._filecache is None:
from metaflow.client.filecache import FileCache
cls._filecache = FileCache()
info = metadata.get('code-package')
env_id = metadata.get('conda_env_id')
if info is None or env_id is None:
return {'type': 'conda'}
info = json.loads(info)
with cls._filecache.get_data(info['ds_type'], flow_name,
info['sha']) as f:
tar = tarfile.TarFile(fileobj=f)
conda_file = tar.extractfile(CONDA_MAGIC_FILE)
if conda_file is None:
return {'type': 'conda'}
info = json.loads(conda_file.read().decode('utf-8'))
new_info = {
'type': 'conda',
'explicit': info[env_id]['explicit'],
'deps': info[env_id]['deps']
}
return new_info
def package(match, deps, name, version):
'''(?P<name>[^-]+)-(?P<version>.+).tar'''
print('Installing %s in version %s...' % (name, version))
pkg_dir = os.path.join(tools_dir, name)
if os.path.exists(pkg_dir):
shutil.rmtree(pkg_dir)
tarfile.TarFile(match).extractall()
extracted_dir = name if os.path.isdir(name) else '%s-%s' % (name, version)
os.chdir(extracted_dir)
os.environ['PREFIX'] = tools_dir
os.environ['PROPATH'] = "."
if os.path.isfile('{0}/progress.cfg.edit'.format(dlc)):
os.environ['PROCFG'] = '{0}/progress.cfg.edit'.format(dlc)
subprocess.call([sys.executable, './build.py'])
if not os.path.exists(pkg_dir):
print('Installing %s failed' % name)
else:
open(pkg_dir + '/.version', 'w').write(version)
os.chdir('..')
shutil.rmtree(extracted_dir)
def tar_compiled(file, dir, expression='^.+$'):
"""
used to tar a compiled application.
the content of models, views, controllers is not stored in the tar file.
"""
tar = tarfile.TarFile(file, 'w')
for file in listdir(dir, expression, add_dirs=True):
filename = os.path.join(dir, file)
if os.path.islink(filename):
continue
if os.path.isfile(filename) and file[-4:] != '.pyc':
if file[:6] == 'models':
continue
if file[:5] == 'views':
continue
if file[:11] == 'controllers':
continue
if file[:7] == 'modules':
continue
tar.add(filename, file, False)
tar.close()
def test__extractNameVersion_archive_w_pkg_info_version_first(self):
import tarfile
import tempfile
from compoze._compat import BytesIO
tested = self._makeOne()
tfile = tempfile.NamedTemporaryFile(suffix='.tgz')
archive = tarfile.TarFile(fileobj=tfile, mode='w')
buffer = BytesIO()
buffer.writelines([
b'Metadata-Version: 1.0\n',
b'Version: 3.14\n',
b'Name: testpackage\n',
])
size = buffer.tell()
buffer.seek(0)
info = tarfile.TarInfo('testpackage.egg-info/PKG-INFO')
info.size = size
archive.addfile(info, buffer)
archive.close()
tfile.flush()
self.assertEqual(tested._extractNameVersion(tfile.name),
('testpackage', '3.14'))
def file_tar(_orig, dest, **kwargs):
"""Create tar file"""
tarobj = tarfile.TarFile(dest, "w")
tree = kwargs.get("tree")
parents = tree.items[kwargs.get("step")]["parents"]
arcnames = [
os.path.join(tree.get_dirname(parent),
os.path.basename(tree.items[parent]["fullname"]))
for parent in parents
]
fullnames = [tree.items[parent]["fullname"] for parent in parents]
prefix = os.path.commonprefix(arcnames)
if prefix in arcnames:
prefix = os.path.dirname(prefix)
for arcname, fullname in zip(arcnames, fullnames):
arcname = get_available(os.path.relpath(arcname, prefix),
tarobj.getnames())
tarobj.add(fullname, arcname=arcname, recursive=False)
tarobj.close()
return dest
def extractCover(tmp_file_name, original_file_extension):
if use_comic_meta:
archive = ComicArchive(tmp_file_name)
cover_data = None
for index, name in enumerate(archive.getPageNameList()):
ext = os.path.splitext(name)
if len(ext) > 1:
extension = ext[1].lower()
if extension == '.jpg' or extension == '.jpeg':
cover_data = archive.getPage(index)
break
else:
if original_file_extension.upper() == '.CBZ':
cf = zipfile.ZipFile(tmp_file_name)
for name in cf.namelist():
ext = os.path.splitext(name)
if len(ext) > 1:
extension = ext[1].lower()
if extension == '.jpg' or extension == '.jpeg':
cover_data = cf.read(name)
break
elif original_file_extension.upper() == '.CBT':
cf = tarfile.TarFile(tmp_file_name)
for name in cf.getnames():
ext = os.path.splitext(name)
if len(ext) > 1:
extension = ext[1].lower()
if extension == '.jpg' or extension == '.jpeg':
cover_data = cf.extractfile(name).read()
break
prefix = os.path.dirname(tmp_file_name)
if cover_data:
tmp_cover_name = prefix + '/cover' + extension
image = open(tmp_cover_name, 'wb')
image.write(cover_data)
image.close()
else:
tmp_cover_name = None
return tmp_cover_name
def test_build_checkout_tarball_stdout(datafiles, cli):
project = str(datafiles)
tarball = os.path.join(cli.directory, "tarball.tar")
result = cli.run(project=project, args=["build", "target.bst"])
result.assert_success()
builddir = os.path.join(cli.directory, "build")
assert os.path.isdir(builddir)
assert not os.listdir(builddir)
checkout_args = ["artifact", "checkout", "--tar", "-", "target.bst"]
result = cli.run(project=project, args=checkout_args, binary_capture=True)
result.assert_success()
with open(tarball, "wb") as f:
f.write(result.output)
with tarfile.TarFile(tarball) as tar:
assert os.path.join(".", "usr", "bin", "hello") in tar.getnames()
assert os.path.join(".", "usr", "include", "pony.h") in tar.getnames()
def test__extractNameVersion_archive_w_setup_at_root(self):
import tarfile
import tempfile
from compoze._compat import BytesIO
tested = self._makeOne()
tfile = tempfile.NamedTemporaryFile(suffix='.tgz')
archive = tarfile.TarFile(fileobj=tfile, mode='w')
dinfo = tarfile.TarInfo('testpackage')
dinfo.type = tarfile.DIRTYPE
dinfo.mode = 0o777
archive.addfile(dinfo)
buffer = BytesIO()
buffer.write(_DUMMY_SETUP)
size = buffer.tell()
buffer.seek(0)
finfo = tarfile.TarInfo('setup.py')
finfo.size = size
archive.addfile(finfo, buffer)
archive.close()
tfile.flush()
self.assertEqual(tested._extractNameVersion(tfile.name),
('testpackage', '3.14'))
def create_assets(self, plugin):
os.makedirs(plugin.sourcedir)
fake_java_path = os.path.join(
plugin.installdir,
"usr",
"lib",
"jvm",
"java-{}-openjdk-amd64".format(self.expected_java_version),
"bin",
"java",
)
os.makedirs(os.path.dirname(fake_java_path))
open(fake_java_path, "w").close()
maven_tar_path = os.path.join(
plugin.partdir,
"maven",
"apache-maven-{}-bin.tar.gz".format(plugin.options.maven_version),
)
os.makedirs(os.path.dirname(maven_tar_path))
tarfile.TarFile(maven_tar_path, "w").close()
def unpack_article_from_arxiv(filename, outdir, fileobj=None):
if os.path.exists(outdir):
shutil.rmtree(outdir, True)
os.mkdir(outdir)
if fileobj is None:
try:
fileobj = file(filename, 'r')
except IOError:
return False
try:
gzipobj = gzip.GzipFile(fileobj=fileobj)
except IOError:
return False
try:
tarobj = tarfile.TarFile(fileobj=gzipobj, name=filename)
# Mutliple files
for name in tarobj.getnames():
nname = os.path.normpath(name)
if os.path.isabs(nname) or nname.startswith('..'):
return False
tarobj.extractall(outdir)
except (tarfile.TarError, IOError):
# Single file
try:
gzipobj.seek(0)
outfile = file(os.path.join(outdir, 'main.tex'), 'w')
outfile.write(gzipobj.read())
outfile.close()
gzipobj.close()
except IOError:
return False
return True
def download_data(out_path, url, extract=True, force=False):
pathlib.Path(out_path).mkdir(exist_ok=True)
out_filename = os.path.join(out_path, os.path.basename(url))
if os.path.isfile(out_filename) and not force:
print(f"File {out_filename} exists, skipping download.")
else:
print(f"Downloading {url}...")
with urllib.request.urlopen(url) as response:
with open(out_filename, "wb") as out_file:
shutil.copyfileobj(response, out_file)
print(f"Saved to {out_filename}.")
extracted_dir = None
if extract and out_filename.endswith(".zip"):
print(f"Extracting {out_filename}...")
with zipfile.ZipFile(out_filename, "r") as zipf:
names = zipf.namelist()
zipf.extractall(out_path)
zipinfos = zipf.infolist()
first_dir = next(filter(lambda zi: zi.is_dir(), zipinfos)).filename
extracted_dir = os.path.join(out_path, os.path.dirname(first_dir))
print(f"Extracted {len(names)} to {extracted_dir}")
retval = extracted_dir
if extract and out_filename.endswith((".tar.gz", ".tgz")):
print(f"Extracting {out_filename}...")
with tarfile.TarFile(out_filename, "r") as tarf:
members = tarf.getmembers()
tarf.extractall(out_path)
first_dir = next(filter(lambda ti: ti.isdir(), members)).name
extracted_dir = os.path.join(out_path, os.path.dirname(first_dir))
print(f"Extracted {len(members)} to {extracted_dir}")
retval = extracted_dir
return out_filename, extracted_dir
def model_load():
# Should load and return the model.
# Optional, but if present will be loaded during
# startup in the "default-python" environment.
# Create AML base model
model = ML_model()
# Load keras model
# from pathlib import Path
# base_path = Path(__file__).parent
# import_folder = str((base_path / "model/models").resolve())
import_folder = 'models/models'
files = os.listdir('models/models')
model_file = ''
for ff in files:
if '.tar' in ff:
model_file = ff
break
print(os.path.join(import_folder, model_file))
model_tar = tarfile.TarFile(os.path.join(import_folder, model_file))
mjson = model_tar.getmember('./model.json')
mweights = model_tar.getmember('./model_weights.h5')
js_file = model_tar.extractfile(mjson)
w_file = model_tar.extractfile(mweights)
tmp_name = 'tmp_weights.h5'
tmp_f = open(tmp_name, 'wb')
tmp_f.write(w_file.read())
tmp_f.close()
json_savedModel = js_file.read() # Load the model architecture
model_j = tf.keras.models.model_from_json(json_savedModel)
model_j.load_weights('tmp_weights.h5') # Load the weights
model.model = model_j
os.remove('tmp_weights.h5')
return model
def __init__(self, sigmffile, name=None, fileobj=None):
self.sigmffile = sigmffile
self.name = name
self.fileobj = fileobj
self._check_input()
archive_name = self._get_archive_name()
sigmf_fileobj = self._get_output_fileobj()
sigmf_archive = tarfile.TarFile(mode="w",
fileobj=sigmf_fileobj,
format=tarfile.PAX_FORMAT)
tmpdir = tempfile.mkdtemp()
sigmf_md_filename = archive_name + SIGMF_METADATA_EXT
sigmf_md_path = os.path.join(tmpdir, sigmf_md_filename)
sigmf_data_filename = archive_name + SIGMF_DATASET_EXT
sigmf_data_path = os.path.join(tmpdir, sigmf_data_filename)
with open(sigmf_md_path, "w") as mdfile:
self.sigmffile.dump(mdfile, pretty=True)
shutil.copy(self.sigmffile.data_file, sigmf_data_path)
def chmod(tarinfo):
if tarinfo.isdir():
tarinfo.mode = 0o755 # dwrxw-rw-r
else:
tarinfo.mode = 0o644 # -wr-r--r--
return tarinfo
sigmf_archive.add(tmpdir, arcname=archive_name, filter=chmod)
sigmf_archive.close()
if not fileobj:
sigmf_fileobj.close()
shutil.rmtree(tmpdir)
self.path = sigmf_archive.name
def build_buildcontainer_image(self):
"""
Build in the container engine the builder container
:return: generator of strings
"""
assert_initialized(self.base_path)
client = self.get_client()
with make_temp_dir() as temp_dir:
logger.info('Building Docker Engine context...')
tarball_path = os.path.join(temp_dir, 'context.tar')
tarball_file = open(tarball_path, 'wb')
tarball = tarfile.TarFile(fileobj=tarball_file,
mode='w')
container_dir = os.path.normpath(os.path.join(self.base_path,
'ansible'))
try:
tarball.add(container_dir, arcname='ansible')
except OSError:
raise AnsibleContainerNotInitializedException()
jinja_render_to_temp('ansible-dockerfile.j2', temp_dir,
'Dockerfile')
tarball.add(os.path.join(temp_dir, 'Dockerfile'),
arcname='Dockerfile')
jinja_render_to_temp('hosts.j2', temp_dir, 'hosts',
hosts=extract_hosts_from_docker_compose(
self.base_path))
tarball.add(os.path.join(temp_dir, 'hosts'), arcname='hosts')
tarball.close()
tarball_file = open(tarball_path, 'rb')
logger.info('Starting Docker build of Ansible Container image...')
return [streamline for streamline in
client.build(fileobj=tarball_file,
rm=True,
custom_context=True,
pull=True,
forcerm=True,
tag=self.builder_container_img_tag)]
def install_pkg(pkg_spec, install_path):
data = get_pkg_metadata(pkg_spec)
latest_ver = data["info"]["version"]
packages = data["releases"][latest_ver]
assert len(packages) == 1
package_url = packages[0]["url"]
print("Installing %s %s from %s" % (pkg_spec, latest_ver, package_url))
f1 = url_open(package_url)
s = read_lines(f1)
try:
str1 = zlib.decompress(s, gzdict_sz)
with tempfile.TemporaryFile() as temp_file:
temp_file.write(str1)
temp_file.seek(0)
with tarfile.TarFile(fileobj=temp_file) as tar_file: # Expects a file object
meta = install_tar(tar_file, install_path)
finally:
f1.close()
return meta
def __init__(self, name, url):
tempdir = tempfile.mkdtemp()
buf = StringIO.StringIO()
curl = pycurl.Curl()
curl.setopt(curl.URL, url)
curl.setopt(curl.WRITEDATA, buf)
curl.perform()
curl.close()
try:
with zipfile.ZipFile(buf, 'r') as myzip:
myzip.extractall(tempdir)
except:
buf.seek(0)
with tarfile.TarFile(fileobj=buf, mode='r') as mytar:
mytar.extractall(tempdir)
fname = [
f for f in os.listdir(tempdir) if os.path.splitext(f)[1] == '.csv'
][0]
self.name = name
with open(os.path.join(tempdir, fname), 'r') as tmpfile:
self.header, self.rows = self.process_result(tmpfile.read())
def create_assets(self, plugin):
os.makedirs(plugin.sourcedir)
fake_java_path = os.path.join(
plugin.installdir,
"usr",
"lib",
"jvm",
"java-11-openjdk-amd64",
"jre",
"bin",
"java",
)
os.makedirs(os.path.dirname(fake_java_path))
open(fake_java_path, "w").close()
ant_tar_path = os.path.join(
plugin.partdir,
"ant",
"apache-ant-{}-bin.tar.bz2".format(plugin.options.ant_version),
)
os.makedirs(os.path.dirname(ant_tar_path))
tarfile.TarFile(ant_tar_path, "w").close()
def extract(cls, filename, target_dir):
os.makedirs(target_dir)
if tarfile.is_tarfile(filename):
logger.debug("Extracting [{}] into [{}]".format(
as_info(filename), as_info(target_dir)))
try:
with tarfile.TarFile(filename) as tf:
tf.extractall(target_dir)
except tarfile.ReadError:
command = "tar -xf {filename}".format(filename=filename)
if subprocess.call(shlex.split(command), cwd=target_dir) != 0:
raise LocationException(
"Could not untar downloaded file from [{}]".format(
filename))
if zipfile.is_zipfile(filename):
logger.debug("Extracting [{}] into [{}]".format(
as_info(filename), as_info(target_dir)))
with zipfile.ZipFile(filename) as zf:
zf.extractall(target_dir)
while cls.remove_common_top_directory_under(target_dir):
pass
def test_tarball(self):
with self._temp_filesystem() as fs_path:
fs_prefix = fs_path.lstrip('/')
def strip_fs_prefix(tarinfo):
if tarinfo.path.startswith(fs_prefix + '/'):
tarinfo.path = tarinfo.path[len(fs_prefix) + 1:]
elif fs_prefix == tarinfo.path:
tarinfo.path = '.'
else:
raise AssertionError(
f'{tarinfo.path} must start with {fs_prefix}')
return tarinfo
with tempfile.NamedTemporaryFile() as t:
with tarfile.TarFile(t.name, 'w') as tar_obj:
tar_obj.add(fs_path, filter=strip_fs_prefix)
self._check_item(
TarballItem(from_target='t', into_dir='y', tarball=t.name),
self._temp_filesystem_provides('y'),
{require_directory('y')},
)
def tar_compiled(file, dir, expression='^.+$', exclude_content_from=None):
"""Used to tar a compiled application.
The content of models, views, controllers is not stored in the tar file.
"""
with tarfile.TarFile(file, 'w') as tar:
for file in listdir(dir,
expression,
add_dirs=True,
exclude_content_from=exclude_content_from):
filename = os.path.join(dir, file)
if os.path.islink(filename):
continue
if os.path.isfile(filename) and not file.endswith('.pyc'):
if file.startswith('models'):
continue
if file.startswith('views'):
continue
if file.startswith('controllers'):
continue
if file.startswith('modules'):
continue
tar.add(filename, file, False)
def test_tar_gz_to_different_filename():
with tm.ensure_clean(filename=".foo") as file:
pd.DataFrame(
[["1", "2"]],
columns=["foo", "bar"],
).to_csv(file,
compression={
"method": "tar",
"mode": "w:gz"
},
index=False)
with gzip.open(file) as uncompressed:
with tarfile.TarFile(fileobj=uncompressed) as archive:
members = archive.getmembers()
assert len(members) == 1
content = archive.extractfile(members[0]).read().decode("utf8")
if is_platform_windows():
expected = "foo,bar\r\n1,2\r\n"
else:
expected = "foo,bar\n1,2\n"
assert content == expected
def load_markets(file_paths: List[str]):
for file_path in file_paths:
if os.path.isdir(file_path):
for path in glob.iglob(file_path + '**/**/*.bz2', recursive=True):
with bz2.BZ2File(path, 'rb') as f:
bytes = f.read()
yield bflw.File(path, bytes)
elif os.path.isfile(file_path):
ext = os.path.splitext(file_path)[1]
# iterate through a tar archive
if ext == '.tar':
with tarfile.TarFile(file_path) as archive:
for file in archive:
name = file.name
bytes = bz2.open(archive.extractfile(file)).read()
yield bflw.File(name, bytes)
# or a zip archive
elif ext == '.zip':
with zipfile.ZipFile(file_path) as archive:
for name in archive.namelist():
bytes = bz2.open(archive.open(name)).read()
yield bflw.File(name, bytes)
return None
