def main(self, *srcfiles):
[ self._params.append(param) for param in
[ self._hosts, self._rmt_cmd, self._files, self._jobs ]
]
cat('/home/matt/Business_Licenses.csv') | parallel(self._params) | mawk(self._reorg_cmd)
print test
print test()
def parse_manifest(path):
try:
manifest_data = ast.literal_eval(cat(path))
except Exception as e:
print("Error on parsing {}: {}".format(path, e))
manifest_data = {"error": "cannot parse"}
return manifest_data
def test_create(builddir, runner):
# prepare needed files
with local.cwd(builddir):
(sh.echo["a.c"] > "a.c")()
(sh.echo["b.c"] > "b.c")()
builder = BuildFile(build_dir=builddir, runner=runner)
builder.main(command_line=['-c', '-D', 'build'])
expected_json = {
'tar czvf foo.tar.gz a.c b.c': {
'b.c': 'input-',
'foo.tar.gz': 'output-',
'a.c': 'input-'
},
'.deps_version': 2
}
# assertions
with local.cwd(builddir):
assert_same_json('.deps', expected_json)
assert os.path.isfile('foo.tar.gz')
assert sh.tar("tf", 'foo.tar.gz') == "a.c\nb.c\n"
print(sh.ls("-al"))
assert '"a.c": "input-' in sh.cat(".deps")
sys.exit.assert_called_once_with(0)
# Modify a.c to force rebuilding
(sh.echo["newline"] > "a.c")()
builder.main(command_line=['-D', 'build'])
with local.cwd(builddir):
sh.tar("df", "foo.tar.gz") # ensure tar diff return no difference
def test_out_rich(self):
from plumbum.cmd import cat
with open("temp.txt", "w", encoding="utf8") as f:
f.write(self.richstr)
out = cat("temp.txt")
assert self.richstr in out
def test_server_certs_var(self):
"""Prueba que el servidor habilita autenticación de cert a través de env vars."""
with local.tempdir() as tdir:
with local.cwd(tdir):
self._generate_certs()
certs_var = {name: cat(name) for name in self.cert_files}
self.postgres_container = docker(
"container",
"run",
"-d",
"--network",
"lan",
"-e",
"CERTS=" + json.dumps(certs_var),
"-e",
"POSTGRES_DB=test_db",
"-e",
"POSTGRES_PASSWORD=test_password",
"-e",
"POSTGRES_USER=test_user",
CONF_EXTRA,
self.image,
).strip()
self._check_local_connection()
self._check_password_auth()
self._connect_wan_network()
self._check_cert_auth()
def test_server_certs_var(self):
"""Test server enables cert authentication through env vars."""
with local.tempdir() as tdir:
with local.cwd(tdir):
self._generate_certs()
certs_var = {name: cat(name) for name in self.cert_files}
self.postgres_container = docker(
"container",
"run",
"-d",
"--network",
"lan",
"-e",
"CERTS=" + json.dumps(certs_var),
"-e",
"POSTGRES_DB=test_db",
"-e",
"POSTGRES_PASSWORD=test_password",
"-e",
"POSTGRES_USER=test_user",
self.image,
).strip()
self._check_local_connection()
self._check_password_auth()
self._connect_wan_network()
self._check_cert_auth()
def test_out_rich(self):
from plumbum.cmd import cat
import io
with io.open('temp.txt', 'w', encoding='utf8') as f:
f.write(self.richstr)
out = cat('temp.txt')
assert self.richstr in out
def parse_manifests(manifests):
modules_data = {}
for module_name, manifest_path in manifests.items():
try:
manifest_data = ast.literal_eval(cat(manifest_path))
except Exception:
manifest_data = {"error": "cannot parse"}
modules_data[module_name] = {"manifest": manifest_data}
return modules_data
def test_create(builddir, runner, end_fabricate):
# prepare needed files
with local.cwd(builddir):
(sh.echo["a.c"] > "a.c")()
(sh.echo["b.c"] > "b.c")()
# build.py content >>>>>>>>>>>>>>>>>>>>>
def fabricate_file():
def build():
run('tar', 'czvf', 'foo.tar.gz', 'a.c', 'b.c')
def clean():
autoclean()
return copy(locals())
main(globals_dict=fabricate_file(),
build_dir=builddir,
runner=runner,
command_line=['-c', '-D', 'build'])
end_fabricate()
expected_json = {'tar czvf foo.tar.gz a.c b.c':
{'b.c': 'input-',
'foo.tar.gz': 'output-',
'a.c': 'input-'},
u'.deps_version': 2}
# assertions
with local.cwd(builddir):
assert_same_json('.deps', expected_json)
assert os.path.isfile('foo.tar.gz')
assert sh.tar("tf", 'foo.tar.gz') == "a.c\nb.c\n"
print(sh.ls("-al"))
assert '"a.c": "input-' in sh.cat(".deps")
sys.exit.assert_called_once_with(0)
# Modify a.c to force rebuilding
(sh.echo["newline"] > "a.c")()
main(globals_dict=fabricate_file(),
build_dir=builddir,
runner=runner,
command_line=['-D', 'build'])
end_fabricate()
with local.cwd(builddir):
sh.tar("df", "foo.tar.gz") # ensure tar diff return no difference
def test_rename(builddir, runner):
# prepare needed files
with local.cwd(builddir):
sh.touch('originalfile')
builder = BuildFile(build_dir=builddir, runner=runner)
###### First build ##########
builder.main(command_line=['-D', 'build'])
expected_json = {
".deps_version": 2,
"mv originalfile testfile": {
"originalfile": "input-d41d8cd98f00b204e9800998ecf8427e",
"testfile": "output-d41d8cd98f00b204e9800998ecf8427e"
}
}
# assertions
with local.cwd(builddir):
assert_json_equality('.deps', expected_json)
assert os.path.isfile('testfile')
sys.exit.assert_called_once_with(0)
# update original file to check the rebuild
(sh.echo["newline"] > "originalfile")()
###### Second build ##########
builder.main(command_line=['-D', 'build'])
expected_json = {
".deps_version": 2,
"mv originalfile testfile": {
"originalfile": "input-321060ae067e2a25091be3372719e053",
"testfile": "output-321060ae067e2a25091be3372719e053"
}
}
with local.cwd(builddir):
assert_json_equality('.deps', expected_json)
assert "newline" in sh.cat('testfile')
###### Cleaning ##########
builder.main(command_line=['-D', 'clean'])
with local.cwd(builddir):
assert not os.isfile('testfile')
assert os.isfile('originalfile')
#!/usr/bin/env python3
### Usage: gl_tag=<html-tag=a> gl_prop=<html-property=href> gl_s=<selector=> <url> [<html-file>]
### Use prop WHOLE to return whole tag
from IPython import embed
from bs4 import BeautifulSoup, SoupStrainer
import requests, sys
from urllib.parse import urlparse
from plumbum import local
from plumbum.cmd import cat
url = sys.argv[1]
if len(sys.argv) >= 3:
data = cat(sys.argv[2])
else:
page = requests.get(url)
data = page.text
soup = BeautifulSoup(data, features='lxml')
parts = urlparse(url)
page_links = []
sel = soup
gl_s = local.env.get('gl_s', '')
tag = local.env.get('gl_tag', 'a')
prop = local.env.get('gl_prop', 'href')
if gl_s != '':
sel = soup.select_one(gl_s)
# We can use select to get an array of matches
def test_rename(builddir, runner, end_fabricate):
# build.py content >>>>>>>>>>>>>>>>>>>>>
def fabricate_file():
def build():
run('mv', 'originalfile', 'testfile')
def clean():
autoclean()
# remake the original file
shell('touch', 'originalfile')
return copy(locals())
with local.cwd(builddir):
sh.touch('originalfile')
###### First build ##########
main(globals_dict=fabricate_file(),
#parallel_ok=True,
build_dir=builddir,
runner=runner,
command_line=['-D', 'build'])
end_fabricate()
expected_json = {
".deps_version": 2,
"mv originalfile testfile": {
"originalfile": "input-d41d8cd98f00b204e9800998ecf8427e",
"testfile": "output-d41d8cd98f00b204e9800998ecf8427e"
}
}
# assertions
with local.cwd(builddir):
assert_json_equality('.deps', expected_json)
assert os.path.isfile('testfile')
sys.exit.assert_called_once_with(0)
# update original file to check the rebuild
(sh.echo["newline"] > "originalfile")()
###### Second build ##########
main(globals_dict=fabricate_file(),
#parallel_ok=True,
#jobs=4,
build_dir=builddir,
runner=runner,
command_line=['-D', 'build'])
end_fabricate()
expected_json = {
".deps_version": 2,
"mv originalfile testfile": {
"originalfile": "input-321060ae067e2a25091be3372719e053",
"testfile": "output-321060ae067e2a25091be3372719e053"
}
}
with local.cwd(builddir):
assert_json_equality('.deps', expected_json)
assert "newline" in sh.cat('testfile')
###### Cleaning ##########
main(globals_dict=fabricate_file(),
#parallel_ok=True,
#jobs=4,
build_dir=builddir,
runner=runner,
command_line=['-D', 'clean'])
end_fabricate()
with local.cwd(builddir):
assert not os.isfile('testfile')
assert os.isfile('originalfile')
