def test_filter():
x = range(10)
y = utils.lfilter(b=[1], a=[1], x=x)
assert (np.array(x) == y).all()
x = [1] + [0] * 10
y = utils.lfilter(b=[0.5], a=[1, -0.5], x=x)
assert list(y) == [0.5 ** (i + 1) for i in range(len(x))]
def test_filter():
x = range(10)
y = utils.lfilter(b=[1], a=[1], x=x)
assert (np.array(x) == y).all()
x = [1] + [0] * 10
y = utils.lfilter(b=[0.5], a=[1, -0.5], x=x)
assert list(y) == [0.5**(i + 1) for i in range(len(x))]
def related_article_to_related_articles(related_article_list):
# ll: [{'xlink_href': u'10.7554/eLife.09561', 'related_article_type': u'article-reference', 'ext_link_type': u'doi'}]
def et(struct):
return (struct.get('xlink_href') or '').rsplit('.', 1)[-1] or None
# ll: ['09561'] or None
return lfilter(None, map(et, related_article_list))
def listfiles(path, ext_list=None):
"returns a pair of (basename_list, absolute_path_list) for given dir, optionally filtered by extension"
path_list = lmap(lambda fname: os.path.abspath(join(path, fname)), os.listdir(path))
if ext_list:
path_list = lfilter(lambda path: os.path.splitext(path)[1] in ext_list, path_list)
path_list = sorted(filter(os.path.isfile, path_list))
return lmap(os.path.basename, path_list)
def main(xml_dir, json_output_dir):
paths = lmap(lambda fname: join(xml_dir, fname), os.listdir(xml_dir))
paths = lfilter(lambda path: path.lower().endswith('.xml'), paths)
paths = sorted(paths, reverse=True)
num_processes = 2
Parallel(n_jobs=num_processes)(delayed(render)(path, json_output_dir)
for path in paths)
print('see scrape.log for errors')
def main(xml_dir, json_output_dir, num=None):
paths = lmap(lambda fname: join(xml_dir, fname), os.listdir(xml_dir))
paths = lfilter(lambda path: path.lower().endswith('.xml'), paths)
paths = sorted(paths, reverse=True)
if num:
paths = paths[:num] # only scrape first n articles
num_processes = -1
Parallel(n_jobs=num_processes)(delayed(render)(path, json_output_dir) for path in paths)
print('see scrape.log for errors')
def listfiles(path, ext_list=None):
"returns a pair of (basename_list, absolute_path_list) for given dir, optionally filtered by extension"
path_list = lmap(lambda fname: os.path.abspath(join(path, fname)),
os.listdir(path))
if ext_list:
path_list = lfilter(lambda path: os.path.splitext(path)[1] in ext_list,
path_list)
path_list = sorted(filter(os.path.isfile, path_list))
return lmap(os.path.basename, path_list)
def test_signal():
length = 120
x = np.sign(RandomState(0).normal(size=length))
x[-20:] = 0 # make sure the signal has bounded support
den = np.array([1, -0.6, 0.1])
num = np.array([0.5])
y = utils.lfilter(x=x, b=num, a=den)
lookahead = 2
h = equalizer.train(
signal=y, expected=x, order=len(den), lookahead=lookahead)
assert dsp.norm(h[:lookahead]) < 1e-12
h = h[lookahead:]
assert_approx(h, den / num)
x_ = utils.lfilter(x=y, b=h, a=[1])
assert_approx(x_, x)
def test_signal():
length = 120
x = np.sign(RandomState(0).normal(size=length))
x[-20:] = 0 # make sure the signal has bounded support
den = np.array([1, -0.6, 0.1])
num = np.array([0.5])
y = utils.lfilter(x=x, b=num, a=den)
lookahead = 2
h = equalizer.train(signal=y,
expected=x,
order=len(den),
lookahead=lookahead)
assert dsp.norm(h[:lookahead]) < 1e-12
h = h[lookahead:]
assert_approx(h, den / num)
x_ = utils.lfilter(x=y, b=h, a=[1])
assert_approx(x_, x)
def main(args=None):
target = first(args) or conf.JSON_DIR
if os.path.isdir(target):
paths = lmap(lambda fname: join(target, fname), os.listdir(target))
paths = sorted(paths, reverse=True)
else:
paths = [os.path.abspath(target)]
paths = lfilter(lambda path: path.lower().endswith('.json'), paths)
print('jobs %d' % len(paths))
Parallel(n_jobs=-1)(delayed(job)(path) for path in paths)
print('see validate.log for errors')
def http_download(location):
cred = None
if location.startswith('https://s3-external-1.amazonaws.com/') or location.startswith('https://s3.amazonaws.com/'):
s3_base = urlparse(location).hostname
# if we can find credentials, attach them
session = botocore.session.get_session()
cred = [getattr(session.get_credentials(), attr) for attr in ['access_key', 'secret_key']]
if lfilter(None, cred): # remove any empty values
cred = S3Auth(*cred, service_url=s3_base)
resp = requests.get(location, auth=cred)
if resp.status_code != 200:
raise RuntimeError("failed to download xml from %r, got response code: %s\n%s" % (location, resp.status_code, resp.content))
resp.encoding = 'utf-8'
return resp.text
def validate_gc_data(gc_data):
# we've had one case like this
ensure(gc_data != {}, "glencoe returned successfully, but response is empty")
# we also can't guarantee all of the sources will always be present
known_sources = SOURCES.keys()
for v_id, v_data in gc_data.items():
available_sources = lfilter(lambda mtype: mtype + "_href" in v_data, known_sources)
# fail if we have partial data
msg = "number of available sources less than known sources for %r. missing: %s" % \
(v_id, ", ".join(set(known_sources) - set(available_sources)))
ensure(len(available_sources) == len(known_sources), msg)
def test_commutation():
x = np.random.RandomState(seed=0).normal(size=1000)
b = [1, 1j, -1, -1j]
a = [1, 0.1]
y = utils.lfilter(x=x, b=b, a=a)
y1 = utils.lfilter(x=utils.lfilter(x=x, b=b, a=[1]), b=[1], a=a)
y2 = utils.lfilter(x=utils.lfilter(x=x, b=[1], a=a), b=b, a=[1])
assert_approx(y, y1)
assert_approx(y, y2)
z = utils.lfilter(x=y, b=a, a=[1])
z_ = utils.lfilter(x=x, b=b, a=[1])
assert_approx(z, z_)
def test_commutation():
x = np.random.RandomState(seed=0).normal(size=1000)
b = [1, 1j, -1, -1j]
a = [1, 0.1]
y = utils.lfilter(x=x, b=b, a=a)
y1 = utils.lfilter(x=utils.lfilter(x=x, b=b, a=[1]), b=[1], a=a)
y2 = utils.lfilter(x=utils.lfilter(x=x, b=[1], a=a), b=b, a=[1])
assert_approx(y, y1)
assert_approx(y, y2)
z = utils.lfilter(x=y, b=a, a=[1])
z_ = utils.lfilter(x=x, b=b, a=[1])
assert_approx(z, z_)
def validate_gc_data(gc_data):
# we've had one case like this
ensure(gc_data != {},
"glencoe returned successfully, but response is empty")
# we also can't guarantee all of the sources will always be present
known_sources = SOURCES.keys()
for v_id, v_data in gc_data.items():
available_sources = lfilter(lambda mtype: mtype + "_href" in v_data,
known_sources)
# fail if we have partial data
msg = "number of available sources less than known sources for %r. missing: %s" % \
(v_id, ", ".join(set(known_sources) - set(available_sources)))
assert len(available_sources) == len(known_sources), msg
def resolve_operation_id_using_rest_semantics(self, operation):
# ll: api.xml.search
orig_path = super(BotLaxResolver, self).resolve_operation_id_using_rest_semantics(operation)
bits = orig_path.split('.') # ll ['api', 'xml', 'search']
module = bits[0] # ll: 'api'
# the funcname resolution is wonky, so discard it
method = bits[-1] # ll: 'search'
path = operation.path # ll: /article-json/validation/{filename}
# we want something like:
# /article-json/validation/{filename} => article_json_validation
# /article-json/{filename}/validation => article_json_validation
bits = path.strip('/').split('/') # => ['article-json', 'validation', '{filename}']
bits = lfilter(lambda bit: bit and not bit.startswith('{') and not bit.endswith('}'), bits)
fnname = '_'.join(bits).replace('-', '_') # ll: article_json_validation
return '%s.%s_%s' % (module, method, fnname) # ll: api.post_article_json_validation
def http_download(location):
cred = None
if location.startswith(
'https://s3-external-1.amazonaws.com/') or location.startswith(
'https://s3.amazonaws.com/'):
s3_base = urlparse(location).hostname
# if we can find credentials, attach them
session = botocore.session.get_session()
cred = [
getattr(session.get_credentials(), attr)
for attr in ['access_key', 'secret_key']
]
if lfilter(None, cred): # remove any empty values
cred = S3Auth(*cred, service_url=s3_base)
resp = requests.get(location, auth=cred)
if resp.status_code != 200:
raise RuntimeError(
"failed to download xml from %r, got response code: %s\n%s" %
(location, resp.status_code, resp.content))
resp.encoding = 'utf-8'
return resp.text
def resolve_operation_id_using_rest_semantics(self, operation):
# ll: api.xml.search
orig_path = super(
BotLaxResolver,
self).resolve_operation_id_using_rest_semantics(operation)
bits = orig_path.split('.') # ll ['api', 'xml', 'search']
module = bits[0] # ll: 'api'
# the funcname resolution is wonky, so discard it
method = bits[-1] # ll: 'search'
path = operation.path # ll: /article-json/validation/{filename}
# we want something like:
# /article-json/validation/{filename} => article_json_validation
# /article-json/{filename}/validation => article_json_validation
bits = path.strip('/').split(
'/') # => ['article-json', 'validation', '{filename}']
bits = lfilter(
lambda bit: bit and not bit.startswith('{') and not bit.endswith(
'}'), bits)
fnname = '_'.join(bits).replace('-',
'_') # ll: article_json_validation
return '%s.%s_%s' % (module, method, fnname
) # ll: api.post_article_json_validation
def do_paths(paths, dry_run=False):
ingest_requests = lfilter(None, map(mkreq, paths))
if dry_run:
return ingest_requests
return send_ingest_requests_to_lax(ingest_requests)
def related_article_to_related_articles(related_article_list):
# ll: [{'xlink_href': u'10.7554/eLife.09561', 'related_article_type': u'article-reference', 'ext_link_type': u'doi'}]
def et(struct):
return (struct.get('xlink_href') or '').rsplit('.', 1)[-1] or None
# ll: ['09561'] or None
return lfilter(None, map(et, related_article_list))
def do_paths(paths, dry_run=False):
ingest_requests = lfilter(None, map(mkreq, paths))
if dry_run:
return ingest_requests
return send_ingest_requests_to_lax(ingest_requests)
def test_highpass():
run(1024, chan=lambda x: utils.lfilter(b=[0.9], a=[1.0, 0.1], x=x))
def test_highpass():
run(1024, chan=lambda x: utils.lfilter(b=[0.9], a=[1.0, 0.1], x=x))
