def movie(trakt_slug, people_needed=False):
cache_key = 'movie-{0}'.format(trakt_slug)
cache_init_func = functools.partial(__movie, trakt_slug=trakt_slug)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__movie,
trakt_slug=trakt_slug,
timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.WEEK
result = cache.cache(cache_key, cache_init_func,
cache_init_exception_handler,
cache_init_failure_handler, cache_update_func,
cache_data_expiration)
if people_needed:
cache_key = 'movie-{0}-people'.format(trakt_slug)
cache_init_func = functools.partial(__movie_people,
trakt_slug=trakt_slug)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__movie_people,
trakt_slug=trakt_slug,
timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.WEEK
result['people'] = cache.cache(cache_key, cache_init_func,
cache_init_exception_handler,
cache_init_failure_handler,
cache_update_func,
cache_data_expiration)
return result
def show(trakt_slug, seasons_needed=False):
cache_key = 'show-{0}'.format(trakt_slug)
cache_init_func = functools.partial(__show, trakt_slug=trakt_slug)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__show,
trakt_slug=trakt_slug,
timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.DAY
result = cache.cache(cache_key, cache_init_func,
cache_init_exception_handler,
cache_init_failure_handler, cache_update_func,
cache_data_expiration)
if seasons_needed:
cache_key = 'show-{0}-seasons'.format(trakt_slug)
cache_init_func = functools.partial(__show_seasons,
trakt_slug=trakt_slug,
show_info=result)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__show_seasons,
trakt_slug=trakt_slug,
show_info=result,
timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.DAY
result['seasons'] = cache.cache(cache_key, cache_init_func,
cache_init_exception_handler,
cache_init_failure_handler,
cache_update_func,
cache_data_expiration)
return result
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return
defs = Definitions()
this = defs.get(target)
with app.timer(this, 'Starting assembly'):
with sandbox.setup(this):
for it in this.get('build-depends', []):
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
for it in this.get('contents', []):
component = defs.get(it)
if component.get('build-mode') == 'bootstrap':
continue
assemble(component)
sandbox.install(this, component)
if this.get('build-mode') != 'bootstrap':
sandbox.ldconfig(this)
else:
app.log(this, "No ldconfig because bootstrap mode is engaged")
build(this)
if this.get('devices'):
sandbox.create_devices(this)
cache.cache(this)
def main():
"""
entry point
"""
url = ('https://en.wikipedia.org/wiki/'
'Category:Newspapers_published_in_Kansas')
obj = cache(url)
parser = WikipediaCatParser()
strd = str(obj)
parser.feed(strd)
for idx in parser.index:
obj = parser.index[idx]
for field in ("page", 'siteurl'):
if field in obj:
val = obj[field]
if val:
obj2 = cache(val)
p=WikipediaPageParser()
p.feed(str(obj2))
p.obj['url']=val
#print("FOUND:%s" % str(p.obj))
output = open('Category_Newspapers_published_in_Kansas.yaml', 'w')
output.write(yaml.dump(parser.index, indent=4, default_flow_style=False))
def do_build(defs, component):
app.config['counter'].increment()
with app.timer(component, 'build of %s' % component['cache']):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
def do_build(defs, component):
app.config['counter'].increment()
with app.timer(component, 'build of %s' % component['cache']):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
# needed for artifact splitting
load_manifest(defs, target)
return cache.cache_key(defs, target)
random.seed(datetime.datetime.now())
component = defs.get(target)
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'assembly'):
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble_system_recursively(system)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
splits = None
if component.get('kind') == 'system':
splits = subcomponent.get('artifacts')
sandbox.install(defs, component, subcomponent, splits)
app.config['counter'] += 1
if 'systems' not in component:
with app.timer(component, 'build'):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(defs, component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def info(topic):
name = topic+" info"
fetched = fetch(name)
if fetched:
return fetched
art = wiki.search(topic)
info = {}
info['name'] = art.title
info['text'] = art.sections[0].string
cache(name, info)
return info
def index_object(filename, data,i):
d = data[i]
for k in d.keys() :
if k not in (
'A',
'C',
'contact_page',
'F',
'facebook',
'ksa_site',
'named' ,
'src_url',
'T',
'twitter',
'user_forum',
'V',
'W',
'website',
'Website',
'wikipedia'):
continue
v = d[k]
if not isinstance(v,str):
continue
if v.startswith('No Website') :
continue
v = v.strip().rstrip()
v = v.rstrip("/")
v = v.replace("http://","")
v = v.replace("https://","")
# strip
if not v:
continue
if v == '':
continue
if v == "NONE":
continue
if v[0] == "?":
continue
if v not in index:
index[v]={}
if k not in index[v]:
# now lets make sure we can cache this all
cache.cache("http://%s" %v)
index[v][k]=[]
print ("adding key:'%s' val'%s'" % (k, v))
index[v][k].append(d)
def prereqs(topic):
name = topic+" prereqs"
fetched = fetch(name)
if fetched:
return fetched
art = wiki.search(topic)
reqs = getReqs(art)
for req in reqs:
review(req) # without grabbing return, works as pure caching function
cache(name, reqs)
return reqs
def build(defs, component):
'''Create an artifact for a single component and add it to the cache'''
with claim(defs, component):
app.config['counter'].increment()
with app.timer(component, 'build of %s' % component['cache']):
run_build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
def get_cached_count_dicts(field: str,
key_column: str) -> Dict[str, Dict[str, int]]:
if len(list(
Path(f'{CACHED_DIR}/{field}/{key_column}').glob('*.json'))) == 0:
cache(field, key_column, reset=True)
key_count_dict = {}
for cached_filepath in Path(f'{CACHED_DIR}/{field}/{key_column}').glob(
'*.json'):
key = cached_filepath.stem
with cached_filepath.open('r') as f:
key_count_dict[key] = json.load(f)
return key_count_dict
def quiz(topic):
name = topic+" quiz"
fetched = fetch(name)
if fetched:
return fetched
art = wiki.search(topic)
quiz = {}
quiz['name'] = art.title
quiz['description'] = getDescription(art)
quiz['distractors'] = getDists(art)
quiz['prereqs'] = getReqs(art)
cache(name, quiz)
return quiz
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
component = defs.get(target)
if get_cache(defs, component) or get_remote(defs, component):
return cache_key(defs, component)
random.seed(datetime.datetime.now())
if component.get('arch') and component['arch'] != app.config['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
sandbox.setup(component)
systems = component.get('systems', [])
random.shuffle(systems)
for system in systems:
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble(defs, subsystem)
dependencies = component.get('build-depends', [])
for it in dependencies:
preinstall(defs, component, it)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode', 'staging') != 'bootstrap':
preinstall(defs, component, subcomponent)
if 'systems' not in component:
if is_building(defs, component):
import time
time.sleep(10)
raise Exception
app.config['counter'] += 1
if not get_cache(defs, component):
with app.timer(component, 'build of %s' % component['cache']):
with claim(defs, component):
build(defs, component)
with app.timer(component, 'artifact creation'):
do_manifest(component)
cache(defs, component)
sandbox.remove(component)
return cache_key(defs, component)
def review(topic):
name = topic+" review"
fetched = fetch(name)
if fetched:
return fetched
art = wiki.search(topic)
rev = {}
if art.disambiguation:
raise DisambiguationError()
rev['name'] = art.title
rev['description'] = getDescription(art)
rev['distractors'] = getDists(art)
cache(name, rev)
return rev
def build(defs, component):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(defs, component):
return
with claim(defs, component):
if component.get('kind', 'chunk') == 'chunk':
install_dependencies(defs, component)
with timer(component, 'build of %s' % component['cache']):
run_build(defs, component)
with timer(component, 'artifact creation'):
write_metadata(defs, component)
cache(defs, component)
def build(defs, component):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(defs, component):
return
with claim(defs, component):
if component.get('kind', 'chunk') == 'chunk':
install_dependencies(defs, component)
with timer(component, 'build of %s' % component['cache']):
run_build(defs, component)
with timer(component, 'artifact creation'):
write_metadata(defs, component)
cache(defs, component)
def transfer_values_cache(cache_path, model, images=None, image_paths=None):
"""
This function either loads the transfer-values if they have
already been calculated, otherwise it calculates the values
and saves them to a file that can be re-loaded again later.
Because the transfer-values can be expensive to compute, it can
be useful to cache the values through this function instead
of calling transfer_values() directly on the Inception model.
See Tutorial #08 for an example on how to use this function.
:param cache_path:
File containing the cached transfer-values for the images.
:param model:
Instance of the Inception model.
:param images:
4-dim array with images. [image_number, height, width, colour_channel]
:param image_paths:
Array of file-paths for images (must be jpeg-format).
:return:
The transfer-values from the Inception model for those images.
"""
# Helper-function for processing the images if the cache-file does not exist.
# This is needed because we cannot supply both fn=process_images
# and fn=model.transfer_values to the cache()-function.
def fn():
return process_images(fn=model.transfer_values,
images=images,
image_paths=image_paths)
# Read the transfer-values from a cache-file, or calculate them if the file does not exist.
transfer_values = cache(cache_path=cache_path, fn=fn)
return transfer_values
def main(args=None, axes=None):
if args is not None:
ARGS = args
steps, metrics = cache(ARGS)
# create plot
print(">> Plotting...")
plt.rcParams["font.size"] = 18
if axes is None:
fig, axes = plt.subplots(figsize=(15, 15))
if "performance" in metrics.keys():
performance_plot(axes, steps, metrics["performance"])
elif "network" in ARGS.viz:
network_plot(ARGS, axes, metrics["empirical"])
else:
empirical_plot(ARGS, axes, metrics)
if ARGS.legend:
axes.legend()
# save plot
if ARGS.plot_dir is None:
plot_path = f"{ARGS.save_dir}/{ARGS.experiment}/{ARGS.expid}/img"
else:
plot_path = f"{ARGS.plot_dir}/img"
helper.makedir_quiet(plot_path)
plot_file = f"{plot_path}/{ARGS.viz}{ARGS.suffix}.pdf"
plt.savefig(plot_file)
print(f">> Saving figure to {plot_file}")
def __init__(self, path):
f = open(path)
self.V, self.E, self.R, self.C, self.X = f.readline().split(' ')
self.V, self.E, self.R, self.C, self.X = int(self.V), int(self.E), int(
self.R), int(self.C), int(self.X)
self.videos = []
self.vidreq = [(0, 0)] * self.V # (id vid, nbreq)
self.endPoints = []
self.requests = []
self.caches = []
for i in range(self.C):
self.caches.append(cache.cache(self.X))
vi = f.readline().split(' ')
for i in range(len(vi)):
self.videos.append(video.video(int(vi[i]), i, 0))
for i in range(self.E):
Ld, K = f.readline().split(' ')
Ld, K = int(Ld), int(K)
e = endPoint.endPoint(Ld)
for j in range(K):
c, Lc = f.readline().split(' ')
e.addCacheLat(self.caches[int(c)], int(Lc))
self.endPoints.append(e)
for i in range(self.R):
Rv, Re, Rn = f.readline().split(' ')
self.requests.append(request.request(int(Rv), int(Re), int(Rn)))
for r in self.requests:
self.videos[r.idVid].req += r.nbCall
Length = len(self.videos)
k = 0
while k < Length:
if self.videos[k].req == 0:
self.videos.pop(k)
Length -= 1
else:
k += 1
self.videos = sorted(self.videos, key=lambda v: v.size / v.req)
for e in self.endPoints:
e.savingLat()
for e in self.endPoints:
for c in e.list:
k = 0
while k < Length:
if c[0].capacity >= c[0].size + self.videos[k].size:
c[0].addVideo(self.videos.pop(k))
Length -= 1
else:
k += 1
f.close()
def trove(name) :
headers = {}
key= "wpt2_" + name
#print key
f=lambda : fetch(name)
# cache.delcache(key)
return cache.cache(key,f)
def get_win32_symbols(filepath):
assert os.path.isfile(SIZER_EXE)
return cache(
lambda: run(SIZER_EXE, filepath),
dirname="symanalysis",
hashelems=("get_win32_symbols", file_contents(SIZER_EXE), file_contents(filepath)),
)["val"]
def transfer_values_cache(cache_path, model, images=None, image_paths=None):
"""
This function either loads the transfer-values if they have
already been calculated, otherwise it calculates the values
and saves them to a file that can be re-loaded again later.
Because the transfer-values can be expensive to compute, it can
be useful to cache the values through this function instead
of calling transfer_values() directly on the Inception model.
See Tutorial #08 for an example on how to use this function.
:param cache_path:
File containing the cached transfer-values for the images.
:param model:
Instance of the Inception model.
:param images:
4-dim array with images. [image_number, height, width, colour_channel]
:param image_paths:
Array of file-paths for images (must be jpeg-format).
:return:
The transfer-values from the Inception model for those images.
"""
# Helper-function for processing the images if the cache-file does not exist.
# This is needed because we cannot supply both fn=process_images
# and fn=model.transfer_values to the cache()-function.
def fn():
return process_images(fn=model.transfer_values, images=images, image_paths=image_paths)
# Read the transfer-values from a cache-file, or calculate them if the file does not exist.
transfer_values = cache(cache_path=cache_path, fn=fn)
return transfer_values
def load_records(train=True):
"""
Load the data-records for the data-set. This returns the image ids,
filenames and text-captions for either the training-set or validation-set.
This wraps _load_records() above with a cache, so if the cache-file already
exists then it is loaded instead of processing the original data-file.
:param train:
Bool whether to load the training-set (True) or validation-set (False).
:return:
ids, filenames, captions for the images in the data-set.
"""
if train:
# Cache-file for the training-set data.
cache_filename = "records_train.pkl"
else:
# Cache-file for the validation-set data.
cache_filename = "records_val.pkl"
# Path for the cache-file.
cache_path = os.path.join(data_dir, cache_filename)
# If the data-records already exist in a cache-file then load it,
# otherwise call the _load_records() function and save its
# return-values to the cache-file so it can be loaded the next time.
records = cache(cache_path=cache_path, fn=_load_records, train=train)
return records
########################################################################
def load_cached(cache_path, in_dir):
print("Criando conjuntos de dados a partir dos arquivos em: " + in_dir)
dataset = cache(cache_path=cache_path, fn=DataSet, in_dir=in_dir)
return dataset
def load_cached(cache_path, in_dir):
"""
이미 존재하면 캐쉬 파일을 불러오고, 그렇지 않으면 새로운 객체를 만들고 캐쉬파일에 저장하는
데이터셋 객체를 만들기 위한 Wrapper 함수
만약 파일명의 이름이 매번 데이터셋을 불러올 때 일치하는 것을 보장하기를 원한다면 유용하다
예를 들면, 다른 캐쉬파일에 저장된 변환 값을 갖는 결합 데이터셋 객체를 사용하는 경우,
e.g. 튜토리얼 #09 에서 이 예제를 볼 수 있다.
:param cache_path:
캐쉬파일에 대한 경로
:param in_dir:
데이터셋 안에 파일들에 대한 최상위 경로.
데이터셋 init 메소드에 대한 인자
:return:
데이터셋 객체
"""
print("Creating dataset from the files in: " + in_dir)
# 만약 DataSet(in_dir=data_dir)의 객체 인스턴스가 이미 캐쉬파일안에 존재한다면 불러오고,
# 그렇지 않다면 객체 인스턴스를 만들고 다음번을 위해 캐쉬파일에 저장한다
dataset = cache(cache_path=cache_path,
fn=DataSet, in_dir=in_dir)
return dataset
def __init__(self):
self.RegisterFile = register()
self.Memory = cache()
self.sp = 0x7ffffffc
self.PC = 0
self.IR = 0
self.total_control_ins = 0
def trove(name):
headers = {}
key = "wpt2_" + name
#print key
f = lambda: fetch(name)
# cache.delcache(key)
return cache.cache(key, f)
def load_cached(cache_path, in_dir):
"""
Wrapper-function for creating a DataSet-object, which will be
loaded from a cache-file if it already exists, otherwise a new
object will be created and saved to the cache-file.
This is useful if you need to ensure the ordering of the
filenames is consistent every time you load the data-set,
for example if you use the DataSet-object in combination
with Transfer Values saved to another cache-file, see e.g.
Tutorial #09 for an example of this.
:param cache_path:
File-path for the cache-file.
:param in_dir:
Root-dir for the files in the data-set.
This is an argument for the DataSet-init function.
:return:
The DataSet-object.
"""
print("Creating dataset from the files in: " + in_dir)
# If the object-instance for DataSet(in_dir=data_dir) already
# exists in the cache-file then reload it, otherwise create
# an object instance and save it to the cache-file for next time.
dataset = cache(cache_path=cache_path, fn=DataSet, in_dir=in_dir)
return dataset
########################################################################
def load_cached(cache_path, in_dir):
"""
Wrapper-function for creating a DataSet-object, which will be
loaded from a cache-file if it already exists, otherwise a new
object will be created and saved to the cache-file.
This is useful if you need to ensure the ordering of the
filenames is consistent every time you load the data-set,
for example if you use the DataSet-object in combination
with Transfer Values saved to another cache-file, see e.g.
Tutorial #09 for an example of this.
:param cache_path:
File-path for the cache-file.
:param in_dir:
Root-dir for the files in the data-set.
This is an argument for the DataSet-init function.
:return:
The DataSet-object.
"""
print("Creating dataset from the files in: " + in_dir)
# If the object-instance for DataSet(in_dir=data_dir) already
# exists in the cache-file then reload it, otherwise create
# an object instance and save it to the cache-file for next time.
dataset = cache(cache_path=cache_path,
fn=DataSet, in_dir=in_dir)
return dataset
def load_records(train=True):
"""
Load the data-records for the data-set. This returns the image ids,
filenames and text-captions for either the training-set or validation-set.
This wraps _load_records() above with a cache, so if the cache-file already
exists then it is loaded instead of processing the original data-file.
:param train:
Bool whether to load the training-set (True) or validation-set (False).
:return:
ids, filenames, captions for the images in the data-set.
"""
if train:
# Cache-file for the training-set data.
cache_filename = "records_train.pkl"
else:
# Cache-file for the validation-set data.
cache_filename = "records_val.pkl"
# Path for the cache-file.
cache_path = os.path.join(data_dir, cache_filename)
# If the data-records already exist in a cache-file then load it,
# otherwise call the _load_records() function and save its
# return-values to the cache-file so it can be loaded the next time.
records = cache(cache_path=cache_path,
fn=_load_records,
train=train)
return records
def assemble(defs, target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(defs, target):
return cache.cache_key(defs, target)
component = defs.get(target)
if component.get('arch') and component['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', component.get('arch'))
return None
def assemble_system_recursively(system):
assemble(defs, system['path'])
for subsystem in system.get('subsystems', []):
assemble_system_recursively(subsystem)
with app.timer(component, 'Starting assembly'):
sandbox.setup(component)
for system_spec in component.get('systems', []):
assemble_system_recursively(system_spec)
dependencies = component.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(defs, dependency)
sandbox.install(defs, component, dependency)
contents = component.get('contents', [])
random.shuffle(contents)
for it in contents:
subcomponent = defs.get(it)
if subcomponent.get('build-mode') != 'bootstrap':
assemble(defs, subcomponent)
sandbox.install(defs, component, subcomponent)
app.settings['counter'] += 1
if 'systems' not in component:
build(defs, component)
do_manifest(component)
cache.cache(defs, component,
full_root=component.get('kind') == "system")
sandbox.remove(component)
return cache.cache_key(defs, component)
def __init__(self, coreID, bus, clock, update):
self.ID = coreID
self.myBus = bus
self.update = update
self.myCache = cache.cache(coreID, update)
self.controller = cacheController.controller(self.myCache, self.myBus, self.change_state, update, coreID)
self.myBus.add_ctrl(self.controller)
self.processor = core.processor(coreID, clock, self.controller, update)
def __init__(self):
"""Constructor."""
try:
self.logger = log()
self.session = requestsbak.Session()
self.session.trust_env = False
self.cache = cache()
except Exception as e:
self.logger.error("token: Error in token module constructor: %s" % (e))
def transfer_values_cache(cache_path, model, images=None, image_paths=None):
def fn():
return process_images(fn=model.transfer_values,
images=images,
image_paths=image_paths)
transfer_values = cache(cache_path=cache_path, fn=fn)
return transfer_values
def load_weight(filenames, captions):
cache_filename = 'weight.pkl'
cache_path = os.path.join(dir_path, cache_filename)
weight = cache(cache_path=cache_path,
fn=_tf_idf,
filenames=filenames,
captions=captions)
return weight
def show_season(trakt_slug, season_index):
cache_key = 'show-{0}-{1}'.format(trakt_slug, season_index)
cache_init_func = functools.partial(__show_season, trakt_slug=trakt_slug, season_index=season_index)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__show_season, trakt_slug=trakt_slug, season_index=season_index, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.HOUR * 6
return cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration)
def shows_search(query):
cache_key = 'shows-search-{0}'.format(query)
cache_init_func = functools.partial(__shows_search, query=query)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__shows_search, query=query, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.HOUR
return cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration)
def load_vector(filenames, captions):
cache_filename = "data_vector.pkl"
cache_path = os.path.join(dir_path, cache_filename)
vectors = cache(cache_path=cache_path,
fn=_load_vector,
filenames=filenames,
captions=captions)
return vectors
def process_images_train(): #定义处理训练集图片的函数
print("Processing {0} images in training-set ...".format(len(filenames_train)))
# 缓存文件的路径
cache_path = os.path.join(coco.data_dir, 'inception_coco_train.pkl')
#如果缓存存在的话我们就直接加载它，这样可以更快速的进入模型
transfer_values = cache(cache_path=cache_path,
fn=process_images,
data_dir=coco.train_dir,
filenames=filenames_train)
return transfer_values
def build(dn):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(dn):
return
with claim(dn):
if dn.get('kind', 'chunk') == 'chunk':
install_dependencies(dn)
with timer(dn, 'build of %s' % dn['cache']):
run_build(dn)
with timer(dn, 'artifact creation'):
if dn.get('kind', 'chunk') == 'system':
install_split_artifacts(dn)
write_metadata(dn)
cache(dn)
def build(dn):
'''Create an artifact for a single component and add it to the cache'''
if get_cache(dn):
return
with claim(dn):
if dn.get('kind', 'chunk') == 'chunk':
install_dependencies(dn)
with timer(dn, 'build of %s' % dn['cache']):
run_build(dn)
with timer(dn, 'artifact creation'):
if dn.get('kind', 'chunk') == 'system':
install_split_artifacts(dn)
write_metadata(dn)
cache(dn)
def process_link(l):
f_name_link = l.get("href")
f_name = l.text.strip()
callit = lambda: extract_links(f_name_link)
# cache.delcache("wp"+ f_name)
extract = cache.cache("wp" + f_name, callit)
(name, party, state) = parse_name(f_name)
(link, link2) = parse_link(f_name_link)
####
callit = lambda: runtrove(name)
# troveid = runtrove(name)
# cache.delcache("wpt"+ f_name)
troveid = cache.cache("wpt" + f_name, callit)
wiki = scan_links(extract, l, f_name_link, f_name, name, party, state,
link, link2, troveid)
def load_records(train=True):
if train:
cache_filename = "records_train.pkl"
else:
cache_filename = "records_val.pkl"
cache_path = os.path.join(data_dir, cache_filename)
records = cache(cache_path=cache_path, fn=_load_records, train=train)
return records
def load_cached(cache_path, in_dir):
print("Creating dataset from the files in: " + in_dir)
# If the object-instance for DataSet(in_dir=data_dir) already
# exists in the cache-file then reload it, otherwise create
# an object instance and save it to the cache-file for next time.
dataset = cache(cache_path=cache_path, fn=DataSet, in_dir=in_dir)
return dataset
def __init__(self, remoteShell, domainAdmin="admin", domain=None):
self.remoteShell = remoteShell
self.vastoolPath = "/opt/quest/bin/vastool"
self.domainAdmin = domainAdmin
self.defaultDomain = domain
self.info = info.info(self.run)
self.flush = flush.flush(self.run)
self.create = create.create(self.run, self.defaultDomain)
self.delete = delete.delete(self.run)
self.timesync = timesync.timesync(self.run)
self.nss = nss.nss(self.run)
self.group = group.group(self.run)
self.isvas = isvas.isvas(self.run)
self.list = list.list(self.run)
self.auth = auth.auth(self.run, self.defaultDomain)
self.cache = cache.cache(self.run)
self.configure = configure.configure(self.run)
self.configureVas = configureVas.configureVas(self.run)
self.schema = schema.schema(self.run)
self.merge = merge.merge(self.run)
self.unmerge = unmerge.unmerge(self.run)
self.user = User.user(self.run)
self.ktutil = ktutil.ktutil(self.run)
self.load = load.load(self.run)
self._license = License.License(self.run)
self.License = self._license.License
self.parseLicense = self._license.parseLicense
self.compareLicenses = self._license.compareLicenses
#self.vasUtilities = vasUtilities.vasUtilities(self.remoteShell)
self.unconfigure = unconfigure.unconfigure(self.run)
self.nssdiag = nssdiag(self.run)
isinstance(self.info, info.info)
isinstance(self.flush, flush.flush)
isinstance(self.create, create.create)
isinstance(self.delete, delete.delete)
isinstance(self.timesync, timesync.timesync)
isinstance(self.nss, nss.nss)
isinstance(self.group, group.group)
isinstance(self.isvas, isvas.isvas)
isinstance(self.list, list.list)
isinstance(self.auth, auth.auth)
isinstance(self.cache, cache.cache)
isinstance(self.configure, configure.configure)
isinstance(self.configureVas, configureVas.configureVas)
isinstance(self.schema, schema.schema)
isinstance(self.merge, merge.merge)
isinstance(self.unmerge, unmerge.unmerge)
isinstance(self.user, User.user)
isinstance(self.ktutil, ktutil.ktutil)
isinstance(self.load, load.load)
#isinstance(self.vasUtilities, vasUtilities.vasUtilities)
isinstance(self.unconfigure, unconfigure.unconfigure)
isinstance(self.nssdiag, nssdiag)
def __init__(self, allowTagGuessing=True, allowDisambiguation=True):
"""
Create Analex instance.
"""
self.nounstemmer = stem_noun.nounStemmer()
# to stem nouns
self.verbstemmer = stem_verb.verbStemmer()
# to stem verbs
self.unknownstemmer = stem_unknown.unknownStemmer()
# to stem unknown
self.stopwordsstemmer = stem_stopwords.stopWordStemmer()
# to stem stopwords
self.allowTagGuessing = allowTagGuessing # allow gueesing tags by naftawayh before analyis
# if taggin is disabled, the disambiguation is also disabled
self.allowDisambiguation = allowDisambiguation and allowTagGuessing # allow disambiguation before analyis
# enable the last mark (Harakat Al-I3rab)
self.allowSyntaxLastMark = True
if self.allowTagGuessing:
self.tagger = naftawayh.wordtag.WordTagger()
if self.allowDisambiguation:
self.disambiguator = disambig.disambiguator()
self.debug = False
# to allow to print internal data
self.limit = 10000
# limit words in the text
self.wordcounter = 0
# the words contain arabic letters and harakat.
# the unicode considers arabic harakats as marks not letters,
# then we add harakat to the regluar expression to tokenize
marks = u"".join(
araby.TASHKEEL
) # contains [FATHA,DAMMA,KASRA,SUKUN,DAMMATAN,KASRATAN,FATHATAN,SHADDA])
# used to tokenize arabic text
self.token_pat = re.compile(u"([\w%s]+)" % marks, re.UNICODE)
#used to split text into clauses
self.Clause_pattern = re.compile(
u"([\w%s\s]+)" % (u"".join(araby.TASHKEEL), ), re.UNICODE)
# allow partial vocalization support,
#~The text is analyzed as partial or fully vocalized.
self.partial_vocalization_support = True
#word frequency dictionary
self.wordfreq = wordfreqdictionaryclass.wordfreqDictionary(
'wordfreq', wordfreqdictionaryclass.wordfreq_DICTIONARY_INDEX)
# added to avoid duplicated search in the word frequency database
# used as cache to reduce database access
#added as a global variable to avoid duplucated search in mutliple call of analex
# cache used to avoid duplicata
self.allowCacheUse = True
if self.allowCacheUse:
self.cache = cache.cache()
def assemble(target):
'''Assemble dependencies and contents recursively until target exists.'''
if cache.get_cache(target):
return cache.cache_key(target)
defs = Definitions()
this = defs.get(target)
if this.get('arch') and this['arch'] != app.settings['arch']:
app.log(target, 'Skipping assembly for', this['arch'])
return None
with app.timer(this, 'Starting assembly'):
sandbox.setup(this)
for it in this.get('systems', []):
system = defs.get(it)
assemble(system)
for subsystem in this.get('subsystems', []):
assemble(subsystem)
dependencies = this.get('build-depends', [])
random.shuffle(dependencies)
for it in dependencies:
dependency = defs.get(it)
assemble(dependency)
sandbox.install(this, dependency)
contents = this.get('contents', [])
random.shuffle(contents)
for it in contents:
component = defs.get(it)
if component.get('build-mode') != 'bootstrap':
assemble(component)
sandbox.install(this, component)
build(this)
do_manifest(this)
cache.cache(this, full_root=this.get('kind', None) == "system")
sandbox.remove(this)
return cache.cache_key(this)
def __init__(self, allowTagGuessing=True, allowDisambiguation=True):
"""
Create Analex instance.
"""
self.nounstemmer = stem_noun.nounStemmer()
# to stem nouns
self.verbstemmer = stem_verb.verbStemmer()
# to stem verbs
self.unknownstemmer = stem_unknown.unknownStemmer()
# to stem unknown
self.stopwordsstemmer = stem_stopwords.stopWordStemmer()
# to stem stopwords
self.allowTagGuessing = allowTagGuessing # allow gueesing tags by naftawayh before analyis
# if taggin is disabled, the disambiguation is also disabled
self.allowDisambiguation = allowDisambiguation and allowTagGuessing # allow disambiguation before analyis
# enable the last mark (Harakat Al-I3rab)
self.allowSyntaxLastMark = True
if self.allowTagGuessing:
self.tagger = naftawayh.wordtag.WordTagger()
if self.allowDisambiguation:
self.disambiguator = disambig.disambiguator()
self.debug = False
# to allow to print internal data
self.limit = 10000
# limit words in the text
self.wordcounter = 0
# the words contain arabic letters and harakat.
# the unicode considers arabic harakats as marks not letters,
# then we add harakat to the regluar expression to tokenize
marks = u"".join(araby.TASHKEEL) # contains [FATHA,DAMMA,KASRA,SUKUN,DAMMATAN,KASRATAN,FATHATAN,SHADDA])
# used to tokenize arabic text
self.token_pat = re.compile(u"([\w%s]+)" % marks, re.UNICODE)
# used to split text into clauses
self.Clause_pattern = re.compile(u"([\w%s\s]+)" % (u"".join(araby.TASHKEEL),), re.UNICODE)
# allow partial vocalization support,
# ~The text is analyzed as partial or fully vocalized.
self.partial_vocalization_support = True
# word frequency dictionary
self.wordfreq = wordfreqdictionaryclass.wordfreqDictionary(
"wordfreq", wordfreqdictionaryclass.wordfreq_DICTIONARY_INDEX
)
# added to avoid duplicated search in the word frequency database
# used as cache to reduce database access
# added as a global variable to avoid duplucated search in mutliple call of analex
# cache used to avoid duplicata
self.allowCacheUse = True
if self.allowCacheUse:
self.cache = cache.cache()
def movie(trakt_slug, people_needed=False):
cache_key = 'movie-{0}'.format(trakt_slug)
cache_init_func = functools.partial(__movie, trakt_slug=trakt_slug)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__movie, trakt_slug=trakt_slug, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.WEEK
result = cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration)
if people_needed:
cache_key = 'movie-{0}-people'.format(trakt_slug)
cache_init_func = functools.partial(__movie_people, trakt_slug=trakt_slug)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__movie_people, trakt_slug=trakt_slug, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.WEEK
result['people'] = cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration)
return result
def __init__(self):
print('This simulator simulates an L1 cache with a writeback policy. There are 2 modes, 0 and 1, for reading from a file and for receiving inputs from the command line.')
self.mode = int(input('Enter simulator mode (0 for reading a file and 1 for reading from the command line):'))
association = int(input('Enter the associativity for the cache to be built:'))
num_sets = int(input('Enter the number of sets for the cache to be built:'))
line_size = int(input('Enter the line size for the cache to be built:'))
self.cache_sim = c.cache(num_sets, association, line_size)
if(self.mode):
print('Entering command line mode . . .')
else:
print('Entering read file mode . . .')
def show(trakt_slug, seasons_needed=False):
cache_key = 'show-{0}'.format(trakt_slug)
cache_init_func = functools.partial(__show, trakt_slug=trakt_slug)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__show, trakt_slug=trakt_slug, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.DAY
result = cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration)
if seasons_needed:
cache_key = 'show-{0}-seasons'.format(trakt_slug)
cache_init_func = functools.partial(__show_seasons, trakt_slug=trakt_slug, show_info=result)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__show_seasons, trakt_slug=trakt_slug, show_info=result, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.DAY
result['seasons'] = cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration)
return result
def transfer_values_cache(cache_path, model, images=None, image_paths=None):
# Helper-function for processing the images if the cache-file does not exist.
def fn():
return process_images(fn=model.transfer_values,
images=images,
image_paths=image_paths)
# Read the transfer-values from a cache-file, or calculate them if the file does not exist.
transfer_values = cache(cache_path=cache_path, fn=fn)
return transfer_values
def tastekid_lookup(title, check_cache=True, load_rec_content=False, use_key=True):
if check_cache and cache.in_cache(title):
ret = cache.retrieve_valid(title)
tkterms = config.tk_terms.keys()
ret = {k:v for k,v in ret.items() if k in tkterms}
if ((not load_rec_content) or ('suggestions' in ret)) and ret.keys() == tkterms:
return ret
tkjson = tastekid_json(title, use_key)
if 'Error' in tkjson:
print 'Yarr there be an error fetching tk; trying sneakily'
try:
tkjson = tastekid_json(title, False)['Similar']
except:
print 'Yarr there be another bloody error; grabbing everything fromcache'
ret = cache.retrieve_cached(title) #ignoring cache limits
if ret is not None and 'info' in ret:
ret['info']['suggestions'] = ret['suggestions'] if 'suggestions' in ret else None
ret = ret['info']
print 'check ret:', ret
for k in ret.keys():
if k not in config.tk_terms.keys():
del ret[k]
for k in config.tk_terms.keys():
if k not in ret:
ret[k] = None
return ret
return None
else:
tkjson = tkjson['Similar']
item = extract_terms(tkjson['Info'][0], config.tk_terms) #actual movie
suggestions = [extract_terms(tk, config.tk_terms) for tk in tkjson['Results']]
rec_titles = [r['title'] for r in suggestions]
tk = {'title':item['title'], 'suggestions':rec_titles}
cache.upsert_properties(item)
cache.cache(tk)
for r in suggestions:
cache.upsert_properties(r)
for k in config.tk_terms:
tk[k] = item[k]
return tk
def load_cache():
u'''
'''
print ("lobbist")
cache_lobbist = cache.cache("fec_lobbist", lobbyist)
print ("lobbist bundle")
cache_lobbist_bundle = cache.cache("fec_lobbist_bundle", lobbyist_bundle)
print ("expenses")
cache_expenses = cache.cache("fec_expenses", expenses)
print ("candidate_disbursement")
# cache_can_disburse = cache.cache(
# "fec_all_candidate_disbursement",
# all_candidate_disbursement)
print ("Committee Summary")
cache_committee_summary = cache.cache("CommitteeSummary", committee_summary)
print ("CampaignAndCommitteeSummary")
cache_camp_comm_sum = cache.cache(
"CampaignAndCommitteeSummary",
campaign_and_committee_summary)
print ("CandidateSummary")
cache_candidate_summary = cache.cache("CandidateSummary", candidate_summary)
def shows_trending(page):
if page < 1 or page > 10:
raise exceptions.HTTPError(404)
cache_key = 'shows-trending-{0}'.format(page)
cache_init_func = functools.partial(__shows_list_page, page='/shows/trending', page_index=page)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__shows_list_page, page='/shows/trending', page_index=page, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.HOUR
cache_expiration = None
return cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration, cache_expiration)
def movies_popular(page):
if page < 1 or page > 10:
raise exceptions.HTTPError(404)
cache_key = 'movies-popular-{0}'.format(page)
cache_init_func = functools.partial(__movies_list_page, page='/movies/popular', page_index=page)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__movies_list_page, page='/movies/popular', page_index=page, timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.DAY
cache_expiration = None
return cache.cache(cache_key, cache_init_func, cache_init_exception_handler, cache_init_failure_handler, cache_update_func, cache_data_expiration, cache_expiration)
def findNewEd2k(self, url):
'''
find and cache all ed2k links on a page, but only return new links
'''
links = self.findEd2k(url)
print 'found %i ed2k links' % len(links)
self.ed2k.extend(links)
cache_id = hash(url)
if cache.has_cache(cache_id):
cacheList = cache.load(cache_id)
if cacheList == self.ed2k:
print 'nothing change. ' + url
else:
print 'you has new links ' + url
newLinks = zip(*self.ed2k)[0]
oldLinks = zip(*cacheList)[0]
diff = list(set(newLinks).difference( set(oldLinks) )) # lists difference
for link in diff:
print link
pyperclip.copy(link) # TODO
else:
print 'just cache the links ' + url
cache.cache(self.ed2k, cache_id)
def shows_search(query):
cache_key = 'shows-search-{0}'.format(query)
cache_init_func = functools.partial(__shows_search, query=query)
cache_init_exception_handler = network.http_get_init_exception_handler
cache_init_failure_handler = network.http_get_init_failure_handler
cache_update_func = functools.partial(__shows_search,
query=query,
timeout=network.TIMEOUT_CONNECT)
cache_data_expiration = cache.HOUR
return cache.cache(cache_key, cache_init_func,
cache_init_exception_handler,
cache_init_failure_handler, cache_update_func,
cache_data_expiration)
