def traverse(dct, parent=None):
resources = {}
for name, res in dct.items():
host = url_to_host(self.base_url)
apidoc_url = urljoin(host, res.get('apidoc'))
schema_url = urljoin(host, res.get('schema'))
resource = Resource(
self,
name,
res.get('relativePath'),
schema_url,
apidoc_url,
parent=parent,
)
self._resources[name] = resource
resources[name] = resource
child_resources = {}
for att, val in res.items():
if att == 'children':
child_resources.update(traverse(val, parent=resource))
resource.children = child_resources
return resources
def __init__(self, webdriver, base_url, components, **env):
"""
:Description: Controller for managing components.
:param webdriver: Webdriver for controller and components to reference.
:type webdriver: WebDriver
:param base_url: Base url for navigations, will navigate to this url in init.
:type base_url: basestring
:param components: Component objects to instantiate.
:type components: tuple, list, dict
:param env: Key value pairs to pass to instantiated components.
:type env: **kwargs => dict
"""
self.webdriver = self.__patch_webdriver(webdriver)
self.js = E2EJS(webdriver)
self.base_url = base_url
self.logger = logger
if not isinstance(components, (tuple, list, dict)):
raise TypeError('Components must be either a tuple, list, or dictionary')
self.env = Resource(**env) if env else Resource()
if isinstance(components, dict):
self.components = Resource(**{
name: component(webdriver=self.webdriver, logger=self.logger, env=self.env)}
for name, component in components.iteritems())
else:
self.components = [
component(webdriver=self.webdriver, logger=self.logger, env=self.env) for component in components]
self.webdriver.get(self.base_url)
def __init__(self,
filterchain_name,
serialize=None,
default_media_name=None):
# to limit the infini import, we import in the init
from resource import Resource
self.resource = Resource()
self.filters = []
# {"filter_name":[observator,]}
self.image_observers = {}
self.filter_output_observers = []
self.filterchain_name = filterchain_name
self.original_image_observer = []
self.dct_global_param = {}
self.dct_media_param = {}
# If starting filterchain with empty media_name, we take the default
# media
self.default_media_name = default_media_name
if serialize:
self.deserialize(filterchain_name, serialize)
else:
# add default filter
self.add_filter(Filter(keys.get_empty_filter_name()))
self.do_configure()
def test_02_request_package_name(self):
""" 2: Test request_package_name method from Resource class
Check if returns a value, which is not NULL """
resource = Resource(self.testid)
resource.unpack_object_to_self(resource.load_from_ckan())
package_name = resource.request_package_name()
self.failUnless(package_name, 'package name is NULL!')
def test_should_load_resource(self):
content = "Content"
web = TestWeb()
web.response = FakeResponse(content, "text/html")
resource = Resource("Link", "Target", web)
loaded_content = resource.load()
self.assertEqual(loaded_content, content)
def __init__(self, request):
"""Box"""
resource_provider = ResourceProvider()
resource = Resource(resource_provider)
self.charts = []
self.chart_type = None
self.layout = Layout(request)
self.body = ''
self.javascript = ''
self.chart_name = request.matchdict['box_name']
chart_format = os.path.splitext(request.environ['PATH_INFO'])[1]
# Go through the registry, and find the resource for this box
# XXX This should be a dictionary
for res in RESOURCES_REGISTRY:
if res[0] == self.chart_name:
self.resources = [res]
if chart_format == '.html':
self.render_html(request)
elif chart_format == '.csv':
self.body = resource.get(self.chart_name, CSV, request.matchdict)
elif chart_format == '.json':
self.body = resource.get(self.chart_name, JSON, request.matchdict)
else:
print "Format not supported %s" % chart_format
raise AttributeError
def initialize(self):
''' Restore system to its initial state and create Process 0
Returns process 0
'''
self.PCB = [None] * 16
self.RCB = [None] * 4
#self.ready_list = deque()
self.ready_list = {
2: deque(),
1: deque(),
0: deque()
} #### WHERE YOU LAST LEFT OFF 10.28.19 5:20 AM
## Creating Process 0 and placing it into Ready List
self.PCB[0] = Process(0, 0, 1, 0) ## Process(id/index, state, parent)
self.active_processes += 1
self.ready_list[0].append(self.PCB[0].id)
self.run_proc = self.PCB[self.ready_list[0][0]]
#print(self.run_proc.id)
## Set up all Resources
self.RCB[0] = Resource(0, 1)
self.RCB[1] = Resource(1, 1)
self.RCB[2] = Resource(2, 2)
self.RCB[3] = Resource(3, 3)
return 0
def traverse(dct, parent=None):
resources = {}
for name, res in dct.items():
host = url_to_host(self.base_url)
apidoc_url = urljoin(host, res.get('apidoc'))
schema_url = urljoin(host, res.get('schema'))
resource = Resource(
self,
name,
res.get('relativePath'),
schema_url,
apidoc_url,
parent=parent,
)
self._resources[name] = resource
resources[name] = resource
child_resources = {}
for att, val in res.items():
if att == 'children':
child_resources.update(
traverse(val, parent=resource)
)
resource.children = child_resources
return resources
def assemble(section_name, items):
# 将配置文件的数据组装成对象
resource = Resource()
resource.flag = section_name
for item in items:
setattr(resource, item[0], item[1])
return resource
def __init__(self, request):
"""Box"""
resource_provider = ResourceProvider()
resource = Resource(resource_provider)
self.charts = []
self.chart_type = None
self.layout = Layout(request)
self.body = ''
self.javascript = ''
self.chart_name = request.matchdict['box_name']
chart_format = os.path.splitext(request.environ['PATH_INFO'])[1]
# Go through the registry, and find the resource for this box
# XXX This should be a dictionary
for res in RESOURCES_REGISTRY:
if res[0] == self.chart_name:
self.resources = [res]
if chart_format == '.html':
self.render_html(request)
elif chart_format == '.csv':
self.body = resource.get(self.chart_name, CSV, request.matchdict)
elif chart_format == '.json':
self.body = resource.get(self.chart_name, JSON, request.matchdict)
else:
print "Format not supported %s" % chart_format
raise AttributeError
def judge(cls, t, ti): # 到达时间t，系统时间ti
"""时间和资源判断"""
r_cpu = Resource.get_res()[0]
r_mem = Resource.get_res()[1]
r_io = Resource.get_res()[2]
if t <= ti and Algorithm.ri1 <= r_cpu and Algorithm.ri2 <= r_mem and Algorithm.ri3 <= r_io:
return True
def add_file(self, resource_list=None, dir=None, file=None):
"""Add a single file to resource_list
Follows object settings of set_path, set_md5 and set_length.
"""
try:
if self.exclude_file(file):
self.logger.debug("Excluding file %s" % (file))
return
# get abs filename and also URL
if (dir is not None):
file = os.path.join(dir,file)
if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))):
return
uri = self.mapper.dst_to_src(file)
if (uri is None):
raise Exception("Internal error, mapping failed")
file_stat=os.stat(file)
except OSError as e:
sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e)))
return
timestamp = file_stat.st_mtime #UTC
r = Resource(uri=uri,timestamp=timestamp)
if (self.set_path):
# add full local path
r.path=file
if (self.set_md5):
# add md5
r.md5=compute_md5_for_file(file)
if (self.set_length):
# add length
r.length=file_stat.st_size
resource_list.add(r)
def upload_new_resources(new_resources_df: pd.DataFrame,
firestore_resources: Dict[Resource, str], db,
sheet: gspread.models.Worksheet) -> List[int]:
length = len(new_resources_df.index)
log(f"{length} resources to upload")
added = 0
uploaded_rows = list()
for index, row in new_resources_df.iterrows():
links = Links(row["card link"], row["website"])
resource = Resource(row["resource name"], True, row["description"],
row["image link"], row["category"],
row["tags"].split(", "), links)
try:
if resource not in firestore_resources:
db.collection(FIREBASE_COLLECTION).add(resource.to_dict())
log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}")
else:
db.collection(FIREBASE_COLLECTION).document(
firestore_resources[resource]).set(resource.to_dict())
log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}"
)
except:
log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully"
)
return uploaded_rows
added += 1
uploaded_rows.append(index + 1)
log(f"\nAdded {added} / {length} entries to Firestore")
return uploaded_rows
def import_ds(ds, type=None, parent_id=None):
if type == 'parent':
ds['is_parent'] = 'true'
else:
ds.pop('is_parent', None)
if type == 'child':
ds['parent_dataset'] = parent_id
else:
ds.pop('parent_dataset', None)
dataset_dummy = create_dummy_dataset()
dataset_dummy['title'] = ds['title']
dataset_dummy['owner_org'] = owner_org
# first run to get name created.
ds_created = dataset_dummy.create(create_url, api_key)
# then update the dataset with all info
dataset_full = load_dataset(ds_created)
map_dataset(dataset_full, ds)
dataset_full._update(update_url, api_key)
# add resource
resources = ds.get('distribution', [])
for res in resources:
resource = Resource()
map_resource(resource, res, dataset_full['id'])
# skip and report empty resource
if resource['url']:
res_created = resource.create(resource_url, api_key)
else:
logging.info(' Empty resource skipped for: %s' % ds['title'])
return dataset_full
def __init__(self,
server_host,
server_port=None,
username="******",
password="******",
use_tls=False,
version=API_CURRENT_VERSION):
"""
Creates a Resource object that provides API endpoints.
@param server_host: The hostname of the Cloudera Manager server.
@param server_port: The port of the server. Defaults to 7180 (http) or
7183 (https).
@param username: Login name.
@param password: Login password.
@param use_tls: Whether to use tls (https).
@param version: API version.
@return: Resource object referring to the root.
"""
self._version = version
protocol = use_tls and "https" or "http"
if server_port is None:
server_port = use_tls and 7183 or 7180
base_url = "%s://%s:%s/api/v%s" % \
(protocol, server_host, server_port, version)
client = HttpClient(base_url, exc_class=ApiException)
client.set_basic_auth(username, password, API_AUTH_REALM)
client.set_headers({"Content-Type": "application/json"})
Resource.__init__(self, client)
def add_file(self, resource_list=None, dir=None, file=None):
"""Add a single file to resource_list
Follows object settings of set_path, set_md5 and set_length.
"""
try:
if self.exclude_file(file):
self.logger.debug("Excluding file %s" % (file))
return
# get abs filename and also URL
if (dir is not None):
file = os.path.join(dir, file)
if (not os.path.isfile(file) or
not (self.include_symlinks or not os.path.islink(file))):
return
uri = self.mapper.dst_to_src(file)
if (uri is None):
raise Exception("Internal error, mapping failed")
file_stat = os.stat(file)
except OSError as e:
sys.stderr.write("Ignoring file %s (error: %s)" % (file, str(e)))
return
timestamp = file_stat.st_mtime #UTC
r = Resource(uri=uri, timestamp=timestamp)
if (self.set_path):
# add full local path
r.path = file
if (self.set_md5):
# add md5
r.md5 = compute_md5_for_file(file)
if (self.set_length):
# add length
r.length = file_stat.st_size
resource_list.add(r)
def get_user_resource_tags(username, collectionid, resourceid):
rsrc = Resource()
rsrc.create()
result = rsrc.get(username, collectionid, resourceid)
if not result:
return [{'No tags'}]
else:
return [{'TagText': result[0]['tags']}]
def __init__(self, manager, id):
Resource.__init__(self, manager, id)
self.data = None
self.size = None
self.hash = None
self.controllers = []
self.remaining_controllers = None
logging.debug("New buffer: %s", self)
def test_08_load_mapping(self):
""" 8: Test fetching the wiki page and parsing together """
resource = Resource(self.testid)
resource.wiki_site = wikitools.Wiki(config.wiki_api_url)
resource.wiki_site.login(config.wiki_username, password=config.wiki_password)
mappings = resource.load_mappings()
self.failUnless(mappings, 'No mappings exists!')
def test_06_request_wiki_page(self):
""" 6: Test if wiki page for this Resource exists """
resource = Resource(self.testid)
resource.wiki_site = wikitools.Wiki(config.wiki_api_url)
resource.wiki_site.login(config.wiki_username, password=config.wiki_password)
wiki_page = resource._request_wiki_page()
self.failUnless(wiki_page, "Wiki page does not exist or empty!")
def __init__(self, request, cells):
resource_provider = ResourceProvider()
self.resource = Resource(resource_provider)
self.cells = cells
self.resources = self.get_resources()
self.charts = self.get_charts(request)
self.packages = self.get_packages()
self.javascript = render_javascript(self.charts, self.packages)
def print(self):
print("Peer info: id: " + self.PEER + ", IP: " + self.PORT)
print("Max download rate: " + self.download_rate() + "b/s")
print("Max upload rate: " + self.upload_rate() + "b/s")
print("*** Downloads in progress ***")
print("-File " + Resource.getFileName(filePath) + "total downloaded: ")
print("*** Uploads in progress ***")
print("-File " + Resource.getFileName(filePath) + "total uploaded: ")
def roll_text(self, dt):
if self.text_y < ((self.surface.get_height()) * -1):
self.state = 1
Resource.getInstance().play_sound('sounds/boot/b.ogg')
pg.time.delay(500)
else:
self.text_y -= self.velocity * dt
pass
class DistributionTestCase(unittest.TestCase):
def setUp(self):
self.job = Job()
self.batch = Batch()
self.resource = Resource()
rootDir = os.environ['BOLT_DIR']
self.batch.readConfig(rootDir + configDir + "/" + batchConfig)
self.resource.readConfig(rootDir + configDir + "/" + resourceConfig)
def testParallelTaskDitributionPureMPI(self):
"""Pure MPI task distribution (fully populated)."""
# Set the parallel distribution
self.job.setTasks(1024)
self.job.setTasksPerNode(self.resource.numCoresPerNode())
self.job.setThreads(1)
self.job.setParallelDistribution(self.resource, self.batch)
correct = "aprun -n 1024 -N 32 -S 8 -d 1"
self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct))
def testParallelTaskDitributionHalfPopulate(self):
"""Pure MPI task distribution (half populated)."""
# Set the parallel distribution
self.job.setTasks(1024)
self.job.setTasksPerNode(16)
self.job.setThreads(1)
self.job.setParallelDistribution(self.resource, self.batch)
correct = "aprun -n 1024 -N 16 -S 4 -d 2"
self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct))
def testParallelTaskDitributionTwoThreads(self):
"""Hybrid MPI/OpenMP task distribution (2 OpenMP threads)."""
# Set the parallel distribution
self.job.setTasks(1024)
self.job.setTasksPerNode(16)
self.job.setThreads(2)
self.job.setParallelDistribution(self.resource, self.batch)
correct = "export OMP_NUM_THREADS=2\naprun -n 1024 -N 16 -S 4 -d 2"
self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct))
def testParallelTaskDitributionThreeThreads(self):
"""Hybrid MPI/OpenMP task distribution (3 OpenMP threads)."""
# Set the parallel distribution
self.job.setTasks(1024)
self.job.setTasksPerNode(10)
self.job.setThreads(3)
self.job.setParallelDistribution(self.resource, self.batch)
correct = "export OMP_NUM_THREADS=3\naprun -n 1024 -N 10 -d 3"
self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct))
def _make_ac(self, chara, name):
values = RM.res('ac')
label = ttk.Label(self, text=RM.res(name))
i = chara.get_ac(name)
value = self._get_value(values, i)
self._ac[name] = tk.StringVar(value=value)
combobox = ttk.Combobox(self, values=values,
textvariable=self._ac[name], state='readonly',width=10)
return (label, combobox)
def put_resource(data):
rsrc = Resource(True)
if 'resourceid' in data:
rsrcid = data['resourceid']
else:
rsrcid = str(time.time())
# the last param None is the photo data which we will fill in later
rsrc.put(data['user'], data['collectionid'], data['description'], rsrcid, None)
return rsrcid
def test_01_load_from_ckan (self):
""" 1: Test load_from_ckan method from Resource class
Check if attributes of the Resource object fetched from CKAN exist:
-- revision_id - used for fetching package_id
-- url - used for downloading the data file """
resource = Resource(self.testid)
object = resource.load_from_ckan()
self.failUnless('url' in object, 'Tested object has no url attribute.')
self.failUnless('revision_id' in object, 'Tested object has no revision_id attribute.')
def get(self):
self.response.headers['Content-Type'] = 'text/html'
resources = Resource.getResources()
resources = Resource.getResourcesFor6Through8(resources)
values = {
"title": "Middle School",
"curr_url": self.request.url,
}
self.response.out.write(template.render('html/middle.html', values))
class KBO():
def __init__(self, isSimulation=False, isAPI=False):
if isAPI:
print("====================As API====================")
from Api.api import API
self.resource = Resource()
self.web = Web(resource=self.resource)
self.vision = Vision(resource=self.resource)
#self.run_server(host="166.104.143.103", port=8080)
self.run_server()
elif isSimulation:
print("====================Simulation====================")
self.run_bbox()
else:
print("====================In Local====================")
self.resource = Resource()
self.web = Web(resource=self.resource)
self.vision = Vision(resource=self.resource)
self.tts = TTS(resource=self.resource)
self.run()
def run(self):
self.resource.set_frameno(START_FRAME + 1)
idx = self.web.parsing_before()
web_thread = threading.Thread(target=self.web.parsing_relaytext,
args=(idx, ))
web_thread.start()
vision_thread = threading.Thread(target=self.vision.play)
vision_thread.start()
tts = threading.Thread(target=self.tts.text_2_speech)
tts.start()
play(self.resource)
def run_bbox(self):
play_bbox(frameno=128400)
def run_server(self):
self.resource.set_frameno(START_FRAME + 1)
idx = self.web.parsing_before()
web_thread = threading.Thread(target=self.web.parsing_relaytext,
args=(idx, True))
web_thread.start()
vision_thread = threading.Thread(target=self.vision.play)
vision_thread.start()
play_API(self.resource, host="169.254.17.149", port=8080)
def test_04_get_wiki_url(self):
""" 4: Test if get_wiki_url method from Resource class
returns URI string.
The result depends on config.py file."""
resource = Resource(self.testid)
parsed_url = urlparse(resource.get_wiki_url())
self.failUnless(parsed_url.scheme, "Scheme is missing.")
self.failUnless(parsed_url.netloc, "Net Location (e.g. publicdata.eu) is missing.")
self.failUnless(parsed_url.path, "Path is missing.")
def test_should_update_the_resource_when_it_attempts_a_redirect(self):
web = TestWeb()
url = "http://localhost:5000/redirect"
web.head = self.redirect
resource = Resource('Page', 'http://localhost:5000', web)
resource.isAPage()
web.head = lambda x: FakeResponse('Content', 'application/pdf')
self.assertEqual(resource.name, "redirect.pdf")
self.assertEqual(resource.url, "http://localhost:5000/redirect.pdf")
def get(self):
self.response.headers['Content-Type'] = 'text/html'
resources = Resource.getResources()
resources = Resource.getResourcesFor9Through12(resources)
values = {
"title": "High School",
"curr_url": self.request.url,
"resources": resources
}
self.response.out.write(template.render('html/highschool.html', values))
def upload_new_resources(new_resources_df: pd.DataFrame,
firestore_resources: Dict[Resource, str], db,
sheet: gspread.models.Worksheet) -> List[int]:
length = len(new_resources_df.index)
log(f"{length} resources to upload")
added = 0
uploaded_rows = list()
for index, row in new_resources_df.iterrows():
links = Links(row["card link"], row["website"])
date_created = datetime.now().strftime("%Y/%m/%d %H:%M:%S")
resource = Resource(title=row["resource name"],
reviewed=True,
want_support_with=row["want support with"],
this_resource_offers=row["this resource offers"],
description=row["description"],
img=row["image link"],
category=row["category"],
tags=row["tags"].split(", "),
links=links,
date_created=date_created,
ranking=row["ranking"])
try:
category_document = db.collection(FIREBASE_COLLECTION).document(
resource.category.replace("/ ", "_"))
if resource not in firestore_resources:
category_document.update({
"resource_list":
firestore.ArrayUnion([resource.title])
}) # Update resource list
category_document.update(
{"tag_list":
firestore.ArrayUnion(resource.tags)}) # Update tag list
category_document.collection("resources").add(
resource.to_dict()) # Add new document to collection
log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}/{category_document.id}"
)
else:
resource._date_created = category_document.collection(
"resources").document(firestore_resources[resource]).get(
).to_dict()["dateCreated"]
category_document.collection("resources").document(
firestore_resources[resource]).set(resource.to_dict(
)) # Update old document in collection
log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}/{category_document.id}"
)
except Exception as e:
log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully"
)
print(e)
return uploaded_rows
added += 1
uploaded_rows.append(index + 1)
log(f"\nAdded {added} / {length} entries to Firestore")
return uploaded_rows
def get(self):
self.response.headers['Content-Type'] = 'text/html'
resources = Resource.getResources()
resources = Resource.getResourcesForCollege(resources)
values = {
"title": "College",
"scholarship": self.request.url + "/scholarship",
"curr_url": self.request.url,
"resources": resources
}
self.response.out.write(template.render('html/college.html', values))
def release_task_res(pi):
"""查找任务pi,并释放其分配的系统资源"""
for i in range(len(news_waiting_queues)):
if news_waiting_queues[i][0] == pi:
r_cpu = news_waiting_queues[i][1] # 查看原始信息
r_mem = news_waiting_queues[i][2]
r_io = news_waiting_queues[i][3]
Resource.release_res(r_cpu, r_mem, r_io) # 释放资源
break
def from_disk(self, path, url_prefix, inventory=None):
"""Create or extend inventory with resources from disk scan
Assumes very simple disk path to URL mapping: chop path and
replace with url_path. Returns the new or extended Inventory
object.
If a inventory is specified then items are added to that rather
than creating a new one.
mb = InventoryBuilder()
m = inventory_from_disk('/path/to/files','http://example.org/path')
"""
num = 0
# Either use inventory passed in or make a new one
if (inventory is None):
inventory = Inventory()
# for each file: create Resource object, add, increment counter
for dirpath, dirs, files in os.walk(path, topdown=True):
for file_in_dirpath in files:
try:
if self.exclude_file(file_in_dirpath):
continue
# get abs filename and also URL
file = os.path.join(dirpath, file_in_dirpath)
if (not os.path.isfile(file)
or not (self.include_symlinks
or not os.path.islink(file))):
continue
rel_path = os.path.relpath(file, start=path)
if (os.sep != '/'):
# if directory path sep isn't / then translate for URI
rel_path = rel_path.replace(os.sep, '/')
url = url_prefix + '/' + rel_path
file_stat = os.stat(file)
except OSError as e:
sys.stderr.write("Ignoring file %s (error: %s)" %
(file, str(e)))
continue
mtime = file_stat.st_mtime
lastmod = datetime.fromtimestamp(mtime).isoformat()
r = Resource(uri=url, lastmod=lastmod)
if (self.do_md5):
# add md5
r.md5 = compute_md5_for_file(file)
if (self.do_size):
# add size
r.size = file_stat.st_size
inventory.add(r)
# prune list of dirs based on self.exclude_dirs
for exclude in self.exclude_dirs:
if exclude in dirs:
dirs.remove(exclude)
return (inventory)
def __init__(self, config):
"""
param:
"""
self.config = config
self.exec_times = 0
self.trigger_time = 0
self.name = self.config['main']['name']
self.__state = EngineState_Init
self.engine_instance_vars = dict()
Resource.initialize_local_resources(self.name, self.config)
def get_user_resource_details(username, collectionid, resourceid):
rsrc = Resource()
rsrc.create()
result = rsrc.get(username, collectionid, resourceid)
if not result:
return {'Not found'}
else:
return {
'FileSource': result[0]['hires_url'],
'Description' : 'Powered by PhotoStore Technology',
'Location': 'Palo Alto'
}
def post(self):
lang = self.request.get("lang")
resources = Resource.getResources()
if lang:
resources = Resource.getResourcesForSpanish(resources, True)
resources = Resource.getResourcesFor9Through12(resources)
values = {
"title": "High School",
"curr_url": self.request.url,
"resources": resources
}
self.response.out.write(template.render('html/highschool.html', values))
def get_resource(data):
if not data:
return {'NOTFOUND'}
rsrc = Resource(True)
if 'collectionid' in data:
collectionid = data['collectionid']
else:
collectionid = None
if 'resourceid' in data:
resourceid = data['resourceid']
else:
resourceid = None
return rsrc.get(data['user'], collectionid, resourceid)
def setUp(self):
self._resource1 = Resource('test_resource1', [
Limit(2, timedelta(seconds=1)),
Limit(3, timedelta(seconds=1)),
])
self._resource2 = Resource('test_resource2', [
Limit(6, timedelta(seconds=1)),
Limit(7, timedelta(seconds=1)),
])
self._resource3 = Resource('test_resource3', [
Limit(10, timedelta(seconds=1)),
Limit(10, timedelta(seconds=1)),
])
def post(self):
lang = self.request.get("lang")
resources = Resource.getResources()
if lang:
resources = Resource.getResourcesForSpanish(resources, True)
resources = Resource.getResourcesForCollege(resources)
values = {
"title": "College",
"scholarship": self.request.url + "/scholarship",
"curr_url": self.request.url,
"resources": resources
}
self.response.out.write(template.render('html/college.html', values))
def from_disk(self,path,url_prefix,inventory=None):
"""Create or extend inventory with resources from disk scan
Assumes very simple disk path to URL mapping: chop path and
replace with url_path. Returns the new or extended Inventory
object.
If a inventory is specified then items are added to that rather
than creating a new one.
mb = InventoryBuilder()
m = inventory_from_disk('/path/to/files','http://example.org/path')
"""
num=0
# Either use inventory passed in or make a new one
if (inventory is None):
inventory = Inventory()
# for each file: create Resource object, add, increment counter
for dirpath, dirs, files in os.walk(path,topdown=True):
for file_in_dirpath in files:
try:
if self.exclude_file(file_in_dirpath):
continue
# get abs filename and also URL
file = os.path.join(dirpath,file_in_dirpath)
if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))):
continue
rel_path=os.path.relpath(file,start=path)
if (os.sep != '/'):
# if directory path sep isn't / then translate for URI
rel_path=rel_path.replace(os.sep,'/')
url = url_prefix+'/'+rel_path
file_stat=os.stat(file)
except OSError as e:
sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e)))
continue
mtime = file_stat.st_mtime
lastmod = datetime.fromtimestamp(mtime).isoformat()
r = Resource(uri=url,lastmod=lastmod)
if (self.do_md5):
# add md5
r.md5=compute_md5_for_file(file)
if (self.do_size):
# add size
r.size=file_stat.st_size
inventory.add(r)
# prune list of dirs based on self.exclude_dirs
for exclude in self.exclude_dirs:
if exclude in dirs:
dirs.remove(exclude)
return(inventory)
def test_07_extract_csv_configurations(self):
""" 7: Test the parser for CSV configurations on Wiki page """
resource = Resource(self.testid)
resource.wiki_site = wikitools.Wiki(config.wiki_api_url)
resource.wiki_site.login(config.wiki_username, password=config.wiki_password)
wiki_page = resource._request_wiki_page()
mappings = resource._extract_csv_mappings(wiki_page)
for mapping in mappings:
self.failUnless(mapping['type'] == 'RelCSV2RDF', 'Wrong mapping type, should be RelCSV2RDF.')
self.failUnless(mapping['name'], 'Name of mapping is undefined!')
self.failUnless(mapping['header'], 'Header line is not specified')
self.failUnless(mapping['omitCols'], 'omitCols is not specified')
self.failUnless(mapping['omitRows'], 'omitRows is not specified')
def create_resources_table():
resources_list = [
Resource(
'http://www.ynet.co.il/articles/0,7340,L-4713571,00.html',
'https://images1.ynet.co.il/PicServer4/2014/08/05/5506384/52203970100690640360no.jpg',
'החוש הדומיננטי שיעזור לכם בלימודים',
'החוש הדומיננטי שיעזור לכם בלימודים. אילו טיפים של שימושבחושים יעזרו לכם?'
),
Resource(
'http://www.ynet.co.il/articles/0,7340,L-5045541,00.html',
'https://images1.ynet.co.il/PicServer5/2017/11/23/8172884/817287001000100980704no.jpg',
'"כ"ט בנובמבר: "שמחה שנמשכה ימים ולילות, הייתה אופוריה"',
'ב1947- הם היו ילדים או צעירים בתחילת דרכם,' +
' אבל את היום הגורלי ב29- בנובמבר הם לא שוכחים עד היום.' +
' "כולם היו צמודים לרדיו. אני זוכרת את התפרצות השמחה, ריקודים והתחבקויות."'
),
Resource(
'https://www.calcalist.co.il/world/articles/0,7340,L-3726321,00.html',
'https://images1.calcalist.co.il/PicServer3/2017/11/30/775736/2_l.jpg',
'רוצים נייר טואלט? הזדהו: כך משפרים הסינים את מצב השירותים הציבוריים',
'שבוע קרא נשיא סין שי ג‘ינפינג להמשיך את מהפכת השירותים' +
' הציבוריים עליה הכריז ב-2015. עד כה שופצו ונבנו 68 אלף מתקנים'),
Resource(
'http://www.nrg.co.il/online/13/ART2/902/962.html',
'http://www.nrg.co.il/images/archive/465x349/1/646/416.jpg',
'מחקו לכם הודעה בווטסאפ? עדיין תוכלו לקרוא אותה',
'אפליקציה בשם Notification History מאפשרת למשתמשי אנדרואיד' +
' לקורא את הנתונים הזמניים הנשמרים ביומן הפעילות של הסמארטפון. כולל הודעות מחוקות.'
),
Resource(
'http://www.nrg.co.il/online/55/ART2/904/542.html',
'http://www.nrg.co.il/images/archive/465x349/1/795/429.jpg',
'גם בחורף: זה בדיוק הזמן לקפוץ לאילת',
'העיר הדרומית נעימה לנופש גם בחודשי החורף.' +
' כעת מוצעים מחירים אטרקטיביים במיוחד בחבילות שכוללות מגוון אטרקציות, לינה וטיסות'
),
Resource(
'https://food.walla.co.il/item/3113079',
'https://img.wcdn.co.il/f_auto,w_700/2/5/1/3/2513314-46.jpg',
'12 בתי קפה שמתאימים לעבודה עם לפטופ',
'בין אם אתם סטודנטים או עצמאיים, זה תמיד סיפור למצוא בית קפה נעים וטעים לרבוץ בו.'
+
' קיבצנו עבורכם 12 מקומות אהובים בדיוק למטרה זו, בארבע הערים הגדולות'
),
Resource(
'https://news.walla.co.il/item/3114145',
'https://img.wcdn.co.il/f_auto,w_700/2/4/9/5/2495334-46.jpg',
'שותק על אזריה, נלחם באהוד ברק: בנט מנסה להיבנות כימין ממלכתי',
'כשרגב נלחמת ברעש בתאטרון יפו, בנט משנה בשקט את נהלי סל התרבות כך '
+ 'שהחומרים "השמאלנים" ייפלטו. כשהקשת הפוליטית מתרעמת על דיווחי' +
' ה"דיל" של טראמפ עם הפלסטינים, בנט שותק עד שהרשות תסרב.'),
Resource(
'https://news.walla.co.il/item/3114283',
'https://img.wcdn.co.il/f_auto,w_700/2/5/1/4/2514588-46.jpg',
'רצח בכל שלושה ימים: צרפת יוצאת למאבק באלימות נגד נשים',
'אחרי ש126- נשים נרצחו בידי בני זוגן בשנה שעברה, ' +
'הציג מקרון צעדים חדשים למלחמה בתופעה. "זאת בושה לצרפת,"' +
' אמר הנשיא שאחת מהבטחות הבחירות שלו הייתה להשיג שוויון מגדרי.')
]
return resources_list
def from_disk_add_map(self, resource_list=None, map=None, set_path=False):
"""Add to resource_list with resources from disk scan based one map
If set_path is True then the path attribue will be set with the
local path for each Resource.
"""
# sanity
if (resource_list is None or map is None):
raise ValueError("Must specify resource_list and map")
path=map.dst_path
#print "walking: %s" % (path)
# for each file: create Resource object, add, increment counter
num_files=0
for dirpath, dirs, files in os.walk(path,topdown=True):
for file_in_dirpath in files:
num_files+=1
if (num_files%50000 == 0):
self.logger.info("ResourceListBuilder.from_disk_add_map: %d files..." % (num_files))
try:
if self.exclude_file(file_in_dirpath):
self.logger.debug("Excluding file %s" % (file_in_dirpath))
continue
# get abs filename and also URL
file = os.path.join(dirpath,file_in_dirpath)
if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))):
continue
uri = map.dst_to_src(file)
if (uri is None):
raise Exception("Internal error, mapping failed")
file_stat=os.stat(file)
except OSError as e:
sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e)))
continue
timestamp = file_stat.st_mtime #UTC
r = Resource(uri=uri,timestamp=timestamp)
if (set_path):
r.path=file
if (self.do_md5):
# add md5
r.md5=compute_md5_for_file(file)
if (self.do_length):
# add length
r.length=file_stat.st_size
resource_list.add(r)
# prune list of dirs based on self.exclude_dirs
for exclude in self.exclude_dirs:
if exclude in dirs:
self.logger.debug("Excluding dir %s" % (exclude))
dirs.remove(exclude)
def to_dict(self):
data = Resource.to_dict(self)
print 'Creating Dictionary for JSON output.'
if self.asset:
for k, v in self.asset.to_dict().iteritems():
data['asset_' + k] = v
return data
def init(self):
self.path = os.path.abspath(self.path)
if not os.path.isdir(self.path):
os.makedirs(self.path)
self._threads = []
self._socket = None
self._token_map = {}
self._paths = set()
self.services, services = [], self.services
for s in services:
self.add_service(s['path'], s)
self.resources = Resource.create(self.resources)
for service in self.services:
logging.info("loading service: %s", service.path)
service.deploy()
service_path = os.path.join(self.path, 'services')
if not os.path.exists(service_path):
os.makedirs(service_path)
self.periodically(self.scan, 1)
self.periodically(self.update, 60 * 30)
def test_reponse_with_data_and_status_code(self):
response_data = dict(
complex_response=dict(something='good', something_else='great'))
class MyHandler(BaseHandler):
"""
Handler which returns a response w/ both data and a status code (201)
"""
allowed_methods = ('POST', )
def create(self, request):
resp = rc.CREATED
resp.content = response_data
return resp
resource = Resource(MyHandler)
request = HttpRequest()
request.method = 'POST'
response = resource(request, emitter_format='json')
self.assertEquals(201, response.status_code)
self.assertTrue(isinstance(response.content, str),
"Expected response content to be a string")
# compare the original data dict with the json response
# converted to a dict
self.assertEquals(response_data, simplejson.loads(response.content))
def run(self):
"""
Get doublicates of article of sameas webservice
create datasources with resources
updates resources (download content, save content to disk if it is new or was updated)
"""
self.done = 0
directoryBaseURL = self.config['directoryURL']
dbPediaURL = self.config['dbPediaURL']
directoryURL = "%s%s%s" % (directoryBaseURL, dbPediaURL, self.article)
page = json.load(urllib2.urlopen(directoryURL))
duplicates = page[0]["duplicates"]
self.total = len(duplicates)
# create resources and append resources to datasources
for url in duplicates:
#DEBUG only list freebase and geonames
if True or "freebase" in url or "geonames" in url:
resource = Resource(url)
if resource.domain not in self.datasources:
datasource = Datasource(resource.domain, self.lastdate)
self.datasources[resource.domain] = datasource
datasource.resources.append(resource)
# update datasources, dublicate detection, creation of json
for domain, datasource in self.datasources.iteritems():
if not self._stop.is_set(): #do not proceed if stop is set
datasource.update()
self.done += 1
self.completed = 1
self.callback(self.datasources)
def add_capability_list(self,capability_list=None):
"""Add a capability list
Adds either a CapabiltyList object specified in capability_list
or else creates a Resource with the URI given in capability_list
and adds that to the Source Description
"""
if (hasattr(capability_list,'uri')):
r = Resource( uri=capability_list.uri,
capability=capability_list.capability_name )
if (capability_list.describedby is not None):
r.link_set( rel='describedby', href=capability_list.describedby )
else:
r = Resource( uri=capability_list,
capability='capabilitylist')
self.add( r )
def get_resources(self, limit=None, owner=None):
"""
Gets a list of all resources. Warning: This takes a long time to complete
:return: Resource[]
"""
resources = list()
start_time = time.time()
logging.info(
"Grabbing {limit} resources from www.hydroshare.org:".format(
limit=limit if limit else ''))
for resource_json in self.client.resources(owner=owner):
if isinstance(limit, int):
limit -= 1
elif isinstance(limit, int) and limit < 1:
break
logging.info(json.dumps(resource_json))
resource = Resource(self.client,
raw=resource_json,
**resource_json)
resources.append(resource)
elapsed_time = time.time() - start_time
et_string = time.strftime("%H:%M:%S", time.gmtime(elapsed_time))
logging.info("Total time to get HydroShare resources: {time}".format(
time=et_string))
return resources
def __init__(self, filterchain_name, serialize=None,
default_media_name=None):
# to limit the infini import, we import in the init
from resource import Resource
self.resource = Resource()
self.filters = []
# {"filter_name":[observator,]}
self.image_observers = {}
self.filter_output_observers = []
self.filterchain_name = filterchain_name
self.original_image_observer = []
self.dct_global_param = {}
self.dct_media_param = {}
# If starting filterchain with empty media_name, we take the default
# media
self.default_media_name = default_media_name
if serialize:
self.deserialize(filterchain_name, serialize)
else:
# add default filter
self.add_filter(Filter(keys.get_empty_filter_name()))
self.do_configure()
def handler(is_running, request_q, response_q):
resource_map = {
'user': UserResource,
'session': SessionResource,
'tag': TagResource,
}
while True and is_running.value:
try:
# read one incoming request
try:
request = request_q.get(block=True, timeout=0.001)
except:
continue
# process incoming request and generate response
http_request = Request(request['raw'])
if http_request.path and http_request.path[0] in resource_map:
resource = resource_map[http_request.path[0]](http_request)
else:
resource = Resource(http_request)
response = resource.response.raw()
# send back response
response_q.put({'id': request['id'], 'raw': response})
except KeyboardInterrupt:
pass
def __init__(self):
"""
Structure of dct_execution
{"execution_name" : {KEY_FILTERCHAIN : ref, KEY_MEDIA : ref}}
"""
self.dct_exec = {}
self.config = Configuration()
self.resource = Resource()
# all record history, contains:
# {"time": ..., "media_name": ..., "path": ...}
self.lst_record_historic = []
self.count_keys = defaultdict(int)
self._is_keep_alive_media = self.config.get_is_keep_alive_media()
self.old_rec_dir_path = self.config.get_path_save_record()
# tcp server for output observer
self.nb_observer_client = 0
self.server_observer = Server()
self.server_observer.start("", 5030)
self.notify_event_client = {}
self.id_client_notify = 0
self.publisher = self.resource.get_publisher()
# launch command on start
thread.start_new_thread(self.config.get_dct_cmd_on_start(), (self,))
class TestResource(object):
def setup(self):
self.resource = Resource('/tmp/a', dummy_compute_digest)
def teardown(self):
print ("Resource teardown")
def test_get(self):
self.resource.compute_digest()
result = self.resource.get()
assert result[1] == '/tmp/a'
assert result[0] == 'digest'
def test_to_string(self):
self.resource.compute_digest()
result = str(self.resource)
assert result == '/tmp/a --> digest'
def __iter__(self):
"""
Implements a generator to process one file at a time.
:return:
"""
for f in self.path.glob('**/*'):
if f.is_file() and not os.stat(str(f.resolve())).st_size == 0:
yield Resource(str(f.resolve()), DiskCrawler.compute_digest)
def add_changed_resources(self, resources, change=None):
"""Add items from a ResourceContainer resources to this ChangeList
If change is specified then the attribute is set in the Resource
objects created.
"""
for resource in resources:
rc = Resource( resource=resource, change=change )
self.add(rc)
def __init__(self):
super().__init__()
self.image = Resource.getInstance().get_image('temp/menu1.png')
self.image = pg.transform.smoothscale(
self.image, (int(self.image.get_width() * 0.45),
int(self.image.get_height() * 0.45))).convert()
self.rect = self.image.get_rect()
self.dirty = 1
def scheduling(cls, pi, t, ti): # 任务pi,到达时间t，系统时间ti
"""调度单个任务"""
# 获取任务所需的系统资源
Algorithm.get_pi_resource(pi)
if Algorithm.judge(t, ti):
Resource.allocate_res(Algorithm.ri1, Algorithm.ri2,
Algorithm.ri3) # 分配系统资源给任务pi
i = Task.find_pi(Task.waiting_queues, pi)[1]
print("Scheduling task: ", Task.waiting_queues[i])
Task.add_running_pi(pi, ti) # 将任务pi加入到运行队列
Task.slow_down.append(Task.waiting_queues[i][7]) # 将减慢比加入列表
average_slow_down = round(
sum(Task.slow_down) / len(Task.slow_down), 2) # 计算平均减慢比
Task.average_slow_down.append(average_slow_down) # 加入列表
Task.del_waiting_pi(pi) # 将任务pi从等待队列中剔除
