def traverse(dct, parent=None): resources = {} for name, res in dct.items(): host = url_to_host(self.base_url) apidoc_url = urljoin(host, res.get('apidoc')) schema_url = urljoin(host, res.get('schema')) resource = Resource( self, name, res.get('relativePath'), schema_url, apidoc_url, parent=parent, ) self._resources[name] = resource resources[name] = resource child_resources = {} for att, val in res.items(): if att == 'children': child_resources.update(traverse(val, parent=resource)) resource.children = child_resources return resources
def __init__(self, webdriver, base_url, components, **env): """ :Description: Controller for managing components. :param webdriver: Webdriver for controller and components to reference. :type webdriver: WebDriver :param base_url: Base url for navigations, will navigate to this url in init. :type base_url: basestring :param components: Component objects to instantiate. :type components: tuple, list, dict :param env: Key value pairs to pass to instantiated components. :type env: **kwargs => dict """ self.webdriver = self.__patch_webdriver(webdriver) self.js = E2EJS(webdriver) self.base_url = base_url self.logger = logger if not isinstance(components, (tuple, list, dict)): raise TypeError('Components must be either a tuple, list, or dictionary') self.env = Resource(**env) if env else Resource() if isinstance(components, dict): self.components = Resource(**{ name: component(webdriver=self.webdriver, logger=self.logger, env=self.env)} for name, component in components.iteritems()) else: self.components = [ component(webdriver=self.webdriver, logger=self.logger, env=self.env) for component in components] self.webdriver.get(self.base_url)
def __init__(self, filterchain_name, serialize=None, default_media_name=None): # to limit the infini import, we import in the init from resource import Resource self.resource = Resource() self.filters = [] # {"filter_name":[observator,]} self.image_observers = {} self.filter_output_observers = [] self.filterchain_name = filterchain_name self.original_image_observer = [] self.dct_global_param = {} self.dct_media_param = {} # If starting filterchain with empty media_name, we take the default # media self.default_media_name = default_media_name if serialize: self.deserialize(filterchain_name, serialize) else: # add default filter self.add_filter(Filter(keys.get_empty_filter_name())) self.do_configure()
def test_02_request_package_name(self): """ 2: Test request_package_name method from Resource class Check if returns a value, which is not NULL """ resource = Resource(self.testid) resource.unpack_object_to_self(resource.load_from_ckan()) package_name = resource.request_package_name() self.failUnless(package_name, 'package name is NULL!')
def test_should_load_resource(self): content = "Content" web = TestWeb() web.response = FakeResponse(content, "text/html") resource = Resource("Link", "Target", web) loaded_content = resource.load() self.assertEqual(loaded_content, content)
def __init__(self, request): """Box""" resource_provider = ResourceProvider() resource = Resource(resource_provider) self.charts = [] self.chart_type = None self.layout = Layout(request) self.body = '' self.javascript = '' self.chart_name = request.matchdict['box_name'] chart_format = os.path.splitext(request.environ['PATH_INFO'])[1] # Go through the registry, and find the resource for this box # XXX This should be a dictionary for res in RESOURCES_REGISTRY: if res[0] == self.chart_name: self.resources = [res] if chart_format == '.html': self.render_html(request) elif chart_format == '.csv': self.body = resource.get(self.chart_name, CSV, request.matchdict) elif chart_format == '.json': self.body = resource.get(self.chart_name, JSON, request.matchdict) else: print "Format not supported %s" % chart_format raise AttributeError
def initialize(self): ''' Restore system to its initial state and create Process 0 Returns process 0 ''' self.PCB = [None] * 16 self.RCB = [None] * 4 #self.ready_list = deque() self.ready_list = { 2: deque(), 1: deque(), 0: deque() } #### WHERE YOU LAST LEFT OFF 10.28.19 5:20 AM ## Creating Process 0 and placing it into Ready List self.PCB[0] = Process(0, 0, 1, 0) ## Process(id/index, state, parent) self.active_processes += 1 self.ready_list[0].append(self.PCB[0].id) self.run_proc = self.PCB[self.ready_list[0][0]] #print(self.run_proc.id) ## Set up all Resources self.RCB[0] = Resource(0, 1) self.RCB[1] = Resource(1, 1) self.RCB[2] = Resource(2, 2) self.RCB[3] = Resource(3, 3) return 0
def traverse(dct, parent=None): resources = {} for name, res in dct.items(): host = url_to_host(self.base_url) apidoc_url = urljoin(host, res.get('apidoc')) schema_url = urljoin(host, res.get('schema')) resource = Resource( self, name, res.get('relativePath'), schema_url, apidoc_url, parent=parent, ) self._resources[name] = resource resources[name] = resource child_resources = {} for att, val in res.items(): if att == 'children': child_resources.update( traverse(val, parent=resource) ) resource.children = child_resources return resources
def assemble(section_name, items): # 将配置文件的数据组装成对象 resource = Resource() resource.flag = section_name for item in items: setattr(resource, item[0], item[1]) return resource
def judge(cls, t, ti): # 到达时间t,系统时间ti """时间和资源判断""" r_cpu = Resource.get_res()[0] r_mem = Resource.get_res()[1] r_io = Resource.get_res()[2] if t <= ti and Algorithm.ri1 <= r_cpu and Algorithm.ri2 <= r_mem and Algorithm.ri3 <= r_io: return True
def add_file(self, resource_list=None, dir=None, file=None): """Add a single file to resource_list Follows object settings of set_path, set_md5 and set_length. """ try: if self.exclude_file(file): self.logger.debug("Excluding file %s" % (file)) return # get abs filename and also URL if (dir is not None): file = os.path.join(dir,file) if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))): return uri = self.mapper.dst_to_src(file) if (uri is None): raise Exception("Internal error, mapping failed") file_stat=os.stat(file) except OSError as e: sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e))) return timestamp = file_stat.st_mtime #UTC r = Resource(uri=uri,timestamp=timestamp) if (self.set_path): # add full local path r.path=file if (self.set_md5): # add md5 r.md5=compute_md5_for_file(file) if (self.set_length): # add length r.length=file_stat.st_size resource_list.add(r)
def upload_new_resources(new_resources_df: pd.DataFrame, firestore_resources: Dict[Resource, str], db, sheet: gspread.models.Worksheet) -> List[int]: length = len(new_resources_df.index) log(f"{length} resources to upload") added = 0 uploaded_rows = list() for index, row in new_resources_df.iterrows(): links = Links(row["card link"], row["website"]) resource = Resource(row["resource name"], True, row["description"], row["image link"], row["category"], row["tags"].split(", "), links) try: if resource not in firestore_resources: db.collection(FIREBASE_COLLECTION).add(resource.to_dict()) log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}") else: db.collection(FIREBASE_COLLECTION).document( firestore_resources[resource]).set(resource.to_dict()) log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}" ) except: log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully" ) return uploaded_rows added += 1 uploaded_rows.append(index + 1) log(f"\nAdded {added} / {length} entries to Firestore") return uploaded_rows
def import_ds(ds, type=None, parent_id=None): if type == 'parent': ds['is_parent'] = 'true' else: ds.pop('is_parent', None) if type == 'child': ds['parent_dataset'] = parent_id else: ds.pop('parent_dataset', None) dataset_dummy = create_dummy_dataset() dataset_dummy['title'] = ds['title'] dataset_dummy['owner_org'] = owner_org # first run to get name created. ds_created = dataset_dummy.create(create_url, api_key) # then update the dataset with all info dataset_full = load_dataset(ds_created) map_dataset(dataset_full, ds) dataset_full._update(update_url, api_key) # add resource resources = ds.get('distribution', []) for res in resources: resource = Resource() map_resource(resource, res, dataset_full['id']) # skip and report empty resource if resource['url']: res_created = resource.create(resource_url, api_key) else: logging.info(' Empty resource skipped for: %s' % ds['title']) return dataset_full
def __init__(self, server_host, server_port=None, username="******", password="******", use_tls=False, version=API_CURRENT_VERSION): """ Creates a Resource object that provides API endpoints. @param server_host: The hostname of the Cloudera Manager server. @param server_port: The port of the server. Defaults to 7180 (http) or 7183 (https). @param username: Login name. @param password: Login password. @param use_tls: Whether to use tls (https). @param version: API version. @return: Resource object referring to the root. """ self._version = version protocol = use_tls and "https" or "http" if server_port is None: server_port = use_tls and 7183 or 7180 base_url = "%s://%s:%s/api/v%s" % \ (protocol, server_host, server_port, version) client = HttpClient(base_url, exc_class=ApiException) client.set_basic_auth(username, password, API_AUTH_REALM) client.set_headers({"Content-Type": "application/json"}) Resource.__init__(self, client)
def add_file(self, resource_list=None, dir=None, file=None): """Add a single file to resource_list Follows object settings of set_path, set_md5 and set_length. """ try: if self.exclude_file(file): self.logger.debug("Excluding file %s" % (file)) return # get abs filename and also URL if (dir is not None): file = os.path.join(dir, file) if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))): return uri = self.mapper.dst_to_src(file) if (uri is None): raise Exception("Internal error, mapping failed") file_stat = os.stat(file) except OSError as e: sys.stderr.write("Ignoring file %s (error: %s)" % (file, str(e))) return timestamp = file_stat.st_mtime #UTC r = Resource(uri=uri, timestamp=timestamp) if (self.set_path): # add full local path r.path = file if (self.set_md5): # add md5 r.md5 = compute_md5_for_file(file) if (self.set_length): # add length r.length = file_stat.st_size resource_list.add(r)
def get_user_resource_tags(username, collectionid, resourceid): rsrc = Resource() rsrc.create() result = rsrc.get(username, collectionid, resourceid) if not result: return [{'No tags'}] else: return [{'TagText': result[0]['tags']}]
def __init__(self, manager, id): Resource.__init__(self, manager, id) self.data = None self.size = None self.hash = None self.controllers = [] self.remaining_controllers = None logging.debug("New buffer: %s", self)
def test_08_load_mapping(self): """ 8: Test fetching the wiki page and parsing together """ resource = Resource(self.testid) resource.wiki_site = wikitools.Wiki(config.wiki_api_url) resource.wiki_site.login(config.wiki_username, password=config.wiki_password) mappings = resource.load_mappings() self.failUnless(mappings, 'No mappings exists!')
def test_06_request_wiki_page(self): """ 6: Test if wiki page for this Resource exists """ resource = Resource(self.testid) resource.wiki_site = wikitools.Wiki(config.wiki_api_url) resource.wiki_site.login(config.wiki_username, password=config.wiki_password) wiki_page = resource._request_wiki_page() self.failUnless(wiki_page, "Wiki page does not exist or empty!")
def __init__(self, request, cells): resource_provider = ResourceProvider() self.resource = Resource(resource_provider) self.cells = cells self.resources = self.get_resources() self.charts = self.get_charts(request) self.packages = self.get_packages() self.javascript = render_javascript(self.charts, self.packages)
def print(self): print("Peer info: id: " + self.PEER + ", IP: " + self.PORT) print("Max download rate: " + self.download_rate() + "b/s") print("Max upload rate: " + self.upload_rate() + "b/s") print("*** Downloads in progress ***") print("-File " + Resource.getFileName(filePath) + "total downloaded: ") print("*** Uploads in progress ***") print("-File " + Resource.getFileName(filePath) + "total uploaded: ")
def roll_text(self, dt): if self.text_y < ((self.surface.get_height()) * -1): self.state = 1 Resource.getInstance().play_sound('sounds/boot/b.ogg') pg.time.delay(500) else: self.text_y -= self.velocity * dt pass
class DistributionTestCase(unittest.TestCase): def setUp(self): self.job = Job() self.batch = Batch() self.resource = Resource() rootDir = os.environ['BOLT_DIR'] self.batch.readConfig(rootDir + configDir + "/" + batchConfig) self.resource.readConfig(rootDir + configDir + "/" + resourceConfig) def testParallelTaskDitributionPureMPI(self): """Pure MPI task distribution (fully populated).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(self.resource.numCoresPerNode()) self.job.setThreads(1) self.job.setParallelDistribution(self.resource, self.batch) correct = "aprun -n 1024 -N 32 -S 8 -d 1" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct)) def testParallelTaskDitributionHalfPopulate(self): """Pure MPI task distribution (half populated).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(16) self.job.setThreads(1) self.job.setParallelDistribution(self.resource, self.batch) correct = "aprun -n 1024 -N 16 -S 4 -d 2" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct)) def testParallelTaskDitributionTwoThreads(self): """Hybrid MPI/OpenMP task distribution (2 OpenMP threads).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(16) self.job.setThreads(2) self.job.setParallelDistribution(self.resource, self.batch) correct = "export OMP_NUM_THREADS=2\naprun -n 1024 -N 16 -S 4 -d 2" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct)) def testParallelTaskDitributionThreeThreads(self): """Hybrid MPI/OpenMP task distribution (3 OpenMP threads).""" # Set the parallel distribution self.job.setTasks(1024) self.job.setTasksPerNode(10) self.job.setThreads(3) self.job.setParallelDistribution(self.resource, self.batch) correct = "export OMP_NUM_THREADS=3\naprun -n 1024 -N 10 -d 3" self.assertEqual(self.job.runLine, correct, "Value= '{0}', Expected= '{1}'".format(self.job.runLine, correct))
def _make_ac(self, chara, name): values = RM.res('ac') label = ttk.Label(self, text=RM.res(name)) i = chara.get_ac(name) value = self._get_value(values, i) self._ac[name] = tk.StringVar(value=value) combobox = ttk.Combobox(self, values=values, textvariable=self._ac[name], state='readonly',width=10) return (label, combobox)
def put_resource(data): rsrc = Resource(True) if 'resourceid' in data: rsrcid = data['resourceid'] else: rsrcid = str(time.time()) # the last param None is the photo data which we will fill in later rsrc.put(data['user'], data['collectionid'], data['description'], rsrcid, None) return rsrcid
def test_01_load_from_ckan (self): """ 1: Test load_from_ckan method from Resource class Check if attributes of the Resource object fetched from CKAN exist: -- revision_id - used for fetching package_id -- url - used for downloading the data file """ resource = Resource(self.testid) object = resource.load_from_ckan() self.failUnless('url' in object, 'Tested object has no url attribute.') self.failUnless('revision_id' in object, 'Tested object has no revision_id attribute.')
def get(self): self.response.headers['Content-Type'] = 'text/html' resources = Resource.getResources() resources = Resource.getResourcesFor6Through8(resources) values = { "title": "Middle School", "curr_url": self.request.url, } self.response.out.write(template.render('html/middle.html', values))
class KBO(): def __init__(self, isSimulation=False, isAPI=False): if isAPI: print("====================As API====================") from Api.api import API self.resource = Resource() self.web = Web(resource=self.resource) self.vision = Vision(resource=self.resource) #self.run_server(host="166.104.143.103", port=8080) self.run_server() elif isSimulation: print("====================Simulation====================") self.run_bbox() else: print("====================In Local====================") self.resource = Resource() self.web = Web(resource=self.resource) self.vision = Vision(resource=self.resource) self.tts = TTS(resource=self.resource) self.run() def run(self): self.resource.set_frameno(START_FRAME + 1) idx = self.web.parsing_before() web_thread = threading.Thread(target=self.web.parsing_relaytext, args=(idx, )) web_thread.start() vision_thread = threading.Thread(target=self.vision.play) vision_thread.start() tts = threading.Thread(target=self.tts.text_2_speech) tts.start() play(self.resource) def run_bbox(self): play_bbox(frameno=128400) def run_server(self): self.resource.set_frameno(START_FRAME + 1) idx = self.web.parsing_before() web_thread = threading.Thread(target=self.web.parsing_relaytext, args=(idx, True)) web_thread.start() vision_thread = threading.Thread(target=self.vision.play) vision_thread.start() play_API(self.resource, host="169.254.17.149", port=8080)
def test_04_get_wiki_url(self): """ 4: Test if get_wiki_url method from Resource class returns URI string. The result depends on config.py file.""" resource = Resource(self.testid) parsed_url = urlparse(resource.get_wiki_url()) self.failUnless(parsed_url.scheme, "Scheme is missing.") self.failUnless(parsed_url.netloc, "Net Location (e.g. publicdata.eu) is missing.") self.failUnless(parsed_url.path, "Path is missing.")
def test_should_update_the_resource_when_it_attempts_a_redirect(self): web = TestWeb() url = "http://localhost:5000/redirect" web.head = self.redirect resource = Resource('Page', 'http://localhost:5000', web) resource.isAPage() web.head = lambda x: FakeResponse('Content', 'application/pdf') self.assertEqual(resource.name, "redirect.pdf") self.assertEqual(resource.url, "http://localhost:5000/redirect.pdf")
def get(self): self.response.headers['Content-Type'] = 'text/html' resources = Resource.getResources() resources = Resource.getResourcesFor9Through12(resources) values = { "title": "High School", "curr_url": self.request.url, "resources": resources } self.response.out.write(template.render('html/highschool.html', values))
def upload_new_resources(new_resources_df: pd.DataFrame, firestore_resources: Dict[Resource, str], db, sheet: gspread.models.Worksheet) -> List[int]: length = len(new_resources_df.index) log(f"{length} resources to upload") added = 0 uploaded_rows = list() for index, row in new_resources_df.iterrows(): links = Links(row["card link"], row["website"]) date_created = datetime.now().strftime("%Y/%m/%d %H:%M:%S") resource = Resource(title=row["resource name"], reviewed=True, want_support_with=row["want support with"], this_resource_offers=row["this resource offers"], description=row["description"], img=row["image link"], category=row["category"], tags=row["tags"].split(", "), links=links, date_created=date_created, ranking=row["ranking"]) try: category_document = db.collection(FIREBASE_COLLECTION).document( resource.category.replace("/ ", "_")) if resource not in firestore_resources: category_document.update({ "resource_list": firestore.ArrayUnion([resource.title]) }) # Update resource list category_document.update( {"tag_list": firestore.ArrayUnion(resource.tags)}) # Update tag list category_document.collection("resources").add( resource.to_dict()) # Add new document to collection log(f"\tAdded {row['resource name']} to {FIREBASE_COLLECTION}/{category_document.id}" ) else: resource._date_created = category_document.collection( "resources").document(firestore_resources[resource]).get( ).to_dict()["dateCreated"] category_document.collection("resources").document( firestore_resources[resource]).set(resource.to_dict( )) # Update old document in collection log(f"\tUpdated {row['resource name']} in {FIREBASE_COLLECTION}/{category_document.id}" ) except Exception as e: log(f"Error uploading data to firestore. {added} / {length} resources uploaded successfully" ) print(e) return uploaded_rows added += 1 uploaded_rows.append(index + 1) log(f"\nAdded {added} / {length} entries to Firestore") return uploaded_rows
def get(self): self.response.headers['Content-Type'] = 'text/html' resources = Resource.getResources() resources = Resource.getResourcesForCollege(resources) values = { "title": "College", "scholarship": self.request.url + "/scholarship", "curr_url": self.request.url, "resources": resources } self.response.out.write(template.render('html/college.html', values))
def release_task_res(pi): """查找任务pi,并释放其分配的系统资源""" for i in range(len(news_waiting_queues)): if news_waiting_queues[i][0] == pi: r_cpu = news_waiting_queues[i][1] # 查看原始信息 r_mem = news_waiting_queues[i][2] r_io = news_waiting_queues[i][3] Resource.release_res(r_cpu, r_mem, r_io) # 释放资源 break
def from_disk(self, path, url_prefix, inventory=None): """Create or extend inventory with resources from disk scan Assumes very simple disk path to URL mapping: chop path and replace with url_path. Returns the new or extended Inventory object. If a inventory is specified then items are added to that rather than creating a new one. mb = InventoryBuilder() m = inventory_from_disk('/path/to/files','http://example.org/path') """ num = 0 # Either use inventory passed in or make a new one if (inventory is None): inventory = Inventory() # for each file: create Resource object, add, increment counter for dirpath, dirs, files in os.walk(path, topdown=True): for file_in_dirpath in files: try: if self.exclude_file(file_in_dirpath): continue # get abs filename and also URL file = os.path.join(dirpath, file_in_dirpath) if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))): continue rel_path = os.path.relpath(file, start=path) if (os.sep != '/'): # if directory path sep isn't / then translate for URI rel_path = rel_path.replace(os.sep, '/') url = url_prefix + '/' + rel_path file_stat = os.stat(file) except OSError as e: sys.stderr.write("Ignoring file %s (error: %s)" % (file, str(e))) continue mtime = file_stat.st_mtime lastmod = datetime.fromtimestamp(mtime).isoformat() r = Resource(uri=url, lastmod=lastmod) if (self.do_md5): # add md5 r.md5 = compute_md5_for_file(file) if (self.do_size): # add size r.size = file_stat.st_size inventory.add(r) # prune list of dirs based on self.exclude_dirs for exclude in self.exclude_dirs: if exclude in dirs: dirs.remove(exclude) return (inventory)
def __init__(self, config): """ param: """ self.config = config self.exec_times = 0 self.trigger_time = 0 self.name = self.config['main']['name'] self.__state = EngineState_Init self.engine_instance_vars = dict() Resource.initialize_local_resources(self.name, self.config)
def get_user_resource_details(username, collectionid, resourceid): rsrc = Resource() rsrc.create() result = rsrc.get(username, collectionid, resourceid) if not result: return {'Not found'} else: return { 'FileSource': result[0]['hires_url'], 'Description' : 'Powered by PhotoStore Technology', 'Location': 'Palo Alto' }
def post(self): lang = self.request.get("lang") resources = Resource.getResources() if lang: resources = Resource.getResourcesForSpanish(resources, True) resources = Resource.getResourcesFor9Through12(resources) values = { "title": "High School", "curr_url": self.request.url, "resources": resources } self.response.out.write(template.render('html/highschool.html', values))
def get_resource(data): if not data: return {'NOTFOUND'} rsrc = Resource(True) if 'collectionid' in data: collectionid = data['collectionid'] else: collectionid = None if 'resourceid' in data: resourceid = data['resourceid'] else: resourceid = None return rsrc.get(data['user'], collectionid, resourceid)
def setUp(self): self._resource1 = Resource('test_resource1', [ Limit(2, timedelta(seconds=1)), Limit(3, timedelta(seconds=1)), ]) self._resource2 = Resource('test_resource2', [ Limit(6, timedelta(seconds=1)), Limit(7, timedelta(seconds=1)), ]) self._resource3 = Resource('test_resource3', [ Limit(10, timedelta(seconds=1)), Limit(10, timedelta(seconds=1)), ])
def post(self): lang = self.request.get("lang") resources = Resource.getResources() if lang: resources = Resource.getResourcesForSpanish(resources, True) resources = Resource.getResourcesForCollege(resources) values = { "title": "College", "scholarship": self.request.url + "/scholarship", "curr_url": self.request.url, "resources": resources } self.response.out.write(template.render('html/college.html', values))
def from_disk(self,path,url_prefix,inventory=None): """Create or extend inventory with resources from disk scan Assumes very simple disk path to URL mapping: chop path and replace with url_path. Returns the new or extended Inventory object. If a inventory is specified then items are added to that rather than creating a new one. mb = InventoryBuilder() m = inventory_from_disk('/path/to/files','http://example.org/path') """ num=0 # Either use inventory passed in or make a new one if (inventory is None): inventory = Inventory() # for each file: create Resource object, add, increment counter for dirpath, dirs, files in os.walk(path,topdown=True): for file_in_dirpath in files: try: if self.exclude_file(file_in_dirpath): continue # get abs filename and also URL file = os.path.join(dirpath,file_in_dirpath) if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))): continue rel_path=os.path.relpath(file,start=path) if (os.sep != '/'): # if directory path sep isn't / then translate for URI rel_path=rel_path.replace(os.sep,'/') url = url_prefix+'/'+rel_path file_stat=os.stat(file) except OSError as e: sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e))) continue mtime = file_stat.st_mtime lastmod = datetime.fromtimestamp(mtime).isoformat() r = Resource(uri=url,lastmod=lastmod) if (self.do_md5): # add md5 r.md5=compute_md5_for_file(file) if (self.do_size): # add size r.size=file_stat.st_size inventory.add(r) # prune list of dirs based on self.exclude_dirs for exclude in self.exclude_dirs: if exclude in dirs: dirs.remove(exclude) return(inventory)
def test_07_extract_csv_configurations(self): """ 7: Test the parser for CSV configurations on Wiki page """ resource = Resource(self.testid) resource.wiki_site = wikitools.Wiki(config.wiki_api_url) resource.wiki_site.login(config.wiki_username, password=config.wiki_password) wiki_page = resource._request_wiki_page() mappings = resource._extract_csv_mappings(wiki_page) for mapping in mappings: self.failUnless(mapping['type'] == 'RelCSV2RDF', 'Wrong mapping type, should be RelCSV2RDF.') self.failUnless(mapping['name'], 'Name of mapping is undefined!') self.failUnless(mapping['header'], 'Header line is not specified') self.failUnless(mapping['omitCols'], 'omitCols is not specified') self.failUnless(mapping['omitRows'], 'omitRows is not specified')
def create_resources_table(): resources_list = [ Resource( 'http://www.ynet.co.il/articles/0,7340,L-4713571,00.html', 'https://images1.ynet.co.il/PicServer4/2014/08/05/5506384/52203970100690640360no.jpg', 'החוש הדומיננטי שיעזור לכם בלימודים', 'החוש הדומיננטי שיעזור לכם בלימודים. אילו טיפים של שימושבחושים יעזרו לכם?' ), Resource( 'http://www.ynet.co.il/articles/0,7340,L-5045541,00.html', 'https://images1.ynet.co.il/PicServer5/2017/11/23/8172884/817287001000100980704no.jpg', '"כ"ט בנובמבר: "שמחה שנמשכה ימים ולילות, הייתה אופוריה"', 'ב1947- הם היו ילדים או צעירים בתחילת דרכם,' + ' אבל את היום הגורלי ב29- בנובמבר הם לא שוכחים עד היום.' + ' "כולם היו צמודים לרדיו. אני זוכרת את התפרצות השמחה, ריקודים והתחבקויות."' ), Resource( 'https://www.calcalist.co.il/world/articles/0,7340,L-3726321,00.html', 'https://images1.calcalist.co.il/PicServer3/2017/11/30/775736/2_l.jpg', 'רוצים נייר טואלט? הזדהו: כך משפרים הסינים את מצב השירותים הציבוריים', 'שבוע קרא נשיא סין שי ג‘ינפינג להמשיך את מהפכת השירותים' + ' הציבוריים עליה הכריז ב-2015. עד כה שופצו ונבנו 68 אלף מתקנים'), Resource( 'http://www.nrg.co.il/online/13/ART2/902/962.html', 'http://www.nrg.co.il/images/archive/465x349/1/646/416.jpg', 'מחקו לכם הודעה בווטסאפ? עדיין תוכלו לקרוא אותה', 'אפליקציה בשם Notification History מאפשרת למשתמשי אנדרואיד' + ' לקורא את הנתונים הזמניים הנשמרים ביומן הפעילות של הסמארטפון. כולל הודעות מחוקות.' ), Resource( 'http://www.nrg.co.il/online/55/ART2/904/542.html', 'http://www.nrg.co.il/images/archive/465x349/1/795/429.jpg', 'גם בחורף: זה בדיוק הזמן לקפוץ לאילת', 'העיר הדרומית נעימה לנופש גם בחודשי החורף.' + ' כעת מוצעים מחירים אטרקטיביים במיוחד בחבילות שכוללות מגוון אטרקציות, לינה וטיסות' ), Resource( 'https://food.walla.co.il/item/3113079', 'https://img.wcdn.co.il/f_auto,w_700/2/5/1/3/2513314-46.jpg', '12 בתי קפה שמתאימים לעבודה עם לפטופ', 'בין אם אתם סטודנטים או עצמאיים, זה תמיד סיפור למצוא בית קפה נעים וטעים לרבוץ בו.' + ' קיבצנו עבורכם 12 מקומות אהובים בדיוק למטרה זו, בארבע הערים הגדולות' ), Resource( 'https://news.walla.co.il/item/3114145', 'https://img.wcdn.co.il/f_auto,w_700/2/4/9/5/2495334-46.jpg', 'שותק על אזריה, נלחם באהוד ברק: בנט מנסה להיבנות כימין ממלכתי', 'כשרגב נלחמת ברעש בתאטרון יפו, בנט משנה בשקט את נהלי סל התרבות כך ' + 'שהחומרים "השמאלנים" ייפלטו. כשהקשת הפוליטית מתרעמת על דיווחי' + ' ה"דיל" של טראמפ עם הפלסטינים, בנט שותק עד שהרשות תסרב.'), Resource( 'https://news.walla.co.il/item/3114283', 'https://img.wcdn.co.il/f_auto,w_700/2/5/1/4/2514588-46.jpg', 'רצח בכל שלושה ימים: צרפת יוצאת למאבק באלימות נגד נשים', 'אחרי ש126- נשים נרצחו בידי בני זוגן בשנה שעברה, ' + 'הציג מקרון צעדים חדשים למלחמה בתופעה. "זאת בושה לצרפת,"' + ' אמר הנשיא שאחת מהבטחות הבחירות שלו הייתה להשיג שוויון מגדרי.') ] return resources_list
def from_disk_add_map(self, resource_list=None, map=None, set_path=False): """Add to resource_list with resources from disk scan based one map If set_path is True then the path attribue will be set with the local path for each Resource. """ # sanity if (resource_list is None or map is None): raise ValueError("Must specify resource_list and map") path=map.dst_path #print "walking: %s" % (path) # for each file: create Resource object, add, increment counter num_files=0 for dirpath, dirs, files in os.walk(path,topdown=True): for file_in_dirpath in files: num_files+=1 if (num_files%50000 == 0): self.logger.info("ResourceListBuilder.from_disk_add_map: %d files..." % (num_files)) try: if self.exclude_file(file_in_dirpath): self.logger.debug("Excluding file %s" % (file_in_dirpath)) continue # get abs filename and also URL file = os.path.join(dirpath,file_in_dirpath) if (not os.path.isfile(file) or not (self.include_symlinks or not os.path.islink(file))): continue uri = map.dst_to_src(file) if (uri is None): raise Exception("Internal error, mapping failed") file_stat=os.stat(file) except OSError as e: sys.stderr.write("Ignoring file %s (error: %s)" % (file,str(e))) continue timestamp = file_stat.st_mtime #UTC r = Resource(uri=uri,timestamp=timestamp) if (set_path): r.path=file if (self.do_md5): # add md5 r.md5=compute_md5_for_file(file) if (self.do_length): # add length r.length=file_stat.st_size resource_list.add(r) # prune list of dirs based on self.exclude_dirs for exclude in self.exclude_dirs: if exclude in dirs: self.logger.debug("Excluding dir %s" % (exclude)) dirs.remove(exclude)
def to_dict(self): data = Resource.to_dict(self) print 'Creating Dictionary for JSON output.' if self.asset: for k, v in self.asset.to_dict().iteritems(): data['asset_' + k] = v return data
def init(self): self.path = os.path.abspath(self.path) if not os.path.isdir(self.path): os.makedirs(self.path) self._threads = [] self._socket = None self._token_map = {} self._paths = set() self.services, services = [], self.services for s in services: self.add_service(s['path'], s) self.resources = Resource.create(self.resources) for service in self.services: logging.info("loading service: %s", service.path) service.deploy() service_path = os.path.join(self.path, 'services') if not os.path.exists(service_path): os.makedirs(service_path) self.periodically(self.scan, 1) self.periodically(self.update, 60 * 30)
def test_reponse_with_data_and_status_code(self): response_data = dict( complex_response=dict(something='good', something_else='great')) class MyHandler(BaseHandler): """ Handler which returns a response w/ both data and a status code (201) """ allowed_methods = ('POST', ) def create(self, request): resp = rc.CREATED resp.content = response_data return resp resource = Resource(MyHandler) request = HttpRequest() request.method = 'POST' response = resource(request, emitter_format='json') self.assertEquals(201, response.status_code) self.assertTrue(isinstance(response.content, str), "Expected response content to be a string") # compare the original data dict with the json response # converted to a dict self.assertEquals(response_data, simplejson.loads(response.content))
def run(self): """ Get doublicates of article of sameas webservice create datasources with resources updates resources (download content, save content to disk if it is new or was updated) """ self.done = 0 directoryBaseURL = self.config['directoryURL'] dbPediaURL = self.config['dbPediaURL'] directoryURL = "%s%s%s" % (directoryBaseURL, dbPediaURL, self.article) page = json.load(urllib2.urlopen(directoryURL)) duplicates = page[0]["duplicates"] self.total = len(duplicates) # create resources and append resources to datasources for url in duplicates: #DEBUG only list freebase and geonames if True or "freebase" in url or "geonames" in url: resource = Resource(url) if resource.domain not in self.datasources: datasource = Datasource(resource.domain, self.lastdate) self.datasources[resource.domain] = datasource datasource.resources.append(resource) # update datasources, dublicate detection, creation of json for domain, datasource in self.datasources.iteritems(): if not self._stop.is_set(): #do not proceed if stop is set datasource.update() self.done += 1 self.completed = 1 self.callback(self.datasources)
def add_capability_list(self,capability_list=None): """Add a capability list Adds either a CapabiltyList object specified in capability_list or else creates a Resource with the URI given in capability_list and adds that to the Source Description """ if (hasattr(capability_list,'uri')): r = Resource( uri=capability_list.uri, capability=capability_list.capability_name ) if (capability_list.describedby is not None): r.link_set( rel='describedby', href=capability_list.describedby ) else: r = Resource( uri=capability_list, capability='capabilitylist') self.add( r )
def get_resources(self, limit=None, owner=None): """ Gets a list of all resources. Warning: This takes a long time to complete :return: Resource[] """ resources = list() start_time = time.time() logging.info( "Grabbing {limit} resources from www.hydroshare.org:".format( limit=limit if limit else '')) for resource_json in self.client.resources(owner=owner): if isinstance(limit, int): limit -= 1 elif isinstance(limit, int) and limit < 1: break logging.info(json.dumps(resource_json)) resource = Resource(self.client, raw=resource_json, **resource_json) resources.append(resource) elapsed_time = time.time() - start_time et_string = time.strftime("%H:%M:%S", time.gmtime(elapsed_time)) logging.info("Total time to get HydroShare resources: {time}".format( time=et_string)) return resources
def handler(is_running, request_q, response_q): resource_map = { 'user': UserResource, 'session': SessionResource, 'tag': TagResource, } while True and is_running.value: try: # read one incoming request try: request = request_q.get(block=True, timeout=0.001) except: continue # process incoming request and generate response http_request = Request(request['raw']) if http_request.path and http_request.path[0] in resource_map: resource = resource_map[http_request.path[0]](http_request) else: resource = Resource(http_request) response = resource.response.raw() # send back response response_q.put({'id': request['id'], 'raw': response}) except KeyboardInterrupt: pass
def __init__(self): """ Structure of dct_execution {"execution_name" : {KEY_FILTERCHAIN : ref, KEY_MEDIA : ref}} """ self.dct_exec = {} self.config = Configuration() self.resource = Resource() # all record history, contains: # {"time": ..., "media_name": ..., "path": ...} self.lst_record_historic = [] self.count_keys = defaultdict(int) self._is_keep_alive_media = self.config.get_is_keep_alive_media() self.old_rec_dir_path = self.config.get_path_save_record() # tcp server for output observer self.nb_observer_client = 0 self.server_observer = Server() self.server_observer.start("", 5030) self.notify_event_client = {} self.id_client_notify = 0 self.publisher = self.resource.get_publisher() # launch command on start thread.start_new_thread(self.config.get_dct_cmd_on_start(), (self,))
class TestResource(object): def setup(self): self.resource = Resource('/tmp/a', dummy_compute_digest) def teardown(self): print ("Resource teardown") def test_get(self): self.resource.compute_digest() result = self.resource.get() assert result[1] == '/tmp/a' assert result[0] == 'digest' def test_to_string(self): self.resource.compute_digest() result = str(self.resource) assert result == '/tmp/a --> digest'
def __iter__(self): """ Implements a generator to process one file at a time. :return: """ for f in self.path.glob('**/*'): if f.is_file() and not os.stat(str(f.resolve())).st_size == 0: yield Resource(str(f.resolve()), DiskCrawler.compute_digest)
def add_changed_resources(self, resources, change=None): """Add items from a ResourceContainer resources to this ChangeList If change is specified then the attribute is set in the Resource objects created. """ for resource in resources: rc = Resource( resource=resource, change=change ) self.add(rc)
def __init__(self): super().__init__() self.image = Resource.getInstance().get_image('temp/menu1.png') self.image = pg.transform.smoothscale( self.image, (int(self.image.get_width() * 0.45), int(self.image.get_height() * 0.45))).convert() self.rect = self.image.get_rect() self.dirty = 1
def scheduling(cls, pi, t, ti): # 任务pi,到达时间t,系统时间ti """调度单个任务""" # 获取任务所需的系统资源 Algorithm.get_pi_resource(pi) if Algorithm.judge(t, ti): Resource.allocate_res(Algorithm.ri1, Algorithm.ri2, Algorithm.ri3) # 分配系统资源给任务pi i = Task.find_pi(Task.waiting_queues, pi)[1] print("Scheduling task: ", Task.waiting_queues[i]) Task.add_running_pi(pi, ti) # 将任务pi加入到运行队列 Task.slow_down.append(Task.waiting_queues[i][7]) # 将减慢比加入列表 average_slow_down = round( sum(Task.slow_down) / len(Task.slow_down), 2) # 计算平均减慢比 Task.average_slow_down.append(average_slow_down) # 加入列表 Task.del_waiting_pi(pi) # 将任务pi从等待队列中剔除