def getCitedInPapers(MyWebdriver, logPath='', warningPath=''): citedInPapers = [] warningInfo = 'Can not get the citedInPapers from this page, for this page has no citedInPapers' tools.warning(warningInfo, warningPath) return citedInPapers
def getAbstract(MyWebdriver, logPath='', warningPath=''): abstract = '' #获取abstract try: MyWebdriver.WebDriverWait_until( 30, lambda x: x.find_element_by_xpath( '/html/body/font/div/font[2]/div')) abstractTag = MyWebdriver.find_element_by_xpath( '/html/body/font/div/font[2]/div') except Exception as e: warningInfo = 'Can not get the abstract from this page\n Failed info: {0}'.format( repr(e)) tools.warning(warningInfo, warningPath) else: abstracts = (abstractTag.text.split('\n')[1:]) if len( abstractTag.text.split('\n')) > 0 else [] for abs in abstracts: abstract += abs successInfo = 'Successfully get the abstract from this page' if abstract == '': successInfo = '!!!Successfully get the abstract from this page, but the abstract is None' tools.log(successInfo, logPath) # print(abstract) return abstract
def __init__(self, name, dict, ast_params): self.access = "rw" self.desc = "" self.hide = False self.init = None self.mandatory = True self.name = name self.owned = True self.serialize = True self.type = "" for key in dict: if not key in [ 'access', 'desc', 'hide', 'init', 'mandatory', 'owned', 'serialize', 'type', ]: warning('unknown Attribute attribute: ' + key + ' from ' + name) self.__dict__[key] = dict[key] self.ast_params = ast_params
def process_dataset(self, workdir, db, url, extractor): '''Extract territories from a given file for a given level with a given extractor function''' loaded = 0 filename = join(workdir, basename(url)) with fiona.open('/', vfs='zip://{0}'.format(filename), encoding='utf8') as collection: info('Extracting {0} elements from {1} ({2} {3})'.format( len(collection), basename(filename), collection.driver, to_string(collection.crs) )) for polygon in collection: try: zone = extractor(polygon) if not zone: continue zone['keys'] = dict((k, v) for k, v in zone.get('keys', {}).items() if v is not None) geom = shape(polygon['geometry']) if extractor.simplify: geom = geom.simplify(extractor.simplify) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': warning('Unsupported geometry type "{0}" for "{1}"'.format(geom.geom_type, zone['name'])) continue zoneid = '/'.join((self.id, zone['code'])) zone.update(_id=zoneid, level=self.id, geom=geom.__geo_interface__) db.find_one_and_replace({'_id': zoneid}, zone, upsert=True) loaded += 1 except Exception as e: error('Error extracting polygon {0}: {1}', polygon['properties'], str(e)) info('Loaded {0} zones for level {1} from file {2}'.format(loaded, self.id, filename)) return loaded
def __init__(self, device=None, dtype=None, noblock=False, relink=False): """ Immutable initialization constructor. Args: device Device (either instance, formatted name or None) to use dtype Datatype to use, None for PyTorch default noblock To try and avoid using blocking memory transfer operations from the host relink Relink instead of copying by default in some assignment operations """ # Convert formatted device name to device instance if device is None: # Use default device device = type(self).default_device if isinstance(device, str): # Warn if CUDA is requested but not available if not torch.cuda.is_available() and device[:4] == "cuda": device = "cpu" tools.warning( "CUDA is unavailable on this node, falling back to CPU in the configuration", context="experiments") # Convert device = torch.device(device) # Resolve the current default dtype if unspecified if dtype is None: dtype = torch.get_default_dtype() # Finalization self._args = { "device": device, "dtype": dtype, "non_blocking": noblock } self.relink = relink
def compute_epoch(self): """ Compute and append the epoch number, if not already done. Returns: self """ column_name = "Epoch number" # Check if already there if column_name in self.data.columns: return # Compute epoch number if self.json is None or "dataset" not in self.json: tools.warning( "No valid JSON-formatted configuration, cannot compute the epoch number" ) return dataset_name = self.json["dataset"] training_size = { "mnist": 60000, "fashionmnist": 60000, "cifar10": 50000, "cifar100": 50000 }.get(dataset_name, None) if training_size is None: tools.warning( "Unknown dataset %r, cannot compute the epoch number" % dataset_name) return self.data[ column_name] = self.data["Training point count"] / training_size # Return self to enable chaining return self
def getCitedInPapers(MyWebdriver, logPath='', warningPath=''): #获取citedInPapers citedInPapers = [] try: citedInPapersBtn = MyWebdriver.find_element_by_xpath( '//*[@id="tab-1016-btnInnerEl"]') citedInPapersBtn.click() #等待referencesTable加载出来 #有些页面无abstract和references,需进行处理 MyWebdriver.WebDriverWait_until( 30, lambda x: x.find_element_by_xpath( '//*[@id="cf_layoutareacitedby"]/div/table')) except Exception as e: #无citedInPapers情况,获取到’Citings are not available‘获取’loading‘ warningInfo = 'Can not get citings from this papers\n Failed info: {0}'.format( repr(e)) tools.warning(warningInfo, warningPath) else: #成功加载了citedInPapers citedInPapersTable = MyWebdriver.find_element_by_xpath( '//*[@id="cf_layoutareacitedby"]/div/table') citedInPapersTable = citedInPapersTable.text.split('\n') i = 0 while i < len(citedInPapersTable): citedInPapers.append(citedInPapersTable[i].strip(' ')) i += 1 logInfo = 'Successfully get the citings from this page' if len(citedInPapers) == 0: logInfo = '!!!Successfully get the citings, but the citings in None' tools.log(logInfo, logPath) return citedInPapers
def check_job(self): if (self.nb_requested_cores < 0 or self.nb_requested_nodes < 0 or self.nb_cores_allocated < 0): tools.error("") if (type(self.used_mem) is str): tools.error("Job : " + self.id + " the " + self.date.strftime('%Y') + self.date.strftime('%m') + self.date.strftime('%d') + " : Error Used Memory") self.used_mem = 0 if (self.used_mem < 0): tools.error("Job : " + self.id + " : Negative Used Memory") self.used_mem = 0 if (self.used_walltime < 0): tools.error("Job : " + self.id + " : Negative Used Walltime") self.used_walltime = 0 if (self.requested_walltime < 0): tools.error("Job : " + self.id + " : Negative Requested Walltime") self.requested_walltime = 0 if (self.requested_mem < 0): tools.error("Job : " + self.id + " : Negative Requested Mem") self.requested_mem = 0 if (self.name == ""): tools.warning("Job without Name") if (self.completion_time < self.start_time): tools.error("Job : " + self.id + " : completion time before start time")
def getPdfURL(MyWebdriver, logPath='', warningPath=''): pdfURL = '' #获取pdfURL try: MyWebdriver.WebDriverWait_until( 30, lambda x: x.find_element_by_xpath( '/html/body/font/div/font[2]/table/tbody/tr/td[2]/table/tbody/tr/td/font/a' )) downloadTag = MyWebdriver.find_element_by_xpath( '/html/body/font/div/font[2]/table/tbody/tr/td[2]/table/tbody/tr/td/font/a' ) except Exception as e: warningInfo = 'Can not get the pdfURL from this page\n Failed info:{0}'.format( repr(e)) tools.warning(warningInfo, warningPath) else: pdfURL = downloadTag.get_attribute('href') successInfo = 'Successfully get the pdfURL from this page' if pdfURL == None: pdfURL = '' successInfo = '!!!Successfully get the pdfURL from this page, but the pdfURL is None' tools.log(successInfo, logPath) # print(pdfURL) return pdfURL
def fetch_flag_or_blazon(self): sparql_query = SPARQL_IMAGE_TEMPLATE.substitute( resource_url=self.resource_url) parameters = { 'default-graph-uri': 'http://fr.dbpedia.org', 'query': sparql_query, 'format': 'json' } flag_or_blazon = {} try: response = requests.get(SPARQL_SERVER, params=parameters) except requests.exceptions.ReadTimeout: warning('Timeout:', SPARQL_SERVER, parameters) return flag_or_blazon try: data = response.json() except json.decoder.JSONDecodeError: warning('JSON Error:', SPARQL_SERVER, parameters, response.text) return flag_or_blazon try: results = data['results']['bindings'][0] except IndexError: return flag_or_blazon if 'flag' in results: flag_name = results['flag']['value'].replace(' ', '_') flag_or_blazon['flag'] = flag_name if 'blazon' in results: blazon_name = results['blazon']['value'].replace(' ', '_') flag_or_blazon['blazon'] = blazon_name return flag_or_blazon
def calc_max_ratio(self, nowarn=False): """ Compute the maximum ratio std dev. / norm theoretically supported by the GAR, cache the result. Args: nowarn Do not issue a warning if the GAR does not have a known ratio Returns: Maximum ratio, None if unavailable """ # Fast path if self.thresh is not None: if self.thresh < 0: # Unavailable return None return self.thresh # Compute and cache threshold if self.json is None or not all(name in self.json for name in ("gar", "nb_workers", "nb_decl_byz")): tools.warning("No valid JSON-formatted configuration, cannot compute the maximum variance-norm ratio for the GAR") return g = self.json["gar"] rule = aggregators.gars.get(g, None) if rule is not None and rule.upper_bound is not None: n = self.json["nb_workers"] f = self.json["nb_decl_byz"] d = experiments.Model(self.json["model"], **self.json["model_args"], config=experiments.Configuration(device="cpu")).get().numel() self.thresh = rule.upper_bound(n, f, d) else: if not nowarn: tools.warning(f"GAR {g!r} has no known ratio threshold") self.thresh = -1 # Return threshold if self.thresh < 0: return None return self.thresh
def _run(topdir, name, seed, device, command): """ Run the attack experiments with the given named parameters. Args: topdir Parent result directory name Experiment unique name seed Experiment seed device Device on which to run the experiments command Command to run """ # Add seed to name name = "%s-%d" % (name, seed) # Process experiment with tools.Context(name, "info"): finaldir = topdir / name # Check whether the experiment was already successful if finaldir.exists(): tools.info("Experiment already processed.") return # Move-make the pending result directory resdir = move_directory(topdir / f"{name}.pending") resdir.mkdir(mode=0o755, parents=True) # Build the command args = command.build(seed, device, resdir) # Launch the experiment and write the standard output/error tools.trace((" ").join(shlex.quote(arg) for arg in args)) cmd_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if cmd_res.returncode == 0: tools.info("Experiment successful") else: tools.warning("Experiment failed") finaldir = topdir / f"{name}.failed" move_directory(finaldir) resdir.rename(finaldir) (finaldir / "stdout.log").write_bytes(cmd_res.stdout) (finaldir / "stderr.log").write_bytes(cmd_res.stderr)
def register(name, unchecked, check, upper_bound=None): """ Simple registration-wrapper helper. Args: name GAR name unchecked Associated function (see module description) check Parameter validity check function upper_bound Compute the theoretical upper bound on the ratio non-Byzantine standard deviation / norm to use this aggregation rule: (n, f, d) -> float """ global gars # Check if name already in use if name in gars: tools.warning("Unable to register %r GAR: name already in use" % name) return # Closure wrapping the call with checks def checked(**kwargs): # Check parameter validity message = check(**kwargs) if message is not None: raise tools.UserException( "Aggregation rule %r cannot be used with the given parameters: %s" % (name, message)) # Aggregation (hard to assert return value, duck-typing is allowed...) return unchecked(**kwargs) # Select which function to call by default func = checked if __debug__ else unchecked # Bind all the (sub) functions to the selected function setattr(func, "check", check) setattr(func, "checked", checked) setattr(func, "unchecked", unchecked) setattr(func, "upper_bound", upper_bound) # Export the selected function with the associated name gars[name] = func
def fetch_population_or_area(self): sparql_query = SPARQL_POPULATION_TEMPLATE.substitute( resource_url=self.resource_url) parameters = { 'default-graph-uri': 'http://fr.dbpedia.org', 'query': sparql_query, 'format': 'json' } result = {} try: response = requests.get(SPARQL_SERVER, params=parameters) except requests.exceptions.ReadTimeout: warning('Timeout:', SPARQL_SERVER, parameters) return result try: data = response.json() except json.decoder.JSONDecodeError: warning('JSON Error:', SPARQL_SERVER, parameters, response.text) return result try: results = data['results']['bindings'][0] except IndexError: return result if 'population' in results: result['population'] = int(results['population']['value']) if 'area' in results: result['area'] = int(round(float(results['area']['value']))) return result
def getPdfURL(MyWebdriver, logPath='', warningPath=''): pdfBaseURL = 'https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=' #获取当前页面url(因为会跳转),所以在此取得实际url current_url = MyWebdriver.current_url() #获取论文的number,用于拼接pdf页面的url paperNumber = current_url.split('/')[-1].split('?')[0] pdfHtmlURL = pdfBaseURL + paperNumber # print('pdfHtmlURL: ' + pdfHtmlURL) pdfURL = '' #无法打开pdfHtml页面 response = tools.requestsGet(pdfHtmlURL, headers=headers, times=2, logPath=logPath, warningPath=warningPath) if response == '': warningInfo = 'Can not get the pdfURL from this page, for failed to get the pdfHtml {0}'.format(pdfHtmlURL) tools.warning(warningInfo, warningPath) return pdfURL #获取pdfURL try: pdfHtml = etree.HTML(response.content) iframe = pdfHtml.xpath('//iframe')[0] except Exception as e: warningInfo = 'Can not get the pdfURL from this page {0}\n Failed info: {1}'.format(pdfHtmlURL, repr(e)) tools.warning(warningInfo, warningPath) else: pdfURL = iframe.get('src').split('?')[0] successInfo = 'Successfully get the pdfURL from this page' if pdfURL == '': successInfo = '!!!Successfully get the pdfURL from this page, but the pdfURL is None' tools.log(successInfo, logPath) # print(pdfURL) return pdfURL
def getAbstract(MyWebdriver, logPath='', warningPath=''): #获取abstract abstract = '' try: #等待abstractBtn加载出来 MyWebdriver.WebDriverWait_until( 30, lambda x: x.find_element_by_xpath('//*[@id="tab-1011-btnEl"]')) abstractBtn = MyWebdriver.find_element_by_xpath( '//*[@id="tab-1011-btnEl"]') abstractBtn.send_keys(Keys.ENTER) #等待abstract加载出来 MyWebdriver.WebDriverWait_until( 30, lambda x: x.find_element_by_xpath( '//*[@id="cf_layoutareaabstract"]/div/div')) except Exception as e: warningInfo = 'Can not find the abstract element from this page\n Failed info: {0}'.format( repr(e)) tools.warning(warningInfo, warningPath) else: abstract = MyWebdriver.find_element_by_xpath( '//*[@id="cf_layoutareaabstract"]/div/div') abstract = abstract.text logInfo = 'Successfully get the abstract from this page' if abstract == 'An abstract is not available.': abstract = '' logInfo = '!!!Successfully get the abstract, but abstract is None' # print(abstract) tools.log(logInfo, logPath) return abstract
def getReferences(MyWebdriver, logPath='', warningPath=''): #获取references references = [] try: referencesBtn = MyWebdriver.find_element_by_xpath( '//*[@id="tab-1015-btnEl"]') referencesBtn.send_keys(Keys.ENTER) #等待referencesTable加载出来 #有些页面无abstract和references,需进行处理 MyWebdriver.WebDriverWait_until( 30, lambda x: x.find_element_by_xpath( '//*[@id="cf_layoutareareferences"]/div/table')) except Exception as e: #无references情况,获取到‘loading' 或 ’References are not available‘ warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format( repr(e)) tools.warning(warningInfo, warningPath) else: #成功加载了references #处理references原始数据 referencesTable = MyWebdriver.find_element_by_xpath( '//*[@id="cf_layoutareareferences"]/div/table') referencesTable = referencesTable.text.split('\n') i = 1 while i < len(referencesTable): references.append(referencesTable[i].strip(' ')) i += 2 # print(references[-1]) logInfo = 'Successfully get the references from this page' if len(references) == 0: logInfo = '!!!Successfully get the references, but references is None' tools.log(logInfo, logPath) return references
def compute_lr(self): """ Compute and append the learning rate, if not already done. Returns: self """ column_name = "Learning rate" # Check if already there if column_name in self.data.columns: return # Compute epoch number if self.json is None or "learning_rate" not in self.json: tools.warning( "No valid JSON-formatted configuration, cannot compute the learning rate" ) return lr = self.json["learning_rate"] lr_decay = self.json.get("learning_rate_decay", 0) lr_delta = self.json.get("learning_rate_decay_delta", 1) if lr_decay > 0: self.data[column_name] = lr / ( (self.data.index // lr_delta * lr_delta) / lr_decay + 1) else: self.data[column_name] = lr # Return self to enable chaining return self
def attach_and_clean_iris(db, filename): info('Attaching French IRIS to their region') processed = 0 for zone in db.find({'level': iris.id}): candidates_ids = [p for p in zone['parents'] if p.startswith(town.id)] if len(candidates_ids) < 1: warning('No parent candidate found for: {0}'.format(zone['_id'])) continue town_id = candidates_ids[0] town_zone = db.find_one({'_id': town_id}) if not town_zone: warning('Town {0} not found'.format(town_id)) continue if zone.get('_type') == 'Z': name = town_zone['name'] else: name = ''.join((town_zone['name'], ' (', zone['name'], ')')) ops = { '$addToSet': {'parents': {'$each': town_zone['parents']}}, '$set': {'name': name}, '$unset': {'_town': 1, '_type': 1} } if db.find_one_and_update({'_id': zone['_id']}, ops): processed += 1 success('Attached {0} french IRIS to their parents'.format(processed))
def account_data(username): """ Will determine user origin (SISC, VSC or CECI) and return a dictionnary with user associated data Args: username: username Returns: A dictionnary with information about the user. User data dictionnary keys with values: 'username', 'home_dir', 'work_dir', 'ssh_key', 'origin' User data dictionnary keys with booleans: 'sisc', 'vsc', 'ceci' """ # Check the cache if username in _USERS_CACHE: return _USERS_CACHE[username] # Init user entry user = dict() user['username'] = username user['univ'] = 'unknown' user['group'] = 'unknown' # Collect finger data if finger(user) == 0: user['origin'] = 'unknown' user['group'] = username user['univ'] = 'unknown' elif IS_VSC.match(username) != None: user['origin'] = 'vsc' user['group'] = username matches = GET_VSC_UNIV.match(user['ldap_home']) if matches == None: tools.warning('failed to extract university for VSC user ' + username + ' from home dir ' + user['ldap_home']) else: user['univ'] = matches.group(1) elif IS_ULB.match(user['home_path']): user['univ'] = 'ulb' user['origin'] = 'sisc' elif IS_VUB.match(user['home_path']): user['univ'] = 'vub' user['origin'] = 'sisc' elif IS_SCC.match(user['home_path']): user['univ'] = 'sisc' user['origin'] = 'sisc' else: user['univ'] = 'unknown' tools.warning('failed to determine university for user ' + username + ' from home dir ' + user['home_path']) # print "Added user ", user['username'], ' - ', user['group'] global _USERS_CACHE _USERS_CACHE[username] = user return user
def search(title, logPath='', warningPath=''): def compareTitle(title, resultTitle): title, resultTitle = [ x.replace(' ', '')\ .replace('-', '')\ .replace(',', '')\ .replace(':', '')\ .replace('.', '')\ .lower() for x in [title, resultTitle] ] return resultTitle.find(title) == 0 title = title.strip('.') baseSearchUrl = 'https://dl.acm.org/results.cfm' headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36', 'referer': 'https://dl.acm.org/results.cfm?', 'upgrade-insecure-requests': '1', } data = { 'query': title, } newURL = '' response = tools.requestsGet(baseSearchUrl, headers=headers, params=data, times=2, logPath=logPath, warningPath=warningPath) #请求搜索页面失败 if response == '': warningInfo = 'Failed search <{0}> in ACM, For can not open the search page'.format( title) tools.warning(warningInfo, warningPath) return newURL #请求搜索页面成功 html = etree.HTML(response.content) resultTitles = html.xpath('//*[@id="results"]/div[5]/div[1]/a/text()') #有对应正确结果 if (len(resultTitles) > 0) and compareTitle(title, resultTitles[0]): newURL = html.xpath('//*[@id="results"]/div[5]/div[1]/a')[0].get( 'href') newURL = 'https://dl.acm.org/' + newURL logInfo = 'Successfully find <{0}> in ACM,and newURL is {1}'.format( title, newURL) tools.log(logInfo, logPath) #无对应正确结果 else: warningInfo = 'Failed to find <{0}> in ACM, For none matched result'.format( title) tools.log(warningInfo, warningPath) return newURL
def getReferences(MyWebdriver, logPath='', warningPath=''): references = [] #获取referencesBtn这个按钮元素,若元素存在,可点击滚动到references区域;若不存在,该页面无references referencesBtn = None try: MyWebdriver.WebDriverWait_until(30, lambda x: x.find_element_by_xpath('//*[@id="document-tabs"]/div/a')) btns = MyWebdriver.find_elements_by_xpath('//*[@id="document-tabs"]/div/a') except Exception as e: warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format(repr(e)) tools.warning(warningInfo, warningPath) return references else: for btn in btns: if btn.text == 'References': referencesBtn = btn break #无referencesBtn这个元素 if referencesBtn == None: warningInfo = 'Can not get references from this page, for this page has no references' tools.warning(warningInfo, warningPath) return references #referencesBtn元素存在 try: referencesBtn.send_keys(Keys.ENTER) except Exception as e: warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format(repr(e)) tools.warning(warningInfo, warningPath) else: try: referencesTag = MyWebdriver.find_elements_by_xpath('//*[@id="references-section-container"]/div/div/xpl-reference-item-migr/div/div/span[2]') except Exception as e: warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format(repr(e)) tools.warning(warningInfo, warningPath) else: length = len(referencesTag) if length == 0: logInfo = '!!!Successfully get the references, but the references is None' tools.log(logInfo, logPath) else: referencesTag = referencesTag[int(length/2) : ] for referenceTag in referencesTag: #可能会出现‘element can not attach’的错误 try: references.append(referenceTag.text) except Exception as e: warningInfo = 'Can not get the {0}th reference from this page\n Failed info: {1}'.format(referencesTag.index(referenceTag), repr(e)) tools.warning(warningInfo, warningPath) logInfo = 'Successfully get the references from this page' tools.log(logInfo, logPath) return references
def has_known_ratio(self): """ Check whether the session's GAR has a known ratio. Returns: Whether the session's GAR has a known ratio """ if self.json is None or "gar" not in self.json: tools.warning("No valid JSON-formatted configuration, cannot tell whether the associated GAR has a ratio") return g = self.json["gar"] rule = aggregators.gars.get(g, None) return rule is not None and rule.upper_bound is not None
def finalize(self, title, xlabel, ylabel, zlabel=None, xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, legend=None): """ Finalize the plot, can be done only once and would prevent further inclusion. Args: title Plot title xlabel Label for the x-axis ylabel Label for the y-axis zlabel Label for the twin y-axis, if any xmin Minimum for abscissa, if any xmax Maximum for abscissa, if any ymin Minimum for ordinate, if any ymax Maximum for ordinate, if any zmin Minimum for second ordinate, if any zmax Maximum for second ordinate, if any legend List of strings (one per 'include', in call order) to use as legend Returns: self """ # Fast path if self._fin: return self # Plot the legend def generator_sum(gen): res = None while True: try: val = next(gen) if res is None: res = val else: res += val except StopIteration: return res (self._ax if self._tax is None else self._tax).legend(generator_sum(ax.get_legend_handles_labels()[0] for ax in self._axs.values()), generator_sum(ax.get_legend_handles_labels()[1] for ax in self._axs.values()) if legend is None else legend, loc="best") # Plot the grid and labels self._ax.grid() self._ax.set_xlabel(xlabel) self._ax.set_ylabel(ylabel) self._ax.set_title(title) if zlabel is not None: if self._tax is None: tools.warning(f"No secondary y-axis found, but its label {zlabel!r} was provided") else: self._tax.set_ylabel(zlabel) elif self._tax is not None: tools.warning(f"No label provided for the secondary y-axis; using label {ylabel!r} from the primary") self._tax.set_ylabel(ylabel) self._ax.set_xlim(left=xmin, right=xmax) self._ax.set_ylim(bottom=ymin, top=ymax) if self._tax is not None: self._tax.set_ylim(bottom=zmin, top=zmax) # Mark finalized self._fin = True # Return self for chaining return self
def process_dataset(self, workdir, db, url, extractor): '''Extract territories from a given file for a given level with a given extractor function''' loaded = 0 filename = join(workdir, basename(url)) # Identify the shapefile to avoid multiple file error on GDAL 2 with ZipFile(filename) as z: candidates = [n for n in z.namelist() if n.endswith('.shp')] if len(candidates) != 1: raise ValueError( 'Unable to find a unique shpaefile into {0}'.format( filename)) shp = candidates[0] with fiona.open('/{0}'.format(shp), vfs='zip://{0}'.format(filename), encoding='utf8') as collection: info('Extracting {0} elements from {1} ({2} {3})'.format( len(collection), basename(filename), collection.driver, to_string(collection.crs))) for polygon in collection: try: zone = extractor(polygon) if not zone: continue zone['keys'] = dict( (k, v) for k, v in zone.get('keys', {}).items() if v is not None) geom = shape(polygon['geometry']) if extractor.simplify: geom = geom.simplify(extractor.simplify) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) elif geom.geom_type != 'MultiPolygon': warning( 'Unsupported geometry type "{0}" for "{1}"'.format( geom.geom_type, zone['name'])) continue zoneid = '/'.join((self.id, zone['code'])) zone.update(_id=zoneid, level=self.id, geom=geom.__geo_interface__) db.find_one_and_replace({'_id': zoneid}, zone, upsert=True) loaded += 1 except Exception as e: error('Error extracting polygon {0}: {1}', polygon['properties'], str(e)) info('Loaded {0} zones for level {1} from file {2}'.format( loaded, self.id, filename)) return loaded
def _extract_compute_resources(req_res, job): """ Extract all the details from the job requested resources. Args: req_res: requested resources job: job object Returns: seconds """ # global IS_NIC, GET_PPN, IS_INT res_parts = req_res.split(':') if len(res_parts) == 1: job['nb-requested-nodes'] += 1 job['nb-requested-cores'] += 1 job['invalid-requested-resources'] = True tools.warning('found invalid requested compute resources: ' + req_res + ' for job ' + job['id'] + ' in file ' + fileinput.filename()) else: # print 'Req res: ', val, ' nodes = ', res_parts[0], ' - cores = ', res_parts[1] # Get requested node(s) if IS_NIC.match(res_parts[0]) is not None: job['nb-requested-nodes'] += 1 elif IS_INT.match(res_parts[0]) is not None: job['nb-requested-nodes'] += int(res_parts[0]) else: tools.warning( "couldn't determine requested node resource from: " + req_res + ' for job ' + job['id'] + ' by user ' + job['user'], ' in file ' + fileinput.filename() + '. Job skipped.') # Get requested core(s) matches = GET_PPN.match(res_parts[1]) nb_cores = 1 req_feat = '' if matches is None: req_feat = res_parts[1] # Case neednodes=1:gpgpu else: nb_cores = int(matches.group(1)) try: req_feat = res_parts[2] except IndexError: req_feat = '' job['nb-requested-cores'] += job['nb-requested-nodes'] * nb_cores job['requested-features'].append(req_feat) return
def _run(name, seed, device, params): """ Run the attack experiments with the given named parameters. Args: name Experiment unique name seed Experiment seed device Device on which to run the experiments params Named parameters """ # Add seed to name name = "%s-%d" % (name, seed) # Process experiment with tools.Context(name, "info"): # Build and set the result directory result_dir = args.data_directory / name if result_dir.exists(): tools.info("Experiment already processed.") return result_dir.mkdir(mode=0o755, parents=True) # Add the missing options params["seed"] = str(seed) params["device"] = device params["result-directory"] = str(result_dir) # Launch the experiment and write the standard output/error def is_multi_param(param): return any(isinstance(param, typ) for typ in (list, tuple)) def param_to_str(param): if is_multi_param(param): return (" ").join(shlex.quote(str(val)) for val in param) return shlex.quote(str(param)) tools.trace("python3 -OO attack.py %s" % (" ").join("--%s %s" % (key, param_to_str(val)) for key, val in params.items())) command = ["python3", "-OO", "attack.py"] for key, val in params.items(): command.append("--%s" % (key, )) if is_multi_param(val): for subval in val: command.append(str(subval)) else: command.append(str(val)) cmd_res = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) if cmd_res.returncode == 0: tools.info("Experiment successful") else: tools.warning("Experiment failed") (result_dir / "stdout.log").write_bytes(cmd_res.stdout) (result_dir / "stderr.log").write_bytes(cmd_res.stderr)
def add_custom_datasets(name, module, _): nonlocal self # Check if has exports, fallback otherwise exports = getattr(module, "__all__", None) if exports is None: tools.warning(f"Dataset module {name!r} does not provide '__all__'; falling back to '__dict__' for name discovery") exports = (name for name in dir(module) if len(name) > 0 and name[0] != "_") # Register the association 'name -> constructor' for all the datasets exported = False for dataset in exports: # Check dataset name type if not isinstance(dataset, str): tools.warning(f"Dataset module {name!r} exports non-string name {dataset!r}; ignored") continue # Recover instance from name constructor = getattr(module, dataset, None) # Check instance is callable (it's only an heuristic...) if not callable(constructor): continue # Register callable with composite name exported = True fullname = f"{name}-{dataset}" if fullname in self.__datasets: tools.warning(f"Unable to make available dataset {dataset!r} from module {name!r}, as the name {fullname!r} already exists") continue self.__datasets[fullname] = constructor if not exported: tools.warning(f"Dataset module {name!r} does not export any valid constructor name through '__all__'")
def add_custom_models(name, module, _): nonlocal self # Check if has exports, fallback otherwise exports = getattr(module, "__all__", None) if exports is None: tools.warning("Model module %r does not provide '__all__'; falling back to '__dict__' for name discovery" % name) exports = (name for name in dir(module) if len(name) > 0 and name[0] != "_") # Register the association 'name -> constructor' for all the models exported = False for model in exports: # Check model name type if not isinstance(model, str): tools.warning("Model module %r exports non-string name %r; ignored" % (name, model)) continue # Recover instance from name constructor = getattr(module, model, None) # Check instance is callable (it's only an heuristic...) if not callable(constructor): continue # Register callable with composite name exported = True fullname = "%s-%s" % (name, model) if fullname in self.__models: tools.warning("Unable to make available model %r from module %r, as the name %r already exists" % (model, name, fullname)) continue self.__models[fullname] = constructor if not exported: tools.warning("Model module %r does not export any valid constructor name through '__all__'" % name)
def __init__(self, name, dict, ast_params): self.access = "rw" self.desc = "" self.hide = False self.init = None self.mandatory = True self.name = name self.owned = True self.serialize = True self.type = "" for key in dict: if not key in ["access", "desc", "hide", "init", "mandatory", "owned", "serialize", "type"]: warning("unknown Attribute attribute: " + key + " from " + name) self.__dict__[key] = dict[key] self.ast_params = ast_params
def register(name, unchecked, check, upper_bound=None, influence=None): """ Simple registration-wrapper helper. Args: name GAR name unchecked Associated function (see module description) check Parameter validity check function upper_bound Compute the theoretical upper bound on the ratio non-Byzantine standard deviation / norm to use this aggregation rule: (n, f, d) -> float influence Attack acceptation ratio function """ global gars # Check if name already in use if name in gars: tools.warning("Unable to register %r GAR: name already in use" % name) return # Export the selected function with the associated name gars[name] = make_gar(unchecked, check, upper_bound=upper_bound, influence=influence)
def get_default_root(self): """ Lazy-initialize and return the default dataset root directory path. Returns: '__default_root' """ # Fast-path already loaded if self.__default_root is not None: return self.__default_root # Generate the default path self.__default_root = pathlib.Path(__file__).parent / "datasets" / "cache" # Warn if the path does not exist and fallback to '/tmp' if not self.__default_root.exists(): tmpdir = tempfile.gettempdir() tools.warning(f"Default dataset root {str(self.__default_root)!r} does not exist, falling back to local temporary directory {tmpdir!r}", context="experiments") self.__default_root = pathlib.Path(tmpdir) # Return the path return self.__default_root
def summary_thread(coord, mngr, sess, path, rstrd): """ Summary thread entry point. Args: coord Coordinator to use mngr Graph manager to use sess Session to use path Path to the manager to use rstrd Whether the model was just restored from a checkpoint """ global args delta = args.summary_delta period = args.summary_period if delta < 0 and period < 0: # Effectively disabled tools.info("Summary saving is effectively disabled") return if mngr.summary_tn is None: tools.warning("No summary to save") return if rstrd: last_step = sess.run(mngr.step) last_time = time.time() else: last_step = -delta last_time = -period # Save summaries with mngr.graph.as_default(): with tf.summary.FileWriter(args.summary_dir, graph=mngr.graph) as writer: writer.add_session_log(tf.SessionLog(status=tf.SessionLog.START), sess.run(mngr.step)) while True: time.sleep(config.thread_idle_delay) step = sess.run(mngr.step) now = time.time() stop = coord.should_stop() if stop or (delta >= 0 and step - last_step >= delta) or ( period >= 0. and now - last_time >= period): writer.add_summary(sess.run(mngr.summary_tn), step) tools.info("Summaries saved (took " + repr(time.time() - now) + " s)") last_step = sess.run(mngr.step) last_time = time.time() if stop: break writer.add_session_log(tf.SessionLog(status=tf.SessionLog.STOP), step)
def attach_canton_parents(db, filename): info('Attaching French Canton to their parents') canton_processed = 0 for zone in db.find({'level': canton.id}): candidates_ids = [p for p in zone['parents'] if p.startswith(county.id)] if len(candidates_ids) < 1: warning('No parent candidate found for: {0}'.format(zone['_id'])) continue county_id = candidates_ids[0] county_zone = db.find_one({'_id': county_id}) ops = { '$addToSet': {'parents': {'$each': county_zone['parents']}}, '$unset': {'_dep': 1} } if db.find_one_and_update({'_id': zone['_id']}, ops): canton_processed += 1 success('Attached {0} french cantons to their parents'.format(canton_processed))
def __init__(self, name, dict, ast_params): self.ast_params = ast_params self.attributes = [] self.clone_by_ref = False self.cloner_prologue = "" # Is the class concrete? (Default is false.) self.concrete = False self.declAttribute = "" self.derived = [] self.desc = "" self.hide = False self.includes = {} self.inline = {} self.name = name self.super = "" self.super_non_nodes = [] for key in dict: # Catch duplicate keys. if isinstance(key, tuple): (realkey, value) = key error("duplicate key: " + name + "::" + realkey) if key not in [ "attributes", "concrete", "declAttribute", "default", "desc", "hide", "inline", "printer", "super", "cloner_prologue", "clone_by_ref", ]: warning("unknown Node attribute: " + name + "::" + key) self.__dict__[key] = dict[key] # If we have only one super-class, it has been parsed as a single # value, but we want a list. if not isinstance(self.super, list): self.super = [self.super] self.attributes = map(self.attribute_of_dict, self.attributes)
def __init__(self, name, dict, ast_params): self.access = "rw" self.desc = "" self.hide = False self.init = None self.mandatory = True self.name = name self.owned = True self.serialize = True self.type = "" for key in dict: if not key in [ 'access', 'desc', 'hide', 'init', 'mandatory', 'owned', 'serialize', 'type', ]: warning ('unknown Attribute attribute: ' + key + ' from ' + name) self.__dict__[key] = dict[key] self.ast_params = ast_params
def build_aggregate(self, code, name, zones, properties, db): geoms = [] populations = [] areas = [] for zoneid in zones: # Resolve wildcard if zoneid.endswith('/*'): level = zoneid.replace('/*', '') ids = db.distinct('_id', {'level': level}) resolved = self.build_aggregate(code, name, ids, properties, db) geoms.append(shape(resolved['geom'])) if resolved.get('population'): populations.append(resolved['population']) if resolved.get('area'): areas.append(resolved['area']) else: zone = db.find_one({'_id': zoneid}) if not zone: warning('Zone {0} not found'.format(zoneid)) continue shp = shape(zone['geom']) if not shp.is_valid: warning('Skipping invalid polygon for {0}'.format(zone['name'])) continue if shp.is_empty: warning('Skipping empty polygon for {0}'.format(zone['name'])) continue geoms.append(shp) if zone.get('population'): populations.append(zone['population']) if zone.get('area'): areas.append(zone['area']) geom = cascaded_union(geoms) if geom.geom_type == 'Polygon': geom = MultiPolygon([geom]) data = { '_id': '/'.join((self.id, code)), 'code': code, 'level': self.id, 'name': name, 'population': sum(populations), 'area': sum(areas), 'geom': geom.__geo_interface__ } data.update(properties) return data