def getCitedInPapers(MyWebdriver, logPath='', warningPath=''): citedInPapers = [] warningInfo = 'Can not get the citedInPapers from this page, for this page has no citedInPapers' tools.warning(warningInfo, warningPath) return citedInPapers
def getAbstract(MyWebdriver, logPath='', warningPath=''):
abstract = ''
#获取abstract
try:
MyWebdriver.WebDriverWait_until(
30, lambda x: x.find_element_by_xpath(
'/html/body/font/div/font[2]/div'))
abstractTag = MyWebdriver.find_element_by_xpath(
'/html/body/font/div/font[2]/div')
except Exception as e:
warningInfo = 'Can not get the abstract from this page\n Failed info: {0}'.format(
repr(e))
tools.warning(warningInfo, warningPath)
else:
abstracts = (abstractTag.text.split('\n')[1:]) if len(
abstractTag.text.split('\n')) > 0 else []
for abs in abstracts:
abstract += abs
successInfo = 'Successfully get the abstract from this page'
if abstract == '':
successInfo = '!!!Successfully get the abstract from this page, but the abstract is None'
tools.log(successInfo, logPath)
# print(abstract)
return abstract
def __init__(self, name, dict, ast_params):
self.access = "rw"
self.desc = ""
self.hide = False
self.init = None
self.mandatory = True
self.name = name
self.owned = True
self.serialize = True
self.type = ""
for key in dict:
if not key in [
'access',
'desc',
'hide',
'init',
'mandatory',
'owned',
'serialize',
'type',
]:
warning('unknown Attribute attribute: ' + key + ' from ' +
name)
self.__dict__[key] = dict[key]
self.ast_params = ast_params
def process_dataset(self, workdir, db, url, extractor):
'''Extract territories from a given file for a given level with a given extractor function'''
loaded = 0
filename = join(workdir, basename(url))
with fiona.open('/', vfs='zip://{0}'.format(filename), encoding='utf8') as collection:
info('Extracting {0} elements from {1} ({2} {3})'.format(
len(collection), basename(filename), collection.driver, to_string(collection.crs)
))
for polygon in collection:
try:
zone = extractor(polygon)
if not zone:
continue
zone['keys'] = dict((k, v) for k, v in zone.get('keys', {}).items() if v is not None)
geom = shape(polygon['geometry'])
if extractor.simplify:
geom = geom.simplify(extractor.simplify)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
warning('Unsupported geometry type "{0}" for "{1}"'.format(geom.geom_type, zone['name']))
continue
zoneid = '/'.join((self.id, zone['code']))
zone.update(_id=zoneid, level=self.id, geom=geom.__geo_interface__)
db.find_one_and_replace({'_id': zoneid}, zone, upsert=True)
loaded += 1
except Exception as e:
error('Error extracting polygon {0}: {1}', polygon['properties'], str(e))
info('Loaded {0} zones for level {1} from file {2}'.format(loaded, self.id, filename))
return loaded
def __init__(self, device=None, dtype=None, noblock=False, relink=False):
""" Immutable initialization constructor.
Args:
device Device (either instance, formatted name or None) to use
dtype Datatype to use, None for PyTorch default
noblock To try and avoid using blocking memory transfer operations from the host
relink Relink instead of copying by default in some assignment operations
"""
# Convert formatted device name to device instance
if device is None:
# Use default device
device = type(self).default_device
if isinstance(device, str):
# Warn if CUDA is requested but not available
if not torch.cuda.is_available() and device[:4] == "cuda":
device = "cpu"
tools.warning(
"CUDA is unavailable on this node, falling back to CPU in the configuration",
context="experiments")
# Convert
device = torch.device(device)
# Resolve the current default dtype if unspecified
if dtype is None:
dtype = torch.get_default_dtype()
# Finalization
self._args = {
"device": device,
"dtype": dtype,
"non_blocking": noblock
}
self.relink = relink
def compute_epoch(self):
""" Compute and append the epoch number, if not already done.
Returns:
self
"""
column_name = "Epoch number"
# Check if already there
if column_name in self.data.columns:
return
# Compute epoch number
if self.json is None or "dataset" not in self.json:
tools.warning(
"No valid JSON-formatted configuration, cannot compute the epoch number"
)
return
dataset_name = self.json["dataset"]
training_size = {
"mnist": 60000,
"fashionmnist": 60000,
"cifar10": 50000,
"cifar100": 50000
}.get(dataset_name, None)
if training_size is None:
tools.warning(
"Unknown dataset %r, cannot compute the epoch number" %
dataset_name)
return
self.data[
column_name] = self.data["Training point count"] / training_size
# Return self to enable chaining
return self
def getCitedInPapers(MyWebdriver, logPath='', warningPath=''):
#获取citedInPapers
citedInPapers = []
try:
citedInPapersBtn = MyWebdriver.find_element_by_xpath(
'//*[@id="tab-1016-btnInnerEl"]')
citedInPapersBtn.click()
#等待referencesTable加载出来
#有些页面无abstract和references，需进行处理
MyWebdriver.WebDriverWait_until(
30, lambda x: x.find_element_by_xpath(
'//*[@id="cf_layoutareacitedby"]/div/table'))
except Exception as e:
#无citedInPapers情况，获取到’Citings are not available‘获取’loading‘
warningInfo = 'Can not get citings from this papers\n Failed info: {0}'.format(
repr(e))
tools.warning(warningInfo, warningPath)
else:
#成功加载了citedInPapers
citedInPapersTable = MyWebdriver.find_element_by_xpath(
'//*[@id="cf_layoutareacitedby"]/div/table')
citedInPapersTable = citedInPapersTable.text.split('\n')
i = 0
while i < len(citedInPapersTable):
citedInPapers.append(citedInPapersTable[i].strip(' '))
i += 1
logInfo = 'Successfully get the citings from this page'
if len(citedInPapers) == 0:
logInfo = '!!!Successfully get the citings, but the citings in None'
tools.log(logInfo, logPath)
return citedInPapers
def check_job(self):
if (self.nb_requested_cores < 0 or self.nb_requested_nodes < 0
or self.nb_cores_allocated < 0):
tools.error("")
if (type(self.used_mem) is str):
tools.error("Job : " + self.id + " the " +
self.date.strftime('%Y') + self.date.strftime('%m') +
self.date.strftime('%d') + " : Error Used Memory")
self.used_mem = 0
if (self.used_mem < 0):
tools.error("Job : " + self.id + " : Negative Used Memory")
self.used_mem = 0
if (self.used_walltime < 0):
tools.error("Job : " + self.id + " : Negative Used Walltime")
self.used_walltime = 0
if (self.requested_walltime < 0):
tools.error("Job : " + self.id + " : Negative Requested Walltime")
self.requested_walltime = 0
if (self.requested_mem < 0):
tools.error("Job : " + self.id + " : Negative Requested Mem")
self.requested_mem = 0
if (self.name == ""):
tools.warning("Job without Name")
if (self.completion_time < self.start_time):
tools.error("Job : " + self.id +
" : completion time before start time")
def getPdfURL(MyWebdriver, logPath='', warningPath=''):
pdfURL = ''
#获取pdfURL
try:
MyWebdriver.WebDriverWait_until(
30, lambda x: x.find_element_by_xpath(
'/html/body/font/div/font[2]/table/tbody/tr/td[2]/table/tbody/tr/td/font/a'
))
downloadTag = MyWebdriver.find_element_by_xpath(
'/html/body/font/div/font[2]/table/tbody/tr/td[2]/table/tbody/tr/td/font/a'
)
except Exception as e:
warningInfo = 'Can not get the pdfURL from this page\n Failed info:{0}'.format(
repr(e))
tools.warning(warningInfo, warningPath)
else:
pdfURL = downloadTag.get_attribute('href')
successInfo = 'Successfully get the pdfURL from this page'
if pdfURL == None:
pdfURL = ''
successInfo = '!!!Successfully get the pdfURL from this page, but the pdfURL is None'
tools.log(successInfo, logPath)
# print(pdfURL)
return pdfURL
def fetch_flag_or_blazon(self):
sparql_query = SPARQL_IMAGE_TEMPLATE.substitute(
resource_url=self.resource_url)
parameters = {
'default-graph-uri': 'http://fr.dbpedia.org',
'query': sparql_query,
'format': 'json'
}
flag_or_blazon = {}
try:
response = requests.get(SPARQL_SERVER, params=parameters)
except requests.exceptions.ReadTimeout:
warning('Timeout:', SPARQL_SERVER, parameters)
return flag_or_blazon
try:
data = response.json()
except json.decoder.JSONDecodeError:
warning('JSON Error:', SPARQL_SERVER, parameters, response.text)
return flag_or_blazon
try:
results = data['results']['bindings'][0]
except IndexError:
return flag_or_blazon
if 'flag' in results:
flag_name = results['flag']['value'].replace(' ', '_')
flag_or_blazon['flag'] = flag_name
if 'blazon' in results:
blazon_name = results['blazon']['value'].replace(' ', '_')
flag_or_blazon['blazon'] = blazon_name
return flag_or_blazon
def calc_max_ratio(self, nowarn=False):
""" Compute the maximum ratio std dev. / norm theoretically supported by the GAR, cache the result.
Args:
nowarn Do not issue a warning if the GAR does not have a known ratio
Returns:
Maximum ratio, None if unavailable
"""
# Fast path
if self.thresh is not None:
if self.thresh < 0: # Unavailable
return None
return self.thresh
# Compute and cache threshold
if self.json is None or not all(name in self.json for name in ("gar", "nb_workers", "nb_decl_byz")):
tools.warning("No valid JSON-formatted configuration, cannot compute the maximum variance-norm ratio for the GAR")
return
g = self.json["gar"]
rule = aggregators.gars.get(g, None)
if rule is not None and rule.upper_bound is not None:
n = self.json["nb_workers"]
f = self.json["nb_decl_byz"]
d = experiments.Model(self.json["model"], **self.json["model_args"], config=experiments.Configuration(device="cpu")).get().numel()
self.thresh = rule.upper_bound(n, f, d)
else:
if not nowarn:
tools.warning(f"GAR {g!r} has no known ratio threshold")
self.thresh = -1
# Return threshold
if self.thresh < 0:
return None
return self.thresh
def _run(topdir, name, seed, device, command):
""" Run the attack experiments with the given named parameters.
Args:
topdir Parent result directory
name Experiment unique name
seed Experiment seed
device Device on which to run the experiments
command Command to run
"""
# Add seed to name
name = "%s-%d" % (name, seed)
# Process experiment
with tools.Context(name, "info"):
finaldir = topdir / name
# Check whether the experiment was already successful
if finaldir.exists():
tools.info("Experiment already processed.")
return
# Move-make the pending result directory
resdir = move_directory(topdir / f"{name}.pending")
resdir.mkdir(mode=0o755, parents=True)
# Build the command
args = command.build(seed, device, resdir)
# Launch the experiment and write the standard output/error
tools.trace((" ").join(shlex.quote(arg) for arg in args))
cmd_res = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if cmd_res.returncode == 0:
tools.info("Experiment successful")
else:
tools.warning("Experiment failed")
finaldir = topdir / f"{name}.failed"
move_directory(finaldir)
resdir.rename(finaldir)
(finaldir / "stdout.log").write_bytes(cmd_res.stdout)
(finaldir / "stderr.log").write_bytes(cmd_res.stderr)
def register(name, unchecked, check, upper_bound=None):
""" Simple registration-wrapper helper.
Args:
name GAR name
unchecked Associated function (see module description)
check Parameter validity check function
upper_bound Compute the theoretical upper bound on the ratio non-Byzantine standard deviation / norm to use this aggregation rule: (n, f, d) -> float
"""
global gars
# Check if name already in use
if name in gars:
tools.warning("Unable to register %r GAR: name already in use" % name)
return
# Closure wrapping the call with checks
def checked(**kwargs):
# Check parameter validity
message = check(**kwargs)
if message is not None:
raise tools.UserException(
"Aggregation rule %r cannot be used with the given parameters: %s"
% (name, message))
# Aggregation (hard to assert return value, duck-typing is allowed...)
return unchecked(**kwargs)
# Select which function to call by default
func = checked if __debug__ else unchecked
# Bind all the (sub) functions to the selected function
setattr(func, "check", check)
setattr(func, "checked", checked)
setattr(func, "unchecked", unchecked)
setattr(func, "upper_bound", upper_bound)
# Export the selected function with the associated name
gars[name] = func
def fetch_population_or_area(self):
sparql_query = SPARQL_POPULATION_TEMPLATE.substitute(
resource_url=self.resource_url)
parameters = {
'default-graph-uri': 'http://fr.dbpedia.org',
'query': sparql_query,
'format': 'json'
}
result = {}
try:
response = requests.get(SPARQL_SERVER, params=parameters)
except requests.exceptions.ReadTimeout:
warning('Timeout:', SPARQL_SERVER, parameters)
return result
try:
data = response.json()
except json.decoder.JSONDecodeError:
warning('JSON Error:', SPARQL_SERVER, parameters, response.text)
return result
try:
results = data['results']['bindings'][0]
except IndexError:
return result
if 'population' in results:
result['population'] = int(results['population']['value'])
if 'area' in results:
result['area'] = int(round(float(results['area']['value'])))
return result
def getPdfURL(MyWebdriver, logPath='', warningPath=''):
pdfBaseURL = 'https://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber='
#获取当前页面url（因为会跳转），所以在此取得实际url
current_url = MyWebdriver.current_url()
#获取论文的number，用于拼接pdf页面的url
paperNumber = current_url.split('/')[-1].split('?')[0]
pdfHtmlURL = pdfBaseURL + paperNumber
# print('pdfHtmlURL: ' + pdfHtmlURL)
pdfURL = ''
#无法打开pdfHtml页面
response = tools.requestsGet(pdfHtmlURL, headers=headers, times=2, logPath=logPath, warningPath=warningPath)
if response == '':
warningInfo = 'Can not get the pdfURL from this page, for failed to get the pdfHtml {0}'.format(pdfHtmlURL)
tools.warning(warningInfo, warningPath)
return pdfURL
#获取pdfURL
try:
pdfHtml = etree.HTML(response.content)
iframe = pdfHtml.xpath('//iframe')[0]
except Exception as e:
warningInfo = 'Can not get the pdfURL from this page {0}\n Failed info: {1}'.format(pdfHtmlURL, repr(e))
tools.warning(warningInfo, warningPath)
else:
pdfURL = iframe.get('src').split('?')[0]
successInfo = 'Successfully get the pdfURL from this page'
if pdfURL == '':
successInfo = '!!!Successfully get the pdfURL from this page, but the pdfURL is None'
tools.log(successInfo, logPath)
# print(pdfURL)
return pdfURL
def getAbstract(MyWebdriver, logPath='', warningPath=''):
#获取abstract
abstract = ''
try:
#等待abstractBtn加载出来
MyWebdriver.WebDriverWait_until(
30, lambda x: x.find_element_by_xpath('//*[@id="tab-1011-btnEl"]'))
abstractBtn = MyWebdriver.find_element_by_xpath(
'//*[@id="tab-1011-btnEl"]')
abstractBtn.send_keys(Keys.ENTER)
#等待abstract加载出来
MyWebdriver.WebDriverWait_until(
30, lambda x: x.find_element_by_xpath(
'//*[@id="cf_layoutareaabstract"]/div/div'))
except Exception as e:
warningInfo = 'Can not find the abstract element from this page\n Failed info: {0}'.format(
repr(e))
tools.warning(warningInfo, warningPath)
else:
abstract = MyWebdriver.find_element_by_xpath(
'//*[@id="cf_layoutareaabstract"]/div/div')
abstract = abstract.text
logInfo = 'Successfully get the abstract from this page'
if abstract == 'An abstract is not available.':
abstract = ''
logInfo = '!!!Successfully get the abstract, but abstract is None'
# print(abstract)
tools.log(logInfo, logPath)
return abstract
def fetch_flag_or_blazon(self):
sparql_query = SPARQL_IMAGE_TEMPLATE.substitute(
resource_url=self.resource_url)
parameters = {
'default-graph-uri': 'http://fr.dbpedia.org',
'query': sparql_query,
'format': 'json'
}
flag_or_blazon = {}
try:
response = requests.get(SPARQL_SERVER, params=parameters)
except requests.exceptions.ReadTimeout:
warning('Timeout:', SPARQL_SERVER, parameters)
return flag_or_blazon
try:
data = response.json()
except json.decoder.JSONDecodeError:
warning('JSON Error:', SPARQL_SERVER, parameters, response.text)
return flag_or_blazon
try:
results = data['results']['bindings'][0]
except IndexError:
return flag_or_blazon
if 'flag' in results:
flag_name = results['flag']['value'].replace(' ', '_')
flag_or_blazon['flag'] = flag_name
if 'blazon' in results:
blazon_name = results['blazon']['value'].replace(' ', '_')
flag_or_blazon['blazon'] = blazon_name
return flag_or_blazon
def fetch_population_or_area(self):
sparql_query = SPARQL_POPULATION_TEMPLATE.substitute(
resource_url=self.resource_url)
parameters = {
'default-graph-uri': 'http://fr.dbpedia.org',
'query': sparql_query,
'format': 'json'
}
result = {}
try:
response = requests.get(SPARQL_SERVER, params=parameters)
except requests.exceptions.ReadTimeout:
warning('Timeout:', SPARQL_SERVER, parameters)
return result
try:
data = response.json()
except json.decoder.JSONDecodeError:
warning('JSON Error:', SPARQL_SERVER, parameters, response.text)
return result
try:
results = data['results']['bindings'][0]
except IndexError:
return result
if 'population' in results:
result['population'] = int(results['population']['value'])
if 'area' in results:
result['area'] = int(round(float(results['area']['value'])))
return result
def getReferences(MyWebdriver, logPath='', warningPath=''):
#获取references
references = []
try:
referencesBtn = MyWebdriver.find_element_by_xpath(
'//*[@id="tab-1015-btnEl"]')
referencesBtn.send_keys(Keys.ENTER)
#等待referencesTable加载出来
#有些页面无abstract和references，需进行处理
MyWebdriver.WebDriverWait_until(
30, lambda x: x.find_element_by_xpath(
'//*[@id="cf_layoutareareferences"]/div/table'))
except Exception as e:
#无references情况，获取到‘loading' 或 ’References are not available‘
warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format(
repr(e))
tools.warning(warningInfo, warningPath)
else:
#成功加载了references
#处理references原始数据
referencesTable = MyWebdriver.find_element_by_xpath(
'//*[@id="cf_layoutareareferences"]/div/table')
referencesTable = referencesTable.text.split('\n')
i = 1
while i < len(referencesTable):
references.append(referencesTable[i].strip(' '))
i += 2
# print(references[-1])
logInfo = 'Successfully get the references from this page'
if len(references) == 0:
logInfo = '!!!Successfully get the references, but references is None'
tools.log(logInfo, logPath)
return references
def compute_lr(self):
""" Compute and append the learning rate, if not already done.
Returns:
self
"""
column_name = "Learning rate"
# Check if already there
if column_name in self.data.columns:
return
# Compute epoch number
if self.json is None or "learning_rate" not in self.json:
tools.warning(
"No valid JSON-formatted configuration, cannot compute the learning rate"
)
return
lr = self.json["learning_rate"]
lr_decay = self.json.get("learning_rate_decay", 0)
lr_delta = self.json.get("learning_rate_decay_delta", 1)
if lr_decay > 0:
self.data[column_name] = lr / (
(self.data.index // lr_delta * lr_delta) / lr_decay + 1)
else:
self.data[column_name] = lr
# Return self to enable chaining
return self
def attach_and_clean_iris(db, filename):
info('Attaching French IRIS to their region')
processed = 0
for zone in db.find({'level': iris.id}):
candidates_ids = [p for p in zone['parents'] if p.startswith(town.id)]
if len(candidates_ids) < 1:
warning('No parent candidate found for: {0}'.format(zone['_id']))
continue
town_id = candidates_ids[0]
town_zone = db.find_one({'_id': town_id})
if not town_zone:
warning('Town {0} not found'.format(town_id))
continue
if zone.get('_type') == 'Z':
name = town_zone['name']
else:
name = ''.join((town_zone['name'], ' (', zone['name'], ')'))
ops = {
'$addToSet': {'parents': {'$each': town_zone['parents']}},
'$set': {'name': name},
'$unset': {'_town': 1, '_type': 1}
}
if db.find_one_and_update({'_id': zone['_id']}, ops):
processed += 1
success('Attached {0} french IRIS to their parents'.format(processed))
def account_data(username):
""" Will determine user origin (SISC, VSC or CECI) and
return a dictionnary with user associated data
Args:
username: username
Returns:
A dictionnary with information about the user.
User data dictionnary keys with values: 'username', 'home_dir', 'work_dir', 'ssh_key', 'origin'
User data dictionnary keys with booleans: 'sisc', 'vsc', 'ceci'
"""
# Check the cache
if username in _USERS_CACHE:
return _USERS_CACHE[username]
# Init user entry
user = dict()
user['username'] = username
user['univ'] = 'unknown'
user['group'] = 'unknown'
# Collect finger data
if finger(user) == 0:
user['origin'] = 'unknown'
user['group'] = username
user['univ'] = 'unknown'
elif IS_VSC.match(username) != None:
user['origin'] = 'vsc'
user['group'] = username
matches = GET_VSC_UNIV.match(user['ldap_home'])
if matches == None:
tools.warning('failed to extract university for VSC user ' +
username + ' from home dir ' + user['ldap_home'])
else:
user['univ'] = matches.group(1)
elif IS_ULB.match(user['home_path']):
user['univ'] = 'ulb'
user['origin'] = 'sisc'
elif IS_VUB.match(user['home_path']):
user['univ'] = 'vub'
user['origin'] = 'sisc'
elif IS_SCC.match(user['home_path']):
user['univ'] = 'sisc'
user['origin'] = 'sisc'
else:
user['univ'] = 'unknown'
tools.warning('failed to determine university for user ' + username +
' from home dir ' + user['home_path'])
# print "Added user ", user['username'], ' - ', user['group']
global _USERS_CACHE
_USERS_CACHE[username] = user
return user
def search(title, logPath='', warningPath=''):
def compareTitle(title, resultTitle):
title, resultTitle = [ x.replace(' ', '')\
.replace('-', '')\
.replace(',', '')\
.replace(':', '')\
.replace('.', '')\
.lower()
for x in [title, resultTitle]
]
return resultTitle.find(title) == 0
title = title.strip('.')
baseSearchUrl = 'https://dl.acm.org/results.cfm'
headers = {
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.96 Safari/537.36',
'referer': 'https://dl.acm.org/results.cfm?',
'upgrade-insecure-requests': '1',
}
data = {
'query': title,
}
newURL = ''
response = tools.requestsGet(baseSearchUrl,
headers=headers,
params=data,
times=2,
logPath=logPath,
warningPath=warningPath)
#请求搜索页面失败
if response == '':
warningInfo = 'Failed search <{0}> in ACM, For can not open the search page'.format(
title)
tools.warning(warningInfo, warningPath)
return newURL
#请求搜索页面成功
html = etree.HTML(response.content)
resultTitles = html.xpath('//*[@id="results"]/div[5]/div[1]/a/text()')
#有对应正确结果
if (len(resultTitles) > 0) and compareTitle(title, resultTitles[0]):
newURL = html.xpath('//*[@id="results"]/div[5]/div[1]/a')[0].get(
'href')
newURL = 'https://dl.acm.org/' + newURL
logInfo = 'Successfully find <{0}> in ACM,and newURL is {1}'.format(
title, newURL)
tools.log(logInfo, logPath)
#无对应正确结果
else:
warningInfo = 'Failed to find <{0}> in ACM, For none matched result'.format(
title)
tools.log(warningInfo, warningPath)
return newURL
def getReferences(MyWebdriver, logPath='', warningPath=''):
references = []
#获取referencesBtn这个按钮元素，若元素存在，可点击滚动到references区域；若不存在，该页面无references
referencesBtn = None
try:
MyWebdriver.WebDriverWait_until(30, lambda x: x.find_element_by_xpath('//*[@id="document-tabs"]/div/a'))
btns = MyWebdriver.find_elements_by_xpath('//*[@id="document-tabs"]/div/a')
except Exception as e:
warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format(repr(e))
tools.warning(warningInfo, warningPath)
return references
else:
for btn in btns:
if btn.text == 'References':
referencesBtn = btn
break
#无referencesBtn这个元素
if referencesBtn == None:
warningInfo = 'Can not get references from this page, for this page has no references'
tools.warning(warningInfo, warningPath)
return references
#referencesBtn元素存在
try:
referencesBtn.send_keys(Keys.ENTER)
except Exception as e:
warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format(repr(e))
tools.warning(warningInfo, warningPath)
else:
try:
referencesTag = MyWebdriver.find_elements_by_xpath('//*[@id="references-section-container"]/div/div/xpl-reference-item-migr/div/div/span[2]')
except Exception as e:
warningInfo = 'Can not get the references from this page\n Failed info: {0}'.format(repr(e))
tools.warning(warningInfo, warningPath)
else:
length = len(referencesTag)
if length == 0:
logInfo = '!!!Successfully get the references, but the references is None'
tools.log(logInfo, logPath)
else:
referencesTag = referencesTag[int(length/2) : ]
for referenceTag in referencesTag:
#可能会出现‘element can not attach’的错误
try:
references.append(referenceTag.text)
except Exception as e:
warningInfo = 'Can not get the {0}th reference from this page\n Failed info: {1}'.format(referencesTag.index(referenceTag), repr(e))
tools.warning(warningInfo, warningPath)
logInfo = 'Successfully get the references from this page'
tools.log(logInfo, logPath)
return references
def has_known_ratio(self):
""" Check whether the session's GAR has a known ratio.
Returns:
Whether the session's GAR has a known ratio
"""
if self.json is None or "gar" not in self.json:
tools.warning("No valid JSON-formatted configuration, cannot tell whether the associated GAR has a ratio")
return
g = self.json["gar"]
rule = aggregators.gars.get(g, None)
return rule is not None and rule.upper_bound is not None
def finalize(self, title, xlabel, ylabel, zlabel=None, xmin=None, xmax=None, ymin=None, ymax=None, zmin=None, zmax=None, legend=None):
""" Finalize the plot, can be done only once and would prevent further inclusion.
Args:
title Plot title
xlabel Label for the x-axis
ylabel Label for the y-axis
zlabel Label for the twin y-axis, if any
xmin Minimum for abscissa, if any
xmax Maximum for abscissa, if any
ymin Minimum for ordinate, if any
ymax Maximum for ordinate, if any
zmin Minimum for second ordinate, if any
zmax Maximum for second ordinate, if any
legend List of strings (one per 'include', in call order) to use as legend
Returns:
self
"""
# Fast path
if self._fin:
return self
# Plot the legend
def generator_sum(gen):
res = None
while True:
try:
val = next(gen)
if res is None:
res = val
else:
res += val
except StopIteration:
return res
(self._ax if self._tax is None else self._tax).legend(generator_sum(ax.get_legend_handles_labels()[0] for ax in self._axs.values()), generator_sum(ax.get_legend_handles_labels()[1] for ax in self._axs.values()) if legend is None else legend, loc="best")
# Plot the grid and labels
self._ax.grid()
self._ax.set_xlabel(xlabel)
self._ax.set_ylabel(ylabel)
self._ax.set_title(title)
if zlabel is not None:
if self._tax is None:
tools.warning(f"No secondary y-axis found, but its label {zlabel!r} was provided")
else:
self._tax.set_ylabel(zlabel)
elif self._tax is not None:
tools.warning(f"No label provided for the secondary y-axis; using label {ylabel!r} from the primary")
self._tax.set_ylabel(ylabel)
self._ax.set_xlim(left=xmin, right=xmax)
self._ax.set_ylim(bottom=ymin, top=ymax)
if self._tax is not None:
self._tax.set_ylim(bottom=zmin, top=zmax)
# Mark finalized
self._fin = True
# Return self for chaining
return self
def process_dataset(self, workdir, db, url, extractor):
'''Extract territories from a given file for a given level with a given extractor function'''
loaded = 0
filename = join(workdir, basename(url))
# Identify the shapefile to avoid multiple file error on GDAL 2
with ZipFile(filename) as z:
candidates = [n for n in z.namelist() if n.endswith('.shp')]
if len(candidates) != 1:
raise ValueError(
'Unable to find a unique shpaefile into {0}'.format(
filename))
shp = candidates[0]
with fiona.open('/{0}'.format(shp),
vfs='zip://{0}'.format(filename),
encoding='utf8') as collection:
info('Extracting {0} elements from {1} ({2} {3})'.format(
len(collection), basename(filename), collection.driver,
to_string(collection.crs)))
for polygon in collection:
try:
zone = extractor(polygon)
if not zone:
continue
zone['keys'] = dict(
(k, v) for k, v in zone.get('keys', {}).items()
if v is not None)
geom = shape(polygon['geometry'])
if extractor.simplify:
geom = geom.simplify(extractor.simplify)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
elif geom.geom_type != 'MultiPolygon':
warning(
'Unsupported geometry type "{0}" for "{1}"'.format(
geom.geom_type, zone['name']))
continue
zoneid = '/'.join((self.id, zone['code']))
zone.update(_id=zoneid,
level=self.id,
geom=geom.__geo_interface__)
db.find_one_and_replace({'_id': zoneid}, zone, upsert=True)
loaded += 1
except Exception as e:
error('Error extracting polygon {0}: {1}',
polygon['properties'], str(e))
info('Loaded {0} zones for level {1} from file {2}'.format(
loaded, self.id, filename))
return loaded
def _extract_compute_resources(req_res, job):
""" Extract all the details from the job requested resources.
Args:
req_res: requested resources
job: job object
Returns:
seconds
"""
# global IS_NIC, GET_PPN, IS_INT
res_parts = req_res.split(':')
if len(res_parts) == 1:
job['nb-requested-nodes'] += 1
job['nb-requested-cores'] += 1
job['invalid-requested-resources'] = True
tools.warning('found invalid requested compute resources: ' + req_res +
' for job ' + job['id'] + ' in file ' +
fileinput.filename())
else:
# print 'Req res: ', val, ' nodes = ', res_parts[0], ' - cores = ', res_parts[1]
# Get requested node(s)
if IS_NIC.match(res_parts[0]) is not None:
job['nb-requested-nodes'] += 1
elif IS_INT.match(res_parts[0]) is not None:
job['nb-requested-nodes'] += int(res_parts[0])
else:
tools.warning(
"couldn't determine requested node resource from: " + req_res +
' for job ' + job['id'] + ' by user ' + job['user'],
' in file ' + fileinput.filename() + '. Job skipped.')
# Get requested core(s)
matches = GET_PPN.match(res_parts[1])
nb_cores = 1
req_feat = ''
if matches is None:
req_feat = res_parts[1] # Case neednodes=1:gpgpu
else:
nb_cores = int(matches.group(1))
try:
req_feat = res_parts[2]
except IndexError:
req_feat = ''
job['nb-requested-cores'] += job['nb-requested-nodes'] * nb_cores
job['requested-features'].append(req_feat)
return
def _run(name, seed, device, params):
""" Run the attack experiments with the given named parameters.
Args:
name Experiment unique name
seed Experiment seed
device Device on which to run the experiments
params Named parameters
"""
# Add seed to name
name = "%s-%d" % (name, seed)
# Process experiment
with tools.Context(name, "info"):
# Build and set the result directory
result_dir = args.data_directory / name
if result_dir.exists():
tools.info("Experiment already processed.")
return
result_dir.mkdir(mode=0o755, parents=True)
# Add the missing options
params["seed"] = str(seed)
params["device"] = device
params["result-directory"] = str(result_dir)
# Launch the experiment and write the standard output/error
def is_multi_param(param):
return any(isinstance(param, typ) for typ in (list, tuple))
def param_to_str(param):
if is_multi_param(param):
return (" ").join(shlex.quote(str(val)) for val in param)
return shlex.quote(str(param))
tools.trace("python3 -OO attack.py %s" %
(" ").join("--%s %s" % (key, param_to_str(val))
for key, val in params.items()))
command = ["python3", "-OO", "attack.py"]
for key, val in params.items():
command.append("--%s" % (key, ))
if is_multi_param(val):
for subval in val:
command.append(str(subval))
else:
command.append(str(val))
cmd_res = subprocess.run(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if cmd_res.returncode == 0:
tools.info("Experiment successful")
else:
tools.warning("Experiment failed")
(result_dir / "stdout.log").write_bytes(cmd_res.stdout)
(result_dir / "stderr.log").write_bytes(cmd_res.stderr)
def add_custom_datasets(name, module, _):
nonlocal self
# Check if has exports, fallback otherwise
exports = getattr(module, "__all__", None)
if exports is None:
tools.warning(f"Dataset module {name!r} does not provide '__all__'; falling back to '__dict__' for name discovery")
exports = (name for name in dir(module) if len(name) > 0 and name[0] != "_")
# Register the association 'name -> constructor' for all the datasets
exported = False
for dataset in exports:
# Check dataset name type
if not isinstance(dataset, str):
tools.warning(f"Dataset module {name!r} exports non-string name {dataset!r}; ignored")
continue
# Recover instance from name
constructor = getattr(module, dataset, None)
# Check instance is callable (it's only an heuristic...)
if not callable(constructor):
continue
# Register callable with composite name
exported = True
fullname = f"{name}-{dataset}"
if fullname in self.__datasets:
tools.warning(f"Unable to make available dataset {dataset!r} from module {name!r}, as the name {fullname!r} already exists")
continue
self.__datasets[fullname] = constructor
if not exported:
tools.warning(f"Dataset module {name!r} does not export any valid constructor name through '__all__'")
def add_custom_models(name, module, _):
nonlocal self
# Check if has exports, fallback otherwise
exports = getattr(module, "__all__", None)
if exports is None:
tools.warning("Model module %r does not provide '__all__'; falling back to '__dict__' for name discovery" % name)
exports = (name for name in dir(module) if len(name) > 0 and name[0] != "_")
# Register the association 'name -> constructor' for all the models
exported = False
for model in exports:
# Check model name type
if not isinstance(model, str):
tools.warning("Model module %r exports non-string name %r; ignored" % (name, model))
continue
# Recover instance from name
constructor = getattr(module, model, None)
# Check instance is callable (it's only an heuristic...)
if not callable(constructor):
continue
# Register callable with composite name
exported = True
fullname = "%s-%s" % (name, model)
if fullname in self.__models:
tools.warning("Unable to make available model %r from module %r, as the name %r already exists" % (model, name, fullname))
continue
self.__models[fullname] = constructor
if not exported:
tools.warning("Model module %r does not export any valid constructor name through '__all__'" % name)
def __init__(self, name, dict, ast_params):
self.access = "rw"
self.desc = ""
self.hide = False
self.init = None
self.mandatory = True
self.name = name
self.owned = True
self.serialize = True
self.type = ""
for key in dict:
if not key in ["access", "desc", "hide", "init", "mandatory", "owned", "serialize", "type"]:
warning("unknown Attribute attribute: " + key + " from " + name)
self.__dict__[key] = dict[key]
self.ast_params = ast_params
def register(name, unchecked, check, upper_bound=None, influence=None):
""" Simple registration-wrapper helper.
Args:
name GAR name
unchecked Associated function (see module description)
check Parameter validity check function
upper_bound Compute the theoretical upper bound on the ratio non-Byzantine standard deviation / norm to use this aggregation rule: (n, f, d) -> float
influence Attack acceptation ratio function
"""
global gars
# Check if name already in use
if name in gars:
tools.warning("Unable to register %r GAR: name already in use" % name)
return
# Export the selected function with the associated name
gars[name] = make_gar(unchecked, check, upper_bound=upper_bound, influence=influence)
def get_default_root(self):
""" Lazy-initialize and return the default dataset root directory path.
Returns:
'__default_root'
"""
# Fast-path already loaded
if self.__default_root is not None:
return self.__default_root
# Generate the default path
self.__default_root = pathlib.Path(__file__).parent / "datasets" / "cache"
# Warn if the path does not exist and fallback to '/tmp'
if not self.__default_root.exists():
tmpdir = tempfile.gettempdir()
tools.warning(f"Default dataset root {str(self.__default_root)!r} does not exist, falling back to local temporary directory {tmpdir!r}", context="experiments")
self.__default_root = pathlib.Path(tmpdir)
# Return the path
return self.__default_root
def summary_thread(coord, mngr, sess, path, rstrd):
""" Summary thread entry point.
Args:
coord Coordinator to use
mngr Graph manager to use
sess Session to use
path Path to the manager to use
rstrd Whether the model was just restored from a checkpoint
"""
global args
delta = args.summary_delta
period = args.summary_period
if delta < 0 and period < 0: # Effectively disabled
tools.info("Summary saving is effectively disabled")
return
if mngr.summary_tn is None:
tools.warning("No summary to save")
return
if rstrd:
last_step = sess.run(mngr.step)
last_time = time.time()
else:
last_step = -delta
last_time = -period
# Save summaries
with mngr.graph.as_default():
with tf.summary.FileWriter(args.summary_dir,
graph=mngr.graph) as writer:
writer.add_session_log(tf.SessionLog(status=tf.SessionLog.START),
sess.run(mngr.step))
while True:
time.sleep(config.thread_idle_delay)
step = sess.run(mngr.step)
now = time.time()
stop = coord.should_stop()
if stop or (delta >= 0 and step - last_step >= delta) or (
period >= 0. and now - last_time >= period):
writer.add_summary(sess.run(mngr.summary_tn), step)
tools.info("Summaries saved (took " +
repr(time.time() - now) + " s)")
last_step = sess.run(mngr.step)
last_time = time.time()
if stop:
break
writer.add_session_log(tf.SessionLog(status=tf.SessionLog.STOP),
step)
def attach_canton_parents(db, filename):
info('Attaching French Canton to their parents')
canton_processed = 0
for zone in db.find({'level': canton.id}):
candidates_ids = [p for p in zone['parents'] if p.startswith(county.id)]
if len(candidates_ids) < 1:
warning('No parent candidate found for: {0}'.format(zone['_id']))
continue
county_id = candidates_ids[0]
county_zone = db.find_one({'_id': county_id})
ops = {
'$addToSet': {'parents': {'$each': county_zone['parents']}},
'$unset': {'_dep': 1}
}
if db.find_one_and_update({'_id': zone['_id']}, ops):
canton_processed += 1
success('Attached {0} french cantons to their parents'.format(canton_processed))
def __init__(self, name, dict, ast_params):
self.ast_params = ast_params
self.attributes = []
self.clone_by_ref = False
self.cloner_prologue = ""
# Is the class concrete? (Default is false.)
self.concrete = False
self.declAttribute = ""
self.derived = []
self.desc = ""
self.hide = False
self.includes = {}
self.inline = {}
self.name = name
self.super = ""
self.super_non_nodes = []
for key in dict:
# Catch duplicate keys.
if isinstance(key, tuple):
(realkey, value) = key
error("duplicate key: " + name + "::" + realkey)
if key not in [
"attributes",
"concrete",
"declAttribute",
"default",
"desc",
"hide",
"inline",
"printer",
"super",
"cloner_prologue",
"clone_by_ref",
]:
warning("unknown Node attribute: " + name + "::" + key)
self.__dict__[key] = dict[key]
# If we have only one super-class, it has been parsed as a single
# value, but we want a list.
if not isinstance(self.super, list):
self.super = [self.super]
self.attributes = map(self.attribute_of_dict, self.attributes)
def __init__(self, name, dict, ast_params):
self.access = "rw"
self.desc = ""
self.hide = False
self.init = None
self.mandatory = True
self.name = name
self.owned = True
self.serialize = True
self.type = ""
for key in dict:
if not key in [
'access',
'desc',
'hide',
'init',
'mandatory',
'owned',
'serialize',
'type',
]:
warning ('unknown Attribute attribute: ' + key + ' from ' + name)
self.__dict__[key] = dict[key]
self.ast_params = ast_params
def build_aggregate(self, code, name, zones, properties, db):
geoms = []
populations = []
areas = []
for zoneid in zones:
# Resolve wildcard
if zoneid.endswith('/*'):
level = zoneid.replace('/*', '')
ids = db.distinct('_id', {'level': level})
resolved = self.build_aggregate(code, name, ids, properties, db)
geoms.append(shape(resolved['geom']))
if resolved.get('population'):
populations.append(resolved['population'])
if resolved.get('area'):
areas.append(resolved['area'])
else:
zone = db.find_one({'_id': zoneid})
if not zone:
warning('Zone {0} not found'.format(zoneid))
continue
shp = shape(zone['geom'])
if not shp.is_valid:
warning('Skipping invalid polygon for {0}'.format(zone['name']))
continue
if shp.is_empty:
warning('Skipping empty polygon for {0}'.format(zone['name']))
continue
geoms.append(shp)
if zone.get('population'):
populations.append(zone['population'])
if zone.get('area'):
areas.append(zone['area'])
geom = cascaded_union(geoms)
if geom.geom_type == 'Polygon':
geom = MultiPolygon([geom])
data = {
'_id': '/'.join((self.id, code)),
'code': code,
'level': self.id,
'name': name,
'population': sum(populations),
'area': sum(areas),
'geom': geom.__geo_interface__
}
data.update(properties)
return data
