def init_vk(self):
"""Инициализация сессии ВК API"""
if self.token:
self.api_session = TokenSession(access_token=self.token,
driver=RatedDriver())
self.group_api = aiovk.API(self.api_session)
if self.login and self.password:
self.login = self.login
self.password = self.password
self.api_session = ImplicitSession(
self.login,
self.password,
self.appid,
scope=self.scope,
driver=RatedDriver()) # all scopes
self.user_api = aiovk.API(self.api_session)
if not self.user_api and not self.group_api:
fatal(
'Вы попытались инициализировать объект класса VkPlus без данных для авторизации!'
)
# Паблик API используется для методов, которые не нуждаются в регистрации (users.get и т.д)
# Используется только при access_token сообщества вместо аккаунта
if self.token:
self.public_api_session = TokenSession(driver=RatedDriver())
self.public_api = aiovk.API(self.public_api_session)
def __init__(self, in_base_path, in_path, out_path):
self.directory = os.path.basename(in_path);
self.in_path = in_path;
self.out_path = out_path + "/" + self.directory;
self.small_dirname = "small";
self.date_str = "";
utils.makedirs(self.out_path + "/" + self.small_dirname);
try:
self.date = time.strptime(self.directory[:10], "%Y-%m-%d");
except ValueError as e:
self.date = None;
pass;
self.attachments = [];
for f in os.listdir(in_path):
f_full = in_path + "/" + f;
if (os.path.isfile(f_full)):
if (f[-4:] == ".mml"):
if (hasattr(self, "page_path")):
utils.fatal("multipe content found in " + in_path);
self.page_path = f_full;
self.processHeader();
else:
self.attachments.append(f);
self.processAttachments();
def __init__(self):
if not os.path.exists(CONFIG_PATH):
info("No configuration was found, first time setup is required!")
print(" Please input your web admin details below.")
config = self.construct_config_interactive()
with open(CONFIG_PATH, 'w') as config_file:
config.write(config_file)
try:
self.config = configparser.ConfigParser()
self.config.read(CONFIG_PATH)
except configparser.DuplicateOptionError as e:
fatal(
"Configuration error(s) found!\nSection '{}' has a duplicate setting: '{}'."
.format(e.section, e.option))
die(CONFIG_DIE_MESG, pause=True)
config_errors = self.validate_config(self.config)
if config_errors:
fatal("Configuration error(s) found!")
for error in config_errors:
print("\t\t" + error)
die(CONFIG_DIE_MESG, pause=True)
async def method(self, key: str, data: dict = {}):
"""Выполняет метод API VK с опциональными параметрами"""
# Если у нас TOKEN, то для всех остальных методов,
# кроме разрешённых, вызовем публичное API
if key not in self.group_methods and self.token and 'message' not in key:
api_method = self.public_api
else:
api_method = self.api
try:
return await api_method(key, **data)
except (asyncio.TimeoutError, json.decoder.JSONDecodeError):
# Пытаемся отправить запрос к API ещё раз
return await api_method(key, **data)
except aiovk.exceptions.VkAuthError:
message = 'TOKEN' if self.token else 'LOGIN и PASSWORD'
fatal(f"Произошла ошибка при авторизации API, "
"проверьте значение полей {message} в settings.py!")
except aiovk.exceptions.VkAPIError as ex:
if ex.error_code == 9:
# Flood error - слишком много одинаковых сообщений
if 'message' not in data:
return {}
data['message'] += f'\n Анти-флуд (API): {self.anti_flood()}'
try:
# Пытаемся отправить сообщение ещё раз
await self.method('messages.send', data)
except aiovk.exceptions.VkAPIError:
# В реальности такое будет очень редко
# или вообще не случится
hues.error('Обход анти-флуда API не удался =(')
else:
hues.error(
f"Произошла ошибка при вызове метода API {key} с значениями {data}:\n{ex}"
)
return {}
def execute(self,
purpose,
sindarin,
fifo=None,
proc_id=None,
options='',
analysis=''):
cmd = 'nice -n 9 ' + self.binary + ' ' + sindarin + ' ' + options
if (purpose == 'histograms'):
cmd = 'export RIVET_ANALYSIS_PATH=../../rivet; ' + cmd
yoda_file = '../../rivet/' + fifo.replace('hepmc', 'yoda')
cmd = cmd + ' & rivet --quiet -H ' + yoda_file + ' -a ' + analysis + ' ' + fifo
num = ' in ' + str(proc_id) if proc_id is not None else ''
ut.logger.info('Calling subprocess ' + cmd + num)
try:
return_code = self.call(cmd)
except Exception as e:
ut.fatal('Exception occured: ' + str(e) +
'Whizard failed on executing ' + sindarin + num)
return FAIL
else:
if not ut.grep('ERROR', 'whizard.log'):
ut.logger.info('Whizard finished' + num)
ut.touch('done')
return return_code
else:
ut.fatal('ERROR in whizard.log of ' + sindarin + num)
return FAIL
async def init_long_polling(self, update=0):
"""Функция для инициализации Long Polling"""
retries = 5
for x in range(retries):
result = await self.vk.method('messages.getLongPollServer',
{'use_ssl': 1})
if result:
break
if not result:
fatal("Не удалось получить значения Long Poll сервера!")
if update == 0:
# Если нам нужно инициализировать с нуля, меняем сервер
self.longpoll_server = "https://" + result['server']
if update in (0, 3):
# Если нам нужно инициализировать с нуля, или ошибка 3
self.longpoll_key = result['key']
self.last_ts = result['ts'] # Последний timestamp
elif update == 2:
# Если ошибка 2 - то нужен новый ключ
self.longpoll_key = result['key']
self.longpoll_values = {
'act': 'a_check',
'key': self.longpoll_key,
'ts': self.last_ts,
'wait': 20, # Тайм-аут запроса
'mode': 2,
'version': 1
}
def main():
if len(sys.argv) != 2:
usage()
mnt = sys.argv[1]
try:
useraufs.umount(mnt)
except useraufs.Error, e:
fatal(e)
def create_layer(ods, ilayer, layerName, geom_name, create_geom, options):
ofrmt = ods.GetDriver().GetName()
# determine geometry type
if geom_name or not create_geom:
feat_defn = ilayer.GetLayerDefn()
if geom_name:
idx = feat_defn.GetGeomFieldIndex(geom_name)
else:
idx = 0
if idx > -1:
geom_type = feat_defn.GetGeomFieldDefn(idx).GetType()
else:
# warning("Layer '%s': geometry '%s' not available" % (layerName, geom_name))
geom_type = ilayer.GetGeomType()
idx = 0
if ofrmt in ('PostgreSQL', 'OCI'):
remove_option(options, 'GEOMETRY_NAME')
options.append('GEOMETRY_NAME=%s' %
feat_defn.GetGeomFieldDefn(idx).GetName().lower())
else:
geom_type = ogr.wkbNone
# create new layer
olayer = ods.CreateLayer(layerName, ilayer.GetSpatialRef(), geom_type,
options)
if not olayer:
fatal("Unable to create layer '%'" % layerName)
# create attributes
feat_defn = ilayer.GetLayerDefn()
for i in range(feat_defn.GetFieldCount()):
ifield = feat_defn.GetFieldDefn(i)
ofield = ogr.FieldDefn(ifield.GetNameRef(), ifield.GetType())
ofield.SetWidth(ifield.GetWidth())
if ofrmt == 'ESRI Shapefile':
# StringList not supported by Esri Shapefile
if ifield.GetType() in (ogr.OFTIntegerList, ogr.OFTRealList,
ogr.OFTStringList):
ofield.SetType(ogr.OFTString)
olayer.CreateField(ofield)
# create also geometry attributes
if not geom_name and \
olayer.TestCapability(ogr.OLCCreateGeomField):
for i in range(feat_defn.GetGeomFieldCount()):
geom_defn = feat_defn.GetGeomFieldDefn(i)
if geom_name and geom_defn.GetName() != geom_name:
continue
olayer.CreateGeomField(feat_defn.GetGeomFieldDefn(i))
return olayer
def main():
args = sys.argv[1:]
if len(args) < 1:
usage()
mnt = args[0]
operations = args[1:]
try:
useraufs.remount(mnt, operations)
except useraufs.Error, e:
fatal(e)
def main():
args = sys.argv[1:]
if len(args) < 1:
usage()
mnt = args[0]
operations = args[1:]
try:
useraufs.remount(mnt, operations)
except useraufs.Error, e:
fatal(e)
def is_valid_wizard_sindarin(proc_dict, template_sindarin):
purpose = proc_dict['purpose']
proc_id = ut.get_process(template_sindarin)
valid = True
if proc_id != 'PROCESS' and \
proc_id != template_sindarin.replace('-template.sin', ''):
ut.fatal('The process doesnt have the same name as the file')
valid = False
if purpose == 'scan' and not ut.grep('#SETSCAN', template_sindarin):
ut.fatal('Your purpose is scan but the sindarin has no #SETSCAN')
valid = False
elif (purpose == 'nlo' or purpose == 'nlo_combined') and \
not is_nlo_calculation(template_sindarin):
ut.fatal('Your purpose is nlo* but the sindarin has no nlo command')
valid = False
fks_mapping = ut.get_string("fks_mapping_type", template_sindarin)
resonance_set_in_sindarin = fks_mapping == '"resonances"'
fks_method = proc_dict.get('fks_method', 'default')
resonance_not_set_in_json = fks_method != 'resonances'
if resonance_set_in_sindarin and resonance_not_set_in_json:
ut.fatal(
'You set fks_mapping_type to resonances but havent set it in the run.json'
)
valid = False
return valid
def fill_all_scan_runs(proc_name, proc_dict):
try:
scans = proc_dict['scans']
except KeyError:
ut.fatal('Aborting: You want a scan but have not set a scans array')
return []
else:
list_of_scans = [
fill_scan_runs(proc_name, proc_dict, scan) for scan in scans
]
full_combination = [sa for sa in dt.Scan_all(list_of_scans)]
runs = unpack_proc_ids(full_combination)
runs = fill_with_copies(proc_dict, runs)
return runs
def init_settings(self):
"""Функция инициализации файла настроек и его создания"""
# Если у нас есть только settings.py.sample
if isfile('settings.py.sample') and not isfile('settings.py'):
try:
shutil.copy('settings.py.sample', 'settings.py')
except Exception:
fatal('Я не могу копировать файлы в текущей папке, '
'проверьте ваши права на неё!')
hues.info('Был создан файл settings.py, '
'не забудьте добавить данные для авторизации!')
exit()
# Если у нас уже есть settings.py
elif isfile('settings.py'):
import settings
try:
self.BLACKLIST = settings.BLACKLIST
self.PREFIXES = settings.PREFIXES
self.LOG_MESSAGES = settings.LOG_MESSAGES
self.LOG_COMMANDS = settings.LOG_COMMANDS
self.APP_ID = settings.APP_ID
self.SCOPE = settings.SCOPE
self.FLOOD_INTERVAL = settings.FLOOD_INTERVAL
self.USERS = settings.USERS
self.PROXIES = settings.PROXIES
if not self.USERS:
fatal("Проверьте, что у есть LOGIN и PASSWORD, или же TOKEN в файле settings.py!"
"Без них бот работать НЕ СМОЖЕТ.")
except (ValueError, AttributeError, NameError):
fatal('Проверьте содержимое файла settings.py, возможно вы удалили что-то нужное!')
# Если не нашли ни settings.py, ни settings.py.sample
else:
fatal("settings.py и settings.py.sample не найдены, возможно вы их удалили?")
async def auth_captcha_is_needed(response, login_form_data, captcha_url, session):
response_url_dict = get_url_query(response.url)
captcha_form_action = get_form_action((await response.text()))
if not captcha_form_action:
fatal('Cannot find form url in captcha')
captcha_url = '%s?s=%s&sid=%s' % (captcha_url, response_url_dict['s'], response_url_dict['sid'])
login_form_data['captcha_sid'] = response_url_dict['sid']
login_form_data['captcha_key'] = await vkplus.enter_captcha(captcha_url)
async with session.post(captcha_form_action, data=login_form_data) as resp:
await resp.text()
def setup_sindarin(proc_dict):
if not proc_dict.get('disabled', False):
ut.logger.info('Setting up sindarins of ' + str(proc_dict))
whizard_folder = 'whizard'
with ut.cd(whizard_folder):
base_sindarin = proc_dict['process'] + '.sin'
template_sindarin = base_sindarin.replace('.sin', '-template.sin')
check_for_valid_wizard_sindarin(proc_dict, template_sindarin)
integration_sindarin = base_sindarin.replace(
'.sin', '-integrate.sin')
template_present = os.path.isfile(template_sindarin)
scan = proc_dict['purpose'] == 'scan'
test_soft = proc_dict['purpose'] == 'test_soft'
if template_present:
if proc_dict['purpose'] == 'integration' or scan or test_soft:
create_integration_sindarin(
integration_sindarin, template_sindarin,
proc_dict['process'], proc_dict['adaption_iterations'],
proc_dict.get('integration_iterations', ' '))
multiply_sindarins(integration_sindarin, proc_dict,
proc_dict.get('scale_variation', False),
proc_dict['nlo_type'])
elif proc_dict['purpose'] == 'histograms' or proc_dict[
'purpose'] == 'events':
create_simulation_sindarin(
base_sindarin, template_sindarin, proc_dict['process'],
proc_dict['adaption_iterations'],
proc_dict.get('integration_iterations',
' '), proc_dict['events_per_batch'])
multiply_sindarins(base_sindarin, proc_dict,
proc_dict.get('scale_variation', False),
proc_dict['nlo_type'])
else:
if scan:
ut.fatal('You have to supply ' + template_sindarin +
' for a scan')
else:
fallback = integration_sindarin + ' and ' + base_sindarin
if os.path.isfile(integration_sindarin) and os.path.isfile(
base_sindarin):
ut.logger.info('Didnt find ' + template_sindarin +
', will use ' + fallback)
return
else:
ut.fatal('Didnt find ' + template_sindarin + ' nor ' +
fallback)
else:
ut.logger.info('Skipping ' + proc_dict['process'] +
' because it is disabled')
def __get_players(info_tree):
players = []
# Empty servers only have a single <td> with an empty server message
is_empty_path = "//table[@id=\"players\"]/tbody//td"
is_empty_result = info_tree.xpath(is_empty_path)
if len(is_empty_result) == 1:
return players
theads_path = "//table[@id=\"players\"]/thead//th//text()"
theads_result = [head.lower() for head in info_tree.xpath(theads_path)]
if theads_result:
name_col = theads_result.index("name") \
if "name" in theads_result else None
perk_col = theads_result.index("perk") \
if "perk" in theads_result else None
dosh_col = theads_result.index("dosh") \
if "dosh" in theads_result else None
health_col = theads_result.index("health") \
if "health" in theads_result else None
kills_col = theads_result.index("kills") \
if "kills" in theads_result else None
ping_col = theads_result.index("ping") \
if "ping" in theads_result else None
# admin_col = theads_result.index("admin") \
# if "admin" in theads_result else None
else:
fatal("Couldn't find server info headings")
return players
# xpath to <td>s and retrieve text manually to catch empty cells
trows_path = "//table[@id=\"players\"]/tbody/tr"
trows_result = info_tree.xpath(trows_path)
for player_row in trows_result:
player_columns = player_row.xpath("td")
player = ConstPlayer(
str((player_columns[name_col].text if name_col else "Unnamed")
or "Unnamed"),
str((player_columns[perk_col].text if perk_col else "Unknown")
or "Unknown"),
int((player_columns[kills_col].text if perk_col else 0) or 0),
int((player_columns[health_col].text if perk_col else 0) or 0),
int((player_columns[dosh_col].text if perk_col else 0) or 0),
int((player_columns[ping_col].text if ping_col else 0) or 0))
players.append(player)
return players
def init_settings(self):
"""Функция инициализации файла настроек и его создания"""
# Если у нас есть только settings.py.sample
if isfile('settings.py.sample') and not isfile('settings.py'):
try:
shutil.copy('settings.py.sample', 'settings.py')
except Exception:
fatal('Я не могу копировать файлы в текущей папке, '
'проверьте ваши права на неё!')
fatal('Был создан файл settings.py, '
'не забудьте добавить данные для авторизации!')
# Если у нас уже есть settings.py
elif isfile('settings.py'):
import settings
try:
self.BLACKLIST = settings.BLACKLIST
self.PREFIXES = settings.PREFIXES
self.LOG_MESSAGES = settings.LOG_MESSAGES
self.LOG_COMMANDS = settings.LOG_COMMANDS
self.NEED_CONVERT = settings.NEED_CONVERT
self.APP_ID = settings.APP_ID
self.SCOPE = settings.SCOPE
self.FLOOD_INTERVAL = settings.FLOOD_INTERVAL
# По умолчанию - False
self.IS_TOKEN = False
# Если в настройках есть токен
if settings.TOKEN:
self.IS_TOKEN = True
self.TOKEN = settings.TOKEN
# Или логин и пароль
if settings.LOGIN and settings.PASSWORD:
self.VK_LOGIN = settings.LOGIN
self.VK_PASSWORD = settings.PASSWORD
if not self.TOKEN and not (self.VK_LOGIN and self.VK_PASSWORD):
fatal(
"Проверьте, что у есть LOGIN и PASSWORD, или же TOKEN в файле settings.py!"
"Без них бот работать НЕ СМОЖЕТ.")
except (ValueError, AttributeError, NameError):
fatal(
'Проверьте содержимое файла settings.py, возможно вы удалили что-то нужное!'
)
# Если не нашли ни settings.py, ни settings.py.sample
else:
fatal(
"settings.py и settings.py.sample не найдены, возможно вы их удалили?"
)
def __init__(self, run_json, verbose):
if not verbose:
devnull = open(os.devnull, 'w')
self.out = devnull
self.err = devnull
self.call = lambda cmd: subprocess.call(
cmd, shell=True, stderr=self.err, stdout=self.out)
else:
self.call = lambda cmd: subprocess.call(cmd, shell=True)
self.binary = run_json.get('whizard', 'whizard')
if not spawn.find_executable(self.binary):
ut.fatal('No valid whizard found. You gave whizard = ' +
self.binary)
self.call = lambda cmd: FAIL
else:
ut.logger.info('Using ' + self.binary)
def get_steps(rnge, start, stop):
steps, stepsize, given_range = None, None, None
try:
steps = rnge['steps']
stepsize = (stop - start) / steps
except KeyError:
try:
stepsize = rnge['stepsize']
steps = (stop - start) / stepsize
except KeyError:
given_range = rnge['range']
except KeyError:
ut.fatal(
'Aborting: You have to give either steps, stepsize or explicit range'
)
return steps, stepsize, given_range
def match_cb(m, self=self):
do_esc = True;
s = m.group(1);
if (s[0] == "-"):
do_esc = False;
s = s[1:];
# mouhahaha
try:
r = eval(s, globals(), context);
if (do_esc):
#print("before " + r);
r = escape(r);
#print("after " + r);
return r;
except:
utils.fatal("error on eval(" + m.group(1) + ")");
async def method(self, key: str, data=None):
"""Выполняет метод API VK с дополнительными параметрами"""
if data is None:
data = {}
# Если мы работаем от имени группы
if self.token:
# Если метод доступен от имени группы - используем API группы
if is_available_from_group(key):
api_method = self.group_api
# Если метод доступен от паблик апи - используем его
elif is_available_from_public(key):
api_method = self.public_api
elif self.user_api:
api_method = self.user_api
else:
hues.warn(f'Метод {key} нельзя вызвать от имени сообщества!')
return {}
else:
api_method = self.user_api
try:
return await api_method(key, **data)
except (asyncio.TimeoutError, json.decoder.JSONDecodeError):
# Пытаемся отправить запрос к API ещё раз
return await api_method(key, **data)
except aiovk.exceptions.VkAuthError:
message = 'TOKEN' if self.token else 'LOGIN и PASSWORD'
fatal("Произошла ошибка при авторизации API, "
f"проверьте значение {message} в settings.py!")
except aiovk.exceptions.VkAPIError as ex:
# Код 9 - Flood error - слишком много одинаковых сообщений
if not ex.error_code == 9:
hues.error("Произошла ошибка при вызове метода API "
f"{key} с значениями {data}:\n{ex}")
return {}
if 'message' not in data:
return {}
data['message'] += f'\n Анти-флуд (API): {self.anti_flood()}'
try:
# Пытаемся отправить сообщение ещё раз
await self.method('messages.send', data)
except aiovk.exceptions.VkAPIError:
# Не знаю, может ли это случиться, или нет
hues.error('Обход анти-флуда API не удался =(')
return {}
def fill_scan_runs(proc_name, proc_dict, scan):
runs = []
try:
scan_object = scan['scan_object']
except KeyError:
ut.fatal('Aborting: You want a scan but have not set a scan_object')
raise
ranges = scan['ranges']
for rnge in ranges:
start = float(rnge.get('start', 0.0))
stop = float(rnge.get('stop', 0.0))
range_type = rnge['type']
steps, stepsize, given_range = get_steps(rnge, start, stop)
step_range = get_step_range(range_type, start, stop, steps, stepsize,
given_range)
runs += [(str(sr).replace('-', 'MINUS') + '-' + scan_object, proc_name,
proc_dict) for sr in step_range]
return runs
async def init_long_polling(self):
"""Функция для инициализации Long Polling"""
result = await self.vk.method('messages.getLongPollServer',
{'use_ssl': 1})
if not result:
fatal("Не удалось получить значения Long Poll сервера!")
self.longpoll_server = "https://" + result['server']
self.longpoll_key = result['key']
self.last_ts = result['ts'] # Последний timestamp
self.longpoll_values = {
'act': 'a_check',
'key': self.longpoll_key,
'ts': self.last_ts,
'wait': 20, # Тайм-аут запроса
'mode': 2,
'version': 1
}
def add(self, rule):
"""Add a rule"""
if isinstance(rule, FileRule):
self._file_rules.append(rule)
elif isinstance(rule, LexRule):
self._lex_rules.append(rule)
elif isinstance(rule, SyntaxRule):
self._syntax_rules.append(rule)
elif isinstance(rule, SyntaxNodeRule):
self._syntax_node_rules.append(rule)
elif isinstance(rule, SemRule):
self._sem_rules.append(rule)
elif isinstance(rule, SemNodeRule):
self._sem_node_rules.append(rule)
elif isinstance(rule, SynthesisRule):
self._synth_rules.append(rule)
else:
fatal("unknown class for rule {0}".format(rule.rulename))
rule.set_runner(self)
def get_step_range(scan_type, start, stop, steps, stepsize, given_range):
if scan_type == 'logarithmic':
step_range = logspace(log10(start),
log10(stop),
num=steps,
endpoint=True,
base=10.0)
elif scan_type == 'logarithmic2':
step_range = logspace(log2(start),
log2(stop),
num=steps,
endpoint=True,
base=2.0)
elif scan_type == 'linear':
step_range = arange(start, stop, float(stepsize))
elif scan_type == 'explicit':
step_range = given_range
else:
ut.fatal('Aborting: Unknown scan type')
return step_range
def processAttachments(self):
if (not hasattr(self, 'thumbnail')):
for f in self.attachments:
if (os.path.splitext(f)[0] == "thumbnail"):
self.thumbnail = f;
if (not hasattr(self, 'thumbnail')):
if (len(self.attachments) == 0):
utils.fatal("no thumbnail can be found in " + self.in_path + "\n");
print("warning: using " + self.attachments[0] + " as thumbnail");
self.thumbnail = self.attachments[0];
for f in self.attachments:
attachment_path = self.out_path + "/" + f;
if (not os.path.isfile(attachment_path)):
print("processing attachment " + attachment_path + "...");
shutil.copy(self.in_path + "/" + f, attachment_path);
if (is_picture(f)):
self.processPicture(f);
def processHeader(self):
fd = utils.open_file(self.page_path, "rb");
header_json="";
m="";
in_header = True;
for line in fd:
if (in_header):
if (line == "\n"):
in_header = False;
header_json += line;
else:
m += line;
try:
self.header = json.loads(header_json);
except ValueError as e:
utils.fatal("malformed content header in " + self.page_path + ":\n" + str(e));
for k in self.header:
setattr(self, k, self.header[k]);
self.html = markdown.markdown(m);
def __init__(self, config_filename, skip_setup=False, reconfigure=False):
if not os.path.exists(config_filename) or reconfigure:
if not reconfigure:
info(
_("No configuration was found, first time setup is "
"required!"))
if not skip_setup:
config = self.construct_config_interactive()
else:
config = self.construct_config_template()
with open(config_filename, 'w') as config_file:
config.write(config_file)
if skip_setup:
info(
_("Guided setup was skipped, a template has been "
"generated."))
die(
_("Setup is not complete yet, please amend '{}' with your "
"server details.").format(CONFIG_PATH_DISPLAY))
try:
self.config = configparser.ConfigParser()
self.config.read(config_filename)
except configparser.DuplicateOptionError as e:
fatal(
_("Configuration error(s) found!\nSection '{}' has a "
"duplicate setting: '{}'.").format(e.section, e.option))
die(CONFIG_DIE_MESG, pause=True)
config_errors = self.validate_config(self.config)
if config_errors:
fatal(_("Configuration error(s) found!"))
for error in config_errors:
print("\t\t" + error)
die(CONFIG_DIE_MESG, pause=True)
async def login(username, password, session):
captcha_url = 'https://m.vk.com/captcha.php'
url_login = "******"
login_form_data = {
'email': username,
'pass': password,
}
async with session.get(url_login) as resp:
html = await resp.text()
login_form_action = get_form_action(html)
if not login_form_action:
fatal("VK changed authentication flow")
async with session.post(login_form_action, data=login_form_data) as resp:
html = await resp.text()
response_url_query = get_url_query(resp.url)
cookies = [cookie.key for cookie in session.cookie_jar]
if 'remixsid' in cookies or 'remixsid6' in cookies:
return
if 'sid' in response_url_query:
await auth_captcha_is_needed(resp, login_form_data, captcha_url, session)
elif response_url_query.get('act') == 'authcheck':
await auth_check_is_needed(await resp.text(), session)
elif 'security_check' in response_url_query:
hues.error("Phone number is needed")
else:
hues.error("Authorization error (incorrect password)")
def generate_thumbnail(_in, _out_dir):
# XXX: not nice but much faster if we have them generated already
if (os.path.isfile(_out_dir + "/thumbnail_green.png")):
return;
_tmp_square = tempfile.mkstemp(suffix=".png", prefix="mbonnin.net")[1];
_tmp_round = tempfile.mkstemp(suffix=".png", prefix="mbonnin.net")[1];
_tmp_round_bw = tempfile.mkstemp(suffix=".png", prefix="mbonnin.net")[1];
# make input square
proc = utils.execute_shell("convert -resize 256x256^ -gravity center -extent 256x256 " + _in + " " + _tmp_square);
if (proc.returncode != 0):
utils.fatal("could not square thumbnail " + _in +": " + proc.err);
# clip
proc = utils.execute_shell("convert " + in_dir + "/thumbnail_clip_mask.png " + _tmp_square + " -compose src-in -composite " + _tmp_round);
if (proc.returncode != 0):
utils.fatal("could not generate_thumbnail " + _in +": " + proc.err);
# desaturate
desaturate(_tmp_round, _tmp_round_bw);
for color in ["green", "yellow"]:
for bw in ["", "_bw"]:
if (bw == "_bw"):
r = _tmp_round_bw;
else:
r = _tmp_round;
_out = _out_dir + "/thumbnail_" + color + bw + ".png";
proc = utils.execute_shell("convert " + in_dir + "/thumbnail_border_" + color + ".png " + r + " -composite " + _out);
if (proc.returncode != 0):
utils.fatal("could not generate_thumbnail " + _in +": " + proc.err);
os.remove(_tmp_square);
os.remove(_tmp_round);
os.remove(_tmp_round_bw);
def test_logging():
warning("something", log=True, display=True)
debug("something", log=True, display=True)
info("something", log=True, display=True)
fatal("something", log=True, display=True)
def main():
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], 'h', ['udba='])
except getopt.GetoptError, e:
usage(e)
udba_level = None
for opt, val in opts:
if opt == '--udba':
if val not in ('none', 'reval', 'inotify'):
usage("invalid udba level (%s)" % val)
udba_level = val
elif opt == '-h':
usage()
if len(args) < 2:
usage()
mnt = args[0]
branches = args[1:]
try:
useraufs.mount(mnt, branches, udba=udba_level)
except useraufs.Error, e:
fatal(e)
if __name__=="__main__":
main()
args = parser.parse_args()
run_json = whizard_wizard.retrieve_and_validate_run_json(
args.process_folder)
whizard_wizard.setup_sindarins(run_json)
else:
run_json = None
args = None
run_json = comm.bcast(run_json, root=0)
args = comm.bcast(args, root=0)
whizard = whizard_wizard.Whizard(run_json, args.verbose)
comm.Barrier()
runs = whizard_wizard.fill_all_runs(run_json)
results = mpi_map(whizard.run_process, runs)
if comm.Get_rank() == 0:
logger.info('This is the MPI master: All processes returned.')
for res, run in zip(results, runs):
if res != whizard_wizard.SUCCESS:
fatal('Unsuccessful run occured. Return code: ' + str(res) +
' in this run ' + str(run))
logger.info("""
#==============================================================================#
# DONE #
#==============================================================================#
""")
def execute(self, files):
inputs = []
props = ['synth']
# First file
for filename in files:
if filename.startswith('--'):
# Handle properties
if filename == '--import':
props = ['import']
elif filename == '--synth':
props = ['synth']
elif filename == '--top':
props = ['synth', 'top']
elif filename == '--tb':
props = ['tb']
else:
fatal("unknown property '{0}'".format(filename))
continue
# Read the file
fid = thin.Get_Identifier(filename.encode('utf-8'))
fe = thin.Read_Source_File(0, fid)
if fe == thin.No_Source_File_Entry:
fatal('cannot open {0}'.format(filename))
fbuf = thin.Get_File_Buffer(fe)
flen = thin.Get_File_Length(fe)
# Not very efficient (it copies the string), but let's use it
# for now.
filebuf = ctypes.string_at(fbuf, flen)
input = RuleInput(filename, fe)
input.filebuf = filebuf
input.props = props
inputs.append(input)
if 'import' not in input.props:
self._nbr_files += 1
# Again, not very efficient (creates the substrings).
flines = filebuf.splitlines(True)
loc = Location(filename)
for r in self._file_rules:
r.check(loc, flines)
# Then tokens
thin.Scanner.Flag_Comment.value = True
for input in inputs:
if 'import' not in input.props:
thin.Scanner.Set_File(input.fe)
filebuf = input.filebuf
while True:
thin.Scanner.Scan()
tok = thin.Scanner.Current_Token.value
loc = TokLocation(input.filename,
thin.Scanner.Get_Current_Line(),
thin.Scanner.Get_Token_Column(),
thin.Scanner.Get_Token_Position(),
thin.Scanner.Get_Position())
if tok == tokens.Tok.Comment:
input.comments[loc.line] = (loc.start, loc.end)
for r in self._lex_rules:
r.check(loc, filebuf, tok)
if tok == tokens.Tok.Eof:
break
thin.Scanner.Close_File()
if not (self._syntax_rules or self._syntax_node_rules or
self._sem_rules or self._sem_node_rules or self._synth_rules):
return
# Then syntax
# The parser doesn't handle comments
thin.Scanner.Flag_Comment.value = False
# Keep extra locations
thin.Flags.Flag_Elocations.value = True
# Keep all parenthesis
thin.Parse.Flag_Parse_Parenthesis.value = True
# Be sure to initialize std and work (and only once).
# Humm, not very elegant.
if thin.Get_Libraries_Chain() == thin.Null_Iir:
thin.analyze_init()
for input in inputs:
thin.Scanner.Set_File(input.fe)
loc = Location(input.filename)
input.ast = thin.Parse.Parse_Design_File()
if 'import' not in input.props:
for r in self._syntax_rules:
r.check(input, input.ast)
if self._syntax_node_rules:
for n in thinutils.nodes_iter(input.ast):
for r in self._syntax_node_rules:
r.check(loc, n)
thin.Scanner.Close_File()
# Then semantic
if self._sem_rules or self._sem_node_rules or self._synth_rules:
# Reduce Canon
thin.Canon.Flag_Concurrent_Stmts.value = False
thin.Canon.Flag_Configurations.value = False
thin.Canon.Flag_Associations.value = False
# First add all units in the work library, so that they they are
# known by the analyzer.
for input in inputs:
unit_ast = iirs.Get_First_Design_Unit(input.ast)
while unit_ast != thin.Null_Iir:
# Detach the unit from its design file
next_unit_ast = iirs.Get_Chain(unit_ast)
iirs.Set_Chain(unit_ast, thin.Null_Iir)
# Add
thin.Add_Design_Unit_Into_Library(unit_ast, False)
input.units_ast.append(unit_ast)
unit_ast = next_unit_ast
# Handle all unit
for input in inputs:
if 'import' not in input.props:
for unit in input.units_ast:
if iirs.Get_Library_Unit(unit) == thin.Null_Iir:
# Over-written.
# FIXME: remove from the list ?
continue
# Be sure the unit was analyzed. It could have been
# already analyzed if referenced. And a unit cannot be
# analyzed twice.
if iirs.Get_Date_State(unit) == iirs.Date_State.Parse:
thin.Finish_Compilation(unit, False)
iirs.Set_Date_State(unit, iirs.Date_State.Analyze)
for r in self._sem_rules:
r.check(input, unit)
for n in thinutils.nodes_iter(unit):
for r in self._sem_node_rules:
r.check(input, n)
for input in inputs:
if 'synth' in input.props:
for unit in input.units_ast:
if iirs.Get_Library_Unit(unit) == thin.Null_Iir:
# Over-written.
continue
for r in self._synth_rules:
r.check(input, unit)
async def get_token(username, password, app_id, scope):
url_get_token = "https://oauth.vk.com/authorize"
session = aiohttp.ClientSession()
await login(username, password, session)
token_data = {
"client_id": app_id,
"redirect_uri": "https://oauth.vk.com/blank.html?",
"response_type": "token",
"scope": scope,
"display": "mobile",
"v": 5.64
}
headers = {
"user-agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36",
"accept-language":
"ru-RU,ru;q=0.8,en-US;q=0.6,en;q=0.4",
"accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
}
async with session.post(url_get_token, data=token_data,
headers=headers) as resp:
html = await resp.text()
response_url_query1 = get_url_query(resp.url)
if resp.history:
response_url_query2 = get_url_query(
resp.history[-1].headers["Location"])
else:
response_url_query2 = {}
if 'access_token' in response_url_query1:
return response_url_query1['access_token']
elif 'access_token' in response_url_query2:
return response_url_query2['access_token']
else:
form_action = get_form_action(html)
if form_action:
async with session.post(form_action, headers=headers) as resp:
html = await resp.text()
response_url_query1 = get_url_query(resp.url)
if resp.history:
response_url_query2 = get_url_query(
resp.history[-1].headers["Location"])
else:
response_url_query2 = {}
if 'access_token' in response_url_query1:
return response_url_query1['access_token']
elif 'access_token' in response_url_query2:
return response_url_query2['access_token']
fatal("Can't get token!")
print 'search for ', result_path + '/' + p + '.dat'
complete = complete and os.path.isfile(result_path + '/' + p +
'.dat')
return complete
else:
return False
def get_RES(c):
return "grep RES " + c + "-*/whizard.log | sed 's/^.*RES //'"
try:
process_folder = sys.argv[1]
except:
fatal('You have to give me the process directory as argument')
with cd(process_folder):
result_path = 'scan-results'
mkdirs(result_path)
run_json = load_json('run.json')
processes = [
p['process'] for p in run_json['processes'] if p['purpose'] == 'scan'
]
uses_scale_variation = [
p.get('scale_variation', False) for p in run_json['processes']
]
is_separate_nlo = [p['nlo_type'] == 'nlo' for p in run_json['processes']]
dirs = []
for p in run_json['processes']:
if p['purpose'] == 'scan':
dirs += get_full_proc_names(p['process'], p)
def desaturate(_in, _out):
proc = utils.execute_shell("convert " + _in + " -colorspace Gray " + _out);
if (proc.returncode != 0):
utils.fatal("could not desaturate " + _in +": " + proc.err);