def refresh(self, attempts=10):
"""Refreshes the rss feed.
Refreshes the contents received from the RSS feed and sets 'self.feed'
to the new feed contents.
Args:
attempts (int): number of attempts to refresh contents before
stopping stopping.
"""
for _ in range(attempts):
feed_update = feedparser.parse(self.rss_url)
# feedparser creates a 'status' attribute if there is a response
if 'status' in feed_update:
if feed_update.status == 304:
continue
elif feed_update.status == 200:
self.feed = feed_update
return
else:
LOGGER.error(f"feed_update: {feed_update.bozo_exception}")
LOGGER.info(f"{attempts} unnsuccessful attempts at refreshing data.")
def update_app(os_name: str) -> None:
"""
Скачивание программы обновления и запуск обновлений
:param os_name: имя OS
:return:
"""
LOGGER.info(f'Клонируем проект {os_name}')
response = requests.get(REPO_URL_UPDATER)
with tempfile.TemporaryFile() as file:
file.write(response.content)
with zipfile.ZipFile(file) as fzip:
fzip.extractall(path)
if os_name == 'Windows':
command = os.path.join(path_to_updater, UPDATE_WIN)
subprocess.Popen(command, cwd=path_to_updater)
exit_ex()
if os_name == 'Linux':
os.system(f'chmod -R 775 {path_to_updater}')
showwarning(
'Обновление', WARNING_MSG['Update_app']['Linux'].format(
branch=REPO_BRANCH_UPDATER, file=UPDATE_LINUX))
exit_ex()
def create_customer(self, data: dict):
try:
LOGGER.debug('Creating customer service')
self.__validations.data(data=data)
customer = Customer(**data)
customer = self.__database.save(document=customer)
return self.__parser.raw_to_json(
data=customer.to_mongo().to_dict())
except NotUniqueError:
LOGGER.info('Email already used')
raise EmailAlreadyUsedException()
except ValidationError as exception:
LOGGER.debug(traceback.format_exc())
LOGGER.debug('Invalid fields value')
raise InvalidFieldsValuesException(validations=exception.to_dict())
except MissingRequiredFieldsException as exception:
raise exception
except Exception as exception:
LOGGER.debug(traceback.format_exc())
LOGGER.critical(exception)
raise exception
def create_entry(data):
"""Creates a key/value store of data for a SEC filing
The data that comes from the RSS feed does not contain detailed information
on the owner of the new issuance, and details about the issuer such as their
stock ticker. This function creates a filing entry first hierchy that are
described by the filing attributes such as issuer, owner, and the details
of each transaction.
Args:
data (feedparser.FeedParserDict): a key/value store that desribes the
typical features of an RSS feed. Derived from parsing an RSS feed.
Returns:
dict: a key/value store that describes the attributes of an SEC filing.
"""
for tag in data.tags:
form_type = tag.term if tag.label == 'form type' else None
if form_type is None:
LOGGER.info("No form type detected: {accession_num}.")
return dict(
created=datetime.now(),
accession_num=data.summary.split('<b>AccNo:</b> ')[1][:20],
updated=data.updated,
datalink=data.links[0].href,
issuer=dict().fromkeys(ISSUER_DSCR + ADDRESS_DSCR, None),
owner=dict().fromkeys(OWNER_DSCR + ADDRESS_DSCR, None),
nonderivatives=[],
derivatives=[],
form_type=form_type,
)
def update_customer(self, data: dict, customer_id: str):
try:
is_object_id(object_id=customer_id)
self.__validations.data(data=data, is_update=True)
LOGGER.debug(f'Updating customer service {customer_id}')
customer = self.__database.update_customer_by_id(
customer_id=customer_id, data=data)
return self.__parser.raw_to_json(
data=customer.to_mongo().to_dict())
except NotUniqueError:
LOGGER.info('Email already used')
raise EmailAlreadyUsedException()
except ValueError:
LOGGER.debug(traceback.format_exc())
LOGGER.debug('Invalid fields value')
raise InvalidFieldsValuesException(validations={'field': ''})
except ValidationError as exception:
LOGGER.debug(traceback.format_exc())
LOGGER.debug('Invalid fields value')
raise InvalidFieldsValuesException(
validations={exception.field_name: ''})
except NotFoundException as exception:
raise exception
except Exception as exception:
LOGGER.debug(traceback.format_exc())
LOGGER.critical(exception)
raise exception
def remove_favorite(self, product_id: str, customer_id: str):
try:
LOGGER.debug('Removing product to customer list')
self.__validations.validate_customer_product_id(product_id=product_id, customer_id=customer_id)
customer = self.__database.get_customer_by_id(customer_id=customer_id)
product = self.__database.get_product_by_id(product_id=product_id)
customer = self.__database.pull_product_to_favorites(customer=customer, product=product)
return self.__parser.parse_customer_favorites(customer=customer)
except NotUniqueError:
LOGGER.info('Email already used')
raise EmailAlreadyUsedException()
except ValidationError as exception:
LOGGER.debug(traceback.format_exc())
LOGGER.debug('Invalid fields value')
raise InvalidFieldsValuesException(validations=exception.to_dict())
except MissingRequiredFieldsException as exception:
raise exception
except Exception as exception:
LOGGER.debug(traceback.format_exc())
LOGGER.critical(exception)
raise exception
def about():
about = {
"version": PINE_EVE_VERSION_STR,
"eve_version": eve.__version__,
"flask_version": flask_version
}
LOGGER.info(about)
return jsonify(about)
def word_tokenize_list(sentences_list):
tokenized_list = list()
total = len(sentences_list)
for index, each in enumerate(sentences_list):
tokenized_list.append(word_tokenize(each))
LOGGER.info('tokenized : {}/{}'.format(index, total))
return tokenized_list
def get_race_venues(self, html, today=False) -> list:
race_links = []
if not html:
LOGGER.info("Cannot scrape the HTML for %s" % self.url)
return race_links
soup = BS(html, "html.parser")
if not today:
race_results_window = soup.find('section',
attrs={"id": 'archiveFormList'})
result_card_sections = race_results_window.find_all(
'div',
class_='w-results-holder')[1:] if race_results_window else []
ttl_lnk = 'results-title'
else:
race_results_window = soup.find('div', class_='w-cards-results')
result_card_sections = race_results_window.find_all(
'section') if race_results_window else []
ttl_lnk = 'w-racecard-grid-race-inactive'
for result_card_section in result_card_sections:
if not today:
sections = result_card_section.find_all('section')
else:
sections = result_card_sections
for section in sections:
title_links = section.find_all(
'a', class_=ttl_lnk) + section.find_all(
'a', class_='w-racecard-grid-race-result')
if not today:
if section.h2:
if 'IRE' in section.h2.text:
break
else:
if 'IRE' in section.h3.a.text:
break
for title_link in title_links:
url = 'https://www.timeform.com' + title_link['href']
if not today:
race_title_text = title_link.text.upper()
else:
race_title_text = title_link['title'].upper()
cleaned_race_definition_word_list = race_title_text.split()
race_type_name = get_race_type_name(
cleaned_race_definition_word_list)
race_links.append((url, race_type_name.replace('\'', '')))
return race_links
def __init__(self, schema_name):
'''
initialization of schema name
:param schema_name:
'''
self.schema = SCHEMAS[schema_name]
LOGGER.info("initated schema is:{0}".format(schema_name))
self.ignore_keys = ["default", "unique", "data_relation"]
self.schema = self.delete_keys_from_dict(self.schema, self.ignore_keys)
self.schema_name = schema_name
LOGGER.info("validating schema:{0}".format(self.schema))
self.v = FabDataValidator(self.schema)
def delete_all_records(self, name_table: str) -> None:
"""
Функция удаления всех записей в таблице
:param name_table: имя очищаемой таблицы
:return: None
"""
LOGGER.warning(f'Начинаю удаление таблицы {name_table}')
self.remote_control_bd.execute(f'DROP TABLE IF EXISTS {name_table}')
self.connect_bd.commit()
MainDB()
LOGGER.info('Успешно удалена')
def show_train_stats(epoch, iteration, losses, y_true, y_pred):
# compute mean statistics
loss = np.mean(losses)
accuracy = accuracy_score(y_true, y_pred)
LOGGER.info(
'Epoch={}, Iter={:,}, Mean Training Loss={:.4f}, Accuracy={:.4f}, '
.format(epoch, iteration, loss, accuracy))
add_metric_summaries('train', iteration, {
'cross_entropy': loss,
'accuracy': accuracy
})
LOGGER.info('\n{}'.format(
classification_report(y_true, y_pred, digits=3)))
def delete_record_in_bd(self, tb_name: str, where: str) -> None:
"""
Функция удаления записи из таблицы
:param tb_name: имя таблицы из которой удаляется запись
:param where: параметры по которым искать нужную запись
:return:
"""
LOGGER.warning(
f'Начинаю удаление записи where={where}, таблица GetRequestsApi'
)
self.remote_control_bd.execute(f'DELETE FROM {tb_name} WHERE {where}')
self.connect_bd.commit()
LOGGER.info('Успешно удалена')
def get_word2vec(self):
LOGGER.info('Loading Word2Vec Vocabulary')
word2vec = KeyedVectors.load_word2vec_format(WORD2VEC, binary=True)
LOGGER.info('Finished loading Word2Vec Vocabulary. {} loaded'.format(
word2vec.vectors.shape[0]))
chosen_word_index_set = set()
for index, each_word in enumerate(word2vec.index2word):
if not index % 100000:
LOGGER.info('checked {}/{}'.format(index,
len(word2vec.index2word)))
if each_word in self.vocab_set and each_word not in self.reserved_vocab_set:
chosen_word_index_set.add(index)
LOGGER.info('Found {}/{} of dataset vocab'.format(
len(chosen_word_index_set), len(self.vocab_set)))
chosen_word_index_list = list(chosen_word_index_set)
self.embedding_vocab = [
word2vec.index2word[each] for each in chosen_word_index_list
]
self.all_vocab = self.reserved_vocab + self.embedding_vocab
self.vocab2id = {
vocab: index
for index, vocab in enumerate(self.all_vocab)
}
word_embeddings = word2vec.vectors[chosen_word_index_list]
reserved_embeddings = np.random.random(
(len(self.reserved_vocab), word_embeddings.shape[1]))
word_embeddings_all = np.concatenate(
(reserved_embeddings, word_embeddings), axis=0)
return word_embeddings_all
def validate_schema(self, item, ignore_keys=[]):
'''
check given payload with defined schema
:param
item: input http payload , ignore keys are not handled by cerberus,
we should remove them to check data schema
ignore_keys: keys to delete from payload to validate payload with schema
:return:
'''
LOGGER.info("------------->ignoring keys are:{0}".format(ignore_keys))
self.v.validate(item)
LOGGER.info("{0} schema validation errors:{1}".format(
self.schema_name, self.v.errors))
return self.v.errors
def __extract(self):
"""Retrieve raw filing data from each of the RSS feed entries"""
# Navigate to the page containing links the data in different formats.
LOGGER.info(f"Navigating to: {self.entry['datalink']}")
htmldat = urllib.request.urlopen(self.entry['datalink'])
page = BeautifulSoup(htmldat, 'lxml')
# Used to locate the 'a' html that contains a link to downlad the filing
# for the accession number in question
search_word = self.entry['accession_num'] + '.' + 'txt'
textpath = page.find('a', string=search_word, href=True)['href']
url = SEC_BASE_URL + textpath
LOGGER.info(f"Navigating to: {url}")
self.text_data = urllib.request.urlopen(url)
def pad(self, X, padding_size):
padded_X = list()
sentences_cut_counter = 0
for each_sent in X:
if len(each_sent) <= padding_size:
padded_sent = each_sent + [self.vocab2id[self.PADDING]
] * (padding_size - len(each_sent))
else:
padded_sent = each_sent[:padding_size]
sentences_cut_counter += 1
padded_X.append(padded_sent)
LOGGER.info(
'number of shortened sentences : {}'.format(sentences_cut_counter))
return padded_X
def __init__(self, update: int, OS_NAME: str):
"""
Создание главного окна, вкладок и управление функциями
:param update: нужно ли проверить обновление
:param OS_NAME: имя операционной системы
:return
"""
super().__init__()
self.OS = OS_NAME
self.app_ico = self.get_app_ico()
self.initialize_ui()
# Панель вкладок в окне
notebook = ttk.Notebook(self)
self.main_book = ttk.Frame(notebook, padding=15)
self.parsing_book = ttk.Frame(notebook)
notebook.add(self.main_book, text='Главная')
notebook.add(self.parsing_book, text='Парсинг')
notebook.pack(expand=True, fill='both')
# Панель вкладок во вкладке парсинг
notebook_parsing = ttk.Notebook(self.parsing_book)
self.parsing_book_groups = ttk.Frame(notebook_parsing, padding=15)
self.parsing_book_by_groups = ttk.Frame(notebook_parsing, padding=10)
notebook_parsing.add(self.parsing_book_groups,
text='Парсинг по группам')
notebook_parsing.add(self.parsing_book_by_groups,
text='Парсинг по критериям')
notebook_parsing.pack(expand=True, fill='both', side='top')
self.update()
self.build_app()
self.function_windows = FunctionsForWindows()
if update == 1:
LOGGER.info('Начинаем процесс проверки обновлений')
self.function_windows.check_update(os_name=self.OS)
self.mainloop()
def send_dl_emails(**kwargs):
LOGGER.info("email sending with payload:{0}".format(kwargs))
msg = Message(kwargs['title'],
sender=kwargs['sender'],
recipients=kwargs['recipients'])
with open(kwargs['template'], 'r') as _file:
html_data = _file.read()
if 'user_id' in kwargs:
html_data = html_data.format(server_url=kwargs['server_url'],
token=kwargs['token'],
user_id=kwargs['user_id'],
email=kwargs['email'],
last_name=kwargs['last_name'],
first_name=kwargs['first_name'])
else:
html_data = html_data.format(server_url=kwargs['server_url'],
token=kwargs['token'],
email=kwargs['email'],
last_name=kwargs['last_name'],
first_name=kwargs['first_name'])
msg.html = html_data
with mail.record_messages() as outbox:
mail.send(msg)
if not len(outbox):
LOGGER.info("email sent failed with token:{0}".format(
kwargs['token']))
return False
else:
LOGGER.info("email sent with token:{0}".format(kwargs['token']))
return True
def update_table(self,
tb_name: str,
update_row: Dict[str, Union[str, int]],
where: str = None) -> None:
"""
функция выполняющая обноление данных таблицы
:param tb_name: имя таблицы
:param update_row: словарь {колонка: значение}
:param where: параметры выборки запии default: не используется
:return:
"""
update_row_values = self.__get_update_row__(tb_name,
update_row,
where=where)
request = 'UPDATE {tb_name} SET {set}'
if where is not None:
request += f' WHERE {where}'
SET = []
for key, value in update_row_values.items():
if type(value) == int:
SET += [f'{key}={value}']
else:
SET += [f'{key}="{value}"']
SET = ', '.join(SET)
request = request.format(tb_name=tb_name, set=SET)
LOGGER.info(f'Обновляю данные {request}')
self.remote_control_bd.execute(request)
self.connect_bd.commit()
LOGGER.warning(f'Обновлены данные таблицы {tb_name}')
LOGGER.warning('Очистка данных')
del tb_name, update_row, where, request,
gc.collect()
def validate(epoch, iteration, batcher, best_loss, patience):
"""Validate the model on validation set."""
losses, y_true, y_pred = list(), list(), list()
for (X_batch_sent, y_true_batch), _ in batcher:
y_pred_batch, loss_batch = session.run(
[
graph.get_tensor_by_name('mlp/y_pred:0'),
graph.get_tensor_by_name('loss/loss:0')
],
feed_dict={
'inputs/x_sent:0': X_batch_sent,
'inputs/y:0': y_true_batch,
'inputs/dropout:0': 1
})
losses.extend(loss_batch.tolist())
y_pred.extend(np.argmax(y_pred_batch, axis=1))
y_true.extend(np.argmax(y_true_batch, axis=1))
# compute mean statistics
loss = np.mean(losses)
accuracy = accuracy_score(y_true, y_pred)
LOGGER.info(
'Epoch={}, Iter={:,}, Validation Loss={:.4f}, Accuracy={:.4f}'.
format(epoch, iteration, loss, accuracy))
add_metric_summaries('valid', iteration, {
'cross_entropy': loss,
'validation_accuracy': accuracy
})
LOGGER.info('\n{}'.format(
classification_report(y_true, y_pred, digits=3)))
if loss < best_loss:
LOGGER.info('Best score Loss so far, save the model.')
save()
best_loss = loss
if iteration * 2 > patience:
patience = iteration * 2
LOGGER.info('Increased patience to {:,}'.format(patience))
return best_loss, patience
def train(training_settings):
LOGGER.info('Start Training')
X_sent1_train, X_sent2_train, y_train, \
X_sent1_dev, X_sent2_dev, y_dev, \
X_sent1_test, X_sent2_test, y_test = get_dataset(PROCESSED_DATA_FILE)
LOGGER.info('Loaded dataset')
all_sentences = X_sent1_train + X_sent2_train + X_sent1_dev + X_sent2_dev
label_encoder = LabelBinarizer()
label_encoder.fit(y_train + y_dev + y_test)
y_train = label_encoder.transform(y_train)
y_dev = label_encoder.transform(y_dev)
# y_test = label_encoder.transform(y_test)
vocab = [each for each_sent in all_sentences for each in each_sent]
vocab_processor = VocabProcessor(vocab)
embedding = vocab_processor.get_word2vec()
training_settings['reserved_vocab_length'] = len(vocab_processor.embedding_vocab)
training_settings['pretrained_vocab_length'] = len(vocab_processor.reserved_vocab)
X_sent_transformed_train = vocab_processor.transform(X_sent1_train, X_sent2_train)
X_sent_transformed_dev = vocab_processor.transform(X_sent1_dev, X_sent2_dev)
padding_size = max([len(each) for each in X_sent_transformed_train + X_sent_transformed_dev])
training_settings['maximum_sent_length'] = padding_size
X_sent_padded_train = vocab_processor.pad(X_sent_transformed_train, padding_size)
X_sent_padded_dev = vocab_processor.pad(X_sent_transformed_dev, padding_size)
X_sent_train_input, y_train = shuffle(X_sent_padded_train, y_train)
X_sent_dev_input, y_dev = shuffle(X_sent_padded_dev, y_dev)
train_batcher = batcher([X_sent_train_input, y_train], training_settings['batch_size'], infinite=True)
valid_batcher = batcher([X_sent_dev_input, y_dev], training_settings['batch_size'])
train_number_of_instance = len(X_sent_train_input)
training_settings['classes_num'] = len(label_encoder.classes_)
LOGGER.info('Number of training instances : {}'.format(train_number_of_instance))
train_network(training_settings,
train_batcher,
list(valid_batcher),
embedding,
train_number_of_instance)
def get_html_page(self, url):
try:
html_page_request = self.engine.get(url=url,
headers=HEADERS,
cookies=COOKIES,
timeout=11)
except CloudConnectionError:
LOGGER.info(
"HTML page could not be fetched because of the Connection Error, settings Cookies have been used instead"
)
return 0
except requests.exceptions.ReadTimeout:
LOGGER.info("The program has freezed and got Timeout Error")
return {}
if html_page_request.status_code == 200:
return html_page_request.text
LOGGER.info(
"HTML page could not be fetched because of the response %d" %
html_page_request.status_code)
time.sleep(10)
return 0
size_now, size_peak = tracemalloc.get_traced_memory()
size_now = size_now // 1024
size_peak = size_peak // 1024
scheduler_logger.warning(
f'Использовалось: {size_last}Mib, Теперь: {size_now}Mib, '
f'В пике: {size_peak}Mib')
master = Tk()
master.overrideredirect(True)
error_logger = LOGGER('App', 'error')
try:
app_brain = BrainForApp(master)
except SystemExit:
error_logger.info('Закрытие программы')
pass
except MemoryError as error:
try:
master.destroy()
except TclError:
pass
size_now, peak = tracemalloc.get_traced_memory()
size_now = size_now // 1024
peak = peak // 1024
showerror(
'Ошибка',
f'Недостаточно оперативной памяти!\n\nИспользуется: {size_now}Mib'
f', В пике: {peak}Mib\n\n{error}')
error_logger.error('Нехватка памяти: Используется - '
f'{size_now}Mib, В пике - {peak}Mib --> {error}')
def main(LOGGER=LOGGER,MONGO=MONGO,*args):
PROJECT_HOME = os.path.abspath(os.path.dirname(__file__))
start = time.time()
LOGGER.debug('--Start--')
if args:
sys.argv.extend(*args)
parser = argparse.ArgumentParser()
parser.add_argument(
'--bibcode-files',
nargs='*',
default=CLASSIC_BIBCODES.values(),
dest='updateTargets',
help='full paths to bibcode files'
)
parser.add_argument(
'--bibcodes',
nargs='*',
default=None,
dest='targetBibcodes',
help='Only analyze the specified bibcodes'
)
parser.add_argument(
'--async',
default=False,
action='store_true',
dest='async',
help='start in async mode'
)
parser.add_argument(
'--load-records-from-files',
nargs='*',
default=None,
dest='load_from_files',
help='Load XML records from files via pickle instead of ADSExports',
)
args = parser.parse_args()
LOGGER.debug('Recieved args (%s)' % (args))
for target in args.updateTargets:
targetRecords = []
LOGGER.info('Working on bibcodes in %s' % target)
s = time.time() #Let's eventually use statsd for these timers :)
with cd(PROJECT_HOME):
with open(target) as fp:
records = []
for line in fp:
if not line or line.startswith("#"):
continue
r = tuple(line.strip().split('\t'))
if args.targetBibcodes:
if r[0] in args.targetBibcodes:
records.append(r)
else:
records.append(r)
if args.async and len(records) >= BIBCODES_PER_JOB:
#We will miss the last batch of records unless it the total is evenly divisible by BIBCODES_PER_JOB
publish(records)
records = []
#TODO: Throttling?
LOGGER.debug('[%s] Read took %0.1fs' % (target,(time.time()-s)))
#Publish any leftovers in case the total was not evenly divisibly
if args.async:
if records:
publish(records)
else:
s = time.time()
records = utils.findChangedRecords(records,LOGGER,MONGO)
LOGGER.info('[%s] Found %s records to be updated in %0.1fs' % (target,len(records),(time.time()-s)))
if args.load_from_files:
records,targets = utils.readRecordsFromFiles(records,args.load_from_files,LOGGER)
else:
records,targets = utils.readRecords(records,LOGGER)
s = time.time()
records = utils.updateRecords(records,targets,LOGGER)
LOGGER.info('[%s] Updating %s records took %0.1fs' % (target,len(records),(time.time()-s)))
s = time.time()
utils.mongoCommit(records,LOGGER,MONGO)
LOGGER.info('Wrote %s records to mongo in %0.1fs' % (len(records),(time.time()-s)))
LOGGER.debug('--End-- (%0.1fs)' % (time.time()-start))
def get_records(self,
tb_name: str,
select: List[str] = None,
one_record: bool = False,
where: str = None,
order: str = None,
limit: Union[str, int] = None,
dictionary: bool = True) -> Union[List[dict], dict]:
"""
Функция получения записей из базы данных
:param tb_name: название таблицы
:param select: имена колонн, которые нужно взять default: все
:param one_record: булево одну ли брать запись default: False
:param where: параметр выборки default: не используется
:param order: параметр сортировки default: не используется
:param limit: параметр ограничения кол-ва строк default: не используется
:param dictionary: параметр определения вида выходных даных,
если False, то вернётся читый результат SQL, если True, то словарь
{столбец: значение} default True
:return:
"""
LOGGER.warning(f'Начинаю брать данные из {tb_name}')
response = []
select = self.__get_select__(tb_name, select)
LOGGER.info('Собираю запрос')
request = f'SELECT {", ".join(select)} FROM {tb_name}'
if where is not None:
request += f' WHERE {where}'
if order is not None:
request += f' ORDER BY {order}'
if limit is not None:
request += f' LIMIT {limit}'
LOGGER.info(f'Получаю данные {request}')
self.remote_control_bd.execute(request)
if one_record is True:
records = [self.remote_control_bd.fetchone()]
else:
record = [list(self.remote_control_bd.fetchone())]
__record__ = self.remote_control_bd.fetchall()
if len(__record__) > 0:
records = record + __record__
else:
records = record
LOGGER.info('Обрабатываю')
if dictionary is False:
response = records
else:
for i in range(len(records)):
record = records[i]
if len(record) == 0 or len(record) < len(select):
continue
response.append({})
for k in range(len(select)):
name = select[k]
response[i][name] = record[k]
del record
LOGGER.warning(f'Успешно получены данные из {tb_name}')
LOGGER.warning('Очистка данных')
del tb_name, select, one_record, where, limit, dictionary, request, \
records
gc.collect()
return response[0] if len(response) == 1 else response
def get(url, *args):
r = requests.get(url, args)
LOGGER.info('{r} GET {url}'.format(r=r, url=url))
return r
def is_valid_registry(registry: str):
match = re.match('\\d{4}-\\d', registry)
if not match:
LOGGER.info('Invalid registry, raising exception')
raise InvalidRecordException()
class ConfigureVkApi:
"""
Класс отвечающий за нстройку инструментов для запросов к API Vk
"""
def __init__(self, ignore_existing_token: bool = False):
self.logger = LOGGER('config_vk_api', 'vk_api')
get_requests_db = GetRequestsToDB()
user_data_table_value = get_requests_db.get_records(
tb_name=get_requests_db.userdata,
one_record=True,
select=['access_token'])
token = user_data_table_value['access_token']
self.__additional_windows = AdditionalWindows
if ignore_existing_token is False:
if (token is None) or (token == DEFAULT_VALUE_FOR_BD):
token = self.get_token()
else:
token = self.get_token()
if (token is not None) or (token != DEFAULT_VALUE_FOR_BD):
is_donat = self.check_is_donat(token)
if is_donat is False:
token = None
self.token = token
if self.token is not None:
vk_session = vk_api.VkApi(token=self.token)
self.vk_tool = vk_api.tools.VkTools(vk_session)
if ignore_existing_token is True:
showinfo('Авторизовались', 'Вы удачно авторизовались!')
self.logger.info('Получен vk_tool и сам токен')
else:
self.logger.error('vk_tool не удалось получить')
self.vk_tool = None
del get_requests_db, user_data_table_value
def get_token(self) -> Union[str, None]:
"""
Функция получения токнеа пользователя
:return:
"""
showinfo('Получение токена!', INFO_MSG['VK_API']['get_token'])
web_open_new_tab(HTTP_GET_TOKEN)
token = self.__additional_windows().get_token()
token = self.preparation_final_token(token)
if token == DEFAULT_VALUE_FOR_BD:
LOGGER.warning(
'При выполнении функции get_token был получен невалидный токен'
)
return None
params = {'v': VERSION_API, 'access_token': token}
try:
request = requests.get(
HTTP_FOR_REQUESTS.format(method='users.get'),
params=params).json()
except ConnectionError:
showerror(
'Нет подключения',
'Не возиожно авторизоваться, нетп подключения к интернету')
return None
if request.get('error'):
showerror(
'Авторизация не удалась',
'Неверный токен авторизации, произошла ошибка, '
'повторите попытку')
return None
update_requests_db = UpdateRequestsToDB()
update_requests_db.update_table(tb_name=update_requests_db.userdata,
update_row={'access_token': token})
del request
return token
@staticmethod
def check_is_donat(token: str) -> bool:
"""
Функция проверки оплаты подписки на программу пользователем
:param token: токен пользователя
:return:
"""
params = {
'v': VERSION_API,
'access_token': token,
'owner_id': ID_GROUP_VK
}
try:
request = requests.get(
HTTP_FOR_REQUESTS.format(method='donut.isDon'),
params=params).json()
except ConnectionError:
showerror(
'Нет подключения',
'Невозможно авторизоваться, нет подключения к интернету')
return False
if request.get('error'):
showerror('Ошибка',
f'Произошла непредвиденная ошибка {request["error"]}')
response = request.get('response')
if int(response) == 1:
return True
else:
get_requests_db = GetRequestsToDB()
__start = GetRequestsToDB().get_records(
select=['start_free_version'],
one_record=True,
tb_name=get_requests_db.settings)['start_free_version']
if __start is None:
warning = WARNING_MSG['VK_API']['is_not_donat_free']
showwarning('Пробная версия!',
warning.format(min=TIME_FREE_VERSION // 60))
start_free_version = time_now()
update_request_db = UpdateRequestsToDB()
update_request_db.update_table(
tb_name=update_request_db.settings,
update_row={'start_free_version': int(start_free_version)})
return True
else:
time_use_free_version = ceil(time_now()) - int(__start)
if time_use_free_version >= TIME_FREE_VERSION:
warning = WARNING_MSG['VK_API']['is_not_donat']
showwarning('Пробная версия!', warning)
return False
else:
time_left = TIME_FREE_VERSION - time_use_free_version
warning = WARNING_MSG['VK_API']['is_not_donat_free']
showwarning('Пробная версия!',
warning.format(min=time_left // 60))
return True
def preparation_final_token(self, token: str) -> str:
"""
Функция обработки ссылки и получения из неё токена
:param token: ссылка с токеном
:return:
"""
token = token.split('access_token=')
if len(token) == 2:
token = token[1].split('&')[0]
return token
showwarning('Не смог распознать токен',
WARNING_MSG['VK_API']['non_inspected_token'])
self.logger.warning(
'При выполнении preparation_final_token, не смог распознать токен')
return DEFAULT_VALUE_FOR_BD
def check_update(self, os_name: str, call: bool = False) -> None:
"""
Проверка наличия обновлений
:param os_name: имя OS
:param call: булево принудительно ли отправлен запрос на проверку
обновлений default: False
:return:
"""
version = os.path.join(path_to_version, 'version.txt')
try:
LOGGER.info('Клонируем version')
response = requests.get(REPO_URL_VERSION)
with tempfile.TemporaryFile() as file:
file.write(response.content)
with zipfile.ZipFile(file) as fzip:
fzip.extractall(path)
except ConnectionError as error:
LOGGER.error(f'Произошла ошибка при клонировании проекта {error}')
if call is True:
showerror(
'Невозможно выполнить обновление',
ERROR_MSG['Update_app']['Bad_connect'].format(VERSION))
return
with open(version, 'r', encoding='utf-8') as file:
file = file.readline().strip().split('&')
shutil.rmtree(path_to_version, ignore_errors=True, onerror=None)
version = file[0].strip()
v_int = [int(item) for item in version.split('.')]
version_old = [item for item in VERSION.split('.')]
v_old_int = [int(item) for item in version_old]
info = file[1]
condition_1 = v_int[0] > v_old_int[0]
condition_2 = v_int[0] >= v_old_int[0] and v_int[1] > v_old_int[1]
condition_3 = \
v_int[0] >= v_old_int[0] and v_int[1] >= v_old_int[1] and \
v_int[2] > v_old_int[2]
need_update = (False, True)[condition_1 or condition_2 or condition_3]
if (call is True) and (need_update is False):
showinfo('Обновление не требуется',
INFO_MSG['Update_app']['Not_need_update'].format(version))
if need_update is True:
answer = askyesnocancel(
'Требуется обновление',
ASKS['Update_app']['Need_update'].format(version=version,
info=info))
if answer is False:
return
if answer is None:
LOGGER.info('Отмена автообновлений')
update_request_db = UpdateRequestsToDB()
update_request_db.update_table(
tb_name=update_request_db.settings,
update_row={'auto_update': 0})
return
if answer is True:
try:
self.update_app(os_name)
except ConnectionError as error:
LOGGER.error(f'Невозможно обновиться {os_name} -> {error}')
showerror(
'Невозможно выполнить обновление',
ERROR_MSG['Update_app']['Bad_connect'].format(VERSION))
class BrainForApp:
"""
Класс отвечающий за настройку и запуск приложения
"""
def __init__(self, window_preview):
"""
Создаёт превью и проверяет нужные настройки для программы, а также
запускает её
:param window_preview: объект окна превью
"""
self.logger = LOGGER('main', 'main')
png_preview_open, png_preview = self.preview_image_open()
self.preview_image_set(png_preview_open, png_preview, window_preview)
window_preview.update()
tracemalloc.start()
time.sleep(2)
MainDB()
get_requests_db = GetRequestsToDB()
settings = get_requests_db.get_records(
tb_name=get_requests_db.settings,
one_record=True,
select=['first_start', 'auto_update'])
first_start = settings['first_start']
auto_update = settings['auto_update']
if first_start == 1:
self.logger.info('Первый запуск')
window_preview.destroy()
done = AdditionalWindows().person_and_agreement_data()
if done is True:
update_requests_db = UpdateRequestsToDB()
update_requests_db.update_table(
tb_name=update_requests_db.settings,
update_row={'first_start': 0})
self.logger.warning('Очистка от лишних файлов в директории')
list_path = os.listdir(path)
if REPO_BRANCH_UPDATER in list_path:
rmtree(REPO_BRANCH_UPDATER, ignore_errors=True, onerror=None)
if REPO_BRANCH_VERSION in list_path:
rmtree(REPO_BRANCH_VERSION, ignore_errors=True, onerror=None)
if REPO_BRANCH_MASTER in list_path:
rmtree(REPO_BRANCH_MASTER, ignore_errors=True, onerror=None)
try:
self.logger.warning('Закрытие окна первью')
window_preview.destroy()
except TclError:
pass
del settings, first_start, list_path, window_preview
gc.collect()
self.logger.info('Создание задачи scheduler')
scheduler = BackgroundScheduler()
scheduler.start()
scheduler.add_job(__scheduler__, 'interval', minutes=1)
self.logger.info('Запуск приложения')
from windows import App
App(auto_update, OS)
self.logger.info('Закрытие приложения')
def preview_image_open(self):
"""
Возвращает первью картинку
"""
while True:
try:
png_preview_open = Image.open(
os.path.join(path_to_dir_ico, 'preview.png'))
png_preview = ImageTk.PhotoImage(png_preview_open)
return png_preview_open, png_preview
except FileNotFoundError as err:
self.logger.error(str(err))
@staticmethod
def preview_image_set(png_preview_open, png_preview, window_preview):
"""
Устанавливает размеры окна, ставит его по середине, устанавливает
картинку как фон
"""
x_img, y_img = png_preview_open.size
x = (window_preview.winfo_screenwidth() - x_img) // 2
y = (window_preview.winfo_screenheight() - y_img) // 2
window_preview.geometry("%ix%i+%i+%i" % (x_img, y_img, x, y))
Label(window_preview, image=png_preview).pack(side='top')