def __init__(self, session: orm.session.Session, page_id: int, title: str, namespace: int, text: str):
self.page_id = page_id
self.title = title
self.namespace = namespace
self.text = text
session.add(self)
def pacu_session(db: orm.session.Session):
query: orm.Query = db.query(PacuSession)
assert query.count() == 0
pacu_session = PacuSession()
db.add(pacu_session)
yield pacu_session
def create_all_from_xml(cls, session: orm.session.Session, breakcount: int = 10000) -> None:
with open(XML_DUMP_FILENAME, 'rb') as f:
for i, element in enumerate(yield_page_elements(f), 1):
Page.create_from_element(session, element)
if not i % breakcount:
session.Commit()
session.Commit()
def ImportRepo(session: orm.session.Session,
language: scrape_repos_pb2.LanguageToClone,
metafile: pathlib.Path, pool: multiprocessing.Pool) -> None:
"""Import contentfiles from repository.
Args:
session: A database session to import to.
language: The language specification for the repo.
metafile: The repo metafile.
pool: A multiprocessing pool.
"""
meta = pbutil.FromFile(metafile, scrape_repos_pb2.GitHubRepoMetadata())
clone_dir = metafile.parent / f'{meta.owner}_{meta.name}'
repo = contentfiles.GitHubRepository.GetOrAdd(session, meta)
repo.language = language.language
for importer in language.importer:
if not importer.source_code_pattern:
logging.error('No source_code_pattern specified! Stopping now.')
return
pat = importer.source_code_pattern
pat = f'{clone_dir}/{pat[1:]}' if pat[
0] == '^' else f'{clone_dir}/{pat}'
cmd = [
'find',
str(clone_dir), '-type', 'f', '-regex', pat, '-not', '-path',
'*/.git/*'
]
logging.debug('$ %s', ' '.join(cmd))
paths = subprocess.check_output(
cmd, universal_newlines=True).rstrip().split('\n')
if len(paths) == 1 and not paths[0]:
logging.debug('No files to import from %s', clone_dir)
return
logging.info("Importing %s '%s' files from %s ...",
humanize.intcomma(len(paths)),
importer.source_code_pattern, clone_dir)
all_files_relpaths = public.GetAllFilesRelativePaths(clone_dir)
jobs = [
scrape_repos_pb2.ImportWorker(
clone_from_url=meta.clone_from_url,
clone_dir=str(clone_dir),
abspath=p,
all_files_relpaths=all_files_relpaths,
preprocessors=importer.preprocessor,
) for p in paths
]
bar = progressbar.ProgressBar(max_value=len(jobs))
for outputs in bar(pool.imap_unordered(ImportWorker, jobs)):
for output in outputs:
session.add(output)
def unmake(cls: Type[T], session: o.session.Session, **kwargs) -> None:
"""
Find the item with the specified name, and delete it if it exists.
:param session: The session to be used in the query and creation.
:param kwargs: Arguments to use in the :meth:`~sqlalchemy.orm.query.Query.filter_by` clause and in the item
constructor.
"""
# Find the item
item = session.query(cls).filter_by(**kwargs).one_or_none()
# Delete the item
if item is not None:
session.delete(item)
def update(self, database: orm.session.Session, commit: bool = True, **kwargs) -> None:
""" Instead of requiring three lines to update a single field inside
a database session, this method updates a single field in one line.
Example usage:
session.update(database, field_name={'json': ...}) """
for key, value in kwargs.items():
value = stringify_datetime(value)
setattr(self, key, value)
database.add(self)
if commit:
database.commit()
def make(cls: Type[T], session: o.session.Session, **kwargs) -> T:
"""
Find the item with the specified name, or create it if it doesn't exist.
:param session: The session to be used in the query and creation.
:param kwargs: Arguments to use in the :meth:`~sqlalchemy.orm.query.Query.filter_by` clause and in the item
constructor.
:return: The retrieved or created item.
"""
# Find the item
item = session.query(cls).filter_by(**kwargs).one_or_none()
# Create the item
if item is None:
item = cls(**kwargs)
session.add(item)
# Return the item
return item
def activate(self, database: orm.session.Session) -> None:
for other_session in database.query(PacuSession).filter(PacuSession.id != self.id):
other_session.is_active = False
database.add(other_session)
self.is_active = True
database.add(self)
database.commit()
def ExportDatabase(session: orm.session.Session,
export_path: pathlib.Path) -> None:
"""Export the contents of a database to a directory."""
query = session.query(contentfiles.ContentFile)
logging.info('Exporting %s files to %s ...',
humanize.intcomma(query.count()), export_path)
for contentfile in query:
path = export_path / (contentfile.sha256_hex + '.txt')
logging.debug(path)
with open(path, 'w') as f:
f.write(contentfile.text)
def __init__(
self,
session: orm.session.Session,
code: str,
canonicalName: str,
characters: str = None,
direction: str = 'ltr',
character_category = True,
otherNames: typing.List[str] = [],
systems: typing.List[str] = [],
parent: 'Script' = None,
wikipedia_article: str = None):
self._code = code
self._canonical_name = canonicalName
self._characters = characters
self._direction = direction
self._other_names = json.dumps(otherNames)
self._systems = json.dumps(systems)
self._parent = parent
self._wikipedia_article = wikipedia_article
session.add(self)
def _single_team_match_from_fixture(
cls, session: orm.session.Session, fixture_data: pd.Series, at_home: bool
) -> TeamMatch:
team_prefix = "home" if at_home else "away"
team_name = fixture_data[f"{team_prefix}_team"]
team = (
session.execute(sql.select(Team).where(Team.name == team_name))
.scalars()
.one()
)
team_score = fixture_data.get(f"{team_prefix}_score", 0)
return TeamMatch(team=team, at_home=at_home, score=team_score)
def IsInDatabase(
cls,
session: orm.session.Session,
proto: scrape_repos_pb2.GitHubRepoMetadata,
) -> bool:
# TODO(cec): At some point it would be nice to have an expiration date for
# scraped repos, after which the contents are considered "stale" and the
# repo is re-scraped. This would require extending the behaviour of
# ShouldImportRepo() to check the expiry date.
instance = (
session.query(cls.clone_from_url)
.filter(cls.clone_from_url == proto.clone_from_url)
.first()
)
return True if instance else False
def ExportDatabase(
session: orm.session.Session, export_path: pathlib.Path
) -> None:
"""Export the contents of a database to a directory."""
query = session.query(contentfiles.ContentFile)
app.Log(
1,
"Exporting %s files to %s ...",
humanize.Commas(query.count()),
export_path,
)
for contentfile in query:
path = export_path / (contentfile.sha256 + ".txt")
app.Log(2, path)
with open(path, "w") as f:
f.write(contentfile.text)
def __init__(self, sess: orm.session.Session):
"""Initializes info object
:param orm.session.Session sess: session object
"""
self._ver_major: int = 0
self._ver_minor: int = 0
self._ver_micro: int = 0
self.databases: typ.Sequence[str] = []
self.database_tables: typ.Dict[str, typ.List[str]] = []
self.data_dir: str = None
self.script_dir: str = osp.dirname(__file__)
self.query_dir: str = osp.join(osp.dirname(__file__), u"../query")
self.have_archive: bool = False
self.have_bdb: bool = False
self.have_federated_engine: bool = True
self.have_innodb: bool = True
self.have_myisam: bool = True
self.have_ndb_cluster: bool = False
self.max_connections: int = 0
self.read_buffer_size: int = 0
self.read_rnd_buffer_size: int = 0
self.sort_buffer_size: int = 0
self.thread_stack: int = 0
self.join_buffer_size: int = 0
self.record_buffer: int = 0
self.record_rnd_buffer: int = 0
self.temp_table_size: int = 0
self.max_heap_table_size: int = 0
self.key_buffer_size: int = 0
self.binlog_cache_size: int = 0
self.long_query_time: int = 0
self.log_slow_queries: bool = False
self.wait_timeout: int = 0
self.interactive_timeout: int = 0
self.innodb_buffer_pool_size: int = 0
self.innodb_buffer_pool_instances: int = 0
self.innodb_buffer_pool_chunk_size: int = 0
self.innodb_additional_mem_pool_size: int = 0
self.innodb_log_buffer_size: int = 0
self.query_cache_size: int = 0
self.query_cache_type: int = 0
self.query_cache_limit: int = 0
self.ariadb_pagecache_buffer_size: int = 0
self.key_cache_block_size: int = 0
self.open_files_limit: int = 0
self.innodb_log_file_size: int = 0
self.innodb_log_files_in_group: int = 0
self.innodb_thread_concurrency: int = 0
self.innodb_file_per_table: bool = True
self.log_bin: bool = False
self.have_threadpool: bool = False
self.thread_pool_size: int = 0
self.version: str = None
self.performance_schema: bool = True
self.have_galera: bool = False
self.have_ariadb: bool = False
self.have_tokudb: bool = False
self.have_xtradb: bool = False
self.have_rocksdb: bool = False
self.have_spider: bool = False
self.have_connect: bool = False
self.wsrep_provider_options: str = None
self.wsrep_causal_reads: str = None
self.wsrep_cluster_name: str = None
self.wsrep_on: bool = False
self.wsrep_cluster_address: str = None
self.wsrep_node_name: str = None
self.wsrep_notify_cmd: str = None
self.wsrep_sst_method: str = None
self.wsrep_osu_method: str = None
self.wsrep_max_ws_size: int = None
self.log_error_file: str = None
self.ignore_builtin_innodb: bool = False
self.gtid_mode: str = None
self.gtid_strict_mode: str = None
self.binlog_format: str = None
self.innodb_flush_log_at_trx_commit: bool = False
self.slaves: typ.Dict[str, str] = {}
self.replicas: typ.Dict[str, str] = {}
self.read_only: bool = False
self.thread_cache_size: int = 0
self.thread_handling: str = None
self.concurrent_insert: bool = False
version_query_file: str = osp.join(self.query_dir, u"version-query.sql")
with open(version_query_file, mode=u"r", encoding=u"utf-8") as vqf:
version_query: str = vqf.read()
result = sess.execute(version_query)
Version = clct.namedtuple(u"Version", result.keys())
versions: typ.Sequence[str] = [
Version(*version).VERSION.split("-")[0].split(".")
for version in result.fetchall()
]
self._ver_major, self._ver_minor, self._ver_micro = [
int(version)
for version in versions[0]
]
def get_module_pages(cls, session: orm.session.Session) -> typing.List[Page]:
return session.query(cls).filter_by(namespace=828).filter(cls.title.notlike('%/documentation')).all()
def IsInDatabase(cls, session: orm.session.Session,
proto: scrape_repos_pb2.GitHubRepoMetadata) -> bool:
instance = session.query(cls).filter_by(
**cls._GetArgsFromProto(proto)).first()
return True if instance else False
def prep_db(session: orm.session.Session):
doctor1 = Doctor(id=1, name='Айболит')
doctor2 = Doctor(id=2, name='Сеченов')
doctor3 = Doctor(id=96881373, name='EugenyBobylev')
session.bulk_save_objects([doctor1, doctor2, doctor3])
session.commit()
job1 = IncartJob(id='1', snippet='job_1')
job2 = IncartJob(id='2', snippet='job_2')
session.bulk_save_objects([job1, job2])
session.commit()
job_doctor: JobDoctor = JobDoctor(job_id="1", doctor_id=1)
job_doctor.request_id = '1'
session.add(job_doctor)
session.commit()
job_doctor2 = JobDoctor(job_id="1", doctor_id=2)
job_doctor2.request_id = '2'
job_doctor2.request_sended = datetime.datetime.utcnow()
session.add(job_doctor2)
session.commit()
def get_all_as_dict(cls, session: orm.session.Session) -> typing.Dict:
return {script.code: script for script in session.query(cls).all()}
def placeholders(session: orm.session.Session):
# Users
users = [
User(418878871, "teadove", "Петер", "Ибрагимов Петер Ильгизович",
"+79778725196"),
User(346235776, "tainella", "Amelia Zagadka",
"Полей-Добронравова Амелия Вадимовна", "+79262895840"),
User(1409549287, "teameekey", "Петер 2.0", "Иванов Иван Иванович"),
User(1624330498, 'elfen_clan', 'Каэде', "Антон Семёнов")
]
session.add_all(users)
# Spheres
spheres = [
Sphere("Дизайн"),
Sphere('SMM'),
Sphere("Разработка ПО под Windows"),
Sphere("Мобильная разработка"),
Sphere("Консультирование"),
Sphere("CRM"),
Sphere("Разработка телеграм ботов"),
Sphere("Фронтенд")
]
session.add_all(spheres)
# Specialist
specialist = Specialist(users[0])
session.add(specialist)
# Spheres to specialist
spheres_to_specialist = [
SphereToSpecialist(spheres[0], specialist),
SphereToSpecialist(spheres[2], specialist),
SphereToSpecialist(spheres[4], specialist),
SphereToSpecialist(spheres[3], specialist),
SphereToSpecialist(spheres[-1], specialist)
]
session.add_all(spheres_to_specialist)
# Representative
representatives = [
Representative(users[1], "ООО название компании"),
Representative(users[3], "ООО моя оборона")
]
session.add_all(representatives)
# Moderator
moderator = Moderator(users[2], is_admin=True)
session.add(moderator)
# Task
tasks = [
Task(
"Разработать телеграм бота", representatives[0],
"Разработать ТГ бота для ведения учёта посещений "
"собраний, полное ТЗ при контакте"),
Task(
"Разработать дизайн для сайта",
representatives[0],
"Разработать дизайн для сайта НКО для сборов пожертвований для бездомных",
specialist=specialist,
status='in_work'),
Task("Разработать сайт для НКО",
representatives[0],
status="awaiting_specialist")
]
session.add_all(tasks)
# SphereToTask
sphere_to_tasks = [
SphereToTask(spheres[-2], tasks[0]),
SphereToTask(spheres[-1], tasks[1]),
SphereToTask(spheres[0], tasks[1])
]
session.add_all(sphere_to_tasks)
session.commit()