def delete_database(self, database_id):
database = Database.query.filter_by(database_id=database_id).first()
if database:
database.is_deleted = True
db.session.commit()
print(convert_query_result2dict(database))
return convert_query_result2dict(database)
def update_project_status(self, project_id, status):
project = Project.query.filter_by(project_id=project_id,
is_deleted=0).first()
if project:
project.status = status
db.session.commit()
return convert_query_result2dict(project)
def delete_project(self, project_id):
project = Project.query.filter_by(project_id=project_id,
is_deleted=0).first()
if project:
project.is_deleted = 1
db.session.commit()
return convert_query_result2dict(project)
def create_gov_rule(self, pattern, correct_word, domain):
domain = urlparse(domain).netloc
rule = GovLexicon(pattern=pattern,
correct_word=correct_word,
domain=domain)
db.session.add(rule)
db.session.commit()
return convert_query_result2dict(rule)
def update_database(self, *, database_id, database_name, config):
database = Database.query.filter_by(database_id=database_id,
is_deleted=0).first()
if not database:
return None
database.database_name = database_name
database.config = config
db.session.commit()
return convert_query_result2dict(database)
def get(self, project_id, limit=10, offset=0):
all = Task.query.filter_by(project_id=project_id).order_by(
Task.update_timestamp.desc()).limit(limit).offset(offset).all()
all_dict = convert_query_result2dict(all)
result = {
'total_count': self.count(project_id=project_id),
'count': len(all),
'result': all_dict,
}
return result
def update_project(self, *, project_id, project_name, setting, cron_config,
tag):
project = Project.query.filter_by(project_id=project_id,
is_deleted=0).first()
if not project:
return None
project.project_name = project_name
project.setting = setting
project.cron_config = cron_config
project.tag = tag
db.session.commit()
return convert_query_result2dict(project)
def create_project(self, *, project_name, setting, cron_config, tag):
if not setting.get('spidercls'):
spider_name = setting.get('spider_name')
if spider_name[0].isdigit():
spider_name = 'a' + spider_name
rules = setting.get('rules')
start_url = setting.get('start_url')
mail_to = setting.get('information_config').get('email')
script_type = setting.get('script_type')
spidercls = create_script(script_name=spider_name,
rules=rules,
start_url=start_url,
mail_to=mail_to,
script_type=script_type)
setting['spidercls'] = spidercls
project = Project(project_name=project_name,
setting=setting,
cron_config=cron_config,
tag=tag)
db.session.add(project)
db.session.commit()
return convert_query_result2dict(project)
def get_database(self, database_id):
return convert_query_result2dict(
Database.query.filter_by(database_id=database_id,
is_deleted=0).first())
def create_database(self, database_name, config):
new_db = Database(database_name=database_name, config=config)
db.session.add(new_db)
db.session.commit()
result = convert_query_result2dict(new_db)
return result
def get_all_databases(self):
all = Database.query.filter_by(is_deleted=0).all()
return convert_query_result2dict(all)
def get_all_gov_lexicons(self):
all = GovLexicon.query.all()
return convert_query_result2dict(all)
def get_project(self, project_id):
return convert_query_result2dict(
Project.query.filter_by(project_id=project_id,
is_deleted=0).first())
def get_all_projects(self):
all = Project.query.filter_by(is_deleted=0).all()
return convert_query_result2dict(all)
