class Create:
def __init__(self):
self.__DB = DB().db
self.cursor = self.__DB.cursor()
def creat_ipsegment_table(self):
"""
创建系统需要的255张基础IP表
:return: None
"""
for i in range(1, 255):
if i == 127:
pass
else:
table_structure = 'IPsegment_%s' % str(i)
sql = """CREATE TABLE IF NOT EXISTS `ip_domains`.`{table_name}` (
`id` int(32) NOT NULL AUTO_INCREMENT,
`origin_ip` varchar(255),
`domain` varchar(255),
`time` datetime(0) NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP(0),
PRIMARY KEY (`id`)
);""".format(table_name=table_structure)
print sql
try:
self.cursor.execute(sql)
print table_structure
except Exception as e:
print e
return e
def commit(self):
self.__DB.commit()
return None
def update(_id, **params):
update_data = {}
if params.get("name"):
update_data["name"] = params["name"]
if not update_data:
return False
_DB = DB.query(Isp).filter(Isp.id == _id).update(update_data)
DB.commit()
return _DB
def upgrade(app):
""" data migration to update the Tag.count fields """
db_session = DB(app.config['DATABASE']).get_session()
existing_posts = db_session.query(Post).all()
for post in existing_posts:
post.summary_html = markup_to_html(post.format, post.summary)
post.content_html = markup_to_html(post.format, post.content)
db_session.commit()
def create(**params):
insert_data = {}
if params.get("name"):
insert_data["name"] = params["name"]
if not insert_data:
return False
_DB = Isp(**insert_data)
DB.add(_DB)
DB.commit()
return _DB
def upgrade(app):
""" data migration to update the Tag.count fields """
db_session = DB(app.config['DATABASE']).get_session()
existing_tags = db_session.query(Tag).all()
for tag in existing_tags:
tag.count = db_session.query(Post, Status).filter(and_(Post.status_id==Status.id,
Status.value=='public',
)).filter(Post.tags.contains(tag)).count()
db_session.commit()
def add_initial_data():
"""Insert initial data into the database"""
# open database session
db_session = DB(db).get_session()
# ask user for an admin username and password
username = raw_input('Please enter the admin username: '******'Please enter the admin password: '******'SECRET_KEY'], password))
db_session.add(u)
# create statuses
s1 = Status('draft')
s2 = Status('private')
s3 = Status('public')
db_session.add(s1)
db_session.add(s2)
db_session.add(s3)
# create formats
f = Format('rest')
f2 = Format('markdown')
db_session.add(f)
db_session.add(f2)
# Tags
t1 = Tag('imposter')
t2 = Tag('weblog')
# build initial post and put it in the database
initial_post_summary = """
Installed Correctly!
"""
initial_post_content = """
Imposter was installed correctly!
This is just a sample post to show Imposter works.
**Have a lot of fun blogging!**
"""
p1 = Post('Welcome to Imposter!', initial_post_summary, initial_post_content)
p1.slug = slugify(p1.title)
p1.createdate = datetime.now()
p1.lastmoddate = datetime.now()
p1.pubdate = datetime.now()
p1.format = f
p1.status = s3
p1.user = u
p1.tags = [t1, t2]
p1.compile()
db_session.add(p1)
db_session.commit()
def update(uid, **params):
update_data = {}
if params.get("passwd"):
update_data["passwd"] = params["passwd"]
if params.get("real_name"):
update_data["real_name"] = params["real_name"]
if params.get("status"):
update_data["status"] = params["status"]
if not update_data:
return False
_DB = DB.query(Users).filter(Users.id == uid).update(update_data)
DB.commit()
return _DB
def create(**params):
insert_data = {}
if not params.get("uname") or not params.get("passwd"):
return False
insert_data["uname"] = params["uname"]
insert_data["passwd"] = params["passwd"]
if params.get("real_name"):
insert_data["real_name"] = params["real_name"]
if params.get("status"):
insert_data["status"] = params["status"]
_DB = Users(**insert_data)
DB.add(_DB)
DB.commit()
return _DB
def install_scraper(scraper):
DB.connect()
scraper_id = DB.query("SELECT scraper_id FROM scrapers WHERE service=?",
[scraper.service])
if not scraper_id:
kodi.set_property("new_scraper", "true", 'script.module.scrapecore')
settings_definition = ''
for s in scraper.settings_definition:
settings_definition += "\n\t\t" + s
settings_definition = settings_definition.replace(
"{NAME}", scraper.name)
settings_definition = settings_definition.replace(
"{SERVICE}", scraper.service)
DB.execute(
"INSERT INTO scrapers(service, name, settings, enabled) VALUES(?,?,?,1)",
[scraper.service, scraper.name, settings_definition])
DB.commit()
def remove(isp_id):
_DB = DB.query(Isp).filter(Isp.id == isp_id).delete()
DB.commit()
return _DB
def toggle_scraper(scraper_id):
DB.execute("UPDATE scrapers SET enabled=ABS(1-enabled) WHERE scraper_id=?",
[scraper_id])
DB.commit()
class VereseBot(telegram.Bot):
def __init__(self):
self.commands = []
self._stay_awake = 30
self.db = DB()
self.import_commands()
# Connect to Telegram
super(VereseBot, self).__init__(token=config.token)
self.queue = {}
def import_commands(self):
for command_file in sorted(os.listdir('commands')):
if command_file.endswith('.pyc') or command_file.startswith('__'):
continue
mod = importlib.import_module('commands.{}'.format(command_file[:-3]))
for member_name, member_type in inspect.getmembers(mod):
if inspect.isclass(member_type) and issubclass(member_type, BotCommand):
self.commands.append(member_type)
def say(self, message, text, reply_markup=None, markdown=False,
reply_to_message=True):
# The telegram library doesn't play well with unicode, oh well.
text = text.encode('utf-8') if isinstance(text, unicode) else text
if reply_markup and reply_to_message:
reply_to_message_id = message.message_id
else:
reply_to_message_id = None
parse_mode = telegram.ParseMode.MARKDOWN if markdown else None
return self.sendMessage(chat_id=message.chat.id,
text=text,
reply_to_message_id=reply_to_message_id,
reply_markup=reply_markup,
parse_mode=parse_mode)
def get_updates(self):
logger.debug('')
last_update = self.db.root.last_update
updates = self.getUpdates(offset=last_update)
return updates
def poll(self):
updates = self.get_updates()
self.process_updates(updates)
if updates:
self._stay_awake += 30
else:
if not self._stay_awake:
logger.debug('Entering sleep mode')
return 4
self._stay_awake = self._stay_awake - 1 if self._stay_awake > 0 else 0
return 1
def process_updates(self, updates):
last_update = self.db.root.last_update
for update in filter(lambda x: x.update_id > last_update, updates):
self.process_message(update.message)
self.db.root.last_update = update.update_id
self.db.commit()
def process_message(self, message):
# Cleanup old messages in queue
now = datetime.utcnow()
tdelta = timedelta(seconds=config.command_timeout)
self.queue = {key: value for (key, value) in self.queue.items()
if value['timestamp'] + tdelta > now}
# Update stats
self.db.root.stats['number_of_messages'] += 1
# Register user
user = self.db.get_or_create_user(message.from_user.id)[0]
user.update(message.from_user.first_name,
message.from_user.last_name,
message.from_user.username)
# Register tab
tab, created = self.db.get_or_create_tab(message.chat.id)
# Register user in tab
tab.register_user(user.id)
reply_to_id = False
if isinstance(message.chat, telegram.user.User):
reply_to_id = message.message_id
elif message.reply_to_message:
reply_to_id = message.reply_to_message.message_id
if reply_to_id:
key = '{}_{}'.format(message.chat.id, reply_to_id)
if key in self.queue:
k = self.queue.pop(key)
logger.debug('Calling queued {}'.format(k))
k['command'](message)
return
for cmd in self.commands:
if cmd.match(message):
logger.debug('Calling process_{}'.format(cmd))
cmd(bot=self)(message)
break
else:
self.say(message, 'Are you drunk matey?')
logger.debug('No command found: {}'.format(message.message_id))
return
def remove(business_id):
_DB = DB.query(Application).filter(Application.id == business_id).delete()
DB.commit()
return _DB
def delete_scraper(service):
DB.execute("DELETE FROM scrapers WHERE service=?", [service])
DB.commit()
write_settings_file()
def build_dependency_list(self, addon_id, url, full_name, master):
#if test in ['xbmc.python', 'xbmc.gui'] or kodi.get_condition_visiblity('System.HasAddon(%s)' % addon_id) == 1: return True
if addon_id in self.installed_list:
kodi.log('Dependency is already installed: %s' % addon_id)
return True
user, repo = full_name.split("/")
kodi.log('Finding dependencies for: %s' % addon_id)
if master:
self.sources[addon_id] = {"type": SOURCES.REPO, "url": url, "user": user, "repo": repo, "version": ""}
xml_str = github.find_xml(full_name)
self.sources[addon_id]['version'] = github.get_version_by_xml(BeautifulSoup(xml_str))
else:
version = downloader.download(url, addon_id, self._destination, True, self.quiet)
src_file = kodi.vfs.join("special://home/addons", addon_id)
kodi.vfs.join(src_file, "addon.xml")
xml = kodi.vfs.read_file(kodi.vfs.join(src_file, "addon.xml"), soup=True)
self.save_source(addon_id, {"type": SOURCES.ZIP, "url": url, "user": user, "repo": repo, "version": version})
for dep in xml.findAll('import'):
test = dep['addon']
try:
if dep['optional'].lower() == 'true':
if kodi.get_setting('install_optional') == 'false':
continue
elif kodi.get_setting('prompt_optional') == "true":
c = kodi.dialog_confirm("Install Optional Dependency", dep['name'], dep['addon'])
if not c: continue
except:
pass
if test in ['xbmc.python', 'xbmc.gui'] or kodi.get_condition_visiblity('System.HasAddon(%s)' % test) == 1 or test in self.installed_list:
kodi.log('Dependency is already installed: %s' % test)
continue
self.required_addons += [test]
if test not in self.available_addons:
self.unmet_addons += [test]
else:
self.sources[test] = {"type": SOURCES.DEFAULT, "url": self.source_table[test]}
kodi.log("%s dependency met in %s" % (test, self.source_table[test]))
def user_resolver(user, unmet):
dep_url, dep_filename, dep_full_name, version = github.find_zip(user, unmet)
if dep_url:
kodi.log("%s found in %s repo" % (unmet, user))
self.met_addons.append(unmet)
user, repo = dep_full_name.split("/")
self.sources[unmet] = {"type": SOURCES.ZIP, "url": dep_url, "user": user, "repo": repo, "version": ""}
kodi.log("%s dependency met in %s" % (unmet, dep_url))
return True
return False
def github_resolver(unmet):
results = github.web_search(unmet)
c = kodi.dialog_select("GitHub Search Results for %s" % unmet, [r['full_name'] for r in results['items']])
if c is not False:
dep = results['items'][c]
dep_url = url = "https://github.com/%s/archive/master.zip" % (dep['full_name'])
self.met_addons.append(unmet)
dep_filename = "%s.zip" % unmet
self.sources[unmet] = {"type": SOURCES.REPO, "url": dep_url, "user": user, "repo": repo, "version": ""}
kodi.log("%s dependency met in %s" % (unmet, dep_url))
self.install_addon(unmet, dep_url, dep['full_name'], master=True)
return True
return False
for unmet in self.unmet_addons:
# Now attempt to locate dependencies from available sources
# The addons that can be found in any enabled repos will be installed at the end.
# check if this exists in users root repo
if kodi.get_setting('source_user') == 'true':
if user_resolver(user, unmet): continue
# check if this exists in tva root repo
if kodi.get_setting('source_tva') == 'true':
if user_resolver(tva_user, unmet): continue
# check if this exists on github
if kodi.get_setting('source_github') == 'true':
if github_resolver(unmet): continue
self.unmet_addons = list(set(self.unmet_addons) - set(self.met_addons))
if len(self.unmet_addons):
self.install_error = True
if not self.quiet:
kodi.close_busy_dialog()
kodi.raise_error("", "Unmet Dependencies:", "See log or install manually", ','.join(self.unmet_addons))
kodi.log("Unmet Dependencies for addon install: %s" % addon_id) # % self.addon_id)
kodi.log(','.join(self.unmet_addons))
inserts = [(a, ) for a in self.unmet_addons]
DB.execute_many("INSERT INTO failed_depends(addon_id) VALUES(?)", inserts)
DB.commit()
self.completed.append(addon_id)
def save_source(self, addon_id, source):
DB.execute("REPLACE INTO install_history(addon_id, source) VALUES(?,?)", [addon_id, json.dumps(source)])
DB.commit()
def remove(business_id):
_DB = DB.query(Users).filter(Users.id == business_id).delete()
DB.commit()
return _DB
def main():
print('Main started')
db = DB()
cursor = db.getCursor()
# driver = geckodriver.create_web_driver()
driver = chrome_webdriver.create_web_driver()
spreadsheet = CompanySpreadsheet()
company_info.update_company_info()
# company_positions = spreadsheet.get_companies_positions()
# for i in range(len(company_positions)):
# company_name = company_positions[i]['company_name']
# company_website = company_positions[i]['company_website']
# url = company_positions[i]['posting_url']
# status = company_positions[i]['status']
# update = company_positions[i]['update_entry']
# query = ("SELECT * FROM company WHERE name = %s LIMIT 1")
# cursor.execute(query, (company_name,))
# result = cursor.fetchone()
# # Insert new entry
# if result == None:
# query = ("INSERT INTO company (name, website_url, posting_url, posting_content, status) VALUES (%s, %s, %s, %s, %s)")
# if url == '':
# args = (company_name, company_website, '', '', status)
# else:
# hashed_content = hash_url_content(driver, url, i)
# args = (company_name, company_website, url, hashed_content, status)
# cursor.execute(query, args)
# spreadsheet.set_row_updated_false(i + 2)
# # Update entry
# elif str_to_bool(update) == True:
# query = ("UPDATE company SET website_url = %s, posting_url = %s, posting_content = %s, status = %s WHERE name = %s")
# if url == '':
# args = (company_website, '', '', status, company_name)
# else:
# hashed_content = hash_url_content(driver, url, i)
# args = (company_website, url, hashed_content, status, company_name)
# cursor.execute(query, args)
# spreadsheet.set_row_updated_false(i + 2)
# # Check entry
# else:
# if url == '':
# if result[3] != '':
# spreadsheet.set_row_status_bad(i + 2)
# else:
# spreadsheet.set_row_status_good(i + 2)
# else:
# hashed_content = hash_url_content(driver, url, i)
# if hashed_content == result[4]:
# spreadsheet.set_row_status_good(i + 2)
# else:
# spreadsheet.set_row_status_bad(i + 2)
db.commit()
db.close()
driver.quit()
def remove(idc_id):
_DB = DB.query(Idc).filter(Idc.id == idc_id).delete()
DB.commit()
return _DB
