def parse_page(self, response):
rootEl = pq(response.body_as_unicode())
listEl = rootEl(".mywifeichiran .wifecon")
for itemEl in listEl:
anchorEl = pq(pq(itemEl)('a')[0])
href = anchorEl.attr('href')
portraiturl = anchorEl('img').attr('src')
saveto = os.path.join(saving_dir, href.replace("/", "-"))
need_crawl = False
if not os.path.exists(saveto) or not os.path.exists(os.path.join(saveto, info_filename)):
need_crawl = True
else:
allfiles = os.listdir(saveto)
if download_thumb and not thumb_filename in allfiles:
need_crawl = True
elif download_topview and not topview_filename in allfiles:
need_crawl = True
else:
info = UserConfig(os.path.join(saveto, info_filename))
desired_images = info.get('desired_images')
if download_gallery and len(filter(lambda fname: re.match(r"\d+\.(jpg|jpeg|png|gif)", fname.lower()), allfiles)) < desired_images:
need_crawl = True
if need_crawl:
# If the directory does not exists, crawl it
yield scrapy.Request( rooturl + href, meta = {
'cookiejar': response.meta['cookiejar'],
'href': href,
'saveto': saveto,
'portraiturl': portraiturl
}, callback = self.parse )
def __init__(self):
self.log = logger.get_logger('mail_installer')
self.app_dir = paths.get_app_dir(APP_NAME)
self.app_data_dir = paths.get_data_dir(APP_NAME)
self.app_url = urls.get_app_url(APP_NAME)
self.app_domain_name = urls.get_app_domain_name(APP_NAME)
self.platform_data_dir = paths.get_data_dir('platform')
self.device_domain_name = urls.get_device_domain_name()
self.database_path = '{0}/database'.format(self.app_data_dir)
self.config_path = join(self.app_data_dir, 'config')
self.config = Config(self.config_path)
self.user_config = UserConfig(self.app_data_dir)
def read_config_file(self):
"""
Parses the xml config file and return the configuration
@return: A Config object with the configuration data
"""
self._add_missing_conf_fields()
tree = eT.ElementTree(file=self.conf_file)
root = tree.getroot()
conf_dict = {}
for element in root:
conf_dict[element.tag] = element.text
conf = UserConfig(conf_dict)
return conf
def install(self):
home_folder = join('/home', USER_NAME)
linux.useradd(USER_NAME, home_folder=home_folder)
self.regenerate_config()
fs.makepath(join(self.app_data_dir, 'config'))
fs.makepath(join(self.app_data_dir, 'redis'))
fs.makepath(join(self.app_data_dir, 'log'))
fs.makepath(join(self.app_data_dir, 'nginx'))
fs.makepath(join(self.app_data_dir, 'tmp'))
fs.chownpath(self.app_data_dir, USER_NAME, recursive=True)
if 'SNAP' not in environ:
self.log.info(fs.chownpath(self.app_dir, USER_NAME,
recursive=True))
self.log.info("setup systemd")
if not UserConfig(self.app_data_dir).is_activated():
database_init(self.log, self.app_dir, self.app_data_dir,
self.database_path, USER_NAME)
def build_config(self, config):
defaultSection = 'Sigmauser'
defaultConfig = {
'username' : '',
'password' : '',
'update_timeout' : '180',
'update_time' : '0',
'update_hash' : '',
'update_auto' : '1',
'update_force' : '0',
'update_data' : '',
'update_msg' : '',
'update_login' : '0',
'app_version' : __version__,
'app_delete' : '0',
'debug_disablepause' : '0',
'debug_toast' : '0',
'debug_serverout' : '0',
'debug_forceexit' : '0',
'debug_errortimeout' : '10'
}
self.userConfig = UserConfig(config, defaultSection, defaultConfig)
from subprocess import call
from builder import Config
from config import UserConfig
user_config = UserConfig()
config = Config(user_config.BASEDIR)
def install_package(package_name):
if not package_name:
print "You need to specify a valid package name"
else:
call([config.pip, 'install', package_name.lower()])
# end of file
info_filename = user_config.get("mywifecc", "info_filename")
dirs = os.listdir(saving_dir)
dirs = filter(lambda x: os.path.isdir(os.path.join(saving_dir, x)), dirs)
# sess = requests.session()
# sess.post(rooturl + "/login/comp", { "login_id": username, "passwd": password })
prgbar = None
tmpfile = None
total = 0
downloaded = 0
for dirname in dirs:
info = UserConfig(os.path.join(saving_dir, dirname, info_filename))
filenames = os.listdir(os.path.join(saving_dir, dirname))
number = info.get("number")
pageurl = info.get("pageurl")
def parse_videourl(url):
if url is None:
return url
if re.match(r"^(http|https)://.*", url) is None:
url = rooturl + re.sub(r"^/+", "", url)
return url
videourl1 = parse_videourl(info.get("video"))
videourl2 = parse_videourl(info.get("video2"))
total_of_this = len(filter(lambda x: not x is None,
[videourl1, videourl2]))
import imp
import os.path
from app import db
from migrate.versioning import api
from config import UserConfig
config = UserConfig()
SQLALCHEMY_DATABASE_URI = config.SQLALCHEMY_DATABASE_URI
SQLALCHEMY_MIGRATE_REPO = config.SQLALCHEMY_MIGRATE_REPO
def db_create():
# This creates the new database.
db.create_all()
# If no repo existed, the creation will prepare for the first migration.
if not os.path.exists(SQLALCHEMY_MIGRATE_REPO):
api.create(SQLALCHEMY_MIGRATE_REPO, 'database repository')
api.version_control(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO)
print '\nDatabase creation completed\n'
else:
api.version_control(
SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO,
api.db_version(SQLALCHEMY_DATABASE_URI, SQLALCHEMY_MIGRATE_REPO))
def db_migrate():
# This is used for database migration. Newly created database should go through this as well.
migration = SQLALCHEMY_MIGRATE_REPO + '/versions/%03d_migration.py' % (
def parse(self, response):
rootEl = pq(response.body_as_unicode())
titleEl = rootEl("head title")
titleText = titleEl.text()
m = re.match(r"No\.(\d+)[\u3000]*([^\|]+).*", titleText)
number = int(m.group(1))
name = m.group(2)
name = name.replace(u"蒼い再会", "")
name = re.sub(u"[\u3000\s]+$", "", re.sub(u"^[\u3000\s]+", "", name))
intro = rootEl(".modelsamplephototop").text()
intro = intro.replace(r"^\s+", "").replace(r"\s+$", "").replace(r"\n+", "\n").replace("\n", "\\n")
videoEl = rootEl(".modelsamplephototop #video")
egvideourl = videoEl.attr('src')
name_en = re.match(r"^.*\/([^\/]+)\.\w+$", egvideourl).group(1).lower()
name_en = re.sub(r"_\d+(m|k)$", "", name_en)
name_en = re.sub(r"_$", "", name_en)
saikai = ("_saikai" in name_en)
name_en = re.sub(r"_saikai$", "", name_en)
name_en = re.sub(r"_$", "", name_en)
name_en = re.sub(r"_+", "_", name_en)
name_en = name_en.split("_")
name_en = map(lambda word: word[0].upper() + word[1:], name_en)
name_en = " ".join(name_en)
videourl = pq(rootEl(".modeldlicon a")[0]).attr("href")
videourl2 = rootEl(".modeldlicon2 a")
if len(videourl2) > 0:
videourl2 = pq(videourl2[0]).attr("href")
else:
videourl2 = None
saveto = response.meta['saveto']
if not os.path.exists(saveto):
os.makedirs(saveto)
def download_img(url, filename = None):
if filename is None:
m = re.match(r"^.*\/([^\/]+)$", url)
filename = m.group(1)
savepath = os.path.join(saveto, filename)
if not os.path.exists(savepath):
return scrapy.Request( url, meta = {
'cookiejar': response.meta['cookiejar'],
'savepath': savepath
}, callback = self.parse_download )
else:
return None
portraiturl = response.meta['portraiturl']
if download_thumb and not os.path.exists(os.path.join(saveto, thumb_filename)):
yield download_img(portraiturl, thumb_filename)
topphotourl = videoEl.attr('poster')
if download_topview and not os.path.exists(os.path.join(saveto, topview_filename)):
yield download_img(topphotourl, topview_filename)
imgEls = rootEl(".modelphoto .fancybox img")
if download_gallery:
for photoEl in imgEls:
photourl = pq(photoEl).attr('src')
yield download_img(photourl)
desired_images = len(imgEls)
infoconf = UserConfig()
infoconf.set("pageurl", response.url)
infoconf.set("number", number)
infoconf.set("href", response.meta['href'])
infoconf.set("name", name)
infoconf.set("name_en", name_en)
infoconf.set("saikai", saikai)
infoconf.set("intro", intro)
infoconf.set("video", videourl)
infoconf.set("video2", videourl2)
infoconf.set("egvideo", egvideourl)
infoconf.set("desired_images", desired_images)
infoconf.save(os.path.join(saveto, info_filename))
videos = os.listdir(videodir)
def en_name(url):
name = url.split("/")[-1].split(".")[0].lower()
name = re.sub(r"_\d+(m|k)", "", name)
name = re.sub(r"_saikai", "", name)
name = re.sub(r"_$", "", name)
return name
for i in xrange(len(videos)):
videos[i] = (videos[i], en_name(videos[i]))
for dirname in dirs:
info = UserConfig(os.path.join(saving_dir, dirname, info_filename))
print dirname
egvideourl = info.get("egvideo")
if egvideourl is None: continue
the_enname = en_name(egvideourl)
matched_videos = filter(lambda x: x[1] == the_enname, videos)
videos = filter(lambda x: x[1] != the_enname, videos)
for mvideo in matched_videos:
os.rename(os.path.join(videodir, mvideo[0]),
os.path.join(saving_dir, dirname, mvideo[0]))
print "Didn't match:"
for video in videos:
print video[0]
def configure(self):
self.prepare_storage()
UserConfig(self.app_data_dir).set_activated(True)
def db_migrate(self):
if not UserConfig(self.app_data_dir).is_activated():
self.initialize()
def __init__(self, base):
self.config = Config(base)
self.user_config = UserConfig()
self.base = base
class MailInstaller:
def __init__(self):
self.log = logger.get_logger('mail_installer')
self.app_dir = paths.get_app_dir(APP_NAME)
self.app_data_dir = paths.get_data_dir(APP_NAME)
self.app_url = urls.get_app_url(APP_NAME)
self.app_domain_name = urls.get_app_domain_name(APP_NAME)
self.platform_data_dir = paths.get_data_dir('platform')
self.device_domain_name = urls.get_device_domain_name()
self.database_path = '{0}/database'.format(self.app_data_dir)
self.config_path = join(self.app_data_dir, 'config')
self.config = Config(self.config_path)
self.user_config = UserConfig(self.app_data_dir)
def regenerate_configs(self):
variables = {
'app_dir': self.app_dir,
'app_data_dir': self.app_data_dir,
'db_psql_path': self.database_path,
'db_psql_port': PSQL_PORT,
'db_name': DB_NAME,
'db_user': DB_USER,
'db_password': DB_PASS,
'platform_data_dir': self.platform_data_dir,
'device_domain_name': self.device_domain_name,
'app_domain_name': self.app_domain_name,
'timezone': get_localzone()
}
templates_path = join(self.app_dir, 'config.templates')
gen.generate_files(templates_path, self.config_path, variables)
self.log.info(fs.chownpath(self.config_path, USER_NAME, recursive=True))
def install(self):
linux.useradd('maildrop')
linux.useradd('dovecot')
linux.useradd(USER_NAME)
fs.makepath(join(self.app_data_dir, 'nginx'))
self.regenerate_configs()
fs.chownpath(self.app_data_dir, USER_NAME, recursive=True)
data_dirs = [
join(self.app_data_dir, 'config'),
join(self.app_data_dir, 'log'),
join(self.app_data_dir, 'spool'),
join(self.app_data_dir, 'dovecot'),
join(self.app_data_dir, 'dovecot', 'private'),
join(self.app_data_dir, 'data')
]
for data_dir in data_dirs:
fs.makepath(data_dir)
fs.chownpath(data_dir, USER_NAME)
box_data_dir = join(self.app_data_dir, 'box')
fs.makepath(box_data_dir)
fs.chownpath(box_data_dir, 'dovecot', recursive=True)
dovecot_lda_error_log = join(self.app_data_dir, 'log', 'dovecot-lda.error.log')
fs.touchfile(dovecot_lda_error_log)
fs.chownpath(dovecot_lda_error_log, 'dovecot')
dovecot_lda_info_log = join(self.app_data_dir, 'log', 'dovecot-lda.info.log')
fs.touchfile(dovecot_lda_info_log)
fs.chownpath(dovecot_lda_info_log, 'dovecot')
self.log.info("setup configs")
if not self.user_config.is_activated():
self.database_init(self.database_path, USER_NAME)
def configure(self):
if not self.user_config.is_activated():
self.initialize(self.config, self.user_config, DB_NAME, DB_USER, DB_PASS)
self.prepare_storage()
try:
ports.add_port(25, 'TCP')
ports.add_port(110, 'TCP')
ports.add_port(143, 'TCP')
ports.add_port(587, 'TCP')
except Exception, e:
self.log.warn("failed to add ports: {0}".format(e.message))
class SigmaWeb():
userConfig = None
service = None
kivyApp = None
GUI = None
def __init__(self):
#Inicia o app carregando os objetos que vai precisar
self.GUI = GUI('res/screens.kv')
self.service = Service()
self.kivyApp = KivyApp()
#Configura o objeto do kivy e inicia o programa (isso vai chamar o on_start())
self.kivyApp.root = self.GUI.root
self.kivyApp.use_kivy_settings = False
self.kivyApp.parent = self
self.kivyApp.run()
'''
'' KIVY CALLBACKS
'''
'''
Chamada no momento que o usuario abre o app
'''
def on_start(self):
Debug().note("on_start()")
'''
Faz algumas correcoes no arquivo de configuracao se a versao anterior era mais antiga que a atual
'''
if ProgramVersionGreater(__version__, self.userConfig.getConfig('app_version')):
Debug().warn("Deletando configuracoes de versao antiga!")
username = self.userConfig.getConfig('username')
password = self.userConfig.getConfig('password')
self.userConfig.clearConfig()
self.userConfig.setConfig('username', username)
self.userConfig.setConfig('password', password)
if self.userConfig.getConfig('username') != '': self.userConfig.setConfig('update_login', '1')
self.userConfig.setConfig('app_delete', '0')
'''
Verifica se o usuario ja realizou o login
'''
if self.userConfig.getConfig('username') == '':
self._clearConfig()
self.GUI.setWindow(screenLogin)
else:
if self.userConfig.getConfig('update_login') == '0':
self.service.start(self.userConfig.exportConfig(), (self.userConfig.getConfig('update_auto')=='0'))
self.GUI.setProperty("msg_loading", "Carregando notas... Aguarde!")
else:
self.service.start(self.userConfig.exportConfig(), False)
self.GUI.setProperty("msg_loading", "[b]Buscando notas no sistema[/b]\n\nDependendo da carga no servidor\nisto pode demorar")
if self.GUI.getWindow() == 'NoneType': self.GUI.setWindow(screenLoading)
def on_stop(self):
Debug().note("on_stop()")
self.userConfig.write()
if (self.userConfig.getConfig('debug_forceexit')=='1'): Debug().warn("debug_forceexit = 1")
self.service.stop(((self.userConfig.getConfig('update_auto')=='0') or (self.userConfig.getConfig('app_delete')=='1') or (self.userConfig.getConfig('debug_forceexit')=='1')))
Debug().note("Aplicativo foi finalizado com sucesso!")
def on_pause(self):
Debug().note("on_pause()")
if self.userConfig.getConfig('debug_disablepause')=='0':
self.userConfig.write()
self.service.stop((self.userConfig.getConfig('update_auto')=='0' and self.userConfig.getConfig('update_login') == '0'))
Debug().note("Aplicativo foi pausado com sucesso!")
return True
else: return False
def on_resume(self):
Debug().note("on_resume()")
self.on_start()
def on_update(self, *args):
keys = self.service.getKeys()
if keys is not None:
for keypair in keys:
key, value = keypair
self.userConfig.setConfig(key, value)
'''
Mensagens que o service manda para avisa o usuario de algum acontecimento
'''
if key == 'update_msg':
'''
Erro no servidor (durante login). Faz logoff do usuario e mostra uma mensagem na tela de login
'''
if (self.userConfig.getConfig('update_login') == '1') and (value[:46] == '[color=ff0000][b]Erro no servidor![/b][/color]'):
self.service.stop()
self._clearConfig()
self.GUI.setProperty('msg_error', 'Erro ao acessar o sistema. \nTente novamente mais tarde')
self.GUI.setWindow(screenLogin)
else:
#Mensagem nao eh um erro entao mostra ela com a cor normal
self.GUI.setProperty('usermsg', value)
#Dados que o service baixou do usuario
elif (key == 'update_data'):
self.GUI.setProperty('userdata', value)
if (platform == 'android') and (self.GUI.getWindow() is not screenMain) and (self.userConfig.getConfig('update_login') == '1'): #Reinicia o server caso seja android (Muda a mensagem de login)
self.userConfig.setConfig('update_login', '0')
if self.userConfig.getConfig('update_auto')=='0':
Debug().error("Reinicializando service...")
self.service.start(self.userConfig.exportConfig(), True)
elif key == 'update_auto':
self.GUI.setProperty('update_auto', value)
elif key == 'username':
if value == '':
self._clearConfig()
self.service.stop()
self.GUI.setProperty('msg_error', 'Login ou senha incorreto')
self.GUI.setWindow(screenLogin)
elif key == 'app_delete':
Debug().error("Iniciando o procedimento do app_delete...")
self._clearConfig()
self.kivyApp.stop()
if self.GUI.getWindow() is not screenMain:
if (self.userConfig.getConfig('update_data') != '') and (self.userConfig.getConfig('username') != ''):
self.GUI.setProperty('userdata', self.userConfig.getConfig('update_data'))
self.GUI.setWindow(screenMain)
self.GUI.setProperty('update_auto', self.userConfig.getConfig('update_auto')) #Seta o estado inicial do botao
self.GUI.setProperty('usermsg', self.userConfig.getConfig('update_msg'))
def build_settings(self, settings):
jsonConfig = open('res/config.json', 'r').read()
jsonConfigDebug = open('res/config_debug.json', 'r').read()
settings.add_json_panel('Configuracoes', self.userConfig.kivyConfig, data=jsonConfig)
settings.add_json_panel('Debug', self.userConfig.kivyConfig, data=jsonConfigDebug)
def build_config(self, config):
defaultSection = 'Sigmauser'
defaultConfig = {
'username' : '',
'password' : '',
'update_timeout' : '180',
'update_time' : '0',
'update_hash' : '',
'update_auto' : '1',
'update_force' : '0',
'update_data' : '',
'update_msg' : '',
'update_login' : '0',
'app_version' : __version__,
'app_delete' : '0',
'debug_disablepause' : '0',
'debug_toast' : '0',
'debug_serverout' : '0',
'debug_forceexit' : '0',
'debug_errortimeout' : '10'
}
self.userConfig = UserConfig(config, defaultSection, defaultConfig)
def on_config_change(self, config, section, key, value):
self.service.setKey(key, value)
if key == 'app_delete':
Debug().error("Iniciando o procedimento do app_delete...")
self._clearConfig()
self.kivyApp.stop()
elif key == 'update_timeout':
if int(value) < 30:
self.userConfig.setConfig('update_timeout', '30')
elif key == 'update_auto':
if (self._toggleService(value)): self.GUI.setProperty('update_auto', value)
def on_event(self, *args):
type = args[0]
if type == 'Login':
type, username, password = args
if (username=='') or (password==''): self.GUI.setProperty("msg_error", "Preenchas seus dados")
else:
self._clearConfig()
self.userConfig.setConfig('username', username)
self.userConfig.setConfig('password', RSACrypto('res/sigmawebplus-server.pub').encrypt(password))
self.userConfig.setConfig('update_login', '1')
self.service.start(self.userConfig.exportConfig(), False)
self.GUI.setWindow(screenLoading)
self.GUI.setProperty("msg_loading", "[b]Buscando notas no sistema[/b]\n\nDependendo da carga no servidor\nisto pode demorar")
elif type == 'Reload':
if self.service.getKey('update_force') == '0': self.service.setKey('update_force', '1')
elif type == 'SwitchPanel':
pass
elif type == 'ServiceToggle':
type, value = args
return self._toggleService(value)
def _toggleService(self, value):
if (platform == 'android') and (not self.service.isAlive()): return False
self.userConfig.setConfig('update_auto', str(value)) #Atualiza arquivo de config
self.service.setKey('update_auto', str(value)) #Atualiza service
self.GUI.setProperty('toggleupdate', str(value))
if self.service.isAlive() and (platform == 'android'):
if (value == '0') and ((self.service.getState() == STATE_CONNECTEDREMOTE) or (self.service.getState() == STATE_CONNECTEDANDROID)):
self.service.start(self.userConfig.exportConfig(), True)
if (platform=='android') and (self.userConfig.getConfig('debug_toast')=='1'): AndroidWrapper().Toast('Monitor de notas desativado')
elif (value == '1') and (self.service.getState() == STATE_CONNECTEDTHREAD):
self.service.start(self.userConfig.exportConfig())
if (platform=='android') and (self.userConfig.getConfig('debug_toast')=='1'): AndroidWrapper().Toast('Monitor de notas ativado')
return True
def _clearConfig(self):
self.userConfig.setConfig('username', '')
self.userConfig.setConfig('password', '')
self.userConfig.setConfig('update_time', '0')
self.userConfig.setConfig('update_hash', '')
self.userConfig.setConfig('update_data', '')
self.userConfig.setConfig('update_msg', '')
self.userConfig.setConfig('update_login', '0')