def test_validator(self):
url = URL('www.google.com')
self.assertFalse(url.isValid())
url = URL('google.com')
self.assertFalse(url.isValid())
url = URL('https://www.google.com/googleplus/whousesthis')
self.assertTrue(url.isValid())
url = URL(
'https://www.google.com/googleplus/whousesthis?true=false¬=really'
)
self.assertTrue(url.isValid())
url = URL('https://www.google.com/')
self.assertTrue(url.isValid())
def main():
url1 = URL("example.com")
url2 = URL(" ")
url3 = URL("http://example.com/")
url4 = URL("http://www.example.com")
url5 = URL("http://z.com/")
url6 = URL("http://example.com alsdfkj")
url7 = URL("http://example.com ()")
url8 = URL("google.com")
#testing validity
if not url1.isValid():
print "Pass first test"
else:
print "Failed first test"
if not url2.isValid():
print "Pass second test"
else:
print "Failed second test"
if url3.isValid():
print "Pass third test"
else:
print "Failed third test"
if not url4.isValid():
print "Pass fourth test"
else:
print "Failed fourth test"
if url5.isValid():
print "Pass fifth test"
else:
print "Failed fifth test"
if not url6.isValid():
print "Pass sixth test"
else:
print "Failed sixth test"
if not url7.isValid():
print "Pass seventh test"
else:
print "Failed seventh test"
if url3 == url3:
print "Pass eighth test"
else:
print "Failed eighth test"
if url3 < url5:
print "Pass ninth test"
else:
print "Failed ninth test"
if url1 != url3:
print "Pass tenth test"
else:
print "Pass tenth test"
def test_canonicalizer(self):
url = URL('www.google.com').getNormalized()
self.assertEqual(url, 'http://www.google.com/')
url = URL('google.com').getNormalized()
self.assertEqual(url, 'http://google.com/')
url = URL(
'https://www.google.com/googleplus/whousesthis').getNormalized()
self.assertEqual(url, 'https://www.google.com/googleplus/whousesthis')
url = URL(
'https://www.google.com/googleplus/whousesthis?true=false¬=really'
).getNormalized()
self.assertEqual(
url,
'https://www.google.com/googleplus/whousesthis?true=false¬=really'
)
def set_url(self, url):
if isinstance(url, URL):
self.url = str(url)
self.url_split = url
else:
self.url = url
self.url_split = URL(self.url)
def fetch_url(protocol,
hostname,
port,
useragent,
timeout,
retries,
storage=[]):
url_obj = URL(protocol, hostname, str(port), useragent, timeout,
retries) # define url object
url_obj.getdata()
storage.append(url_obj.result)
def _get_urls(self):
urls = []
for element in self._get_elements('Url'):
template = element.getAttribute('template')
type = element.getAttribute('type')
if template and type:
url = URL()
url.template = template
url.type = type
urls.append(url)
return urls
def test_eq(self):
url1 = URL('www.google.com')
url2 = URL('www.google.com/')
self.assertTrue(url2.__eq__(url1))
url1 = URL('google.com')
url2 = URL('www.google.com/')
self.assertFalse(url2.__eq__(url1))
url1 = URL('www.google.com/hello')
url2 = URL('www.google.com/')
self.assertFalse(url2.__eq__(url1))
url1 = URL('https://www.google.com/')
url2 = URL('http://www.google.com/')
self.assertFalse(url2.__eq__(url1))
url1 = URL('google.com')
url2 = URL('google.com')
self.assertTrue(url2.__eq__(url1))
def process_header(self, ctx: Context):
host_url = URL(ctx.header.host)
cfg = self.cfg_by_domain(host_url.host)
ctx.cfg = cfg
if cfg.get('x-forward-for'):
ctx.header.args['X-Forwarded-For'] = ctx.src_addr[0]
if cfg.get('-'):
for arg in cfg['-']:
utils.del_key(ctx.header.args, arg)
def __process(self, task):
"""
Collect urls from the web page.
"""
task_url = URL(task)
soup = bs(self.__response.content, "lxml")
# Get all links from current page.Remove duplicated links.
url_set = set(
item.get("href") for item in soup.find_all(lambda tag: tag.get(
"href") and "javascript" not in tag.get("href")))
target = set()
# Construct new target links
for item in url_set:
u = URL(item)
if u.netloc and u.netloc not in Worker.Worker_basedomain:
continue
else:
u.standardize(task_url.url)
std_url = u.url
target.add(std_url)
return target
def construct_mongo_url():
""" Construct URL for connecting to MongoDB."""
url = URL('')
if MONGO_CONFIG.host in ['127.0.0.1', 'localhost']:
url.scheme = 'mongodb'
url.netloc = f'{MONGO_CONFIG.host}:{MONGO_CONFIG.port}'
else:
url.scheme = 'mongodb+srv'
url.netloc = f'{MONGO_CONFIG.username}:{MONGO_CONFIG.password}@{MONGO_CONFIG.host}'
url.path = MONGO_CONFIG.database
url.query_param.set('retryWrites', 'true')
url.query_param.set('w', 'majority')
return str(url)
def config(self, **config):
self._config = config
for key, value in config.items():
self.__dict__[key] = value
if not self.path:
raise ValueError("Path is Missing")
if not self.urls:
raise ValueError("Uri is Missing")
self._urls = [URL(url) for url in self.urls]
self.init_service()
def setupURLs(self):
self.urls = []
origins = self.getOrigins()
destination = self.destinations[0]
for origin in origins:
startdate = self.getStartDate()
enddate = self.getEndDate()
url = URL({
"origin": origin,
"destination": destination,
"date1": startdate,
"date2": enddate
})
self.urls.append(url)
def test_gt(self):
url1 = URL('www.google.com')
url2 = URL('www.google.com/hello/world')
self.assertTrue(url2.__gt__(url1))
url1 = URL('www.google.com')
url2 = URL('www.google.com/?hello=world')
self.assertTrue(url2.__gt__(url1))
url1 = URL('www.google.com/a')
url2 = URL('www.google.com/b/oh')
self.assertTrue(url2.__gt__(url1))
url1 = URL('www.foogle.com/what')
url2 = URL('www.google.com/')
self.assertTrue(url2.__gt__(url1))
def main():
# system prompt
if len(sys.argv) is not 2:
print 'use: python main.py <input-file>\n'
exit(1)
# read input file
f = open(sys.argv[1])
lines = readfile(f)
# create URL objects
urls = [URL(x) for x in lines]
# count occurrences of original and canonical URLS
dic_source = {}
dic_canonical = {}
for url in urls:
if url.getURL() in dic_source:
dic_source[url.getURL()] = dic_source[url.getURL()] + 1
else:
dic_source[url.getURL()] = 1
if url.getNormalized() in dic_canonical:
dic_canonical[
url.getNormalized()] = dic_canonical[url.getNormalized()] + 1
else:
dic_canonical[url.getNormalized()] = 1
# print properties of each URL read
for url in urls:
print "Source: " + url.getURL()
if url.isValid():
print "Valid: True"
else:
print "Valid: False"
print "Canonical: " + url.getNormalized()
if dic_source[url.getURL()] == 1:
print "Source unique: True"
else:
print "Source unique: False"
if dic_canonical[url.getNormalized()] == 1:
print "Canonicalized URL unique: True\n"
else:
print "Canonicalized URL unique: False\n"
def forward(self, ctx):
up_url = URL(ctx.cfg['upstream'])
if up_url.protocol == Protocol.unix:
right_conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
right_conn.connect(up_url.host)
else:
right_conn = socket.socket()
right_conn.connect((up_url.host, up_url.port))
right_conn.sendall(ctx.header.encode())
t = spawn(self.relay, (ctx.left_conn, right_conn))
self.relay(right_conn, ctx.left_conn)
t.join()
ctx.left_conn.close()
right_conn.close()
def __make_request(self, task):
"""
Send request to target.
:return: None
"""
self.__response = None
try:
self.__response = self.__session.get(task)
except requests.exceptions.HTTPError as e:
# If the remote server return a response with 4xx or 5xx code,
# requests will raise a HTTPError
self.__response = e.response
except requests.exceptions.RequestException as e:
msg = "{url} - {error}".format(url=task.encode("utf-8"), error=e)
Worker.Worker_error_logger.warning(msg)
Worker.Worker_debug_logger.warning(msg)
raise TaskAbort()
except Exception as e:
# In case of unexpected errors.
msg = "{url} - An unexpected error occurred: {err}".format(
url=task.encode("utf-8"), err=e)
Worker.Worker_sys_logger.exception(msg)
Worker.Worker_debug_logger.exception(msg)
raise TaskAbort()
else:
# If we are redirected to a domain that not in basedomain ,
# ignore it.
if Worker.Worker_basedomain and URL(
self.__response.url
).netloc not in Worker.Worker_basedomain:
raise TaskAbort()
else:
# Deal with http errors
if 400 <= self.__response.status_code <= 599:
msg = "{url} - {status}".format(
url=task.encode("utf-8"),
status=self.__response.status_code)
Worker.Worker_error_logger.warning(msg)
Worker.Worker_debug_logger.warning(msg)
raise TaskAbort()
def main():
filename = None
if len(sys.argv) is not 2:
print 'Usage: python main.py input-file'
exit(1)
inputfile = open(sys.argv[1])
strings = get_strings(inputfile)
urls = [URL(x) for x in strings]
normalized = {}
original = {}
for url in urls:
if url.getURL() in original:
original[url.getURL()] = original[url.getURL()] + 1
else:
original[url.getURL()] = 1
if url.getNormalized() in normalized:
normalized[
url.getNormalized()] = normalized[url.getNormalized()] + 1
else:
normalized[url.getNormalized()] = 1
for url in urls:
print "Source: " + url.getURL()
if url.isValid():
print "Valid: True"
else:
print "Valid: False"
print "Canonical: " + url.getNormalized()
if original[url.getURL()] == 1:
print "Source unique: True"
else:
print "Source unique: False"
if normalized[url.getNormalized()] == 1:
print "Canonicalized unique: True"
else:
print "Canonicalized unique: False"
def send_subscription_confirmation(cls, email: str,
subs_lst: List[DB.EmailSubscription]):
""" Send the email for confirmation of email subscription."""
confirmation_url = URL(
f'https://{FRONTEND_BASE_URI}/visa/email/subscription')
confirmation_url.query_param.set('email', email)
for visa_type, code, till in subs_lst:
confirmation_url.query_param.append('visa_type', visa_type.value)
confirmation_url.query_param.append('code', code.value)
confirmation_url.query_param.append('till', till)
subscription_str = '<ul>\n{}\n</ul>'.format('\n'.join([
'<li>{} Visa at {} till {}.</li>'.format(
VISA_TYPE_DETAILS[vt],
next((e.name_en
for e in USEmbassy.get_embassy_lst() if e.code == ec),
'None'),
tl.strftime('%Y/%m/%d') if tl != datetime.max else 'FOREVER',
) for vt, ec, tl in subs_lst
]))
content = SUBSCRIPTION_CONFIRMATION_CONTENT.format(
user=email.split('@')[0],
email=email,
subscription_str=subscription_str,
confirmation_url=confirmation_url,
)
for _ in range(10): # for robust
sent = cls.send_email(
title=SUBSCRIPTION_CONFIRMATION_TITLE.format(email=email),
content=content,
receivers=[email])
if sent:
break
else:
sent = False
return sent
from url import URL from quiz import Quiz url_generator = URL() url_generator.choose_category() url_generator.choose_type() url_generator.choose_difficulty() url_generator.choose_number_of_questions() # url_generator.number_of = 5 # url_generator.category = "any" # url_generator.difficulty = "any" # url_generator.type = "any" api_url = url_generator.generate() quiz = Quiz(api_url) quiz.ask_question() quiz.get_user_answer()
def menu_feed(self, email, senha):
self.email = email
self.senha = senha
self.user = self.buscar_usuario(email)
print('FEED')
print('1 - Publicar')
print('2 - Adicionar amigo')
print('3 - Buscar email de seu amigo')
print('4 - Mudar profissão')
print('5 - Listar minhas informações')
print('6 - Desfazer amizade')
print('7 - Listar amigos')
print('8 - Alterar sua senha')
print('9 - Excuir conta')
print('10 - BATE-PAPO')
print('11 - Adicionar anime com site')
print('12 - Buscar anime ou site pelo nome')
print('13 - Criar evento')
print('14 - Listar eventos futuros')
print('15 - Listar eventos passados ')
print('x - Sair')
opcao1 = input('Digite a opção (feed): ').lower()
while opcao1 != 'x':
if opcao1 == '1':
texto = input('Texto: ')
'''local=input('Local: ')
amigo=input('Email Amigo: ')'''
print(self.publicar(self.user, texto))
if opcao1 == '2':
email_amigo = input(
'Digite o email da pessoa que deseja adicionar: ').lower()
self.usuarioDAO.inserir_amigo(self.email, email_amigo)
print('Agora ' + str(self.buscar_usuario(email_amigo).nome) +
' é seu amigo')
if opcao1 == '3':
nome_amigo = input('Digite o nome do seu amigo: ')
print(self.buscar_amigo(nome_amigo))
if opcao1 == '4':
nova_profissao = input('Digite sua nova profissão: ')
self.user.set_profissao(nova_profissao)
self.usuarioDAO.set_profissao(nova_profissao, self.email)
print('Agora sua nova profissao é ' + nova_profissao)
if opcao1 == '5':
print(self.usuarioDAO.listar_dados(self.email))
if opcao1 == '6':
email_amigo = input('Digite o email do seu amigo: ').lower()
self.usuarioDAO.desfazer_amizade(self.email, email_amigo)
print('Você desfez amizade com ' +
self.buscar_usuario(email_amigo).nome)
if opcao1 == '7':
for amigo in self.usuarioDAO.listar_amigos(self.user.email):
print(amigo)
if opcao1 == '8':
sua_senha = input('Digite sua senha: ')
if sua_senha == self.senha:
n_senha = input('Digite a nova senha: ')
n_senha1 = input('Confirme a nova senha: ')
if n_senha != n_senha1:
print('As senhas não conferem')
n_senha = input('Digite a nova senha: ')
n_senha1 = input('Confirme a nova senha: ')
self.usuarioDAO.set_senha(self.email, n_senha)
print('Senha alterada com sucesso!')
if opcao1 == '9':
senha = input('Por questão de segurança digite sua senha: ')
cursor = self.usuarioDAO.conexao.cursor()
cursor.execute('SELECT * FROM login')
for tupla in cursor.fetchall():
if str(tupla[0]) == str(email):
if str(tupla[1]) == self.senha:
self.usuarioDAO.excuir_conta(email)
print('Usuário excluído')
self.menu_inicial()
else:
print('Senha incorreta')
if opcao1 == '10':
print('BATE-PAPO')
print('1 - Iniciar conversa')
print('2 - Entrar em conversa')
'''print('3 - Buscar mensagem')
print('4 - Apagar conversa')
print('5 - Apagar todas conversas')'''
print('x - Voltar')
opcao2 = input('Digite sua opção: ').lower()
while opcao2 != 'x':
if opcao2 == '1':
email_d = input('Digite o email da pessoa: ')
texto = input('Digite a mensagem: ')
mensagem = Mensagem(texto, self.email, email_d)
self.mensagemDAO.enviar(mensagem)
if opcao2 == '2':
print('CONVERSAS')
for nome in self.mensagemDAO.nomes_conversas(
self.email):
print(str(nome[0]))
nome = input(
'Digite o nome da pessoa (Para sair digite x): ')
if nome == 'x':
self.menu_feed(self.email, self.senha)
pessoa = self.usuarioDAO.buscar_pelo_nome(nome)
if pessoa == False:
print('Nome incorreto')
opcao2 = '2'
return opcao2
self.conversa(pessoa, self.user)
conversar = input(
'Você deseja continuar conversa? sim ou nao: '
).lower()
while conversar == 'sim':
texto = input('Digite a mensagem: ')
mensagem = Mensagem(texto, self.email,
pessoa.email)
self.mensagemDAO.enviar(mensagem)
self.conversa(pessoa, self.user)
conversar = input(
'Você deseja continuar conversa? sim ou nao: ')
if opcao2 == '3':
pass
if opcao2 == '4':
pass
if opcao2 == '5':
pass
print('BATE-PAPO')
print('1 - Iniciar conversa')
print('2 - Entrar em conversa')
'''print('3 - Buscar mensagem')
print('4 - Apagar conversa')
print('5 - Apagar todas conversas')'''
print('x - Voltar')
opcao2 = input('Digite sua opção: ').lower()
return self.menu_feed(self.email, self.senha)
if opcao1 == '11':
nome = input('Digite o nome do anime: ').lower()
url = input('Digite a URL do site do anime: ')
site = URL(url, nome)
self.urlDAO.inserir(site)
print('Anime cadastrado')
if opcao1 == '12':
nome = input('Digite o nome do anime: ').lower()
animes = self.urlDAO.listar_nome(nome)
num = 1
for anime in animes:
print(str(num) + ' - ' + str(anime.nome))
num += 1
opcao = input('Digite o número do anime: ')
for x in range(1, len(animes) + 1):
x -= 1
for anime in animes:
if opcao == str(x + 1):
anime.abrir()
if opcao1 == '13':
nome = input('Digite o nome do evento: ')
tema = input('Digite o tema do evento: ')
local = input('Digite o local do evento: ')
data = input('Digite a data do evento (DD/MM/AAAA): ')
hora_ini = input(
'Digite o horário de início do evento (HH:MM): ')
hora_fim = input('Digite o horário de fim do evento (HH:MM): ')
try:
self.criar_evento(nome, tema, local, data, hora_ini,
hora_fim)
except ValueError:
print(
'Não foi possível cadastrar usuário. Erro: Data inserida de forma incorreta'
)
if opcao1 == '14':
eventos = self.eventoDAO.listar()
for e in eventos:
print('Nome:', e.nome)
print('Tema:', e.tema)
print('Local:', e.local)
print('Data:', e.data)
print('Horário de início:', e.hora_ini)
print('Horário de fim:', e.hora_fim)
print(' ')
if opcao1 == '15':
eventos = self.eventoDAO.listar_passados()
for e in eventos:
print('Nome:', e.nome)
print('Tema:', e.tema)
print('Local:', e.local)
print('Data:', e.data)
print('Horário de início:', e.hora_ini)
print('Horário de fim:', e.hora_fim)
print(' ')
if opcao1 == 'x':
self.menu_inicial()
print('FEED')
print('1 - Publicar')
print('2 - Adicionar amigo')
print('3 - Buscar email de seu amigo')
print('4 - Mudar profissão')
print('5 - Listar minhas informações')
print('6 - Desfazer amizade')
print('7 - Listar amigos')
print('8 - Alterar sua senha')
print('9 - Excuir conta')
print('10 - BATE-PAPO')
print('11 - Adicionar anime com site')
print('12 - Buscar anime ou site pelo nome')
print('13 - Criar evento')
print('14 - Listar eventos futuros')
print('15 - Listar eventos passados ')
print('x - Sair')
opcao1 = input('Digite a opção (feed): ').lower()
return opcao1
for row in csvreader:
legit_urls.append(row[0])
with open(more_legit_url_data, 'r', encoding='utf8') as more_legit_file:
csvreader = csv.reader(more_legit_file)
data_list = list(csvreader)
for row in data_list[:50000]:
legit_urls.append(row[1])
num_legit_urls = len(legit_urls)
num_phish_urls = len(
phish_urls) if len(phish_urls) <= len(legit_urls) else len(legit_urls)
print(f'num legit: {num_legit_urls}')
print(f'num phish: {num_phish_urls}')
# iterate through urls, making url objects
print('setting up urls')
url_objs = [URL(u, 0).to_json() for u in phish_urls[:num_phish_urls]] + \
[URL(u, 1).to_json() for u in legit_urls[:num_legit_urls]]
# bulk save them into mongodb databases
print('inserting urls')
new_result = urls.insert_many(url_objs)
print(f'Number of inserts: {len(new_result.inserted_ids)}')
# Disconnect from MongoDB
client.close()
def send_unsubscription_confirmation(cls, email: str):
""" Send the email for confirmation of email unsubscription. """
subs_lst_by_email = DB.Subscription.get_subscriptions_by_email(email)
if len(subs_lst_by_email
) == 0: # If the user has no subscription/email doesn't exist
for _ in range(10):
sent = cls.send_email(
title=UNSUBSCRIPTION_EMPTY_SUBS_TITLE.format(email=email),
content=UNSUBSCRIPTION_EMPTY_SUBS_CONTENT.format(
user=email.split('@')[0],
email=email,
base_uri=FRONTEND_BASE_URI),
receivers=[email],
)
if sent:
break
else:
sent = False
return sent
unsubs_url = URL(
f'https://{FRONTEND_BASE_URI}/visa/email/unsubscription'
) # Unsubscription confirmation url
unsubs_url.query_param.set('email', email)
unsubs_all_url = unsubs_url.copy()
unsubs_info = []
for subs in subs_lst_by_email:
url = unsubs_url.copy()
url.query_param.set('visa_type', subs['visa_type'])
url.query_param.set('code', subs['embassy_code'])
url.query_param.set('till', subs['till'])
unsubs_info.append((subs['visa_type'], subs['embassy_code'],
subs['till'], subs['expired'], url))
unsubs_all_url.query_param.append('visa_type', subs['visa_type'])
unsubs_all_url.query_param.append('code', subs['embassy_code'])
unsubs_all_url.query_param.append('till', subs['till'])
unsubscription_str = '{}'.format('\n'.join([
'<li>{} Visa at {} {} on {}: click <a href="{}">this link</a> to unsubscribe.</li>'
.format(
VISA_TYPE_DETAILS[vt],
next((e.name_en
for e in USEmbassy.get_embassy_lst() if e.code == ec),
'None'),
'expired' if exp else 'expiring',
tl.strftime('%Y/%m/%d') if tl.year < 9999 else 'FOREVER',
url,
) for vt, ec, tl, exp, url in unsubs_info
]))
content = UNSUBSCRIPTION_CONFIRMATION_CONTENT.format(
user=email.split('@')[0],
email=email,
unsubscription_str=unsubscription_str,
unsubscribe_all_url=unsubs_all_url,
)
for _ in range(10):
sent = cls.send_email(title=UNSUBSCRIPTION_CONFIRMATION_TITLE,
content=content,
receivers=[email])
if sent:
break
else:
sent = False
return sent
def main(
self,
path=None,
*names,
run_nbinit=True,
all=False,
**kwargs,
):
log = self.print
log(f'URLLoader.main({self}, path={path}, names={names}, all={all}, **{kwargs}')
gist = kwargs.get('gist')
github = kwargs.get('github')
gitlab = kwargs.get('gitlab')
pkg = kwargs.get('pkg')
url = urlparse(path)
if url.scheme or gist or github or gitlab:
domain = url.netloc
from _gist import Commit, File, Gist, chars
gist_attrs = Gist.parse_url(path, throw=False)
if gist_attrs or gist:
assert not github
assert not gitlab
assert not pkg
gist_attrs = merge(gist_attrs, kwargs, 'user', 'commit', 'file')
if gist:
maybe_user = maybe(f'(?P<user>{chars})/')
id_re = f'(?P<id>{chars})'
m = match(f'^{maybe_user}{id_re}$', gist)
if not m: raise Exception(f'Unrecognized gist: {gist}')
user = m.group('user')
id = m.group('id')
gist_attrs = merge(gist_attrs, dict(id=id, user=user))
log(f'gist_attrs: {gist_attrs}')
obj = Gist.from_dict(skip_cache=opts.skip_cache, **gist_attrs)
if isinstance(obj, Commit):
commit = obj
log(f'Parsed commit: {commit}')
gist = commit.gist
name = gist.module_name
elif isinstance(obj, File):
file = obj
commit = file.commit
log(f'Parsed commit: {commit} (file {file})')
name = file.module_fullname
else:
raise Exception(f'Unrecognized gist object: {obj}')
if name in sys.modules:
mod = sys.modules[name]
log(f'Found loaded gist module: {mod}')
else:
log(f'Loading gist module {name} (commit {commit})')
node = GitNode(commit)
with importer.tmp_path(node):
[ top, *pieces ] = name.split('.')
mod_path = [top]
fullname = top
while True:
spec = importer.find_spec(name)
if not spec:
raise Exception(f'Failed to find spec for {name} (commit {commit})')
mod = importer.create_module(spec)
importer.exec_module(mod)
if not pieces: break
basename = pieces.pop(0)
mod_path += [basename]
fullname += f'.{basename}'
elif domain == 'github.com':
raise NotImplementedError
elif domain == 'gitlab.com':
raise NotImplementedError
elif match(r'https?', url.scheme):
from url import URL
url = URL(path)
# Force materialization of the URL's content to the on-disk cache
url.content
path = url._dir / 'content'
mod = self.url_mod(url, path)
else:
mod = self.url_mod(url, path)
if not names and not all: return mod
dct = mod.__dict__
if hasattr(mod, '__all__'):
members = mod.__all__
else:
members = [ name for name in dct if not name.startswith('_') ]
import_all = '*' in names or all is True or all == '*'
if import_all:
names = [ name for name in names if not name == '*' ]
not_found = list(set(names).difference(members))
if not_found:
raise Exception('Names not found: %s' % ','.join(not_found))
if not import_all:
members = names
log(f'Bubbling up {members}')
update = { name: dct[name] for name in members }
stk = stack()
cur_file = stk[0].filename
cur_dir = Path(cur_file).parent
frame_info = next(frame for frame in stk if Path(frame.filename).parent != cur_dir)
log(f'Frame: {frame_info}, {frame_info.filename}')
frame_info.frame.f_globals.update(update)
return mod
from url import URL
def get_strings(f):
strings = []
line = f.readline()
while len(line) > 0:
# do not add empty line
if len(line) > 1:
strings.append(line[:len(line) - 1])
line = f.readline()
return strings
if __name__ == "__main__":
filename = None
if len(sys.argv) not in (3,3):
print 'Usage: python main.py input-file output-file'
exit(1)
inputfile = open(sys.argv[1])
strings = get_strings(inputfile)
urls = [URL(x) for x in strings]
outputfile = open(sys.argv[2], 'w+')
results = algorithms.analyzeURLs(urls)
for item in results:
outputfile.write('%s\n' % item)
inputfile.close()
outputfile.close()
def runTest(self):
assert URL(original).getNormalized() == normalized, \
(original, normalized, URL(original).getNormalized())
def testValidity(url, expected):
if URL(url).isValid() == expected:
print "+ Validity PASS"
else:
print "- Validity FAIL"
def testNormalization(url, expected):
result = URL(url).getNormalized()
if result == expected:
print "+ Normalization PASS"
else:
print "- Normalization FAIL: result was " + result
def __montar_objeto_site(self, tupla):
return URL(tupla['url'], tupla['nome'])
def runTest(self):
assert (URL(value).isValid()) == expected, \
(expected, value, URL(value).isValid())
""" Everything that's static variable or shared by other logic.
No need to refactor if it's less than 1 kLOC
"""
import os
import enum
from collections import namedtuple
from dotenv import load_dotenv
from url import URL
load_dotenv()
MEMBER_URL = URL('{}/v5/members/'.format(os.getenv('API_BASE_URL')))
CHALLENGE_URL = URL('{}/v5/challenges/?{}'.format(os.getenv('API_BASE_URL'), os.getenv('DEFAULT_CHALLENGE_QUERY')))
RESOURCE_URL = URL('{}/v5/resources/?perPage=5000'.format(os.getenv('API_BASE_URL')))
AUTH_TOKEN = os.getenv('JWT') and 'Bearer {}'.format(os.getenv('JWT'))
MongoConfig = namedtuple('MongoConfig', ['host', 'port', 'username', 'password', 'database'])
MONGO_CONFIG = MongoConfig(
host=os.getenv("MONGO_HOST"),
port=os.getenv("MONGO_PORT") and int(os.getenv("MONGO_PORT")),
username=os.getenv('MONGO_USER'),
password=os.getenv('MONGO_PSWD'),
database=os.getenv("MONGO_DATABASE"),
)
# Some meta data from topcoder.com, manually written here because it's pretty short
DETAILED_STATUS = [
'New',
'Draft',
'Cancelled',
'Active',
'Completed',
