def new_url():
slug = request.form["slug"]
url = request.form["url"]
print(url, slug)
if slug == "":
slug = generateSlug(6)
urlArr.append(Url(url, slug))
return "http://localhost:5000/" + slug
def getHtml(urlNum, allUrlList, urlImgQueue, urlQueue):
while (urlNum < 20 and urlQueue.not_empty):
url = urlQueue.get()
thisHtml = Html(url)
html = thisHtml.getHtml()
if (html != None):
thisUrl = Url(html, allUrlList, urlQueue, urlImgQueue, urlNum)
allUrlList, urlQueue, urlImgQueue, urlNum = thisUrl.getUrl()
def main():
#Defining Variables
covers_url = 'https://www.covers.com/Sports/mlb/PrintSheetHtml?isPrevious=False'
urlObj = Url(covers_url)
parser = Parser()
#Logic
bs_obj = urlObj.getBSoup()
oddTable = bs_obj.findAll('tr', attrs={'class': 'trBackGround'})
evenTable = bs_obj.findAll('tr', attrs={'class': ''})
#oddThread and evenThreads are list of lists DS.
#which contains the game threads
oddThreads = parser.parseTable(oddTable)
evenThreads = parser.parseTable(evenTable)
def main():
u = Url()
m = Message()
last_update_id = ''
while True:
print("Updates")
updates = u.get_updates(last_update_id)
print(updates)
if len(updates['result']) > 0:
last_update_id = m.get_last_update_id(updates) + 1
m.handle_updates(updates)
time.sleep(0.5)
def create_graph():
graph = nx.DiGraph()
dev_directory = "DEV/"
graph.add_nodes_from(range(1, len(mapping)+1))
for direct in os.listdir(dev_directory):
if direct != '.DS_Store':
for file in os.listdir(dev_directory + direct):
# posting(file) returns url and stuff for DOC_ID_DICT and for tokenizer
temp = Url(dev_directory + direct + '/' + file)
if temp.get_url() in mapping:
list_of_links = get_all_sublinks(temp.get_html(), temp.get_url())
graph.add_edges_from([(mapping[temp.get_url()], final_edge)for final_edge in list_of_links])
if mapping[temp.get_url()] == 34480 or mapping[temp.get_url()] == 34479:
return graph
def get_all_files(dev_directory):
file_count_name = 'indexes2/inverted_index_'
file_count = 0
file_count_name_count = 1
doc_id = 1
inverted_index = defaultdict(list)
reader = Html_Reader()
for direct in os.listdir(dev_directory):
if direct != '.DS_Store':
for file in os.listdir(dev_directory + direct):
# posting(file) returns url and stuff for DOC_ID_DICT and for tokenizer
temp = Url(dev_directory + direct + '/' + file)
try:
read = reader.read_file(temp.get_html(), doc_id,
inverted_index)
if read:
# write to a file the current inverted index, if it is above a certain file count
file_count += 1
DOC_ID_DICT[doc_id] = temp.get_url()
doc_id += 1
except Exception as e:
with open('error.txt', 'w+') as error_file:
error_file.write(str(e) + str(temp.get_url()) + "\n")
if file_count == 1000:
write_to_index(inverted_index, file_count_name_count,
file_count_name)
inverted_index = defaultdict(list)
elif file_count > 1000:
file_count_name_count += 1
file_count = 0
# adds the current dict to a file for a partial index
# change file_count_name also to write to a different file
write_to_index(inverted_index, file_count_name_count, file_count_name)
write_doc_ids(DOC_ID_DICT)
def TestValidator():
tests = [(True, Url("ftp://ftp.is.co.za/rfc/rfc1808.txt")),
(True, Url("http://www.ietf.org/rfc/rfc2396.txt")),
(False, Url("ldap://[2001:db8::7]/c=GB?objectClass?one")),
(False, Url("mailto:[email protected]")),
(False, Url("news:comp.infosystems.www.servers.unix")),
(False, Url("tel:+1-816-555-1212")),
(False, Url("telnet://192.0.2.16:80/")),
(False,
Url("urn:oasis:names:specification:docbook:dtd:xml:4.1.2"))]
suite = unittest.TestSuite()
def testcase(expected, value):
class test(unittest.TestCase):
def runTest(self):
self.assertEqual(value.isValid(), expected,
"Url: {}".format(value.url))
return test()
for (expected, value) in tests:
suite.addTest(testcase(expected, value))
return suite
def TestNormalize():
tests1 = [(False, Url("http://:@example.com/")),
(False, Url("http://@example.com/")),
(False, Url("http://example.com")),
(False, Url("HTTP://example.com/")),
(False, Url("http://EXAMPLE.COM/")),
(False, Url("http://example.com/%7Ejane")),
(False, Url("http://example.com/?q=%C7")),
(False, Url("http://example.com/?q=%5c")),
(False, Url("http://example.com/?q=C%CC%A7")),
(False, Url("http://example.com/a/../a/b")),
(False, Url("http://example.com/a/./b")),
(False, Url("http://example.com:80/")),
(True, Url("http://example.com/")),
(True, Url("http://example.com/?q=%C3%87")),
(True, Url("http://example.com/?q=%E2%85%A0")),
(True, Url("http://example.com/?q=%5C")),
(True, Url("http://example.com/~jane")),
(True, Url("http://example.com/a/b")),
(True, Url("http://example.com:8080/")),
(True, Url("http://*****:*****@example.com/")),
(True, Url("ftp://ftp.is.co.za/rfc/rfc1808.txt")),
(True, Url("http://www.ietf.org/rfc/rfc2396.txt")),
(True, Url("ldap://[2001:db8::7]/c=GB?objectClass?one")),
(True, Url("mailto:[email protected]")),
(True, Url("news:comp.infosystems.www.servers.unix")),
(True, Url("tel:+1-816-555-1212")),
(True, Url("telnet://192.0.2.16:80/")),
(True,
Url("urn:oasis:names:specification:docbook:dtd:xml:4.1.2")),
(True, Url("http://127.0.0.1/")),
(False, Url("http://127.0.0.1:80/")),
(True, Url("http://www.w3.org/2000/01/rdf-schema#")),
(False, Url("http://example.com:081/"))]
suite = unittest.TestSuite()
def testcase(expected, value):
class test(unittest.TestCase):
def runTest(self):
self.assertTrue(
(value.getNormalized() == value.url) == expected)
return test()
for (expected, value) in tests1:
suite.addTest(testcase(expected, value))
return suite
from Arquivo import Arquivo
from Url import Url
url = Url('https://docs.python.org/3/')
file = Arquivo('teste')
list = url.getWords()
for word in list:
file.addRegistro(url.indexaPalavra(word))
def __init__(self, site, url, postfix=''):
self.url = Url(site, url, postfix)
def getItem(self):
return Item(Request(None,self.method,
[Header("Content-Type","Content-Type","application/json","text")],
Url(self.host,self.path)),self.path)
'Sec-Fetch-User':
'******',
'Upgrade-Insecure-Requests':
'1',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
'(KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'
}
RELEVANT_TITLES = [
'אחריות', 'דגם', 'זיכרון', 'כונן קשיח', 'משקל', 'סוללה', 'מעבד',
'מאיץ גרפי'
]
NUMBER_OF_CATALOG_PAGES_TO_SCRAPE = 2
URL = Url(
"https://ksp.co.il/?select=.268..271..211..5870..2640..347..1798..4517..442..2639.&kg=&list=&sort=2&glist=0&uin=0&txt_search=&buy=&minprice=0&maxprice=0&intersect=&rintersect=&store_real="
)
def run_with_process_pool(
executor, function,
*args) -> Coroutine[concurrent.futures.Executor, Callable, List[Any]]:
"""Uses a concurrent.futures.ProcessPoolExecuter to run the parameter 'function'
with parameters 'args' with a process pool, in parralel.
returns an awaitable task representing the 'function' running in parralel.
Args:
executor (concurrent.futures.ProcessPoolExecuter): executes the process pool
function (typing.Callable): function to run in parralel
*args: parameters to pass to function
def __init__(self):
from cdebug import CDebug
self.log = CDebug('AniDUBadv.log', 'MAIN')
del CDebug
self.log('Initialization')
self.adv_log = False
self.addon_data_dir = fs_enc(
xbmc.translatePath(Main.__settings__.getAddonInfo('profile')))
self.images_dir = os.path.join(self.addon_data_dir, 'images')
self.torrents_dir = os.path.join(self.addon_data_dir, 'torrents')
self.library_dir = os.path.join(self.addon_data_dir, 'library')
self.icon = os.path.join(Main.__root__, 'icon.png')
self.fanart = os.path.join(Main.__root__, 'fanart.jpg')
self.source_quality = int(
Main.__settings__.getSetting("source_quality"))
self.show_rating = bool(
Main.__settings__.getSetting("show_rating").lower() == 'true')
self.show_history = bool(
Main.__settings__.getSetting("show_history").lower() == 'true')
self.show_search = bool(
Main.__settings__.getSetting("show_search").lower() == 'true')
self.show_peers = bool(
Main.__settings__.getSetting("show_peers").lower() == 'true')
self.progress = xbmcgui.DialogProgress()
if not os.path.exists(self.addon_data_dir):
self.log('Creating folder: ' + self.addon_data_dir)
os.makedirs(self.addon_data_dir)
if not os.path.exists(self.images_dir):
self.log('Creating folder: ' + self.images_dir)
os.mkdir(self.images_dir)
if not os.path.exists(self.torrents_dir):
self.log('Creating folder: ' + self.torrents_dir)
os.mkdir(self.torrents_dir)
if not os.path.exists(self.library_dir):
self.log('Creating folder: ' + self.library_dir)
os.mkdir(self.library_dir)
self.params = {
'mode': 'main',
'url': Main.site_url,
'param': '',
'page': 1
}
args = urlparse.parse_qs(sys.argv[2][1:])
for a in args:
self.params[a] = args[a][0]
from Url import Url
self.url = Url(
use_auth=True,
auth_state=bool(
self.__settings__.getSetting("auth").lower() == 'true'))
del Url
self.url.auth_url = Main.site_url
self.url.auth_post_data = {
'login_name': self.__settings__.getSetting('login'),
'login_password': self.__settings__.getSetting('password'),
'login': '******'
}
self.url.sid_file = fs_enc(
os.path.join(xbmc.translatePath('special://temp/'), 'anidub.sid'))
self.url.download_dir = self.addon_data_dir
self.url.cb_auth_ok = self._save_auth_setting
if not self.__settings__.getSetting(
"login") or not self.__settings__.getSetting("password"):
show_message('Авторизация', 'Укажите логин и пароль')
self.params['mode'] = 'check_settings'
return
if not self.url.auth_state:
if not self.url.auth_try():
self.params['mode'] = 'check_settings'
show_message('Ошибка', 'Проверьте логин и пароль')
self.log('Wrong authentication credentials (init)')
return
if Main.__settings__.getSetting("engine") != '':
Main.__settings__.setSetting("engine", '')
self.params['mode'] = 'p2psettings'
self.res_list = [
'bd1080', 'tv1080', 'bd720', 'tv720', 'dvd720', 'dvd480', 'hwp',
'psp', ''
]
from AnimeDB import AnimeDB
self.DB = AnimeDB(
fs_dec(os.path.join(self.addon_data_dir, 'anidata.db')))
del AnimeDB
class Message:
str_help = ""
u = Url()
def __init__(self):
self.str_help = """
/new NOME DUEDATE
/todo ID
/doing ID
/done ID
/delete ID
/list
/rename ID NOME
/dependson ID ID...
/duplicate ID
/priority ID PRIORITY{low, medium, high}
/help
"""
@staticmethod
def get_last_update_id(updates):
update_ids = []
for update in updates['result']:
update_ids.append(int(update["update_id"]))
return max(update_ids)
def deps_text(self, task, chat, preceed=''):
text = ''
for i in range(len(task.dependencies.split(',')[:-1])):
line = preceed
query = db.session.query(Task).filter_by(id=int(
task.dependencies.split(',')[:-1][i]),
chat=chat)
dep = query.one()
icon = '\U0001F195'
if dep.status == 'DOING':
icon = '\U000023FA'
elif dep.status == 'DONE':
icon = '\U00002611'
if i + 1 == len(task.dependencies.split(',')[:-1]):
line += '└── [[{}]] {} {}\n'.format(dep.id, icon, dep.name)
line += self.deps_text(dep, chat, preceed + ' ')
else:
line += '├── [[{}]] {} {}\n'.format(dep.id, icon, dep.name)
line += self.deps_text(dep, chat, preceed + '│ ')
text += line
return text
def handle_updates(self, updates):
def new_assigment(msg, chat):
duedate = msg.split(' ', 1)[1]
msg = msg.split(' ', 1)[0]
task = Task(chat=chat,
name=msg,
status='TODO',
dependencies='',
parents='',
priority='',
duedate=duedate)
db.session.add(task)
db.session.commit()
make_github_issue(task.name, '')
self.u.send_message(
"New task *TODO* [[{}]] {} - {}".format(
task.id, task.name, task.duedate), chat)
def rename_assigment(msg, chat):
text = ''
if msg != '':
if len(msg.split(' ', 1)) > 1:
text = msg.split(' ', 1)[1]
msg = msg.split(' ', 1)[0]
if not msg.isdigit():
self.u.send_message("You must inform the task id", chat)
else:
task_id = int(msg)
query = db.session.query(Task).filter_by(id=task_id, chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
self.u.send_message(
"_404_ Task {} not found x.x".format(task_id), chat)
return
if text == '':
self.u.send_message(
"You want to modify task {}, but you didn't provide any new text"
.format(task_id), chat)
return
old_text = task.name
task.name = text
db.session.commit()
self.u.send_message(
"Task {} redefined from {} to {}".format(
task_id, old_text, text), chat)
def duplicate_assigment(msg, chat):
if not msg.isdigit():
self.u.send_message("You must inform the task id", chat)
else:
task_id = int(msg)
query = db.session.query(Task).filter_by(id=task_id, chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
self.u.send_message(
"_404_ Task {} not found x.x".format(task_id), chat)
return
dtask = Task(chat=task.chat,
name=task.name,
status=task.status,
dependencies=task.dependencies,
parents=task.parents,
priority=task.priority,
duedate=task.duedate)
db.session.add(dtask)
for t in task.dependencies.split(',')[:-1]:
qy = db.session.query(Task).filter_by(id=int(t), chat=chat)
t = qy.one()
t.parents += '{},'.format(dtask.id)
db.session.commit()
self.u.send_message(
"New task *TODO* [[{}]] {}".format(dtask.id, dtask.name),
chat)
def delete_assigment(msg, chat):
if not msg.isdigit():
self.u.send_message("You must inform the task id", chat)
else:
task_id = int(msg)
query = db.session.query(Task).filter_by(id=task_id, chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
self.u.send_message(
"_404_ Task {} not found x.x".format(task_id), chat)
return
for t in task.dependencies.split(',')[:-1]:
qy = db.session.query(Task).filter_by(id=int(t), chat=chat)
t = qy.one()
t.parents = t.parents.replace('{},'.format(task.id), '')
db.session.delete(task)
db.session.commit()
self.u.send_message("Task [[{}]] deleted".format(task_id),
chat)
def todo_assigment(msg, chat):
id_list = msg.split(" ")
for id in id_list:
if not id.isdigit():
send_message("You must inform the task id", chat)
else:
task_id = int(id)
query = db.session.query(Task).filter_by(id=task_id,
chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
send_message(
"_404_ Task {} not found x.x".format(task_id),
chat)
return
task.status = 'TODO'
db.session.commit()
send_message(
"*TODO* task [[{}]] {}".format(task.id, task.name),
chat)
def doing_assigment(msg, chat):
id_list = msg.split(" ")
for id in id_list:
if not id.isdigit():
send_message("You must inform the task id", chat)
else:
task_id = int(id)
query = db.session.query(Task).filter_by(id=task_id,
chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
send_message(
"_404_ Task {} not found x.x".format(task_id),
chat)
return
task.status = 'DOING'
db.session.commit()
send_message(
"*DOING* task [[{}]] {}".format(task.id, task.name),
chat)
def done_assigment(msg, chat):
id_list = msg.split(" ")
for id in id_list:
if not id.isdigit():
send_message("You must inform the task id", chat)
else:
task_id = int(id)
query = db.session.query(Task).filter_by(id=task_id,
chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
send_message(
"_404_ Task {} not found x.x".format(task_id),
chat)
return
task.status = 'DONE'
db.session.commit()
send_message(
"*DONE* task [[{}]] {}".format(task.id, task.name),
chat)
def list_assigment(msg, chat):
a = ''
a += '\U0001F4CB Task List\n'
query = db.session.query(Task).filter_by(
parents='', chat=chat).order_by(Task.id)
for task in query.all():
icon = '\U0001F195'
if task.status == 'DOING':
icon = '\U000023FA'
elif task.status == 'DONE':
icon = '\U00002611'
a += '[[{}]] {} {} - {}\n'.format(task.id, icon, task.name,
task.duedate)
a += self.deps_text(task, chat)
self.u.send_message(a, chat)
a = ''
a += '\U0001F4DD _Status_\n'
query = db.session.query(Task).filter_by(
status='TODO', chat=chat).order_by(Task.id)
a += '\n\U0001F195 *TODO*\n'
for task in query.all():
print(task.name)
a += '[[{}]] {} - {}\n'.format(task.id, task.name,
task.duedate)
query = db.session.query(Task).filter_by(priority='high',
chat=chat).order_by(
Task.id)
a += '\U0001F6F0 *HIGH*\n'
for task in query.all():
a += '[[{}]] {} - {}\n'.format(task.id, task.name,
task.duedate)
query = db.session.query(Task).filter_by(priority='medium',
chat=chat).order_by(
Task.id)
a += '\U0001F6F0 *MEDIUM*\n'
for task in query.all():
a += '[[{}]] {} - {}\n'.format(task.id, task.name,
task.duedate)
query = db.session.query(Task).filter_by(
priority='low', chat=chat).order_by(Task.id)
a += '\U0001F6F0 *LOW*\n'
for task in query.all():
a += '[[{}]] {} - {}\n'.format(task.id, task.name,
task.duedate)
query = db.session.query(Task).filter_by(
status='DOING', chat=chat).order_by(Task.id)
a += '\n\U000023FA *DOING*\n'
for task in query.all():
a += '[[{}]] {} - {}\n'.format(task.id, task.name,
task.duedate)
query = db.session.query(Task).filter_by(
status='DONE', chat=chat).order_by(Task.id)
a += '\n\U00002611 *DONE*\n'
for task in query.all():
a += '[[{}]] {} - {}\n'.format(task.id, task.name,
task.duedate)
self.u.send_message(a, chat)
def existing_dependent_task(task, task_dependent):
query = db.session.query(Task).filter_by(id=task)
task_dependency = query.one()
dependencies_task = task_dependency.dependencies.split(",")
return str(task_dependent) in dependencies_task
def dependson_assigment(msg, chat):
text = ''
if msg != '':
if len(msg.split(' ', 1)) > 1:
text = msg.split(' ', 1)[1]
msg = msg.split(' ', 1)[0]
if not msg.isdigit():
return "You must inform the task id"
else:
task_id = int(msg)
query = db.session.query(Task).filter_by(id=task_id, chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
return "_404_ Task {} not found x.x".format(task_id)
if text == '':
for i in task.dependencies.split(',')[:-1]:
i = int(i)
q = db.session.query(Task).filter_by(id=i, chat=chat)
task_dependency = query.one()
task_dependency.parents = task_dependency.parents.replace(
'{},'.format(task.id), '')
task.dependencies = ''
return "Dependencies removed from task {}".format(
task_id, chat)
elif existing_dependent_task(text, task_id):
return "Task {} already have a dependency of task {}".format(
text, task_id, chat)
else:
for depid in text.split(' '):
if not depid.isdigit():
return "All dependencies ids must be numeric, and not {}".format(
depid)
else:
depid = int(depid)
query = db.session.query(Task).filter_by(id=depid,
chat=chat)
try:
task_dependency = query.one()
task_dependency.parents += str(task.id) + ','
except sqlalchemy.orm.exc.NoResultFound:
return "_404_ Task {} not found x.x".format(
depid)
continue
deplist = task.dependencies.split(',')
if str(depid) not in deplist:
task.dependencies += str(depid) + ','
if text == '':
task.priority = ''
self.u.send_message(
"_Cleared_ all priorities from task {}".format(
task_id), chat)
else:
if text.lower() not in ['high', 'medium', 'low']:
self.u.send_message(
"The priority *must be* one of the following: high, medium, low",
chat)
else:
task.priority = text.lower()
self.u.send_message(
"*Task {}* priority has priority *{}*".format(
task_id, text.lower()), chat)
db.session.commit()
return "Task {} dependencies up to date".format(task_id)
def priority_assigment(msg, chat):
text = ''
if msg != '':
if len(msg.split(' ', 1)) > 1:
text = msg.split(' ', 1)[1]
msg = msg.split(' ', 1)[0]
if not msg.isdigit():
self.u.send_message("You must inform the task id", chat)
else:
task_id = int(msg)
query = db.session.query(Task).filter_by(id=task_id, chat=chat)
try:
task = query.one()
except sqlalchemy.orm.exc.NoResultFound:
self.u.send_message(
"_404_ Task {} not found x.x".format(task_id), chat)
return
if text == '':
task.priority = ''
self.u.send_message(
"_Cleared_ all priorities from task {}".format(
task_id), chat)
else:
if text.lower() not in ['high', 'medium', 'low']:
self.u.send_message(
"The priority *must be* one of the following: high, medium, low",
chat)
else:
task.priority = text.lower()
self.u.send_message(
"*Task {}* priority has priority *{}*".format(
task_id, text.lower()), chat)
db.session.commit()
for update in updates["result"]:
if 'message' in update:
message = update['message']
elif 'edited_message' in update:
message = update['edited_message']
else:
print('Can\'t process! {}'.format(update))
return
try:
text = message["text"]
except:
message["text"] = ""
command = message["text"].split(" ", 1)[0]
msg = ''
if len(message["text"].split(" ", 1)) > 1:
msg = message["text"].split(" ", 1)[1].strip()
chat = message["chat"]["id"]
print(command, msg, chat)
if command == '/new':
new_assigment(msg, chat)
elif command == '/rename':
rename_assigment(msg, chat)
elif command == '/duplicate':
duplicate_assigment(msg, chat)
elif command == '/delete':
delete_assigment(msg, chat)
elif command == '/todo':
todo_assigment(msg, chat)
elif command == '/doing':
doing_assigment(msg, chat)
elif command == '/done':
done_assigment(msg, chat)
elif command == '/list':
list_assigment(msg, chat)
elif command == '/dependson':
dependson_assigment(msg, chat)
elif command == '/priority':
priority_assigment(msg, chat)
elif command == '/start':
self.u.send_message(
"Welcome! Here is a list of things you can do.", chat)
self.u.send_message(self.str_help, chat)
elif command == '/help':
self.u.send_message("Here is a list of things you can do.",
chat)
self.u.send_message(self.str_help, chat)
else:
self.u.send_message(
"I'm sorry dave. I'm afraid I can't do that.", chat)
def put(self):
return Item(
Request(self.auth, "PUT", self.header,
Url(self.urlBase, self.name + "/1")), "put " + self.name)
def post(self):
return Item(
Request(self.auth, "POST", self.header,
Url(self.urlBase, self.name + "/")), "post " + self.name)
def getById(self):
return Item(
Request(self.auth, "GET", self.header,
Url(self.urlBase, self.name + "/1")), "get " + self.name)
def search(self):
return Item(
Request(self.auth, "GET", self.header,
Url(self.urlBase, self.name + "/search")),
"search " + self.name)
def getAll(self):
return Item(
Request(self.auth, "GET", self.header,
Url(self.urlBase, self.name + "/")), "list " + self.name)
Side Effects:
Prints stats to stdout
"""
def DisplayStats(url_stats):
for stat in url_stats:
print "Source: {}".format(stat['source'])
print "Valid: {}".format(stat['valid'])
print "Canonical: {}".format(stat['canonicalized'])
print "Source unique: {}".format(stat['source_unique'])
print "Canonicalized URL unique: {}\n".format(
stat['canonicalized_unique'])
if __name__ == "__main__":
if len(sys.argv) != 2:
print "Usage: " + argv[0] + " filename\n"
sys.exit(1)
try:
file = open(sys.argv[1], "rb")
except IOError:
print "Could not open " + filename + " for reading\n"
sys.exit(1)
urls = [Url(line.strip()) for line in file]
stats = Url.GetStats(urls)
DisplayStats(stats)
exit(0)
def patch(self):
return Item(
Request(self.auth, "PATCH", self.header,
Url(self.urlBase, self.name + "/1")), "patch " + self.name)
def library():
type = request.vars.type or 'image'
sourceMediaType = request.vars.source_type or None
if request.post_vars.query == '':
query = ''
else:
query = request.post_vars.query or request.get_vars.query or ''
if query != session.priorQuery:
index = 0
else:
index = request.post_vars.index or request.get_vars.index or 0
session.priorQuery = query
count = request.post_vars.count or request.get_vars.count or 20
total = None
if index:
index = int(index)
if count:
count = int(count)
# Initialize output variables
subtitle = ''
total = ''
media = []
if type == 'image':
# Retrieve images
heading = 'Images'
from Images import Images
imageObj = Images(db)
# images = imageObj.search_images_by_owner(session.organization, query, index, count)
images = imageObj.search_images_by_owner(1, query, index, count)
total = images['count']
media = images['images']
elif type == 'video_embedded':
# Retrieve videos
heading = 'Videos'
from VideoEmbedded import VideoEmbedded
videoObj = VideoEmbedded(db)
# videos = videoObj.search_videos_by_owner(session.organization, query, index, count)
videos = videoObj.search_videos_by_owner(1, query, index, count)
total = videos['count']
media = videos['videos']
elif type == 'audio':
# Retrieve audio clips - not implemented yet
heading = 'Audio'
media = []
elif type == 'url':
# Retrieve all URLs
heading = 'URLs'
from Url import Url
urlObj = Url(db)
# urls = urlObj.search_urls_by_owner(session.organization, query, index, count)
urls = urlObj.search_urls_by_owner(1, query, index, count)
total = urls['count']
media = urls['urls']
if count > total:
count = total
return api_response(title='Media Library',
heading=heading,
type=type,
media=media,
query=query,
index=index,
count=count,
total=total)
l = Locations()
provinces = l.get_provinces()
province_ids = l.get_province_ids()
# generate a string of provinces to prompt user with
provinces_string = "\n"
for province_no in provinces:
provinces_string += str(province_no) + ": " + (
provinces[province_no] + "\n")
# second arg is the province number
province_num = input("Enter a province number from:" +
provinces_string)
# generate the kijiji url
websiteObject = Url()
websiteObject.set_product_name(product)
websiteObject.set_province_num(province_num)
websiteObject.set_provinces(provinces)
websiteObject.set_province_ids(province_ids)
website = websiteObject.generate()
# webscrape the product from the website
cheapest_product = cheapest(website, product)
# get the info
product_name = cheapest_product.get_name()
product_price = cheapest_product.get_price()
product_url = cheapest_product.get_url()
def delete(self):
return Item(
Request(self.auth, "DELETE", self.header,
Url(self.urlBase, self.name + "/1")),
"delete " + self.name)
