def get_delimiter(dialect):
"""Get the string delimiter for the dataset file(s)."""
val = clean_input("delimiter (press return to skip): ", ignore_empty=True)
if val == "" or val == []:
# return and dont add key to dialect dict if empty val
return
dialect['delimiter'] = val
def get(self, tag_name):
tag_name = utils.clean_input(tag_name)
entries = models.get_entries(equal_conditions=[('tags', tag_name)])
ctx = {}
ctx['entries'] = entries
self.render_template('posts.html', ctx)
def get_header_rows(dialect):
"""Get number of rows considered as the header."""
val = clean_input("header_rows (int) (press return to skip): ",
ignore_empty=True, dtype=int)
if val == "" or val == []:
# return and dont add key to dialect dict if empty val
return
dialect['header_rows'] = val
def auth_check(username, session):
data = db.select('auth_user', where="username='******'" % clean_input(username))
try:
data = data[0]
except IndexError:
return False
else:
return str(data.session) == str(session)
def check_login(data):
try:
user_data = db.select('auth_user', where="username='******'" % clean_input(data.username))[0]
if password_to_md5(data.password) == user_data.password:
return True
else:
return False
except IndexError:
return False
def get_fixed_width(dialect):
"""Set fixed_width property."""
val = clean_input(
"fixed_width (bool = True/False) (press return to skip): ",
ignore_empty=True,
dtype=bool
)
if val == "" or val == []:
# return and dont add key to dialect dict if empty val
return
dialect['fixed_width'] = val
def get_do_not_bulk_insert(dialect):
"""Set do_not_bulk_insert property."""
val = clean_input(
"do_not_bulk_insert (bool = True/False) (press return to skip): ",
ignore_empty=True,
dtype=bool
)
if val == "" or val == []:
# return and dont add key to dialect dict if empty val
return
dialect['do_not_bulk_insert'] = val
def get_contains_pk(dialect):
"""Set contains_pk property."""
val = clean_input(
"contains_pk (bool = True/False) (press return to skip): ",
ignore_empty=True,
dtype=bool
)
if val == "" or val == []:
# return and dont add key to dialect dict if empty val
return
dialect['contains_pk'] = val
def post(self):
errors = []
username = self.request.get('username'); logging.error(username)
email = self.request.get('email'); logging.error(email)
password = self.request.get('password'); logging.error(password)
confirm_password = self.request.get('confirm_password'); logging.error(confirm_password)
if (username and email and password and (password == confirm_password)):
username = utils.clean_input(username)
email = utils.clean_input(email)
user = models.create_user(username, email, password)
if user:
self.set_user(user)
self.redirect('/blog/posts')
else:
errors.append('An error occured, please try again.')
if not username: errors.append('Username is required.')
if not password:
errors.append('Password is required.')
elif not password == confirm_password:
errors.append('Passwords do not match.')
self.render_template('register.html', {'errors': errors})
def check_login(username, password):
user_data = db.select(USERS, where='username="******"' % clean_input(username))
try:
user_data = user_data[0]
encode = user_data.password
user_id = user_data.id
except IndexError:
return False, user_pass_not_match
if not check_password(encode=encode, raw_password=password):
return False, user_already_exist
else:
return True, user_id
def post(self):
user = self.get_user()
if not user:
self.redirect('/blog/login?redirect=%s' % self.request.url)
errors = []
title = self.request.get('title')
body = self.request.get('body')
tags = self.request.get('entry_tags')
slug = self.request.get('slug')
entry_id = self.request.get('entry_id')
entry_type = self.request.get('entry_type')
if (title and body and tags):
title = utils.clean_input(title)
# line below commented out to prevent escaping of raw html within markdown
# body = utils.clean_input(body)
slug = utils.clean_input(slug)
if entry_id:
try: entry_id = int(utils.clean_input(entry_id))
except TypeError: entry_id = 0
if tags:
tags = tags.split(",")
tags = [utils.clean_input(tag.strip()) for tag in tags]
entry_type = (entry_type in ['article', 'note', 'log'] and entry_type or '')
entry = models.save_entry(title=title,
body=body,
slug=slug,
author=user.user_id,
tags=tags,
entry_id=(entry_id or 0),
entry_type=entry_type)
self.redirect('/blog/entry/view/%d' % entry.entry_id)
if not title: errors.append('Title is required')
if not body: errors.append('Body is required.')
if not tags: errors.append('At least one tag is required')
self.render_template('post_entry.html', {'errors': errors,
'title': title,
'body': body,
'entry_tags': tags,
'slug': slug})
def get_nulls(dialect):
"""Get list of strings that denote missing value in the dataset."""
val = clean_input(
"missing values (separated by ';') (press return to skip): ",
split_char=';',
ignore_empty=True
)
if val == "" or val == []:
# return and dont add key to dialect dict if empty val
return
dialect['missingValues'] = val
# change list to single value if size == 1
if len(dialect['missingValues']) == 1:
dialect['missingValues'] = dialect['missingValues'][0]
def post(self):
errors = []
username = self.request.get('username');logging.error(username)
password = self.request.get('password');logging.error(password)
redirect = self.request.get('redirect');logging.error(redirect)
if (username and password):
username = utils.clean_input(username)
user = models.get_user(username, password)
if user:
self.set_user(user)
self.redirect((str(redirect) or '/blog/posts'))
errors.append('Login Error: incorrect username or password.')
if not username: errors.append('Username is required.')
if not password: errors.append('Password is required.')
self.render_template('login.html', {'errors': errors, 'redirect': redirect})
def get_replace_columns(dialect):
"""Get the replace values for columns from the user."""
val = clean_input("replace_columns (separated by "
"';', with comma-separated values) "
"(press return to skip): ",
split_char=';', ignore_empty=True)
if val == "" or val == []:
# return and don't add key to dialect dict if empty val
return
dialect['replace_columns'] = []
for v in val:
try:
pair = v.split(',')
dialect['replace_columns'].append((pair[0].strip(), pair[1].strip()))
except IndexError:
continue
def number_to_words(self, number):
# Clean input (remove any non alphanumeric characters and make uppercase)
number = utils.clean_input(number)
# Check input
if not utils.is_valid_input(number, regex="[\W\D]"):
return None, None
# returns a string word
word, digits = self.find_word(number)
if word:
number = utils.rreplace(number, digits, word, 1)
return word, number
else:
return None, None
def all_wordifications(self, number):
# Clean input (remove any non alphanumeric characters and make uppercase)
number = utils.clean_input(number)
# Check input
if not utils.is_valid_input(number, regex="[\W\D]"):
return None
# returns a dict of {"#####":["WORD1","WORD2"]...}
solutions = self.find_words(number)
if solutions:
numbers = self.create_all_wordifications(number, solutions)
return numbers
else:
# print("No words found")
return None
def do_filecount(self, args):
"""Process a document, count pages
Syntax:
filecount <filename>
"""
#Clean input
filepath = utils.clean_input(args)
#check if file exists
if not os.path.exists(filepath):
print 'Could not find a file at <{}>'.format(filepath)
return
#Make sure this is a file, not a direcotyr
if os.path.isdir(filepath):
print 'This path point to a directory, not a file.',
print 'Use the "dircount" command instead.'
return
#check file extension is PDF
if not filepath.endswith('.pdf'):
print 'Invalid extension'
return
#Get counts
s, count = utils.ProcessPDF(filepath, self.largeformatsize)
#Print out counts
lout = self._log_line
if s:
lout('\nFile Stats for <{}>\n'.format(filepath))
for k,v in count.iteritems():
lout('{}: \t{}\n'.format(k,v))
else:
#function returned error, handle here
error = count
lout(error + '\n')
def do_dircount(self, args):
"""Process a directory and all the files inside
Syntax:
dircount <x:\\path\\to\\dir\\>
"""
#Clean input
dirpath = utils.clean_input(args)
#Check that directory exists
if not os.path.exists(dirpath):
print 'Could not find specific path at <{}>'.format(dirpath)
return
#Make sure this is a file, not a direcotyr
if os.path.isfile(dirpath):
print 'This path point to a file, not a directory.'
print 'Use the "filecount" command instead.'
return
#Get counts
errors, counts = utils.ParseDir(dirpath, self.largeformatsize)
#Print output
self._log_output (dirpath, counts, errors)
def save_session(username):
session = make_session()
db.update('auth_user', where="username='******'" % clean_input(username), session=session)
return session
def edit_dict(obj, tabwidth=0):
"""
Recursive helper function for edit_json() to edit a datapackage.JSON script file.
"""
for key, val in obj.copy().items():
print('\n' + " " * tabwidth + "->" + key + " (", type(val), ") :\n")
if isinstance(val, list):
for v in val:
print(" " * tabwidth + str(v) + '\n\n')
elif isinstance(val, dict):
for item in val.items():
print(" " * tabwidth + str(item) + '\n\n')
else:
print(" " * tabwidth + str(val) + '\n\n')
while True:
try:
if isinstance(val, dict):
if val != {}:
print(" '" + key + "' has the following keys:\n" +
str(obj[key].keys()) + "\n")
do_edit = clean_input(
"Edit the values for these sub-keys of " + key +
"? (y/N): ")
if do_edit.lower() in ['y', 'yes']:
edit_dict(obj[key], tabwidth + 1)
print("Select one of the following for the key '" + key +
"': \n")
print("1. Add an item")
print("2. Modify an item")
print("3. Delete an item")
print("4. Remove from script")
print("5. Continue (no changes)\n")
selection = clean_input("\nYour choice: ")
if selection == '1':
add_key = clean_input('Enter new key: ')
add_val = clean_input('Enter new value: ')
obj[key][add_key] = add_val
elif selection == '2':
mod_key = clean_input('Enter the key: ')
if mod_key not in val:
print("Invalid input! Key not found.")
continue
mod_val = clean_input('Enter new value: ')
obj[key][mod_key] = mod_val
elif selection == '3':
del_key = clean_input('Enter key to be deleted: ')
if del_key not in val:
print("Invalid key: Not found")
continue
print("Removed " + str(del_key) + " : " +
str(obj[key].pop(del_key)))
elif selection == '4':
do_remove = clean_input(
"Are you sure "
"(completely remove this entry)? (y/n): ")
if do_remove.lower() in ['y', 'yes']:
obj.pop(key)
print("Removed " + key + " from script.\n")
else:
print("Aborted.")
sleep(1)
elif selection == '5' or selection == "":
pass
else:
raise RuntimeError("Invalid input!")
elif isinstance(val, list):
for i in range(len(val)):
print(i + 1, '. ', str(val[i]))
if isinstance(val[i], dict):
do_edit = clean_input("\nEdit this dict in '" +
key + "'? (y/N): ")
if do_edit.lower() in ['y', 'yes']:
edit_dict(obj[key][i], tabwidth + 2)
print("Select one of the following for the key '" + key +
"': \n")
print("1. Add an item")
print("2. Delete an item")
print("3. Remove from script")
print("4. Continue (no changes)\n")
selection = clean_input("\nYour choice: ")
if selection == '1':
add_val = clean_input('Enter new value: ')
obj[key].append(add_val)
elif selection == '2':
del_val = clean_input('Enter value to be deleted: ')
if del_val not in obj[key]:
print("Invalid value: Not found.")
continue
obj[key].remove(del_val)
print("Removed " + str(del_val))
elif selection == '3':
do_remove = clean_input(
"Are you sure "
"(completely remove this entry)? (y/n): ")
if do_remove.lower() in ['y', 'yes']:
obj.pop(key)
print("Removed " + key + " from script.\n")
else:
print("Aborted.")
sleep(1)
elif selection == '4' or selection == "":
pass
else:
raise RuntimeError("Invalid input!")
else:
print("Select one of the following for the key '" + key +
"': \n")
print("1. Modify value")
print("2. Remove from script")
print("3. Continue (no changes)\n")
selection = clean_input("\nYour choice: ")
if selection == '1':
new_val = clean_input('Enter new value: ')
obj[key] = new_val
elif selection == '2':
do_remove = clean_input(
"Are you sure (completely remove this entry)? (y/n): "
)
if do_remove.lower() in ['y', 'yes']:
obj.pop(key)
print("Removed " + key + " from script.\n")
else:
print("Aborted.")
sleep(1)
elif selection == '3' or selection == "":
pass
else:
raise RuntimeError("Invalid input!")
break
except RuntimeError:
continue
def auth_check(username, session):
data = db.select('auth_user', where="username='******'" % clean_input(username))[0]
return str(data.session) == str(session)
def create_json():
"""
Creates datapackage.JSON script.
http://specs.frictionlessdata.io/data-packages/#descriptor-datapackagejson
Takes input from user via command line.
"""
contents = {}
tableurls = {}
invalid_name = True
script_exists = True
while script_exists or invalid_name:
contents['name'] = clean_input("name (a short unique identifier;"
" only lowercase letters and - allowed): ")
invalid_name = re.compile(r'[^a-z-]').search(contents['name'])
if invalid_name:
print("name can only contain lowercase letters and -")
continue
script_exists = contents['name'].lower() in short_names()
if script_exists:
print("Dataset already available. Check the list or try a different name")
contents['title'] = clean_input("title: ", ignore_empty=True)
contents['description'] = clean_input("description: ", ignore_empty=True)
contents['citation'] = clean_input("citations (separated by ';'): ",
split_char=';', ignore_empty=True)
contents['homepage'] = clean_input("homepage (for the entire dataset): ", ignore_empty=True)
contents['keywords'] = clean_input("keywords (separated by ';'): ",
split_char=';', ignore_empty=True)
contents['resources'] = []
contents['retriever'] = "True"
contents['retriever_minimum_version'] = "2.0.dev"
contents['encoding'] = clean_input("encoding: ", ignore_empty=True)
if is_empty(clean_input("encoding: ", ignore_empty=True)):
contents['encoding'] = ENCODING
contents['version'] = "1.0.0"
# Add tables
while True:
addtable = clean_input("\nAdd Table? (y/N): ")
if addtable.lower() in ["n", "no"]:
break
elif addtable.lower() not in ["y", "yes"]:
print("Not a valid option\n")
continue
else:
table = dict()
table['name'] = clean_input("table-name: ")
table['url'] = clean_input("table-url: ")
table['dialect'] = {}
tableurls[table['name']] = table['url']
# get table properties (dialect)
# refer retriever.lib.table.Table
get_replace_columns(table['dialect'])
get_nulls(table['dialect'])
get_delimiter(table['dialect'])
get_do_not_bulk_insert(table['dialect'])
get_contains_pk(table['dialect'])
get_fixed_width(table['dialect'])
get_header_rows(table['dialect'])
# set table schema
table['schema'] = {}
table['schema']["fields"] = []
print("Enter columns [format = name, type, (optional) size] (press return to skip):\n\n")
while True:
# get column list (optional)
try:
col_list = clean_input("", split_char=',', ignore_empty=True)
if not col_list:
break
if not isinstance(col_list, list):
raise Exception
col_list = [c.strip() for c in col_list]
col_obj = dict() # dict to store column data
col_obj["name"] = col_list[0]
col_obj["type"] = col_list[1]
if len(col_list) > 2:
if type(eval(col_list[2])) != int:
raise Exception
col_obj["size"] = col_list[2]
table["schema"]["fields"].append(col_obj)
except:
print("Exception occured. Check the input format again.\n")
pass
isCT = clean_input(
"Add crosstab columns? (y,N): ", ignore_empty=True)
if isCT.lower() in ["y", "yes"]:
ct_column = clean_input("Crosstab column name: ")
ct_names = []
print("Enter names of crosstab column values (Press return after each name):\n")
name = clean_input()
while name != "":
ct_names.append(name)
name = clean_input()
table['schema']['ct_column'] = ct_column
table['schema']['ct_names'] = ct_names
contents['resources'].append(table)
give_message = clean_input(
"Would you like to add a Message? (y,N): ", ignore_empty=True)
if give_message.lower() in ["y", "yes"]:
contents['message'] = clean_input("Provide your Message: ", ignore_empty=True)
contents['urls'] = tableurls
file_name = contents['name'] + ".json"
file_name = file_name.replace('-', '_')
with open(os.path.join('scripts', file_name), 'w') as output_file:
json.dump(contents,
output_file,
sort_keys=True,
indent=4,
separators=(',', ': '))
output_file.write('\n')
print("\nScript written to " + file_name)
output_file.close()
