def list_reports(page, filter_id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'for_view': True}
data_dict = {}
if db.related_extra_table is None:
db.init_db(context['model'])
res = db.RelatedExtra.getALL(**data_dict)
res = [x for x in res if x.key == 'report']
if filter_id != '':
res = [x for x in res if x.related_id == filter_id]
length = len(res)
result = []
try:
page = int(page)
except ValueError:
page = 1
for i in range(page*10-10,page*10):
try:
result.append(res[i])
except IndexError:
pass
if page > length//10+1:
base.abort(400, ('"page" parameter out of range'))
#res = res[page-1*10:page*10+4]
return {'reports': result, 'count': length, 'delall': (length > 0) and (filter_id != ''), 'related_id':filter_id}
def search_add(context, data_dict):
'''
Add an item to the search_history for the current user.
:param content: Search query to add to history
:type content: string
'''
try:
tk.check_access('ckanext_search_history_add', context, data_dict)
except tk.NotAuthorized:
#JOE#
#tk.abort(401, tk._('Not authorized to add history item'))
pass
if db.search_history_table is None:
db.init_db(context['model'])
content = tk.get_or_bust(data_dict, 'content')
username = context.get('user')
#JOE#
#user_id = new_authz.get_user_id_for_username(username, allow_none=False)
user_id = new_authz.get_user_id_for_username(username, allow_none=True)
search_history = db.SearchHistory()
search_history.content = content
search_history.user_id = user_id
session = context['session']
session.add(search_history)
session.commit()
return db.table_dictize(search_history, context)
def _pages_list(context, data_dict):
search = {}
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
ordered = data_dict.get('order')
private = data_dict.get('private', True)
if ordered:
search['order'] = True
if not org_id:
search['group_id'] = None
try:
p.toolkit.check_access('ckanext_pages_update', context, data_dict)
if not private:
search['private'] = False
except p.toolkit.NotAuthorized:
search['private'] = False
else:
group = context['model'].Group.get(org_id)
user = context['user']
member = new_authz.has_user_permission_for_group_or_org(
group.id, user, 'read')
search['group_id'] = org_id
if not member:
search['private'] = False
out = db.Page.pages(**search)
return [{'title': pg.title,
'content': pg.content,
'name': pg.name,
'group_id': pg.group_id,
} for pg in out]
def main():
logger.info('Starting follow neighbour script')
try:
db.init_db()
lat, lon = config.TWEET_HOME_GEO
radius = '100km'
geocode = ','.join([str(lat), str(lon), radius])
registered_users = [
user.user_id for user in db.session.query(User).all()]
results = getTweets([config.TWEET_LOCATION.split(',')[0]],
result_type='recent')
near_tweets = []
for tweet in results:
if tweet.get('place'):
if tweet.get('place').get('full_name') == 'San Francisco, CA':
near_tweets.append(tweet)
near_tweets[0]
authors = [tweet.get('user') for tweet in near_tweets]
authors_id_list = set([author.get('id_str') for author in authors])
follow_list = [author for author in authors_id_list
if author not in registered_users]
logger.info('Found %d new neighbours' % len(follow_list))
followUsers(follow_list[:30])
except Exception, err:
logger.error('%s: %s' % (Exception, err))
def create_application():
global app, db, UserManager
if not config.check():
return None
if not os.path.exists(config.get('base', 'cache_dir')):
os.makedirs(config.get('base', 'cache_dir'))
import db
db.init_db()
app = Flask(__name__)
app.secret_key = '?9huDM\\H'
if config.get('base', 'log_file'):
import logging
from logging.handlers import TimedRotatingFileHandler
handler = TimedRotatingFileHandler(config.get('base', 'log_file'), when = 'midnight')
handler.setLevel(logging.WARNING)
app.logger.addHandler(handler)
app.teardown_request(teardown)
import frontend
import api
return app
def main():
logger.info("Starting interventions retweeting (pid %s)" % os.getpid())
twitter_stream = TwitterStream(domain="stream.twitter.com", app=1)
db.init_db()
while 1:
for tag_date, tag in TAGS:
retweets = 0
while tag_date == datetime.datetime.now().date():
try:
for status in twitter_stream.stream.statuses.filter(track=tag):
sleep(5)
tweet_id = status.get("id_str")
retweet_ids = [tweet.tweet_id for tweet in db.session.query(ReTweet).all()]
if tweet_id in retweet_ids:
continue
screen_name = status.get("user").get("screen_name")
# Retweet 10 first
if retweets < 5:
reTweet(tweet_id)
logger.info("Retweeted: %s" % tweet_id)
retweets += 1
# Retweet 50 if not BOT
elif retweets <= 16 and screen_name not in BOTS:
reTweet(tweet_id)
logger.info("Retweeted: %s" % tweet_id)
retweets += 1
# Favorite tweet
favorite(tweet_id)
if tag_date != datetime.datetime.now().date():
break
except:
print "Stream connection timeout, restarting..."
def page_privacy(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
out = db.Page.get(group_id=org_id, name=page)
if out and out.private == False:
return {'success': True}
# CCG change: permit anonymous listing of available pages
if page == '' and out is None:
return {'success': True}
# no org_id means it's a universal page
if not org_id:
if out and out.private:
return {'success': False}
return {'success': True}
group = context['model'].Group.get(org_id)
user = context['user']
authorized = authz.has_user_permission_for_group_or_org(group.id,
user,
'read')
if not authorized:
return {'success': False,
'msg': p.toolkit._(
'User %s not authorized to read this page') % user}
else:
return {'success': True}
def __init__(self):
self.verify_https = os.environ.get('OAUTHLIB_INSECURE_TRANSPORT', '') == ""
if self.verify_https and os.environ.get("REQUESTS_CA_BUNDLE", "").strip() != "":
self.verify_https = os.environ["REQUESTS_CA_BUNDLE"].strip()
self.jwt_enable = six.text_type(os.environ.get('CKAN_OAUTH2_JWT_ENABLE', toolkit.config.get('ckan.oauth2.jwt.enable',''))).strip().lower() in ("true", "1", "on")
self.legacy_idm = six.text_type(os.environ.get('CKAN_OAUTH2_LEGACY_IDM', toolkit.config.get('ckan.oauth2.legacy_idm', ''))).strip().lower() in ("true", "1", "on")
self.authorization_endpoint = six.text_type(os.environ.get('CKAN_OAUTH2_AUTHORIZATION_ENDPOINT', toolkit.config.get('ckan.oauth2.authorization_endpoint', ''))).strip()
self.token_endpoint = six.text_type(os.environ.get('CKAN_OAUTH2_TOKEN_ENDPOINT', toolkit.config.get('ckan.oauth2.token_endpoint', ''))).strip()
self.profile_api_url = six.text_type(os.environ.get('CKAN_OAUTH2_PROFILE_API_URL', toolkit.config.get('ckan.oauth2.profile_api_url', ''))).strip()
self.client_id = six.text_type(os.environ.get('CKAN_OAUTH2_CLIENT_ID', toolkit.config.get('ckan.oauth2.client_id', ''))).strip()
self.client_secret = six.text_type(os.environ.get('CKAN_OAUTH2_CLIENT_SECRET', toolkit.config.get('ckan.oauth2.client_secret', ''))).strip()
self.scope = six.text_type(os.environ.get('CKAN_OAUTH2_SCOPE', toolkit.config.get('ckan.oauth2.scope', ''))).strip()
self.rememberer_name = six.text_type(os.environ.get('CKAN_OAUTH2_REMEMBER_NAME', toolkit.config.get('ckan.oauth2.rememberer_name', 'auth_tkt'))).strip()
self.profile_api_user_field = six.text_type(os.environ.get('CKAN_OAUTH2_PROFILE_API_USER_FIELD', toolkit.config.get('ckan.oauth2.profile_api_user_field', ''))).strip()
self.profile_api_fullname_field = six.text_type(os.environ.get('CKAN_OAUTH2_PROFILE_API_FULLNAME_FIELD', toolkit.config.get('ckan.oauth2.profile_api_fullname_field', ''))).strip()
self.profile_api_mail_field = six.text_type(os.environ.get('CKAN_OAUTH2_PROFILE_API_MAIL_FIELD', toolkit.config.get('ckan.oauth2.profile_api_mail_field', ''))).strip()
self.profile_api_groupmembership_field = six.text_type(os.environ.get('CKAN_OAUTH2_PROFILE_API_GROUPMEMBERSHIP_FIELD', toolkit.config.get('ckan.oauth2.profile_api_groupmembership_field', ''))).strip()
self.sysadmin_group_name = six.text_type(os.environ.get('CKAN_OAUTH2_SYSADMIN_GROUP_NAME', toolkit.config.get('ckan.oauth2.sysadmin_group_name', ''))).strip()
self.redirect_uri = urljoin(urljoin(toolkit.config.get('ckan.site_url', 'http://localhost:5000'), toolkit.config.get('ckan.root_path')), constants.REDIRECT_URL)
# Init db
db.init_db(model)
missing = [key for key in REQUIRED_CONF if getattr(self, key, "") == ""]
if missing:
raise ValueError("Missing required oauth2 conf: %s" % ", ".join(missing))
elif self.scope == "":
self.scope = None
def setup():
"""Set up the file system, GPG, and database."""
create_directories()
init_gpg()
init_db()
# Do tests that should always run on app startup
crypto_util.do_runtime_tests()
def datarequest_comment_show(context, data_dict):
'''
Action to retrieve a comment. Access rights will be checked before getting the
comment and a NotAuthorized exception will be risen if the user is not allowed
to get the comment
:param id: The ID of the comment to be retrieved
:type id: string
:returns: A dict with the following fields: id, user_id, datarequest_id, time
and comment
:rtype: dict
'''
model = context['model']
comment_id = data_dict.get('id', '')
# Check id
if not comment_id:
raise tk.ValidationError(['Comment ID has not been included'])
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.DATAREQUEST_COMMENT_SHOW, context, data_dict)
# Get comments
result = db.Comment.get(id=comment_id)
if not result:
raise tk.ObjectNotFound('Comment %s not found in the data base' % comment_id)
return _dictize_comment(result[0])
def datarequest_comment_list(context, data_dict):
'''
Action to retrieve all the comments of a data request. Access rights will be checked before
getting the comments and a NotAuthorized exception will be risen if the user is not
allowed to read the comments
:param datarequest_id: The ID of the datarequest whose comments want to be retrieved
:type id: string
:returns: A list with all the comments of a data request. Every comment is a dict with the
following fields: id, user_id, datarequest_id, time and comment
:rtype: list
'''
model = context['model']
datarequest_id = data_dict.get('datarequest_id', '')
# Check id
if not datarequest_id:
raise tk.ValidationError('Data Request ID has not been included')
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.DATAREQUEST_COMMENT_LIST, context, data_dict)
# Get comments
comments_db = db.Comment.get_ordered_by_date(datarequest_id=datarequest_id)
comments_list = []
for comment in comments_db:
comments_list.append(_dictize_comment(comment))
return comments_list
def reset(args):
"""Clears the SecureDrop development applications' state, restoring them to
the way they were immediately after running `setup_dev.sh`. This command:
1. Erases the development sqlite database file.
2. Regenerates the database.
3. Erases stored submissions and replies from the store dir.
"""
# Erase the development db file
if not hasattr(config, 'DATABASE_FILE'):
raise Exception("TODO: ./manage.py doesn't know how to clear the db "
'if the backend is not sqlite')
try:
os.remove(config.DATABASE_FILE)
except OSError:
pass
# Regenerate the database
init_db()
# Clear submission/reply storage
try:
os.stat(config.STORE_DIR)
except OSError:
pass
else:
for source_dir in os.listdir(config.STORE_DIR):
try:
# Each entry in STORE_DIR is a directory corresponding
# to a source
shutil.rmtree(os.path.join(config.STORE_DIR, source_dir))
except OSError:
pass
return 0
def _pages_update(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
# we need the page in the context for name validation
context['page'] = page
context['group_id'] = org_id
data, errors = df.validate(data_dict, schema, context)
if errors:
raise p.toolkit.ValidationError(errors)
out = db.Page.get(group_id=org_id, name=page)
if not out:
out = db.Page()
out.group_id = org_id
out.name = page
items = ['title', 'content', 'name', 'private', 'order']
for item in items:
setattr(out, item, data.get(item))
out.modified = datetime.datetime.utcnow()
out.user_id = p.toolkit.c.userobj.id
out.save()
session = context['session']
session.add(out)
session.commit()
def update_db(self):
tracks = explore.explore_collection(MUSIC_PATH)
if not os.path.isfile(DBFILE) or os.path.getsize(DBFILE == 0):
db.init_db(DBFILE)
db.create_tracks(DBFILE, tracks)
else:
db.update_tracks(DBFILE, tracks)
def is_dataset_acquired(pkg_dict):
db.init_db(model)
if tk.c.user:
return len(db.AllowedUser.get(package_id=pkg_dict["id"], user_name=tk.c.user)) > 0
else:
return False
def main(sqlite_path):
if path.exists(db.db_path):
print(db.db_path, 'already exists')
return 1
db.init_db(True)
csv.register_dialect('wormholes',
lineterminator='\n', quoting=csv.QUOTE_MINIMAL, skipinitialspace=True, strict=True)
wormholes = get_wormholes('setup_data/wormholes.csv')
conn = sqlite3.connect(sqlite_path)
conn.row_factory = sqlite3.Row
rows = conn.execute('''
SELECT solarSystemID, solarSystemName, security, regionName, wormholeClassId FROM mapSolarSystems
JOIN mapRegions ON mapRegions.regionID = mapSolarSystems.regionID
JOIN mapLocationWormholeClasses ON mapRegions.regionId = mapLocationWormholeClasses.locationId
''')
for row in rows:
id = int(row['solarSystemID'])
name = row['solarSystemName']
security = row['security']
region = row['regionName']
wormhole = wormholes.get(id)
if wormhole:
assert name == wormhole['name']
whclass = wormhole['class']
effect = wormhole['effect']
static1 = wormhole['static1']
static2 = wormhole['static2']
else:
if security >= 0.45:
whclass = 'highsec'
elif security > 0.0:
whclass = 'lowsec'
else:
whclass = 'nullsec'
effect = static1 = static2 = None
db.SolarSystem(name, id, region, whclass, effect, static1, static2).save()
for wh_type in get_wh_types('setup_data/wh_types.csv'):
"""
id = wh_type['id']
name = wh_type['name']
src = wh_type['src']
dest = wh_type['dest']
lifetime = wh_type['lifetime']
jump_mass = wh_type['jump_mass']
max_mass = wh_type['max_mass']
"""
db.WHType(**wh_type).save()
username = input('username: '******'password: '******'map', b'[]')
return 0
def _pages_show(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
out = db.Page.get(group_id=org_id, name=page)
if out:
out = db.table_dictize(out, context)
return out
def reports_num(related_id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'for_view': True}
data_dict = {'related_id': related_id, 'key':'report'}
if db.related_extra_table is None:
db.init_db(context['model'])
res = db.RelatedExtra.get(**data_dict)
return len(res)
def get_tag_info(context, data_dict):
'''
This function retrieves extra information about given tag_id and
possibly more filtering criterias.
'''
if db.tag_info_table is None:
db.init_db(context['model'])
res = db.TagInfo.get(**data_dict)
return res
def get_allowed_users(key, data, errors, context):
pkg_id = data[('id',)]
db.init_db(context['model'])
users = db.AllowedUser.get(package_id=pkg_id)
for i, user in enumerate(users):
data[(key[0], i)] = user.user_name
def main():
'''Script is used to post an image from the renvotion of the bot's
fictional restaurant.'''
try:
sleep_time = randrange(120)
logger.info(
'Starting post renovation progress script by sleeping %s minutes'
% sleep_time
)
sleep(sleep_time*60)
logger.info('Waking up post_renovation script.')
tweets = twitter_api.statuses.user_timeline(screen_name=USER,
count=200
)
filtered_tweets = []
for tweet in tweets:
text = tweet.get('text')
if ('#renovation' in text
or '#comingsoon' in text
or '#refurbishment' in text):
if tweet.get('entities').get('media'):
for media in tweet.get('entities').get('media'):
filtered_tweets.append(tweet)
# TODO: Check with previous tweets...
db.init_db()
filtered_statuses = db.session.query(Status).all()
# source ids of previously posted statuses
filtered_statuses = [status.source_id for status in filtered_statuses
if status.source_id]
# ids of found tweets
filtered_tweet_ids = [tweet.get('id_str') for tweet in filtered_tweets]
# Removing previously posted source tweets
possible_statuses = [tweet_id for tweet_id in filtered_tweet_ids
if tweet_id not in filtered_statuses]
# Setting the new tweet id, to the newest possible one
possible_statuses = sorted(possible_statuses)
if possible_statuses:
new_source_id = possible_statuses[0]
# Getting the source tweet
new_source = [tweet for tweet in filtered_tweets
if tweet.get('id_str') == new_source_id][0]
# TODO: Find next image to post
medias = new_source.get('entities').get('media')
media_url = None
for media in medias:
media_url = media.get('media_url')
break
image = urlopen(media_url)
params = {"media[]": image.read(),
"status": generate_text(),
'source_id': new_source_id
}
# Post tweet
updateStatus(params)
except Exception, err:
logger.error('%s: %s' % (Exception, err))
def command_line():
if not config.check():
sys.exit(1)
db.init_db()
if len(sys.argv) > 1:
CLI().onecmd(' '.join(sys.argv[1:]))
else:
CLI().cmdloop()
def reported_id(related_id, user_id):
context = {'model': model, 'session': model.Session,
'user': c.user or c.author, 'auth_user_obj': c.userobj,
'for_view': True}
data_dict = {'related_id': related_id, 'key':'reported_by'}
if db.related_extra_table is None:
db.init_db(context['model'])
res = db.RelatedExtra.get(**data_dict)
res = [x for x in res if x.value.split('*')[0] == user_id]
return res[0].value.split('*')[1]
def _pages_delete(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
out = db.Page.get(group_id=org_id, name=page)
if out:
session = context['session']
session.delete(out)
session.commit()
def acquisitions_list(context, data_dict):
'''
API to retrieve the list of datasets that have been acquired by a certain user
:parameter user: The user whose acquired dataset you want to retrieve. This parameter
is optional. If you don't include this identifier, the system will use the one
of the user that is performing the request
:type user: string
:return: The list of datarequest that has been acquired by the specified user
:rtype: list
'''
if data_dict is None:
data_dict = {}
if 'user' not in data_dict and 'user' in context:
data_dict['user'] = context['user']
plugins.toolkit.check_access(constants.ACQUISITIONS_LIST, context.copy(), data_dict)
# Init db
db.init_db(context['model'])
# Init the result array
result = []
# Check that the user exists
try:
plugins.toolkit.get_validator('user_name_exists')(data_dict['user'], context.copy())
except Exception:
raise plugins.toolkit.ValidationError('User %s does not exist' % data_dict['user'])
# Get the datasets acquired by the user
query = db.AllowedUser.get(user_name=data_dict['user'])
# Get the datasets
for dataset in query:
try:
dataset_show_func = 'package_show'
func_data_dict = {'id': dataset.package_id}
internal_context = context.copy()
# Check that the the dataset can be accessed and get its data
# FIX: If the check_access function is not called, an exception is risen.
plugins.toolkit.check_access(dataset_show_func, internal_context, func_data_dict)
dataset_dict = plugins.toolkit.get_action(dataset_show_func)(internal_context, func_data_dict)
# Only packages with state == 'active' can be shown
if dataset_dict.get('state', None) == 'active':
result.append(dataset_dict)
except Exception:
pass
return result
def _pages_list(context, data_dict):
search = {}
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
ordered = data_dict.get('order')
order_publish_date = data_dict.get('order_publish_date')
page_type = data_dict.get('page_type')
private = data_dict.get('private', True)
if ordered:
search['order'] = True
if page_type:
search['page_type'] = page_type
if order_publish_date:
search['order_publish_date'] = True
if not org_id:
search['group_id'] = None
try:
p.toolkit.check_access('ckanext_pages_update', context, data_dict)
if not private:
search['private'] = False
except p.toolkit.NotAuthorized:
search['private'] = False
else:
group = context['model'].Group.get(org_id)
user = context['user']
member = authz.has_user_permission_for_group_or_org(
group.id, user, 'read')
search['group_id'] = org_id
if not member:
search['private'] = False
out = db.Page.pages(**search)
out_list = []
# cdc
for pg in out:
parser = HTMLFirstImage()
parser.feed(pg.content)
img = parser.first_image
pg_row = {'title': pg.title,
'ename': pg.ename,
'cname': pg.cname,
'content': pg.content,
'econtent': pg.econtent,
'name': pg.name,
'publish_date': pg.publish_date.isoformat() if pg.publish_date else None,
'group_id': pg.group_id,
'page_type': pg.page_type,
}
if img:
pg_row['image'] = img
extras = pg.extras
if extras:
pg_row.update(json.loads(pg.extras))
out_list.append(pg_row)
return out_list
def datarequest_close(context, data_dict):
'''
Action to close a data request. Access rights will be checked before closing the
data request. If the user is not allowed, a NotAuthorized exception will be risen.
:param id: The id of the data request to be closed
:type id: string
:param accepted_dataset_id: The ID of the dataset accepted as solution for this
data request
:type accepted_dataset_id: string
:returns: A dict with the data request (id, user_id, title, description,
organization_id, open_time, accepted_dataset, close_time, closed)
:rtype: dict
'''
model = context['model']
session = context['session']
datarequest_id = data_dict.get('id', '')
# Check id
if not datarequest_id:
raise tk.ValidationError('Data Request ID has not been included')
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.DATAREQUEST_CLOSE, context, data_dict)
# Get the data request
result = db.DataRequest.get(id=datarequest_id)
if not result:
raise tk.ObjectNotFound('Data Request %s not found in the data base' % datarequest_id)
# Validate data
validator.validate_datarequest_closing(context, data_dict)
data_req = result[0]
# Was the data request previously closed?
if data_req.closed:
raise tk.ValidationError(['This Data Request is already closed'])
data_req.closed = True
data_req.accepted_dataset_id = data_dict.get('accepted_dataset_id', None)
data_req.close_time = datetime.datetime.now()
session.add(data_req)
session.commit()
return _dictize_datarequest(data_req)
def start_standalone_server(host=None, port=None, debug=True):
if not config.check():
sys.exit(1)
if not os.path.exists(config.get('base', 'cache_dir')):
os.makedirs(config.get('base', 'cache_dir'))
import db
from web import app
db.init_db()
app.run(host=host, port=port, debug=debug)
def __init__(self):
self.authorization_endpoint = config.get('ckan.oauth2.authorization_endpoint', None)
self.token_endpoint = config.get('ckan.oauth2.token_endpoint', None)
self.profile_api_url = config.get('ckan.oauth2.profile_api_url', None)
self.client_id = config.get('ckan.oauth2.client_id', None)
self.client_secret = config.get('ckan.oauth2.client_secret', None)
self.scope = config.get('ckan.oauth2.scope', '').decode()
self.rememberer_name = config.get('ckan.oauth2.rememberer_name', None)
# Init db
db.init_db(model)
def datarequest_create(context, data_dict):
'''
Action to create a new dara request. The function checks the access rights
of the user before creating the data request. If the user is not allowed
a NotAuthorized exception will be risen.
In addition, you should note that the parameters will be checked and an
exception (ValidationError) will be risen if some of these parameters are
not valid.
:param title: The title of the data request
:type title: string
:param description: A brief description for your data request
:type description: string
:param organiztion_id: If you want to create the data request in a specific
organization.
:type organization_id: string
:returns: A dict with the data request (id, user_id, title, description,
organization_id, open_time, accepted_dataset, close_time, closed)
:rtype: dict
'''
model = context['model']
session = context['session']
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.DATAREQUEST_CREATE, context, data_dict)
# Validate data
validator.validate_datarequest(context, data_dict)
# Store the data
data_req = db.DataRequest()
_undictize_datarequest_basic(data_req, data_dict)
data_req.user_id = context['auth_user_obj'].id if context['auth_user_obj'] else 'anonymous'
data_req.open_time = datetime.datetime.now()
data_req.visibility = constants.DataRequestState.visible.value
context['ignore_auth'] = config.get('ckan.datarequests.ignore_auth', False)
user = context['user']
if context['ignore_auth'] == False and not authz.is_sysadmin(user):
data_req.visibility = constants.DataRequestState.hidden.value
session.add(data_req)
session.commit()
return _dictize_datarequest(data_req)
def listings():
offersList = []
init_db()
#imgresult = Imagesmod.query.first()
#imgresult.image
#imgretrieved = image.load_img(BytesIO(imgresult.image))
#get all ads
results = OfferModel.query.all()
#for each ad get all images belonging to it and add as List
for result in results:
imagesList = db_session.query(ImagesModel).join(OfferModel).filter(
OfferModel.id == result.id)
result.imagesList = list(imagesList)
offersList.append(result)
print(offersList)
return render_template("listings.html", offersList=offersList)
def __init__(self, bot_key, db_path):
init_db(db_path)
self._updater = Updater(bot_key, use_context=True)
self._dp = self._updater.dispatcher
self._dp.add_handler(CommandHandler(
"start",
self._start,
pass_user_data=True))
self._dp.add_handler(MessageHandler(
Filters.text,
self._start,
pass_user_data=True))
self._dp.add_error_handler(self._error)
def ocr_images(file_id):
# init db
conn, c = init_db()
conn.row_factory = lambda cursor, row: row[0]
values = (file_id, )
# only OCR pages with boxes
results = c.execute(
"""SELECT id, image_path FROM page WHERE box = 1 and file_id = ?""",
values).fetchall()
# init pyocr
tools = pyocr.get_available_tools()
tool = tools[0]
langs = tool.get_available_languages()
lang = langs[0]
for page_id, image_path in results:
current_app.logger.info('Running OCR on image id %s (%s)' %
(page_id, image_path))
# load image
image = cv2.imread(image_path)
text_image = PIL_Image.fromarray(image)
# run ocr
body = tool.image_to_string(text_image,
lang=lang,
builder=pyocr.builders.TextBuilder())
# save results
c.execute("""UPDATE page SET body = ? WHERE id = ?""", (body, page_id))
conn.commit()
conn.close()
def test_redirect_url():
db_path = 'test_redirect_url.json'
remove_path(db_path)
db = init_db(db_path)
url_items = [{
'url': 'https://about.google.com/', # 301 HTTP code
'regex': 'Policy',
}]
correct_output = {
'matches': [{
'text': 'Policy',
'start': 24355,
'end': 24361,
}],
}
create_process(url_items, db)
output_items = db.all()
assert (len(output_items) > 0)
item = output_items[0]
assert (len(correct_output['matches']) == len(item['matches']))
for i in range(len(item['matches'])):
for key in item['matches'][i].keys():
assert (
correct_output['matches'][i][key] == item['matches'][i][key])
remove_path(db_path)
def get_most_recent():
"""
Get the most recent three items that have been analyzed.
"""
db_recent = DB.init_db(config.get("details_db")).product_details
res = list(
db_recent.find({
"status": "ready"
}).sort('timestamp', pymongo.DESCENDING))
items = []
try:
for i in range(3):
obj = res[i]
title = obj.get("product_name")
_max_length = config.get("misc").get("max_title_length")
if len(title) > _max_length:
title = title[:_max_length]
title += "..."
item = {
"image_url": obj.get("img"),
"title": title,
"product_url": obj.get("url"),
"sku": obj.get("sku")
}
items.append(item)
except IndexError:
pass
return items
def get_answer(question, sku):
"""
Open a websocket and keep the connection alive for the duration
of the session so that the sku model is loaded only once.
"""
response = {"status": __error__}
try:
db_votes = DB.init_db(config.get("votes_db"))
inf = inference.Inference(sku)
answer, confidence = inf.infer(question)
cluster = qna_clustering.Cluster({
"question": question,
"answer": answer,
"sku": sku
})
votes = cluster.get_votes(db_votes)
response = {
"confidence": confidence,
"answer": answer,
"status": "OK",
}
response.update(votes)
except Exception as e:
msg = '{}'.format(type(e).__name__, e.args[0])
logger.exception(msg)
return response
def get_status(sku):
response = {"status": __in_queue__}
status = None
try:
status = list(db_status.find({"sku": sku}))[0]
status = status.get("msg")
db_details = DB.init_db(config.get("details_db")).product_details
product = list(db_details.find({"sku": sku}))
product_url, product_name, image_url = "", "", ""
if product:
product_name = product[0].get("product_name")
product_url = product[0].get("url")
image_url = product[0].get("img")
logger.info("Status for {}: {}".format(sku, status))
return {
"status": status,
"product_name": product_name,
"product_url": product_url,
"image_url": image_url,
}
except IndexError:
# this happens due to a race condition because the sku hasn't been
# added to the database yet or because it simply doesn't exist. The
# second case only true if the URL has been typed in manually or
# bookmarked but the sku is missing from the URL.
logger.warning(
"Product status not yet available for sku {}".format(sku))
_set_status(__in_queue__, sku)
response = {"status": status}
except Exception as e:
logger.exception(e)
response = {"status": __error__}
return response
def run(script, debug):
_, session = init_db()
command_obj = get_command(session, script)
lines = load_file(command_obj.filepath)
header, content = separate_headers(lines)
if header_check(session, header, script, content):
command_traitement(content, debug=debug)
def _get_product_details(source, url, sku):
"""
Scrape product metadata.
:param url: canonical product url
:return number of reviews and product name
"""
sc = scraper.Scraper(source=source)
response = sc.get_request(url)
pr = parser.Parser(sku=sku, source=source)
res = pr.parse(response, init=True)
if res:
# Save it to the database
db_details = DB.init_db(config.get("details_db"))
db_details = db_details.product_details
record = {
"status": "processing",
"url": url,
"product_name": res.get("product_name"),
"review_count": res.get("review_count"),
"review_page_count": res.get("page_count"),
"source": source,
"sku": sku,
"img": res.get("img_url"),
"timestamp": time.time(),
}
db_details.insert_one(record)
logger.info("Saved new product details: ")
logger.info(record)
return res
def startup():
err = init_db(app)
if err:
app.logger.error(f'DB: {err}')
sys.exit(1)
app.logger.info('DB connect')
proc.start()
def _db_product_details(sku):
"""
Return product details from the database.
"""
db_details = DB.init_db(config.get("details_db")).product_details
product = list(db_details.find({"sku": sku}))
if product:
product_name = product[0].get("product_name")
product_url = product[0].get("url")
image_url = product[0].get("img")
review_count = product[0].get("review_count")
page_count = product[0].get("review_page_count")
valid = all(
[product_name, product_url, image_url, review_count, page_count])
if valid:
return {
"product_name": product_name,
"review_count": review_count,
"page_count": page_count,
}
else:
logger.info(
"Unable to validate product details for {}. Deleting entry".
format(sku))
db_details.delete_one({"sku": sku})
return {}
def home():
conn, c = init_db()
conn.row_factory = lambda cursor, row: row[0]
files = c.execute(
"""SELECT pdf_path, xlsx_path, title, subject, author FROM file"""
).fetchall()
return render_template('home.html', files=files)
def _update_details_db(sku):
"""
Update the status field of
"""
db_details = DB.init_db(config.get("details_db")).product_details
record = {"status": "ready"}
db_details.update_one({"sku": sku}, {"$set": record})
def main():
mydb = db.init_db()
s = db.select(mydb.curs)
products = p.list_of_products()
v = view.view(s)
c = controller(v, s, mydb, products)
c.main_menu_loop()
def _pages_update(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
# we need the page in the context for name validation
context['page'] = page
context['group_id'] = org_id
data, errors = df.validate(data_dict, schema, context)
if errors:
raise p.toolkit.ValidationError(errors)
out = db.Page.get(group_id=org_id, name=page)
if not out:
out = db.Page()
out.group_id = org_id
out.name = page
items = [
'title',
'content',
'name',
'private',
'order',
'page_type',
'publish_date',
'image_url',
]
for item in items:
# backward compatible with older version where page_type does not exist
setattr(out, item,
data.get(item, 'page' if item == 'page_type' else None))
extras = {}
extra_keys = set(schema.keys()) - set(items + ['id', 'created'])
for key in extra_keys:
if key in data:
extras[key] = data.get(key)
out.extras = json.dumps(extras)
out.modified = datetime.datetime.utcnow()
out.user_id = p.toolkit.c.userobj.id
out.save()
session = context['session']
session.add(out)
session.commit()
def update_datarequest_comment(context, data_dict):
'''
Action to update a comment of a data request. Access rights will be checked
before updating the comment and a NotAuthorized exception will be risen if
the user is not allowed to update the comment
:param id: The ID of the comment to be updated
:type id: string
:param comment: The updated comment
:type comment: string
:returns: A dict with the data request comment (id, user_id, datarequest_id,
time and comment)
:rtype: dict
'''
model = context['model']
session = context['session']
comment_id = data_dict.get('id', '')
if not comment_id:
raise tk.ValidationError([tk._('Comment ID has not been included')])
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.UPDATE_DATAREQUEST_COMMENT, context, data_dict)
# Get the data request
result = db.Comment.get(id=comment_id)
if not result:
raise tk.ObjectNotFound(tk._('Comment %s not found in the data base') % comment_id)
comment = result[0]
# Validate data
validator.validate_comment(context, data_dict)
# Set the data provided by the user in the data_red
_undictize_comment_basic(comment, data_dict)
session.add(comment)
session.commit()
return _dictize_comment(comment)
def datarequest_create(context, data_dict):
'''
Action to create a new dara request. The function checks the access rights
of the user before creating the data request. If the user is not allowed
a NotAuthorized exception will be risen.
In addition, you should note that the parameters will be checked and an
exception (ValidationError) will be risen if some of these parameters are
not valid.
:param title: The title of the data request
:type title: string
:param description: A brief description for your data request
:type description: string
:param organiztion_id: If you want to create the data request in a specific
organization.
:type organization_id: string
:returns: A dict with the data request (id, user_id, title, description,
organization_id, open_time, accepted_dataset, close_time, closed)
:rtype: dict
'''
model = context['model']
session = context['session']
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.DATAREQUEST_CREATE, context, data_dict)
# Validate data
validator.validate_datarequest(context, data_dict)
# Store the data
data_req = db.DataRequest()
_undictize_datarequest_basic(data_req, data_dict)
data_req.user_id = context['auth_user_obj'].id
data_req.open_time = datetime.datetime.now()
session.add(data_req)
session.commit()
return _dictize_datarequest(data_req)
def _pages_show(context, data_dict):
if db.pages_table is None:
db.init_db(context['model'])
org_id = data_dict.get('org_id')
page = data_dict.get('page')
order = data_dict.get('order')
lang = data_dict.get('lang')
out = None
if page or org_id:
out = db.Page.get(group_id=org_id, name=page)
elif lang and order:
out = db.Page.get(order=order, lang=lang)
if out:
out = db.table_dictize(out, context)
return out
def datarequest_comment(context, data_dict):
'''
Action to create a comment in a data request. Access rights will be checked
before creating the comment and a NotAuthorized exception will be risen if
the user is not allowed to create the comment
:param datarequest_id: The ID of the datarequest to be commented
:type id: string
:param comment: The comment to be added to the data request
:type comment: string
:returns: A dict with the data request comment (id, user_id, datarequest_id,
time and comment)
:rtype: dict
'''
model = context['model']
session = context['session']
datarequest_id = data_dict.get('datarequest_id', '')
# Check id
if not datarequest_id:
raise tk.ValidationError(
[tk._('Data Request ID has not been included')])
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.DATAREQUEST_COMMENT, context, data_dict)
# Validate comment
validator.validate_comment(context, data_dict)
# Store the data
comment = db.Comment()
_undictize_comment_basic(comment, data_dict)
comment.user_id = context['auth_user_obj'].id
comment.time = datetime.datetime.now()
session.add(comment)
session.commit()
return _dictize_comment(comment)
def init_db():
# Make sure we have the tables up-to-date in the DB
db.init_db()
SITE_ROOT = os.path.realpath(os.path.dirname(__file__))
main_dir = os.path.join(SITE_ROOT, "views", "resources", "GEOJSON")
# Some GeoJSON files are not meant to be used in the graph, and thus should not be imported in
# the DB. Don't even try, this will raise an error
excluded_filenames = ['CONTOUR.geojson']
filenames = [
filename for filename in os.listdir(main_dir)
if filename.endswith('.geojson') and filename not in excluded_filenames
]
for filename in filenames:
with open(os.path.join(main_dir, filename)) as file:
insert_features_in_db(filename, json.loads(file.read()))
print(f"Inserted features of {filename}")
def app():
db_fd, db_path = tempfile.mkstemp()
app = create_app({
'TESTING': True,
'DATABASE': db_path,
# Set the folder path for testing. All uploaded files will be saved here
'UPLOAD_FOLDER': './testing_files'
})
with app.app_context():
init_db()
yield app
os.close(db_fd)
os.unlink(db_path)
def approve_worker(self, worker, force=False):
''' Approve worker '''
assignment_id = worker['assignmentId']
init_db()
found_worker = False
parts = Participant.query.\
filter(Participant.assignmentid == assignment_id).\
filter(Participant.status.in_([3, 4])).\
all()
# Iterate through all the people who completed this assignment.
# This should be one person, and it should match the person who
# submitted the HIT, but that doesn't always hold.
status_report = ''
for part in parts:
if part.workerid == worker['workerId']:
found_worker = True
success = self.amt_services.approve_worker(assignment_id)
if success:
part.status = 5
db_session.add(part)
db_session.commit()
status_report = 'approved worker {} for assignment {}'.format(part.workerid, assignment_id)
else:
error_msg = '*** failed to approve worker {} for assignment {}'.format(part.workerid, assignment_id)
raise Exception(error_msg)
else:
status_report = 'found unexpected worker {} for assignment {}'.format(part.workerid, assignment_id)
if not found_worker:
# approve workers not found in DB if the assignment id has been specified
if force:
success = self.amt_services.approve_worker(assignment_id)
if success:
_status_report = 'approved worker {} for assignment {} but not found in DB'.format(worker['workerId'], assignment_id)
status_report = '\n'.join([status_report,_status_report])
else:
error_msg = '*** failed to approve worker {} for assignment {}'.format(worker['workerId'], assignment_id)
raise Exception(error_msg)
# otherwise don't approve, and print warning
else:
_status_report = 'worker {} not found in DB for assignment {}. Not automatically approved. Use --force to approve anyway.'.format(worker['workerId'], assignment_id)
if status_report:
status_report = '\n'.join([status_report,_status_report])
else:
status_report = _status_report
return status_report
def telephone_n(message: Message):
global telephone
telephone = int(message.text)
init_db()
add_reg_info_in_db(chat_id=message.chat.id, company_id=CompanyID, name=name, surname=surname,
telephone=telephone, day=day, start_time=start_time, end_time=end_time)
bot.send_message(message.from_user.id, "Ձեր անձնական տվյալներն են \n\n Անուն : " + name + "\n Ազգանուն : " +
surname + "\n Կազմակերպության ID : " + str(CompanyID) + "\n Աշխատանքային գրաֆիկ : " + day + " " +
start_time + "-" + end_time +
"\n Հեռախոսահամար ։ 0" + str(telephone))
whole_info = "Նոր գրանցում \n\n Անուն : " + name + "\n Ազգանուն : " + surname + "\n Կազմակերպության ID : " + str(CompanyID) + "\n Աշխատանքային գրաֆիկ : " + "\n Հեռախոսահամար ։ 0" + str(telephone)
mail = fromCompanyId_adminCheck(company_id=CompanyID)
if(mail):
sendMailAfterRegister(mail, whole_info)
else:
print("something happened, we didn't send")
def command(value, vars):
from lmparser import load_file, header_check, separate_headers, command_traitement
_, session = init_db()
command_obj = get_command(session, value)
lines = load_file(command_obj.filepath)
header, content = separate_headers(lines)
if header_check(session, header, value, content):
command_traitement(content)
return
def telephone_n(message: Message):
global telephone
telephone = int(message.text)
init_db()
add_reg_info_in_db(chat_id=message.chat.id,
company_id=CompanyID,
name=name,
surname=surname,
telephone=telephone,
day=day,
start_time=start_time,
end_time=end_time)
bot.send_message(
message.from_user.id, "Ձեր անձնական տվյալներն են \n\n Անուն : " +
name + "\n Ազգանուն : " + surname + "\n Կազմակերպության ID : " +
str(CompanyID) + "\n Աշխատանքային գրաֆիկ : " + day + " " + start_time +
"-" + end_time + "\n Հեռախոսահամար ։ 0" + str(telephone))
"""քոմենտել եմ որտև Էռոռա տալիս """
def _nlp_reset(sku):
"""
Clear the database of any existing sentences for sku.
:param sku: product sku
"""
db_sents = DB.init_db(config.get("sent_db")).sentences
db_sents.delete_many({"sku": sku})
logger.info("Cleared sentence table for " + sku)
def create_app():
"""
Initialize Flask and setup database
"""
app = CustomFlask(__name__)
app.config.from_mapping(SECRET_KEY=os.urandom(16),
CACHED_TIME=time.time(),
THEME_DIR="./data/theme.json")
# Init database
import db
db.init_db()
# Init github cache
from utils.github import update_public_repos
db_conn = db.get_db()
update_public_repos(db_conn)
# Init meidum cache
db_conn = db.get_db()
with open(app.config["THEME_DIR"], "r") as f:
data = json.load(f)
medium_url = data["medium_url"]
from utils.medium import update_articles
update_articles(db_conn, medium_url)
# Register blueprints
import home
app.register_blueprint(home.bp)
import auth
app.register_blueprint(auth.bp)
# Initialize login manager
login_manager = LoginManager()
login_manager.init_app(app)
@login_manager.user_loader
def user_loader(username):
db_conn = db.get_db()
return get_user(db_conn, username)
return app
def unfollow_datarequest(context, data_dict):
'''
Action to unfollow a data request. Access rights will be cheked before
unfollowing a datarequest and a NotAuthorized exception will be risen if
the user is not allowed to unfollow the given datarequest. ValidationError
will be risen if the datarequest ID is not included in the request.
ObjectNotFound will be risen if the user is not following the given
datarequest.
:param id: The ID of the datarequest to be unfollowed
:type id: string
:returns: True
:rtype: bool
'''
model = context['model']
session = context['session']
datarequest_id = data_dict.get('id', '')
if not datarequest_id:
raise tk.ValidationError(
[tk._('Data Request ID has not been included')])
# Init the data base
db.init_db(model)
# Check access
tk.check_access(constants.UNFOLLOW_DATAREQUEST, context, data_dict)
# Is already following?
user_id = context['auth_user_obj'].id
result = db.DataRequestFollower.get(datarequest_id=datarequest_id,
user_id=user_id)
if not result:
raise tk.ObjectNotFound(
[tk._('The user is not following the given Data Request')])
follower = result[0]
session.delete(follower)
session.commit()
return True
def slack_channels_show(context, data_dict):
'''
Action to retrieve the slack notification channels. The only required
parameter is the name of the organization passed as id. A NotFound
exception will be risen if the organization id is not found.
Access rights will be checked before returning the information and an
exception will be risen (NotAuthorized) if the user is not authorized.
:param context: the context of the request
:type context: dict
:param data_dict: Contains the following
organization_id: The ID of the organization
:type data_dict: dict
:returns: A list of the slack notification details(id,
organization_id, webhook_url, slack_channel)
:rtype: list
'''
model = context['model']
organization_id = data_dict.get('organization_id')
success = data_dict.get('success', False)
if not organization_id and not success:
raise toolkit.ValidationError(
toolkit._('Organization ID has not been included'))
# Init the data base
db.init_db(model)
# Check access
if not success:
toolkit.check_access(constants.MANAGE_NOTIFICATIONS, context,
data_dict)
# Get the available slack channels
result = db.Org_Slack_Details.get(organization_id=organization_id)
if result:
slack_channels = [
_dictize_slack_details(channel) for channel in result
]
else:
slack_channels = []
return slack_channels
def search_list(context, data_dict):
'''
List the search history
:param limit: The number of items to show (optional, default: 10)
:type limit: int
'''
tk.check_access('ckanext_search_history_list', context, data_dict)
if db.search_history_table is None:
db.init_db(context['model'])
username = context.get('user')
user = new_authz.get_user_id_for_username(username, allow_none=False)
limit = data_dict.get('limt')
history = db.SearchHistory.search_history(user_id=user, limit=limit)
result = []
if history:
for item in history:
result.append(db.table_dictize(item, context))
return result
