def __init__(self):
Configuration.__init__(self)
self.topicindex = self.config['topicindex']
self.database = 'datasets'
self.storedata = []
self.data = []
self.links = {}
def test_validate_uri(self):
self.clioinfra = Configuration()
if self.clioinfra.not_valid_uri('passwd'):
print self.clioinfra.config['error']
if self.clioinfra.not_valid_uri('`wget'):
print self.clioinfra.config['error']
if self.clioinfra.not_valid_uri('|'):
print self.clioinfra.config['error']
class URITestClass(ConfigTestClass):
def test_validate_uri(self):
self.clioinfra = Configuration()
if self.clioinfra.not_valid_uri('passwd'):
print self.clioinfra.config['error']
if self.clioinfra.not_valid_uri('`wget'):
print self.clioinfra.config['error']
if self.clioinfra.not_valid_uri('|'):
print self.clioinfra.config['error']
def test_validate_uri(self):
self.clioinfra = Configuration()
self.assertFalse(self.clioinfra.is_valid_uri('http://cliodomain.com/xcxc;`some code'))
# check if passwd can be recognized in uri
self.assertFalse(self.clioinfra.are_parameters_valid('passwd'), 'passwd recognition problem')
# wget instance in uri
self.assertFalse(self.clioinfra.are_parameters_valid('wget` '), 'wget recognition problem')
self.assertFalse(self.clioinfra.are_parameters_valid('curl '), 'curl recognition problem')
# other characters are allowed
self.assertTrue(self.clioinfra.are_parameters_valid("http://data.cliodomain.org/collabs/data/static/"), 'False recognition of characters in url')
# shell injection check
self.assertFalse(self.clioinfra.are_parameters_valid("http://data.cliodomain.org/?export?fileId=12;%cat%20/etc/passw1d%20"), 'False recognition of characters in url')
def test_validate_uri(self):
self.clioinfra = Configuration()
self.assertFalse(
self.clioinfra.is_valid_uri(
'http://cliodomain.com/xcxc;`some code'))
# check if passwd can be recognized in uri
self.assertFalse(self.clioinfra.are_parameters_valid('passwd'),
'passwd recognition problem')
# wget instance in uri
self.assertFalse(self.clioinfra.are_parameters_valid('wget` '),
'wget recognition problem')
self.assertFalse(self.clioinfra.are_parameters_valid('curl '),
'curl recognition problem')
# other characters are allowed
self.assertTrue(
self.clioinfra.are_parameters_valid(
"http://data.cliodomain.org/collabs/data/static/"),
'False recognition of characters in url')
# shell injection check
self.assertFalse(
self.clioinfra.are_parameters_valid(
"http://data.cliodomain.org/?export?fileId=12;%cat%20/etc/passw1d%20"
), 'False recognition of characters in url')
def testsearch(self):
settings = Configuration()
dv = "National"
dv = "Micro"
sconnection = ExtrasearchAPI(settings.config['dataverseroot'], dv)
p = sconnection.read_all_datasets()
self.assertTrue(bool(sconnection.read_all_datasets()))
# test if dataset is private
self.assertTrue(bool(sconnection.has_restricted_data("V4Q8XE")))
# test if dataset is public
self.assertFalse(bool(sconnection.has_restricted_data("8FCYOX")))
# test full handle pid
self.assertTrue(
bool(sconnection.has_restricted_data("hdl:10622/V4Q8XE")))
def login(settings=''):
name = ''
clioinfra = Configuration()
opensession = {}
openldap = OpenLDAP()
if request.args.get('project'):
opensession['project'] = request.args.get('project')
else:
opensession['project'] = ''
try:
thisuser = openldap.authentificate(request.form.get('login'),
request.form.get('password'))
if thisuser[0][1]['uid'][0]:
session['name'] = thisuser[0][1]['displayName'][0]
session['uid'] = thisuser[0][1]['uid'][0]
if opensession['project']:
session['project'] = opensession['project']
name = str(thisuser[0][1]['displayName'][0])
try:
projectname = session['project']
except:
projectname = ''
if projectname:
sandboxflag = re.search("sandbox", request.url)
if sandboxflag:
projecturl = "%s/%s" % (clioinfra.config['apiroot'],
projectname)
else:
projecturl = "%s/%s" % (clioinfra.config['proxy'], projectname)
return redirect(projecturl, code=302)
else:
return make_response(
render_template('iish/login.html', username=name))
except:
return make_response(render_template('iish/login.html'))
def test_settings(self):
self.clioinfra = Configuration()
self.assertTrue(self.clioinfra.config['dataverseroot'])
self.assertTrue(self.clioinfra.config['apiroot'])
self.assertTrue(self.clioinfra.config['key'])
def upload_file():
if request.method == 'POST':
file = request.files['files']
if file and allowed_file(file.filename):
filename = secure_filename(file.filename)
file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
return redirect("%spics/%s" % (request.url_root, filename))
return 'Upload form'
@app.route('/pics/<filename>')
def uploaded_file(filename):
return send_from_directory(app.config['UPLOAD_FOLDER'], filename)
clioinfra = Configuration()
app.secret_key = clioinfra.config['secretkey']
@app.route('/')
def browse(settings=''):
session['name'] = ''
return make_response(render_template('iish/projects.html'))
@app.route('/logout', methods=['GET'])
def logout(settings=''):
session['name'] = ''
try:
if 'project' in session:
sandboxflag = re.search("sandbox", request.url)
def geofilter():
pids = []
ctrlist = {}
ctrfilter = []
settings = Configuration()
clioinfra = Dataset()
clioindex = clioinfra.clioindex()
columns = ['1', 'Webmapper code', 'Webmapper numeric code', 'ccode', 'country name', 'start year', 'end year']
(classification, geodataset, title, units) = content2dataframe(settings.config, settings.config['geocoderhandle'])
settings = DataFilter(request.args)
if settings.selected():
pids = clioinfra.findhandles(settings.selected())
datasets = clioinfra.retrievedatasets(pids)
selection = []
for item in datasets:
dataset = datasets[item]
dataset.columns = dataset.ix[1]
dataset = dataset.convert_objects(convert_numeric=True)
dataset.index = dataset['Webmapper numeric code']
if (settings.minyear()):
dataset = dataset.loc[dataset['start year'] >= settings.minyear()]
dataset = dataset.loc[dataset['start year'] <= settings.maxyear()]
for col in columns:
dataset = dataset.drop(col, axis=1)
dataset['total'] = dataset.sum(axis=1)
dataset = dataset.ix[dataset['total'] > 0]
selection.append(dataset.index)
for row in selection:
for countryID in row:
if countryID not in ctrlist:
ctrlist[countryID] = countryID
ctrfilter.append(countryID)
geodataset = geodataset.convert_objects(convert_numeric=True)
geodataset = geodataset.loc[geodataset['start year'] >= settings.minyear()]
geodataset = geodataset.loc[geodataset['start year'] <= settings.maxyear()]
if settings.showframe():
geodataset.index = geodataset['Webmapper numeric code']
if ctrfilter:
geodataset = geodataset.ix[ctrfilter]
(geocoder, geolist, oecd) = buildgeocoder(geodataset, settings.config, settings.countryfilter())
data = json.dumps(geocoder, encoding="utf-8", sort_keys=True, indent=4)
return Response(data, mimetype='application/json')
def geofilter():
pids = []
ctrlist = {}
ctrfilter = []
settings = Configuration()
clioinfra = Dataset()
clioindex = clioinfra.clioindex()
columns = [
'1', 'Webmapper code', 'Webmapper numeric code', 'ccode',
'country name', 'start year', 'end year'
]
(classification, geodataset, title,
units) = content2dataframe(settings.config,
settings.config['geocoderhandle'])
settings = DataFilter(request.args)
if settings.selected():
pids = clioinfra.findhandles(settings.selected())
datasets = clioinfra.retrievedatasets(pids)
selection = []
for item in datasets:
dataset = datasets[item]
dataset.columns = dataset.ix[1]
dataset = dataset.convert_objects(convert_numeric=True)
dataset.index = dataset['Webmapper numeric code']
if (settings.minyear()):
dataset = dataset.loc[
dataset['start year'] >= settings.minyear()]
dataset = dataset.loc[
dataset['start year'] <= settings.maxyear()]
for col in columns:
dataset = dataset.drop(col, axis=1)
dataset['total'] = dataset.sum(axis=1)
dataset = dataset.ix[dataset['total'] > 0]
selection.append(dataset.index)
for row in selection:
for countryID in row:
if countryID not in ctrlist:
ctrlist[countryID] = countryID
ctrfilter.append(countryID)
geodataset = geodataset.convert_objects(convert_numeric=True)
geodataset = geodataset.loc[geodataset['start year'] >= settings.minyear()]
geodataset = geodataset.loc[geodataset['start year'] <= settings.maxyear()]
if settings.showframe():
geodataset.index = geodataset['Webmapper numeric code']
if ctrfilter:
geodataset = geodataset.ix[ctrfilter]
(geocoder, geolist, oecd) = buildgeocoder(geodataset, settings.config,
settings.countryfilter())
data = json.dumps(geocoder, encoding="utf-8", sort_keys=True, indent=4)
return Response(data, mimetype='application/json')
def __init__(self):
Configuration.__init__(self)
self.geocoderhandle = self.config['geocoderhandle']
(self.classification, self.geodataset, self.title, self.units) = content2dataframe(self.config, self.geocoderhandle)
def test_ldap(self):
self.clioinfra = Configuration()
self.openldap = OpenLDAP()
thisuser = self.openldap.searchuser("testusername")
thisuser = self.openldap.authentificate("testusername", "passwd")
print thisuser
def datasetspace(settings=''):
(where, query, datasets, metadata, s, permissions) = ({}, '', [], [], {}, 'yes')
where = {'collab': '', 'iish': '', 'harvard': ''}
pagetitle = "Public datasets"
config = configuration()
if config['error']:
return config['error']
root = config['apiroot']
dataversename = 'global'
if request.args.get('dv'):
dataversename = request.args.get('dv')
if request.args.get('q'):
query = request.args.get('q')
if request.args.get('permissions'):
permissions = request.args.get('permissions')
if request.args.get('where'):
where[request.args.get('where')] = 'checked="checked"'
settings = Configuration()
sconnection = ExtrasearchAPI(settings.config['dataverseroot'], dataversename)
if where['harvard']:
# Extract host for Dataverse connection
findhost = re.search('(http\:\/\/|https\:\/\/)(.+)', settings.config['harvarddataverse'])
if findhost:
settings.config['dataversehostname'] = findhost.group(2)
connection = Connection(settings.config['dataversehostname'], settings.config['harvardkey'])
else:
try:
connection = Connection(config['hostname'], settings.config['key'])
except:
return 'Error: no connection to Dataverse. Please try later...'
handlestr = ''
if query:
s['q'] = query
metadata = search_by_keyword(connection, s)
else:
try:
dataverse = connection.get_dataverse(dataversename)
item = dataverse.get_contents()
active = 'yes'
except:
active = None
if active:
try:
for item in dataverse.get_contents():
handlestr+= item['identifier'] + ' '
active = 'yes'
except:
active = None
if not active:
handlestr = sconnection.read_all_datasets()
if handlestr:
s['q'] = handlestr
s['per_page'] = 100
metadata = search_by_keyword(connection, s)
#return str(metadata['items'])
for dataset in metadata['items']:
active = ''
# Private datasets
if permissions == 'closed':
pagetitle = "Restricted datasets"
try:
if (sconnection.has_restricted_data(dataset['global_id'])):
active = 'yes'
except:
active = ''
# Public data
else:
try:
if not (sconnection.has_restricted_data(dataset['global_id'])):
active = 'yes'
except:
active = ''
if active:
try:
for author in dataset['authors']:
dataset['author'] = str(author) + ', '
dataset['author'] = dataset['author'][:-2]
except:
dataset['author'] = str(dataset['description'])
datasets.append(dataset)
if where['harvard']:
datasets.append(dataset)
(username, projectname) = ('','')
fields = {}
resp = make_response(render_template('search.html', projectname=projectname, username=username, datasets=datasets, searchq=query, pagetitle=pagetitle, where=where, fields=fields))
return resp
def __init__(self):
Configuration.__init__(self)
self.geocoderhandle = self.config['geocoderhandle']
(self.classification, self.geodataset, self.title,
self.units) = content2dataframe(self.config, self.geocoderhandle)