def get(environment_id):
with make_session() as session:
data = session.query(Environment).filter(
Environment.id == environment_id).first() # type: Environment
if data is None:
raise NotFound("Requested environment does not exist")
return marshal(data[0], environment_fields)
def install(username, password):
if app.config['DATABASE_PATH'] is not None:
if os.path.exists(app.config['DATABASE_PATH']):
confirm = raw_input(
'A database already exists. Remove it? [y/N] ').lower()
if confirm != 'y':
print 'Aborting install.'
return
app.logger.warning('Removing database.')
db.make_session()
db.init_db()
admin = users.User(username, password)
db.session.add(admin)
db.session.commit()
def get(pack_id):
with make_session() as session:
data = session.query(Pack).filter(
Pack.id == pack_id).first() # type: Pack
if data is None:
raise NotFound("Requested pack does not exist")
return marshal(data, pack_fields)
def delete(pack_id):
with make_session() as session:
data = session.query(Pack).filter(
Pack.id == pack_id).first() # type: Pack
if data is None:
raise NotFound("Requested pack does not exist")
session.delete(data)
return make_empty_response()
def patch(environment_id, name):
with make_session() as session:
data = session.query(Environment).filter(
Environment.id == environment_id).first() # type: Environment
if data is None:
raise NotFound("Requested environment does not exist")
data.name = name
return make_empty_response()
def get(template_id, with_text):
with make_session() as session:
template = session.query(Template).filter(
Template.id == template_id).first() # type: Template
if template is None:
raise NotFound("Requested template does not exist")
return marshal(
template,
template_fields_with_text if with_text else template_fields)
def delete(environment_id):
with make_session() as session:
data = session.query(Environment).filter(
Environment.id == environment_id).first() # type: Environment
if data is None:
raise NotFound("Requested environment does not exist")
if data.in_use():
raise Conflict("Requested environment is in use")
session.delete(data)
return make_empty_response()
def load_template(filename: str):
with open(filename) as file:
template = yaml.load(file)
tpl = Template(name=template['name'], text=template['text'])
for varname, properties in template['variables'].items():
tpl.variables.append(
Variable(name=varname, description=properties['description']))
with make_session() as session:
session.add(tpl)
def delete(template_id):
with make_session() as session:
template = session.query(Template).filter(
Template.id == template_id).first() # type: Template
if template is None:
raise NotFound("Requested template does not exist")
for variable in template.variables:
if variable.in_use():
raise Conflict(
"Cannot delete the template because one or more variables are in use"
)
session.delete(template)
return make_empty_response()
def put(name, text):
try:
with make_session() as session:
if session.query(
session.query(Template).filter(
Template.name == name.strip()).exists()).scalar():
raise Conflict(
"A template with the same name already exists")
template = Template(name=name.strip(), text=text)
session.add(template)
session.commit()
return make_id_response(template.id)
except IntegrityError:
raise InternalServerError(
"Could not create the requested template")
def patch(template_id, name, text, variables):
with make_session() as session:
template = session.query(Template).filter(
Template.id == template_id).first() # type: Template
if template is None:
raise NotFound("Requested template does not exist")
if variables != missing:
if 'delete' in variables:
for d in variables['delete']:
to_delete = session.query(Variable).filter(
Variable.id == d)
if to_delete.first().in_use():
raise Conflict(
'Cannot delete variable because it is in use')
to_delete.delete()
if 'update' in variables:
for u in variables['update']:
if 'description' in u:
variable = session.query(Variable) \
.filter(Variable.id == u['id']) \
.first()
variable.description = u['description'].strip()
if 'name' in u:
if variable.name != u[
'name'] and variable.in_use():
raise Conflict(
'Cannot change name of a variable that\'s in use'
)
variable.name = u['name']
if 'create' in variables:
for c in variables['create']:
if 'description' in c:
session.add(
Variable(template=template,
name=c['name'].strip(),
description=c['description'].strip()))
if text != missing:
template.text = text
if name != missing:
template.name = name
return make_id_response(template_id)
def get_history_for_entity(cls, limit=10, with_details=False):
transaction = transaction_class(cls)
transaction_changes = cls.__versioned__['transaction_changes']
with make_session() as session:
transactions = (session
.query(transaction)
.join(transaction.changes)
.filter(transaction_changes.entity_name.in_([cls.__name__]))
.order_by(transaction.issued_at.desc())
.limit(limit)
.all())
return [{
'transaction_id': transaction.id,
'issued_at': transaction.issued_at.isoformat(),
'remote_address': transaction.remote_addr,
'changed_entities': _marshal_changed_entities(transaction, with_details)
} for transaction in transactions]
def get(pack_name, environment_name):
with make_session() as session:
pack = session.query(Pack).filter(
Pack.name == pack_name).first() # type: Pack
if pack is None:
raise NotFound('Requested pack not found')
environment = session.query(Environment).filter(
Environment.name ==
environment_name).first() # type: Environment
if environment is None:
raise NotFound('Requested environment not found')
result = []
for template in pack.get_templates():
result.append({
'name': template.name,
'text': instantiate(pack, template, environment)
})
return result
def seed_data():
load_template('seeds/apache-vhost.template.yml')
with make_session() as session:
template = Template(name="test",
text="whoa = {{xxx_var}};"
" template={{_meta.template}};"
" environment={{_meta.environment}};"
" pack={{_meta.pack}}")
session.add(template)
variable = Variable(name="xxx_var",
description="me gusta",
template=template)
session.add(variable)
env = Environment(name="rotlicht")
session.add(env)
pack = Pack(name="fun")
session.add(pack)
instance = Instance(name="some/instance", pack=pack, template=template)
value = Value(variable=variable,
environment=env,
instance=instance,
data="injection")
session.add(value)
import tarfile
from pprint import pprint
import time
import gzip
import sqlalchemy
import os
# Custom Imports
sys.path.append('../database/')
from db import make_session, Tweet, User
# Parameters
INFILE = '../../data/pablos_stuff/rnd_usr_sample/users-2016-10-19.json'
# Set up sqlalchemy session
session = make_session()
with io.open(INFILE, 'r') as infile:
tweet_count = 0
duplicates = 0
for i, line in enumerate(infile):
try:
user = json.loads(line)
tweet_count += 1
except json.decoder.JSONDecodeError as e:
print(e)
continue
uid = int(user['id_str'][0])
import db
from db.models import User
db.init_db()
session = db.make_session()
user = session.query(User).filter(User.name == '0000').first()
if user is None:
user = User(name='0000', password='******', is_admin=True)
session.add(user)
session.commit()
else:
print('user exists, change another username')
def get():
with make_session() as session:
return marshal(
session.query(Environment).all(), environment_fields)
def get_session():
session = db.make_session()
try:
yield session
finally:
session.close()
def get(template_id):
with make_session() as session:
return marshal(
session.query(Variable).filter(
Variable.template_id == template_id).all(),
variable_fields)
def get():
with make_session() as session:
return marshal(session.query(Template).all(), template_fields)
def put(name: str):
environment = Environment(name=name.strip())
with make_session() as session:
session.add(environment)
session.commit()
return make_id_response(environment.id)
def put(name):
pack = Pack(name=name.strip())
with make_session() as session:
session.add(pack)
session.commit()
return make_id_response(pack.id)
def get():
with make_session() as session:
return marshal(session.query(Pack).all(), pack_fields)
outfile.write(outline.format('id', 'hit', 'hit_de', 'recaptured'))
else:
with io.open(OUTFILE, 'r') as outfile:
id_generator.n_samples = file_len(outfile) - 1
id_generator.sampled_ids = get_sampled_ids(outfile)
# Set up authentication and api
acc = tc.credentials['coll_1']
auth = tweepy.OAuthHandler(acc['consumer_key'], acc['consumer_secret'])
auth.set_access_token(acc['access_token'], acc['access_token_secret'])
api = tweepy.API(auth, parser=tweepy.parsers.JSONParser())
# Get all ids from pablos sample
## Connect to sql engine
_, engine = make_session('../../database/db_credentials')
query = "SELECT id FROM users WHERE data_group = 'de_panel';"
captured_1 = pd.read_sql_query(sql=query, con=engine)
sample_1 = set(captured_1.id)
limits = api.rate_limit_status()
remaining = limits['resources']['users']['/users/lookup']['remaining']
for i in count():
# Sleep random time (Expectation 1 second == 900 requests per 15 minutes)
time.sleep(1 + np.random.exponential(scale=0.1))
# Generate a batch of unique random ids
id_batch = [str(a) for a in islice(id_generator, 100)]
from copy import copy
from gensim import corpora
from gensim.matutils import corpus2csc
# Custom imports
sys.path.append('../database')
from db import make_session, Tweet, data_frame
sys.path.append('/home/flinder/Dropbox/current_projects/text_utils')
from text_utils import Cleaner, n_grams, url_regex
# ---------------------------------------------
# Data preprocessing
# ---------------------------------------------
# Database connection
_, engine = make_session()
# Get number of tweets by language
query = ("SELECT count(*), lang "
"FROM tweets "
"WHERE data_group = 'de_panel' "
"AND created_at >= '2015-01-01' "
"AND created_at < '2016-01-01' "
"GROUP BY lang "
"ORDER BY count DESC; ")
df_language = pd.read_sql(sql=query, con=engine)
print('Most common languages:')
print(df_language.head())
print('Number of languages:')
print(df_language.shape)
