def get(environment_id): with make_session() as session: data = session.query(Environment).filter( Environment.id == environment_id).first() # type: Environment if data is None: raise NotFound("Requested environment does not exist") return marshal(data[0], environment_fields)
def install(username, password): if app.config['DATABASE_PATH'] is not None: if os.path.exists(app.config['DATABASE_PATH']): confirm = raw_input( 'A database already exists. Remove it? [y/N] ').lower() if confirm != 'y': print 'Aborting install.' return app.logger.warning('Removing database.') db.make_session() db.init_db() admin = users.User(username, password) db.session.add(admin) db.session.commit()
def get(pack_id): with make_session() as session: data = session.query(Pack).filter( Pack.id == pack_id).first() # type: Pack if data is None: raise NotFound("Requested pack does not exist") return marshal(data, pack_fields)
def delete(pack_id): with make_session() as session: data = session.query(Pack).filter( Pack.id == pack_id).first() # type: Pack if data is None: raise NotFound("Requested pack does not exist") session.delete(data) return make_empty_response()
def patch(environment_id, name): with make_session() as session: data = session.query(Environment).filter( Environment.id == environment_id).first() # type: Environment if data is None: raise NotFound("Requested environment does not exist") data.name = name return make_empty_response()
def get(template_id, with_text): with make_session() as session: template = session.query(Template).filter( Template.id == template_id).first() # type: Template if template is None: raise NotFound("Requested template does not exist") return marshal( template, template_fields_with_text if with_text else template_fields)
def delete(environment_id): with make_session() as session: data = session.query(Environment).filter( Environment.id == environment_id).first() # type: Environment if data is None: raise NotFound("Requested environment does not exist") if data.in_use(): raise Conflict("Requested environment is in use") session.delete(data) return make_empty_response()
def load_template(filename: str): with open(filename) as file: template = yaml.load(file) tpl = Template(name=template['name'], text=template['text']) for varname, properties in template['variables'].items(): tpl.variables.append( Variable(name=varname, description=properties['description'])) with make_session() as session: session.add(tpl)
def delete(template_id): with make_session() as session: template = session.query(Template).filter( Template.id == template_id).first() # type: Template if template is None: raise NotFound("Requested template does not exist") for variable in template.variables: if variable.in_use(): raise Conflict( "Cannot delete the template because one or more variables are in use" ) session.delete(template) return make_empty_response()
def put(name, text): try: with make_session() as session: if session.query( session.query(Template).filter( Template.name == name.strip()).exists()).scalar(): raise Conflict( "A template with the same name already exists") template = Template(name=name.strip(), text=text) session.add(template) session.commit() return make_id_response(template.id) except IntegrityError: raise InternalServerError( "Could not create the requested template")
def patch(template_id, name, text, variables): with make_session() as session: template = session.query(Template).filter( Template.id == template_id).first() # type: Template if template is None: raise NotFound("Requested template does not exist") if variables != missing: if 'delete' in variables: for d in variables['delete']: to_delete = session.query(Variable).filter( Variable.id == d) if to_delete.first().in_use(): raise Conflict( 'Cannot delete variable because it is in use') to_delete.delete() if 'update' in variables: for u in variables['update']: if 'description' in u: variable = session.query(Variable) \ .filter(Variable.id == u['id']) \ .first() variable.description = u['description'].strip() if 'name' in u: if variable.name != u[ 'name'] and variable.in_use(): raise Conflict( 'Cannot change name of a variable that\'s in use' ) variable.name = u['name'] if 'create' in variables: for c in variables['create']: if 'description' in c: session.add( Variable(template=template, name=c['name'].strip(), description=c['description'].strip())) if text != missing: template.text = text if name != missing: template.name = name return make_id_response(template_id)
def get_history_for_entity(cls, limit=10, with_details=False): transaction = transaction_class(cls) transaction_changes = cls.__versioned__['transaction_changes'] with make_session() as session: transactions = (session .query(transaction) .join(transaction.changes) .filter(transaction_changes.entity_name.in_([cls.__name__])) .order_by(transaction.issued_at.desc()) .limit(limit) .all()) return [{ 'transaction_id': transaction.id, 'issued_at': transaction.issued_at.isoformat(), 'remote_address': transaction.remote_addr, 'changed_entities': _marshal_changed_entities(transaction, with_details) } for transaction in transactions]
def get(pack_name, environment_name): with make_session() as session: pack = session.query(Pack).filter( Pack.name == pack_name).first() # type: Pack if pack is None: raise NotFound('Requested pack not found') environment = session.query(Environment).filter( Environment.name == environment_name).first() # type: Environment if environment is None: raise NotFound('Requested environment not found') result = [] for template in pack.get_templates(): result.append({ 'name': template.name, 'text': instantiate(pack, template, environment) }) return result
def seed_data(): load_template('seeds/apache-vhost.template.yml') with make_session() as session: template = Template(name="test", text="whoa = {{xxx_var}};" " template={{_meta.template}};" " environment={{_meta.environment}};" " pack={{_meta.pack}}") session.add(template) variable = Variable(name="xxx_var", description="me gusta", template=template) session.add(variable) env = Environment(name="rotlicht") session.add(env) pack = Pack(name="fun") session.add(pack) instance = Instance(name="some/instance", pack=pack, template=template) value = Value(variable=variable, environment=env, instance=instance, data="injection") session.add(value)
import tarfile from pprint import pprint import time import gzip import sqlalchemy import os # Custom Imports sys.path.append('../database/') from db import make_session, Tweet, User # Parameters INFILE = '../../data/pablos_stuff/rnd_usr_sample/users-2016-10-19.json' # Set up sqlalchemy session session = make_session() with io.open(INFILE, 'r') as infile: tweet_count = 0 duplicates = 0 for i, line in enumerate(infile): try: user = json.loads(line) tweet_count += 1 except json.decoder.JSONDecodeError as e: print(e) continue uid = int(user['id_str'][0])
import db from db.models import User db.init_db() session = db.make_session() user = session.query(User).filter(User.name == '0000').first() if user is None: user = User(name='0000', password='******', is_admin=True) session.add(user) session.commit() else: print('user exists, change another username')
def get(): with make_session() as session: return marshal( session.query(Environment).all(), environment_fields)
def get_session(): session = db.make_session() try: yield session finally: session.close()
def get(template_id): with make_session() as session: return marshal( session.query(Variable).filter( Variable.template_id == template_id).all(), variable_fields)
def get(): with make_session() as session: return marshal(session.query(Template).all(), template_fields)
def put(name: str): environment = Environment(name=name.strip()) with make_session() as session: session.add(environment) session.commit() return make_id_response(environment.id)
def put(name): pack = Pack(name=name.strip()) with make_session() as session: session.add(pack) session.commit() return make_id_response(pack.id)
def get(): with make_session() as session: return marshal(session.query(Pack).all(), pack_fields)
outfile.write(outline.format('id', 'hit', 'hit_de', 'recaptured')) else: with io.open(OUTFILE, 'r') as outfile: id_generator.n_samples = file_len(outfile) - 1 id_generator.sampled_ids = get_sampled_ids(outfile) # Set up authentication and api acc = tc.credentials['coll_1'] auth = tweepy.OAuthHandler(acc['consumer_key'], acc['consumer_secret']) auth.set_access_token(acc['access_token'], acc['access_token_secret']) api = tweepy.API(auth, parser=tweepy.parsers.JSONParser()) # Get all ids from pablos sample ## Connect to sql engine _, engine = make_session('../../database/db_credentials') query = "SELECT id FROM users WHERE data_group = 'de_panel';" captured_1 = pd.read_sql_query(sql=query, con=engine) sample_1 = set(captured_1.id) limits = api.rate_limit_status() remaining = limits['resources']['users']['/users/lookup']['remaining'] for i in count(): # Sleep random time (Expectation 1 second == 900 requests per 15 minutes) time.sleep(1 + np.random.exponential(scale=0.1)) # Generate a batch of unique random ids id_batch = [str(a) for a in islice(id_generator, 100)]
from copy import copy from gensim import corpora from gensim.matutils import corpus2csc # Custom imports sys.path.append('../database') from db import make_session, Tweet, data_frame sys.path.append('/home/flinder/Dropbox/current_projects/text_utils') from text_utils import Cleaner, n_grams, url_regex # --------------------------------------------- # Data preprocessing # --------------------------------------------- # Database connection _, engine = make_session() # Get number of tweets by language query = ("SELECT count(*), lang " "FROM tweets " "WHERE data_group = 'de_panel' " "AND created_at >= '2015-01-01' " "AND created_at < '2016-01-01' " "GROUP BY lang " "ORDER BY count DESC; ") df_language = pd.read_sql(sql=query, con=engine) print('Most common languages:') print(df_language.head()) print('Number of languages:') print(df_language.shape)