Esempio n. 1
0
    def __init__(self, config, sgdb=None):
        Main._check_configuration(config)

        self.cli = CLI()
        self.config = config
        self.log = LOG(self.config.get("log_dir", None))

        self.sgdb = sgdb
        if self.sgdb is None and not self.config.get("new_migration", None):
            if self.config.get("database_engine") == 'mysql':
                from mysql import MySQL
                self.sgdb = MySQL(config)
            elif self.config.get("database_engine") == 'oracle':
                from oracle import Oracle
                self.sgdb = Oracle(config)
            elif self.config.get("database_engine") == 'mssql':
                from mssql import MSSQL
                self.sgdb = MSSQL(config)
            elif self.config.get("database_engine") == 'postgresql':
                from postgresql import PostgreSQL
                self.sgdb = PostgreSQL(config)
            else:
                raise Exception("engine not supported '%s'" % self.config.get("database_engine"))

        self.db_migrate = SimpleDBMigrate(self.config)
Esempio n. 2
0
def on_message(client, userdata, msg):
    print "Message"
    print(msg.topic + "\nAES Message:" + str(msg.payload))
    encryptedData = msg.payload
    db = PostgreSQL()
    encryptor = AESCipher(konfig.AESPassword)
    decryptedData = encryptor.decrypt(encryptedData)
    if "select" in decryptedData.lower():
        print "Sorgu:" + decryptedData + "\n"
Esempio n. 3
0
def query_db(query, args=(), one=False):
    with PostgreSQL(PARAMS) as cur:
        cur.execute(query, args)
        rv = cur.fetchall()
    return (rv[0] if rv else None) if one else rv
Esempio n. 4
0
# from flask_wtf import csrf
from flask_wtf.csrf import CSRFProtect

from urllib import parse, request as req
from postgresql import PostgreSQL
import jwt, json

load_dotenv(find_dotenv(), override=True)

UPLOAD_FOLDER = 'image'
ALLOWED_EXTENSIONS = set(['txt', 'pdf', 'png', 'jpg', 'jpeg', 'gif'])

app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['WTF_CSRF_CHECK_DEFAULT'] = False
db = PostgreSQL()


@app.route('/')
@app.route('/index', methods=['GET'])
def index():
    return 'Chill~~~, jemput aku di kos dong'


@app.route('/login', methods=['POST'])
def login():
    try:
        login_dictionary = {}

        if ('password' in request.form):
            login_dictionary['password'] = request.form['password']
Esempio n. 5
0
def num_rows_database(table):
    with PostgreSQL(PARAMS) as cur:
        query = 'select * from {}'.format(table)
        cur.execute(query)
        rv = cur.fetchall()
    return len(rv)
Esempio n. 6
0
def row_exists(tabel, title):
    with PostgreSQL(PARAMS) as cur:
        query = 'select exists(select 1 from {} where title=%s)'.format(tabel)
        cur.execute(query, (title, ))
        return cur.fetchone()[0]
Esempio n. 7
0
def database_manipulation(query, args=()):
    with PostgreSQL(PARAMS) as cur:
        cur.execute(query, args)
Esempio n. 8
0
from datetime import datetime
from collections import defaultdict
import tensorflow as tf
import os
import netCDF4 as nc
import numpy as np
import xarray as xr

from datetime import timedelta, datetime
from scipy.stats import percentileofscore

from postgresql import PostgreSQL
from data_helpers import load_embeddings_and_mapping, get_word_vector
from config import (SEQUENCE_LENGTH, EMBEDDING_DIM, SAMPLE_SETS)

pg = PostgreSQL('classification')


class BasinHydrology():
    def __init__(self,
                 name_rainfall_dataset,
                 data,
                 basin_id,
                 correct_rainfall=True,
                 discard_below_or_equal_to_value=-1):
        self.name_rainfall_dataset = name_rainfall_dataset
        self.precipitation = data
        self.basin_id = basin_id
        self.correct_rainfall = correct_rainfall
        self.discard_below_or_equal_to_value = discard_below_or_equal_to_value
Esempio n. 9
0
class Main(object):

    def __init__(self, config, sgdb=None):
        Main._check_configuration(config)

        self.cli = CLI()
        self.config = config
        self.log = LOG(self.config.get("log_dir", None))

        self.sgdb = sgdb
        if self.sgdb is None and not self.config.get("new_migration", None):
            if self.config.get("database_engine") == 'mysql':
                from mysql import MySQL
                self.sgdb = MySQL(config)
            elif self.config.get("database_engine") == 'oracle':
                from oracle import Oracle
                self.sgdb = Oracle(config)
            elif self.config.get("database_engine") == 'mssql':
                from mssql import MSSQL
                self.sgdb = MSSQL(config)
            elif self.config.get("database_engine") == 'postgresql':
                from postgresql import PostgreSQL
                self.sgdb = PostgreSQL(config)
            else:
                raise Exception("engine not supported '%s'" % self.config.get("database_engine"))

        self.db_migrate = SimpleDBMigrate(self.config)

    def execute(self):
        self._execution_log('\nStarting DB migration on host/database "%s/%s" with user "%s"...' % (self.config.get('database_host'), self.config.get('database_name'), self.config.get('database_user')), "PINK", log_level_limit=1)        
        if self.config.get("new_migration", None):
            self._create_migration()
        else:
            self._migrate()
        self._execution_log("\nDone.\n", "PINK", log_level_limit=1)

    @staticmethod
    def _check_configuration(config):
        if not isinstance(config, Config):
            raise Exception("config must be an instance of simple_db_migrate.config.Config")

        required_configs = ['database_host', 'database_name', 'database_user', 'database_password', 'database_migrations_dir', 'database_engine', 'schema_version']
        if config.get("new_migration", None):
            required_configs = ['database_migrations_dir']

        for key in required_configs:
            #check if config has the key, if do not have will raise exception
            config.get(key)

    def _create_migration(self):
        migrations_dir = self.config.get("database_migrations_dir")
        new_file = Migration.create(self.config.get("new_migration", None), migrations_dir[0], self.config.get("database_script_encoding", "utf-8"), self.config.get("utc_timestamp", False))
        self._execution_log("- Created file '%s'" % (new_file), log_level_limit=1)

    def _migrate(self):
        destination_version = self._get_destination_version()
        current_version = self.sgdb.get_current_schema_version()

        # do it!
        self._execute_migrations(current_version, destination_version)

    def _get_destination_version(self):
        label_version = self.config.get("label_version", None)
        schema_version = self.config.get("schema_version", None)

        destination_version = None
        destination_version_by_label = None
        destination_version_by_schema = None

        if label_version is not None:
            destination_version_by_label = self.sgdb.get_version_number_from_label(label_version)
            """
            if specified label exists at database and schema version was not specified,
            is equivalent to run simple-db-migrate with schema_version equals to the version with specified label
            """
            if destination_version_by_label is not None and schema_version is None:
                schema_version = destination_version_by_label
                self.config.update("schema_version", destination_version_by_label)

        if schema_version is not None and self.sgdb.get_version_id_from_version_number(schema_version):
            destination_version_by_schema = schema_version

        if label_version is None:
            if schema_version is None:
                destination_version = self.db_migrate.latest_version_available()
            elif destination_version_by_schema is None:
                destination_version = schema_version
            else:
                destination_version = destination_version_by_schema
        else:
            if schema_version is None:
                destination_version = self.db_migrate.latest_version_available()
            elif (destination_version_by_label is None) or (destination_version_by_schema == destination_version_by_label):
                destination_version = schema_version

        if (destination_version_by_schema is not None) and (destination_version_by_label is not None) and (destination_version_by_schema != destination_version_by_label):
            raise Exception("label (%s) and schema_version (%s) don't correspond to the same version at database" % (label_version, schema_version))

        if (schema_version is not None and label_version is not None) and ((destination_version_by_schema is not None and destination_version_by_label is None) or (destination_version_by_schema is None and destination_version_by_label is not None)):
            raise Exception("label (%s) or schema_version (%s), only one of them exists in the database" % (label_version, schema_version))

        if destination_version is not '0' and not (self.db_migrate.check_if_version_exists(destination_version) or self.sgdb.get_version_id_from_version_number(destination_version)):
            raise Exception("version not found (%s)" % destination_version)

        return destination_version

    def _get_migration_files_to_be_executed(self, current_version, destination_version, is_migration_up):
        if current_version == destination_version and not self.config.get("force_execute_old_migrations_versions", False):
            return []

        schema_migrations = self.sgdb.get_all_schema_migrations()

        # migration up
        if is_migration_up:
            available_migrations = self.db_migrate.get_all_migrations()
            remaining_migrations = Lists.subtract(available_migrations, schema_migrations)
            remaining_migrations_to_execute = [migration for migration in remaining_migrations if migration.version <= destination_version]
            return remaining_migrations_to_execute

        # migration down...
        destination_version_id = self.sgdb.get_version_id_from_version_number(destination_version)
        try:
            migration_versions = self.db_migrate.get_all_migration_versions()
        except:
            migration_versions = []
        down_migrations_to_execute = [migration for migration in schema_migrations if migration.id > destination_version_id]
        force_files = self.config.get("force_use_files_on_down", False)
        for migration in down_migrations_to_execute:
            if not migration.sql_down or force_files:
                if migration.version not in migration_versions:
                    raise Exception("impossible to migrate down: one of the versions was not found (%s)" % migration.version)
                migration_tmp = self.db_migrate.get_migration_from_version_number(migration.version)
                migration.sql_up = migration_tmp.sql_up
                migration.sql_down = migration_tmp.sql_down
                migration.file_name = migration_tmp.file_name

        down_migrations_to_execute.reverse()
        return down_migrations_to_execute

    def _execute_migrations(self, current_version, destination_version):
        """
        passed a version:
            this version don't exists in the database and is younger than the last version -> do migrations up until this version
            this version don't exists in the database and is older than the last version -> do nothing, is a unpredictable behavior
            this version exists in the database and is older than the last version -> do migrations down until this version

        didn't pass a version -> do migrations up until the last available version
        """

        is_migration_up = True
        # check if a version was passed to the program
        if self.config.get("schema_version"):
            # if was passed and this version is present in the database, check if is older than the current version
            destination_version_id = self.sgdb.get_version_id_from_version_number(destination_version)
            if destination_version_id:
                current_version_id = self.sgdb.get_version_id_from_version_number(current_version)
                # if this version is previous to the current version in database, then will be done a migration down to this version
                if current_version_id > destination_version_id:
                    is_migration_up = False
            # if was passed and this version is not present in the database and is older than the current version, raise an exception
            # cause is trying to go down to something that never was done
            elif current_version > destination_version:
                raise Exception("Trying to migrate to a lower version wich is not found on database (%s)" % destination_version)

        # getting only the migration sql files to be executed
        migrations_to_be_executed = self._get_migration_files_to_be_executed(current_version, destination_version, is_migration_up)

        self._execution_log("- Current version is: %s" % current_version, "GREEN", log_level_limit=1)

        if migrations_to_be_executed is None or len(migrations_to_be_executed) == 0:
            self._execution_log("- Destination version is: %s" % current_version, "GREEN", log_level_limit=1)
            self._execution_log("\nNothing to do.\n", "PINK", log_level_limit=1)
            return

        self._execution_log("- Destination version is: %s" % (is_migration_up and migrations_to_be_executed[-1].version or destination_version), "GREEN", log_level_limit=1)

        up_down_label = is_migration_up and "up" or "down"
        if self.config.get("show_sql_only", False):
            self._execution_log("\nWARNING: database migrations are not being executed ('--showsqlonly' activated)", "YELLOW", log_level_limit=1)
        else:
            self._execution_log("\nStarting migration %s!" % up_down_label, log_level_limit=1)

        self._execution_log("*** versions: %s\n" % ([ migration.version for migration in migrations_to_be_executed]), "CYAN", log_level_limit=1)

        sql_statements_executed = []
        for migration in migrations_to_be_executed:
            sql = is_migration_up and migration.sql_up or migration.sql_down

            if not self.config.get("show_sql_only", False):
                self._execution_log("===== executing %s (%s) =====" % (migration.file_name, up_down_label), log_level_limit=1)

                label = None
                if is_migration_up:
                    label = self.config.get("label_version", None)

                try:
                    self.sgdb.change(sql, migration.version, migration.file_name, migration.sql_up, migration.sql_down, is_migration_up, self._execution_log, label)
                except Exception, e:
                    self._execution_log("===== ERROR executing %s (%s) =====" % (migration.abspath, up_down_label), log_level_limit=1)
                    raise e

                # paused mode
                if self.config.get("paused_mode", False):
                    raw_input("* press <enter> to continue... ")

            # recording the last statement executed
            sql_statements_executed.append(sql)

        if self.config.get("show_sql", False) or self.config.get("show_sql_only", False):
            self._execution_log("__________ SQL statements executed __________", "YELLOW", log_level_limit=1)
            for sql in sql_statements_executed:
                self._execution_log(sql, "YELLOW", log_level_limit=1)
            self._execution_log("_____________________________________________", "YELLOW", log_level_limit=1)