def execute(self, reader):
if len(self.params) < 1:
die("Graph requires units [--graph file.csv units by f1]")
val = self.params[1].lower().strip()
if len(self.params) > 3 and self.params[2].lower() == "by":
key = self.params[3].lower()
else:
die("by flag required")
headers = self.clean_headers(reader.headers)
self.in_headers([key],headers)
self.in_headers([val],headers)
data = {}
for line in reader:
try:
value = float(line[headers[val]])
except:
print val + ' not a numerical field at line: '+ str(reader.count+1)
continue
lkey = line[headers[key]]
if lkey not in data:
data[lkey] = value
else:
data[lkey] = data[lkey] + value
print self.get_url(data)
def execute(self,reader):
if len(self.params) < 2:
die("Aggregation requires a field list [--aggregate file.csv units by f1,f2]")
agg = self.params[1].lower().strip()
if len(self.params) > 3 and self.params[2].lower() == "by":
keys = map( lambda x: x.lower().strip(), self.params[3].split(','))
else:
keys = []
headers = self.clean_headers(reader.headers)
self.in_headers([agg],headers)
self.in_headers(keys,headers)
counts = {}
for line in reader:
if line is None : continue
try:
incr = float( line[headers[agg]] )
except ValueError:
print agg + ' not a numerical field at line: '+ str(reader.count+1)
continue
key = self.get_keys_hash(keys, line, headers)
if key not in counts:
counts[key] = incr
else:
counts[key] = counts[key] + incr
keys.append(agg)
print ", ".join(keys)
for key in counts:
print key + str(counts[key])
def buildReader(self, path, masklist):
if os.path.isfile(path) is False:
die("Invalid input file at: "+path)
masks = []
for mask in masklist:
masks.append( Mask.build(mask, masklist[mask]) )
return Reader(path, masks)
def H_ERROR(self, keys, headers):
error = 'Field list not included in Header set\n'
error += "Input:\n"
error += "\t"+str(keys)+"\n"
error += "Headers:\n"
for x in headers:
error += "\t'" + x + "'\n"
die(error)
def make_zip(_file):
# type: (string) -> bool
zf = ZipFile(_file + '.zip', mode='w')
try:
zf.write(_file + '.xml', compress_type=ZIP_DEFLATED)
except RuntimeError:
die('Ошибка архивации файлов [' + _file + '] при подготовке файлов к обмену')
finally:
zf.close()
return True
def make_import(_exchange_url, _exchange_path, _login, _password):
try:
start_time = time.time()
print colored(
'Запущен обмен ' +
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + '\n',
'blue')
print 'Точка обмена: ' + colored(exchange_url + exchange_path,
'blue') + '\n'
print 'Логин: ' + colored(login, 'blue') + '\n\nПароль: ' + colored(
password, 'blue') + '\n'
print 'Рабочая директория: ', colored(os.getcwd(), 'blue') + '\n'
files_for_send = get_exchange_files()
print 'Обнаруженные файлы для импорта: ', colored(
files_for_send, 'blue'), '\n'
for file_for_upload in files_for_send:
exchange_begin_time = time.time()
print colored('Начало импорта [' + file_for_upload + ']', 'green')
session_id = step1(_exchange_url, _exchange_path, _login,
_password)
res = step2(_exchange_url, _exchange_path, session_id)
if is_exists_directories_with_files_for_upload(
) and res['zip'] != 'yes':
die('Присутствуют папки с файлами, их можно загрузить только архивом.\nПроверьте настройки компонента'
)
if res['zip'] == 'yes':
file_for_upload += '.zip'
else:
file_for_upload += '.xml'
step3(_exchange_url, _exchange_path, session_id, file_for_upload)
step4(_exchange_url, _exchange_path, session_id,
file_for_upload.replace('.zip', '.xml'))
print colored(
'Конец импорта.\n' + ("Время выполнения - %s секунд(ы)\n" %
(time.time() - exchange_begin_time)),
'green')
print colored(
'Обмен завершен ' +
datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + '\n' +
("Время выполнения - %s секунд(ы)\n" % (time.time() - start_time)),
'blue')
return True
except requests.exceptions.ConnectionError:
print colored(
'Ошибка подключения к точке обмена. Проверьте параметры подключения.\nОбмен прерван',
'red')
return False
def check_backup_files(self, backup_files):
# check backup_files parameter
if type(backup_files) != list:
print('2nd parameter must be a list!')
die()
for i in backup_files:
if not path.isfile(i):
print('Backup path does not exist:')
print(i)
die()
def check_timestamp(self, timestamp):
# check timestamp parameter
error_text = """3rd parameter should be a string number list following the format ['HHMM']\nHH: hour\nMM: minute\ne.g. ['1200', '1300', '1400', '1500', '1600', '1700', '1800', '1900']"""
for i in timestamp:
if bool(re.search("\d{4}", i)) and len(i) == 4 and int(i[:2]) <= 23 and int(i[2:]) <= 59:
pass
else:
print(error_text)
die()
timestamp = unique_list(timestamp)
timestamp.sort()
def create_root_backup_folder(self, backup_drive):
# create BACKUPS and log folder (check backup_drive parameter)
if not re.search("^[A-Z]$", backup_drive):
print('1st parameter must be a single uppercase character!')
die()
else:
self.backup_drive = backup_drive
try:
makedirs(self.backup_drive + r':\BACKUPS\log')
except:
pass
def get_url(self, data):
url = "https://chart.googleapis.com/chart?chs=500x200&cht=lc&chds=a&chd=t:"
vals = ""
if len(data) < 2:
die("Need 2+ values to graph, investigate with aggregation\n"+str(data))
for key in sorted(data.iterkeys()):
vals = vals + str(data[key]) + ","
url = url + vals[:len(vals)-1]
conn = httplib.HTTPSConnection("www.googleapis.com")
headers = {'Content-Type': 'application/json'}
params = '{"longUrl": \"'+url+'\"}'
conn.request("POST","/urlshortener/v1/url",params,headers)
return json.loads(conn.getresponse().read())['id']
def step2(_exchange_url, _exchange_path, _cookie):
# type: (string, string) -> dict
print colored('Шаг #2: Инициализация', 'blue')
params = urllib.urlencode({'type': 'catalog', 'mode': 'init'})
url = 'http://' + _exchange_url + _exchange_path + '?' + params
result = requests.get(url, headers={'Cookie': _cookie}).text
print result + '\n'
# выбираем полученные парметры
temp = findall(r"[\w]+", result)
# проверка что параметры получены
if temp[0] != 'zip' and temp[2] != 'file_limit':
die('Ошибка инициализации\nОбмен завершен')
else:
print 'Инициализация выполнена успешно\n'
# Сохранение полученных параметров
return dict(zip=temp[1], file_limit=temp[3])
def is_masked(self, line, rheaders):
if len(self.params) < 1:
die("Unique mask requires parameters [--unique p1,p2,p3]")
parameters = map(lambda x: x.lower(), self.params[0].split(','))
headers = {}
for k,v in enumerate(rheaders):
headers[v.lower()] = rheaders[v]
out = ""
for param in parameters:
try:
val = line[headers[param]].lower()
except KeyError:
die("Bad parameters: \n"+str(parameters))
out = out + val
if out in self.set:
return True
else:
self.set.add(out)
return False
def step1(_exchange_url, _exchange_path, _login, _password):
# type: (string, string, string, string, string) -> string
print colored('Шаг #1: Авторизация', 'blue')
params = urllib.urlencode({'type': 'catalog', 'mode': 'checkauth'})
url = 'http://' + _exchange_url + _exchange_path + '?' + params
auth = 'Basic ' + string.strip(base64.encodestring(_login + ':' + _password))
r = requests.get(url, headers={'Authorization': auth}).text
print r + '\n'
temp = r.split('\n')
if temp[0] != 'success':
print colored('Авторизация провалена\nОбмен завершен ' + datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
'blue')
die('Авторизация провалена\nОбмен завершен')
else:
print 'Авторизация выполнена успешно\n'
# Идентификатор сессии
return temp[1] + '=' + temp[2]
def getArgs(self, args):
cmds = self.get_cmds(args)
if len(cmds) < 1 or cmds.keys()[0].startswith('h'):
die(Strat.Help())
nargs = self.build_args(cmds)
if len(nargs["strategy"]) != 1:
die("Exactly ONE valid execution flag is required, see usage (-h)")
strat = nargs["strategy"].keys()[0]
if len(nargs["strategy"][strat]) < 1:
die("File path required for this execution flag, see usage (-h)")
nargs["file"] = nargs["strategy"][strat][0]
return nargs
def __init__(self):
user = getpass.getuser()
self.directory=[]
if 'root' in user:
print("This cannot be run as root, run as your regular user")
die(1)
elif path.exists('.dockercompilesetupcomplete'):
print("It appears that you've already run the setup for this once before. Do you want to run it again?")
rerun = raw_input("(y/N) ")
if "y" in rerun.lower():
self.dirsource()
self.dircompile()
self.dockerrun(user, "first")
elif "n" in rerun.lower():
print("Setup is now exiting. Be Well " + user)
die(0)
elif len(rerun) == 0:
print("Setup is now exiting. Be Well " + user)
die(0)
else:
self.dirsource()
self.dircompile()
self.dockerrun(user, "first")
def is_masked(self, line, rheaders):
if len(self.params) < 1:
die("Filter mask requires parameters [--filter p1=regex*1,p2=[regex]+2]")
parameters = self.params[0].split(',')
headers = {}
for k,v in enumerate(rheaders):
headers[v.lower()] = rheaders[v]
for param in parameters:
bits = param.split('=')
col = bits[0].lower()
reg = bits[1].lower()
if not col in headers:
die(col + " not in Header set")
val = line[headers[col]].lower()
# print val + " " + regex
try:
regex = re.compile(reg.lower())
except:
die("Bad regex: "+reg)
if re.search(regex, val) is None:
return True
return False
def execute(self,reader):
die("no strategy")
def open(self, path):
try:
return open(path, 'rU')
except:
die("Bad file path" + path)
import contextlib
import datetime
import os
import string
import time
import urllib
from re import findall
import sys
from sys import exit as die
from sys import stderr
from zipfile import ZipFile, ZIP_DEFLATED
try:
import requests
except ImportError:
die('В системе не установлен модуль requests\nДля установки выполните команду\npip install requests'
)
try:
from termcolor import colored
except ImportError:
die('В системе не установлен модуль termcolor\nДля установки выполните команду\npip install termcolor'
)
class UploadInChunks(object):
def __init__(self, filename, chunksize=1 << 13):
self.filename = filename
self.chunksize = chunksize
self.totalsize = os.path.getsize(filename)
self.readsofar = 0
prefix = getopt.getopt(argv[1:], None, ['prefix='])[0][1][1]
except:
prefix = '/usr/local'
# The Get?ID's Get?StructByName functions return a structure of
# useless (to us) information and one useful thing: the ?ID at the subscript
# [2].
def GetGIDByName(name):
return GetGroupStructByName(name)[2]
def GetUIDByName(name):
return GetUserStructByName(name)[2]
# This needs to be run by root, and this tests if it is running as root.
if GetProcessUID() != GetUIDByName('root'):
die("This must be run by root!\nQuitting...")
# Verifies that we have the correct versions of the correct libraries.
try:
import sqlalchemy
if '0.6' not in sqlalchemy.__version__:
die("Your SQLAlchemy is too old or too new!\nYou need 0.6.0 or newer \
but older then 0.7.0.")
except ImportError:
die("You need SQLAlchemy 0.6.0 or newer but a older version then 0.7.0.")
import pygtk
try:
pygtk.require('2.0')
except AssertionError:
die()
#!/usr/bin/env python
import sqlalchemy, subprocess, time, sqlalchemy.orm, gtk
from sqlalchemy.ext.declarative import declarative_base as _Base
from sys import exit as die
#Checks to see if the SQLAlchemy version is in the right range.
if "0.6" not in sqlalchemy.__version__:
die("Your SQLAlchemy is too old or too new!\nYou need 0.6.0 or newer \
but older then 0.7.0.")
#Converts the magic function to the magic class.
DeclarativeBase = _Base()
# This flag gets funged by the installer to denote the install status of the
# program.
installed = False
if not installed:
die("This needs to be installed before you can use it.")
engine = sqlalchemy.create_engine(
'sqlite:///%s/share/btrfsguitools/snapshot.db' % (prefix))
DBSession = sqlalchemy.orm.scoped_session(sqlalchemy.orm.sessionmaker(
bind=engine))
class SQLAlchemyMagic(DeclarativeBase, object):
"Uses SQLAlchemy's declarative extension to map a database to a Python" + \
" class in order to store btrfs snapshots."
# Sets up the table.
__tablename__ = "snapshots"
def main():
parser = ArgumentParser(
description=
'Sign up for all Carleton email lists. You must have a list of lists email in your inbox, unless run with the --with-new-lists option.'
)
parser.add_argument(
'-r',
'--request-lists',
action='store_true',
help='Just request the list of lists email and do nothing else.')
parser.add_argument(
'-n',
'--with-new-lists',
action='store_true',
help=
'Request the list of lists email, and then wait for a response and subscribe to all.'
)
parser.add_argument('-u',
'--username',
nargs=1,
metavar='USERNAME',
help='Gmail address (e.g. [email protected])')
parser.add_argument('-p',
'--password',
nargs=1,
metavar='PASSWORD',
help='Gmail password')
parser.add_argument(
'-c',
'--creds-file',
nargs=1,
metavar='CREDS_FILE',
help='JSON document with keys "username" and "password"')
args = parser.parse_args()
if args.creds_file is not None:
with open(args.creds_file[0], 'r') as f:
creds = json.load(f)
username = creds['username']
password = creds['password']
elif args.username and args.password:
username = args.username[0]
password = args.password[0]
else:
sys.exit(
'You must specify a --creds-file or a --username and --password')
subscriber = Subscriber(username, password)
if args.request_lists and args.with_new_lists:
sys.die('Choose at most one of --request-lists and --with-new-lists')
if args.request_lists:
subscriber.request_lists()
elif args.with_new_lists:
lists = subscriber.new_lists()
subscriber.subscribe(lists)
else:
lists = subscriber.get_lists()
if lists is None:
sys.die('Run with --request-lists first')
subscriber.subscribe(lists)
import corr, adc5g, httplib
import matplotlib.pyplot as plt
import numpy as np
import scipy.optimize
import sys, time
if len(sys.argv) < 3:
sys.die('Usage: python ./collect_mark6_data.py <numberOfSnapshots> <dataName>')
r2 = corr.katcp_wrapper.FpgaClient('r2dbe-1')
r2.wait_connected()
rpt = int(sys.argv[1])
for r in range(rpt):
# time.sleep(1)
#x0 correspons to if0 and x1 corresponds to if1
x0 = np.array(adc5g.get_snapshot(r2, 'r2dbe_snap_8bit_0_data'))
x1 = np.array(adc5g.get_snapshot(r2, 'r2dbe_snap_8bit_1_data'))
if r > 0:
x0full = np.column_stack((x0full, x0))
x1full = np.column_stack((x1full, x1))
else:
x0full = x0
x1full = x1
print x0full.shape
np.save('dataSamp_' + sys.argv[2] + '_if0full.npy', x0full)
#!/usr/bin/env python
from __future__ import print_function
from sys import argv, stderr, exit as die
import pip
import json
try:
from sql_util import read, readWithParams, executeQuery, executeWithParams, createQuery
import mysql.connector
from flask import Flask, jsonify, request, session, g, redirect, url_for, abort, render_template, flash
except:
print("Missing requirements\n", stderr)
die(-1)
app = Flask(__name__)
app.config.from_object(__name__)
apiJsonObject = None
queryJsonObject = None
@app.route('/')
def home():
return render_template('index.html')
@app.route('/about')
def about():
return render_template('about.html')
@app.route('/hateabase/api/v1.0/insert/', methods=["POST"])
def insertOffenses():
params = request.form.getlist('params', type = str)
SQL = ("INSERT INTO OFFENSES "
