def main():
try:
datasets_dir = os.environ['DATASETS_DIR']
except KeyError:
datasets_dir = '../../../../'
print('running etl...')
country_measure, country_ent = run(DOCID, SHEET_COUNTRY, ['geo', 'time'],
None, COLUMN_TO_CONCEPT, OUT_DIR)
region_measure, region_ent = run(DOCID, SHEET_REGION, ['geo', 'time'],
{'geo': 'world_4region'},
COLUMN_TO_CONCEPT, OUT_DIR)
global_measure, global_ent = run(DOCID, SHEET_GLOBAL, ['geo', 'time'],
{'geo': 'global'}, COLUMN_TO_CONCEPT,
OUT_DIR)
measures_df = pd.concat([country_measure, region_measure, global_measure],
ignore_index=True)
open_numbers_df = pd.read_csv(
osp.join(datasets_dir, 'open-numbers/ddf--open_numbers/',
'ddf--concepts.csv'))
(pd.concat([measures_df, open_numbers_df], ignore_index=True,
sort=True).drop_duplicates().to_csv(osp.join(
OUT_DIR, 'ddf--concepts.csv'),
index=False))
for ent_file in glob.glob(
osp.join(datasets_dir, 'open-numbers/ddf--open_numbers/',
'ddf--entities--geo--*.csv')):
shutil.copy(ent_file, OUT_DIR)
def gcloud_dns_list(zone): r = lib.run( [ 'gcloud', 'dns', 'record-sets', 'list', '-z', zone, '--format', 'json' ], stdout=subprocess.PIPE ) r = lib.attrconvert(json.loads(r.stdout)) return r
def copy_objdir_files(dest_dir, config):
for d in open(os.path.join(config['index_path'], 'objdir-dirs')).readlines():
d = d.strip()
run(['mkdir', '-p', os.path.join(dest_dir, d)])
paths = open(os.path.join(config['index_path'], 'objdir-files')).readlines()
for path in paths:
path = path.strip()
source = path.replace('__GENERATED__', config['objdir_path'])
try:
data = open(source).read()
except:
continue
dest = os.path.join(dest_dir, path)
f = open(dest, 'w')
f.write(data)
f.close()
def _update(dataset, unit, message):
"""
Update repo with given unit of dataset
"""
conf = get_config()
if dataset not in conf['datasets']:
print("Unknown datasets. Please use 'show' to see the available datasets")
return
workingdir = os.path.join(datadir, dataset)
detail = conf['datasets'][dataset]
try:
with cwd(workingdir):
# Add the unit to the
print(os.getcwd())
cmd = ['dvc', "add", "data/{}".format(unit)]
run(cmd)
cmd = ['git', "add", "datasets"]
run(cmd)
if message is None:
message = "Automated commit of the dataset update"
cmd = ['git', "commit", '-a', '-m', message]
run(cmd)
cmd = ['git', "push", "origin"]
run(cmd)
#cmd = ["dvc", "push"]
#run(cmd)
except:
traceback.print_exc()
#}debug
else:
print(message)
if __name__ == '__main__':
import os
if len(sys.argv) > 1:
fpname = sys.argv[1]
if '/' not in fpname:
fpname = 'recount/' + fpname
else:
fpname = None
reset()
lib.run('skada.js', conf, on_message)
try:
while 1:
input()
if fout:
s = summ()
sys.stderr.write(s)
fwrite(fout, '"%s"' % s)
fout.close()
sys.stderr.write('[+] fclose\n')
fout = None
reset()
except:
sys.stderr.write(summ())
exit()
import os
import os.path
import sys
import subprocess
import json
import collections
from lib import run
config = json.load(open(sys.argv[1]))
tree_name = sys.argv[2]
repo_path = config['trees'][tree_name]['files_path']
stdout = run('git ls-files --recurse-submodules', shell=True, cwd=repo_path)
lines = stdout.split('\n')
# Comm-central has a mozilla/ subrepo
if tree_name == 'comm-central':
stdout = run('git ls-files',
shell=True,
cwd=os.path.join(repo_path, 'mozilla'))
lines2 = stdout.split('\n')
lines2 = ['mozilla/' + f for f in lines2 if f]
lines = lines + lines2
files = []
js = []
idl = []
ipdl = []
import os
import os.path
import sys
import subprocess
import json
import collections
from lib import run
config = json.load(open(sys.argv[1]))
tree_name = sys.argv[2]
repo_path = config['trees'][tree_name]['files_path']
stdout = run('git ls-files', shell=True, cwd=repo_path)
lines = stdout.split('\n')
files = []
js = []
idl = []
ipdl = []
dirs = collections.OrderedDict()
ipdl_dirs = collections.OrderedDict()
for line in lines:
path = line.strip()
if not path:
continue
#!D:\ProgramFilesx86\Python35-32\python.exe #$Header: //MyDataDepot/Projects/repeatability/site/cgi-bin/ShowFormFields/run.py#1 $ import sys sys.path.insert(0, '..') from lib import run fmt = b"Content-type: text/plain; charset=utf-8\n\n" cmd = "java -cp . com.repeatability.dblp.ShowFormFields" run(fmt, cmd)
import sys
import conf
import lib
if len(sys.argv) > 1:
mod = sys.argv[1]
if '.js' not in mod:
mod = 'mod/%s.js' % mod
lib.run(mod, conf)
sys.stdin.read()
import os
import os.path
import sys
import subprocess
import json
import collections
from lib import run
config = json.load(open(sys.argv[1]))
tree_name = sys.argv[2]
repo_path = config['trees'][tree_name]['files_path']
if tree_name == 'comm-central':
files1 = run('hg locate', shell=True, cwd=repo_path)
lines1 = files1.split('\n')
files2 = run('hg locate',
shell=True,
cwd=os.path.join(repo_path, 'mozilla'))
lines2 = files2.split('\n')
lines2 = ['mozilla/' + f for f in lines2 if f]
lines = lines1 + lines2
else:
stdout = run('git ls-files', shell=True, cwd=repo_path)
lines = stdout.split('\n')
files = []
js = []
import os
import os.path
import sys
import subprocess
import json
import collections
from lib import run
config = json.load(open(sys.argv[1]))
tree_name = sys.argv[2]
repo_path = config['trees'][tree_name]['files_path']
stdout = run('git ls-files', shell=True, cwd=repo_path)
lines = stdout.split('\n')
# Comm-central has a mozilla/ subrepo
if tree_name == 'comm-central':
stdout = run('git ls-files',
shell=True,
cwd=os.path.join(repo_path, 'mozilla'))
lines2 = stdout.split('\n')
lines2 = ['mozilla/' + f for f in lines2 if f]
lines = lines + lines2
files = []
js = []
idl = []
ipdl = []
from lib import run from config import Conf import router run(host=Conf.HOST, port=Conf.PORT, debug=Conf.DEBUG, reloader=Conf.RELOAD)
config_fname = sys.argv[1]
livegrep_config = {
'name': 'Searchfox',
'repositories': [],
'fs_paths': [],
}
config = json.load(open(config_fname))
repos = config['trees']
for key in repos:
repo_name = key
if 'git_path' in repos[key]:
run(['ln', '-s', repos[key]['git_path'], '/tmp/dummy/%s' % key])
livegrep_config['repositories'].append({
'name': key,
'path': repos[key]['git_path'],
'revisions': ['HEAD']
})
else:
run(['ln', '-s', repos[key]['files_path'], '/tmp/dummy/%s' % key])
# If we don't include the trailing '/', then all search
# results will include an initial slash in their paths.
livegrep_config['fs_paths'].append({
'name': key,
'path': repos[key]['files_path'] + '/'
})
def fig2():
"""Generate th raster plot"""
spikes = run(ext_w=0.2, inh_w=0.2, linear=True)
spikes_nl = run(ext_w=0.2, inh_w=0.2, linear=False)
pop(spikes, save="spikes_l.png")
pop(spikes_nl, save="spikes_nl.png")
def gcloud_dns_txn(zone, op, *args): r = lib.run( [ 'gcloud', 'dns', 'record-sets', 'transaction', op, '-z', zone, *args ] ) return r
below = 1000000
max_n = 1
max_ratio = 0
# Fastest way: using n/phi(n) = PI(p|n, p / p - 1), all p have to be smallest as possible to maximum n/phi
m = 1
for n in prime.primes_above(1):
if m * n > below:
return m
m *= n
# Brute force
for n in xrange(2, below):
phi = n
for p in lib.integer_factorization(n).keys():
phi /= p
phi *= p - 1
#print n, phi
ratio = 1.0 * n / phi
if ratio > max_ratio:
max_n = n
max_ratio = ratio
return max_n
#profile.run('e60()')
lib.run(e60)
config = json.load(open(sys.argv[2]))
tree_name = sys.argv[3]
# Dynamically import the repo_files.py script from the tree's scripts in the
# config repo.
try:
repo_files = runpy.run_path(
os.path.join(config_repo, tree_name, 'repo_files.py'))
except FileNotFoundError:
# For simplicity allow the tree config to not have the script, in which case
# we fall back to some default behaviour.
repo_files = {}
tree_config = config['trees'][tree_name]
tree_repo = tree_config['files_path']
lines = run(['git', 'ls-files', '--recurse-submodules'],
cwd=tree_repo).splitlines()
if 'modify_file_list' in repo_files:
lines = repo_files['modify_file_list'](lines, config=tree_config)
files = []
js = []
idl = []
ipdl = []
dirs = collections.OrderedDict()
ipdl_dirs = collections.OrderedDict()
for line in lines:
path = line.strip()
if not path:
def modify_file_list(files, config=None, **kwargs):
# Also grab the file list from the mozilla/ subrepo
subrepo_path = os.path.join(config['files_path'], 'mozilla')
sub_files = run(['git', 'ls-files'], cwd=subrepo_path).splitlines()
sub_files = [b'mozilla/' + f for f in sub_files if f]
return files + sub_files
l.info(f"measurement: {cfg.measurement}")
cfg.subnet = IPv4(cfg.subnet)
l.info(f"local subnet: {cfg.subnet}")
l.info(f"unresolved local aux mappings: {cfg.aux}")
cfg.subnets_addn = [IPv4(s) for s in cfg.get("subnets_addn", [])]
l.info(f"additional local subnets: {cfg.subnets_addn}")
cfg.numeric_dst_hosts = [IPv4(s) for s in cfg.get("numeric_dst_hosts", [])]
l.info(f"not resolving dst-address for hosts: {cfg.numeric_dst_hosts}")
l.info(f"influxdb instance from {cfg.influx.uri}")
l.info(f"influxdb raw data RP: {cfg.influx.rp_raw}")
l.info(
f"influxdb aggregated data RPs: {[ x.rp for x in cfg.influx.aggregations ]}"
)
if os.isatty(sys.stderr.fileno()):
cfg_highlight_on = lib.run(["tput", "bold"], stdout=subprocess.PIPE).stdout
cfg_highlight_off = lib.run(["tput", "sgr0"],
stdout=subprocess.PIPE).stdout
else:
cfg_highlight_on = ""
cfg_highlight_off = ""
if "verbose" in cfg:
if cfg.verbose:
l.setLevel(logging.DEBUG)
else:
if os.isatty(sys.stderr.fileno()):
l.setLevel(logging.DEBUG)
# Up to this point, we did not perform any meaningful things.
# Check if we actually have any flows to report.
import os
import os.path
import sys
import subprocess
import json
import collections
from lib import run
config = json.load(open(sys.argv[1]))
tree_name = sys.argv[2]
repo_path = config['trees'][tree_name]['files_path']
if tree_name == 'comm-central':
files1 = run('hg locate', shell=True, cwd=repo_path)
lines1 = files1.split('\n')
files2 = run('hg locate', shell=True, cwd=os.path.join(repo_path, 'mozilla'))
lines2 = files2.split('\n')
lines2 = [ 'mozilla/' + f for f in lines2 if f ]
lines = lines1 + lines2
else:
stdout = run('git ls-files', shell=True, cwd=repo_path)
lines = stdout.split('\n')
files = []
js = []
idl = []
ipdl = []
import lib
import evo
import packer
import requests as rq
ind1 = lib.run()
Evo = evo.Evo(200)
ind2 = Evo.run()
Packer = packer.Packer()
payload = Packer.pack(ind1, ind2)
response = rq.post("https://cit-home1.herokuapp.com/api/ga_homework",
json=payload)
print(response.content)
