def __init__(self, config, auth, shared_storage):
self.logger = logging.getLogger()
self.config = config
self.shared_storage = shared_storage
self.symbol = self.config.symbol
self.auth = auth
self.subscribed = False
self.orders_received = False
self.events = {
"order-received": self.process_new_received,
"modify-received": self.process_amend_received,
"cancel-received": self.process_cancel_received,
"accepted": self.process_accept,
"rejected": self.process_new_rejection,
"modify-rejected": self.process_amend_rejection,
"canceled": self.process_elim,
"cancel-rejected": self.process_elim_reject,
"filled": self.process_fill,
}
if self.symbol is None:
self.config.symbol = None
elif type(self.config.symbol) is not list:
self.config.symbol = [self.config.symbol]
def setup_custom_logger(name):
formatter = logging.Formatter(
fmt='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger(name)
logger.setLevel(logging.DEBUG)
return logger
def __init__(self, cfg, exchange_adapter):
self.logger = logging.getLogger()
self._load_configuration(cfg)
self.config = cfg
self.exchange_adapter = exchange_adapter
self.exchange_adapter.set_order_update_callback(self.on_market_update)
self.orders_manager = orders_manager(self.exchange_adapter)
if self.cancel_orders_on_start is True:
self.exchange_adapter.cancel_orders_on_start = True
else:
self.exchange_adapter.cancel_orders_on_start = False
self.update_orders = False
self.started_time = time.time()
self.last_amend_time = None
self.reconnecting = False
self.active = True
self.tob = None
self.num_of_sent_orders = 0
self.cancel_all_request_was_sent = False
self.user_asks = self.config.orders.asks
self.user_bids = self.config.orders.bids
def __init__(self, exchange_adapter):
self.exchange_adapter = exchange_adapter
try:
self.exch_name = self.exchange_adapter.config.name
except AttributeError:
self.exch_name = ""
self.orders = {}
self.live_orders_ids = []
self.orders_states = {}
self.orderid_to_orderid_map = {}
self.order_to_event = {}
self.order_to_strategy_type = {}
self.ids_to_fills = {}
self.update_type_to_state = {
new_order_ack: event.on_insert_ack,
new_order_nack: event.on_insert_nack,
order_elim_ack: event.on_cancel_ack,
order_elim_nack: event.on_cancel_nack,
order_fill_ack: event.on_fill,
order_full_fill_ack: event.on_full_fill,
amend_ack: event.on_amend_ack,
amend_ack_on_partial: event.on_amend_partial_ack,
amend_nack: event.on_amend_nack,
}
self.logger = logging.getLogger()
def __init__(self):
self.logger = logging.getLogger('db.py')
self.con = MySQLdb.connect(config.get_db("host"),
config.get_db("user"),
config.get_db("pass"),
config.get_db("database"),
charset="utf8")
self.cursor = self.con.cursor()
def __init__(self):
self.INSERT_PRICE = "insert into stock_price_history (`code`,`open` ,`close` ,`price` ,`high`,`low`,`buy_price` ,`sell_price`,`deal_number`,`deal_money` ,`buy_one_number`,`buy_one_price`,`buy_two_number`,`buy_two_price`,`buy_three_number`,`buy_three_price`,`buy_four_number`,`buy_four_price`,`buy_five_number` ,`buy_five_price` ,`sell_one_number` ,`sell_one_price` ,`sell_two_number`,`sell_two_price` ,`sell_three_number` ,`sell_three_price` ,`sell_four_number`,`sell_four_price` ,`sell_five_number` ,`sell_five_price`,`date`,`time`,`other`) values (%s) on duplicate key update `other`='00'"
self.INSERT_TODAY_PRICE = "insert into stock_price_newest (`code`,`open` ,`close` ,`price` ,`high`,`low`,`buy_price` ,`sell_price`,`deal_number`,`deal_money` ,`buy_one_number`,`buy_one_price`,`buy_two_number`,`buy_two_price`,`buy_three_number`,`buy_three_price`,`buy_four_number`,`buy_four_price`,`buy_five_number` ,`buy_five_price` ,`sell_one_number` ,`sell_one_price` ,`sell_two_number`,`sell_two_price` ,`sell_three_number` ,`sell_three_price` ,`sell_four_number`,`sell_four_price` ,`sell_five_number` ,`sell_five_price`,`date`,`time`,`other`) values (%s) on duplicate key update `other`='00'"
self.TRUNCATE_TODAY_PRICE = "truncate table stock_price_newest"
self.UPDATE_NAME = "update stock set name='%s' where code='%s'"
self.stock_url = "http://hq.sinajs.cn/list="
self.logger = logging.getLogger('price.py')
reload(sys)
sys.setdefaultencoding('utf-8')
def addNetworkHandler(comm, boxinfo):
# add in the beanstalk logger if applicable
from logger import logging
tlog = logging.getLogger("pipeline")
network_handler = NetworkHandler(comm, boxinfo)
network_handler.setLevel(logging.INFO)
network_handler.setFormatter(log_formatter)
tlog.addHandler(network_handler)
def __init__(self, config, auth, ws, shared_storage):
self.logger = logging.getLogger()
self.config = config
self.ws = ws
self.shared_storage = shared_storage
self.auth = auth
self.headers = {'content-type': 'application/json'}
def __init__(self):
self.count = 0
self.buildlocks = {}
self.idlock = Lock()
self.rsynclock = Lock()
self.mod_buildlocks = Lock()
self.logger = logging.getLogger('pipeline').getChild('locks')
def __init__(self):
self.msg_callback = None
self.stop = False
self.last_hb_time = time.time()
self.cancel_orders_on_start = False
self.started = False
self.reconnecting = False
self.ready_to_listen = asyncio.Event()
self.logger = logging.getLogger()
def addNetworkHandler(app, name):
"""
Create the network based log handler. It takes a celery app and a unique
log-producer name (such as worker-<UUID>-0) and puts the message on a
queue to gateway so that they are stored and displayed
"""
# add in the celery logger if applicable
from logger import logging
tlog = logging.getLogger("pipeline")
network_handler = NetworkHandler(app, name)
network_handler.setLevel(logging.INFO)
network_handler.setFormatter(log_formatter)
tlog.addHandler(network_handler)
def check_paths():
logger = logging.getLogger()
if not path.exists(_CONF_DIR):
mkdir(_CONF_DIR)
logger.debug('Creata la cartella "%s".' % _CONF_DIR)
if not path.exists(OUTBOX):
mkdir(OUTBOX)
logger.debug('Creata la cartella "%s".' % OUTBOX)
if not path.exists(SENT):
mkdir(SENT)
logger.debug('Creata la cartella "%s".' % SENT)
def check_paths():
logger = logging.getLogger()
if not path.exists(_CONF_DIR):
mkdir(_CONF_DIR)
logger.debug('Creata la cartella "%s".' % _CONF_DIR)
if not path.exists(OUTBOX):
mkdir(OUTBOX)
logger.debug('Creata la cartella "%s".' % OUTBOX)
if not path.exists(SENT):
mkdir(SENT)
logger.debug('Creata la cartella "%s".' % SENT)
def __init__(self, cfg):
self.logger = logging.getLogger()
self.is_active = True
self.exchange_adapter = emx_adapter(cfg.adapter)
try:
strategy_name = cfg.strategy.name
except AttributeError:
self.logger.exception("strategy was not found")
raise Exception("strategy was not found")
try:
self.strategy = strategies_factory[strategy_name](cfg.strategy, self.exchange_adapter)
except KeyError:
self.logger.exception("strategy was not found in a factory")
raise Exception("strategy was not found in a factory")
# from threading import Thread, Lock
import sys
import time
import simplejson
import traceback
from config import *
import network
import rsync_tools
import compute
import database
import kimobjects
import kimapi
from logger import logging
logger = logging.getLogger("pipeline").getChild("pipeline")
PIPELINE_WAIT = 1
PIPELINE_TIMEOUT = 60
PIPELINE_MSGSIZE = 2**20
PIPELINE_JOB_TIMEOUT = 3600*24
buildlock = Lock()
loglock = Lock()
def getboxinfo():
os.system("cd /home/vagrant/openkim-pipeline; git log -n 1 | grep commit | sed s/commit\ // > /persistent/setuphash")
info = {}
things = ['sitename', 'username', 'boxtype',
'ipaddr', 'vmversion', 'setuphash', 'uuid',
'gitargs', 'gitbranch', 'githost']
import requests
import pprint
from os import environ
from os import system
import json
import time
import sys
from logger import logging
formatter = logging.Formatter(fmt='%(asctime)s %(levelname)-8s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger('zookeeper-launch')
logger.setLevel(logging.DEBUG)
pp = pprint.PrettyPrinter(indent=4)
discovery_ep = environ.get('DISCOVERY_EP') + '/subscriptions/' + environ.get(
'TENANT_ID')
logger.info("Discovery endpoint %s", discovery_ep)
headers = json.loads('{"Content-type": "application/json"}')
getUrl = discovery_ep + '/getpods'
response = requests.get(getUrl, headers=headers)
response.raise_for_status()
#pp.pprint(response.json())
role_name = environ.get('ROLE_NAME')
logger.info("Role %s", role_name)
lExpectedHosts = {}
for pod in response.json():
if pod["DesiredStatus"] != 1:
#! /usr/bin/env python
""" Simple wrapper for udunits2-bin for converting arbitrary units to SI units """
VERSION = 0.3
import re
import subprocess
import warnings
warnings.simplefilter("ignore")
from logger import logging
logger = logging.getLogger("pipeline").getChild("kimunits")
logger.setLevel(logging.DEBUG)
class UnitConversion(Exception):
""" Class for unit conversion errors """
_output_expression_default = re.compile(
"You have: You want: "
"(?:(?P<value>(?:[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)) )?"
"(?P<unit>\S+)", )
_output_expression_convert = re.compile(
"You have: You want: "
".*?= " #with original stuff
"(?:(?P<value>(?:[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?)) )?"
"(?P<unit>\S+)", )
def convert_udunits2(from_value,
def __init__(self, exchange_name=None):
self.ws = None
self.session = None
self.exchange_name = exchange_name
self.logger = logging.getLogger()
import shutil
import subprocess
import dircache
import glob
import os
from packaging import version
from contextlib import contextmanager
import database
import kimapi
from template import template_environment
import config as cf
from config import __kim_api_version_spec__
from logger import logging
logger = logging.getLogger("pipeline").getChild("kimobjects")
#------------------------------------------------
# Base KIMObject
#------------------------------------------------
class KIMObject(object):
""" The base KIMObject that all things inherit from
Attributes:
required_leader
the required two letter leader for all kim codes, meant to be overridden
by subclassers
makeable
marks the type of kimobject as makeable or not, to be overriden by subclassers
path
from discord.ext import commands
from formatting.markdown import MD
from logger import logging
logger = logging.getLogger(f"robotina.{__name__}")
class CommandErrorHandler(commands.Cog):
def __init__(self, bot):
self.bot = bot
logger.debug(f"{self.__class__.__name__} initialized")
@commands.Cog.listener()
async def on_command_error(self, ctx, error):
await ctx.send(error)
class Status(commands.Cog):
def __init__(self, bot):
self.bot = bot
logger.debug(f"{self.__class__.__name__} initialized")
@commands.command(aliases=["stat", "st"])
async def status(self, ctx):
await ctx.send(f"latency: {self.bot.latency:0.3f}s")
class Greetings(commands.Cog):
def __init__(self, bot):
self.bot = bot
one unit to another
* asedata - the dictionary of reference data contained within ASE
"""
import os
import jinja2
import json
import clj
from functools import partial
from kimquery import query
from kimunits import convert
import database
import kimobjects
import config as cf
from logger import logging
logger = logging.getLogger("pipeline").getChild("template")
#-----------------------------------------
# New Template functions
#-----------------------------------------
def path(cand):
obj = kimobjects.kim_obj(cand)
try:
p = obj.executable
except AttributeError:
p = obj.path
logger.debug("thinks the path is %r",p)
return p
def latestversion(kim):
from os import listdir from threading import Thread, Event from flask import Flask, render_template, request, redirect, send_file, jsonify from flask_socketio import SocketIO from util import Util from vault import Vault from yara_create import Rule, YaraFiles from config import CAP_PATH, SESSION_CACHE_PATH, CARVED_DIR from logger import logging, LOG_FILE, FORMATTER, TIMESTAMP, LOG_LEVEL logger = logging.getLogger(__name__) logger.setLevel(LOG_LEVEL) formatter = logging.Formatter(FORMATTER, TIMESTAMP) file_handler = logging.FileHandler(LOG_FILE) file_handler.setFormatter(formatter) logger.addHandler(file_handler) app = Flask(__name__) app.config["SECRET_KEY"] = "secret!" app.config["DEBUG"] = False app.config["SERVER_NAME"] = "127.0.0.1:8000" socketio = SocketIO(app, async_mode=None)
"""
Simple set of tools for having rsync commands go through
"""
import config as cf
from logger import logging
logger = logging.getLogger("pipeline").getChild("rsync_tools")
import os
import subprocess
import tempfile
from database import parse_kim_code
from functools import partial
# --delete ensures that we delete files that aren't on remote
RSYNC_FLAGS = "-vvrLhzREc --progress --stats -e "
RSYNC_FLAGS += "'ssh -i "+cf.GLOBAL_KEY+" -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no'"
RSYNC_FLAGS += " --exclude-from="+cf.RSYNC_EXCLUDE_FILE
RSYNC_ADDRESS = cf.RSYNC_USER+"@"+cf.RSYNC_HOST
RSYNC_PATH = RSYNC_ADDRESS+":"+cf.RSYNC_REMOTE_ROOT
RSYNC_LOG_FILE_FLAG = "--log-file={}/rsync.log".format(cf.KIM_LOG_DIR)
RSYNC_LOG_PIPE_FLAG = " >> {} 2>&1".format(cf.KIM_LOG_DIR+"/rsync_stdout.log")
if cf.PIPELINE_GATEWAY:
READ_PENDING = os.path.join(RSYNC_PATH, "/curators-to-pipeline-interface/pending/./")
READ_APPROVED = os.path.join(RSYNC_PATH, "/curators-to-pipeline-interface/approved/./")
if cf.PIPELINE_DEBUG:
WRITE_RESULTS = os.path.join(RSYNC_PATH, "/pipeline/test-result-uploads-dbg/incoming/./")
else:
WRITE_RESULTS = os.path.join(RSYNC_PATH, "/pipeline/test-result-uploads/incoming/./")
def setup_celery_logging(**kwargs):
return logging.getLogger('pipeline')
import pymongo
import os
import re
import datetime
import util
import kimunits
import database
import config as cf
from logger import logging
logger = logging.getLogger('pipeline').getChild('mongodb')
client = pymongo.MongoClient(host=cf.MONGODB_HOSTNAME)
db = client[cf.MONGODB_DATABASE]
PATH_RESULT = cf.RSYNC_LOCAL_ROOT
PATH_APPROVED = cf.RSYNC_LOCAL_ROOT
def config_edn(flname):
with open(flname) as f:
doc = util.loadedn(f)
doc.setdefault("created_on", str(datetime.datetime.fromtimestamp(os.path.getctime(flname))))
return doc
def parse_kim_code(kim_code):
RE_KIMID = r"(?:([_a-zA-Z][_a-zA-Z0-9]*?)__)?([A-Z]{2})_([0-9]{12})(?:_([0-9]{3}))?"
return re.match(RE_KIMID,kim_code).groups()
def drop_tables(ask=True):
if ask:
check = raw_input("Are you sure? [y/n] ")
from copy import copy
from datetime import datetime
from game.board import Board
from game.trie import Trie
from logger import logging
logger = logging.getLogger("start_game")
class Game:
def __init__(self, board, trie, min_len=0, allow_repetitions=True):
self.min_len = min_len
self.allow_repetitions = allow_repetitions
self.board = board
self.trie = trie
self.found = {}
def dfs(self, trie, cur, visited, words, word_i):
cur_c = self.board.get_c(cur)
trie = trie.get(cur_c)
if not trie:
return
visited[cur] = True
word_i.append(cur)
if Trie.is_word(trie) and len(word_i) >= self.min_len:
if self.allow_repetitions:
words.append(word_i)
else:
import glob
import os
import itertools
from packaging import version
from contextlib import contextmanager
from collections import defaultdict
import util
import database
import kimapi
from template import template_environment
import config as cf
from config import __kim_api_version_spec__
from logger import logging
logger = logging.getLogger("pipeline").getChild("kimobjects")
#------------------------------------------------
# Base KIMObject
#------------------------------------------------
class KIMObject(object):
""" The base KIMObject that all things inherit from
Attributes:
required_leader
the required two letter leader for all kim codes, meant to be overridden
by subclassers
makeable
marks the type of kimobject as makeable or not, to be overriden by subclassers
path
the full path to the directory associated with the kim object
"""
Simple set of tools for having rsync commands go through
"""
import config as cf
import network
from logger import logging
logger = logging.getLogger("pipeline").getChild("rsync_tools")
import os
import sys
import subprocess
import tempfile
from database import parse_kim_code
from functools import partial
# # For Director and Worker
#
# - Expecting directory ~/.ssh/controlmasters to exist, will be created if missing.
#
# - ~/.ssh/config should contain these lines:
#
# ControlMaster auto
#
# Host gateway-host-for-rsync
# ControlPath ~/.ssh/controlmasters/gateway-rsync
# UserKnownHostsFile /dev/null
# StrictHostKeyChecking no
# HostName gateway.pipeline.openkim.org
# User gateway-user1
# IdentityFile ~/.ssh/ssh-key-for-gateway
import pymongo
import os
import yaml
import json
from logger import logging
logger = logging.getLogger('pipeline').getChild('connect')
client = pymongo.MongoClient()
db = client.database
objs = db.objs
results = db.results
errors = db.errors
verifications = db.verifications
# encoding: utf-8
"""
es_util.py
Created by Cuong Pham on 2012-09-06.
Copyright (c) 2012 ChonGiaDung.vn. All rights reserved.
"""
from logger import logging
from elasticsearch import helpers, Elasticsearch
import time
import threading
import json
from decorators import command
logging.getLogger('elasticsearch').setLevel(logging.WARNING)
class ESClient(object):
def __init__(self, hosts, batchSize=1000, **kwargs):
self.esConn = Elasticsearch(hosts, **kwargs)
self.bulker = ListBulker()
self.batchSize = batchSize
self.ID_FIELD = "_id"
def _isOk(self, response):
return response.get('acknowledged', False)
def createIndex(self,
indexName="test",
body=None,
communication between directors, workers, and webiste through
the beanstalk daemon.
Any of the classes below rely on a secure public key to open an ssh
tunnel to the remote host. It then connects to the beanstalkd
across this tunnel.
"""
import os
import time
import json
from subprocess import Popen, check_call, CalledProcessError
from string import Template
import config as cf
from logger import logging
logger = logging.getLogger("pipeline").getChild("network")
#=============================================================================
# Functions for formulating messages to different parts of the system
#=============================================================================
def transform_priority(priority):
priorities = {
"immediate": 0, "very high": 0.01, "high": 0.1,
"normal": 1, "low": 10, "very low": 100
}
if isinstance(priority, str):
priority = int(1e6*priorities.get(priority, 1))
return priority
def director_update_message(itemid, status='approved', force=False, priority='normal'):
import os
import discord
from discord.ext import commands
from dotenv import load_dotenv
from bot_commands import COGS
from logger import logging
logger = logging.getLogger("robotina.start")
load_dotenv()
TOKEN = os.getenv('DISCORD_TOKEN')
intents = discord.Intents(messages=True, guilds=True)
bot = commands.Bot(command_prefix='!', intents=intents)
for cog in COGS:
bot.add_cog(cog(bot))
@bot.event
async def on_ready():
logger.info("Robonita is ready")
bot.run(TOKEN)
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from arp import do_arping
from logger import logging
from threading import Thread
import ipcalc
import ping
import re
import socket
MAX = 128
logger = logging.getLogger()
class sendit(Thread):
def __init__ (self, ip):
Thread.__init__(self)
self.ip = ip
self.status = 0
self.elapsed = 0
def run(self):
try:
self.elapsed = ping.do_one("%s" % self.ip, 1)
if (self.elapsed > 0):
self.status = 1
except Exception as e:
logger.debug('Errore durante il ping dell\'host %s: %s' % (self.ip, e))
# ============================================================================
# GameMode Class
# ============================================================================
from logger import logging, log
import time
from AudioPlay import AudioPlay
battLog = logging.getLogger('BATTERY')
batt_handler = logging.FileHandler(
'/home/pi/Documents/ovaom/logs/battery_monitor.log')
battLog.addHandler(batt_handler)
INACTIVE_THRESHOLD = 5 # in seconds
class GameMode(object):
''' GameMode Class: puzzleMode and jungleMode must inherit from this class '''
instrument = [{
'active': 0,
'maxPreset': 1,
'currentPreset': 0,
'lastSeen': -1,
'battery': 0,
} for i in range(0, 4)]
prev_offline_objects = []
prevMonitoring = 0
def __init__(self, net):
self.net = net
If we are told that a new test result has come in:
* Get all (te, mo) pairs that depend on this result
* See if they need to be updated (are they latest?)
* Send these updates through the original channels
Local dependency resolution
"""
import config as cf
import kimobjects
import kimquery
import kimapi
from logger import logging
logger = logging.getLogger("pipeline").getChild("dependencies")
# TODO -- implement blacklist in mongodb to not run some tests
def result_inqueue(test, model):
query = {
"database": "job", "query": {"test": str(test), "model": str(model)},
"project": ["status"], "limit": 1
}
status = kimquery.query(query, decode=True)
return len(status) > 0
def result_exists(test, model):
query = {
"project": ["test-result-id"], "database": "obj", "query": {
import sqlite3
import hashlib
import cPickle as pickle
from itertools import chain
import config as cf
import kimapi
import kimobjects
import database
import dependencies
import pipeline
import rsync_tools
import network
from logger import logging
logger = logging.getLogger("pipeline").getChild("director")
#==================================================================
# director class for the pipeline
#==================================================================
class Director(pipeline.Agent):
"""
The Director object, knows to listen to incoming jobs, computes
dependencies and passes them along to workers
"""
def __init__(self, *args, **kwargs):
super(Director, self).__init__(name="director", num=0, *args, **kwargs)
self.logger.info("Initializing KIM Director")
self.db = KIMObjectDB(cf.DIRECTOR_DB)
# start by getting into sync with the shared repository and making
#! /usr/bin/env python
""" Simple wrapper for executable for converting arbitrary units to SI units """
VERSION = 0.3
import re
import math
import subprocess
import warnings
warnings.simplefilter("ignore")
from logger import logging
logger = logging.getLogger("pipeline").getChild("kimunits")
logger.setLevel(logging.DEBUG)
class UnitConversion(Exception):
""" Class for unit conversion errors """
_units_output_expression = re.compile("(?P<value>(?:[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?))(?: (?P<unit>.+))?")
def linear_fit(x, y):
"""
Perform a linear fit between x,y, returning the average error for each data
point as well. This is written this way so as to not add a numpy dependency
"""
n = len(x)
xx = sum(map(lambda x: x**2, x)) - sum(x)**2 / n
yy = sum(map(lambda x: x**2, y)) - sum(y)**2 / n
xy = sum(map(lambda x,y : x*y, x, y)) - sum(x) * sum(y) / n
a, b = sum(y)/n - xy / xx * sum(x) / n, xy / xx
yhat = map(lambda x: a + b*x, x)
yerr = math.sqrt(sum(map(lambda y,yh: (y - yh)**2/y**2, y, yhat)) / n)
"""
Methods that deal with the KIM API directly. Currently these are methods
that build the libraries and use the Python interface kimservice
to test if tests and models match.
"""
import os
import sys
from subprocess import check_output, check_call, CalledProcessError
from contextlib import contextmanager
import packaging.specifiers, packaging.version
from functools import partial
import config as cf
from config import __pipeline_version_spec__, __kim_api_version_spec__
from logger import logging
logger = logging.getLogger("pipeline").getChild("kimapi")
#======================================
# API build utilities
#======================================
MAKE_LOG = os.path.join(cf.LOG_DIR, "make.log")
@contextmanager
def in_dir(path):
cwd = os.getcwd()
os.chdir(path)
try:
yield
except Exception as e:
raise e
finally:
import json
import os
from logger import logging
logger = logging.getLogger("trie")
class Trie:
END_OF_WORD = "!"
def __init__(self):
self._trie = {}
@property
def trie(self):
return self._trie
def save_to_file(self, path="trie.json"):
with open(path, 'w') as outfile:
json.dump(self._trie, outfile)
logger.info(f"save: location={path}, size={os.stat(path).st_size}")
return path
def load_from_file(self, path="trie.json"):
with open(path) as json_file:
self._trie = json.load(json_file)
logger.info(f"load: location={path}, size={os.stat(path).st_size}")
return path
def generate_from_dict_file(self,
dict_path="dictionary.txt",
- if there are results outstanding, do not submit the original pair
(since the acceptance of the test result will trigger it again)
If we are told that a new test result has come in:
* Get all (te, mo) pairs that depend on this result
* See if they need to be updated (are they latest?)
* Send these updates through the original channels
Local dependency resolution
"""
import kimobjects
import kimquery
from logger import logging
logger = logging.getLogger("pipeline").getChild("dependencies")
def result_inqueue(test, model):
query = {
"database": "job",
"test": str(test),
"model": str(model),
"project": ["tube"],
"limit": 1
}
tube = kimquery.query(query, decode=True)
return len(tube) > 0
import pymongo
import os
import yaml
import json
from logger import logging
logger = logging.getLogger('pipeline').getChild('mongofill')
from processkimfiles import eatfile, configtojson
client = pymongo.MongoClient()
db = client.test_database
#tests = db.tests
#tests.insert(post)
REPO_DIR = os.path.abspath("/home/vagrant/openkim-repository/")
objs = db.objs
objs.drop()
results = db.results
results.drop()
logger.info("Filling with test results")
leaders = ('tr', 'vr')
for leader in leaders:
for folder in os.listdir(os.path.join(REPO_DIR, leader)):
logger.info("On %s ", folder)
path = os.path.join(REPO_DIR, leader, folder)
info = configtojson(os.path.join(path, 'kimspec.ini'))
info['path'] = os.path.join(leader, folder)
info['uuid'] = folder
resultobj = objs.insert(info)
with open(os.path.join(path, 'results.yaml')) as f:
import time
import subprocess
import threading
import shutil
import json
import traceback
from contextlib import contextmanager
import util
import kimunits
import kimquery
import kimobjects
import config as cf
from logger import logging
logger = logging.getLogger("pipeline").getChild("compute")
#================================================================
# a class to be able to timeout on a command
#================================================================
class Command(object):
def __init__(self, cmd, stdin=None, stdout=None, stderr=None):
"""
A class to provide time limits to sub processes. Accepts
a command as an array (similar to check_output) and file
handles with which to communicate on stdin, stdout, stderr
"""
self.cmd = cmd
self.process = None
self.stdin = stdin
self.stdout = stdout
import urllib
import urllib2
import itertools
from config import PipelineQueryError
import config as cf
import json
import os
import sys
import ssl
from logger import logging
logger = logging.getLogger("pipeline").getChild("kimquery")
import sys
def open_url(url, data, header):
request = urllib2.Request(url, data, header)
if cf.PIPELINE_QUERY_USE_SSL:
response = urllib2.urlopen(request)
else:
context = ssl._create_unverified_context()
response = urllib2.urlopen(request, context=context)
answer = response.read()
response.close()
return answer
def prepare_dns():
resolv = "/etc/resolv.conf"
dnsline = "nameserver 127.0.0.1"
'''
import socket
import time
import copy
import utility as util
# from digitalIO import DIO
from constants import Constant as const
from vergence_version import ValidateVergenceVersion
from global_parameters import Globals as globls
from data_collection import DataFileObj
from constants import Constant, ConfigLabels
from logger import logging
import numpy as np
log = logging.getLogger(__name__)
class ArbitratorServer:
def __init__(self):
# self.DIO_obj = DIO()
# for programing printer port
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# sock.bind(('', globls.config_param['arbitrator_service']['local_port']))
sock.bind(('100.1.1.3',
globls.config_param['arbitrator_service']['local_port']))
sock_eye = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
# sock_eye.bind(('', globls.config_param['arbitrator_service']['local_port_eye']))
import zipfile
from collections import OrderedDict
from HTMLParser import HTMLParser
from StringIO import StringIO
from urllib2 import urlopen, URLError, HTTPError
from logger import logging
from lxml import etree
from lxml.etree import tostring
from jinja2 import Environment, FileSystemLoader
log = logging.getLogger('development')
def create():
env = Environment(loader=FileSystemLoader("."))
try:
configuration = env.get_template('WebGrab++.config.xml.template')
sections = json.loads(open('template_variables.json').read())
country_name = sections['country_name']
if country_name is None:
raise Exception(sections['no_country_config'])
if ' ' in country_name:
log.error("Please do the following"
"Download the zip file and place it on the channels "
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
import ConfigParser
from logger import logging
from os import path
import paths
logger = logging.getLogger()
configerror = 00000 # codice di errore utilizzato in presenza errori di errorcoder.py
class Errorcoder:
'''
Nel file di configurazione di possono creare delle sezioni, al'interno
delle quali avviene la ricerca come se fosse un dizionario si può associare
una sezione a ciascun set di codici di errore utilizzato dai vari operatori.
In alternativa ciascuna sezione può essere associata alla tipologia di
errore, I/O FTP PING.
'''
def __init__(self, filename):
self._filename = filename
the task name 'pipeline.tasks.{object-type-name}_{function-name}'.
Concretely, the `run` function of Worker is listed as {'worker': ['run']} in
the registry and is mapped to the task 'pipeline.tasks.worker_run'.
Finally, functions for monitoring the Celery application while its running are
provided at the end of the file. There allow for the watching of task
generation and consumption. It is used by the Gateway to keep track of job
status and used by the website to send realtime updates to users on the web.
"""
import database
import kimobjects
import config as cf
from app import app
from logger import logging
logger = logging.getLogger('pipeline').getChild('tasks')
import time
import threading
from celery import Task
# conditional import since importing mongodb, Gateway tries to connect to the
# database while loading, causing issues in the director, worker
Gateway, Director, Worker = [dict for i in xrange(3)]
GWorker, GDirector, GGateway = [dict(obj=None) for i in xrange(3)]
if cf.PIPELINE_GATEWAY:
from gateway import Gateway, mongodb
elif cf.PIPELINE_DIRECTOR:
from director import Director
elif cf.PIPELINE_WORKER:
"""
Set of methods for querying the database in one form or the other
As well as parsing and handling kim_codes
Currently these calls mostly glob on the database, could be replaced by something more elegant later
"""
import re
import os
import glob
import operator
import config as cf
from logger import logging
logger = logging.getLogger("pipeline").getChild("database")
#-------------------------------------------------
# Helper routines (probably move)
#-------------------------------------------------
#KIMID matcher ( optional name __) (prefix ) ( number )( opt version )
RE_KIMID = r"^(?:([_a-zA-Z][_a-zA-Z0-9]*?)__)?([A-Z]{2})_([0-9]{12})(?:_([0-9]{3}))?$"
RE_UUID = r"^([A-Z]{2}_[0-9]{12}_[0-9]{3}-and-[A-Z]{2}_[0-9]{12}_[0-9]{3}-[0-9]{5,})(-error)?$"
RE_ERROR = r"^([A-Z]{2}_[0-9]{12}_[0-9]{3}-and-[A-Z]{2}_[0-9]{12}_[0-9]{3}-[0-9]{5,})-error$"
RE_UUID_LEX = r"^([A-Z]{2}_[0-9]{12}_[0-9]{3})-and-([A-Z]{2}_[0-9]{12}_[0-9]{3})-[0-9]{5,}(-error)?$"
LEADERS_OBJS = ["mo", "md", "te", "td", "tv", "mv"]
LEADERS_DATA = ["tr", "vr", "rd", "er"]
def jobid_to_kimcodes(jobid):
codes = re.match(RE_UUID_LEX, jobid)
def __init__(self):
self.validate_url = "http://basic.10jqka.com.cn/"
self.logger = logging.getLogger('code.py')