def copy_dynamic_files(agent, src_base, dst_base, host, username, copy_type): """ Copy dynamic agent files to/from a remote machine. agent - agent name src_base - base source directory to concatenate to listed paths dst_base - base destination directory to concatenate to listed paths host - remote host to connect to username - remote username used in the connection copy_type - either 'local' when copying FROM remote machine or 'remote' when copying TO remote machine """ scoutlog.info('copying dynamic files of %s' % agent) dynamic_path = path(RULES_DIR, agent, 'dynamic') if not os.path.isfile(dynamic_path): return False with open(path(RULES_DIR, agent, 'dynamic'), 'r') as f: base_paths = f.read().splitlines() path_list = [] for p in base_paths: # Get tuples with remote source and local destination path_list.append((path(src_base, p), path(dst_base, p))) if copy_type == 'local': # Copy to local machine remote_get(path_list, host, username) elif copy_type == 'remote': # Copy to remote machine remote_put(path_list, host, username)
def purge(self, name, sender=None, locale="en"): """ Remove an agent's configuration files. """ self.set_locale(locale) if not self.has_permissions(sender): print(MSG_NO_PERM) return self.feedback(MSG_NO_PERM, sender) # Uninstall the agent self.remove(name) # Remove config files confpath = path(ZAM_INFO, name + ".conffiles") if not os.path.isfile(confpath): msg = _("Agent %s has no config files") % name print(msg) return self.feedback(msg, sender) with open(confpath, "r") as conflist: for cf in conflist.read().splitlines(): c = path(env["ZOE_HOME"], cf) print("Removing %s" % c) try: os.remove(c) except: # Nothing to remove? pass os.remove(confpath) msg = _("Agent %s purged") % name print(msg) return self.feedback(msg, sender)
def firstrun(here): # First run. print("Welcome to Baked Beans!") data = {} message(""" To get started, a few pieces of information are needed so they can be filled in for each new project you start. If you're already on GitHub, you can fill in your user ID and as much information as possible will be looked up automatically for you. Or you can leave it blank and enter the rest manually. """) username = input("Enter your GitHub user ID (optional): ") if username: get_github_info(data, username) prompt_missing_info(data, 'name', """ your full name, as you would like it to appear on projects you publish """) prompt_missing_info(data, 'url', """ a base URL for your projects, including http:// or https:// as desired. The default project URL for each new project will be of the form <base url>/<project name>. """) prompt_missing_info(data, 'email', """ an email where you can be reached for project support """) copy_template_tree(path(here, 'tin.template'), path(here, 'tin'), data) if not DEVELOPMENT_VERSION: rmtree(path(here, 'tin.template')) print("Great! Now, let's set up your first project...")
def launch(self, parser): """ Launch an agent. name* - unique name of the agent sender - sender of the message src - channel from which the message was obtained """ name, sender, src = self.multiparse( parser, ['name', 'sender', 'src']) self.set_locale(sender) if not self.has_permissions(sender): self.logger.info("%s tried to launch an agent" % sender) return self.feedback(_("You don't have permissions to do that"), sender, src) if self.running(name): self.logger.info("'%s' is already running" % name) return self.feedback( _("Agent '%s' is already running") % name, sender, src) agent_dir = path(env["ZOE_HOME"], "agents", name) if not os.path.isdir(agent_dir): self.logger.info("Directory for '%s' does not exist" % name) return self.feedback( _("Agent '%s' does not exist!") % name, sender, src) # Redirect stdout and stderr to zam's log log_file = open(path(env["ZOE_LOGS"], "zam.log"), "a") proc = subprocess.Popen([ZOE_LAUNCHER, "launch-agent", name], stdout=log_file, stderr=log_file, cwd=env["ZOE_HOME"]) zconf = self.read_conf() # Force the agent to register port = zconf["agent " + name]["port"] launch_msg = { "dst": "server", "tag": "register", "name": name, "host": env["ZOE_SERVER_HOST"], "port": port } self.sendbus( self.feedback(_("Launching agent '%s'") % name, sender, src).msg()) return zoe.MessageBuilder(launch_msg)
def main(datestamp, sleep_interval = 1): 'Download given a datestamp.' # Set the directories. index_dir = path('downloads', 'index', datestamp) image_dir = path('downloads', 'images') # Download the index. index = get('http://www.postsecret.com/', cachedir = index_dir) # Download the new images. html = fromstring(index.read()) srcs = html.xpath('//img/@src') for src in srcs: get(src, cachedir = image_dir) sleep(sleep_interval)
def setUpClass(self): """ Sets up the test class. """ self.cpath = "ExtFunctionTests.ExtFunctionTest2" self.dir = build_ext('array_shared', 'ExtFunctionTests.mo') self.fpath = path(self.dir, "ExtFunctionTests.mo")
def stop(self, parser): """ Stop an agent's execution. name* - unique name of the agent sender - sender of the message src - channel from which the message was obtained """ name, sender, src = self.multiparse( parser, ['name', 'sender', 'src']) self.set_locale(sender) if not self.has_permissions(sender): self.logger.info("%s tried to stop an agent" % sender) return self.feedback(_("You don't have permissions to do that"), sender, src) if not self.running(name): self.logger.info("'%s' is not running" % name) return self.feedback( _("Agent '%s' is not running") % name, sender, src) # Redirect stdout and stderr to zam's log log_file = open(path(env["ZOE_LOGS"], "zam.log"), "a") proc = subprocess.Popen([ZOE_LAUNCHER, "stop-agent", name], stdout=log_file, stderr=log_file, cwd=env["ZOE_HOME"]) return self.feedback(_("Stopping agent '%s'") % name, sender, src)
def setUpClass(self): """ Sets up the test class. Check timeout of infinite loop during constant evaluation. """ self.cpath = "ExtFunctionTests.ExternalInfinityTest" self.dir = build_ext('array_shared', 'ExtFunctionTests.mo') self.fpath = path(self.dir, "ExtFunctionTests.mo")
def _createFileIfSpecialDir(self, dirName, marker, fileName = None): if fileName == None: fileName = "file-" + os.path.basename(dirName) r = re.compile(marker + r".*") if r.match(os.path.basename(dirName)): open(path(dirName, fileName), 'a').close()
def __rsync(self, donorURL, acceptorURL, ignoreItemsPool): rsyncShellCommand = ["rsync", "--verbose", "--progress", "--stats", "--compress", "--compress-level=9", "--recursive", "--times", "--perms", "--links", "-L", "--exclude \'" + "\' --exclude \'".join(ignoreItemsPool) + "\'", "--rsh=\"ssh -i '%s' -C\"" % self.__findSSHKeyPath(), "\'" + path(donorURL, "") + "\'*", "\'" + acceptorURL + "\'"] sys.stdout.write("\nINFO:%s:%s: " % (__file__, Helper().lineno()) + "Is going to launch shell command: %s\n\n" % " ".join(rsyncShellCommand)) exitCode = os.system(" ".join(rsyncShellCommand)) sys.stdout.write("\nINFO:%s:%s: " % (__file__, Helper().lineno()) + "Shell command finished" + " with exit code '%s'.\n" % exitCode) self.__validateRSyncExitCode(exitCode) return
def copy_template_tree(source, there, data): """Interface to shutil.copytree. Interpolates into .template files as they are found, removing .template filename extensions from the copies, and gracefully handles an already-existing destination folder.""" def do_copy(s, t): copytree(s, t, copy_function=partial(template_copy, data)) if exists(there): # Dump everything into the existing directory, one piece at a time. for item in directory_contents(source): item, target = path(source, item), path(there, item) if is_directory(item): do_copy(item, target) else: template_copy(data, item, target) else: do_copy(source, there)
def check_dir(self, user): """ Check if the user has a directory inside the todo tree and create it if it is not present. """ user_path = path(TODO_PATH, user) if not os.path.isdir(user_path): os.mkdir(user_path)
def test_ModelicaUtilities(self): """ Test compiling a model with external functions using the functions in ModelicaUtilities. """ fpath = path(get_files_path(), 'Modelica', "ExtFunctionTests.mo") cpath = "ExtFunctionTests.ExtFunctionTest3" jmu_name = compile_fmu(cpath, fpath) model = load_fmu(jmu_name)
def _createIgnoreItemsInDir(self, dirName): for name in self.td.ignoreNamesPool[:-1]: newPath = path(dirName, name) if not os.path.exists(newPath): os.mkdir(newPath) self._createFileIfSpecialDir(dirName, "", fileName = self.td.ignoreNamesPool[-1])
def handle_uploaded_file(f,name, task_translation): #save_path = path( SITE_ROOT , 'taskresources', 'Image', task_id , task_language ) save_path = path( SITE_ROOT , 'resources', 'task', str(task_translation.task_id) , task_translation.language_locale_id, 'resources' ) # Check if upload folder of a specific task already exists and create it, if it doesn't. if not os.path.exists(save_path): os.makedirs(save_path) # Write file to disk with open(path(save_path, name), 'wb+') as destination: for chunk in f.chunks(): destination.write(chunk) # Write filename to DB resource = Resources(filename = name, type = "image", task = task_translation.task, language = task_translation.language_locale) resource.save() return save_path
def _mkTestSubDirSet(self, dirName, quantity, depth): self._createFileIfSpecialDir(dirName, "0") if depth == 0: return for idx in range(0, quantity): if idx%2 == 0: newPath = path(dirName, str(idx) + " with space in name") else: newPath = path(dirName, str(idx)) not os.path.exists(newPath) and os.mkdir(newPath, 0700) if idx%2 == 1: self._createIgnoreItemsInDir(newPath) self._mkTestSubDirSet(newPath, quantity, depth - 1 )
def running(self, name): """ Check if an agent is running. """ # We depend on the .pid files here pid_path = path(env["ZOE_VAR"], name + ".pid") if os.path.isfile(pid_path): return True return False
def purge(self, parser): """ Remove an agent's configuration files. name* - unique name of the agent sender - sender of the message src - channel from which the message was obtained """ name, sender, src = self.multiparse( parser, ['name', 'sender', 'src']) self.set_locale(sender) if not self.has_permissions(sender): self.logger.info("%s tried to purge an agent" % sender) return self.feedback(_("You don't have permissions to do that"), sender, src) # Uninstall the agent self.remove(parser) # Remove config files confpath = path(ZAM_INFO, name + ".conffiles") if not os.path.isfile(confpath): self.logger.info("'%s' has no config files" % name) return self.feedback( _("Agent '%s' has no config files") % name, sender, src) with open(confpath, "r") as conflist: for cf in conflist.read().splitlines(): c = path(env["ZOE_HOME"], cf) self.logger.debug("Removing %s" % c) try: os.remove(c) except: # Nothing to remove? pass os.remove(confpath) self.logger.info("Agent '%s' purged" % name) return self.feedback(_("Agent '%s' purged") % name, sender, src)
def move_files(self, name, updating=False): """ Move the files and directories to their corresponding ZOE_HOME counterpart. To be used only by install() and update() Returns the destination file list """ source_dir = path(ZAM_TEMP, name) # Generate list of source files src_list = [] for d in os.listdir(source_dir): if os.path.isdir(path(source_dir, d)) and d not in [".git", "zam"]: subdir = path(source_dir, d) for root, dirs, files in os.walk(subdir): for f in files: src_list.append(path(root, f)) if updating: # Diff list diff_list = [] for src in src_list: stripped = src.replace(source_dir, "") stripped = self.remove_slash(stripped) diff_list.append(stripped) # Compare file lists and remove those not present in the update alist_path = path(ZAM_INFO, name + ".list") with open(alist_path, "r") as alist: for f in [p for p in alist.read().splitlines() if p not in diff_list]: l = path(env["ZOE_HOME"], f) # Remove final file os.remove(l) # Remove the generated tree dirs = os.path.split(l) while dirs[0] != "/": if os.listdir(dirs[0]): break shutil.rmtree(dirs[0]) dirs = os.path.split(dirs[0]) # Move files file_list = [] for src in src_list: stripped = src.replace(source_dir, "") stripped = self.remove_slash(stripped) dst = os.path.dirname(path(env["ZOE_HOME"], stripped)) try: os.makedirs(dst) except: # Tree already exists? pass shutil.copy(src, dst) file_list.append(stripped) return file_list
def list_exists(self, user, tlist): """ Determine whether the specified list exists for the given user or not. """ list_path = path(TODO_PATH, user, tlist) if not os.path.isfile(list_path): return False return True
def write_list(self, tasks, user, tlist): """ Write back a list of tasks to the specified list. The tasks are alphabetically sorted before stored in the list. """ list_path = path(TODO_PATH, user, tlist) with open(list_path, "w") as ulist: for task in sorted(tasks): ulist.write("%s\n" % task)
def read_list(self, user, tlist): """ Read the list for the specified user line by line and return all the tasks. """ list_path = path(TODO_PATH, user, tlist) with open(list_path, "r") as ulist: tasks = ulist.read().splitlines() return tasks
def launch(self, name, sender=None, locale="en"): """ Launch an agent. """ self.set_locale(locale) if not self.has_permissions(sender): print(MSG_NO_PERM) return self.feedback(MSG_NO_PERM, sender) if self.running(name): msg = _("Agent %s is already running") % name print(msg) return self.feedback(msg, sender) agent_dir = path(env["ZOE_HOME"], "agents", name) if not os.path.isdir(agent_dir): msg = _("Agent %s does not exist!") % name print(msg) return self.feedback(msg, sender) # Redirect stdout and stderr to zam's log log_file = open(path(env["ZOE_LOGS"], "zam.log"), "a") proc = subprocess.Popen([path(env["ZOE_HOME"], "zoe.sh"), "launch-agent", name], stdout=log_file, stderr=log_file, cwd=env["ZOE_HOME"]) zconf = self.read_conf() # Force the agent to register port = zconf["agent " + name]["port"] launch_msg = { "dst": "server", "tag": "register", "name": name, "host": env["ZOE_SERVER_HOST"], "port": port } return [ self.feedback(_("Launching agent %s") % name, sender), zoe.MessageBuilder(launch_msg) ]
def dashboard(project): """ Index of the homepage. """ g.selected_tab = "dashboard" # Check if project argument is correct project_query = Project.select().where(Project.slug == project).first() if project_query is None: flash("invalid project") return redirect(url_for("projects")) session["project"] = project_query health = dict(total=0, success=0, warning=0, error=0) # pylint: disable=R0204 #disabling warning about redefining settings. I do this on purpose project_path = path(project_query.path, "build.abc") settings = Anemone.abcfile.parse(project_path) if settings is None: flash("Could not parse settings file, prehaps the file path ({}) is wrong" .format(project_path)) settings = dict() else: settings = settings.m_nodes # pylint: enable=R0204 query = (Job .select() .where(Job.project == project_query) .order_by(-Job.started.is_null(), -Job.started)) entries = [] count = 0 for job in query: span = job.ended status = job.get_status() if job.started is not None: if job.ended is not None: span = job.ended - job.started if status is 1: health["total"] += 1 health["success"] += 1 elif status is 2: health["total"] += 1 health["warning"] += 1 elif status is 3: health["total"] += 1 health["error"] += 1 if count < JOBS_PER_PAGE: entries.append(dict(id=job.id, status=status, name=job.name, start=job.started, end=job.ended, span=span)) return render_template("dashboard.html", entries=entries, buildconf=settings, health=health)
def stop(self, name, sender=None, locale="en"): """ Stop an agent's execution. """ self.set_locale(locale) if not self.has_permissions(sender): print(MSG_NO_PERM) return self.feedback(MSG_NO_PERM, sender) if not self.running(name): msg = _("Agent %s is not running") % name print(msg) return self.feedback(msg, sender) # Redirect stdout and stderr to zam's log log_file = open(path(env["ZOE_LOGS"], "zam.log"), "a") proc = subprocess.Popen([path(env["ZOE_HOME"], "zoe.sh"), "stop-agent", name], stdout=log_file, stderr=log_file, cwd=env["ZOE_HOME"]) return self.feedback(_("Stopping agent %s") % name, sender)
def GetUserTempDir(): import stdpaths if getattr(sys, 'is_portable', False): base_temp = stdpaths.temp / 'digsby' else: base_temp = stdpaths.userlocaldata pth = path.path(base_temp) / 'temp' if not pth.isdir(): os.makedirs(pth) return pth
def close_tunnel(conf, name): """ Close the SSH tunnel to a given outpost. conf - ConfigParser section with the data of the outpost name - name of the outpost """ if not os.path.isfile(path(env['ZOE_VAR'], name + '.pid')): # No tunnel currently open? scoutlog.error('there is no tunnel open for outpost %s' % name) return False # Add script logs to scout log log_file = open(path(env["ZOE_LOGS"], "scout.log"), "a") # Stop the process proc = subprocess.Popen([ZOE_LAUNCHER, "stop-agent", name], stdout=log_file, stderr=log_file, cwd=env["ZOE_HOME"]) proc.wait() scoutlog.info('tunnel to outpost %s should be closed now' % name) return True
def fetch(self, name, source): """ Download the source of the agent to var/zam/name. """ temp = path(ZAM_TEMP, name) alist = self.read_list() try: if not source: src = alist[name]["source"] else: src = source except: return -1 return subprocess.call(["git", "clone", src, temp])
def main(): src = path.path("src") dst = path.path("dst") if not dst.exists(): dst.mkdir() for file in dst.files(): file.remove() md_files = [file for file in src.listdir() if file.ext in [".txt", ".md"]] md_files.sort(key=lambda file: file.getsize()) for md_file in md_files: namebase = md_file.namebase print "{0:20} ...".format(namebase[:20]), js_file = namebase + ".js" error = False try: sh.pandoc("-t", "json", "-o", dst / js_file, md_file) json1 = json.load(open(dst / js_file), object_pairs_hook=pandoc.Map) doc = pandoc.to_pandoc(json1) py_file = namebase + ".py" output = open(dst / py_file, "w") output.write(repr(doc)) js2_file = namebase + "2" + ".js" json2 = pandoc.to_json(doc) json.dump(json2, open(dst /js2_file, "w")) sh.pandoc("-s", "-t", "markdown", "-o", dst / namebase + ".txt", "-f", "json", dst / js2_file) except Exception: error = True if not error and json1 == json2: print "OK" else: print "FAIL"
def file_loaded(self, services, globals, filename=None): domains = globals.get('Domains', {}) if not domains: return p = lambda s: normpath(path(dirname(filename), s)) for domain, data in domains.items(): if not data: continue if 'cert' in data: data['cert'] = open(p(data['cert']), 'r').read() if 'key' in data: key_paths = [p(data['key'])] if 'KEY_PATH' in os.environ: key_paths.append(path(os.environ['KEY_PATH'], data['key'])) key = None for candidate in key_paths: if exists(candidate): key = open(candidate, 'r').read() break if not key: raise ValueError('key not found in: %s' % key_paths) data['key'] = key
def setUpClass(self): self.fpath = path(path_to_mofiles, "ExtFunctionTests.mo")
writer.writeheader() for test in test_name: row = {} row['test_name'] = test for measure in measures: row[measure] = np.median(performance_details[test][measure]) row[measure + '_IQR'] = np.percentile( performance_details[test][measure], 75) - np.percentile( performance_details[test][measure], 25) writer.writerow(row) if __name__ == '__main__': csv_proc = CSV_PROC() # """all the variables below are a dict of lists.""" env_dir = path(data_dir, 'MT_data', '3rdRun') MT_result = path(env_dir, 'raw_3rd.csv') golden_truthe_csv = path(data_dir, 'MT_data', 'golden_query_true_answers_expectations.csv') answers_csv = path(env_dir, 'extracted_answers.csv') feauralized_answers_csv = path(env_dir, 'feauralized_answers.csv') complete_featuer_csv = path(env_dir, 'complete_featuers.csv') # be careful that MT_result should be ranged by **HIT_ID** in acsending order. ## Manual operation from Excel is needed!! answers, gold_answers = get_MT_answers(MT_result) csv_proc.transDict2Csv(answers, answers_csv) golden_truthes = get_golden_truthes(golden_truthe_csv) pre, rec, f1 = evaluate_answer_quality(gold_answers, golden_truthes)
def exists(self): return path(self.file)
from prepare_data import data_dir from os.path import join as path import plotly plotly.tools.set_credentials_file(username='******', api_key='czrCH0mQHmX5HLXSHBqS') import plotly.plotly as py import plotly.graph_objs as go import cPickle env_dir = path(data_dir, 'MT_data', '3rdRun') details_path = path(env_dir, 'performance_details_tree.p') details = cPickle.load(open(details_path, 'rb')) # n1, n2, n3, n4 = 'MS_alone', 'Lit', 'MT-wo-MS', 'Combined' # t1, t2, t3, t4 = 'Single Feature:\nmergeable_state', 'Quantitative Features:\nLiterature', 'Quanlitative Features:\nMTurk', 'Combination of Qualitat Last Two' n1, n2, n3 = 'Lit', 'MT-wo-MS', 'Combined' t1, t2, t3 = 'Quantitative Features\n(From API Mining)', 'Qualitative Features\n(From MTurk Answers)', 'Combined Features\n(Quan. & Qual.)' #x = ['day 1', 'day 1', 'day 1', 'day 1', 'day 1', 'day 1', # 'day 2', 'day 2', 'day 2', 'day 2', 'day 2', 'day 2'] l = len(details[n1]['precision']) # x = [t1] * l + [t2] * l + [t3] * l + [t4] * l x = [t1] * l + [t2] * l + [t3] * l Precision = go.Box( y = sorted(details[n1]['precision']) + sorted(details[n2]['precision']) + sorted(details[n3]['precision']), x=x, name='Precision', marker=dict( color='#3D9970' )
def setUpClass(self): """ Sets up the test class. """ self.cpath = "ExtFunctionTests.ExtFunctionTest2" self.fpath = path(path_to_mofiles, "ExtFunctionTests.mo")
def setUpClass(self): """ Sets up the test class. Check timeout of infinite loop during constant evaluation. """ self.fpath = path(path_to_mofiles, "ExtFunctionTests.mo")
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import sys import base64 import threading import telebot import time import zoe import subprocess from os import environ as env from os.path import join as path from telebot import util from zoe.deco import Agent, Message with open(path(env['ZOE_HOME'], 'etc', 'tgbot.conf'), 'r') as f: TG_TOKEN = f.readline().strip() @Agent(name='tg') class Tgbot: def __init__(self): self._starttime = time.time() self._sleeptime = 10000 # Non-threaded (better fro skipping exceptions) self.bot = telebot.TeleBot(TG_TOKEN, threaded=False) self.bot.set_update_listener(self._tg_msg) self._bot_me = self.bot.get_me()
def setUpClass(cls): """ Sets up the test class. """ cls.fpath = path(path_to_mofiles, 'Asserts.mo')
def setUpClass(cls): """ Sets up the test class. """ cls.fpath = path(path_to_mofiles, "ExtFunctionTests.mo")
import sys sys.path.append('./lib') import gettext import threading import zoe from infocards.archive import Archive from os import environ as env from os.path import join as path from zoe.deco import Agent, Message from zoe.models.users import Users gettext.install("archivist") with open(path(env["ZOE_HOME"], "etc", "archivist.conf"), "r") as f: DB_PATH = f.readline().strip() LOCALEDIR = path(env["ZOE_HOME"], "locale") ZOE_LOCALE = env["ZOE_LOCALE"] or "en" LOCK = threading.Lock() @Agent(name="archivist") class Archivist: @Message(tags=["add-section"]) def add_card_to_section(self, parser): """ Adds a card to the given section. cid* - card id
def delete(self): if path(self.file): try:remove(self.file) except:pass