def test_simple_HTTP(self) :
# ============== Setting up an HTTP server at 'http://localhost:8001/' in current directory
try :
PORT = int(sys.argv[1])
except :
PORT = randint(1025, 65535)
try :
Handler = SimpleHTTPServer.SimpleHTTPRequestHandler
httpd = SocketServer.TCPServer(("", PORT), Handler)
except :
Handler = http.server.SimpleHTTPRequestHandler
httpd = socketserver.TCPServer(("", PORT), Handler)
print ("Serving at port %d" % PORT)
http_thread = Thread( target = httpd.serve_forever, )
http_thread.daemon = True
# ============== Starting the HTTP server
http_thread.start()
# ============== Wait until HTTP server is ready
sleep(1)
with remote_repo(['test_package'], base_url = 'http://localhost:%d/' % PORT) :
from test_package import module1
self.assertTrue(module1.dummy_str) # If this point is reached then the module1 is imported succesfully!
def test_zip_import(self):
self.assertFalse('test_package' in sys.modules)
httpimport.INSECURE = True
with httpimport.remote_repo(
base_url='http://localhost:%d/test_package.zip' % self.PORT, ):
import test_package
# If this point is reached then the module1 is imported succesfully!
self.assertTrue('test_package' in sys.modules)
def test_simple_HTTP_fail(self):
httpimport.INSECURE = True
with httpimport.remote_repo(base_url='http://localhost:%d/' %
self.PORT):
try:
import test_package_nonexistent
except (ImportError, KeyError) as e:
self.assertTrue(e)
def test_zip_import_w_pwd(self):
self.assertFalse('test_package' in sys.modules)
httpimport.INSECURE = True
with httpimport.remote_repo(
base_url='http://*****:*****@ssw0rd!' # <--- Correct Password
):
import test_package
# If this point is reached then the module1 is imported succesfully!
self.assertTrue('test_package' in sys.modules)
def OnRegisteredAutomata():
global Automata
import httpimport
with httpimport.remote_repo([
'utils'
], "https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-automata/main/automata/"
):
import utils
from utils import loadAutomata
Automata = loadAutomata()
return Automata
def test_autodetect_corrupt_file(self):
self.assertFalse('test_package' in sys.modules)
httpimport.INSECURE = True
try:
with httpimport.remote_repo(
base_url='http://localhost:%d/test_package.corrupted.tar' %
self.PORT, ):
import test_package
except (ImportError, KeyError) as e:
self.assertTrue(e)
# If this point is reached then the module1 is imported succesfully!
self.assertFalse('test_package' in sys.modules)
def test_tarxz_import(self):
self.assertFalse('test_package' in sys.modules)
if httpimport.LEGACY: # Pass the test in Python2, which does not support tar.xz lzma
self.assertTrue(True)
return
httpimport.INSECURE = True
with httpimport.remote_repo(
base_url='http://localhost:%d/test_package.tar.xz' %
self.PORT, ):
import test_package
# If this point is reached then the module1 is imported succesfully!
self.assertTrue('test_package' in sys.modules)
def test_remote_import():
url = "https://gist.githubusercontent.com/zhqu1148980644/" \
"2eafbe8d25883919ecf11a729f1fdb9a/raw/23f8e087ceba040492c78565f81c764bce9b65c5"
with httpimport.remote_repo(['testtask'], url):
from testtask import task1, task2
@flow
def myflow(name):
return [name | task1, name] | task2 | view
names = Channel.values("qwe", 'asd', 'zxcx', 'hhh')
workflow = myflow(names)
run(workflow)
def test_enc_zip_import_w_pwd_wrong(self):
self.assertFalse('test_package' in sys.modules)
httpimport.INSECURE = True
try:
with httpimport.remote_repo(
base_url='http://localhost:%d/test_package.enc.zip' %
self.PORT,
zip_pwd=b'XXXXXXXX' # <--- Wrong Password
):
import test_package
except RuntimeError:
pass # <--- zipfile module fails for wrong password
# If this point is reached then the module1 is imported succesfully!
self.assertFalse('test_package' in sys.modules)
def process_item(record):
import httpimport
with httpimport.remote_repo([
'stop_words'
], "https://raw.githubusercontent.com/explosion/spaCy/master/spacy/lang/en/"
):
import stop_words
from stop_words import STOP_WORDS
with httpimport.remote_repo([
'utils'
], "https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-automata/main/automata/"
):
import utils
from utils import loadAutomata, find_matches
from string import punctuation
import itertools
import re
debug = enable_debug()
global Automata
if not Automata:
Automata = loadAutomata()
global rconn
if not rconn:
rconn = connecttoRedis()
shard_id = hashtag()
article_id = record['key'].split(':')[1]
if debug:
log(f"Matcher received {record['key']} and my {shard_id}")
for each_key in record['value']:
sentence_key = record['key'] + f':{each_key}'
tokens = set(record['value'][each_key].split(' '))
processed = execute('SISMEMBER',
'processed_docs_stage3{%s}' % shard_id,
sentence_key)
if not processed:
if debug:
log("Matcher: length of tokens " + str(len(tokens)))
tokens.difference_update(set(punctuation))
tokens.difference_update(STOP_WORDS)
matched_ents = find_matches(" ".join(tokens), Automata)
if len(matched_ents) < 1:
if debug:
log("Error matching sentence " + sentence_key)
else:
if debug:
log("Matcher: Matching sentence " + sentence_key)
for pair in itertools.combinations(matched_ents, 2):
source_entity_id = pair[0][0]
destination_entity_id = pair[1][0]
label_source = pair[0][1]
label_destination = pair[1][1]
source_canonical_name = re.sub('[^A-Za-z0-9]+', ' ',
str(label_source))
destination_canonical_name = re.sub(
'[^A-Za-z0-9]+', ' ', str(label_destination))
#TODO: this candidate for rgsync
# execute('XADD', 'nodes_matched_{%s}' % shard_id, '*','node_id',f'{source_entity_id}','node_name',f'{source_canonical_name}')
# execute('XADD', 'nodes_matched {%s}' % shard_id,'*','node_id',f'{destination_entity_id}','node_name',f'{destination_canonical_name}')
year = rconn.hget(f"article_id:{article_id}", 'year')
if not year:
year = '2021'
execute('XADD', 'edges_matched_{%s}' % shard_id, '*',
'source', f'{source_entity_id}', 'destination',
f'{destination_entity_id}', 'source_name',
source_canonical_name, 'destination_name',
destination_canonical_name, 'rank', 1, 'year',
year)
#FIXME: this breaks design pattern of NLP processing to support microservices pattern on front end
rconn.zincrby(
f'edges_scored:{source_entity_id}:{destination_entity_id}',
1, sentence_key)
execute('SADD', 'processed_docs_stage3{%s}' % shard_id,
sentence_key)
else:
if debug:
log(f"Matcher Alteady processed {sentence_key}")
#!flask/bin/python
from flask import Flask, jsonify, request, abort
from flask_cors import CORS, cross_origin
app = Flask(__name__)
app.config['SECRET_KEY'] = 'P3JqafOaPHmi7DV96aZA'
app.config.update(dict(PREFERRED_URL_SCHEME='https'))
CORS(app, supports_credentials=True)
import httpimport
with httpimport.remote_repo([
'utils'
], "https://raw.githubusercontent.com/applied-knowledge-systems/the-pattern-automata/main/automata/"
):
import utils
from utils import loadAutomata, find_matches
from common.utils import *
import os
config_switch = os.getenv('DOCKER', 'local')
REDISGRAPH_PORT = os.getenv('REDISGRAPH_PORT', "9001")
if config_switch == 'local':
startup_nodes = [{
"host": "127.0.0.1",
"port": "30001"
}, {
"host": "127.0.0.1",
"port": "30002"
}, {
from httpimport import remote_repo
import httpimport
httpimport.INSECURE = True
with remote_repo(['hello', 'neat'], base_url='http://127.0.0.1:5002'):
from hello import hi
import neat
neat.Neat().print_neat()
print(dir(hi))
hi.HelloWorld().print_hello()
import_list = ['hello', 'neat']
obj_dict = {}
for item in import_list:
obj_dict[item] = httpimport.load(item, 'http://127.0.0.1:5002')
obj_dict['neat'].Neat().print_neat()
print(dir(obj_dict['hello'].hi))
obj_dict['hello'].hi.HelloWorld().print_hello()
import multiprocessing
from ctypes import c_ulong, windll
from mod_control import control_version, control_hhtpimport, VariableUrlGitLab
httpimport.INSECURE = True
logger = logging.getLogger()
logger.setLevel(logging.INFO)
log_format = '%(asctime)s: %(message)s'
logging.basicConfig(format=log_format,
filename=os.path.join(os.path.abspath(os.curdir),
'log.txt'),
datefmt='%Y-%m-%d %H:%M:%S')
url = f"http://{base64.b64decode(VariableUrlGitLab.HOST).decode()}"
with httpimport.remote_repo([
"verify_txt", "log_in", "hundred_cart", "check_element_in_list",
"read_id_archive", "send_id_cart_nomenklature", "utils"
], url):
from verify_txt import verify
from log_in import log_in
from hundred_cart import hundred_cart
from check_element_in_list import check_element_in_list
from read_id_archive import read_id_archive
from send_id_cart_nomenklature import send_id_cart_nomenklature
from utils import errMessage, succesfully, func_list_tread, moving_all
class Variable:
"""
List of all variables
"""
# Don't touch! ------->
import httpimport
with httpimport.remote_repo(
['Simple_CNN'],
'https://raw.githubusercontent.com/khliland/OrangeCNN/master'):
from Simple_CNN import *
# <------- Don't touch!
######################
## Apply simple CNN ##
######################
classify = True
epochs = 100
batch_size = 64
learning_rate = 0.0001
# Don't touch! ------->
(X_train, Y_train, X_test, Y_test) = dataPrep(in_data)
history = CNN(classify,
X_train=X_train,
Y_train=Y_train,
X_test=X_test,
Y_test=Y_test,
batch_size=batch_size,
epochs=epochs,
lr=learning_rate)
out_data = Table.from_numpy(
None,
np.hstack([
np.array(history.history['accuracy'])[:, np.newaxis],
# In[1]:
import os
from argparse import Namespace
import numpy as np
import httpimport
import torch
import torch.optim as optim
from tqdm import tqdm_notebook
# import data preprocessing and modeling functions from https://github.com/jasoriya/CS6120-PS2-support/tree/master/utils
with httpimport.remote_repo(['data_vectorization','model','helper'], 'https://raw.githubusercontent.com/jasoriya/CS6120-PS2-support/master/utils/'):
from data_vectorization import Vocabulary, SequenceVocabulary, SurnameVectorizer, SurnameDataset, generate_batches
from model import SurnameGenerationModel, sample_from_model, decode_samples
from helper import make_train_state, update_train_state, normalize_sizes, compute_accuracy, sequence_loss, set_seed_everywhere, handle_dirs
# In[3]:
args = Namespace(
# Data and Path information
surname_csv="https://raw.githubusercontent.com/jasoriya/CS6120-PS2-support/master/data/surnames/surnames_with_splits.csv",
vectorizer_file="vectorizer.json",
model_state_file="model.pth",
save_dir= "/", # give path here
# Model hyper parameters
import os
import sys
import time
import Queue
import threading
import cStringIO
import collections
# remote imports
import httpimport
httpimport.INSECURE = True
with httpimport.remote_repo(['pyHook','pythoncom','pyxhook'], base_url='http://localhost:8000'):
if os.name == 'nt':
for module in ['pyHook','pythoncom']:
try:
exec "import %s" in globals()
except ImportError:
util.debug("Error: unable to import '%s'" % module)
else:
for module in ['pyxhook']:
try:
exec "import %s" in globals()
except ImportError:
util.debug("Error: unable to import '%s'" % module)
# byob
import os
import sys
import time
import pickle
import socket
import struct
import httpimport
# byob
import util
# remote imports
with httpimport.remote_repo(['cv2'], base_url='http://localhost:8000'):
try:
import cv2
except ImportError:
util.debug("Error: unable to import '%s'" % 'cv2')
def image(*args, **kwargs):
try:
dev = cv2.VideoCapture(0)
r, f = dev.read()
dev.release()
if not r:
util.debug(f)
return "Unable to access webcam"
png = util.png(f)
import base64
import pickle
import _winreg
import threading
import cStringIO
import collections
# remote imports
import httpimport
httpimport.INSECURE = True
with httpimport.remote_repo([
'Crypto', 'Crypto.Util', 'Crypto.Cipher.AES', 'Crypto.Hash.HMAC',
'Crypto.Hash.MD5', 'Crypto.PublicKey.RSA', 'Crypto.Cipher.PKCS1_OAEP'
],
base_url='http://localhost:8000'):
for module in [
'Crypto', 'Crypto.Util', 'Crypto.Cipher.AES', 'Crypto.Hash.HMAC',
'Crypto.Hash.MD5', 'Crypto.PublicKey.RSA',
'Crypto.Cipher.PKCS1_OAEP'
]:
try:
exec "import %s" % module
except ImportError:
print("Error: unable to import '%s'" % module)
# byob
import util
def test_simple_HTTP(self):
httpimport.INSECURE = True
with httpimport.remote_repo(base_url='http://localhost:%d/' %
self.PORT):
import test_package
self.assertTrue(test_package)
def test_dependent_HTTP(self):
httpimport.INSECURE = True
with httpimport.remote_repo(base_url='http://localhost:%d/' %
self.PORT):
import dependent_module
self.assertTrue(dependent_module)
#!/usr/bin/env python
# coding: utf-8
# Training *character-level* language models.
# We will train unigram, bigram, and trigram character-level models on a collection of books from Project Gutenberg. We will then use these trained English language models to distinguish English documents from Brazilian Portuguese documents in the test set.
# In[28]:
import pandas as pd
import httpimport
import sklearn_pandas
from sklearn.model_selection import train_test_split
with httpimport.remote_repo([
'lm_helper'
], 'https://raw.githubusercontent.com/jasoriya/CS6120-PS2-support/master/utils/'
):
from lm_helper import get_train_data, get_test_data
# In[55]:
# get the train and test data
train = get_train_data()
test, test_files = get_test_data()
train, held_out = train_test_split(train, test_size=0.20)
# Collecting some statistics on the unigram, bigram, and trigram character counts.
# In[31]:
import re
https://github.com/colental/byob
Copyright (c) 2018 Daniel Vega-Myhre
"""
from __future__ import print_function
# standard library
import os
# remote imports
import httpimport
httpimport.INSECURE = True
with httpimport.remote_repo(['win32com'], base_url):
for module in ['win32com.shell.shell']:
try:
exec "import %s" % module in globals()
print("%s imported successfully." % module)
except ImportError:
print("%s import failed." % module)
# byob
import util
@util.config(platforms=['win32'], command=True, usage='escalate')
def escalate(self):
"""
