def __init__(self,
config_path='./config.json',
thumbnail=False,
sample=None):
self.config = None
# standard lmdb environment for storing biblio entries by uuid
self.env = None
# lmdb environment for storing mapping between doi/pmcid and uuid
self.env_doi = None
# lmdb environment for keeping track of failures
self.env_fail = None
self._load_config(config_path)
# boolean indicating if we want to generate thumbnails of front page of PDF
self.thumbnail = thumbnail
self._init_lmdb()
# if a sample value is provided, indicate that we only harvest the indicated number of PDF
self.sample = sample
self.s3 = None
if self.config["bucket_name"] is not None and len(
self.config["bucket_name"]) is not 0:
self.s3 = S3.S3(self.config)
def __init__(self, config_path='./config.json'):
self.config = None
self._load_config(config_path)
self.s3 = None
if self.config["bucket_name"] is not None and len(
self.config["bucket_name"]) is not 0:
self.s3 = S3.S3(self.config)
def onSubscribe(message):
command = Box(json.loads(str(message, 'utf-8')))
startTime = datetime.datetime.strptime(command.startTime, "%Y/%m/%d %H:%M:%S")
seconds = command.seconds
endTime = startTime + timedelta(seconds=seconds)
fileName = "/tmp/output.mp4"
mp4 = Mp4.Mp4(dataPath)
mp4.create(startTime, endTime, fileName)
print("{} created.".format(fileName))
s3 = S3.S3(identityPoolId)
key = "{}.mp4".format(startTime)
s3.putObject(bucketName, key, fileName)
def __init__(self, config_path='./config.json'):
self.config = None
# standard lmdb environment for storing processed biblio entry uuid
self.env = None
# lmdb environment for keeping track of PDF annotation failures
self.env_fail = None
self._load_config(config_path)
self._init_lmdb()
if self.config['bucket_name'] is not None and len(
self.config['bucket_name']) > 0:
self.s3 = S3.S3(self.config)
def main(args):
word_embs = word_embeddings.load_embeddings(args.embs_path)
with open(args.output_jsonl, 'w') as out:
with open(args.input_jsonl, 'r') as f:
for line in f:
instance = json.loads(line)
# The input summaries to S3 are lists of sentences. The example
# just passes the whole text in as 1 sentence without pre-sentence tokenizing
# it, so we will do the same. But the input summaries are expected
# to just be 1 string each, so we wrap them in an extra list
summary = [instance['summary']]
references = [[reference]
for reference in instance['references']]
pyr, resp = S3.S3(references, summary, word_embs,
args.model_folder)
out.write(json.dumps({'pyr': pyr, 'resp': resp}) + '\n')
def __init__(self, config_path='./config.json'):
self.config = None
# standard lmdb environment for storing biblio entries by uuid
self.env = None
# lmdb environment for storing mapping between doi and uuid
self.env_doi = None
# lmdb environment for keeping track of failures
self.env_fail = None
self._load_config(config_path)
self._init_lmdb()
self.s3 = None
if self.config["bucket_name"] is not None and len(
self.config["bucket_name"]) is not 0:
self.s3 = S3.S3(self.config)
def __init__(self, config_path='./config.json'):
self.config = None
# standard lmdb environment for keeping track of the status of processing
self.env_software = None
self._load_config(config_path)
self._init_lmdb()
if self.config['bucket_name'] is not None and len(
self.config['bucket_name']) > 0:
self.s3 = S3.S3(self.config)
self.mongo_db = None
# load blacklist
self.blacklisted = []
with open("resources/covid_blacklist.txt", "r") as blackfile:
for line in blackfile:
line = line.replace(" ", "").strip()
if not line.startswith("#"):
self.blacklisted.append(line)
print("blacklist size:", len(self.blacklisted))
def lambda_handler(event, context):
dataModel = event
alert = Alert.Alert(dataModel)
s3 = S3.S3()
resultList = []
if dataModel["config"]["initObject"]:
resultList.append(alert.backgroundObjectRegister())
backgroundEvent = s3.readJson("backgroundEvent.json")
lastNotificationTimestamp = s3.readJson("lastNotificationTimestamp.json")
result = alert.cameraCoveredDetect(lastNotificationTimestamp,
backgroundEvent)
if result is not None:
resultList.append(result)
backgroundObjectList = s3.readJson("backgroundObjectList.json")
result = alert.cameraMovedDetect(lastNotificationTimestamp,
backgroundEvent, backgroundObjectList)
if result is not None:
resultList.append(result)
result = alert.backgroundObjectLostDetect(backgroundObjectList,
backgroundEvent)
if result is not None:
resultList.append(result)
result = alert.abnormalObjectStayDetect(backgroundObjectList,
backgroundEvent)
if result is not None:
resultList.append(result)
dataModel["result"] = resultList
return dataModel
def run_example(embs_path, model_folder): word_embs = word_embeddings.load_embeddings(embs_path) result = S3.S3(references, system_summary, word_embs, model_folder) print "RESULTS: " print "-- (trained on normalized pyramid): ", result[0] print "-- (trained on normalized responsiveness): ", result[1]
eval_id = 'ev-' + base64.b32encode(os.urandom(10))
ml.create_evaluation(EvaluationId=eval_id,
EvaluationName=name + " evaluation",
MLModelId=model_id,
EvaluationDataSourceId=test_ds_id)
print("Created Evaluation %s" % eval_id)
return eval_id
if __name__ == "__main__":
try:
data_s3_url = S3_URI
#schema_fn = "temData.csv.schema"
schema_fn = DATA_SCHEMA
recipe_fn = "recipe.json"
if len(sys.argv) > 2:
name = sys.argv[1]
else:
name = "Marketing sample"
except:
raise
newS3 = S3.S3(S3_FILE_NAME)
newS3.S3_BUCKET_NAME = S3_BUCKET_NAME
newS3.uploadData()
print "upload success"
model_id = build_model(data_s3_url, schema_fn, recipe_fn, name=name)
#print("""\nFor the next step in the demo, run:
#python use_model.py %s 0.77 s3://your-bucket/ml-output/""" % model_id)
def __init__(self, dataModel):
self.__s3 = S3.S3()
self.__dataModel = dataModel
def test_create_a_fake_bucket():
s= S3()
def RunFileAnalysis(self):
# Main program/ file analysis will happen here.
uploadFileNames = ""
directoryValid = False
# check to see if the directory entry box has been set. If not, prompt the user.
# Repeat until the directory is set
folderDirectory = self.getSelectedFolderDirectory()
# if the directory is not set
if (folderDirectory == "INVALID"):
tkMessageBox.showinfo("Alert!",
"Please select a file directory to analyze")
# if directory is successfully set
else:
uploadFileNames = self.getAllFileNamesInDirectory(folderDirectory)
directoryValid = True
print("Valid directory!")
if (directoryValid):
self.t_log.insert(tk.INSERT, "Analyzing files in: ",
"generalStatement")
self.t_log.insert(tk.INSERT, folderDirectory + "\n", "keyword")
self.addTextHighlightingToFileAnalysis()
self.t_log.insert(tk.INSERT, "\n Files\n===================",
"generalStatement")
bucket = S3.S3()
self.createNewS3Bucket(bucket.BucketName)
for fileName in uploadFileNames:
localFilePath = os.path.join(folderDirectory + "\\" + fileName)
response = {}
print("\n-----\n\nUploading '{}' to Amazon S3 bucket '{}'".
format(localFilePath, bucket.BucketName))
# check metadata of local file and corresponding file in amazon s3
try:
# update the current file in the bucket with the local file's contents
bucket.uploadFileToBucket(fileName, localFilePath)
response = self.getBoto3ResponseUsingLocalFileEtag(
bucket.BucketName, fileName, localFilePath)
except ClientError:
# Not found
print(
"No file '{}' found in bucket '{}'. Continuing upload..."
.format(fileName, bucket.BucketName))
self.t_log.insert(tk.INSERT, "\n" + fileName, "fileNew")
self.addTextHighlightingToFileAnalysis()
if response != {}:
# reads the file back and displays the name from the file
print(
"The following was added to your '{}' bucket:".format(
bucket.BucketName))
self.getDetailsFromUploadedFile(fileName,
bucket.BucketName)
db = RDS.RDS()
db.Connect()
# connect to database and add new entry for response
db.uploadBucketFileContentsToDatabase(
response, localFilePath, fileName, bucket.BucketName,
USER.Username, USER.Password, USER.EmailAddress,
USER.PhoneNumber, USER.FirstName, USER.LastName)
self.t_log.insert(tk.INSERT, "\n" + fileName,
db.OutputTextFileStatus)
self.addTextHighlightingToFileAnalysis()
db.Disconnect()
# get the analysis in case the user wishes to export it to a text file
global ANALYSIS
ANALYSIS = str(self.t_log.get(1.0, tk.END))
print('\n\nClosing program...')
