def read_producao(file_name):
producoes = list()
with open(file_name, encoding='Latin-1') as file: # abre o .tsv
for row in file:
cells = str(row).split('\t')
if 'periodicos' in file_name:
producao = MyClasses.Periodico(*cells[:29])
producao.add_authors(*cells[29:-1])
else:
producao = MyClasses.Conferencias(*cells[:32])
producao.add_authors(*cells[32:-1])
producoes.append(producao)
del producoes[0]
return producoes
def read_conferencias(filepath):
conferencias = list()
with open(filepath + 'conferencias.tsv', encoding='Latin-1') as file:
for row in file:
cells = str(row).split('\t')
new_conferencia = MyClasses.Conferencias(*cells[:32])
new_conferencia.add_authors(*cells[32:-1])
conferencias.append(new_conferencia)
del conferencias[0]
return conferencias
def read_periodicos(filepath):
# leitura dos periodicos
periodicos = list()
with open(filepath + 'periodicos.tsv', encoding='Latin-1') as file:
for row in file:
cells = str(row).split('\t')
new_periodico = MyClasses.Periodico(*cells[:29])
new_periodico.add_authors(*cells[29:-1])
periodicos.append(new_periodico)
del periodicos[0]
return periodicos
def convert_to_object(to_convert):
obj_productions = list()
for production in to_convert:
new_prod = MyClasses.Producao(*production[0][:16])
for prod in production:
if prod[16] != '': # detalhamento
new_prod.add_detalhamento(prod[16], prod[17])
elif prod[18] != '':
new_prod.add_autor(prod[18], prod[19], prod[20])
obj_productions.append(new_prod)
return obj_productions
def run():
"""execute the TraCI control loop"""
traci.init(PORT)
step = 0
programManager = MyClasses.ProgramManager(
myP + constants.CONST_CURR_MAP.rsplit('\\', 1)[0] + '\\',
constants.CONST_CURR_MAP.rsplit('\\', 1)[1], configFile)
#This loops through all of the simulaton steps as defined by the configuartion file
while traci.simulation.getMinExpectedNumber() > 0:
traci.simulationStep() #This performs one simulation step
step += 1 #Count the steps
programManager.Update(step)
if (constants.CONST_FLOW_CUT_OFF
and step > constants.CONST_FLOW_SIMULATION_TIME):
break
programManager.OnExit()
traci.close()
sys.stdout.flush()
import csv
import win32com.client as win32
# Import classes
import MyClasses
import MyExceptions
# Initialise
PROJECT_NAME = "ExtractFromTodoist"
CURRENT_DIRECTORY = os.getcwd()
LOG_END = "\n-------------------------"
os.makedirs("app_data", exist_ok=True)
os.makedirs("data", exist_ok=True)
# Initialise class
my_logger_class = MyClasses.MyLogger(PROJECT_NAME)
my_logger = my_logger_class.my_logger
my_main_exception = MyExceptions.MainException(PROJECT_NAME, my_logger,
LOG_END)
my_general_class = MyClasses.AppGeneral(PROJECT_NAME, my_logger, LOG_END)
my_config_class = MyClasses.MyConfig(PROJECT_NAME, my_logger)
my_config = my_config_class.my_config
# Initialise config
if not os.path.isfile("config.ini"):
my_config_class.create_file_specific()
msg_box_res = my_config_class.create_file()
if msg_box_res == 2: # 1 for IDOK, 2 for IDCANCEL
my_general_class.finalise_app(is_app_end=False)
my_logger.info("Reading config file")
my_config.read("config.ini")
def main(videoDetails: vidDets):
## Get blob details
(blobDetails, timeToCutUTC, frameNumberList0, sport, event,
multipleVideoEvent, samplingProportion) = videoDetails
blobOptions = json.loads(blobDetails)
container = blobOptions['container']
fileURL = blobOptions['fileUrl']
fileName = blobOptions['blob']
frameNumberList = json.loads(frameNumberList0)
logging.info(
f"frameNumberList (type: {type(frameNumberList)}, length: {len(frameNumberList)}) received"
)
logging.info(
f"multipleVideoEvent: {multipleVideoEvent}, type: {type(multipleVideoEvent)}"
)
# ## Get clean video name to be used as folder name (without ".mp4" on the end)
# vidName = MyFunctions.cleanUpVidName(fileName.split("/")[-1])[:-4]
## Return the container name and connection string to insert images into
containerOutput, connectionStringOutput, bsaOutput = MyFunctions.getContainerAndConnString(
sport, container)
logging.info(f"containerOutput: {containerOutput}")
logging.info(f"connectionStringOutput: {connectionStringOutput}")
## Create BlockBlobService object to be used to get blob from container
block_blob_serviceINPUT = BlockBlobService(
connection_string=os.getenv("fsevideosConnectionString"))
## Create BlockBlobService object to be used to upload blob to container
block_blob_serviceOUTPUT = BlockBlobService(
connection_string=connectionStringOutput)
logging.info(
f'BlockBlobService created for account "{block_blob_serviceOUTPUT.account_name}"'
)
## Get names of all containers in the blob storage account
containerNames = [
x.name for x in block_blob_serviceOUTPUT.list_containers()
]
logging.info(
f"List of containerNames received, length: {len(containerNames)}")
## Create container (will do nothing if container already exists)
if containerOutput not in containerNames:
logging.info(f"Container '{containerOutput}' doesn't exist")
_ = block_blob_serviceOUTPUT.create_container(
container_name=containerOutput,
public_access=PublicAccess.Blob,
fail_on_exist=False)
logging.info(
f"Container '{containerOutput}' didn't exist, now has been created"
)
else:
logging.info(f"Container '{containerOutput}' exists already'")
## Get SAS file URL
sasFileURL = MyFunctions.get_SAS_URL(
fileURL=fileURL,
block_blob_service=block_blob_serviceINPUT,
container=container)
logging.info(f"sasFileURL: {sasFileURL}")
## Open the video
vidcap = cv2.VideoCapture(sasFileURL)
logging.info("VideoCapture object created")
success, image = vidcap.read()
## Get metadata
fps = vidcap.get(cv2.CAP_PROP_FPS)
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
logging.info('Video metadata acquired')
logging.info(f"frameCount: {str(frameCount)}")
logging.info(f"FPS: {fps}")
## Create variable to keep track of number of images generated
imagesCreated = []
imageNames = []
## If frame count negative, download locally and try again
if frameCount <= 0:
logging.info(
"Frame count not greater than 0, so local download needed (MP4toJPEGs)"
)
with tempfile.TemporaryDirectory() as dirpath:
## Get blob and save to local directory
vidLocalPath = fr"{dirpath}\{fileName}"
# logging.info("About to get connection string")
# logging.info(f"CS: {os.environ['fsevideosConnectionString']}")
logging.info(f"About to save blob to path: '{vidLocalPath}'")
block_blob_serviceINPUT.get_blob_to_path(container_name=container,
blob_name=fileName,
file_path=vidLocalPath,
max_connections=1)
logging.info("Blob saved to path")
with MyClasses.MyVideoCapture(vidLocalPath) as vc1:
for frameNumberName, frameNumber in frameNumberList:
## Create blobs
imageCreated, imageName = MyFunctions.createBlobs(
vidcap=vc1,
frameNumber=frameNumber,
frameNumberName=frameNumberName,
event=event,
fileName=fileName,
block_blob_service=block_blob_serviceOUTPUT,
containerOutput=containerOutput,
multipleVideoEvent=multipleVideoEvent)
imagesCreated.append(imageCreated)
imageNames.append(imageName)
else:
## Loop through the frame numbers
for frameNumberName, frameNumber in frameNumberList:
## Create blobs
imageCreated, imageName = MyFunctions.createBlobs(
vidcap=vidcap,
frameNumber=frameNumber,
frameNumberName=frameNumberName,
event=event,
fileName=fileName,
block_blob_service=block_blob_serviceOUTPUT,
containerOutput=containerOutput,
multipleVideoEvent=multipleVideoEvent)
imagesCreated.append(imageCreated)
imageNames.append(imageName)
logging.info("Finished looping through all the frames")
## Load variables to be returned into one variable
returnMe = json.dumps([
imagesCreated # list of Trues/Falses
,
len(imagesCreated) # length of above list
,
imageNames # list of names of images (whether created or not)
,
containerOutput # container the images were created in
,
bsaOutput # blob storage account the above container is in
])
return returnMe
def main(req: func.HttpRequest) -> func.HttpResponse:
## Get blob details
fileURL = "https://fsevideos.blob.core.windows.net/us-office/Brighton Highlights.mp4"
container = "us-office"
fileName = "Brighton Highlights.mp4"
timeToCutStr = "2095-03-13 00:00:00.00000"
timeToCut = datetime.strptime(timeToCutStr, "%Y-%m-%d %H:%M:%S.%f")
logging.info(f"fileURL: {fileURL}")
logging.info(f"container: {container}")
logging.info(f"fileName: {fileName}")
logging.info(f"timeToCutStr: {timeToCutStr}")
## Open the video
vidcap = cv2.VideoCapture(fileURL)
logging.info(f"VideoCapture object created for {fileURL}")
success, image = vidcap.read()
## Get metadata
fps = vidcap.get(cv2.CAP_PROP_FPS)
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
logging.info('Video metadata acquired')
logging.info(f"frameCount: {str(frameCount)}")
## If frame count negative, download locally and try again
if frameCount <= 0:
logging.info("Frame count greater than 0, so local download needed")
with tempfile.TemporaryDirectory() as dirpath:
## Get blob and save to local directory
vidLocalPath = fr"{dirpath}\{fileName}"
# logging.info("About to get connection string")
# logging.info(f"CS: {os.environ['fsevideosConnectionString']}")
fsevideosConnectionString = "DefaultEndpointsProtocol=https;AccountName=fsevideos;AccountKey=xfYncTDRCowSrISbdsSknM05jqOrJXc4Oavq7BQ56yR7uQ7MCeL5aXmBsbsE+SZ+++xGt2oy6FvrEdpryc+vwQ==;EndpointSuffix=core.windows.net"
logging.info("About to create BlockBlobService")
block_blob_service = BlockBlobService(
connection_string=fsevideosConnectionString)
logging.info("BlockBlobService created")
block_blob_service.get_blob_to_path(container_name=container,
blob_name=fileName,
file_path=vidLocalPath)
logging.info("Blob saved to path")
with MyClasses.MyVideoCapture(vidLocalPath) as vc1:
frameCount = int(vc1.get(cv2.CAP_PROP_FRAME_COUNT))
logging.info(f"(new) frameCount: {str(frameCount)}")
## Get number of frames wanted per second
wantedFPS = 1
takeEveryN = math.floor(fps / wantedFPS)
if timeToCutStr != "2095-03-13 00:00:00.00000":
## Work out when the recording starts based on the filename
vidName = fileName.split("\\")[-1].replace(".mp4", "")
vidName1 = vidName[:vidName.index("-")]
recordingStart = datetime.strptime(
f'{vidName1.split("_")[0]} {vidName1[-4:]}', "%Y%m%d %H%M")
## Work out which frames to reject
frameToCutFrom = int((timeToCut - recordingStart).seconds * fps)
else:
## If last play is my 100th birthday, set a huge number that it'll never reach
frameToCutFrom = 1000000000
logging.info("List of frame numbers about to be generated")
## Create list of frame numbers to be JPEGed
listOfFrameNumbers = [
i for i in range(frameCount)
if (i % takeEveryN == 0) & (i <= frameToCutFrom)
]
logging.info(
f"listOfFrameNumbers created with {len(listOfFrameNumbers)} elements")
return func.HttpResponse(json.dumps(listOfFrameNumbers))
def read_qualis():
with open('qualis-2017-2018.tsv', encoding='Latin-1') as file:
qualis = [MyClasses.Quali(*str(row).split('\t')) for row in file]
del qualis[0]
return qualis
def main(videoDetails: vidDets) -> list:
## Get blob details
(blobDetails, timeToCutUTCStr, frameNumberList, sport, event,
multipleVideoEvent, samplingProportion) = videoDetails
blobOptions = json.loads(blobDetails)
fileURL = blobOptions['fileUrl']
container = blobOptions['container']
fileName = blobOptions['blob']
# if blobOptions['imagesAlreadyCreated'] is not None:
# imagesAlreadyCreated = int(float((blobOptions['imagesAlreadyCreated'])))
# else:
# imagesAlreadyCreated = None
if (blobOptions['imagesAlreadyCreated'] is None) or pd.isna(
blobOptions['imagesAlreadyCreated']):
imagesAlreadyCreated = None
else:
imagesAlreadyCreated = int(float(
(blobOptions['imagesAlreadyCreated'])))
timeToCutUTC = datetime.strptime(timeToCutUTCStr, "%Y-%m-%d %H:%M:%S.%f")
logging.info(f"fileURL: {fileURL}")
logging.info(f"container: {container}")
logging.info(f"fileName: {fileName}")
logging.info(f"timeToCutUTCStr: {timeToCutUTCStr}")
logging.info(f"imagesAlreadyCreated: {imagesAlreadyCreated}")
## Create BlockBlobService object
logging.info("About to create BlockBlobService")
block_blob_service = BlockBlobService(
connection_string=os.environ['fsevideosConnectionString'])
## Get SAS file URL
sasFileURL = MyFunctions.get_SAS_URL(fileURL=fileURL,
block_blob_service=block_blob_service,
container=container)
## Open the video
vidcap = cv2.VideoCapture(sasFileURL)
logging.info(f"VideoCapture object created for {fileURL}")
success, image = vidcap.read()
## Get metadata
fps = vidcap.get(cv2.CAP_PROP_FPS)
fpsInt = int(round(fps, 0))
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
logging.info('Video metadata acquired')
logging.info(f"(initial) frameCount: {str(frameCount)}")
logging.info(f"(initial) FPS: {fps}")
logging.info(f"(initial) int FPS: {fpsInt}")
## If frame count negative, download locally and try again
if frameCount <= 0:
logging.info(
"Frame count not greater than 0, so local download needed (ReturnFrameNumbers)"
)
with tempfile.TemporaryDirectory() as dirpath:
## Get blob and save to local directory
vidLocalPath = fr"{dirpath}\{fileName}"
logging.info("BlockBlobService created")
block_blob_service.get_blob_to_path(container_name=container,
blob_name=fileName,
file_path=vidLocalPath,
max_connections=1)
logging.info("Blob saved to path")
with MyClasses.MyVideoCapture(vidLocalPath) as vc1:
frameCount = int(vc1.get(cv2.CAP_PROP_FRAME_COUNT))
fps = vc1.get(cv2.CAP_PROP_FPS)
fpsInt = int(round(fps, 0))
logging.info(f"(new) frameCount: {str(frameCount)}")
logging.info(f"(new) FPS: {fps}")
logging.info(f"(new) int FPS: {fpsInt}")
if timeToCutUTCStr != "2095-03-13 00:00:00.00000":
utcTZ = pytz.timezone('UTC')
etTZ = pytz.timezone('America/New_York')
## Work out when the recording starts based on the filename
vidName = fileName.split("\\")[-1].replace(".mp4", "")
vidName1 = vidName[:vidName.index("-")]
## Get recording start and then assign it the US Eastern Time time zone
recordingStart = datetime.strptime(
f'{vidName1.split("_")[0]} {vidName1[-4:]}', "%Y%m%d %H%M")
recordingStartET = etTZ.localize(recordingStart)
## Convert it to UTC
recordingStartUTC = recordingStartET.astimezone(utcTZ).replace(
tzinfo=None)
## Work out which frames to reject
frameToCutFrom = int((timeToCutUTC - recordingStartUTC).seconds * fps)
else:
## If last play is my 100th birthday, set a huge number that it'll never reach
frameToCutFrom = 100000000000000
logging.info("List of frame numbers about to be generated")
## Get number of frames wanted per second
if samplingProportion is None:
wantedFPS = 1
else:
wantedFPS = samplingProportion
takeEveryN_requested = math.floor(fpsInt / wantedFPS)
takeEveryN_1FPS = math.floor(fpsInt / 1)
logging.info(f"Taking 1 image for every {takeEveryN_requested} frames")
## Create list of frame numbers, under the assumption of 1 FPS
listOfFrameNumbers_1FPS = [
i for i in range(frameCount)
if (i % takeEveryN_1FPS == 0) & (i <= frameToCutFrom)
]
## If video is shorter than a minute, return all the frames (@ 1 FPS)
if len(listOfFrameNumbers_1FPS) <= 60:
listOfFrameNumbersAndFrames = [
(i, frame) for i, frame in enumerate(listOfFrameNumbers_1FPS, 1)
]
## Otherwise, get the first min @ 1 FPS and the rest @ requested FPS
else:
## Get first minute's frames @ 1 FPS
firstMinute = [
(i, frame)
for i, frame in enumerate(listOfFrameNumbers_1FPS[:60], 1)
]
## Create list of frame numbers to be JPEGed
remainingMinutes = [
(i, frame)
for i, frame in enumerate(listOfFrameNumbers_1FPS[60:], 61)
if (frame % takeEveryN_requested == 0)
]
## Join them together
listOfFrameNumbersAndFrames = firstMinute + remainingMinutes
if imagesAlreadyCreated is not None:
## Some images have already been created, so cut them away
logging.info(
f"Initially, there were {len(listOfFrameNumbersAndFrames)} elements"
)
listOfFrameNumbersAndFrames = listOfFrameNumbersAndFrames[
imagesAlreadyCreated:]
logging.info(
f"Now there are {len(listOfFrameNumbersAndFrames)} elements")
logging.info(
f"listOfFrameNumbers created with {len(listOfFrameNumbersAndFrames)} elements"
)
return json.dumps(listOfFrameNumbersAndFrames)
def main(req: func.HttpRequest) -> func.HttpResponse:
## Get blob details
fileURL = "https://fsevideos.blob.core.windows.net/us-office/Brighton Highlights.mp4"
container = "us-office"
fileName = "Brighton Highlights.mp4"
sport = None
event = None
frameNumberList = [x for x in range(0, 775, 25)]
logging.info(
f"frameNumberList (type: {type(frameNumberList)}, length: {len(frameNumberList)}) received"
)
# ## Get clean video name to be used as folder name (without ".mp4" on the end)
# vidName = MyFunctions.cleanUpVidName(fileName.split("/")[-1])[:-4]
## Return the container name and connection string to insert images into
containerOutput, connectionStringOutput = getContainerAndConnString(
sport, container)
logging.info(f"containerOutput: {containerOutput}")
logging.info(f"connectionStringOutput: {connectionStringOutput}")
## Set the file name to be used
if event is not None:
fileNameFolder = event
else:
## Blob name without ".mp4"
fileNameFolder = fileName.split("/")[-1][:-4]
logging.info(f"fileNameFolder: {fileNameFolder}")
## Create BlockBlobService object to be used to upload blob to container
block_blob_service = BlockBlobService(
connection_string=connectionStringOutput)
logging.info(
f'BlockBlobService created for account "{block_blob_service.account_name}"'
)
# Get names of all containers in the blob storage account
containerNames = [x.name for x in block_blob_service.list_containers()]
## Create container (will do nothing if container already exists)
if containerOutput not in containerNames:
existsAlready = block_blob_service.create_container(
container_name=containerOutput, fail_on_exist=False)
logging.info(
f"Container '{containerOutput}' didn't exist, now has been created"
)
else:
logging.info(f"Container '{containerOutput}' exists already'")
## Open the video
vidcap = cv2.VideoCapture(fileURL)
logging.info("VideoCapture object created")
success, image = vidcap.read()
## Get metadata
fps = vidcap.get(cv2.CAP_PROP_FPS)
frameCount = int(vidcap.get(cv2.CAP_PROP_FRAME_COUNT))
logging.info('Video metadata acquired')
logging.info(f"frameCount: {str(frameCount)}")
logging.info(f"FPS: {fps}")
## If frame count negative, download locally and try again
if frameCount <= 0:
logging.info("Frame count greater than 0, so local download needed")
with tempfile.TemporaryDirectory() as dirpath:
## Get blob and save to local directory
vidLocalPath = fr"{dirpath}\{fileName}"
# logging.info("About to get connection string")
# logging.info(f"CS: {os.environ['fsevideosConnectionString']}")
fsevideosConnectionString = "DefaultEndpointsProtocol=https;AccountName=fsevideos;AccountKey=xfYncTDRCowSrISbdsSknM05jqOrJXc4Oavq7BQ56yR7uQ7MCeL5aXmBsbsE+SZ+++xGt2oy6FvrEdpryc+vwQ==;EndpointSuffix=core.windows.net"
logging.info("About to create BlockBlobService")
block_blob_service = BlockBlobService(
connection_string=fsevideosConnectionString)
logging.info("BlockBlobService created")
block_blob_service.get_blob_to_path(container_name=container,
blob_name=fileName,
file_path=vidLocalPath)
logging.info("Blob saved to path")
with MyClasses.MyVideoCapture(vidLocalPath) as vc1:
for frameNumberName, frameNumber in enumerate(
frameNumberList, 1):
## Create blobs
createBlobs(vc1, frameNumber, frameNumberName,
fileNameFolder, block_blob_service,
containerOutput)
else:
## Loop through the frame numbers
for frameNumberName, frameNumber in enumerate(frameNumberList, 1):
## Create blobs
createBlobs(vidcap, frameNumber, frameNumberName, fileNameFolder,
block_blob_service, containerOutput)
logging.info("Finished looping through frames")
return func.HttpResponse("I think it worked")