def get_comments_by_id(id):
job = get_current_job()
total_comments = get_comments_count(id)
comments_loaded = 0
comments = []
request_parameters = {
'videoId': id,
'part': 'snippet',
'textFormat': 'plainText',
'maxResults': 100
}
next_page_token = None
while True:
response = youtube.commentThreads().list(**request_parameters).execute()
comments_loaded += response['pageInfo']['resultsPerPage']
for item in response['items']:
comments.append(format_comment(item))
next_page_token = response.get('nextPageToken')
if (next_page_token is None):
break
else:
request_parameters['pageToken'] = next_page_token
job.meta['progress'] = round(comments_loaded / total_comments, 2)
job.save_meta()
return comments
def enqueue_with_reservation(func, resources, args=None, kwargs=None, options=None):
"""
Enqueue a message to Pulp workers with a reservation.
This method provides normal enqueue functionality, while also requesting necessary locks for
serialized urls. No two tasks that claim the same resource can execute concurrently. It
accepts resources which it transforms into a list of urls (one for each resource).
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a :func:`_queue_reserved_task` task. See
the docblock on :func:`_queue_reserved_task` for more information on this.
This method creates a :class:`pulpcore.app.models.Task` object. Pulp expects to poll on a
task just after calling this method, so a Task entry needs to exist for it
before it returns.
Args:
func (callable): The function to be run by RQ when the necessary locks are acquired.
resources (list): A list of resources to reserve guaranteeing that only one task reserves
these resources. Each resource can be either a (str) resource URL or a
(django.models.Model) resource instance.
args (tuple): The positional arguments to pass on to the task.
kwargs (dict): The keyword arguments to pass on to the task.
options (dict): The options to be passed on to the task.
Returns (rq.job.job): An RQ Job instance as returned by RQ's enqueue function
Raises:
ValueError: When `resources` is an unsupported type.
"""
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_('Must be (str|Model)'))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
if current_job:
current_task = Task.objects.get(pk=current_job.id)
parent_kwarg['parent'] = current_task
Task.objects.create(pk=inner_task_id, state=TASK_STATES.WAITING,
name=f'{func.__module__}.{func.__name__}', **parent_kwarg)
q = Queue('resource-manager', connection=redis_conn)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
q.enqueue(_queue_reserved_task, args=task_args, job_timeout=TASK_TIMEOUT)
return Job(id=inner_task_id, connection=redis_conn)
def create_task(graph):
"""
# Trip Madrid > New York JFK
graph_before=[
{"order" : ("Stockholm", "New York JFK")},
{"order" : ("Barcelona", "Girona Airport")},
{"order" : ("Madrid", "Barcelona")},
{"order" : ("Girona Airport", "Stockholm")},
]
graph_after=[
{"order" : ("Madrid", "Barcelona")},
{"order" : ("Barcelona", "Girona Airport")},
{"order" : ("Girona Airport", "Stockholm")},
{"order" : ("Stockholm", "New York JFK")},]
"""
graph_before = graph
edges = [tuple(*d.values()) for d in graph_before]
# [('Stockholm', 'New York JFK'), ('Barcelona', 'Girona Airport')...
G = networkx.DiGraph()
G.add_edges_from(edges)
longest_path = networkx.dag_longest_path(G)
graph_after = list(zip(longest_path[:-1], longest_path[1:]))
graph_after = [{'order': edge} for edge in graph_after]
job_id = get_current_job().id
# Get job
job = Job.objects.get(job_id=job_id)
# Edit
job.job_is_finished = True
job.job_ended_at = datetime.datetime.now()
job.job_is_queued = False
job.job_is_started = False
job.job_status = "finished"
job.graph = graph_after
job.save()
return True
# def create_task_2(task_type):
# time.sleep(int(task_type) * 10)
# job_id = get_current_job().id
# # Get job
# job = Job.objects.get(job_id=job_id)
#
# # Edit
# job.job_is_finished = True
# job.job_ended_at = datetime.datetime.now()
# job.job_is_queued = False
# job.job_is_started = False
# job.job_status = "finished"
#
# job.save()
# return True
def _enqueue_with_reservation(
func, resources, args=None, kwargs=None, options=None, task_group=None
):
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_("Must be (str|Model)"))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
json.dumps(args, cls=NonJSONWarningEncoder)
json.dumps(kwargs, cls=NonJSONWarningEncoder)
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(GuidMiddleware.get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
**parent_kwarg,
)
for resource in resources:
reservation_record = ReservedResourceRecord.objects.get_or_create(resource=resource)[0]
TaskReservedResourceRecord.objects.create(resource=reservation_record, task=task)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def create_profile_db_and_connection():
"""
Create a profile db from this tasks UUID and a sqlite3 connection to that databases.
The database produced has three tables with the following SQL format:
The `stages` table stores info about the pipeline itself and stores 3 fields
* uuid - the uuid of the stage
* name - the name of the stage
* num - the number of the stage starting at 0
The `traffic` table stores 3 fields:
* uuid - the uuid of the stage this queue feeds into
* waiting_time - the amount of time the item is waiting in the queue before it enters the stage.
* service_time - the service time the item spent in the stage.
The `system` table stores 3 fields:
* uuid - The uuid of stage this queue feeds into
* length - The length of items in this queue, measured just before each arrival.
* interarrival_time - The amount of time since the last arrival.
"""
debug_data_dir = "/var/lib/pulp/debug/"
pathlib.Path(debug_data_dir).mkdir(parents=True, exist_ok=True)
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
if current_job:
db_path = debug_data_dir + current_job.id
else:
db_path = debug_data_dir + uuid.uuid4()
import sqlite3
global CONN
CONN = sqlite3.connect(db_path)
c = CONN.cursor()
# Create table
c.execute(
"""CREATE TABLE stages
(uuid varchar(36), name text, num int)"""
)
# Create table
c.execute(
"""CREATE TABLE traffic
(uuid varchar(36), waiting_time real, service_time real)"""
)
# Create table
c.execute(
"""CREATE TABLE system
(uuid varchar(36), length int, interarrival_time real)"""
)
return CONN
def current():
"""
Returns:
pulpcore.app.models.Task: The current task.
"""
try:
job_id = get_current_job().id
except AttributeError:
task = None
else:
task = Task.objects.get(job_id=job_id)
return task
def get_current_worker():
"""
Get the rq worker assigned to the current job
Returns:
class:`rq.worker.Worker`: The worker assigned to the current job
"""
for worker in Worker.all():
if worker.get_current_job() == get_current_job():
return worker
return None
def current():
"""
Returns:
pulpcore.app.models.Task: The current task.
"""
try:
task_id = get_current_job().id
except AttributeError:
task = None
else:
task = Task.objects.get(pk=task_id)
return task
def _enqueue_with_reservation(func,
resources,
args=None,
kwargs=None,
options=None,
task_group=None):
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
resources = _validate_and_get_resources(resources)
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
args_as_json = json.dumps(args, cls=UUIDEncoder)
kwargs_as_json = json.dumps(kwargs, cls=UUIDEncoder)
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
args=args_as_json,
kwargs=kwargs_as_json,
reserved_resources_record=resources,
**parent_kwarg,
)
task_args = (func, inner_task_id, resources, args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def _task_id():
"""
The current task ID.
Returns:
str: The current task ID.
Raises:
RuntimeError: When used outside of an RQ task.
"""
try:
return get_current_job().id
except AttributeError:
raise RuntimeError(_('May only be used within a Task.'))
def _hostname():
"""
The worker hostname.
Returns:
str: The worker hostname.
Raises:
RuntimeError: When used outside of an RQ task.
"""
try:
return get_current_job().origin
except AttributeError:
raise RuntimeError(_('May only be used within a Task.'))
def _hostname():
"""
The worker hostname.
Returns:
str: The worker hostname.
Raises:
RuntimeError: When used outside of an RQ task.
"""
try:
return get_current_job().origin
except AttributeError:
raise RuntimeError(_('May only be used within a Task.'))
def _task_id():
"""
The current task ID.
Returns:
str: The current task ID.
Raises:
RuntimeError: When used outside of an RQ task.
"""
try:
return get_current_job().id
except AttributeError:
raise RuntimeError(_('May only be used within a Task.'))
def task_add_retrorule_step(form_data, network_id):
job = get_current_job()
job.meta['progress'] = 'started'
job.save_meta()
clicked_node = form_data['smiles']
x = form_data['x']
y = form_data['y']
data = json.loads(current_app.redis.get(network_id))
graph_dict = json.loads(data['graph_dict'])
attr_dict = json.loads(data['attr_dict'])
target_smiles = data['target_smiles']
network_options = json.loads(data['network_options'])
graph = nx.from_dict_of_lists(graph_dict, create_using=nx.DiGraph)
network = Network(graph=graph, target_smiles=target_smiles)
network.update_settings(network_options)
network.add_attributes(attr_dict)
network.retrorules.retrorules_rxns = current_app.retrorules_rxns
network.retrorules.retrorule_db = current_app.retrorules_db
new_substrate_nodes, new_reaction_nodes = network.retrorules.add_step(
clicked_node)
all_new_nodes = [clicked_node] + new_substrate_nodes + new_reaction_nodes
subgraph = network.graph.subgraph(all_new_nodes)
nodes, edges = network.get_visjs_nodes_and_edges(graph=subgraph)
for i, node in enumerate(nodes):
nodes[i].update({'x': x, 'y': y})
result = {
'nodes': nodes,
'edges': edges,
'to_delete': [],
}
data['graph_dict'] = json.dumps(nx.to_dict_of_lists(network.graph))
data['attr_dict'] = json.dumps(network.attributes_dict())
data['nodes'] = add_new(data['nodes'], nodes)
data['edges'] = add_new(data['edges'], edges)
current_app.redis.mset({network_id: json.dumps(data)})
current_app.redis.expire(network_id, 5 * 60)
return result
def current():
"""
Returns:
pulpcore.app.models.Task: The current task.
"""
try:
if settings.USE_NEW_WORKER_TYPE:
task_id = os.environ["PULP_TASK_ID"]
else:
task_id = get_current_job().id
except (AttributeError, KeyError):
task = None
else:
task = Task.objects.get(pk=task_id)
return task
def emit(self, record):
"""
Log `record` into the `CollectionImport.messages` field of the current task.
Args:
record (logging.LogRecord): The record to log.
"""
# This import cannot occur at import time because Django attempts to instantiate it early
# which causes an unavoidable circular import as long as this needs to import any model
from .models import CollectionImport
collection_import = CollectionImport.objects.get(
task=get_current_job().id)
collection_import.add_log_record(record)
collection_import.save()
def setClustersWork(year, month, day, data=None):
dateForm = '-'.join([str(year), str("%02d" % month), str("%02d" % day)])
print(dateForm)
dcon.saveJobStateToFirebaseDB(dateForm, get_current_job().get_status())
dcon.loadParcelDataFromFirebaseDB(dateForm)
distributer.clustering()
dcon.saveTSPFile('data')
finder = RouteFinder()
for c, fname in enumerate(dcon.getTSPFilenames()):
finder.solve(dateForm, fname)
dcon.saveParcelDataToFirebaseDB(dateForm, c + 1, finder.problem,
finder.route)
print('firebaseDB updated for cluster', c + 1)
dcon.saveJobStateToFirebaseDB(dateForm, "finished")
print('setClusters Done')
def __init__(self):
"""
Create a WorkingDirectory.
Raises:
RuntimeError: When used outside of an RQ task.
"""
try:
job = get_current_job()
self.hostname = job.origin
self.task_id = job.id
except AttributeError:
raise RuntimeError(_("May only be used within a Task."))
self.task_path = os.path.join(get_worker_path(self.hostname),
self.task_id)
super().__init__(self.task_path)
def test_job_access_within_job_function(self):
"""The current job is accessible within the job function."""
# Executing the job function from outside of RQ throws an exception
self.assertIsNone(get_current_job())
# Executing the job function from within the job works (and in
# this case leads to the job ID being returned)
job = Job.create(func=access_self)
job.save()
id = job.perform()
self.assertEqual(job.id, id)
self.assertEqual(job.func, access_self)
# Ensure that get_current_job also works from within synchronous jobs
queue = Queue(async=False)
job = queue.enqueue(access_self)
id = job.perform()
self.assertEqual(job.id, id)
self.assertEqual(job.func, access_self)
def test_job_access_within_job_function(self):
"""The current job is accessible within the job function."""
# Executing the job function from outside of RQ throws an exception
self.assertIsNone(get_current_job())
# Executing the job function from within the job works (and in
# this case leads to the job ID being returned)
job = Job.create(func=access_self)
job.save()
id = job.perform()
self.assertEqual(job.id, id)
self.assertEqual(job.func, access_self)
# Ensure that get_current_job also works from within synchronous jobs
queue = Queue(async=False)
job = queue.enqueue(access_self)
id = job.perform()
self.assertEqual(job.id, id)
self.assertEqual(job.func, access_self)
def __init__(self):
"""
Create a WorkingDirectory.
Raises:
RuntimeError: When used outside of an RQ task.
"""
warnings.warn(
_("WorkingDirectory is deprecated and will be removed in pulpcore==3.12; "
'use tempfile.TemporaryDirectory(dir=".") instead.'),
DeprecationWarning,
)
try:
job = get_current_job()
self.hostname = job.origin
self.task_id = job.id
except AttributeError:
raise RuntimeError(_("May only be used within a Task."))
self.task_path = os.path.join(get_worker_path(self.hostname),
self.task_id)
super().__init__(self.task_path)
def job_func(*args, **kwargs):
current_job = get_current_job()
if not current_job or IS_RUNNING_TESTS:
# in synchronous calls (as wall in TESTS because queued jobs are executed synchronously)
# we don't won't to create another session
return func(*args, **kwargs)
start = time.perf_counter()
started_at = current_job.started_at or datetime.now()
logger.info(
"Started job %s",
func.__name__,
extra={
**job_extra_description(current_job),
"status":
"started",
"waiting_time":
round(
((started_at - current_job.enqueued_at).microseconds) /
1000),
},
)
with current_app.app_context():
result = func(*args, **kwargs)
logger.info(
"Ended job %s",
func.__name__,
extra={
**job_extra_description(current_job),
"status": "ended",
"duration": round((time.perf_counter() - start) * 1000),
},
)
return result
def test_job_access_outside_job_fails(self):
"""The current job is accessible only within a job context."""
self.assertIsNone(get_current_job())
def __init__(self):
self.job = get_current_job()
def import_collection(
artifact_pk,
repository_pk=None,
expected_namespace=None,
expected_name=None,
expected_version=None,
):
"""
Create a Collection from an uploaded artifact and optionally validate its expected metadata.
This task provides optional validation of the `namespace`, `name`, and `version` metadata
attributes. If the Artifact fails validation or parsing, the Artifact is deleted and the
Collection is not created.
This task performs a CollectionImport object get_or_create() to allow import messages to be
logged.
Args:
artifact_pk (str): The pk of the Artifact to create the Collection from.
Keyword Args:
repository_pk (str): Optional. If specified, a new RepositoryVersion will be created for the
Repository and any new Collection content associated with it.
expected_namespace (str): Optional. The namespace is validated against the namespace
specified in the Collection's metadata. If it does not match a ImporterError is
raised.
expected_name (str): Optional. The name is validated against the name specified in the
Collection's metadata. If it does not match a ImporterError is raised.
expected_version (str): Optional. The version is validated against the version specified in
the Collection's metadata. If it does not match a ImporterError is raised.
Raises:
ImporterError: If the `expected_namespace`, `expected_name`, or `expected_version` do not
match the metadata in the tarball.
"""
CollectionImport.objects.get_or_create(task_id=get_current_job().id)
artifact = Artifact.objects.get(pk=artifact_pk)
filename = CollectionFilename(expected_namespace, expected_name, expected_version)
log.info(f"Processing collection from {artifact.file.name}")
import_logger = logging.getLogger("pulp_ansible.app.tasks.collection.import_collection")
with _artifact_guard(artifact):
try:
with artifact.file.open() as artifact_file:
importer_result = process_collection(
artifact_file, filename=filename, logger=import_logger
)
except ImporterError as exc:
log.info(f"Collection processing was not successfull: {exc}")
raise
collection_info = importer_result["metadata"]
with transaction.atomic():
collection, created = Collection.objects.get_or_create(
namespace=collection_info["namespace"], name=collection_info["name"]
)
tags = collection_info.pop("tags")
# Remove fields not used by this model
collection_info.pop("license_file")
collection_info.pop("readme")
# the importer returns many None values. We need to let the defaults in the model prevail
for key in ["description", "documentation", "homepage", "issues", "repository"]:
if collection_info[key] is None:
collection_info.pop(key)
collection_version = CollectionVersion(
collection=collection,
**collection_info,
contents=importer_result["contents"],
docs_blob=importer_result["docs_blob"],
)
collection_version.save()
for name in tags:
tag, created = Tag.objects.get_or_create(name=name)
collection_version.tags.add(tag)
_update_highest_version(collection_version)
collection_version.save() # Save the FK updates
ContentArtifact.objects.create(
artifact=artifact,
content=collection_version,
relative_path=collection_version.relative_path,
)
CreatedResource.objects.create(content_object=collection_version)
if repository_pk:
repository = Repository.objects.get(pk=repository_pk)
content_q = CollectionVersion.objects.filter(pk=collection_version.pk)
with RepositoryVersion.create(repository) as new_version:
new_version.add_content(content_q)
CreatedResource.objects.create(content_object=repository)
def import_collection(
temp_file_pk,
repository_pk=None,
expected_namespace=None,
expected_name=None,
expected_version=None,
):
"""
Create a Collection from an uploaded artifact and optionally validate its expected metadata.
This task provides optional validation of the `namespace`, `name`, and `version` metadata
attributes. If the Artifact fails validation or parsing, the Artifact is deleted and the
Collection is not created.
This task performs a CollectionImport object get_or_create() to allow import messages to be
logged.
Args:
temp_file_pk (str): The pk of the PulpTemporaryFile to create the Collection from.
Keyword Args:
repository_pk (str): Optional. If specified, a new RepositoryVersion will be created for the
Repository and any new Collection content associated with it.
expected_namespace (str): Optional. The namespace is validated against the namespace
specified in the Collection's metadata. If it does not match a ImporterError is
raised.
expected_name (str): Optional. The name is validated against the name specified in the
Collection's metadata. If it does not match a ImporterError is raised.
expected_version (str): Optional. The version is validated against the version specified in
the Collection's metadata. If it does not match a ImporterError is raised.
Raises:
ImporterError: If the `expected_namespace`, `expected_name`, or `expected_version` do not
match the metadata in the tarball.
"""
CollectionImport.objects.get_or_create(task_id=get_current_job().id)
temp_file = PulpTemporaryFile.objects.get(pk=temp_file_pk)
filename = CollectionFilename(expected_namespace, expected_name, expected_version)
log.info(f"Processing collection from {temp_file.file.name}")
user_facing_logger = logging.getLogger("pulp_ansible.app.tasks.collection.import_collection")
try:
with temp_file.file.open() as artifact_file:
importer_result = process_collection(
artifact_file, filename=filename, logger=user_facing_logger
)
artifact = Artifact.from_pulp_temporary_file(temp_file)
importer_result["artifact_url"] = reverse("artifacts-detail", args=[artifact.pk])
collection_version = create_collection_from_importer(importer_result)
except ImporterError as exc:
log.info(f"Collection processing was not successfull: {exc}")
temp_file.delete()
raise
except Exception as exc:
user_facing_logger.error(f"Collection processing was not successfull: {exc}")
temp_file.delete()
raise
ContentArtifact.objects.create(
artifact=artifact,
content=collection_version,
relative_path=collection_version.relative_path,
)
CreatedResource.objects.create(content_object=collection_version)
if repository_pk:
repository = AnsibleRepository.objects.get(pk=repository_pk)
content_q = CollectionVersion.objects.filter(pk=collection_version.pk)
with repository.new_version() as new_version:
new_version.add_content(content_q)
CreatedResource.objects.create(content_object=repository)
def enqueue_with_reservation(
func, resources, args=None, kwargs=None, options=None, task_group=None
):
"""
Enqueue a message to Pulp workers with a reservation.
This method provides normal enqueue functionality, while also requesting necessary locks for
serialized urls. No two tasks that claim the same resource can execute concurrently. It
accepts resources which it transforms into a list of urls (one for each resource).
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a :func:`_queue_reserved_task` task. See
the docblock on :func:`_queue_reserved_task` for more information on this.
This method creates a :class:`pulpcore.app.models.Task` object. Pulp expects to poll on a
task just after calling this method, so a Task entry needs to exist for it
before it returns.
Args:
func (callable): The function to be run by RQ when the necessary locks are acquired.
resources (list): A list of resources to reserve guaranteeing that only one task reserves
these resources. Each resource can be either a (str) resource URL or a
(django.models.Model) resource instance.
args (tuple): The positional arguments to pass on to the task.
kwargs (dict): The keyword arguments to pass on to the task.
options (dict): The options to be passed on to the task.
task_group (pulpcore.app.models.TaskGroup): A TaskGroup to add the created Task to.
Returns (rq.job.job): An RQ Job instance as returned by RQ's enqueue function
Raises:
ValueError: When `resources` is an unsupported type.
"""
if not args:
args = tuple()
if not kwargs:
kwargs = dict()
if not options:
options = dict()
def as_url(r):
if isinstance(r, str):
return r
if isinstance(r, Model):
return util.get_url(r)
raise ValueError(_("Must be (str|Model)"))
resources = {as_url(r) for r in resources}
inner_task_id = str(uuid.uuid4())
resource_task_id = str(uuid.uuid4())
redis_conn = connection.get_redis_connection()
current_job = get_current_job(connection=redis_conn)
parent_kwarg = {}
if current_job:
# set the parent task of the spawned task to the current task ID (same as rq Job ID)
parent_kwarg["parent_task"] = Task.objects.get(pk=current_job.id)
with transaction.atomic():
task = Task.objects.create(
pk=inner_task_id,
_resource_job_id=resource_task_id,
state=TASK_STATES.WAITING,
logging_cid=(GuidMiddleware.get_guid() or ""),
task_group=task_group,
name=f"{func.__module__}.{func.__name__}",
**parent_kwarg,
)
for resource in resources:
reservation_record = ReservedResourceRecord.objects.get_or_create(resource=resource)[0]
TaskReservedResourceRecord.objects.create(resource=reservation_record, task=task)
task_args = (func, inner_task_id, list(resources), args, kwargs, options)
try:
q = Queue("resource-manager", connection=redis_conn)
q.enqueue(
_queue_reserved_task,
job_id=resource_task_id,
args=task_args,
job_timeout=TASK_TIMEOUT,
)
except RedisConnectionError as e:
task.set_failed(e, None)
return Job(id=inner_task_id, connection=redis_conn)
def prepare(my_job_id, video):
"""
prepare a job and return several parameters
:param my_job_id: database id of the job
:param video: video file as bytes array
"""
from project.model import model
# get the current job
job = get_current_job()
job.meta['stage'] = {'name': 'preparing', 'progress': None}
# get the redis id for the job
job_id = str(get_current_job().get_id())
# add to database
job_cache_dir = Path(os.path.join(Config.CACHE_DIR, str(my_job_id)))
# The cache dir will look like that : cache/aabb-ccc-dddd-fff-ggg/ (UUID)
# Create a directory where the cache is stored.
if not job_cache_dir.exists():
os.makedirs(job_cache_dir)
# save the source video in the cache folder
filename = os.path.join(job_cache_dir, Config.SOURCE_VIDEO_FILE)
print("Saving video at %s" % filename)
with (open(filename, 'wb')) as file:
file.write(video)
file.close()
# save thumbnail
videogen = skvideo.io.FFmpegReader(filename)
for frame in videogen.nextFrame():
thumbnail_path = job_cache_dir / Config.THUMBNAIL_FILE
skvideo.io.vwrite(str(thumbnail_path), frame)
videogen.close()
break
# retrieve the fps from the video
fps = videogen.inputfps
# result
result = model.get_result_by_id(my_job_id)
result.result_code = model.ResultCode.pending
model.db.session.commit()
result_cache_dir = Path(os.path.join(job_cache_dir, Config.RESULT_DIR))
if not result_cache_dir.exists():
os.makedirs(result_cache_dir)
# get specific file locations
pose2d_file = result_cache_dir / Config.DATA_2D_FILE
pose3d_file = result_cache_dir / Config.DATA_3D_FILE
# create a thumbnail
job.meta['stage'] = {'name': 'thumbnail'}
job.save_meta()
pose3d_world = None
points_list = None
job.meta['stage'] = {'name': '2d', 'progress': 0}
job.save_meta()
return job, job_id, model, job_cache_dir, pose2d_file, pose3d_file, thumbnail_path, filename, result, \
result_cache_dir, fps
def track_runs(inp: bytes) -> bytes:
job = get_current_job()
db: Redis[bytes] = job.connection
val = db.incrby("sentinel", 1)
time.sleep(0.5)
return str(val).encode()
def test_job_access_outside_job_fails(self):
"""The current job is accessible only within a job context."""
self.assertIsNone(get_current_job())
import logging